{ "generated_note": "Generated by web/build_data.py from web/data_sources.json and web/model_catalog.json; rerun after curation changes.", "index_version": "index-v4.1", "models": [ { "axes": {}, "best_run_id": null, "catalog_id": "Snowflake/Arctic-Text2SQL-R1-7B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5 Coder", "kind": "community", "lane": "answer-only", "model_label": "Arctic Text2SQL R1 7B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "arctic-text2sql-r1-7b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Nexusflow/Athene-V2-Chat", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Athene V2 Chat", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "athene-v2-chat", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "abacusai/bigstral-12b-32k", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral", "kind": "community", "lane": "answer-only", "model_label": "bigstral 12b 32k", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "bigstral-12b-32k", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "prism-ml/Ternary-Bonsai-27B-unpacked", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.6", "kind": "community", "lane": "answer-only", "model_label": "Bonsai 27B Ternary", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "bonsai-27b-ternary", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "CohereLabs/c4ai-command-r-plus", "composite": null, "demo": false, "est_cost_usd": null, "family": "Command R", "kind": "community", "lane": "answer-only", "model_label": "Command R+", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "c4ai-command-r-plus", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "CohereLabs/c4ai-command-r-v01", "composite": null, "demo": false, "est_cost_usd": null, "family": "Command R", "kind": "community", "lane": "answer-only", "model_label": "Command R", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "c4ai-command-r-v01", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "CohereLabs/c4ai-command-r7b-12-2024", "composite": null, "demo": false, "est_cost_usd": null, "family": "Command R", "kind": "community", "lane": "answer-only", "model_label": "Command R7B 12-2024", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "c4ai-command-r7b-12-2024", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "CohereLabs/command-a-plus-05-2026-bf16", "composite": null, "demo": false, "est_cost_usd": null, "family": "Command A", "kind": "community", "lane": "answer-only", "model_label": "Command A Plus 05-2026 BF16", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "command-a-plus-05-2026-bf16", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "ReadyArt/Dark-Scarlett-v0.3-26B-A4B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Dark Scarlett v0.3 26B A4B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "dark-scarlett-v0-3-26b-a4b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "agentica-org/DeepScaleR-1.5B-Preview", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "DeepScaleR 1.5B Preview", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepscaler-1-5b-preview", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1-0528", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1 0528", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1-0528", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1 Distill Llama 70B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1-distill-llama-70b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1 Distill Llama 8B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1-distill-llama-8b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1 Distill Qwen 1.5B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1-distill-qwen-1-5b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1 Distill Qwen 14B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1-distill-qwen-14b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1 Distill Qwen 32B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1-distill-qwen-32b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek R1 Distill Qwen 7B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-r1-distill-qwen-7b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V3", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V3", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V3", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v3", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V3-0324", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V3", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V3 0324", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v3-0324", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V3.1", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V3", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V3.1", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v3-1", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V3.1-Terminus", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V3", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V3.1 Terminus", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v3-1-terminus", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V3.2", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V3", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V3.2", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v3-2", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V3.2-Exp", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V3", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V3.2 Exp", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v3-2-exp", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V4-Flash", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V4", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V4 Flash", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v4-flash", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "deepseek-ai/DeepSeek-V4-Pro", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek V4", "kind": "community", "lane": "answer-only", "model_label": "DeepSeek V4 Pro", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepseek-v4-pro", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "agentica-org/DeepSWE-Preview", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "DeepSWE Preview", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "deepswe-preview", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Devstral-Small-2-24B-Instruct-2512", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Small", "kind": "community", "lane": "answer-only", "model_label": "Devstral Small 2 24B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "devstral-small-2-24b-instruct-2512", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Devstral-Small-2505", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Small", "kind": "community", "lane": "answer-only", "model_label": "Devstral Small 2505", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "devstral-small-2505", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "AliesTaha/fable-traces", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "fable traces", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "fable-traces", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "microsoft/FrogBoss-32B-2510", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "FrogBoss 32B 2510", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "frogboss-32b-2510", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-3-12b-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 3", "kind": "community", "lane": "answer-only", "model_label": "Gemma 3 12B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-3-12b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-3-1b-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 3", "kind": "community", "lane": "answer-only", "model_label": "Gemma 3 1B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-3-1b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-3-27b-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 3", "kind": "community", "lane": "answer-only", "model_label": "Gemma 3 27B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-3-27b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-3-4b-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 3", "kind": "community", "lane": "answer-only", "model_label": "Gemma 3 4B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-3-4b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-3n-E2B-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 3n", "kind": "community", "lane": "answer-only", "model_label": "Gemma 3n E2B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-3n-e2b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-3n-E4B-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 3n", "kind": "community", "lane": "answer-only", "model_label": "Gemma 3n E4B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-3n-e4b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "yuxinlu1/gemma-4-12B-coder-fable5-composer2.5-v1", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Gemma 4 12B Coder Fable Composer", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-12b-coder-fable5-composer2-5-v1", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "agentic_provenance": "project_attested", "axes": { "coding": { "conditional_accuracy": 0.2702702702702703, "hi": 34.45945945945946, "hi_raw": 0.34459459459459457, "lo": 20.27027027027027, "lo_raw": 0.20270270270270271, "n": 148, "n_errors": 0, "n_no_answer": 0, "point": 27.027027027027028, "point_raw": 0.2702702702702703, "raw_accuracy": 0.2702702702702703, "termination_rate": 1.0 }, "instruction": { "conditional_accuracy": 0.6950354609929078, "hi": 72.10884353741497, "hi_raw": 0.7210884353741497, "lo": 61.224489795918366, "lo_raw": 0.6122448979591837, "n": 294, "n_errors": 0, "n_no_answer": 0, "point": 66.66666666666666, "point_raw": 0.6666666666666666, "raw_accuracy": 0.6666666666666666, "termination_rate": 0.9591836734693877 }, "knowledge": { "conditional_accuracy": 0.8172323759791122, "hi": 80.07448125941039, "hi_raw": 0.8007448125941039, "lo": 70.81332466167156, "lo_raw": 0.7081332466167156, "n": 400, "n_errors": 0, "n_no_answer": 16, "point": 75.58422351505216, "point_raw": 0.7558422351505216, "raw_accuracy": 0.7825, "termination_rate": 0.9575 }, "math": { "conditional_accuracy": 0.32682425488180883, "hi": 17.26618705035971, "hi_raw": 0.17266187050359713, "lo": 6.474820143884892, "lo_raw": 0.06474820143884892, "n": 139, "n_errors": 0, "n_no_answer": 87, "point": 11.510791366906476, "point_raw": 0.11510791366906475, "raw_accuracy": 0.11510791366906475, "termination_rate": 0.35251798561151076 }, "tool_use": { "conditional_accuracy": 0.11301369863013698, "hi": 18.33452012383901, "hi_raw": 0.1833452012383901, "lo": 6.993085955312357, "lo_raw": 0.06993085955312357, "n": 146, "n_errors": 0, "n_no_answer": 6, "point": 12.333333333333332, "point_raw": 0.12333333333333332, "raw_accuracy": 0.1095890410958904, "termination_rate": 0.9863013698630136 } }, "axis_status": { "bfcl_multi_turn_long_context": { "axis": "bfcl_multi_turn_long_context", "reason": "not_run", "status": "not_measured" }, "bfcl_single_turn": { "axis": "bfcl_single_turn", "reason": "not_run", "status": "not_measured" }, "call_formatting": { "axis": "call_formatting", "reason": "not_run", "status": "not_measured" }, "coding": { "axis": "coding", "reason": "ok", "status": "measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "ok", "status": "measured" }, "tool_use": { "axis": "tool_use", "reason": "ok", "status": "measured" } }, "best_run_id": "gemma-4-12b-it__gemma-4-12b-it-qat-ud-q4kxl-s2v5", "catalog_id": "google/gemma-4-12B-it", "composite": { "hi": 44.73917327037827, "hi_raw": 0.4473917327037827, "lo": 39.400632850923486, "lo_raw": 0.3940063285092349, "point": 42.034174057819136, "point_raw": 0.42034174057819135 }, "composite_full": { "hi": 44.73917327037827, "hi_raw": 0.4473917327037827, "lo": 39.400632850923486, "lo_raw": 0.3940063285092349, "point": 42.034174057819136, "point_raw": 0.42034174057819135 }, "composite_static": { "hi": 54.96075895024066, "hi_raw": 0.5496075895024066, "lo": 48.921320541530626, "lo_raw": 0.4892132054153063, "point": 51.9344542993144, "point_raw": 0.519344542993144 }, "conformance_status": "headline-comparable", "contamination_label": "unverified", "est_cost_usd": null, "family": "Gemma 4", "gpu": { "driver": "610.62", "name": "NVIDIA GeForce RTX 5090", "vram_gb": 31.8, "vram_mb": 32607 }, "has_code_artifacts": true, "index_version": "index-v4.1", "kind": "maintainer_project", "lane": "bounded-final-v2", "latency_s_median": 60.219, "model_label": "Gemma 4 12B IT", "n_runs": 3, "origin": "project_anchor", "ranked": true, "replicated": false, "runtime": { "ctx_len_configured": 32768, "kv_cache_quant": "k=f16,v=f16", "name": "llama.cpp", "parallel_slots": 1, "version": "b9852/fd1a05791" }, "score_status": "measured", "slug": "gemma-4-12b-it", "static_index_version": "static-suite-v3", "tier": "standard", "tokens_to_answer_median": 8259.0, "tokens_to_answer_p95": 16384.0, "trust_label": "project_anchor", "verdict_source": "verifier", "wall_time_seconds": 61749.93416809998 }, { "axes": {}, "best_run_id": null, "catalog_id": "AEON-7/Gemma-4-12B-it-AEON-Abliterated-K4-BF16", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Gemma 4 12B it AEON Abliterated K4 BF16", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-12b-it-aeon-abliterated-k4-bf16", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "OBLITERATUS/Gemma-4-12B-OBLITERATED", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Gemma 4 12B OBLITERATED", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-12b-obliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-4-26B-A4B-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Gemma 4 26B A4B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-26b-a4b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "TrevorJS/gemma-4-26B-A4B-it-uncensored", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "gemma 4 26B A4B it uncensored", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-26b-a4b-it-uncensored", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "agentic_provenance": "project_attested", "axes": { "coding": { "conditional_accuracy": 0.33783783783783783, "hi": 41.21621621621622, "hi_raw": 0.41216216216216217, "lo": 26.351351351351347, "lo_raw": 0.2635135135135135, "n": 148, "n_errors": 0, "n_no_answer": 0, "point": 33.78378378378378, "point_raw": 0.33783783783783783, "raw_accuracy": 0.33783783783783783, "termination_rate": 1.0 }, "instruction": { "conditional_accuracy": 0.797945205479452, "hi": 84.01360544217688, "hi_raw": 0.8401360544217688, "lo": 74.48979591836735, "lo_raw": 0.7448979591836735, "n": 294, "n_errors": 0, "n_no_answer": 0, "point": 79.25170068027211, "point_raw": 0.7925170068027211, "raw_accuracy": 0.7925170068027211, "termination_rate": 0.9931972789115646 }, "knowledge": { "conditional_accuracy": 0.8875, "hi": 90.73884340226117, "hi_raw": 0.9073884340226117, "lo": 83.72281567670144, "lo_raw": 0.8372281567670145, "n": 400, "n_errors": 0, "n_no_answer": 1, "point": 87.3711500939925, "point_raw": 0.873711500939925, "raw_accuracy": 0.8875, "termination_rate": 1.0 }, "math": { "conditional_accuracy": 0.5150262768508597, "hi": 56.11510791366906, "hi_raw": 0.5611510791366906, "lo": 39.568345323741006, "lo_raw": 0.39568345323741005, "n": 139, "n_errors": 0, "n_no_answer": 6, "point": 48.201438848920866, "point_raw": 0.48201438848920863, "raw_accuracy": 0.48201438848920863, "termination_rate": 0.935251798561151 }, "tool_use": { "conditional_accuracy": 0.16095890410958905, "hi": 24.752688172043015, "hi_raw": 0.24752688172043014, "lo": 11.09074100513992, "lo_raw": 0.1109074100513992, "n": 146, "n_errors": 0, "n_no_answer": 0, "point": 17.65686274509804, "point_raw": 0.1765686274509804, "raw_accuracy": 0.16095890410958905, "termination_rate": 1.0 } }, "axis_status": { "bfcl_multi_turn_long_context": { "axis": "bfcl_multi_turn_long_context", "reason": "not_run", "status": "not_measured" }, "bfcl_single_turn": { "axis": "bfcl_single_turn", "reason": "not_run", "status": "not_measured" }, "call_formatting": { "axis": "call_formatting", "reason": "not_run", "status": "not_measured" }, "coding": { "axis": "coding", "reason": "ok", "status": "measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "ok", "status": "measured" }, "tool_use": { "axis": "tool_use", "reason": "ok", "status": "measured" } }, "best_run_id": "gemma-4-31b-it__gemma-4-31b-it-q4km-s2v5", "catalog_id": "google/gemma-4-31B-it", "composite": { "hi": 55.9553334838678, "hi_raw": 0.559553334838678, "lo": 50.259277560281475, "lo_raw": 0.5025927756028148, "point": 53.12081637550447, "point_raw": 0.5312081637550446 }, "composite_full": { "hi": 55.9553334838678, "hi_raw": 0.559553334838678, "lo": 50.259277560281475, "lo_raw": 0.5025927756028148, "point": 53.12081637550447, "point_raw": 0.5312081637550446 }, "composite_static": { "hi": 67.9233373913961, "hi_raw": 0.679233373913961, "lo": 62.01930257089381, "lo_raw": 0.6201930257089381, "point": 64.94213425230659, "point_raw": 0.6494213425230659 }, "conformance_status": "headline-comparable", "contamination_label": "unverified", "est_cost_usd": null, "family": "Gemma 4", "gpu": { "driver": "610.62", "name": "NVIDIA GeForce RTX 5090", "vram_gb": 31.8, "vram_mb": 32607 }, "has_code_artifacts": true, "index_version": "index-v4.1", "kind": "maintainer_project", "lane": "bounded-final-v2", "latency_s_median": 39.658, "model_label": "Gemma 4 31B IT", "n_runs": 2, "origin": "project_anchor", "ranked": true, "replicated": false, "runtime": { "ctx_len_configured": 32768, "kv_cache_quant": "k=f16,v=f16", "name": "llama.cpp", "parallel_slots": 1, "version": "b9852/fd1a05791" }, "score_status": "measured", "slug": "gemma-4-31b-it", "static_index_version": "static-suite-v3", "tier": "standard", "tokens_to_answer_median": 1918.0, "tokens_to_answer_p95": 9318.0, "trust_label": "project_anchor", "verdict_source": "verifier", "wall_time_seconds": 100257.26 }, { "axes": {}, "best_run_id": null, "catalog_id": "TrevorJS/gemma-4-31B-it-uncensored", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "gemma 4 31B it uncensored", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-31b-it-uncensored", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-4-E2B-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Gemma 4 E2B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-e2b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "google/gemma-4-E4B-it", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Gemma 4 E4B IT", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-e4b-it", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "OBLITERATUS/gemma-4-E4B-it-OBLITERATED", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "Gemma 4 E4B IT OBLITERATED", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-e4b-it-obliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "TrevorJS/gemma-4-E4B-it-uncensored", "composite": null, "demo": false, "est_cost_usd": null, "family": "Gemma 4", "kind": "community", "lane": "answer-only", "model_label": "gemma 4 E4B it uncensored", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gemma-4-e4b-it-uncensored", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "zai-org/GLM-5", "composite": null, "demo": false, "est_cost_usd": null, "family": "GLM 5", "kind": "community", "lane": "answer-only", "model_label": "GLM 5", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "glm-5", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "zai-org/GLM-5.1", "composite": null, "demo": false, "est_cost_usd": null, "family": "GLM 5", "kind": "community", "lane": "answer-only", "model_label": "GLM 5.1", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "glm-5-1", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "zai-org/GLM-5.2", "composite": null, "demo": false, "est_cost_usd": null, "family": "GLM 5", "kind": "community", "lane": "answer-only", "model_label": "GLM 5.2", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "glm-5-2", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Goedel-LM/Goedel-Prover-V2-32B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Goedel Prover V2 32B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "goedel-prover-v2-32b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Goedel-LM/Goedel-Prover-V2-8B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Goedel Prover V2 8B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "goedel-prover-v2-8b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "openai/gpt-oss-120b", "composite": null, "demo": false, "est_cost_usd": null, "family": "GPT OSS", "kind": "community", "lane": "answer-only", "model_label": "GPT OSS 120B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gpt-oss-120b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "openai/gpt-oss-20b", "composite": null, "demo": false, "est_cost_usd": null, "family": "GPT OSS", "kind": "community", "lane": "answer-only", "model_label": "GPT OSS 20B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "gpt-oss-20b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Huihui-Qwen3-14B-abliterated-v2", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Huihui Qwen3 14B abliterated v2", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "huihui-qwen3-14b-abliterated-v2", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Huihui-Qwen3-30B-A3B-Instruct-2507-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Huihui Qwen3 30B A3B Instruct 2507 abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "huihui-qwen3-30b-a3b-instruct-2507-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Huihui-Qwen3-30B-A3B-Thinking-2507-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Huihui Qwen3 30B A3B Thinking 2507 abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "huihui-qwen3-30b-a3b-thinking-2507-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Huihui-Qwen3-4B-Instruct-2507-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Huihui Qwen3 4B Instruct 2507 abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "huihui-qwen3-4b-instruct-2507-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Huihui-Qwen3-Coder-30B-A3B-Instruct-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3 Coder", "kind": "community", "lane": "answer-only", "model_label": "Huihui Qwen3 Coder 30B A3B Instruct abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "huihui-qwen3-coder-30b-a3b-instruct-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Applied-Innovation-Center/Karnak-40B-v1.0", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Karnak 40B v1.0", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "karnak-40b-v1-0", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-3.1-405B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.1", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.1 405B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-1-405b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-3.1-70B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.1", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.1 70B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-1-70b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-3.1-8B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.1", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.1 8B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-1-8b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.1", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.1 Nemotron 70B Instruct HF", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-1-nemotron-70b-instruct-hf", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-3.2-1B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.2", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.2 1B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-2-1b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-3.2-3B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.2", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.2 3B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-2-3b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-3.3-70B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.3", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.3 70B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-3-70b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Llama-3.3-70B-Instruct-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.3", "kind": "community", "lane": "answer-only", "model_label": "Llama 3.3 70B Instruct abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-3-3-70b-instruct-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 4", "kind": "community", "lane": "answer-only", "model_label": "Llama 4 Maverick 17B 128E Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-4-maverick-17b-128e-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 4", "kind": "community", "lane": "answer-only", "model_label": "Llama 4 Scout 17B 16E Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "llama-4-scout-17b-16e-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "bineric/lynx-instruct-30b", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "lynx instruct 30b", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "lynx-instruct-30b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Magistral-Small-2506", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Small", "kind": "community", "lane": "answer-only", "model_label": "Magistral Small 2506", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "magistral-small-2506", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Ministral-3-14B-Instruct-2512", "composite": null, "demo": false, "est_cost_usd": null, "family": "Ministral", "kind": "community", "lane": "answer-only", "model_label": "Ministral 3 14B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "ministral-3-14b-instruct-2512", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Ministral-3-3B-Instruct-2512", "composite": null, "demo": false, "est_cost_usd": null, "family": "Ministral", "kind": "community", "lane": "answer-only", "model_label": "Ministral 3 3B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "ministral-3-3b-instruct-2512", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Ministral-3-8B-Instruct-2512", "composite": null, "demo": false, "est_cost_usd": null, "family": "Ministral", "kind": "community", "lane": "answer-only", "model_label": "Ministral 3 8B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "ministral-3-8b-instruct-2512", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mistral-7B-Instruct-v0.2", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral", "kind": "community", "lane": "answer-only", "model_label": "Mistral 7B Instruct v0.2", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-7b-instruct-v0-2", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mistral-7B-Instruct-v0.3", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral", "kind": "community", "lane": "answer-only", "model_label": "Mistral 7B Instruct v0.3", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-7b-instruct-v0-3", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mistral-Medium-3.5-128B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Medium", "kind": "community", "lane": "answer-only", "model_label": "Mistral Medium 3.5 128B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-medium-3-5-128b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "DavidAU/Mistral-MOE-4X7B-Dark-MultiVerse-Uncensored-Enhanced32-24B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral", "kind": "community", "lane": "answer-only", "model_label": "Mistral MOE 4X7B Dark MultiVerse Uncensored Enhanced32 24B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-moe-4x7b-dark-multiverse-uncensored-enhanced32-24b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mistral-Nemo-Instruct-2407", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Nemo", "kind": "community", "lane": "answer-only", "model_label": "Mistral Nemo 12B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-nemo-instruct-2407", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mistral-Small-3.1-24B-Instruct-2503", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Small", "kind": "community", "lane": "answer-only", "model_label": "Mistral Small 3.1 24B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-small-3-1-24b-instruct-2503", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Small", "kind": "community", "lane": "answer-only", "model_label": "Mistral Small 3.2 24B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-small-3-2-24b-instruct-2506", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mistral-Small-Instruct-2409", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Small", "kind": "community", "lane": "answer-only", "model_label": "Mistral Small Instruct 2409", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mistral-small-instruct-2409", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mixtral-8x22B-Instruct-v0.1", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mixtral", "kind": "community", "lane": "answer-only", "model_label": "Mixtral 8x22B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mixtral-8x22b-instruct-v0-1", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mistralai/Mixtral-8x7B-Instruct-v0.1", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mixtral", "kind": "community", "lane": "answer-only", "model_label": "Mixtral 8x7B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "mixtral-8x7b-instruct-v0-1", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "nvidia/Nemotron-Research-Reasoning-Qwen-1.5B", "composite": null, "demo": false, "est_cost_usd": null, "family": "DeepSeek R1 Distill", "kind": "community", "lane": "answer-only", "model_label": "Nemotron Research Reasoning Qwen 1.5B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "nemotron-research-reasoning-qwen-1-5b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "nex-agi/Nex-N2-mini", "composite": null, "demo": false, "est_cost_usd": null, "family": "Nex N2", "kind": "community", "lane": "answer-only", "model_label": "Nex N2 Mini", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "nex-n2-mini", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "nex-agi/Nex-N2-Pro", "composite": null, "demo": false, "est_cost_usd": null, "family": "Nex N2", "kind": "community", "lane": "answer-only", "model_label": "Nex N2 Pro", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "nex-n2-pro", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "open-thoughts/OpenThinker2-32B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "OpenThinker2 32B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "openthinker2-32b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "open-thoughts/OpenThinker2-7B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "OpenThinker2 7B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "openthinker2-7b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "open-thoughts/OpenThinker3-7B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "OpenThinker3 7B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "openthinker3-7b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "microsoft/Phi-3-medium-4k-instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Phi 3", "kind": "community", "lane": "answer-only", "model_label": "Phi 3 Medium 4K Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "phi-3-medium-4k-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "microsoft/Phi-3-mini-4k-instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Phi 3", "kind": "community", "lane": "answer-only", "model_label": "Phi 3 Mini 4K Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "phi-3-mini-4k-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "microsoft/Phi-3-small-8k-instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Phi 3", "kind": "community", "lane": "answer-only", "model_label": "Phi 3 Small 8K Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "phi-3-small-8k-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "microsoft/phi-4", "composite": null, "demo": false, "est_cost_usd": null, "family": "Phi 4", "kind": "community", "lane": "answer-only", "model_label": "Phi 4", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "phi-4", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "microsoft/Phi-4-mini-instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Phi 4", "kind": "community", "lane": "answer-only", "model_label": "Phi 4 Mini Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "phi-4-mini-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "microsoft/Phi-4-reasoning", "composite": null, "demo": false, "est_cost_usd": null, "family": "Phi 4", "kind": "community", "lane": "answer-only", "model_label": "Phi 4 Reasoning", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "phi-4-reasoning", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-0.5B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 0.5B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-0-5b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-1.5B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 1.5B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-1-5b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-14B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 14B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-14b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-32B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 32B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-32b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Qwen2.5-32B-Instruct-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 32B Instruct abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-32b-instruct-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-3B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 3B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-3b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-72B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 72B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-72b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Qwen2.5-72B-Instruct-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 72B Instruct abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-72b-instruct-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-7B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 7B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-7b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Qwen2.5-7B-Instruct-abliterated-v2", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 7B Instruct abliterated v2", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-7b-instruct-abliterated-v2", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Orion-zhen/Qwen2.5-7B-Instruct-Uncensored", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 7B Instruct Uncensored", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-7b-instruct-uncensored", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-Coder-14B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5 Coder", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 Coder 14B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-coder-14b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-Coder-32B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5 Coder", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 Coder 32B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-coder-32b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-Coder-7B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5 Coder", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 Coder 7B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-coder-7b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-Math-1.5B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5 Math", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 Math 1.5B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-math-1-5b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen2.5-Math-7B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5 Math", "kind": "community", "lane": "answer-only", "model_label": "Qwen2.5 Math 7B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen2-5-math-7b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-0.6B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 0.6B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-0-6b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-1.7B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 1.7B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-1-7b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-14B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 14B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-14b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-235B-A22B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 235B A22B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-235b-a22b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 235B A22B Instruct 2507", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-235b-a22b-instruct-2507", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-235B-A22B-Thinking-2507", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 235B A22B Thinking 2507", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-235b-a22b-thinking-2507", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-30B-A3B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 30B A3B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-30b-a3b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "mlabonne/Qwen3-30B-A3B-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 30B A3B abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-30b-a3b-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 30B A3B Instruct 2507", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-30b-a3b-instruct-2507", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-30B-A3B-Thinking-2507", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 30B A3B Thinking 2507", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-30b-a3b-thinking-2507", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-32B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 32B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-32b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "huihui-ai/Qwen3-32B-abliterated", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 32B abliterated", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-32b-abliterated", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-4B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 4B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-4b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-4B-Instruct-2507", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 4B Instruct 2507", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-4b-instruct-2507", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-4B-SafeRL", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 4B SafeRL", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-4b-saferl", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-4B-Thinking-2507", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 4B Thinking 2507", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-4b-thinking-2507", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3.5-9B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.5", "kind": "community", "lane": "answer-only", "model_label": "Qwen3.5 9B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-5-9b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "agentic_provenance": "project_attested", "axes": { "coding": { "conditional_accuracy": 0.25, "hi": 31.08108108108108, "hi_raw": 0.3108108108108108, "lo": 17.56756756756757, "lo_raw": 0.17567567567567569, "n": 148, "n_errors": 0, "n_no_answer": 0, "point": 24.324324324324326, "point_raw": 0.24324324324324326, "raw_accuracy": 0.25, "termination_rate": 1.0 }, "instruction": { "conditional_accuracy": 0.6797153024911032, "hi": 70.40816326530613, "hi_raw": 0.7040816326530612, "lo": 59.523809523809526, "lo_raw": 0.5952380952380952, "n": 294, "n_errors": 0, "n_no_answer": 0, "point": 64.96598639455783, "point_raw": 0.6496598639455783, "raw_accuracy": 0.6496598639455783, "termination_rate": 0.95578231292517 }, "knowledge": { "conditional_accuracy": 0.8546365914786967, "hi": 87.3711500939925, "hi_raw": 0.873711500939925, "lo": 79.51319904136561, "lo_raw": 0.7951319904136561, "n": 400, "n_errors": 0, "n_no_answer": 1, "point": 83.44217456767905, "point_raw": 0.8344217456767905, "raw_accuracy": 0.8525, "termination_rate": 0.9975 }, "math": { "conditional_accuracy": 0.28106926223726897, "hi": 33.81294964028777, "hi_raw": 0.3381294964028777, "lo": 19.424460431654676, "lo_raw": 0.19424460431654678, "n": 139, "n_errors": 0, "n_no_answer": 9, "point": 26.618705035971225, "point_raw": 0.26618705035971224, "raw_accuracy": 0.26618705035971224, "termination_rate": 0.935251798561151 }, "tool_use": { "conditional_accuracy": 0.13356164383561644, "hi": 20.454032747119463, "hi_raw": 0.20454032747119463, "lo": 8.235294117647058, "lo_raw": 0.08235294117647059, "n": 146, "n_errors": 0, "n_no_answer": 0, "point": 13.96078431372549, "point_raw": 0.1396078431372549, "raw_accuracy": 0.13356164383561644, "termination_rate": 1.0 } }, "axis_status": { "bfcl_multi_turn_long_context": { "axis": "bfcl_multi_turn_long_context", "reason": "not_run", "status": "not_measured" }, "bfcl_single_turn": { "axis": "bfcl_single_turn", "reason": "not_run", "status": "not_measured" }, "call_formatting": { "axis": "call_formatting", "reason": "not_run", "status": "not_measured" }, "coding": { "axis": "coding", "reason": "ok", "status": "measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "ok", "status": "measured" }, "tool_use": { "axis": "tool_use", "reason": "ok", "status": "measured" } }, "best_run_id": "qwen3-6-27b__qwen3-6-27b-q4km-s2v5", "catalog_id": "Qwen/Qwen3.6-27B", "composite": { "hi": 47.046593158143565, "hi_raw": 0.4704659315814357, "lo": 41.66472896537073, "lo_raw": 0.4166472896537073, "point": 44.35140814560549, "point_raw": 0.44351408145605486 }, "composite_full": { "hi": 47.046593158143565, "hi_raw": 0.4704659315814357, "lo": 41.66472896537073, "lo_raw": 0.4166472896537073, "point": 44.35140814560549, "point_raw": 0.44351408145605486 }, "composite_static": { "hi": 57.39457366243394, "hi_raw": 0.5739457366243395, "lo": 51.55504812837714, "lo_raw": 0.5155504812837713, "point": 54.481616089565485, "point_raw": 0.5448161608956549 }, "conformance_status": "headline-comparable", "contamination_label": "unverified", "est_cost_usd": null, "family": "Qwen3.6", "gpu": { "driver": "610.62", "name": "NVIDIA GeForce RTX 5090", "vram_gb": 31.8, "vram_mb": 32607 }, "has_code_artifacts": true, "index_version": "index-v4.1", "kind": "maintainer_project", "lane": "bounded-final-v2", "latency_s_median": 47.375, "model_label": "Qwen3.6 27B", "n_runs": 1, "origin": "project_anchor", "ranked": true, "replicated": false, "runtime": { "ctx_len_configured": 32768, "kv_cache_quant": "k=f16,v=f16", "name": "llama.cpp", "parallel_slots": 1, "version": "b9852/fd1a05791" }, "score_status": "measured", "slug": "qwen3-6-27b", "static_index_version": "static-suite-v3", "tier": "standard", "tokens_to_answer_median": 3287.0, "tokens_to_answer_p95": 9380.0, "trust_label": "project_anchor", "verdict_source": "verifier", "wall_time_seconds": 85345.68213159998 }, { "axes": {}, "best_run_id": null, "catalog_id": "AEON-7/Qwen3.6-27B-AEON-Ultimate-Uncensored-BF16", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.6", "kind": "community", "lane": "answer-only", "model_label": "Qwen3.6 27B AEON Ultimate Uncensored BF16", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-6-27b-aeon-ultimate-uncensored-bf16", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "bytkim/Qwen3.6-27B-MTP-pi-tune", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.6", "kind": "community", "lane": "answer-only", "model_label": "Qwen3.6 27B MTP pi tune", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-6-27b-mtp-pi-tune", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "agentic_provenance": "project_attested", "axes": { "coding": { "conditional_accuracy": 0.2702702702702703, "hi": 34.45945945945946, "hi_raw": 0.34459459459459457, "lo": 20.27027027027027, "lo_raw": 0.20270270270270271, "n": 148, "n_errors": 0, "n_no_answer": 0, "point": 27.027027027027028, "point_raw": 0.2702702702702703, "raw_accuracy": 0.2702702702702703, "termination_rate": 1.0 }, "instruction": { "conditional_accuracy": 0.5971731448763251, "hi": 63.26530612244898, "hi_raw": 0.6326530612244898, "lo": 51.70068027210885, "lo_raw": 0.5170068027210885, "n": 294, "n_errors": 0, "n_no_answer": 0, "point": 57.48299319727891, "point_raw": 0.5748299319727891, "raw_accuracy": 0.5748299319727891, "termination_rate": 0.9625850340136054 }, "knowledge": { "conditional_accuracy": 0.8571428571428571, "hi": 85.96794454888055, "hi_raw": 0.8596794454888055, "lo": 77.82935238723128, "lo_raw": 0.7782935238723128, "n": 400, "n_errors": 0, "n_no_answer": 5, "point": 82.0389690225671, "point_raw": 0.820389690225671, "raw_accuracy": 0.84, "termination_rate": 0.98 }, "math": { "conditional_accuracy": 0.23701610140459062, "hi": 29.496402877697843, "hi_raw": 0.2949640287769784, "lo": 15.827338129496402, "lo_raw": 0.15827338129496402, "n": 139, "n_errors": 0, "n_no_answer": 6, "point": 22.302158273381295, "point_raw": 0.22302158273381295, "raw_accuracy": 0.22302158273381295, "termination_rate": 0.9424460431654677 }, "tool_use": { "conditional_accuracy": 0.1095890410958904, "hi": 17.591745442870902, "hi_raw": 0.175917454428709, "lo": 6.788355416423997, "lo_raw": 0.06788355416423997, "n": 146, "n_errors": 0, "n_no_answer": 2, "point": 11.911764705882351, "point_raw": 0.11911764705882352, "raw_accuracy": 0.1095890410958904, "termination_rate": 1.0 } }, "axis_status": { "bfcl_multi_turn_long_context": { "axis": "bfcl_multi_turn_long_context", "reason": "not_run", "status": "not_measured" }, "bfcl_single_turn": { "axis": "bfcl_single_turn", "reason": "not_run", "status": "not_measured" }, "call_formatting": { "axis": "call_formatting", "reason": "not_run", "status": "not_measured" }, "coding": { "axis": "coding", "reason": "ok", "status": "measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "ok", "status": "measured" }, "tool_use": { "axis": "tool_use", "reason": "ok", "status": "measured" } }, "best_run_id": "qwen3-6-35b-a3b__qwen3-6-35b-a3b-udq4-s2v5", "catalog_id": "Qwen/Qwen3.6-35B-A3B", "composite": { "hi": 44.85608415283716, "hi_raw": 0.4485608415283716, "lo": 39.440871606595834, "lo_raw": 0.39440871606595834, "point": 42.12412562752062, "point_raw": 0.4212412562752062 }, "composite_full": { "hi": 44.85608415283716, "hi_raw": 0.4485608415283716, "lo": 39.440871606595834, "lo_raw": 0.39440871606595834, "point": 42.12412562752062, "point_raw": 0.4212412562752062 }, "composite_static": { "hi": 55.26431126677264, "hi_raw": 0.5526431126677264, "lo": 49.196607455783656, "lo_raw": 0.4919660745578366, "point": 52.19491260140005, "point_raw": 0.5219491260140005 }, "conformance_status": "headline-comparable", "contamination_label": "unverified", "est_cost_usd": null, "family": "Qwen3.6", "gpu": { "driver": "610.62", "name": "NVIDIA GeForce RTX 5090", "vram_gb": 31.8, "vram_mb": 32607 }, "has_code_artifacts": true, "index_version": "index-v4.1", "kind": "maintainer_project", "lane": "bounded-final-v2", "latency_s_median": 19.071, "model_label": "Qwen3.6 35B A3B", "n_runs": 1, "origin": "project_anchor", "ranked": true, "replicated": false, "runtime": { "ctx_len_configured": 32768, "kv_cache_quant": "k=f16,v=f16", "name": "llama.cpp", "parallel_slots": 1, "version": "b9852/fd1a05791" }, "score_status": "measured", "slug": "qwen3-6-35b-a3b", "static_index_version": "static-suite-v3", "tier": "standard", "tokens_to_answer_median": 3639.0, "tokens_to_answer_p95": 9377.0, "trust_label": "project_anchor", "verdict_source": "verifier", "wall_time_seconds": 33278.309414800024 }, { "axes": {}, "best_run_id": null, "catalog_id": "lordx64/Qwen3.6-35B-A3B-Claude-4.7-Opus-Reasoning-Distilled", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.6", "kind": "community", "lane": "answer-only", "model_label": "Qwen3.6 35B A3B Claude 4.7 Opus Reasoning Distilled", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-6-35b-a3b-claude-4-7-opus-reasoning-distilled", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "HauhauCS/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.6", "kind": "community", "lane": "answer-only", "model_label": "Qwen3.6 35B A3B Uncensored HauhauCS Aggressive", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-6-35b-a3b-uncensored-hauhaucs-aggressive", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-8B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 8B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-8b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3 Coder", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 Coder 30B A3B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-coder-30b-a3b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": { "instruction": { "conditional_accuracy": 0.3125, "hi": 36.054421768707485, "hi_raw": 0.36054421768707484, "lo": 25.510204081632654, "lo_raw": 0.25510204081632654, "n": 294, "n_errors": 5, "n_no_answer": 0, "point": 30.612244897959183, "point_raw": 0.30612244897959184, "raw_accuracy": 0.30612244897959184, "termination_rate": 0.9795918367346939 }, "knowledge": { "conditional_accuracy": 0.8459595959595958, "hi": 85.68730343985817, "hi_raw": 0.8568730343985816, "lo": 77.54871127820888, "lo_raw": 0.7754871127820888, "n": 400, "n_errors": 0, "n_no_answer": 2, "point": 81.75832791354473, "point_raw": 0.8175832791354474, "raw_accuracy": 0.8375, "termination_rate": 0.99 }, "tool_calling": { "conditional_accuracy": 0.7545454545454545, "hi": 80.0, "hi_raw": 0.8, "lo": 70.60606060606061, "lo_raw": 0.706060606060606, "n": 330, "n_errors": 0, "n_no_answer": 5, "point": 75.45454545454545, "point_raw": 0.7545454545454545, "raw_accuracy": 0.7545454545454545, "termination_rate": 1.0 } }, "axis_status": { "agentic": { "axis": "agentic", "reason": "ok", "status": "measured" }, "coding": { "axis": "coding", "reason": "not_run", "status": "not_measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "not_run", "status": "not_measured" }, "tool_calling": { "axis": "tool_calling", "reason": "ok", "status": "measured" } }, "best_run_id": "qwen3-coder-next__qwen3-coder-next-q8", "catalog_id": "Qwen/Qwen3-Coder-Next", "composite": null, "composite_full": null, "composite_static": null, "conformance_status": "headline-comparable", "contamination_label": "unverified", "diagnostic_composite": { "hi": 65.65314695321212, "hi_raw": 0.6565314695321212, "lo": 56.29860841145572, "lo_raw": 0.5629860841145572, "point": 61.00260116795033, "point_raw": 0.6100260116795033 }, "est_cost_usd": null, "family": "Qwen3 Coder", "gpu": { "driver": "595.58.03", "name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", "vram_gb": 95.6, "vram_mb": 97887 }, "kind": "community", "lane": "capped-thinking", "latency_s_median": 1.199, "model_label": "Qwen3 Coder Next", "n_runs": 2, "origin": "community", "ranked": false, "replicated": false, "runtime": { "ctx_len_configured": null, "kv_cache_quant": "unknown", "name": null, "parallel_slots": null, "version": null }, "score_status": "measured", "slug": "qwen3-coder-next", "tier": "standard", "tokens_to_answer_median": 276.0, "tokens_to_answer_p95": 3355.8999999999937, "trust_label": "community_re_scored", "wall_time_seconds": 3528.822674885858 }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3 Next", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 Next 80B A3B Instruct", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-next-80b-a3b-instruct", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/Qwen3-Next-80B-A3B-Thinking", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3 Next", "kind": "community", "lane": "answer-only", "model_label": "Qwen3 Next 80B A3B Thinking", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwen3-next-80b-a3b-thinking", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Jackrong/Qwopus3.6-27B-v1-preview", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.6", "kind": "community", "lane": "answer-only", "model_label": "Qwopus 3.6 27B v1 Preview", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwopus3-6-27b-v1-preview", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "agentic_provenance": "project_attested", "axes": { "coding": { "conditional_accuracy": 0.28378378378378377, "hi": 35.810810810810814, "hi_raw": 0.3581081081081081, "lo": 20.945945945945947, "lo_raw": 0.20945945945945946, "n": 148, "n_errors": 0, "n_no_answer": 0, "point": 28.37837837837838, "point_raw": 0.28378378378378377, "raw_accuracy": 0.28378378378378377, "termination_rate": 1.0 }, "instruction": { "conditional_accuracy": 0.6159420289855072, "hi": 63.26530612244898, "hi_raw": 0.6326530612244898, "lo": 52.04081632653062, "lo_raw": 0.5204081632653061, "n": 294, "n_errors": 0, "n_no_answer": 0, "point": 57.82312925170068, "point_raw": 0.5782312925170068, "raw_accuracy": 0.5782312925170068, "termination_rate": 0.9387755102040817 }, "knowledge": { "conditional_accuracy": 0.8505154639175259, "hi": 84.28409789474622, "hi_raw": 0.8428409789474622, "lo": 76.14550573309694, "lo_raw": 0.7614550573309694, "n": 400, "n_errors": 0, "n_no_answer": 12, "point": 80.35512236843277, "point_raw": 0.8035512236843277, "raw_accuracy": 0.825, "termination_rate": 0.97 }, "math": { "conditional_accuracy": 0.3543609556799005, "hi": 33.093525179856115, "hi_raw": 0.33093525179856115, "lo": 18.705035971223023, "lo_raw": 0.18705035971223022, "n": 139, "n_errors": 0, "n_no_answer": 37, "point": 25.899280575539567, "point_raw": 0.2589928057553957, "raw_accuracy": 0.2589928057553957, "termination_rate": 0.697841726618705 }, "tool_use": { "conditional_accuracy": 0.14383561643835616, "hi": 22.224610555684727, "hi_raw": 0.22224610555684726, "lo": 9.403274711946635, "lo_raw": 0.09403274711946635, "n": 146, "n_errors": 0, "n_no_answer": 0, "point": 15.397058823529411, "point_raw": 0.1539705882352941, "raw_accuracy": 0.14383561643835616, "termination_rate": 1.0 } }, "axis_status": { "bfcl_multi_turn_long_context": { "axis": "bfcl_multi_turn_long_context", "reason": "not_run", "status": "not_measured" }, "bfcl_single_turn": { "axis": "bfcl_single_turn", "reason": "not_run", "status": "not_measured" }, "call_formatting": { "axis": "call_formatting", "reason": "not_run", "status": "not_measured" }, "coding": { "axis": "coding", "reason": "ok", "status": "measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "ok", "status": "measured" }, "tool_use": { "axis": "tool_use", "reason": "ok", "status": "measured" } }, "best_run_id": "qwopus3-6-27b-v2-mtp__qwopus3-6-27b-v2-mtp-q4km-s2v5", "catalog_id": "Jackrong/Qwopus3.6-27B-v2-MTP", "composite": { "hi": 46.10909164556549, "hi_raw": 0.4610909164556549, "lo": 40.42161414657037, "lo_raw": 0.4042161414657037, "point": 43.26695249871298, "point_raw": 0.43266952498712985 }, "composite_full": { "hi": 46.10909164556549, "hi_raw": 0.4610909164556549, "lo": 40.42161414657037, "lo_raw": 0.4042161414657037, "point": 43.26695249871298, "point_raw": 0.43266952498712985 }, "composite_static": { "hi": 55.62104131542001, "hi_raw": 0.5562104131542001, "lo": 49.487356574038394, "lo_raw": 0.49487356574038394, "point": 52.5569170571075, "point_raw": 0.525569170571075 }, "conformance_status": "headline-comparable", "contamination_label": "unverified", "est_cost_usd": null, "family": "Qwen3.6", "gpu": { "driver": "610.62", "name": "NVIDIA GeForce RTX 5090", "vram_gb": 31.8, "vram_mb": 32607 }, "has_code_artifacts": true, "index_version": "index-v4.1", "kind": "maintainer_project", "lane": "bounded-final-v2", "latency_s_median": 25.451, "model_label": "Qwopus 3.6 27B v2 MTP", "n_runs": 1, "origin": "project_anchor", "ranked": true, "replicated": false, "runtime": { "ctx_len_configured": 32768, "kv_cache_quant": "k=f16,v=f16", "name": "llama.cpp", "parallel_slots": 1, "version": "b9852/fd1a05791" }, "score_status": "measured", "slug": "qwopus3-6-27b-v2-mtp", "static_index_version": "static-suite-v3", "tier": "standard", "tokens_to_answer_median": 1623.0, "tokens_to_answer_p95": 16384.0, "trust_label": "project_anchor", "verdict_source": "verifier", "wall_time_seconds": 85136.51475749997 }, { "axes": {}, "best_run_id": null, "catalog_id": "Qwen/QwQ-32B", "composite": null, "demo": false, "est_cost_usd": null, "family": "QwQ", "kind": "community", "lane": "answer-only", "model_label": "QwQ 32B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwq-32b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "empero-ai/Qwythos-9B-Claude-Mythos-5-1M", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.5", "kind": "community", "lane": "answer-only", "model_label": "Qwythos 9B Claude Mythos 5 1M", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwythos-9b-claude-mythos-5-1m", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "empero-ai/Qwythos-9B-v2", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.5", "kind": "community", "lane": "answer-only", "model_label": "Qwythos 9B v2", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "qwythos-9b-v2", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "prefeitura-rio/Rio-3.0-Open-Mini", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "Rio 3.0 Open Mini", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "rio-3-0-open-mini", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "BytedTsinghua-SIA/RL-MemoryAgent-14B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "RL MemoryAgent 14B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "rl-memoryagent-14b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "AlexWortega/SIQ-1-35B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3.6", "kind": "community", "lane": "answer-only", "model_label": "SIQ 1 35B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "siq-1-35b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "t-tech/T-pro-it-2.1", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen3", "kind": "community", "lane": "answer-only", "model_label": "T pro it 2.1", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "t-pro-it-2-1", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "SakanaAI/TinySwallow-1.5B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "TinySwallow 1.5B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "tinyswallow-1-5b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Team-ACE/ToolACE-8B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.1", "kind": "community", "lane": "answer-only", "model_label": "ToolACE 8B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "toolace-8b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Vikhrmodels/Vikhr-Llama3.1-8B-Instruct-R-21-09-24", "composite": null, "demo": false, "est_cost_usd": null, "family": "Llama 3.1", "kind": "community", "lane": "answer-only", "model_label": "Vikhr Llama3.1 8B Instruct R 21 09 24", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "vikhr-llama3-1-8b-instruct-r-21-09-24", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Vikhrmodels/Vikhr-Nemo-12B-Instruct-R-21-09-24", "composite": null, "demo": false, "est_cost_usd": null, "family": "Mistral Nemo", "kind": "community", "lane": "answer-only", "model_label": "Vikhr Nemo 12B Instruct R 21 09 24", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "vikhr-nemo-12b-instruct-r-21-09-24", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "vectionlabs/VL-1-Coder", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5 Coder", "kind": "community", "lane": "answer-only", "model_label": "VL 1 Coder", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "vl-1-coder", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "Virtue-AI-HUB/VulnLLM-R-7B", "composite": null, "demo": false, "est_cost_usd": null, "family": "Qwen2.5", "kind": "community", "lane": "answer-only", "model_label": "VulnLLM R 7B", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "vulnllm-r-7b", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "01-ai/Yi-1.5-34B-Chat", "composite": null, "demo": false, "est_cost_usd": null, "family": "Yi 1.5", "kind": "community", "lane": "answer-only", "model_label": "Yi 1.5 34B Chat", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "yi-1-5-34b-chat", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "01-ai/Yi-1.5-6B-Chat", "composite": null, "demo": false, "est_cost_usd": null, "family": "Yi 1.5", "kind": "community", "lane": "answer-only", "model_label": "Yi 1.5 6B Chat", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "yi-1-5-6b-chat", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "01-ai/Yi-1.5-9B-Chat", "composite": null, "demo": false, "est_cost_usd": null, "family": "Yi 1.5", "kind": "community", "lane": "answer-only", "model_label": "Yi 1.5 9B Chat", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "yi-1-5-9b-chat", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": {}, "best_run_id": null, "catalog_id": "altomek/YiSM-34B-0rn", "composite": null, "demo": false, "est_cost_usd": null, "family": "Yi 1.5", "kind": "community", "lane": "answer-only", "model_label": "YiSM 34B 0rn", "n_runs": 0, "ranked": false, "replicated": false, "score_status": "missing", "slug": "yism-34b-0rn", "tier": null, "tokens_to_answer_median": null, "tokens_to_answer_p95": null }, { "axes": { "instruction": { "conditional_accuracy": 0.3402777777777778, "hi": 38.775510204081634, "hi_raw": 0.3877551020408163, "lo": 27.89115646258503, "lo_raw": 0.2789115646258503, "n": 294, "n_errors": 6, "n_no_answer": 0, "point": 33.33333333333333, "point_raw": 0.3333333333333333, "raw_accuracy": 0.3333333333333333, "termination_rate": 0.9795918367346939 }, "knowledge": { "conditional_accuracy": 0.6766917293233082, "hi": 68.56819578949244, "hi_raw": 0.6856819578949244, "lo": 58.46511586468643, "lo_raw": 0.5846511586468643, "n": 400, "n_errors": 0, "n_no_answer": 4, "point": 63.516655827089444, "point_raw": 0.6351665582708944, "raw_accuracy": 0.675, "termination_rate": 0.9975 }, "tool_calling": { "conditional_accuracy": 0.6636363636363637, "hi": 71.21212121212122, "hi_raw": 0.7121212121212122, "lo": 61.212121212121204, "lo_raw": 0.6121212121212121, "n": 330, "n_errors": 0, "n_no_answer": 32, "point": 66.36363636363637, "point_raw": 0.6636363636363637, "raw_accuracy": 0.6636363636363637, "termination_rate": 1.0 } }, "axis_status": { "agentic": { "axis": "agentic", "reason": "ok", "status": "measured" }, "coding": { "axis": "coding", "reason": "not_run", "status": "not_measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "not_run", "status": "not_measured" }, "tool_calling": { "axis": "tool_calling", "reason": "ok", "status": "measured" } }, "best_run_id": "gemma-4-12b-coder-fable5__gemma-4-12b-coder-fable5-Q8_0", "composite": null, "composite_full": null, "composite_static": null, "conformance_status": "headline-comparable", "contamination_label": "unverified", "diagnostic_composite": { "hi": 58.056920050620576, "hi_raw": 0.5805692005062058, "lo": 47.686632425757104, "lo_raw": 0.47686632425757103, "point": 52.909655026067625, "point_raw": 0.5290965502606763 }, "est_cost_usd": null, "family": "Gemma 4 Coder", "gpu": { "driver": "596.36", "name": "NVIDIA GeForce RTX 5090", "vram_gb": 31.8, "vram_mb": 32607 }, "kind": "community", "lane": "capped-thinking", "latency_s_median": 2.578, "model_label": "Gemma 4 12B Coder Fable5", "n_runs": 1, "origin": "community", "ranked": false, "replicated": false, "runtime": { "ctx_len_configured": null, "kv_cache_quant": "unknown", "name": null, "parallel_slots": null, "version": null }, "score_status": "measured", "slug": "gemma-4-12b-coder-fable5", "tier": "standard", "tokens_to_answer_median": 132.0, "tokens_to_answer_p95": 583.0, "trust_label": "community_re_scored", "wall_time_seconds": 4437.270994546001 }, { "axes": { "instruction": { "conditional_accuracy": 0.3275862068965517, "hi": 37.755102040816325, "hi_raw": 0.37755102040816324, "lo": 26.87074829931973, "lo_raw": 0.2687074829931973, "n": 294, "n_errors": 4, "n_no_answer": 0, "point": 32.31292517006803, "point_raw": 0.3231292517006803, "raw_accuracy": 0.3231292517006803, "termination_rate": 0.9863945578231292 }, "knowledge": { "conditional_accuracy": 0.8270676691729322, "hi": 84.56473900376861, "hi_raw": 0.8456473900376861, "lo": 76.14550573309694, "lo_raw": 0.7614550573309694, "n": 400, "n_errors": 0, "n_no_answer": 1, "point": 80.35512236843277, "point_raw": 0.8035512236843277, "raw_accuracy": 0.825, "termination_rate": 0.9975 }, "tool_calling": { "conditional_accuracy": 0.7242424242424242, "hi": 77.27272727272727, "hi_raw": 0.7727272727272727, "lo": 67.57575757575758, "lo_raw": 0.6757575757575758, "n": 330, "n_errors": 0, "n_no_answer": 2, "point": 72.42424242424242, "point_raw": 0.7242424242424242, "raw_accuracy": 0.7242424242424242, "termination_rate": 1.0 } }, "axis_status": { "agentic": { "axis": "agentic", "reason": "ok", "status": "measured" }, "coding": { "axis": "coding", "reason": "not_run", "status": "not_measured" }, "instruction_following": { "axis": "instruction_following", "reason": "ok", "status": "measured" }, "knowledge": { "axis": "knowledge", "reason": "ok", "status": "measured" }, "long_context": { "axis": "long_context", "reason": "not_run", "status": "not_measured" }, "math": { "axis": "math", "reason": "not_run", "status": "not_measured" }, "tool_calling": { "axis": "tool_calling", "reason": "ok", "status": "measured" } }, "best_run_id": "ornith-1-0-35b__ornith-1.0-35b-q6", "composite": null, "composite_full": null, "composite_static": null, "conformance_status": "headline-comparable", "contamination_label": "unverified", "diagnostic_composite": { "hi": 65.18812220990117, "hi_raw": 0.6518812220990117, "lo": 55.52503465609564, "lo_raw": 0.5552503465609564, "point": 60.35657843299841, "point_raw": 0.6035657843299841 }, "est_cost_usd": null, "family": "Ornith", "gpu": { "driver": "595.58.03", "name": "NVIDIA RTX PRO 6000 Blackwell Workstation Edition", "vram_gb": 95.6, "vram_mb": 97887 }, "kind": "community", "lane": "capped-thinking", "latency_s_median": 1.178, "model_label": "Ornith 1.0 35B", "n_runs": 4, "origin": "community", "ranked": false, "replicated": false, "runtime": { "ctx_len_configured": null, "kv_cache_quant": "unknown", "name": null, "parallel_slots": null, "version": null }, "score_status": "measured", "slug": "ornith-1-0-35b", "tier": "standard", "tokens_to_answer_median": 275.0, "tokens_to_answer_p95": 1735.149999999998, "trust_label": "community_re_scored", "wall_time_seconds": 2448.0192751688883 } ], "suite_version": "mixed" }