{"schemaVersion":2,"publishedDate":"2026-10-02","benchmarkWindow":"September 2026","machine":"AMD Ryzen AI Max+ 395 mini PC; 128 GB unified memory","modelBenchmark":{"description":"Twenty automatically graded scenarios; 11–20 are the hard half","scoreMaximum":20,"scenarios":[{"n":1,"label":"Python algorithm (interval merge)","category":"Code","hard":false},{"n":2,"label":"Python bug fix (date arithmetic)","category":"Code","hard":false},{"n":3,"label":"Compound-interest math","category":"Math","hard":false},{"n":4,"label":"Tiered tax calculation","category":"Math","hard":false},{"n":5,"label":"JavaScript ratio stress test","category":"Code","hard":false},{"n":6,"label":"SQL: best value per group","category":"Data","hard":false},{"n":7,"label":"Single tool call","category":"Tools","hard":false},{"n":8,"label":"Bilingual email → JSON","category":"Data","hard":false},{"n":9,"label":"Long context (~18k tokens)","category":"Context","hard":false},{"n":10,"label":"Predict program output","category":"Reasoning","hard":false},{"n":11,"label":"Payment schedule with prepayments","category":"Math","hard":true},{"n":12,"label":"Two-formula penalty comparison","category":"Math","hard":true},{"n":13,"label":"Expression parser, no eval()","category":"Code","hard":true},{"n":14,"label":"JS async retry with backoff","category":"Code","hard":true},{"n":15,"label":"Exact set cover","category":"Reasoning","hard":true},{"n":16,"label":"SQL window functions","category":"Data","hard":true},{"n":17,"label":"Multi-file bug hunt","category":"Code","hard":true},{"n":18,"label":"Multi-hop long context","category":"Context","hard":true},{"n":19,"label":"Multi-step tool use","category":"Tools","hard":true},{"n":20,"label":"Logic puzzle","category":"Reasoning","hard":true}],"records":[{"id":"gemma4-31b-qat","name":"Gemma 4 31B QAT","quant":"UD-Q4_K_XL","status":"current","role":"Planner · reviewer","runs":[20,20,20],"hard":[10,10,10],"scenarios":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"budgetAt":[],"misses":0,"budgetMisses":0,"genTps":10.6,"promptTps":183,"loadSeconds":9,"dates":["2026-09-27","2026-09-28","2026-09-29"],"note":"Perfect in all three runs. Slow, careful, never ran out of budget."},{"id":"gpt-oss-120b","name":"gpt-oss-120b","quant":"MXFP4","status":"current","role":"Builder · daily driver","runs":[19.96,19.72,19.96],"hard":[9.96,9.72,9.96],"scenarios":[1,1,1,1,1,1,1,1,1,1,1,1,0.88,1,1,1,1,1,1,1],"budgetAt":[],"misses":3,"budgetMisses":0,"genTps":48.5,"promptTps":248,"loadSeconds":24,"dates":["2026-09-27","2026-09-28","2026-09-29"],"note":"Only ever dropped points on the expression parser — and still passed most of its hidden cases. Near-perfect at 4–5× Gemma’s speed."},{"id":"glm53-flash","name":"GLM-5.3-Flash","quant":"UD-IQ3_XXS","status":"contender","role":"Heavyweight contender","runs":[20],"hard":[10],"scenarios":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],"budgetAt":[],"misses":0,"budgetMisses":0,"genTps":13.4,"promptTps":60,"loadSeconds":45,"dates":["2026-09-30"],"note":"Froze the GPU twice on a Vulkan build; perfect on ROCm. One run so far."},{"id":"nemotron3-super-120b","name":"Nemotron 3 Super 120B","quant":"UD-Q4_K_XL","status":"contender","role":"Contender","runs":[19,19,17],"hard":[9,9,7],"scenarios":[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,0.67,1,0.67,1,1],"budgetAt":[13,18],"misses":5,"budgetMisses":4,"genTps":16.8,"promptTps":137,"loadSeconds":30,"dates":["2026-09-27","2026-09-28","2026-09-29"],"note":"Strong, but the parser exhausted its budget in every run."},{"id":"muse-glimmer-30b","name":"Muse Glimmer 30B","quant":"Q8_0","status":"contender","role":"Contender","runs":[19],"hard":[9],"scenarios":[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1],"budgetAt":[13],"misses":1,"budgetMisses":1,"genTps":7.6,"promptTps":149,"loadSeconds":12,"dates":["2026-09-26"],"note":"One miss, on the parser, from budget. The slowest model tested."},{"id":"qwen3-coder-next-q4","name":"Qwen3-Coder-Next","quant":"UD-Q4_K_XL","status":"contender","role":"Contender","runs":[18.56,19.04,18.29],"hard":[8.56,9.04,8.79],"scenarios":[1,1,1,1,1,1,1,1,1,0.83,1,1,0.69,1,1,1,0.9,1,1,0.2],"budgetAt":[],"misses":8,"budgetMisses":0,"genTps":46.8,"promptTps":265,"loadSeconds":18,"dates":["2026-09-26","2026-09-28","2026-09-29"],"note":"Fast and steady. Failed the logic puzzle the same way in all three runs."},{"id":"deepseek-v4-flash","name":"DeepSeek V4-Flash","quant":"UD-IQ3_XXS","status":"contender","role":"Contender","runs":[18],"hard":[8],"scenarios":[1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1],"budgetAt":[13,17],"misses":2,"budgetMisses":2,"genTps":11.8,"promptTps":72,"loadSeconds":36,"dates":["2026-09-27"],"note":"Both misses were unfinished answers, not wrong ones."},{"id":"kat-coder-v2.5","name":"KAT-Coder v2.5","quant":"Q8_0","status":"retired","role":"Retired","runs":[17.55],"hard":[7.55],"scenarios":[1,1,1,1,1,1,1,1,1,1,0.83,1,0,1,1,1,0.71,1,1,0],"budgetAt":[13],"misses":4,"budgetMisses":1,"genTps":51.4,"promptTps":368,"loadSeconds":15,"dates":["2026-09-26"]},{"id":"qwen3-coder-next-q6","name":"Qwen3-Coder-Next","quant":"Q6","status":"retired","role":"Retired","runs":[17.36],"hard":[8.16],"scenarios":[1,1,1,1,1,0.2,1,1,1,1,1,1,0.76,0,1,1,1,1,1,0.4],"budgetAt":[],"misses":4,"budgetMisses":0,"genTps":41,"promptTps":226,"loadSeconds":27,"dates":["2026-09-26"],"note":"The larger quant did not beat Q4 here."},{"id":"qwen38-flash-next-ple","name":"Qwen3.8-Flash-Next","quant":"IQ4_XS-PLE · fixed sampling","status":"retired","role":"Retired","runs":[17],"hard":[7],"scenarios":[1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,1,0,1,1,1],"budgetAt":[11,13,17],"misses":3,"budgetMisses":3,"genTps":28.2,"promptTps":274,"loadSeconds":33,"dates":["2026-09-30"]},{"id":"qwen38-flash-next-ud","name":"Qwen3.8-Flash-Next","quant":"UD-IQ4_XS · fixed sampling","status":"retired","role":"Retired","runs":[17],"hard":[7],"scenarios":[1,1,1,1,1,1,1,1,1,1,0,1,0,1,0,1,1,1,1,1],"budgetAt":[11,13,15],"misses":3,"budgetMisses":3,"genTps":23.6,"promptTps":266,"loadSeconds":36,"dates":["2026-09-30"],"note":"Every miss was budget exhaustion — it thinks past its output limit."},{"id":"tiel-coder-35b","name":"Tiel-Coder 35B","quant":"Q8_0","status":"retired","role":"Retired","runs":[16.92],"hard":[7.8],"scenarios":[0.12,1,1,1,1,1,1,1,1,1,1,1,0,1,0.8,0,1,1,1,1],"budgetAt":[13],"misses":4,"budgetMisses":1,"genTps":43.6,"promptTps":284,"loadSeconds":15,"dates":["2026-09-26"]},{"id":"devstral2-24b","name":"Devstral 2 24B","quant":"Q8_0","status":"retired","role":"Retired","runs":[14.76],"hard":[7.76],"scenarios":[1,0.5,0,1,0,0.5,1,1,1,1,0,1,0.76,1,0.8,1,1,1,1,0.2],"budgetAt":[],"misses":8,"budgetMisses":0,"genTps":8.5,"promptTps":217,"loadSeconds":12,"dates":["2026-09-26"],"note":"Better on the hard half than the standard half — its misses were genuine errors."},{"id":"qwen38-27b","name":"Qwen3.8-27B","quant":"Q8_0","status":"audit","role":"Sampling audit open","runs":[14.75],"hard":[6],"scenarios":[1,1,0,1,1,1,1,1,1,0.75,0,1,0,1,0,0,1,1,1,1],"budgetAt":[3,13,15,16],"misses":6,"budgetMisses":4,"genTps":7.5,"promptTps":155,"loadSeconds":12,"dates":["2026-09-26"],"note":"Request settings are still under audit; treat this score with caution."},{"id":"gemma4-26b-a4b","name":"Gemma 4 26B-A4B QAT","quant":"UD-Q4_K_XL","status":"retired","role":"Retired","runs":[14],"hard":[6],"scenarios":[1,0,1,1,1,1,1,1,1,0,1,1,0,1,0,1,1,0,1,0],"budgetAt":[2,10,13,15,18,20],"misses":6,"budgetMisses":6,"genTps":57.5,"promptTps":635,"loadSeconds":6,"dates":["2026-09-27"],"note":"The fastest model tested. All six misses were budget exhaustion."},{"id":"gemma4-12b","name":"Gemma 4 12B QAT","quant":"UD-Q4_K_XL","status":"retired","role":"Retired","runs":[13],"hard":[4],"scenarios":[1,1,1,1,1,1,1,1,1,0,0,1,0,1,0,1,0,0,1,0],"budgetAt":[10,11,13,15,17,18,20],"misses":7,"budgetMisses":7,"genTps":25.5,"promptTps":301,"loadSeconds":6,"dates":["2026-09-30"],"note":"Clean sweep of the standard half; every hard miss was unfinished."},{"id":"mistral-small4-119b","name":"Mistral Small 4 119B","quant":"UD-Q4_K_XL","status":"retired","role":"Retired","runs":[11.42],"hard":[5.17],"scenarios":[1,0.38,0,1,0.38,0.5,1,1,0.5,0.5,0.5,1,0.56,1,0.4,0,0.71,0,1,0],"budgetAt":[],"misses":13,"budgetMisses":0,"genTps":36.1,"promptTps":163,"loadSeconds":30,"dates":["2026-09-27"],"note":"Finished every answer — and got many of them partly wrong."}]},"codingHarnessBenchmark":{"description":"Six repository tasks with hidden tests per model and harness","scoreMaximum":6,"tasks":["Feature: tax rebate + CLI (Python)","Multi-file bug hunt (Python)","Implement JS from a written spec","Cross-file rename + deprecated alias","Messy CSV → JSON script","Debug a timezone / loop bug (Node)"],"openCodePermissionsFix":{"before":1.04,"after":6},"records":[{"model":"gpt-oss-120b","cells":{"Pi":{"score":6,"minutes":4},"OpenCode":{"score":6,"minutes":6},"dsh":{"score":6,"minutes":6},"Hermes":{"score":6,"minutes":11}}},{"model":"Gemma 4 31B QAT","cells":{"Pi":{"score":6,"minutes":24},"OpenCode":{"score":6,"minutes":34},"dsh":{"score":6,"minutes":41},"Hermes":{"score":6,"minutes":26}}},{"model":"Nemotron 3 Super 120B","cells":{"Pi":{"score":6,"minutes":25},"OpenCode":{"score":null,"minutes":null,"note":"Not re-run after the permissions fix"},"dsh":{"score":6,"minutes":26},"Hermes":{"score":6,"minutes":35}}},{"model":"Qwen3-Coder-Next Q4","cells":{"Pi":{"score":5.83,"minutes":4},"OpenCode":{"score":null,"minutes":null,"note":"Not re-run after the permissions fix"},"dsh":{"score":5.83,"minutes":6},"Hermes":{"score":5.83,"minutes":8}}},{"model":"Gemma 4 12B QAT","cells":{"Pi":{"score":5.33,"minutes":33,"note":"One task timed out"},"OpenCode":{"score":6,"minutes":27},"dsh":{"score":6,"minutes":34},"Hermes":{"score":4.15,"minutes":27,"note":"One task timed out"}}}]},"limitations":["Some model configurations have one full run","Quants, inference backends, and sampling differ between configurations","Qwen3.8-27B request settings remain under audit","No real-world assistant or OpenMuse results have been measured"]}