{ "schema": "runlog.work-benchmark.v1", "updated_at": "2026-10-27T16:03:33.490Z", "task_count": 9, "descriptions": [ { "id": "t1", "title": "Transform-aware object positioning", "description": "Repair an object attachment when parent objects have different transforms, while preserving the scene hierarchy." }, { "id": "t2", "title": "Frame-rate-independent timing", "description": "Correct motion calculations across segments with different frame rates, including validation and edge cases." }, { "id": "t3", "title": "Analog character movement", "description": "Implement frame-independent character movement with consistent diagonal speed and proportional analog input." }, { "id": "t4", "title": "Animation keyframes", "description": "Create constant-speed location animation while preserving existing animation on other channels and setting playback correctly." }, { "id": "t5", "title": "Conflicting technical evidence", "description": "Compare conflicting performance evidence, distinguish demonstrated results from missing evidence, and propose a useful validation experiment." }, { "id": "t6", "title": "Schedule and travel planning", "description": "Reconcile revised meeting information with uncertain travel duration, identify scheduling conflicts, and propose a practical plan." }, { "id": "t7", "title": "Quality and cost analysis", "description": "Calculate cost per attempt and per usable answer, apply quality requirements, and distinguish observed results from future reliability." }, { "id": "t8", "title": "Visual storytelling", "description": "Develop a concise, filmable sequence with continuous staging, observable acting, coherent timing and sound." }, { "id": "t9", "title": "Evidence-based professional writing", "description": "Write a tailored professional introduction using only supplied experience and a concrete response to the stated work problem." } ], "methodology": "Nine practical work samples evaluated with RunLog's hybrid objective engine: coding tasks verified in sandboxed execution (unit test assertions); planning, extraction and analysis validated with deterministic schema and DAG invariants; and creative tasks evaluated by a 3-model multi-judge consensus panel (MiniMax, ChatGPT, DeepSeek) with trimmed median aggregation. Output allowances varied where needed. Cost is mean selected-answer cost per 1k answers. Private prompts, fixtures, and reference answers remain protected.", "models": [ { "id": "openai/gpt-6.1-sol-pro", "quality": 98.97777777777777, "mean_answer_usd": 0.011190888888888889, "median_seconds": 10.333250096999109, "released_at": "2026-09-29T17:28:06+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0117955, "median_seconds": 15.656082986388355, "mean_seconds": 14.955078858183697 }, "research": { "samples": 3, "quality": 97.6, "mean_answer_usd": 0.010164666666666667, "median_seconds": 9.839026933070272, "mean_seconds": 9.304383247935524 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.011521, "median_seconds": 12.826949550770223, "mean_seconds": 12.826949550770223 } } }, { "id": "openai/gpt-6-astra-pro", "quality": 98.96666666666667, "mean_answer_usd": 0.052874444444444446, "median_seconds": 9.194387841038406, "released_at": "2026-09-04T20:13:55+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0553225, "median_seconds": 8.708748232806101, "mean_seconds": 11.083292809664272 }, "research": { "samples": 3, "quality": 97.9, "mean_answer_usd": 0.051800000000000006, "median_seconds": 9.194387841038406, "mean_seconds": 8.690338626038283 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.04959, "median_seconds": 42.65972123737447, "mean_seconds": 42.65972123737447 } } }, { "id": "openai/gpt-6-astra", "quality": 98.6, "mean_answer_usd": 0.014861111111111111, "median_seconds": 5.919983279891312, "released_at": "2026-09-04T20:13:58+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.015684999999999998, "median_seconds": 5.129610612988472, "mean_seconds": 7.030636970070191 }, "research": { "samples": 3, "quality": 96.8, "mean_answer_usd": 0.015463333333333332, "median_seconds": 6.031521951314062, "mean_seconds": 6.3284764244842036 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.012310000000000001, "median_seconds": 6.789001947967336, "mean_seconds": 6.789001947967336 } } }, { "id": "meta/muse-spark-1.3", "quality": 98.44444444444444, "mean_answer_usd": 0.005206, "median_seconds": 4.289983201306313, "released_at": "2026-09-02T19:45:59+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00461725, "median_seconds": 4.540046523092315, "mean_seconds": 5.2834693607874215 }, "research": { "samples": 3, "quality": 96, "mean_answer_usd": 0.005534, "median_seconds": 4.289983201306313, "mean_seconds": 4.413822726191332 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0058915, "median_seconds": 4.520669423509389, "mean_seconds": 4.520669423509389 } } }, { "id": "openai/gpt-6.1-sol", "quality": 98.86666666666666, "mean_answer_usd": 0.0028888888888888888, "median_seconds": 6.572628878057003, "released_at": "2026-09-29T17:28:02+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0032370000000000003, "median_seconds": 6.772365388926119, "mean_seconds": 8.079853923991323 }, "research": { "samples": 3, "quality": 97.6, "mean_answer_usd": 0.002696, "median_seconds": 6.187872551847249, "mean_seconds": 5.8724279801050825 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.002482, "median_seconds": 7.436737932963297, "mean_seconds": 7.436737932963297 } } }, { "id": "unbiased/pareto-26.10", "quality": 99.26666666666667, "mean_answer_usd": 0.0011955555555555556, "median_seconds": 8.996804349590093, "released_at": "2026-10-15T14:13:40+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0011740000000000001, "median_seconds": 8.088476417819038, "mean_seconds": 13.445186242694035 }, "research": { "samples": 3, "quality": 97.8, "mean_answer_usd": 0.0009246666666666666, "median_seconds": 8.156525302678347, "mean_seconds": 7.386074838228524 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.001645, "median_seconds": 12.037818462355062, "mean_seconds": 12.037818462355062 } } }, { "id": "openai/gpt-6-sol-pro", "quality": 99.44444444444444, "mean_answer_usd": 0.007438, "median_seconds": 4.063316758256406, "released_at": "2026-09-22T18:13:01+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0069295, "median_seconds": 3.9888283635955304, "mean_seconds": 3.9318418918410316 }, "research": { "samples": 3, "quality": 99, "mean_answer_usd": 0.008378, "median_seconds": 4.214979513082653, "mean_seconds": 4.34264123501877 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.007044999999999999, "median_seconds": 4.245639613829553, "mean_seconds": 4.245639613829553 } } }, { "id": "openai/gpt-5.6-sol", "quality": 99.5, "mean_answer_usd": 0.003957777777777778, "median_seconds": 2.5311845960095525, "released_at": "2026-07-09T09:54:10+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.003649, "median_seconds": 2.0812692353501916, "mean_seconds": 2.2326953588053584 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.0045720000000000005, "median_seconds": 2.8156522917561233, "mean_seconds": 4.8123979902205365 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.003654, "median_seconds": 2.8585862489417195, "mean_seconds": 2.8585862489417195 } } }, { "id": "meta/muse-spark-open", "quality": 99.44444444444444, "mean_answer_usd": 0.004195222222222223, "median_seconds": 2.9144271723926067, "released_at": "2026-10-15T17:57:49+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0046517500000000005, "median_seconds": 2.9312244539614767, "mean_seconds": 3.119161470909603 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.0033886666666666666, "median_seconds": 2.6336775263771415, "mean_seconds": 2.529409523587674 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.0044919999999999995, "median_seconds": 4.570285193389282, "mean_seconds": 4.570285193389282 } } }, { "id": "openai/gpt-6-sol", "quality": 99.61111111111111, "mean_answer_usd": 0.001820111111111111, "median_seconds": 2.7407508101314306, "released_at": "2026-09-22T18:12:55+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00169725, "median_seconds": 2.0639533151406795, "mean_seconds": 2.1965993923367932 }, "research": { "samples": 3, "quality": 99.5, "mean_answer_usd": 0.0021093333333333333, "median_seconds": 2.915681683924049, "mean_seconds": 6.75006851212432 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0016320000000000002, "median_seconds": 3.074045359622687, "mean_seconds": 3.074045359622687 } } }, { "id": "anthropic/claude-opus-4.8", "quality": 93.13333333333334, "mean_answer_usd": 0.011537777777777777, "median_seconds": 4.598947220016271, "released_at": "2026-05-27T18:04:51+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 98.35000000000001, "mean_answer_usd": 0.01296, "median_seconds": 4.163888240931556, "mean_seconds": 5.55782115040347 }, "research": { "samples": 3, "quality": 83.60000000000001, "mean_answer_usd": 0.01264, "median_seconds": 4.986526838969439, "mean_seconds": 6.162752082416167 }, "creative": { "samples": 2, "quality": 97, "mean_answer_usd": 0.007039999999999999, "median_seconds": 4.713738269871101, "mean_seconds": 4.713738269871101 } } }, { "id": "anthropic/claude-opus-4.6", "quality": 99.44444444444444, "mean_answer_usd": 0.012187333333333333, "median_seconds": 8.32150732493028, "released_at": "2026-02-04T15:30:50+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0170025, "median_seconds": 6.4478474704083055, "mean_seconds": 12.592260754900053 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.010002, "median_seconds": 8.55413114093244, "mean_seconds": 9.320686156706264 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.005835, "median_seconds": 7.689294732641429, "mean_seconds": 7.689294732641429 } } }, { "id": "unbiased/pareto-26.9", "quality": 99.63333333333334, "mean_answer_usd": 0.0027013333333333334, "median_seconds": 8.758972641080618, "released_at": "2026-10-13T22:28:57+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00273975, "median_seconds": 8.144112420501187, "mean_seconds": 12.767274425947107 }, "research": { "samples": 3, "quality": 98.9, "mean_answer_usd": 0.0020826666666666667, "median_seconds": 3.2461792598478496, "mean_seconds": 5.331605007406324 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0035525, "median_seconds": 10.081620343029499, "mean_seconds": 10.081620343029499 } } }, { "id": "qwen/qwen3.8-2.4t-a95b", "quality": 99.06666666666666, "mean_answer_usd": 0.007148, "median_seconds": 19.263765481766313, "released_at": "2026-08-12T16:21:42+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0097015, "median_seconds": 25.785130543634295, "mean_seconds": 29.667897914769128 }, "research": { "samples": 3, "quality": 98.2, "mean_answer_usd": 0.004676666666666667, "median_seconds": 13.915692034177482, "mean_seconds": 14.499992620976021 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.005748, "median_seconds": 18.714090107241645, "mean_seconds": 18.714090107241645 } } }, { "id": "google/gemini-3.8-flash", "quality": 53.16111111111111, "mean_answer_usd": 0.001097111111111111, "median_seconds": 3.2103858590126038, "released_at": "2026-09-02T15:14:16+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 50, "mean_answer_usd": 0.00110225, "median_seconds": 2.9234530511312187, "mean_seconds": 3.327473940094933 }, "research": { "samples": 3, "quality": 53.15, "mean_answer_usd": 0.0013296666666666665, "median_seconds": 3.3064062306657434, "mean_seconds": 3.8505027318994203 }, "creative": { "samples": 2, "quality": 59.5, "mean_answer_usd": 0.0007379999999999999, "median_seconds": 3.3222203962504864, "mean_seconds": 3.3222203962504864 } } }, { "id": "meta/muse-wonder", "quality": 99.22222222222223, "mean_answer_usd": 0.024122333333333332, "median_seconds": 10.920539753045887, "released_at": "2026-10-12T12:25:45+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.026687, "median_seconds": 10.689296369673684, "mean_seconds": 13.593913990538567 }, "research": { "samples": 3, "quality": 99, "mean_answer_usd": 0.022206666666666666, "median_seconds": 8.31830042693764, "mean_seconds": 9.466239304126551 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.021866499999999997, "median_seconds": 12.363576960749924, "mean_seconds": 12.363576960749924 } } }, { "id": "google/gemini-4-argon", "quality": 100, "mean_answer_usd": 0.0027593333333333333, "median_seconds": 5.519470416940749, "released_at": "2026-10-13T16:04:37+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0023875, "median_seconds": 5.4606912629678845, "mean_seconds": 5.950322955613956 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.003704666666666667, "median_seconds": 7.860857422929257, "mean_seconds": 8.015009982821843 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.002085, "median_seconds": 5.750284886918962, "mean_seconds": 5.750284886918962 } } }, { "id": "openai/gpt-5.5", "quality": 99.44444444444444, "mean_answer_usd": 0.006137333333333333, "median_seconds": 3.240914815105498, "released_at": "2026-04-24T17:31:33+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.005688749999999999, "median_seconds": 2.2030332079157233, "mean_seconds": 2.547047643107362 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.007535, "median_seconds": 3.8009191658347845, "mean_seconds": 3.6016077573100724 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.004938, "median_seconds": 6.264038297114894, "mean_seconds": 6.264038297114894 } } }, { "id": "anthropic/claude-sonnet-4.6", "quality": 99.22222222222223, "mean_answer_usd": 0.006419, "median_seconds": 3.8978214487433434, "released_at": "2026-02-17T15:43:10+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00932025, "median_seconds": 3.824714757502079, "mean_seconds": 5.634169241180643 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.005051, "median_seconds": 4.1122607621364295, "mean_seconds": 4.206251736264676 }, "creative": { "samples": 2, "quality": 96.5, "mean_answer_usd": 0.0026685, "median_seconds": 3.116093901451677, "mean_seconds": 3.116093901451677 } } }, { "id": "anthropic/claude-fable-5.1", "quality": 99, "mean_answer_usd": 0.023906777777777777, "median_seconds": 6.906312197912484, "released_at": "2026-09-01T18:03:58+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.03052775, "median_seconds": 7.648241804214194, "mean_seconds": 8.440504113794304 }, "research": { "samples": 3, "quality": 98, "mean_answer_usd": 0.022033333333333332, "median_seconds": 6.906312197912484, "mean_seconds": 6.567052457171182 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.013474999999999999, "median_seconds": 6.869636100484058, "mean_seconds": 6.869636100484058 } } }, { "id": "qwen/qwen3.8-max-prime", "quality": 56.138888888888886, "mean_answer_usd": 0.01574, "median_seconds": 11.513919995166361, "released_at": "2026-09-23T19:20:28+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 50, "mean_answer_usd": 0.019964, "median_seconds": 13.019837356405333, "mean_seconds": 15.284894278855063 }, "research": { "samples": 3, "quality": 51.25, "mean_answer_usd": 0.009521333333333333, "median_seconds": 6.445435560308397, "mean_seconds": 7.739610436372459 }, "creative": { "samples": 2, "quality": 75.75, "mean_answer_usd": 0.01662, "median_seconds": 13.044716752134264, "mean_seconds": 13.044716752134264 } } }, { "id": "anthropic/claude-sonnet-5.5", "quality": 72.52037037037037, "mean_answer_usd": 0.004811333333333333, "median_seconds": 4.405064797960222, "released_at": "2026-09-28T18:04:46+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 83.33333333333333, "mean_answer_usd": 0.0053755, "median_seconds": 4.156067326897755, "mean_seconds": 4.327264499734156 }, "research": { "samples": 3, "quality": 48.95, "mean_answer_usd": 0.00509, "median_seconds": 4.405064797960222, "mean_seconds": 4.2540840686609345 }, "creative": { "samples": 2, "quality": 86.25, "mean_answer_usd": 0.003265, "median_seconds": 3.4823778131976724, "mean_seconds": 3.4823778131976724 } } }, { "id": "moonshotai/kimi-k3.1-pro", "quality": 98.52222222222223, "mean_answer_usd": 0.002672, "median_seconds": 6.164420208893716, "released_at": "2026-10-23T17:56:28+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0027215, "median_seconds": 5.314349026186392, "mean_seconds": 6.575520714046434 }, "research": { "samples": 3, "quality": 96.9, "mean_answer_usd": 0.00302, "median_seconds": 6.395660862326622, "mean_seconds": 7.554992540739477 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.002051, "median_seconds": 6.173386584967375, "mean_seconds": 6.173386584967375 } } }, { "id": "xiaomi/mimo-v2.6-pro-ultraspeed", "quality": 99.61111111111111, "mean_answer_usd": 0.0028927777777777775, "median_seconds": 2.576180418021977, "released_at": "2026-09-21T20:07:47+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0031934999999999997, "median_seconds": 3.161336856428534, "mean_seconds": 3.5239842731971294 }, "research": { "samples": 3, "quality": 99.5, "mean_answer_usd": 0.003173, "median_seconds": 5.981656342279166, "mean_seconds": 10.070049350926032 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0018709999999999998, "median_seconds": 1.9029098344035447, "mean_seconds": 1.9029098344035447 } } }, { "id": "minimax/minimax-m3.1-flash:free", "quality": 98.83333333333333, "mean_answer_usd": 0.0005044444444444444, "median_seconds": 4.312944055069238, "released_at": "2026-10-19T03:12:49.000Z", "cost_basis": "usage-priced-equivalent", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000667, "median_seconds": 6.29565278859809, "mean_seconds": 9.197716741356999 }, "research": { "samples": 3, "quality": 96.5, "mean_answer_usd": 0.00037933333333333336, "median_seconds": 4.229555350728333, "mean_seconds": 5.285000783856958 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.00036700000000000003, "median_seconds": 4.7381509186234325, "mean_seconds": 4.7381509186234325 } } }, { "id": "openai/gpt-5.3-codex", "quality": 99.33333333333333, "mean_answer_usd": 0.0039506666666666666, "median_seconds": 3.1349772289395332, "released_at": "2026-02-24T18:52:44+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00447825, "median_seconds": 3.0210631326772273, "mean_seconds": 3.1916452958248556 }, "research": { "samples": 3, "quality": 99, "mean_answer_usd": 0.003993333333333333, "median_seconds": 3.1349772289395332, "mean_seconds": 3.1138640719776354 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.0028315000000000002, "median_seconds": 6.571585078490898, "mean_seconds": 6.571585078490898 } } }, { "id": "anthropic/claude-opus-5.5", "quality": 98.35555555555555, "mean_answer_usd": 0.016316, "median_seconds": 8.093451019842178, "released_at": "2026-09-22T16:32:12+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.018071, "median_seconds": 9.134823992149904, "mean_seconds": 9.37033731979318 }, "research": { "samples": 3, "quality": 96.4, "mean_answer_usd": 0.013859999999999999, "median_seconds": 7.467920176219195, "mean_seconds": 7.388276966754347 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.016489999999999998, "median_seconds": 9.176412814063951, "mean_seconds": 9.176412814063951 } } }, { "id": "qwen/qwen3.8-max-0902", "quality": 98.52222222222223, "mean_answer_usd": 0.008277333333333333, "median_seconds": 21.04031843179837, "released_at": "2026-09-03T21:08:24+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0072115, "median_seconds": 19.14592636586167, "mean_seconds": 21.41647267108783 }, "research": { "samples": 3, "quality": 96.9, "mean_answer_usd": 0.005052666666666666, "median_seconds": 21.04031843179837, "mean_seconds": 18.705225658292573 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.015246, "median_seconds": 43.66311051649973, "mean_seconds": 43.66311051649973 } } }, { "id": "openai/gpt-5.4", "quality": 99.77777777777777, "mean_answer_usd": 0.003519, "median_seconds": 2.725465591996908, "released_at": "2026-03-05T18:12:32+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00339975, "median_seconds": 2.6095179833937436, "mean_seconds": 2.678826486109756 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.004163, "median_seconds": 2.7395234322175384, "mean_seconds": 2.985087961113701 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0027915, "median_seconds": 2.6276048647705466, "mean_seconds": 2.6276048647705466 } } }, { "id": "aion-labs/aion-3.5", "quality": 67.97592592592592, "mean_answer_usd": 0.009387666666666667, "median_seconds": 22.36191289080307, "released_at": "2026-09-23T13:42:41+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 62.5, "mean_answer_usd": 0.01147875, "median_seconds": 27.10239133797586, "mean_seconds": 29.592302950914018 }, "research": { "samples": 3, "quality": 60.483333333333334, "mean_answer_usd": 0.0073160000000000005, "median_seconds": 16.21832475811243, "mean_seconds": 20.180278243962675 }, "creative": { "samples": 2, "quality": 90.16666666666666, "mean_answer_usd": 0.008313, "median_seconds": 20.592085082549602, "mean_seconds": 20.592085082549602 } } }, { "id": "google/gemini-3.5-flash", "quality": 99.44444444444444, "mean_answer_usd": 0.012642, "median_seconds": 8.071772751864046, "released_at": "2026-05-19T12:30:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0145615, "median_seconds": 8.584805516758934, "mean_seconds": 10.270395917003043 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.012156666666666666, "median_seconds": 8.071772751864046, "mean_seconds": 7.878804616319637 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.009531, "median_seconds": 6.875064087100327, "mean_seconds": 6.875064087100327 } } }, { "id": "google/gemini-3.1-pro-preview", "quality": 98.26666666666667, "mean_answer_usd": 0.013225111111111111, "median_seconds": 10.4649300230667, "released_at": "2026-02-19T14:00:27+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.010139, "median_seconds": 9.617419659392908, "mean_seconds": 9.242211845004931 }, "research": { "samples": 3, "quality": 95.8, "mean_answer_usd": 0.015200666666666666, "median_seconds": 11.049867080058903, "mean_seconds": 11.893331872609755 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.016433999999999997, "median_seconds": 15.117507213493809, "mean_seconds": 15.117507213493809 } } }, { "id": "moonshotai/kimi-k2-0905", "quality": 98.44444444444444, "mean_answer_usd": 0.0007556666666666666, "median_seconds": 4.641817349940538, "released_at": "2025-09-04T21:25:47+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00105, "median_seconds": 6.316168909193948, "mean_seconds": 8.691819664440118 }, "research": { "samples": 3, "quality": 97, "mean_answer_usd": 0.0005816666666666666, "median_seconds": 4.614916449878365, "mean_seconds": 4.806716414013256 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.000428, "median_seconds": 3.9919973369687796, "mean_seconds": 3.9919973369687796 } } }, { "id": "anthropic/claude-opus-5", "quality": 98.63333333333334, "mean_answer_usd": 0.012973888888888889, "median_seconds": 6.488785746041685, "released_at": "2026-07-24T17:02:24+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.01529125, "median_seconds": 7.528284835163504, "mean_seconds": 7.387245967518538 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.013648333333333333, "median_seconds": 4.744778689928353, "mean_seconds": 6.829520381676654 }, "creative": { "samples": 2, "quality": 96.1, "mean_answer_usd": 0.007327500000000001, "median_seconds": 4.892796492902562, "mean_seconds": 4.892796492902562 } } }, { "id": "openai/gpt-6-luna-pro", "quality": 99.23333333333333, "mean_answer_usd": 0.0004006666666666667, "median_seconds": 3.699072781018913, "released_at": "2026-09-22T18:13:11+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00038425, "median_seconds": 4.124314347514883, "mean_seconds": 3.9352291038958356 }, "research": { "samples": 3, "quality": 98.7, "mean_answer_usd": 0.000435, "median_seconds": 6.153918628115207, "mean_seconds": 5.633423146481316 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.00038199999999999996, "median_seconds": 2.8412308909464628, "mean_seconds": 2.8412308909464628 } } }, { "id": "openai/gpt-6-luna", "quality": 99.44444444444444, "mean_answer_usd": 0.00009255555555555555, "median_seconds": 1.8392771040089428, "released_at": "2026-09-22T18:13:06+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000083, "median_seconds": 1.4803742349613458, "mean_seconds": 1.647111712838523 }, "research": { "samples": 3, "quality": 99, "mean_answer_usd": 0.00011066666666666667, "median_seconds": 2.1370373792015016, "mean_seconds": 2.7620473131537437 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0000845, "median_seconds": 1.7871809219941497, "mean_seconds": 1.7871809219941497 } } }, { "id": "openai/gpt-5-mini", "quality": 99.48888888888888, "mean_answer_usd": 0.0013257777777777777, "median_seconds": 6.692102524917573, "released_at": "2025-08-07T17:23:27+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0016034999999999999, "median_seconds": 8.149995552143082, "mean_seconds": 7.877964721876197 }, "research": { "samples": 3, "quality": 98.8, "mean_answer_usd": 0.001281, "median_seconds": 6.039770949631929, "mean_seconds": 6.313871583590905 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.0008375, "median_seconds": 8.254505385644734, "mean_seconds": 8.254505385644734 } } }, { "id": "moonshotai/kimi-k3", "quality": 99.45555555555556, "mean_answer_usd": 0.0025483333333333334, "median_seconds": 5.4492962579242885, "released_at": "2026-07-16T15:30:58+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00328575, "median_seconds": 4.630407724995166, "mean_seconds": 4.670705096563324 }, "research": { "samples": 3, "quality": 98.7, "mean_answer_usd": 0.002196, "median_seconds": 5.546785668004304, "mean_seconds": 4.7935854606330395 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.0016020000000000001, "median_seconds": 4.595958489459008, "mean_seconds": 4.595958489459008 } } }, { "id": "fireworks/ember-1", "quality": 78.47777777777777, "mean_answer_usd": 0.004830333333333333, "median_seconds": 2.951362628955394, "released_at": "2026-09-24T00:07:41+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0059924999999999996, "median_seconds": 4.26624234742485, "mean_seconds": 4.612933918251656 }, "research": { "samples": 3, "quality": 50.3, "mean_answer_usd": 0.004199, "median_seconds": 2.951362628955394, "mean_seconds": 3.245137365690122 }, "creative": { "samples": 2, "quality": 77.7, "mean_answer_usd": 0.0034530000000000003, "median_seconds": 3.000895344186574, "mean_seconds": 3.000895344186574 } } }, { "id": "anthropic/claude-sonnet-5", "quality": 98.83333333333333, "mean_answer_usd": 0.006001777777777778, "median_seconds": 5.07968095690012, "released_at": "2026-06-30T18:11:23+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0084065, "median_seconds": 5.980126128997654, "mean_seconds": 7.723019192228094 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.004682666666666667, "median_seconds": 4.20937279984355, "mean_seconds": 4.886020381934941 }, "creative": { "samples": 2, "quality": 97, "mean_answer_usd": 0.003171, "median_seconds": 4.416391093516722, "mean_seconds": 4.416391093516722 } } }, { "id": "reflection/beam-1.0-501b-a23b", "quality": 99.83333333333333, "mean_answer_usd": 0.000481, "median_seconds": 2.9506721519865096, "released_at": "2026-10-16T14:21:39+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00049475, "median_seconds": 2.8728537110146135, "mean_seconds": 3.1155133174033836 }, "research": { "samples": 3, "quality": 99.5, "mean_answer_usd": 0.0005136666666666666, "median_seconds": 2.9506721519865096, "mean_seconds": 2.9526877080400786 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0004045, "median_seconds": 3.018901255680248, "mean_seconds": 3.018901255680248 } } }, { "id": "x-ai/grok-4.5", "quality": 99.56666666666666, "mean_answer_usd": 0.004016555555555555, "median_seconds": 11.856012410949916, "released_at": "2026-07-08T15:05:54+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0050030000000000005, "median_seconds": 14.576277772895992, "mean_seconds": 20.56402082426939 }, "research": { "samples": 3, "quality": 99.7, "mean_answer_usd": 0.003906333333333334, "median_seconds": 10.826932516880333, "mean_seconds": 11.109484286047518 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.002209, "median_seconds": 9.289950557984412, "mean_seconds": 9.289950557984412 } } }, { "id": "xiaomi/mimo-v2.6-pro", "quality": 79.86666666666666, "mean_answer_usd": 0.0003281111111111111, "median_seconds": 4.704617874696851, "released_at": "2026-09-21T20:07:39+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 67.45, "mean_answer_usd": 0.00044325, "median_seconds": 6.378803862491623, "mean_seconds": 7.663013205397874 }, "research": { "samples": 3, "quality": 95, "mean_answer_usd": 0.00029133333333333333, "median_seconds": 4.704617874696851, "mean_seconds": 4.393011094226192 }, "creative": { "samples": 2, "quality": 82, "mean_answer_usd": 0.000153, "median_seconds": 5.234739777399227, "mean_seconds": 5.234739777399227 } } }, { "id": "google/gemini-3.5-flash-lite", "quality": 99.5, "mean_answer_usd": 0.0014292222222222221, "median_seconds": 1.4203814142383635, "released_at": "2026-07-21T15:12:06+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00077675, "median_seconds": 1.2810950297862291, "mean_seconds": 1.4461474784184247 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.0015873333333333334, "median_seconds": 1.146792805287987, "mean_seconds": 1.9249159474857152 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.002497, "median_seconds": 3.6029343474656343, "mean_seconds": 3.6029343474656343 } } }, { "id": "openai/gpt-5.4-mini", "quality": 99.35555555555555, "mean_answer_usd": 0.0009605555555555555, "median_seconds": 1.7676939289085567, "released_at": "2026-03-17T11:49:38+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00100675, "median_seconds": 1.9864051756449044, "mean_seconds": 2.038722346071154 }, "research": { "samples": 3, "quality": 99.4, "mean_answer_usd": 0.0010003333333333333, "median_seconds": 1.7676939289085567, "mean_seconds": 1.7874219633328419 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.0008085, "median_seconds": 1.7771854628808796, "mean_seconds": 1.7771854628808796 } } }, { "id": "deepseek/deepseek-v4-pro", "quality": 80.31111111111112, "mean_answer_usd": 0.00013566666666666667, "median_seconds": 3.6432033888995647, "released_at": "2026-04-24T03:17:59+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 86.25, "mean_answer_usd": 0.00017575, "median_seconds": 5.125746113481, "mean_seconds": 5.352059462340549 }, "research": { "samples": 3, "quality": 65.13333333333334, "mean_answer_usd": 0.00010866666666666667, "median_seconds": 2.975553974043578, "mean_seconds": 3.160763854626566 }, "creative": { "samples": 2, "quality": 91.2, "mean_answer_usd": 0.000096, "median_seconds": 3.3667726281564683, "mean_seconds": 3.3667726281564683 } } }, { "id": "google/gemma-4-31b-it", "quality": 99.44444444444444, "mean_answer_usd": 0.00009466666666666667, "median_seconds": 2.716313829179853, "released_at": "2026-04-02T16:48:06+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00011575000000000001, "median_seconds": 3.602966306498274, "mean_seconds": 3.8139214664697647 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.00009566666666666667, "median_seconds": 3.055032579228282, "mean_seconds": 3.079173158699026 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.000051, "median_seconds": 1.6620835980866104, "mean_seconds": 1.6620835980866104 } } }, { "id": "anthropic/claude-haiku-5.5", "quality": 80.17777777777778, "mean_answer_usd": 0.00021766666666666665, "median_seconds": 2.034678773023188, "released_at": "2026-10-07T18:31:23+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 81, "mean_answer_usd": 0.0002305, "median_seconds": 1.8064515320584178, "mean_seconds": 2.3051896683173254 }, "research": { "samples": 3, "quality": 78.80000000000001, "mean_answer_usd": 0.000248, "median_seconds": 2.059034178033471, "mean_seconds": 2.334378344627718 }, "creative": { "samples": 2, "quality": 80.6, "mean_answer_usd": 0.0001465, "median_seconds": 2.010248405393213, "mean_seconds": 2.010248405393213 } } }, { "id": "deepseek/deepseek-v4.1-flash-1017", "quality": 99.14444444444445, "mean_answer_usd": 0.0003282222222222222, "median_seconds": 4.460890015121549, "released_at": "2026-10-17T08:27:04+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000491, "median_seconds": 3.4937880823854357, "mean_seconds": 9.019258483196609 }, "research": { "samples": 3, "quality": 98.1, "mean_answer_usd": 0.000244, "median_seconds": 4.977873279247433, "mean_seconds": 5.026819714810699 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.000129, "median_seconds": 4.744395773392171, "mean_seconds": 4.744395773392171 } } }, { "id": "google/gemini-3.7-flash", "quality": 99.88888888888889, "mean_answer_usd": 0.0011455555555555555, "median_seconds": 3.425703639164567, "released_at": "2026-08-13T17:03:01+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0011982499999999999, "median_seconds": 3.1937473451253027, "mean_seconds": 3.45685878128279 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.0013946666666666667, "median_seconds": 3.425703639164567, "mean_seconds": 4.002505905460566 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.0006665, "median_seconds": 3.3212749620433897, "mean_seconds": 3.3212749620433897 } } }, { "id": "openai/gpt-5.6-luna", "quality": 99.63333333333334, "mean_answer_usd": 0.00025166666666666664, "median_seconds": 2.277150822803378, "released_at": "2026-07-09T09:54:24+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0002345, "median_seconds": 3.4082611254416406, "mean_seconds": 3.606324812397361 }, "research": { "samples": 3, "quality": 98.9, "mean_answer_usd": 0.0002973333333333333, "median_seconds": 2.1731411782093346, "mean_seconds": 2.154117222254475 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.00021749999999999997, "median_seconds": 2.0536341802217066, "mean_seconds": 2.0536341802217066 } } }, { "id": "deepseek/deepseek-v4.1-flash", "quality": 55.138271604938275, "mean_answer_usd": 0.00026766666666666665, "median_seconds": 4.5729741868563, "released_at": "2026-09-10T06:21:25+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 50, "mean_answer_usd": 0.000362, "median_seconds": 4.006274230778217, "mean_seconds": 6.942706783884205 }, "research": { "samples": 3, "quality": 50.6, "mean_answer_usd": 0.00023866666666666668, "median_seconds": 4.719891793094575, "mean_seconds": 5.230350914100806 }, "creative": { "samples": 2, "quality": 72.22222222222223, "mean_answer_usd": 0.0001225, "median_seconds": 4.204187663504854, "mean_seconds": 4.204187663504854 } } }, { "id": "openai/gpt-5.6-terra", "quality": 99.77777777777777, "mean_answer_usd": 0.002021666666666667, "median_seconds": 2.012293383013457, "released_at": "2026-07-09T09:54:17+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00179875, "median_seconds": 1.8076527141965926, "mean_seconds": 1.805424740887247 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.002366, "median_seconds": 2.531267140991986, "mean_seconds": 2.364870608629038 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.001951, "median_seconds": 2.365935957292095, "mean_seconds": 2.365935957292095 } } }, { "id": "deepseek/deepseek-v4-pro-0813", "quality": 93.21111111111111, "mean_answer_usd": 0.0005597777777777777, "median_seconds": 2.8544705463573337, "released_at": "2026-08-12T15:42:44+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000559, "median_seconds": 2.5137668550014496, "mean_seconds": 2.6384814792545512 }, "research": { "samples": 3, "quality": 81.3, "mean_answer_usd": 0.0006126666666666666, "median_seconds": 2.8544705463573337, "mean_seconds": 2.738289592166742 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.000482, "median_seconds": 4.320629779016599, "mean_seconds": 4.320629779016599 } } }, { "id": "google/gemini-3.6-flash", "quality": 99.61111111111111, "mean_answer_usd": 0.004695111111111111, "median_seconds": 6.322182232979685, "released_at": "2026-07-21T15:12:13+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0042995, "median_seconds": 5.368209549924359, "mean_seconds": 6.170308178756386 }, "research": { "samples": 3, "quality": 99.5, "mean_answer_usd": 0.004428333333333333, "median_seconds": 6.322182232979685, "mean_seconds": 6.040738082490861 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.005886499999999999, "median_seconds": 9.111903970362619, "mean_seconds": 9.111903970362619 } } }, { "id": "tencent/hy4-preview", "quality": 86.15555555555555, "mean_answer_usd": 0.0006748888888888889, "median_seconds": 4.775964486878365, "released_at": "2026-08-28T06:09:35+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 87.1, "mean_answer_usd": 0.000739, "median_seconds": 5.31468386715278, "mean_seconds": 5.0652880277484655 }, "research": { "samples": 3, "quality": 95, "mean_answer_usd": 0.0007556666666666666, "median_seconds": 6.061439763288945, "mean_seconds": 6.040522082398335 }, "creative": { "samples": 2, "quality": 71.00000000000001, "mean_answer_usd": 0.0004255, "median_seconds": 3.968149239430204, "mean_seconds": 3.968149239430204 } } }, { "id": "elorian/glimpse", "quality": 99.56666666666666, "mean_answer_usd": 0.0008567777777777778, "median_seconds": 1.1901616309769452, "released_at": "2026-10-17T03:10:29+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000923, "median_seconds": 1.3358326021116227, "mean_seconds": 1.670553054427728 }, "research": { "samples": 3, "quality": 99.7, "mean_answer_usd": 0.0009413333333333333, "median_seconds": 2.0073367869481444, "mean_seconds": 1.8935798374004662 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.0005975, "median_seconds": 1.110577008454129, "mean_seconds": 1.110577008454129 } } }, { "id": "qwen/qwen3.5-flash-02-23", "quality": 39.72222222222222, "mean_answer_usd": 0.00008755555555555555, "median_seconds": 1.761187041644007, "released_at": "2026-02-25T21:09:36+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.0001395, "median_seconds": 3.4761203275993466, "mean_seconds": 3.4675105357309803 }, "research": { "samples": 3, "quality": 82.5, "mean_answer_usd": 0.000048, "median_seconds": 1.2740482911467552, "mean_seconds": 1.2743251939925055 }, "creative": { "samples": 2, "quality": 55, "mean_answer_usd": 0.000043, "median_seconds": 1.548220579046756, "mean_seconds": 1.548220579046756 } } }, { "id": "qwen/qwen3-235b-a22b-thinking-2507", "quality": 53.46728395061728, "mean_answer_usd": 0.009848222222222222, "median_seconds": 29.853107251226902, "released_at": "2025-07-25T13:19:17+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 55.55555555555556, "mean_answer_usd": 0.0098715, "median_seconds": 23.004308445379138, "mean_seconds": 22.74683467752766 }, "research": { "samples": 3, "quality": 50.55, "mean_answer_usd": 0.008731, "median_seconds": 104.42050890112296, "mean_seconds": 76.48650310731803 }, "creative": { "samples": 2, "quality": 53.66666666666667, "mean_answer_usd": 0.0114775, "median_seconds": 84.76145764393732, "mean_seconds": 84.76145764393732 } } }, { "id": "aion-labs/aion-3.5-mini", "quality": 54.75, "mean_answer_usd": 0.0036428888888888887, "median_seconds": 29.5882805143483, "released_at": "2026-09-23T13:42:42+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 50, "mean_answer_usd": 0.0043875, "median_seconds": 32.561532978666946, "mean_seconds": 49.27139473229181 }, "research": { "samples": 3, "quality": 47.58333333333333, "mean_answer_usd": 0.0033426666666666665, "median_seconds": 37.8406987329945, "mean_seconds": 41.26670706908529 }, "creative": { "samples": 2, "quality": 75, "mean_answer_usd": 0.002604, "median_seconds": 28.167644589208066, "mean_seconds": 28.167644589208066 } } }, { "id": "inclusionai/ling-3.1-flash", "quality": 1.1666666666666667, "mean_answer_usd": 0.00007244444444444445, "median_seconds": 21.721912355162203, "released_at": "2026-10-02T14:07:04+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.00008075, "median_seconds": 10.928401085315272, "mean_seconds": 16.130553240538575 }, "research": { "samples": 3, "quality": 3.5, "mean_answer_usd": 0.00007933333333333333, "median_seconds": 25.03974907193333, "mean_seconds": 35.43267469263325 }, "creative": { "samples": 2, "quality": 0, "mean_answer_usd": 0.0000455, "median_seconds": 17.164789530681446, "mean_seconds": 17.164789530681446 } } }, { "id": "google/gemini-3-flash-preview", "quality": 100, "mean_answer_usd": 0.0008047777777777777, "median_seconds": 1.615984975360334, "released_at": "2025-12-17T15:57:58+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00099075, "median_seconds": 1.8849556103814393, "mean_seconds": 2.1120803457451984 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.0007793333333333334, "median_seconds": 1.615984975360334, "mean_seconds": 1.6916077067144215 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000471, "median_seconds": 1.5317919361405075, "mean_seconds": 1.5317919361405075 } } }, { "id": "moonshotai/kimi-k2.6", "quality": 99.56666666666666, "mean_answer_usd": 0.0010135555555555555, "median_seconds": 4.371059483848512, "released_at": "2026-04-20T15:36:42+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0015854999999999999, "median_seconds": 13.520765985362232, "mean_seconds": 15.074644475243986 }, "research": { "samples": 3, "quality": 98.7, "mean_answer_usd": 0.0006789999999999999, "median_seconds": 3.99838113412261, "mean_seconds": 5.1449258193994565 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0003715, "median_seconds": 3.495849360479042, "mean_seconds": 3.495849360479042 } } }, { "id": "z-ai/glm-5.2", "quality": 99.3111111111111, "mean_answer_usd": 0.0013632222222222223, "median_seconds": 1.0620795162394643, "released_at": "2026-06-16T17:45:30+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0015925, "median_seconds": 1.7997078022453934, "mean_seconds": 2.5957228236366063 }, "research": { "samples": 3, "quality": 98.6, "mean_answer_usd": 0.001428, "median_seconds": 1.0188079220242798, "mean_seconds": 1.2743292685287695 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0008075000000000001, "median_seconds": 1.0014194082468748, "mean_seconds": 1.0014194082468748 } } }, { "id": "deepseek/deepseek-v4-flash-vision-exp", "quality": 99.07777777777778, "mean_answer_usd": 0.00015877777777777779, "median_seconds": 2.722887234762311, "released_at": "2026-08-21T11:26:03+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00017525, "median_seconds": 1.5863412274047732, "mean_seconds": 1.743632556288503 }, "research": { "samples": 3, "quality": 97.9, "mean_answer_usd": 0.00016033333333333332, "median_seconds": 2.722887234762311, "mean_seconds": 3.826770018941412 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0001235, "median_seconds": 3.518133177421987, "mean_seconds": 3.518133177421987 } } }, { "id": "google/gemini-2.5-pro", "quality": 49.891975308641975, "mean_answer_usd": 0.010716333333333333, "median_seconds": 9.887418640311807, "released_at": "2025-06-17T14:12:24+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 44.44444444444444, "mean_answer_usd": 0.01161975, "median_seconds": 11.12599761155434, "mean_seconds": 10.741550283739343 }, "research": { "samples": 3, "quality": 51.25, "mean_answer_usd": 0.010670666666666667, "median_seconds": 8.680203119758517, "mean_seconds": 9.63722692678372 }, "creative": { "samples": 2, "quality": 58.75, "mean_answer_usd": 0.008978, "median_seconds": 9.40052011422813, "mean_seconds": 9.40052011422813 } } }, { "id": "google/gemini-2.5-flash-image", "quality": 99.44444444444444, "mean_answer_usd": 0.0007678888888888889, "median_seconds": 2.2431643838062882, "released_at": "2025-10-07T20:53:51+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00098725, "median_seconds": 1.9859188254922628, "mean_seconds": 3.0385659970343113 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.0006483333333333333, "median_seconds": 2.244449021294713, "mean_seconds": 2.040977549428741 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.0005085, "median_seconds": 3.1156294443644583, "mean_seconds": 3.1156294443644583 } } }, { "id": "z-ai/glm-5.3-prime", "quality": 57.92222222222222, "mean_answer_usd": 0.002524666666666667, "median_seconds": 2.736224167048931, "released_at": "2026-09-23T21:40:51+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 50, "mean_answer_usd": 0.00265375, "median_seconds": 2.5618956051766872, "mean_seconds": 3.1838931359816343 }, "research": { "samples": 3, "quality": 51.1, "mean_answer_usd": 0.0028376666666666667, "median_seconds": 3.489167541731149, "mean_seconds": 3.286763872640828 }, "creative": { "samples": 2, "quality": 84, "mean_answer_usd": 0.001797, "median_seconds": 2.7480383969377726, "mean_seconds": 2.7480383969377726 } } }, { "id": "z-ai/glm-4.7", "quality": 99.33333333333333, "mean_answer_usd": 0.000442, "median_seconds": 5.0126471519470215, "released_at": "2025-12-22T04:33:34+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0006075, "median_seconds": 9.787192068062723, "mean_seconds": 10.175774242263287 }, "research": { "samples": 3, "quality": 99, "mean_answer_usd": 0.00033966666666666666, "median_seconds": 5.0126471519470215, "mean_seconds": 4.980009777937084 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.0002645, "median_seconds": 3.2946460109669715, "mean_seconds": 3.2946460109669715 } } }, { "id": "openai/gpt-5.4-nano", "quality": 89.38888888888889, "mean_answer_usd": 0.00045055555555555555, "median_seconds": 2.911664089653641, "released_at": "2026-03-17T11:49:47+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00052375, "median_seconds": 3.348945560399443, "mean_seconds": 4.181460562278517 }, "research": { "samples": 3, "quality": 69.5, "mean_answer_usd": 0.0004636666666666667, "median_seconds": 2.9514724537730217, "mean_seconds": 4.633414793759584 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.0002845, "median_seconds": 2.264181431615725, "mean_seconds": 2.264181431615725 } } }, { "id": "z-ai/glm-5.3-flash", "quality": 99.33333333333333, "mean_answer_usd": 0.00003233333333333333, "median_seconds": 2.9634678359143436, "released_at": "2026-08-26T13:59:01+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000037, "median_seconds": 2.8634602774400264, "mean_seconds": 7.458998574526049 }, "research": { "samples": 3, "quality": 98, "mean_answer_usd": 0.000035, "median_seconds": 2.9634678359143436, "mean_seconds": 2.8297960509856543 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000019, "median_seconds": 2.754491894505918, "mean_seconds": 2.754491894505918 } } }, { "id": "anthropic/claude-sonnet-4.5", "quality": 99.44444444444444, "mean_answer_usd": 0.005147666666666667, "median_seconds": 7.3167949500493705, "released_at": "2025-09-29T16:01:16+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00690225, "median_seconds": 7.524559239624068, "mean_seconds": 6.602841770392843 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.004318, "median_seconds": 7.707088886294514, "mean_seconds": 7.97434186283499 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.002883, "median_seconds": 5.256912663578987, "mean_seconds": 5.256912663578987 } } }, { "id": "google/gemini-3.1-flash-lite", "quality": 100, "mean_answer_usd": 0.0003522222222222222, "median_seconds": 1.1334088509902358, "released_at": "2026-05-07T15:47:08+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00039825, "median_seconds": 1.200835436815396, "mean_seconds": 1.333986226702109 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.000373, "median_seconds": 1.0608555087819695, "mean_seconds": 1.159807289019227 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.00022899999999999998, "median_seconds": 1.242104056989774, "mean_seconds": 1.242104056989774 } } }, { "id": "google/gemma-4-26b-a4b-it", "quality": 98.87777777777778, "mean_answer_usd": 0.00007211111111111112, "median_seconds": 2.877546523232013, "released_at": "2026-04-03T14:53:09+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000092, "median_seconds": 3.315834072418511, "mean_seconds": 4.185528642730787 }, "research": { "samples": 3, "quality": 98.3, "mean_answer_usd": 0.000069, "median_seconds": 1.9959861417301, "mean_seconds": 2.704209523896376 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.000037000000000000005, "median_seconds": 1.4124884833581746, "mean_seconds": 1.4124884833581746 } } }, { "id": "tencent/hy3", "quality": 70.13333333333334, "mean_answer_usd": 0.00011188888888888888, "median_seconds": 3.1288254726678133, "released_at": "2026-07-06T13:20:48+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 53.75, "mean_answer_usd": 0.000111, "median_seconds": 3.2882958555128425, "mean_seconds": 3.2510248702019453 }, "research": { "samples": 3, "quality": 80.46666666666667, "mean_answer_usd": 0.00013066666666666665, "median_seconds": 3.1288254726678133, "mean_seconds": 3.542404822539538 }, "creative": { "samples": 2, "quality": 87.4, "mean_answer_usd": 0.00008549999999999999, "median_seconds": 3.004105716943741, "mean_seconds": 3.004105716943741 } } }, { "id": "openai/gpt-oss-safeguard-20b", "quality": 75.68148148148148, "mean_answer_usd": 0.0003915555555555556, "median_seconds": 1.580129588022828, "released_at": "2025-10-29T15:47:16+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 83.33333333333333, "mean_answer_usd": 0.00046125, "median_seconds": 1.6117678221780807, "mean_seconds": 1.9122655043611303 }, "research": { "samples": 3, "quality": 49.6, "mean_answer_usd": 0.0003213333333333333, "median_seconds": 1.3743717093020678, "mean_seconds": 1.4142166938011844 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.0003575, "median_seconds": 1.541612099390477, "mean_seconds": 1.541612099390477 } } }, { "id": "z-ai/glm-4.5v", "quality": 69.11111111111111, "mean_answer_usd": 0.0005274444444444444, "median_seconds": 5.364841139409691, "released_at": "2025-08-11T14:24:48+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 61.2, "mean_answer_usd": 0.0006475, "median_seconds": 5.341341420309618, "mean_seconds": 5.464682318386622 }, "research": { "samples": 3, "quality": 64.60000000000001, "mean_answer_usd": 0.00048266666666666667, "median_seconds": 5.474453267175704, "mean_seconds": 14.961671022698283 }, "creative": { "samples": 2, "quality": 91.7, "mean_answer_usd": 0.0003545, "median_seconds": 6.555089934263378, "mean_seconds": 6.555089934263378 } } }, { "id": "mistralai/mistral-large-4-0", "quality": 99.56666666666666, "mean_answer_usd": 0.0017956666666666665, "median_seconds": 2.2882931088097394, "released_at": "2026-10-06T13:42:02+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0024435, "median_seconds": 1.718593826284632, "mean_seconds": 3.510732184513472 }, "research": { "samples": 3, "quality": 99.7, "mean_answer_usd": 0.0015446666666666666, "median_seconds": 3.0336904232390225, "mean_seconds": 3.3021828810063503 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.0008765, "median_seconds": 2.2046166013460606, "mean_seconds": 2.2046166013460606 } } }, { "id": "openai/gpt-5.6-luna-pro", "quality": 99.9, "mean_answer_usd": 0.0012125555555555555, "median_seconds": 3.020866387989372, "released_at": "2026-07-09T09:54:27+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00116175, "median_seconds": 2.7698218659497797, "mean_seconds": 2.951901199412532 }, "research": { "samples": 3, "quality": 99.7, "mean_answer_usd": 0.001366, "median_seconds": 3.020866387989372, "mean_seconds": 2.997277125560989 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.001084, "median_seconds": 3.466969986911863, "mean_seconds": 3.466969986911863 } } }, { "id": "qwen/qwen3.8-flash", "quality": 68.55555555555556, "mean_answer_usd": 0.00014, "median_seconds": 4.248947509098798, "released_at": "2026-08-26T19:37:40+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 67.8, "mean_answer_usd": 0.00016874999999999998, "median_seconds": 5.0047896155156195, "mean_seconds": 6.782554068020545 }, "research": { "samples": 3, "quality": 58.86666666666667, "mean_answer_usd": 0.000132, "median_seconds": 3.8061116957105696, "mean_seconds": 4.529353698560347 }, "creative": { "samples": 2, "quality": 84.60000000000001, "mean_answer_usd": 0.00009449999999999999, "median_seconds": 4.449591351673007, "mean_seconds": 4.449591351673007 } } }, { "id": "stepfun/step-3.7-flash", "quality": 99.06666666666666, "mean_answer_usd": 0.002152888888888889, "median_seconds": 10.301195682026446, "released_at": "2026-05-28T16:17:49+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0027967499999999998, "median_seconds": 16.027684995206073, "mean_seconds": 16.206646000850014 }, "research": { "samples": 3, "quality": 97.2, "mean_answer_usd": 0.001631, "median_seconds": 9.922449574805796, "mean_seconds": 10.715350619362047 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.001648, "median_seconds": 11.592591731110588, "mean_seconds": 11.592591731110588 } } }, { "id": "deepseek/deepseek-chat-v3.1", "quality": 100, "mean_answer_usd": 0.0002502222222222222, "median_seconds": 14.682263563387096, "released_at": "2025-08-21T12:33:48+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000317, "median_seconds": 17.628910216968507, "mean_seconds": 25.02543161716312 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.00022833333333333332, "median_seconds": 9.528085101861507, "mean_seconds": 17.346017201275874 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0001495, "median_seconds": 11.296911324607208, "mean_seconds": 11.296911324607208 } } }, { "id": "deepseek/deepseek-v4-flash", "quality": 68.31111111111112, "mean_answer_usd": 0.00006011111111111111, "median_seconds": 5.257019791286439, "released_at": "2026-04-24T03:17:46+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 76.25, "mean_answer_usd": 0.00007325, "median_seconds": 5.629477960988879, "mean_seconds": 5.9691175509942695 }, "research": { "samples": 3, "quality": 46.333333333333336, "mean_answer_usd": 0.000056, "median_seconds": 6.31768140103668, "mean_seconds": 5.588748725131154 }, "creative": { "samples": 2, "quality": 85.4, "mean_answer_usd": 0.000039999999999999996, "median_seconds": 3.501148525159806, "mean_seconds": 3.501148525159806 } } }, { "id": "qwen/qwen3.7-plus", "quality": 99.55555555555556, "mean_answer_usd": 0.0005802222222222222, "median_seconds": 3.640295200049877, "released_at": "2026-06-03T13:03:03+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00098425, "median_seconds": 10.88117512059398, "mean_seconds": 12.994890452013351 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.000293, "median_seconds": 3.640295200049877, "mean_seconds": 4.123079156658302 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.000203, "median_seconds": 3.3212464065290987, "mean_seconds": 3.3212464065290987 } } }, { "id": "moonshotai/kimi-k2", "quality": 93.26984126984127, "mean_answer_usd": 0.0015255555555555556, "median_seconds": 6.765200742054731, "released_at": "2025-07-11T19:47:32+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00106025, "median_seconds": 9.94197644898668, "mean_seconds": 9.572479487629607 }, "research": { "samples": 3, "quality": 96, "mean_answer_usd": 0.002866, "median_seconds": 7.180241342168301, "mean_seconds": 9.970297899407646 }, "creative": { "samples": 2, "quality": 75.71428571428572, "mean_answer_usd": 0.0004455, "median_seconds": 3.975298889912665, "mean_seconds": 3.975298889912665 } } }, { "id": "google/gemini-3.1-flash-lite-preview", "quality": 100, "mean_answer_usd": 0.000356, "median_seconds": 1.1413925401866436, "released_at": "2026-03-03T04:37:53+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00040425, "median_seconds": 1.2675146495457739, "mean_seconds": 1.351199428550899 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.000393, "median_seconds": 1.148158366791904, "mean_seconds": 1.2907233135774732 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000204, "median_seconds": 1.1103866959456354, "mean_seconds": 1.1103866959456354 } } }, { "id": "openai/gpt-5-nano", "quality": 95.65555555555557, "mean_answer_usd": 0.0005241111111111112, "median_seconds": 14.561993570066988, "released_at": "2025-08-07T17:23:22+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0007155, "median_seconds": 16.870599907357246, "mean_seconds": 16.786830349010415 }, "research": { "samples": 3, "quality": 88.30000000000001, "mean_answer_usd": 0.00044666666666666666, "median_seconds": 10.05037843203172, "mean_seconds": 9.962403070957711 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.0002575, "median_seconds": 4.8751772723626345, "mean_seconds": 4.8751772723626345 } } }, { "id": "google/gemini-2.5-flash", "quality": 99.44444444444444, "mean_answer_usd": 0.0006436666666666666, "median_seconds": 1.296906251925975, "released_at": "2025-06-17T15:01:28+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0007725, "median_seconds": 1.3748913819435984, "mean_seconds": 1.6562789172166958 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.0006706666666666666, "median_seconds": 1.296906251925975, "mean_seconds": 1.5108225701066356 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.0003455, "median_seconds": 1.3012343887239695, "mean_seconds": 1.3012343887239695 } } }, { "id": "qwen/qwen3.8-omni-flash", "quality": 99.05555555555556, "mean_answer_usd": 0.00020422222222222223, "median_seconds": 5.080231613013893, "released_at": "2026-09-21T03:05:54+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000316, "median_seconds": 6.2621354148723185, "mean_seconds": 8.451483569457196 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.000129, "median_seconds": 3.032624182291329, "mean_seconds": 3.5422458327375352 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.0000935, "median_seconds": 3.440282996511087, "mean_seconds": 3.440282996511087 } } }, { "id": "xiaomi/mimo-v2.6-flash", "quality": 98.06666666666666, "mean_answer_usd": 0.000057444444444444444, "median_seconds": 15.731094161979854, "released_at": "2026-09-21T20:07:44+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00005875, "median_seconds": 11.77245675213635, "mean_seconds": 14.240605911822058 }, "research": { "samples": 3, "quality": 95.2, "mean_answer_usd": 0.00007166666666666667, "median_seconds": 15.731094161979854, "mean_seconds": 16.00637740502134 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.0000335, "median_seconds": 21.247065249364823, "mean_seconds": 21.247065249364823 } } }, { "id": "openai/gpt-4.1-mini", "quality": 97.5, "mean_answer_usd": 0.00043788888888888887, "median_seconds": 1.737336364109069, "released_at": "2025-04-14T17:23:01+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00048675, "median_seconds": 1.8618294345214963, "mean_seconds": 3.3321951450780034 }, "research": { "samples": 3, "quality": 92.5, "mean_answer_usd": 0.0004953333333333333, "median_seconds": 1.737336364109069, "mean_seconds": 2.4488411494530737 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000254, "median_seconds": 1.900494830450043, "mean_seconds": 1.900494830450043 } } }, { "id": "openai/gpt-4.1", "quality": 96.57777777777778, "mean_answer_usd": 0.0020391111111111113, "median_seconds": 1.7646328499540687, "released_at": "2025-04-14T17:23:05+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0024515, "median_seconds": 1.714392963098362, "mean_seconds": 2.150879738968797 }, "research": { "samples": 3, "quality": 91.4, "mean_answer_usd": 0.0019973333333333336, "median_seconds": 2.7485987842082977, "mean_seconds": 2.7896910024185977 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.001277, "median_seconds": 1.7616721594240516, "mean_seconds": 1.7616721594240516 } } }, { "id": "deepseek/deepseek-v3.2", "quality": 96.76666666666667, "mean_answer_usd": 0.00018322222222222223, "median_seconds": 8.731968097388744, "released_at": "2025-12-01T13:10:42+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00025475, "median_seconds": 17.871792779536918, "mean_seconds": 15.971008999738842 }, "research": { "samples": 3, "quality": 90.3, "mean_answer_usd": 0.00014, "median_seconds": 5.121913455892354, "mean_seconds": 9.554000391314426 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000105, "median_seconds": 6.0087788321543485, "mean_seconds": 6.0087788321543485 } } }, { "id": "deepseek/deepseek-v3.2-exp", "quality": 100, "mean_answer_usd": 0.00015955555555555555, "median_seconds": 17.535236082971096, "released_at": "2025-09-29T12:54:41+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00020575000000000001, "median_seconds": 18.204952914034948, "mean_seconds": 21.54107358888723 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.00014133333333333332, "median_seconds": 18.339914969168603, "mean_seconds": 16.489542350638658 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.00009449999999999999, "median_seconds": 10.758241838309914, "mean_seconds": 10.758241838309914 } } }, { "id": "qwen/qwen3.8-27b", "quality": 99.77777777777777, "mean_answer_usd": 0.000313, "median_seconds": 12.23341709189117, "released_at": "2026-08-14T15:55:10+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0005762499999999999, "median_seconds": 12.377707273932174, "mean_seconds": 13.157522222725675 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.00012066666666666667, "median_seconds": 11.298602701164782, "mean_seconds": 21.441077284049243 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.00007500000000000001, "median_seconds": 25.325225391658023, "mean_seconds": 25.325225391658023 } } }, { "id": "openai/gpt-oss-120b", "quality": 76.75925925925925, "mean_answer_usd": 0.00007755555555555556, "median_seconds": 5.450171762146056, "released_at": "2025-08-05T17:17:11+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0001, "median_seconds": 6.112645805347711, "mean_seconds": 7.152415256015956 }, "research": { "samples": 3, "quality": 51.5, "mean_answer_usd": 0.000071, "median_seconds": 3.196362820919603, "mean_seconds": 4.18123082878689 }, "creative": { "samples": 2, "quality": 68.16666666666666, "mean_answer_usd": 0.0000425, "median_seconds": 3.4073090879246593, "mean_seconds": 3.4073090879246593 } } }, { "id": "qwen/qwen3.6-35b-a3b", "quality": 99.88888888888889, "mean_answer_usd": 0.0003513333333333333, "median_seconds": 1.1640594908967614, "released_at": "2026-04-27T03:24:15+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0005265000000000001, "median_seconds": 2.594620679738, "mean_seconds": 2.5492784508969635 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.00024933333333333334, "median_seconds": 0.8339216588065028, "mean_seconds": 1.2779875208313267 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.000154, "median_seconds": 0.8509941797237843, "mean_seconds": 0.8509941797237843 } } }, { "id": "z-ai/glm-4.6", "quality": 60.37777777777778, "mean_answer_usd": 0.00035666666666666664, "median_seconds": 5.754874822683632, "released_at": "2025-09-30T12:32:56+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 56.85, "mean_answer_usd": 0.00041975, "median_seconds": 8.382614701986313, "mean_seconds": 13.672231268137693 }, "research": { "samples": 3, "quality": 48.333333333333336, "mean_answer_usd": 0.00033666666666666665, "median_seconds": 4.232173296157271, "mean_seconds": 5.305332814032833 }, "creative": { "samples": 2, "quality": 85.5, "mean_answer_usd": 0.0002605, "median_seconds": 4.0606637368910015, "mean_seconds": 4.0606637368910015 } } }, { "id": "mistralai/devstral-2512", "quality": 98.66666666666667, "mean_answer_usd": 0.00039, "median_seconds": 1.1526924851350486, "released_at": "2025-12-09T13:03:39+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0003195, "median_seconds": 1.1372022496070713, "mean_seconds": 1.1022207781206816 }, "research": { "samples": 3, "quality": 96, "mean_answer_usd": 0.0005653333333333333, "median_seconds": 1.3621778609231114, "mean_seconds": 1.638254713267088 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000268, "median_seconds": 1.1195613129530102, "mean_seconds": 1.1195613129530102 } } }, { "id": "deepseek/deepseek-v4-flash-0731", "quality": 72.60000000000001, "mean_answer_usd": 0.00004644444444444445, "median_seconds": 2.6495346766896546, "released_at": "2026-07-31T06:21:48+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 46.65, "mean_answer_usd": 0.0000555, "median_seconds": 2.4783242864068598, "mean_seconds": 3.8584712335141376 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.000045333333333333335, "median_seconds": 2.6495346766896546, "mean_seconds": 4.023601470204691 }, "creative": { "samples": 2, "quality": 83.4, "mean_answer_usd": 0.00003, "median_seconds": 2.0781292982865125, "mean_seconds": 2.0781292982865125 } } }, { "id": "minimax/minimax-m3", "quality": 98.14444444444445, "mean_answer_usd": 0.0004152222222222222, "median_seconds": 9.326441576238722, "released_at": "2026-05-31T16:36:14+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00051975, "median_seconds": 11.261479509528726, "mean_seconds": 16.139011886320077 }, "research": { "samples": 3, "quality": 94.9, "mean_answer_usd": 0.000387, "median_seconds": 9.326441576238722, "mean_seconds": 10.17146113406246 }, "creative": { "samples": 2, "quality": 99.3, "mean_answer_usd": 0.0002485, "median_seconds": 6.754532492952421, "mean_seconds": 6.754532492952421 } } }, { "id": "inception/mercury-3", "quality": 99.46666666666667, "mean_answer_usd": 0.00010333333333333333, "median_seconds": 0.7380497129634023, "released_at": "2026-10-16T13:45:37+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0001385, "median_seconds": 0.8444183529354632, "mean_seconds": 0.9537368239834905 }, "research": { "samples": 3, "quality": 98.4, "mean_answer_usd": 0.00008333333333333334, "median_seconds": 0.6686841319315135, "mean_seconds": 0.7410775888711214 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000063, "median_seconds": 0.7500935189891607, "mean_seconds": 0.7500935189891607 } } }, { "id": "anthropic/claude-haiku-4.5", "quality": 98.94444444444444, "mean_answer_usd": 0.0018914444444444443, "median_seconds": 3.3169399867765605, "released_at": "2025-10-15T17:00:38+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00260825, "median_seconds": 2.964237042469904, "mean_seconds": 3.746428188169375 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.0015643333333333334, "median_seconds": 3.9766359250061214, "mean_seconds": 3.7644399780159197 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.0009484999999999999, "median_seconds": 2.67180738132447, "mean_seconds": 2.67180738132447 } } }, { "id": "deepseek/deepseek-v3.1-terminus", "quality": 99.69999999999999, "mean_answer_usd": 0.00022611111111111112, "median_seconds": 8.505985083058476, "released_at": "2025-09-22T13:37:55+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00019899999999999999, "median_seconds": 9.777701068902388, "mean_seconds": 8.593102875864133 }, "research": { "samples": 3, "quality": 99.1, "mean_answer_usd": 0.0002643333333333333, "median_seconds": 8.260330399032682, "mean_seconds": 10.693319874349982 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000223, "median_seconds": 10.02395471232012, "mean_seconds": 10.02395471232012 } } }, { "id": "inclusionai/ling-3.0-flash-vl", "quality": 100, "mean_answer_usd": 0.000060222222222222225, "median_seconds": 3.895007752813399, "released_at": "2026-09-10T16:01:54+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00007950000000000001, "median_seconds": 4.10873194783926, "mean_seconds": 8.335091800428927 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.00005333333333333333, "median_seconds": 4.398690106347203, "mean_seconds": 4.115660401837279 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000032, "median_seconds": 3.262016064254567, "mean_seconds": 3.262016064254567 } } }, { "id": "deepseek/deepseek-chat-v3-0324", "quality": 99.88888888888889, "mean_answer_usd": 0.0003062222222222222, "median_seconds": 7.144548393320292, "released_at": "2025-03-24T13:59:15+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00038775, "median_seconds": 11.802267572144046, "mean_seconds": 11.881971337250434 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.0002846666666666667, "median_seconds": 7.753445118200034, "mean_seconds": 9.535668172097454 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.00017549999999999998, "median_seconds": 6.692794619593769, "mean_seconds": 6.692794619593769 } } }, { "id": "mistralai/mistral-medium-3", "quality": 99.5, "mean_answer_usd": 0.00039, "median_seconds": 1.1468684342689812, "released_at": "2025-05-07T14:15:41+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0003195, "median_seconds": 1.1003444602247328, "mean_seconds": 1.0896968325832859 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.0005653333333333333, "median_seconds": 1.3051197780296206, "mean_seconds": 1.621615425683558 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000268, "median_seconds": 1.1209561969153583, "mean_seconds": 1.1209561969153583 } } }, { "id": "mistralai/mistral-small-2603", "quality": 58.111111111111114, "mean_answer_usd": 0.00013533333333333333, "median_seconds": 1.6248900247737765, "released_at": "2026-03-16T21:14:45+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 61.300000000000004, "mean_answer_usd": 0.0001455, "median_seconds": 1.3255799061153084, "mean_seconds": 1.4456610082415864 }, "research": { "samples": 3, "quality": 25.933333333333337, "mean_answer_usd": 0.00014133333333333332, "median_seconds": 1.6248900247737765, "mean_seconds": 1.5346740518386166 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000106, "median_seconds": 2.296510110842064, "mean_seconds": 2.296510110842064 } } }, { "id": "nousresearch/hermes-3-llama-3.1-405b", "quality": 95.34444444444445, "mean_answer_usd": 0.0004605555555555556, "median_seconds": 12.642047369852662, "released_at": "2024-08-16T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000603, "median_seconds": 18.639828785555437, "mean_seconds": 18.096413365448825 }, "research": { "samples": 3, "quality": 88.5, "mean_answer_usd": 0.0003876666666666667, "median_seconds": 5.569898669142276, "mean_seconds": 7.85746394827341 }, "creative": { "samples": 2, "quality": 96.3, "mean_answer_usd": 0.000285, "median_seconds": 5.7837682268582284, "mean_seconds": 5.7837682268582284 } } }, { "id": "upstage/solar-pro4", "quality": 99.30000000000001, "mean_answer_usd": 0.0002122222222222222, "median_seconds": 3.889287155121565, "released_at": "2026-08-10T14:20:36+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00029775, "median_seconds": 6.675077742431313, "mean_seconds": 7.946501294965856 }, "research": { "samples": 3, "quality": 97.9, "mean_answer_usd": 0.000172, "median_seconds": 4.5958117810077965, "mean_seconds": 4.035022878243278 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0001015, "median_seconds": 2.6290498594753444, "mean_seconds": 2.6290498594753444 } } }, { "id": "inclusionai/ling-3.0-flash", "quality": 98.66666666666667, "mean_answer_usd": 0.000015222222222222223, "median_seconds": 1.2696630880236626, "released_at": "2026-07-23T14:56:20+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00001525, "median_seconds": 1.2714127379003912, "mean_seconds": 1.4318247938062996 }, "research": { "samples": 3, "quality": 96, "mean_answer_usd": 0.000017, "median_seconds": 1.2632363070733845, "mean_seconds": 1.606740667950362 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0000125, "median_seconds": 1.426723460899666, "mean_seconds": 1.426723460899666 } } }, { "id": "google/gemini-2.5-flash-lite", "quality": 98.11111111111111, "mean_answer_usd": 0.00020344444444444444, "median_seconds": 1.3883626810275018, "released_at": "2025-07-22T16:04:36+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00031099999999999997, "median_seconds": 2.114047046052292, "mean_seconds": 2.456389593775384 }, "research": { "samples": 3, "quality": 96, "mean_answer_usd": 0.00013933333333333332, "median_seconds": 1.1353035192005336, "mean_seconds": 1.4410669587862988 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.0000845, "median_seconds": 1.1759192263707519, "mean_seconds": 1.1759192263707519 } } }, { "id": "mistralai/mistral-medium-3.1", "quality": 99.5, "mean_answer_usd": 0.00038044444444444443, "median_seconds": 1.2554078586399555, "released_at": "2025-08-13T14:33:59+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0003195, "median_seconds": 1.1585477502085268, "mean_seconds": 1.3316498760832474 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.0005653333333333333, "median_seconds": 1.2554078586399555, "mean_seconds": 1.7030772405366104 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000225, "median_seconds": 1.1218776730820537, "mean_seconds": 1.1218776730820537 } } }, { "id": "qwen/qwen3.7-flash", "quality": 99.67777777777778, "mean_answer_usd": 0.000049444444444444446, "median_seconds": 3.362508454825729, "released_at": "2026-07-27T22:16:01+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000071, "median_seconds": 4.014666917733848, "mean_seconds": 5.630941102397628 }, "research": { "samples": 3, "quality": 99.7, "mean_answer_usd": 0.000036, "median_seconds": 2.412467218004167, "mean_seconds": 2.808216487367948 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.0000265, "median_seconds": 2.9615406848024577, "mean_seconds": 2.9615406848024577 } } }, { "id": "openai/gpt-oss-20b", "quality": 49.28333333333333, "mean_answer_usd": 0.000041666666666666665, "median_seconds": 2.7263004519045353, "released_at": "2025-08-05T17:17:09+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 40, "mean_answer_usd": 0.000057, "median_seconds": 6.277574524981901, "mean_seconds": 7.337497681030072 }, "research": { "samples": 3, "quality": 45.35, "mean_answer_usd": 0.000034, "median_seconds": 2.7263004519045353, "mean_seconds": 3.206338689662516 }, "creative": { "samples": 2, "quality": 73.75, "mean_answer_usd": 0.000022499999999999998, "median_seconds": 2.193077679956332, "mean_seconds": 2.193077679956332 } } }, { "id": "sao10k/l3.1-euryale-70b", "quality": 96.63333333333334, "mean_answer_usd": 0.00036944444444444443, "median_seconds": 5.280517118982971, "released_at": "2024-08-28T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000475, "median_seconds": 9.247041139518842, "mean_seconds": 9.359985004295595 }, "research": { "samples": 3, "quality": 94.9, "mean_answer_usd": 0.000304, "median_seconds": 3.2545796371996403, "mean_seconds": 3.9896272535746298 }, "creative": { "samples": 2, "quality": 92.5, "mean_answer_usd": 0.0002565, "median_seconds": 3.431878002593294, "mean_seconds": 3.431878002593294 } } }, { "id": "mistralai/mistral-large", "quality": 46.577777777777776, "mean_answer_usd": 0.0016216666666666667, "median_seconds": 2.5448006270453334, "released_at": "2024-02-26T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 45.35, "mean_answer_usd": 0.00170875, "median_seconds": 4.273802746320143, "mean_seconds": 5.188048598356545 }, "research": { "samples": 3, "quality": 26.866666666666667, "mean_answer_usd": 0.0017813333333333333, "median_seconds": 2.4593701320700347, "mean_seconds": 3.068722181332608 }, "creative": { "samples": 2, "quality": 78.6, "mean_answer_usd": 0.001208, "median_seconds": 2.7098552179522812, "mean_seconds": 2.7098552179522812 } } }, { "id": "meta-llama/llama-3.3-70b-instruct", "quality": 95.6, "mean_answer_usd": 0.00009555555555555555, "median_seconds": 5.292328028939664, "released_at": "2024-12-06T17:28:57+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000144, "median_seconds": 8.290279818931594, "mean_seconds": 16.48116584017407 }, "research": { "samples": 3, "quality": 86.8, "mean_answer_usd": 0.00006566666666666667, "median_seconds": 3.6732010329142213, "mean_seconds": 5.627706087970485 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0000435, "median_seconds": 3.296466108644381, "mean_seconds": 3.296466108644381 } } }, { "id": "dots-studio/dots-3-note-preview:free", "quality": 45.37777777777778, "mean_answer_usd": 0, "median_seconds": 3.520392674021423, "released_at": "2026-08-14T04:06:01+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 37.45, "mean_answer_usd": 0, "median_seconds": 5.222981275059283, "mean_seconds": 5.024388182093389 }, "research": { "samples": 3, "quality": 48.86666666666667, "mean_answer_usd": 0, "median_seconds": 2.849245955236256, "mean_seconds": 3.2424490048239627 }, "creative": { "samples": 2, "quality": 56, "mean_answer_usd": 0, "median_seconds": 2.955210061511025, "mean_seconds": 2.955210061511025 } } }, { "id": "amazon/nova-2-lite-v1", "quality": 97.48888888888888, "mean_answer_usd": 0.0006202222222222222, "median_seconds": 1.2375379242002964, "released_at": "2025-12-02T17:31:12+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000873, "median_seconds": 1.799191399011761, "mean_seconds": 2.115402907016687 }, "research": { "samples": 3, "quality": 93.8, "mean_answer_usd": 0.000495, "median_seconds": 1.2345817051827908, "mean_seconds": 1.3638093650030594 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.00030250000000000003, "median_seconds": 1.1960869571194053, "mean_seconds": 1.1960869571194053 } } }, { "id": "mistralai/mixtral-8x22b-instruct", "quality": 44.22222222222222, "mean_answer_usd": 0.001324888888888889, "median_seconds": 1.2527015726082027, "released_at": "2024-04-17T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 54.9, "mean_answer_usd": 0.001227, "median_seconds": 1.1919030270073563, "mean_seconds": 1.26536995859351 }, "research": { "samples": 3, "quality": 20.2, "mean_answer_usd": 0.0015503333333333335, "median_seconds": 1.357894935645163, "mean_seconds": 1.4688499976570408 }, "creative": { "samples": 2, "quality": 58.900000000000006, "mean_answer_usd": 0.0011825, "median_seconds": 1.438147742068395, "mean_seconds": 1.438147742068395 } } }, { "id": "qwen/qwen3-235b-a22b-2507", "quality": 44.17777777777778, "mean_answer_usd": 0.00019155555555555554, "median_seconds": 3.89751685783267, "released_at": "2025-07-21T17:39:15+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 33.25, "mean_answer_usd": 0.000297, "median_seconds": 8.754496755078435, "mean_seconds": 16.918683782801963 }, "research": { "samples": 3, "quality": 38, "mean_answer_usd": 0.00012533333333333334, "median_seconds": 2.3914040187373757, "mean_seconds": 3.4414450771485767 }, "creative": { "samples": 2, "quality": 75.3, "mean_answer_usd": 0.00007999999999999999, "median_seconds": 1.9669839311391115, "mean_seconds": 1.9669839311391115 } } }, { "id": "thedrummer/cydonia-24b-v4.1", "quality": 99.27777777777777, "mean_answer_usd": 0.00015911111111111112, "median_seconds": 3.31935108685866, "released_at": "2025-09-27T00:11:18+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000208, "median_seconds": 5.8492719165515155, "mean_seconds": 5.689493098994717 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.000134, "median_seconds": 2.5182479498907924, "mean_seconds": 3.6221019226747253 }, "creative": { "samples": 2, "quality": 99, "mean_answer_usd": 0.000099, "median_seconds": 2.515462512150407, "mean_seconds": 2.515462512150407 } } }, { "id": "qwen/qwen2.5-vl-72b-instruct", "quality": 90.77777777777777, "mean_answer_usd": 0.00027055555555555557, "median_seconds": 3.335109729319811, "released_at": "2025-02-01T11:45:11+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00027925, "median_seconds": 3.7035934664309025, "mean_seconds": 3.7524558095028624 }, "research": { "samples": 3, "quality": 74, "mean_answer_usd": 0.000294, "median_seconds": 3.335109729319811, "mean_seconds": 4.371462244695674 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.00021799999999999999, "median_seconds": 2.517593716038391, "mean_seconds": 2.517593716038391 } } }, { "id": "google/gemma-3-12b-it", "quality": 97.95555555555556, "mean_answer_usd": 0.000043333333333333334, "median_seconds": 3.6915401867590845, "released_at": "2025-03-13T21:50:25+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00004875, "median_seconds": 4.548005319898948, "mean_seconds": 4.503764216322452 }, "research": { "samples": 3, "quality": 97.2, "mean_answer_usd": 0.000044, "median_seconds": 2.9074498610571027, "mean_seconds": 3.5135324746370316 }, "creative": { "samples": 2, "quality": 95, "mean_answer_usd": 0.0000315, "median_seconds": 2.9822569196112454, "mean_seconds": 2.9822569196112454 } } }, { "id": "qwen/qwen3-vl-30b-a3b-instruct", "quality": 94.33333333333333, "mean_answer_usd": 0.00023233333333333333, "median_seconds": 5.630168443080038, "released_at": "2025-10-06T23:47:56+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00035875, "median_seconds": 10.159198620123789, "mean_seconds": 30.07740597729571 }, "research": { "samples": 3, "quality": 83, "mean_answer_usd": 0.00015966666666666665, "median_seconds": 4.959728023037314, "mean_seconds": 8.117994291863093 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.00008850000000000001, "median_seconds": 3.947444331133738, "mean_seconds": 3.947444331133738 } } }, { "id": "qwen/qwen3.5-9b", "quality": 41.37777777777778, "mean_answer_usd": 0.00006788888888888888, "median_seconds": 11.389032318256795, "released_at": "2026-03-10T14:19:56+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 28.65, "mean_answer_usd": 0.00009075, "median_seconds": 15.295091691659763, "mean_seconds": 15.442394413985312 }, "research": { "samples": 3, "quality": 40.46666666666667, "mean_answer_usd": 0.00005766666666666667, "median_seconds": 11.465040570124984, "mean_seconds": 9.739427427606037 }, "creative": { "samples": 2, "quality": 68.2, "mean_answer_usd": 0.000037500000000000003, "median_seconds": 4.837973739951849, "mean_seconds": 4.837973739951849 } } }, { "id": "qwen/qwen3-30b-a3b-instruct-2507", "quality": 95, "mean_answer_usd": 0.00009066666666666667, "median_seconds": 16.14329433394596, "released_at": "2025-07-29T16:36:05+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00011449999999999999, "median_seconds": 19.88341761706397, "mean_seconds": 19.71080845524557 }, "research": { "samples": 3, "quality": 85, "mean_answer_usd": 0.00008433333333333333, "median_seconds": 2.3567279963754117, "mean_seconds": 14.170074868326386 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0000525, "median_seconds": 12.39737473288551, "mean_seconds": 12.39737473288551 } } }, { "id": "perceptron/perceptron-mk1.5", "quality": 55.05555555555556, "mean_answer_usd": 0.00033433333333333335, "median_seconds": 1.2141954810358584, "released_at": "2026-09-25T16:11:01+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.00045575, "median_seconds": 1.8317750424612314, "mean_seconds": 1.9611663953401148 }, "research": { "samples": 3, "quality": 98.5, "mean_answer_usd": 0.00023999999999999998, "median_seconds": 1.2141954810358584, "mean_seconds": 1.1784756062552333 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.00023300000000000003, "median_seconds": 1.1608841510023922, "mean_seconds": 1.1608841510023922 } } }, { "id": "qwen/qwen-2.5-72b-instruct", "quality": 96.38888888888889, "mean_answer_usd": 0.00016233333333333334, "median_seconds": 4.658019149210304, "released_at": "2024-09-19T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000184, "median_seconds": 5.133828041376546, "mean_seconds": 7.869943588739261 }, "research": { "samples": 3, "quality": 89.5, "mean_answer_usd": 0.00016766666666666666, "median_seconds": 4.453408694360405, "mean_seconds": 5.64944636526828 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.00011100000000000001, "median_seconds": 3.411324695451185, "mean_seconds": 3.411324695451185 } } }, { "id": "mistralai/codestral-2508", "quality": 40, "mean_answer_usd": 0.00024333333333333333, "median_seconds": 1.4123754589818418, "released_at": "2025-08-01T20:20:30+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 42.300000000000004, "mean_answer_usd": 0.00029775, "median_seconds": 1.601702485466376, "mean_seconds": 2.0824606714304537 }, "research": { "samples": 3, "quality": 20.400000000000002, "mean_answer_usd": 0.00021133333333333334, "median_seconds": 1.6000170316547155, "mean_seconds": 1.3371640462428331 }, "creative": { "samples": 2, "quality": 64.80000000000001, "mean_answer_usd": 0.0001825, "median_seconds": 1.1817159235943109, "mean_seconds": 1.1817159235943109 } } }, { "id": "nousresearch/hermes-3-llama-3.1-70b", "quality": 90.77777777777777, "mean_answer_usd": 0.00030744444444444444, "median_seconds": 5.305041420273483, "released_at": "2024-08-18T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00038275, "median_seconds": 12.975195104489103, "mean_seconds": 14.02534554398153 }, "research": { "samples": 3, "quality": 74, "mean_answer_usd": 0.00026933333333333334, "median_seconds": 5.230810466222465, "mean_seconds": 5.695140016730875 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.000214, "median_seconds": 4.506323697045445, "mean_seconds": 4.506323697045445 } } }, { "id": "meta-llama/llama-3.1-70b-instruct", "quality": 95.33333333333333, "mean_answer_usd": 0.0001651111111111111, "median_seconds": 4.098976875655353, "released_at": "2024-07-23T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00018525, "median_seconds": 5.8280488778837025, "mean_seconds": 8.31812674482353 }, "research": { "samples": 3, "quality": 96, "mean_answer_usd": 0.00016133333333333334, "median_seconds": 3.913562591653317, "mean_seconds": 5.435773941067358 }, "creative": { "samples": 2, "quality": 85, "mean_answer_usd": 0.0001305, "median_seconds": 4.256364079657942, "mean_seconds": 4.256364079657942 } } }, { "id": "amazon/nova-pro-v1", "quality": 99.1888888888889, "mean_answer_usd": 0.0005508888888888888, "median_seconds": 1.1795074376277626, "released_at": "2024-12-05T22:05:03+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0007307500000000001, "median_seconds": 1.3753385599702597, "mean_seconds": 1.495088277501054 }, "research": { "samples": 3, "quality": 98.9, "mean_answer_usd": 0.00039433333333333334, "median_seconds": 0.9954588250257075, "mean_seconds": 0.9734910307452083 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.000426, "median_seconds": 1.3004736152943224, "mean_seconds": 1.3004736152943224 } } }, { "id": "google/gemma-3-27b-it", "quality": 99.55555555555556, "mean_answer_usd": 0.000050333333333333335, "median_seconds": 5.16260798741132, "released_at": "2025-03-12T05:12:39+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00005625, "median_seconds": 5.297799480613321, "mean_seconds": 7.215939151006751 }, "research": { "samples": 3, "quality": 99, "mean_answer_usd": 0.000048666666666666666, "median_seconds": 5.16260798741132, "mean_seconds": 6.016726430815955 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.000041, "median_seconds": 5.431743064895272, "mean_seconds": 5.431743064895272 } } }, { "id": "upstage/solar-mini4", "quality": 96.47222222222223, "mean_answer_usd": 0.00007266666666666667, "median_seconds": 2.9728024750947952, "released_at": "2026-09-23T10:45:58+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00008875, "median_seconds": 3.275355285964906, "mean_seconds": 4.652705263579264 }, "research": { "samples": 3, "quality": 90.75, "mean_answer_usd": 0.000071, "median_seconds": 3.274990724865347, "mean_seconds": 4.360890458337963 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.000043, "median_seconds": 1.8691444552969187, "mean_seconds": 1.8691444552969187 } } }, { "id": "nvidia/nemotron-3-nano-30b-a3b", "quality": 42.81481481481482, "mean_answer_usd": 0.00005544444444444444, "median_seconds": 1.2344300728291273, "released_at": "2025-12-14T16:54:35+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 33.333333333333336, "mean_answer_usd": 0.0000505, "median_seconds": 0.8486776645295322, "mean_seconds": 1.0590305206133053 }, "research": { "samples": 3, "quality": 51, "mean_answer_usd": 0.00007133333333333332, "median_seconds": 1.2609397317282856, "mean_seconds": 1.3451974291043978 }, "creative": { "samples": 2, "quality": 49.5, "mean_answer_usd": 0.0000415, "median_seconds": 0.976374289020896, "mean_seconds": 0.976374289020896 } } }, { "id": "mistralai/ministral-14b-2512", "quality": 92.25555555555555, "mean_answer_usd": 0.00008722222222222222, "median_seconds": 3.151333475019783, "released_at": "2025-12-02T13:22:15+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00010025, "median_seconds": 3.8951435582712293, "mean_seconds": 6.564728977391496 }, "research": { "samples": 3, "quality": 77.1, "mean_answer_usd": 0.000081, "median_seconds": 3.151333475019783, "mean_seconds": 3.6663914810245237 }, "creative": { "samples": 2, "quality": 99.5, "mean_answer_usd": 0.0000705, "median_seconds": 3.5531286215409636, "mean_seconds": 3.5531286215409636 } } }, { "id": "anthracite-org/magnum-v4-72b", "quality": 99.44444444444444, "mean_answer_usd": 0.0015012222222222222, "median_seconds": 9.77762448694557, "released_at": "2024-10-22T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0017125, "median_seconds": 12.78397824196145, "mean_seconds": 15.913529804092832 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.0015103333333333334, "median_seconds": 12.971942740958184, "mean_seconds": 11.395941561708847 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.001065, "median_seconds": 7.836568020982668, "mean_seconds": 7.836568020982668 } } }, { "id": "cohere/north-mini-code:free", "quality": 61.91111111111112, "mean_answer_usd": 0, "median_seconds": 3.207594132050872, "released_at": "2026-06-17T19:15:48+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 27.650000000000002, "mean_answer_usd": 0, "median_seconds": 6.25470329192467, "mean_seconds": 10.682628524373285 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0, "median_seconds": 2.7362855561077595, "mean_seconds": 3.8271160242147744 }, "creative": { "samples": 2, "quality": 73.30000000000001, "mean_answer_usd": 0, "median_seconds": 2.628845270490274, "mean_seconds": 2.628845270490274 } } }, { "id": "z-ai/glm-4.5-air", "quality": 35.4, "mean_answer_usd": 0.00015322222222222224, "median_seconds": 3.1926648817025125, "released_at": "2025-07-25T19:20:58+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 35.4, "mean_answer_usd": 0.00020075, "median_seconds": 4.420616548974067, "mean_seconds": 5.741289505502209 }, "research": { "samples": 3, "quality": 21, "mean_answer_usd": 0.00012066666666666667, "median_seconds": 3.090190020855516, "mean_seconds": 2.9632680183276534 }, "creative": { "samples": 2, "quality": 57, "mean_answer_usd": 0.000107, "median_seconds": 2.981811430770904, "mean_seconds": 2.981811430770904 } } }, { "id": "nousresearch/hermes-4-405b", "quality": 98.33333333333333, "mean_answer_usd": 0.0008152222222222222, "median_seconds": 4.325308780185878, "released_at": "2025-08-26T19:11:03+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00102575, "median_seconds": 7.056167067494243, "mean_seconds": 7.489449147717096 }, "research": { "samples": 3, "quality": 95, "mean_answer_usd": 0.0007196666666666666, "median_seconds": 3.7762238876894116, "mean_seconds": 3.9310349437097707 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0005375, "median_seconds": 3.0103063988499343, "mean_seconds": 3.0103063988499343 } } }, { "id": "mistralai/voxtral-small-24b-2507", "quality": 98.77777777777777, "mean_answer_usd": 0.00007144444444444445, "median_seconds": 1.1849211622029543, "released_at": "2025-10-30T14:39:04+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00007625000000000001, "median_seconds": 1.5135903283953667, "mean_seconds": 1.6759319168049842 }, "research": { "samples": 3, "quality": 98, "mean_answer_usd": 0.00007566666666666667, "median_seconds": 1.1849211622029543, "mean_seconds": 1.4612956764176488 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.00005550000000000001, "median_seconds": 1.0548517650458962, "mean_seconds": 1.0548517650458962 } } }, { "id": "scaledown/summarize", "quality": 79.32222222222222, "mean_answer_usd": 0.000014888888888888888, "median_seconds": 0.8727567191235721, "released_at": "2026-10-13T12:09:20+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 60, "mean_answer_usd": 0.000014, "median_seconds": 1.4145150275435299, "mean_seconds": 1.3970551734091714 }, "research": { "samples": 3, "quality": 92.5, "mean_answer_usd": 0.00001633333333333333, "median_seconds": 0.9458095580339432, "mean_seconds": 0.9576268487920364 }, "creative": { "samples": 2, "quality": 98.2, "mean_answer_usd": 0.0000145, "median_seconds": 0.6914118544664234, "mean_seconds": 0.6914118544664234 } } }, { "id": "thedrummer/skyfall-36b-v2", "quality": 86.36666666666666, "mean_answer_usd": 0.0003157777777777778, "median_seconds": 3.756043078843504, "released_at": "2025-03-10T19:56:06+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 80, "mean_answer_usd": 0.00043075000000000003, "median_seconds": 5.21726029063575, "mean_seconds": 6.078539362293668 }, "research": { "samples": 3, "quality": 87.1, "mean_answer_usd": 0.00026233333333333333, "median_seconds": 2.2576684742234647, "mean_seconds": 3.040798358308772 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.000166, "median_seconds": 1.9132876063231379, "mean_seconds": 1.9132876063231379 } } }, { "id": "qwen/qwen3-vl-8b-instruct", "quality": 92.58888888888889, "mean_answer_usd": 0.00018144444444444444, "median_seconds": 2.015858568251133, "released_at": "2025-10-14T17:35:08+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.0002075, "median_seconds": 2.3712816485203803, "mean_seconds": 2.6149584028171375 }, "research": { "samples": 3, "quality": 79.1, "mean_answer_usd": 0.00020166666666666664, "median_seconds": 1.5333656831644475, "mean_seconds": 2.0230501433834434 }, "creative": { "samples": 2, "quality": 98, "mean_answer_usd": 0.000099, "median_seconds": 1.0208949418738484, "mean_seconds": 1.0208949418738484 } } }, { "id": "qwen/qwen3-coder-30b-a3b-instruct", "quality": 38.68888888888889, "mean_answer_usd": 0.00008233333333333333, "median_seconds": 7.658601175062358, "released_at": "2025-07-31T14:32:59+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 40, "mean_answer_usd": 0.00011525, "median_seconds": 12.060736660379916, "mean_seconds": 11.472797093680128 }, "research": { "samples": 3, "quality": 36.4, "mean_answer_usd": 0.00006733333333333333, "median_seconds": 7.658601175062358, "mean_seconds": 7.055594421147059 }, "creative": { "samples": 2, "quality": 39.5, "mean_answer_usd": 0.000039, "median_seconds": 4.020321052521467, "mean_seconds": 4.020321052521467 } } }, { "id": "openai/gpt-4o-mini", "quality": 91.5, "mean_answer_usd": 0.00015855555555555555, "median_seconds": 2.454403553158045, "released_at": "2024-07-18T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00018525, "median_seconds": 2.685425538569689, "mean_seconds": 3.517258648062125 }, "research": { "samples": 3, "quality": 74.5, "mean_answer_usd": 0.000143, "median_seconds": 1.9898974210955203, "mean_seconds": 2.64780529246976 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0001285, "median_seconds": 2.9169363256078213, "mean_seconds": 2.9169363256078213 } } }, { "id": "google/gemma-3-4b-it", "quality": 26.711111111111112, "mean_answer_usd": 0.000019666666666666666, "median_seconds": 1.5063243401236832, "released_at": "2025-03-13T22:38:30+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 15.7, "mean_answer_usd": 0.000022, "median_seconds": 1.8240520393010229, "mean_seconds": 3.344238297198899 }, "research": { "samples": 3, "quality": 20.133333333333333, "mean_answer_usd": 0.000019333333333333333, "median_seconds": 1.154984808061272, "mean_seconds": 1.3667102876740198 }, "creative": { "samples": 2, "quality": 58.6, "mean_answer_usd": 0.0000155, "median_seconds": 1.3729775636456907, "mean_seconds": 1.3729775636456907 } } }, { "id": "amazon/nova-lite-v1", "quality": 89.46666666666667, "mean_answer_usd": 0.000051555555555555556, "median_seconds": 1.1578322099521756, "released_at": "2024-12-05T22:22:43+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 85, "mean_answer_usd": 0.00006275000000000001, "median_seconds": 1.5510493845213205, "mean_seconds": 1.4988105786032975 }, "research": { "samples": 3, "quality": 88.4, "mean_answer_usd": 0.00005033333333333333, "median_seconds": 1.1578322099521756, "mean_seconds": 1.1509906298791368 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000031, "median_seconds": 0.9476761175319552, "mean_seconds": 0.9476761175319552 } } }, { "id": "thedrummer/unslopnemo-12b", "quality": 96.66666666666667, "mean_answer_usd": 0.00015133333333333334, "median_seconds": 1.807412937283516, "released_at": "2024-11-08T22:04:08+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00016875, "median_seconds": 2.1377921015955508, "mean_seconds": 2.929026418016292 }, "research": { "samples": 3, "quality": 95, "mean_answer_usd": 0.00016433333333333333, "median_seconds": 1.807412937283516, "mean_seconds": 2.031703599883864 }, "creative": { "samples": 2, "quality": 92.5, "mean_answer_usd": 0.000097, "median_seconds": 1.4083857301156968, "mean_seconds": 1.4083857301156968 } } }, { "id": "mistralai/ministral-8b-2512", "quality": 97.66666666666667, "mean_answer_usd": 0.00006255555555555555, "median_seconds": 1.416570883244276, "released_at": "2025-12-02T13:20:54+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00006125, "median_seconds": 1.3799291625618935, "mean_seconds": 1.680461431038566 }, "research": { "samples": 3, "quality": 93, "mean_answer_usd": 0.00006533333333333333, "median_seconds": 2.599421086255461, "mean_seconds": 2.621300734890004 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000061, "median_seconds": 1.6420383756048977, "mean_seconds": 1.6420383756048977 } } }, { "id": "qwen/qwen-2.5-7b-instruct", "quality": 88.52222222222223, "mean_answer_usd": 0.00005477777777777778, "median_seconds": 1.405072109773755, "released_at": "2024-10-16T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000061000000000000005, "median_seconds": 1.4722073033917695, "mean_seconds": 1.6324421962490305 }, "research": { "samples": 3, "quality": 69.9, "mean_answer_usd": 0.000055333333333333334, "median_seconds": 0.9131949581205845, "mean_seconds": 1.546074778928111 }, "creative": { "samples": 2, "quality": 93.5, "mean_answer_usd": 0.0000415, "median_seconds": 1.1589669333770871, "mean_seconds": 1.1589669333770871 } } }, { "id": "meta-llama/llama-3.1-8b-instruct", "quality": 91.11111111111111, "mean_answer_usd": 0.000014111111111111112, "median_seconds": 5.775404147803783, "released_at": "2024-07-23T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.000015, "median_seconds": 5.253186671528965, "mean_seconds": 6.994051319314167 }, "research": { "samples": 3, "quality": 75, "mean_answer_usd": 0.000015, "median_seconds": 4.42790964897722, "mean_seconds": 6.2391444357732935 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.000011, "median_seconds": 5.921789621002972, "mean_seconds": 5.921789621002972 } } }, { "id": "mistralai/mistral-nemo", "quality": 25.11111111111111, "mean_answer_usd": 0.0000064444444444444445, "median_seconds": 3.1688524279743433, "released_at": "2024-07-19T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 38, "mean_answer_usd": 0.000006750000000000001, "median_seconds": 3.2283977903425694, "mean_seconds": 3.976372987148352 }, "research": { "samples": 3, "quality": 10.333333333333334, "mean_answer_usd": 0.000007, "median_seconds": 3.2889264370314777, "mean_seconds": 3.257859277228514 }, "creative": { "samples": 2, "quality": 21.5, "mean_answer_usd": 0.000005, "median_seconds": 2.5134169473312795, "mean_seconds": 2.5134169473312795 } } }, { "id": "amazon/nova-micro-v1", "quality": 90.17777777777778, "mean_answer_usd": 0.00003355555555555556, "median_seconds": 1.197324275970459, "released_at": "2024-12-05T22:20:37+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 100, "mean_answer_usd": 0.00004, "median_seconds": 1.1236657397821546, "mean_seconds": 1.2011453988961875 }, "research": { "samples": 3, "quality": 72.4, "mean_answer_usd": 0.00003, "median_seconds": 1.197324275970459, "mean_seconds": 1.1298663596001763 }, "creative": { "samples": 2, "quality": 97.2, "mean_answer_usd": 0.000026000000000000002, "median_seconds": 1.051474733510986, "mean_seconds": 1.051474733510986 } } }, { "id": "mistralai/ministral-3b-2512", "quality": 51.733333333333334, "mean_answer_usd": 0.00004266666666666667, "median_seconds": 0.9634624966420233, "released_at": "2025-12-02T13:19:20+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 37.6, "mean_answer_usd": 0.000044999999999999996, "median_seconds": 1.143297813134268, "mean_seconds": 1.1357408987823874 }, "research": { "samples": 3, "quality": 40.4, "mean_answer_usd": 0.00004066666666666667, "median_seconds": 0.9634624966420233, "mean_seconds": 0.9961657258681953 }, "creative": { "samples": 2, "quality": 97, "mean_answer_usd": 0.000041, "median_seconds": 1.0470884954556823, "mean_seconds": 1.0470884954556823 } } }, { "id": "tencent/hunyuan-a13b-instruct", "quality": 87.07777777777777, "mean_answer_usd": 0.00020733333333333332, "median_seconds": 8.309283213689923, "released_at": "2025-07-08T15:14:24+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 92.6, "mean_answer_usd": 0.000259, "median_seconds": 8.949951554415748, "mean_seconds": 10.945723316981457 }, "research": { "samples": 3, "quality": 72.1, "mean_answer_usd": 0.00018733333333333332, "median_seconds": 6.976378052029759, "mean_seconds": 7.454672682254265 }, "creative": { "samples": 2, "quality": 98.5, "mean_answer_usd": 0.000134, "median_seconds": 6.307565982919186, "mean_seconds": 6.307565982919186 } } }, { "id": "bytedance/ui-tars-1.5-7b", "quality": 40.888888888888886, "mean_answer_usd": 0.00006122222222222222, "median_seconds": 2.3608312979340553, "released_at": "2025-07-22T17:24:16+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.00005275, "median_seconds": 2.323040409013629, "mean_seconds": 2.182443137629889 }, "research": { "samples": 3, "quality": 56, "mean_answer_usd": 0.00008733333333333333, "median_seconds": 3.5935791078954935, "mean_seconds": 3.9434570049246154 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.000039, "median_seconds": 2.079418799607083, "mean_seconds": 2.079418799607083 } } }, { "id": "meta-llama/llama-3.2-3b-instruct", "quality": 44.388888888888886, "mean_answer_usd": 0.00008955555555555556, "median_seconds": 1.8683416941203177, "released_at": "2024-09-25T00:00:00+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.0000995, "median_seconds": 1.9471894917078316, "mean_seconds": 1.931502215564251 }, "research": { "samples": 3, "quality": 66.5, "mean_answer_usd": 0.00008833333333333333, "median_seconds": 1.4276537359692156, "mean_seconds": 1.5457906283748646 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0000715, "median_seconds": 1.6611158095765859, "mean_seconds": 1.6611158095765859 } } }, { "id": "moondream/moondream3-preview", "quality": 26.900000000000002, "mean_answer_usd": 0.00036, "median_seconds": 0.822419298812747, "released_at": "2026-10-15T22:13:05+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.0003305, "median_seconds": 1.0763531678821892, "mean_seconds": 1.0519476869376376 }, "research": { "samples": 3, "quality": 41.7, "mean_answer_usd": 0.000357, "median_seconds": 0.7234317711554468, "mean_seconds": 0.7370989210903645 }, "creative": { "samples": 2, "quality": 58.5, "mean_answer_usd": 0.0004235, "median_seconds": 0.8411435757298023, "mean_seconds": 0.8411435757298023 } } }, { "id": "inference-net/schematron-v2-small", "quality": 25.333333333333332, "mean_answer_usd": 0.000042555555555555554, "median_seconds": 0.624553075991571, "released_at": "2026-09-12T01:35:33+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.00003225, "median_seconds": 0.4649536204524338, "mean_seconds": 0.5695808854652569 }, "research": { "samples": 3, "quality": 11, "mean_answer_usd": 0.000049, "median_seconds": 0.8107602228410542, "mean_seconds": 0.7988133342005312 }, "creative": { "samples": 2, "quality": 97.5, "mean_answer_usd": 0.0000535, "median_seconds": 0.7749282813165337, "mean_seconds": 0.7749282813165337 } } }, { "id": "inference-net/schematron-v2-turbo", "quality": 25.644444444444446, "mean_answer_usd": 0.000047555555555555553, "median_seconds": 1.6301661217585206, "released_at": "2026-09-12T01:35:49+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.000042000000000000004, "median_seconds": 1.4651957501191646, "mean_seconds": 1.9759346725186333 }, "research": { "samples": 3, "quality": 16, "mean_answer_usd": 0.000043333333333333334, "median_seconds": 1.9075195319019258, "mean_seconds": 2.065209621253113 }, "creative": { "samples": 2, "quality": 91.4, "mean_answer_usd": 0.000065, "median_seconds": 2.2724043137859553, "mean_seconds": 2.2724043137859553 } } }, { "id": "meta-llama/llama-guard-4-12b", "quality": 5.611111111111111, "mean_answer_usd": 0.00006555555555555556, "median_seconds": 0.43081220611929893, "released_at": "2025-04-30T01:06:33+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 0, "mean_answer_usd": 0.000063, "median_seconds": 0.43754484155215323, "mean_seconds": 0.42780825681984425 }, "research": { "samples": 3, "quality": 3.5, "mean_answer_usd": 0.00006966666666666666, "median_seconds": 0.43081220611929893, "mean_seconds": 0.4341001268476248 }, "creative": { "samples": 2, "quality": 20, "mean_answer_usd": 0.0000645, "median_seconds": 0.3957128745969385, "mean_seconds": 0.3957128745969385 } } }, { "id": "openai/gpt-6.1-terra", "quality": 98.66666666666667, "mean_answer_usd": 0.0014405555555555556, "median_seconds": 3.7373764161020517, "released_at": "2026-10-26T18:13:29+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 97.5, "mean_answer_usd": 0.00153225, "median_seconds": 4.5619717496447265, "mean_seconds": 4.497048853780143 }, "research": { "samples": 3, "quality": 99.33333333333333, "mean_answer_usd": 0.001418, "median_seconds": 3.5435972418636084, "mean_seconds": 3.7023889768558242 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.001291, "median_seconds": 3.6974711299408227, "mean_seconds": 3.6974711299408227 } } }, { "id": "openai/gpt-6.1-terra-pro", "quality": 99.66666666666667, "mean_answer_usd": 0.004198555555555555, "median_seconds": 6.874042389914393, "released_at": "2026-10-26T18:15:56+00:00", "cost_basis": "reported", "categories": { "coding": { "samples": 4, "quality": 99.25, "mean_answer_usd": 0.0042585, "median_seconds": 6.18028205819428, "mean_seconds": 5.954788223607466 }, "research": { "samples": 3, "quality": 100, "mean_answer_usd": 0.004233333333333334, "median_seconds": 7.028569881804287, "mean_seconds": 7.040572887596984 }, "creative": { "samples": 2, "quality": 100, "mean_answer_usd": 0.0040265, "median_seconds": 6.271713332505897, "mean_seconds": 6.271713332505897 } } } ] }