{"request_params":{"needles":8,"mode":"full","model_slugs":null},"available_bins":[8192,16384,32768,65536,131072,262144,524288,1048576],"models":[{"model_slug":"google/gemini-3.7-flash","reasoning_mode":"high","overall_metrics":{"auc_1m":0.7887576853856791,"auc_128k":0.9389654127983247,"cum_avg_1m":0.888115908505733,"cum_avg_128k":0.9604232483607866,"avg_score_overall":0.8302514103027084,"avg_response_time_ms":9937.714489840722,"total_prompt_cost":6.491430750000003,"total_cache_read_cost":18.089336662500028,"total_completion_cost":0.8397956250000002,"total_reasoning_cost":10.3304625,"total_prompt_cost_128k":1.3932555000000002,"total_completion_cost_128k":0.34890187500000003,"total_on_demand_input_cost":187.38479737500043,"total_on_demand_input_cost_128k":9.205049999999998,"total_runs":1161,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000229398697198},"bin_metrics":{"8192":{"avg_score":1,"min_score":1,"max_score":1,"n_tests":80,"stddev":0,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747248,"is_incomplete":false,"total_prompt_cost_bin":0.19932900000000006,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.05245312500000001,"total_reasoning_cost_bin":0.23093062500000006,"total_cost_bin":0.48271275000000013,"total_on_demand_input_cost_bin":0.19932900000000006,"ci_99_lower":1,"ci_99_upper":1,"ci_98_lower":1,"ci_98_upper":1,"ci_95_lower":1,"ci_95_upper":1,"ci_90_lower":1,"ci_90_upper":1,"ci_75_lower":1,"ci_75_upper":1,"ci_50_lower":1,"ci_50_upper":1},"16384":{"avg_score":0.9999417401850224,"min_score":0.9953392148017883,"max_score":1,"n_tests":80,"stddev":0.0005210916265863064,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024478,"is_incomplete":false,"total_prompt_cost_bin":0.40400474999999986,"total_cache_read_cost_bin":0.0009197249999999999,"total_completion_cost_bin":0.05208562499999999,"total_reasoning_cost_bin":0.2607956250000001,"total_cost_bin":0.7178057249999998,"total_on_demand_input_cost_bin":0.41320199999999996,"ci_99_lower":0.9997669607400894,"ci_99_upper":1,"ci_98_lower":0.9997669607400894,"ci_98_upper":1,"ci_95_lower":0.9998252205550671,"ci_95_upper":1,"ci_90_lower":0.9998252205550671,"ci_90_upper":1,"ci_75_lower":0.9998834803700447,"ci_75_upper":1,"ci_50_lower":0.9998834803700447,"ci_50_upper":1},"32768":{"avg_score":0.9856939005035469,"min_score":0.025095228105783463,"max_score":1,"n_tests":136,"stddev":0.11751105692993528,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000035909094991,"is_incomplete":false,"total_prompt_cost_bin":0.3204783750000002,"total_cache_read_cost_bin":0.11320526250000015,"total_completion_cost_bin":0.09933375,"total_reasoning_cost_bin":0.5565431249999999,"total_cost_bin":1.0895605125000003,"total_on_demand_input_cost_bin":1.4525309999999982,"ci_99_lower":0.949985593251117,"ci_99_upper":1,"ci_98_lower":0.9570509389623562,"ci_98_upper":1,"ci_95_lower":0.9642159666979055,"ci_95_upper":1,"ci_90_lower":0.9642848861739314,"ci_90_upper":0.9999972836059683,"ci_75_lower":0.9714236296302474,"ci_75_upper":0.999989134423873,"ci_50_lower":0.9785589348415241,"ci_50_upper":0.9928596015499138},"65536":{"avg_score":0.9307822378626203,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.2526719937132465,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000152878522812,"is_incomplete":false,"total_prompt_cost_bin":0.17456549999999996,"total_cache_read_cost_bin":0.19104551249999974,"total_completion_cost_bin":0.06375,"total_reasoning_cost_bin":0.476311875,"total_cost_bin":0.9056728874999997,"total_on_demand_input_cost_bin":2.085020625,"ci_99_lower":0.8495303983661511,"ci_99_upper":0.9886197410008926,"ci_98_lower":0.850558114678544,"ci_98_upper":0.9882970330962801,"ci_95_lower":0.8616139192427236,"ci_95_upper":0.9769663821848329,"ci_90_lower":0.8735674920122084,"ci_90_upper":0.9766340393737397,"ci_75_lower":0.8962073873824385,"ci_75_upper":0.9652066039044739,"ci_50_lower":0.9082576353185097,"ci_50_upper":0.9536061580676366},"131072":{"avg_score":0.8856983632527436,"min_score":0.007238883059471846,"max_score":1,"n_tests":103,"stddev":0.3146247937495473,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":0.294877875,"total_cache_read_cost_bin":0.4760089499999998,"total_completion_cost_bin":0.08127937500000001,"total_reasoning_cost_bin":0.63705,"total_cost_bin":1.4892162,"total_on_demand_input_cost_bin":5.054967375,"ci_99_lower":0.7922879524787354,"ci_99_upper":0.9526010050357872,"ci_98_lower":0.8015774000116435,"ci_98_upper":0.9512919290448784,"ci_95_lower":0.8192352066258448,"ci_95_upper":0.9419679422388124,"ci_90_lower":0.8288580325599398,"ci_90_upper":0.9330696463243565,"ci_75_lower":0.8480032483410787,"ci_75_upper":0.922587490494779,"ci_50_lower":0.8659777715553538,"ci_50_upper":0.9049763585773106},"262144":{"avg_score":0.8903964994911183,"min_score":0.019755037501454353,"max_score":1,"n_tests":141,"stddev":0.30746023707954634,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000038179365245,"is_incomplete":false,"total_prompt_cost_bin":0.5942969999999992,"total_cache_read_cost_bin":1.3254859875000002,"total_completion_cost_bin":0.09966750000000002,"total_reasoning_cost_bin":0.8793731250000005,"total_cost_bin":2.8988236125,"total_on_demand_input_cost_bin":13.849156875000002,"ci_99_lower":0.8222556055948164,"ci_99_upper":0.9453398022648767,"ci_98_lower":0.8283506782409729,"ci_98_upper":0.9452278328269509,"ci_95_lower":0.8356866843579006,"ci_95_upper":0.9383005274023781,"ci_90_lower":0.8428589726018831,"ci_90_upper":0.9314995534262283,"ci_75_lower":0.8566006934996826,"ci_75_upper":0.9181254589220126,"ci_50_lower":0.8700434733512449,"ci_50_upper":0.9106559341324243},"524288":{"avg_score":0.7772110688857244,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.40538946108198026,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000019078540325,"is_incomplete":false,"total_prompt_cost_bin":3.669304875000002,"total_cache_read_cost_bin":4.271798024999979,"total_completion_cost_bin":0.16374000000000008,"total_reasoning_cost_bin":2.1425474999999987,"total_cost_bin":10.247390399999981,"total_on_demand_input_cost_bin":46.387285125000055,"ci_99_lower":0.7054141097803308,"ci_99_upper":0.8420332386713846,"ci_98_lower":0.7110859131741524,"ci_98_upper":0.8367545950686918,"ci_95_lower":0.7202350851228032,"ci_95_upper":0.8270507378374685,"ci_90_lower":0.73010057204549,"ci_90_upper":0.8191327525993367,"ci_75_lower":0.743936550360759,"ci_75_upper":0.808828212022367,"ci_50_lower":0.7574481188499179,"ci_50_upper":0.7959881348040161},"1048576":{"avg_score":0.6352034578650879,"min_score":0,"max_score":1,"n_tests":300,"stddev":0.47028555151534607,"median_score":1,"q1_score":0.031754132360219955,"q3_score":1,"avg_token_efficiency":1.0000009538490304,"is_incomplete":false,"total_prompt_cost_bin":0.8345733750000011,"total_cache_read_cost_bin":11.710873200000048,"total_completion_cost_bin":0.22748625000000003,"total_reasoning_cost_bin":5.146910625,"total_cost_bin":17.91984345000005,"total_on_demand_input_cost_bin":117.94330537500038,"ci_99_lower":0.5626363251862815,"ci_99_upper":0.7006333228979182,"ci_98_lower":0.5685006022449505,"ci_98_upper":0.6928962359470703,"ci_95_lower":0.5784268233758825,"ci_95_upper":0.6842054757564329,"ci_90_lower":0.5862725108396262,"ci_90_upper":0.6795566265825181,"ci_75_lower":0.6018224546139632,"ci_75_upper":0.6650568828916099,"ci_50_lower":0.6151857413859882,"ci_50_upper":0.6515377106983214}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Google: Gemini 3.7 Flash","provider_name":"Google","run_provider_names":["Google Vertex AI"],"pricing_metadata":{"prompt":3.75e-7,"completion":0.000001875,"reasoning":0.000001875,"has_tiered_pricing":true},"api_created_at":"2026-08-13T17:03:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T01:47:53.883Z"},{"model_slug":"google/gemini-3.7-flash","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.7452715208466707,"auc_128k":0.8946314508550318,"cum_avg_1m":0.8545018349521392,"cum_avg_128k":0.9339023122987336,"avg_score_overall":0.7955668569756247,"avg_response_time_ms":8405.648930512658,"total_prompt_cost":3.972865125000002,"total_cache_read_cost":18.341193225000026,"total_completion_cost":0.8343618750000003,"total_reasoning_cost":4.0390575,"total_prompt_cost_128k":1.408922625,"total_completion_cost_128k":0.34967062500000007,"total_on_demand_input_cost":187.38479737500046,"total_on_demand_input_cost_128k":9.205050000000007,"total_runs":1161,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000229398697194},"bin_metrics":{"8192":{"avg_score":1,"min_score":1,"max_score":1,"n_tests":80,"stddev":0,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747245,"is_incomplete":false,"total_prompt_cost_bin":0.19779937500000006,"total_cache_read_cost_bin":0.00015296249999999998,"total_completion_cost_bin":0.052453124999999996,"total_reasoning_cost_bin":0.13678875000000001,"total_cost_bin":0.3871942125000001,"total_on_demand_input_cost_bin":0.19932900000000006,"ci_99_lower":1,"ci_99_upper":1,"ci_98_lower":1,"ci_98_upper":1,"ci_95_lower":1,"ci_95_upper":1,"ci_90_lower":1,"ci_90_upper":1,"ci_75_lower":1,"ci_75_upper":1,"ci_50_lower":1,"ci_50_upper":1},"16384":{"avg_score":1,"min_score":1,"max_score":1,"n_tests":80,"stddev":0,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000074513502447,"is_incomplete":false,"total_prompt_cost_bin":0.37181437499999986,"total_cache_read_cost_bin":0.0041387624999999996,"total_completion_cost_bin":0.05208562499999999,"total_reasoning_cost_bin":0.15624937500000008,"total_cost_bin":0.5842881375,"total_on_demand_input_cost_bin":0.41320199999999974,"ci_99_lower":1,"ci_99_upper":1,"ci_98_lower":1,"ci_98_upper":1,"ci_95_lower":1,"ci_95_upper":1,"ci_90_lower":1,"ci_90_upper":1,"ci_75_lower":1,"ci_75_upper":1,"ci_50_lower":1,"ci_50_upper":1},"32768":{"avg_score":0.9855039166286588,"min_score":0.013557620346546173,"max_score":1,"n_tests":136,"stddev":0.1190942416441995,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000359090949908,"is_incomplete":false,"total_prompt_cost_bin":0.2745825000000001,"total_cache_read_cost_bin":0.11779485000000015,"total_completion_cost_bin":0.09966375,"total_reasoning_cost_bin":0.3385406250000001,"total_cost_bin":0.8305817250000004,"total_on_demand_input_cost_bin":1.4525309999999994,"ci_99_lower":0.9492793415180024,"ci_99_upper":1,"ci_98_lower":0.9565013276741785,"ci_98_upper":1,"ci_95_lower":0.9637543199104531,"ci_95_upper":1,"ci_90_lower":0.9637749037787537,"ci_90_upper":1,"ci_75_lower":0.9710182554691154,"ci_75_upper":1,"ci_50_lower":0.9782610860488871,"ci_50_upper":0.9927571694202283},"65536":{"avg_score":0.8634647167397335,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.33907722301960935,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000015287852282,"is_incomplete":false,"total_prompt_cost_bin":0.17464049999999992,"total_cache_read_cost_bin":0.19103801249999972,"total_completion_cost_bin":0.06381375000000002,"total_reasoning_cost_bin":0.24178499999999997,"total_cost_bin":0.6712772624999996,"total_on_demand_input_cost_bin":2.085020624999999,"ci_99_lower":0.7493942739056313,"ci_99_upper":0.9440930485623779,"ci_98_lower":0.7688135866415412,"ci_98_upper":0.943713150667191,"ci_95_lower":0.7823228294348947,"ci_95_upper":0.9311327743074199,"ci_90_lower":0.7951222781897249,"ci_90_upper":0.9206186397406547,"ci_75_lower":0.8190153190996224,"ci_75_upper":0.9072298454969903,"ci_50_lower":0.8393088741679949,"ci_50_upper":0.8868506713249885},"131072":{"avg_score":0.8205429281252753,"min_score":0.007622677367180586,"max_score":1,"n_tests":103,"stddev":0.37920762996697255,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000076410572556,"is_incomplete":false,"total_prompt_cost_bin":0.390085875,"total_cache_read_cost_bin":0.4664881499999998,"total_completion_cost_bin":0.08165437500000006,"total_reasoning_cost_bin":0.354271875,"total_cost_bin":1.2925002749999999,"total_on_demand_input_cost_bin":5.054967375000008,"ci_99_lower":0.7250503810398841,"ci_99_upper":0.9058615031257923,"ci_98_lower":0.7271690638825405,"ci_98_upper":0.9050806336736903,"ci_95_lower":0.7443798984790863,"ci_95_upper":0.8875209893784891,"ci_90_lower":0.754304081553964,"ci_90_upper":0.8774489378793962,"ci_75_lower":0.7733856577553914,"ci_75_upper":0.8589488526244503,"ci_50_lower":0.7925064474721755,"ci_50_upper":0.8485591144185712},"262144":{"avg_score":0.8213952719927469,"min_score":0.01864100992679596,"max_score":1,"n_tests":141,"stddev":0.3768966172443341,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000038179365236,"is_incomplete":false,"total_prompt_cost_bin":0.40086412500000085,"total_cache_read_cost_bin":1.3448292750000002,"total_completion_cost_bin":0.09990937500000002,"total_reasoning_cost_bin":0.42409500000000006,"total_cost_bin":2.269697775000001,"total_on_demand_input_cost_bin":13.849156875,"ci_99_lower":0.7392362374185538,"ci_99_upper":0.8969833564155913,"ci_98_lower":0.7457319116344054,"ci_98_upper":0.8907088950469243,"ci_95_lower":0.752831162968056,"ci_95_upper":0.8772978151719092,"ci_90_lower":0.7661901480194631,"ci_90_upper":0.8759064815786555,"ci_75_lower":0.7804520936692727,"ci_75_upper":0.8563264936657874,"ci_50_lower":0.7983421775056644,"ci_50_upper":0.8425027801100402},"524288":{"avg_score":0.7442956472307017,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.42509168997189756,"median_score":1,"q1_score":0.12373737432062626,"q3_score":1,"avg_token_efficiency":1.0000019078540325,"is_incomplete":false,"total_prompt_cost_bin":1.328505,"total_cache_read_cost_bin":4.505878012499976,"total_completion_cost_bin":0.16318687500000006,"total_reasoning_cost_bin":0.8379018750000002,"total_cost_bin":6.835471762499976,"total_on_demand_input_cost_bin":46.387285125000055,"ci_99_lower":0.6745943930109971,"ci_99_upper":0.8114924823848672,"ci_98_lower":0.6770201286856653,"ci_98_upper":0.8039890561785976,"ci_95_lower":0.6869369440884685,"ci_95_upper":0.7964086524643974,"ci_90_lower":0.6942472132873895,"ci_90_upper":0.7887411413314729,"ci_75_lower":0.7086401139143055,"ci_75_upper":0.7772185399364266,"ci_50_lower":0.7239168881721325,"ci_50_upper":0.7633037650072256},"1048576":{"avg_score":0.6008121988999968,"min_score":0.004759071860462427,"max_score":1,"n_tests":300,"stddev":0.476535029326322,"median_score":1,"q1_score":0.030274545308202505,"q3_score":1,"avg_token_efficiency":1.0000009538490304,"is_incomplete":false,"total_prompt_cost_bin":0.8345733750000012,"total_cache_read_cost_bin":11.710873200000048,"total_completion_cost_bin":0.2215950000000001,"total_reasoning_cost_bin":1.5494249999999998,"total_cost_bin":14.31646657500005,"total_on_demand_input_cost_bin":117.94330537500039,"ci_99_lower":0.5227535850908142,"ci_99_upper":0.6712523202359366,"ci_98_lower":0.5305170409482283,"ci_98_upper":0.6585174525133799,"ci_95_lower":0.5462583502552784,"ci_95_upper":0.6503226463466805,"ci_90_lower":0.5559002804333966,"ci_90_upper":0.6432399680664141,"ci_75_lower":0.5671306090144208,"ci_75_upper":0.6306482091140545,"ci_50_lower":0.5810976590383022,"ci_50_upper":0.6180768696517529}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Google: Gemini 3.7 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":3.75e-7,"completion":0.000001875,"reasoning":0.000001875,"has_tiered_pricing":true},"api_created_at":"2026-08-13T17:03:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T02:31:14.486Z"},{"model_slug":"google/gemini-3.7-flash","reasoning_mode":"low","overall_metrics":{"auc_1m":0.6851122538164471,"auc_128k":0.8507018635437091,"cum_avg_1m":0.8076303293256791,"cum_avg_128k":0.8926626499791602,"avg_score_overall":0.744826976096572,"avg_response_time_ms":4627.550259743755,"total_prompt_cost":6.387155625000009,"total_cache_read_cost":18.099764175000026,"total_completion_cost":0.8347575,"total_reasoning_cost":1.9335806249999994,"total_prompt_cost_128k":1.382640375,"total_completion_cost_128k":0.34853625,"total_on_demand_input_cost":187.38479737500035,"total_on_demand_input_cost_128k":9.205049999999996,"total_runs":1161,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000229398697196},"bin_metrics":{"8192":{"avg_score":0.9756173897301779,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.15323977543497122,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747239,"is_incomplete":false,"total_prompt_cost_bin":0.1932168750000001,"total_cache_read_cost_bin":0.0006112124999999999,"total_completion_cost_bin":0.05208937499999999,"total_reasoning_cost_bin":0.06858187500000004,"total_cost_bin":0.31449933750000014,"total_on_demand_input_cost_bin":0.19932900000000006,"ci_99_lower":0.9265795771157136,"ci_99_upper":1,"ci_98_lower":0.9271179803679697,"ci_98_upper":1,"ci_95_lower":0.938907856377773,"ci_95_upper":1,"ci_90_lower":0.9394503281684592,"ci_90_upper":1,"ci_75_lower":0.9515060153556988,"ci_75_upper":1,"ci_50_lower":0.9635617025429383,"ci_50_upper":0.9879443128127605},"16384":{"avg_score":0.9274447010364384,"min_score":0.02010050229728222,"max_score":1,"n_tests":80,"stddev":0.25642882549260954,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000074513502447,"is_incomplete":false,"total_prompt_cost_bin":0.3948074999999998,"total_cache_read_cost_bin":0.0018394499999999999,"total_completion_cost_bin":0.052141875,"total_reasoning_cost_bin":0.07741125,"total_cost_bin":0.5262000749999999,"total_on_demand_input_cost_bin":0.4132019999999997,"ci_99_lower":0.8425352164854994,"ci_99_upper":0.9880242464132607,"ci_98_lower":0.8538361323971767,"ci_98_upper":0.9877899620449171,"ci_95_lower":0.8664060743723531,"ci_95_upper":0.9760260966676287,"ci_90_lower":0.8777363771304955,"ci_90_upper":0.975541218323633,"ci_75_lower":0.8911520898138406,"ci_75_upper":0.9635463195038028,"ci_50_lower":0.9036511157348286,"ci_50_upper":0.9513206548523159},"32768":{"avg_score":0.9716322808526456,"min_score":0.022179193794727325,"max_score":1,"n_tests":136,"stddev":0.1635718325857318,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000035909094991,"is_incomplete":false,"total_prompt_cost_bin":0.28058550000000015,"total_cache_read_cost_bin":0.11719455000000015,"total_completion_cost_bin":0.09943687499999998,"total_reasoning_cost_bin":0.21723937499999985,"total_cost_bin":0.7144563000000002,"total_on_demand_input_cost_bin":1.4525309999999982,"ci_99_lower":0.9287805917691988,"ci_99_upper":1,"ci_98_lower":0.9292534211445052,"ci_98_upper":1,"ci_95_lower":0.9426944886464351,"ci_95_upper":0.9930183153106447,"ci_90_lower":0.9435749845784706,"ci_90_upper":0.9928903546841705,"ci_75_lower":0.9572279310840017,"ci_75_upper":0.9858284564414883,"ci_50_lower":0.9644409049959743,"ci_50_upper":0.9788467717521331},"65536":{"avg_score":0.8060899255358998,"min_score":0.01416765060275793,"max_score":1,"n_tests":85,"stddev":0.39011764036391205,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000152878522817,"is_incomplete":false,"total_prompt_cost_bin":0.26667899999999994,"total_cache_read_cost_bin":0.18183416249999979,"total_completion_cost_bin":0.064273125,"total_reasoning_cost_bin":0.13069687499999996,"total_cost_bin":0.6434831624999997,"total_on_demand_input_cost_bin":2.085020625000001,"ci_99_lower":0.6908825906746857,"ci_99_upper":0.9085999427247136,"ci_98_lower":0.6921271943731562,"ci_98_upper":0.8976670360531219,"ci_95_lower":0.714116110216881,"ci_95_upper":0.8858238326971802,"ci_90_lower":0.7269055910890593,"ci_90_upper":0.8745396722918924,"ci_75_lower":0.7496819651000859,"ci_75_upper":0.8518125974217101,"ci_50_lower":0.782406523881261,"ci_50_upper":0.8395813298137749},"131072":{"avg_score":0.7825289527406392,"min_score":0.008612439967691898,"max_score":1,"n_tests":103,"stddev":0.4062951399942365,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":0.24735150000000003,"total_cache_read_cost_bin":0.4807615874999998,"total_completion_cost_bin":0.08059500000000004,"total_reasoning_cost_bin":0.19230374999999997,"total_cost_bin":1.0010118375,"total_on_demand_input_cost_bin":5.054967374999998,"ci_99_lower":0.6690174103492094,"ci_99_upper":0.8769845811229253,"ci_98_lower":0.6797654328543187,"ci_98_upper":0.8683441335286548,"ci_95_lower":0.6969561448433771,"ci_95_upper":0.8581313407206818,"ci_90_lower":0.7079097627499367,"ci_90_upper":0.8481358420501636,"ci_75_lower":0.7353116995315927,"ci_75_upper":0.8292132845505602,"ci_50_lower":0.7545663846218145,"ci_50_upper":0.8108915347782262},"262144":{"avg_score":0.794948450415164,"min_score":0.011757425963878632,"max_score":1,"n_tests":141,"stddev":0.3958866035717792,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000003817936524,"is_incomplete":false,"total_prompt_cost_bin":0.4975410000000011,"total_cache_read_cost_bin":1.3351615875000007,"total_completion_cost_bin":0.10031625000000002,"total_reasoning_cost_bin":0.22992375000000004,"total_cost_bin":2.162942587500002,"total_on_demand_input_cost_bin":13.849156875000002,"ci_99_lower":0.6992970477427679,"ci_99_upper":0.8775489259175692,"ci_98_lower":0.7063608637253973,"ci_98_upper":0.8702737983000458,"ci_95_lower":0.7260193835181046,"ci_95_upper":0.8564452082737751,"ci_90_lower":0.7344072816358443,"ci_90_upper":0.8497632372721449,"ci_75_lower":0.753833349064283,"ci_75_upper":0.8359659132993522,"ci_50_lower":0.7678756542431178,"ci_50_upper":0.8218148580673051},"524288":{"avg_score":0.6512964970138224,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4621266265064139,"median_score":1,"q1_score":0.04358196072280407,"q3_score":1,"avg_token_efficiency":1.0000019078540323,"is_incomplete":false,"total_prompt_cost_bin":2.8890648750000043,"total_cache_read_cost_bin":4.349822024999979,"total_completion_cost_bin":0.16253625000000005,"total_reasoning_cost_bin":0.3248775,"total_cost_bin":7.726300649999984,"total_on_demand_input_cost_bin":46.38728512500001,"ci_99_lower":0.5694839003234677,"ci_99_upper":0.7312608227871558,"ci_98_lower":0.5740097031298278,"ci_98_upper":0.7230848953615818,"ci_95_lower":0.5880281631065774,"ci_95_upper":0.7086352424545389,"ci_90_lower":0.6003802549456526,"ci_90_upper":0.7004335620785349,"ci_75_lower":0.6136597254417365,"ci_75_upper":0.6847000734540692,"ci_50_lower":0.6302939635431157,"ci_50_upper":0.6706517079176176},"1048576":{"avg_score":0.551484437280645,"min_score":0.0019421246834099293,"max_score":1,"n_tests":300,"stddev":0.48171667053411177,"median_score":1,"q1_score":0.02965727960690856,"q3_score":1,"avg_token_efficiency":1.0000009538490304,"is_incomplete":false,"total_prompt_cost_bin":1.617909375000003,"total_cache_read_cost_bin":11.632539600000047,"total_completion_cost_bin":0.22336875,"total_reasoning_cost_bin":0.6925462499999996,"total_cost_bin":14.166363975000051,"total_on_demand_input_cost_bin":117.94330537500032,"ci_99_lower":0.4732415159069545,"ci_99_upper":0.6169358541159735,"ci_98_lower":0.4842820897741088,"ci_98_upper":0.6115502549560741,"ci_95_lower":0.49831102717784237,"ci_95_upper":0.6031987402247032,"ci_90_lower":0.5062835030478114,"ci_90_upper":0.5941412909515202,"ci_75_lower":0.5196847891435027,"ci_75_upper":0.5818159628643965,"ci_50_lower":0.5321645059700434,"ci_50_upper":0.5709247339260763}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Google: Gemini 3.7 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":3.75e-7,"completion":0.000001875,"reasoning":0.000001875,"has_tiered_pricing":true},"api_created_at":"2026-08-13T17:03:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T01:03:09.397Z"},{"model_slug":"google/gemini-3.6-flash","reasoning_mode":"high","overall_metrics":{"auc_1m":0.6654022346864934,"auc_128k":0.8418586373666831,"cum_avg_1m":0.799374127043638,"cum_avg_128k":0.8925680908589049,"avg_score_overall":0.733362838301139,"avg_response_time_ms":15053.618327657585,"total_prompt_cost":27.30678599999998,"total_cache_read_cost":73.63855934999981,"total_completion_cost":3.3671024999999997,"total_reasoning_cost":31.37492999999999,"total_prompt_cost_128k":7.760926500000004,"total_completion_cost_128k":1.3987574999999999,"total_on_demand_input_cost":763.6923795000018,"total_on_demand_input_cost_128k":36.8202,"total_runs":1170,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227707464873},"bin_metrics":{"8192":{"avg_score":0.987702578259632,"min_score":0.01620626077055931,"max_score":1,"n_tests":80,"stddev":0.10999148383778654,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747248,"is_incomplete":false,"total_prompt_cost_bin":0.7973160000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.20987999999999998,"total_reasoning_cost_bin":1.2603300000000002,"total_cost_bin":2.267526,"total_on_demand_input_cost_bin":0.7973160000000001,"ci_99_lower":0.9508103130385279,"ci_99_upper":1,"ci_98_lower":0.9508103130385279,"ci_98_upper":1,"ci_95_lower":0.963107734778896,"ci_95_upper":1,"ci_90_lower":0.963107734778896,"ci_90_upper":1,"ci_75_lower":0.975405156519264,"ci_75_upper":1,"ci_50_lower":0.975405156519264,"ci_50_upper":1},"16384":{"avg_score":0.9877374999923632,"min_score":0.01899999938905239,"max_score":1,"n_tests":80,"stddev":0.1096791343646707,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024476,"is_incomplete":false,"total_prompt_cost_bin":0.8927444999999998,"total_cache_read_cost_bin":0.0760063499999999,"total_completion_cost_bin":0.20844000000000001,"total_reasoning_cost_bin":1.3464375,"total_cost_bin":2.5236283499999996,"total_on_demand_input_cost_bin":1.652808,"ci_99_lower":0.9509499999694526,"ci_99_upper":1,"ci_98_lower":0.9509499999694526,"ci_98_upper":1,"ci_95_lower":0.9632124999770895,"ci_95_upper":1,"ci_90_lower":0.9632124999770895,"ci_90_upper":1,"ci_75_lower":0.9754749999847263,"ci_75_upper":1,"ci_50_lower":0.9754749999847263,"ci_50_upper":1},"32768":{"avg_score":0.9083901389762211,"min_score":0.014925372786819935,"max_score":1,"n_tests":136,"stddev":0.2830172164791026,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000359090949913,"is_incomplete":false,"total_prompt_cost_bin":1.9683405000000005,"total_cache_read_cost_bin":0.38417835000000133,"total_completion_cost_bin":0.39899249999999997,"total_reasoning_cost_bin":3.282314999999998,"total_cost_bin":6.03382635,"total_on_demand_input_cost_bin":5.810123999999993,"ci_99_lower":0.8335416767754085,"ci_99_upper":0.9649026513962514,"ci_98_lower":0.8510979286632638,"ci_98_upper":0.9585002028763375,"ci_95_lower":0.858052190302474,"ci_95_upper":0.9524804562826722,"ci_90_lower":0.8650142659210628,"ci_90_upper":0.9447409261586419,"ci_75_lower":0.8797476540032008,"ci_75_upper":0.936574868271504,"ci_50_lower":0.8929470823033411,"ci_50_upper":0.9236543321886155},"65536":{"avg_score":0.8056053280939951,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.3911382352558513,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000152878522814,"is_incomplete":false,"total_prompt_cost_bin":1.204365,"total_cache_read_cost_bin":0.71357175,"total_completion_cost_bin":0.2568600000000001,"total_reasoning_cost_bin":2.1675975000000003,"total_cost_bin":4.34239425,"total_on_demand_input_cost_bin":8.3400825,"ci_99_lower":0.6897655897835161,"ci_99_upper":0.9083141160235905,"ci_98_lower":0.6913707210301039,"ci_98_upper":0.8972962928460363,"ci_95_lower":0.7132964360294864,"ci_95_upper":0.8855967778507072,"ci_90_lower":0.7262868602836834,"ci_90_upper":0.874222679150236,"ci_75_lower":0.7491202863083933,"ci_75_upper":0.851474226583891,"ci_50_lower":0.7817705752273254,"ci_50_upper":0.839085135586998},"131072":{"avg_score":0.7734049089723131,"min_score":0.00738688837736845,"max_score":1,"n_tests":103,"stddev":0.41314352756052997,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000076410572554,"is_incomplete":false,"total_prompt_cost_bin":2.8981605000000035,"total_cache_read_cost_bin":1.7321709000000014,"total_completion_cost_bin":0.3245849999999999,"total_reasoning_cost_bin":3.0999525000000006,"total_cost_bin":8.054868900000006,"total_on_demand_input_cost_bin":20.219869500000005,"ci_99_lower":0.6599132564872756,"ci_99_upper":0.8680781538923298,"ci_98_lower":0.6691574796583973,"ci_98_upper":0.8675785950746713,"ci_95_lower":0.6878427560077257,"ci_95_upper":0.8491198639127706,"ci_90_lower":0.705895956642745,"ci_90_upper":0.8394563594064448,"ci_75_lower":0.7261940735297331,"ci_75_upper":0.8204622175612315,"ci_50_lower":0.7451625368422241,"ci_50_upper":0.8017385007148988},"262144":{"avg_score":0.7388424983211787,"min_score":0.0019772613886743784,"max_score":1,"n_tests":141,"stddev":0.43156361376942715,"median_score":1,"q1_score":0.047493401914834976,"q3_score":1,"avg_token_efficiency":1.000003817936524,"is_incomplete":false,"total_prompt_cost_bin":2.835187500000002,"total_cache_read_cost_bin":5.256144000000008,"total_completion_cost_bin":0.40414500000000003,"total_reasoning_cost_bin":4.18566,"total_cost_bin":12.68113650000001,"total_on_demand_input_cost_bin":55.39662749999998,"ci_99_lower":0.6429944608502127,"ci_99_upper":0.8287594822828299,"ci_98_lower":0.6493921515046527,"ci_98_upper":0.8216069090318312,"ci_95_lower":0.6624873525220306,"ci_95_upper":0.8089095219125118,"ci_90_lower":0.67652450666535,"ci_90_upper":0.8002392418130221,"ci_75_lower":0.6970483776995604,"ci_75_upper":0.7801745094293695,"ci_50_lower":0.711667473924987,"ci_50_upper":0.765657901417147},"524288":{"avg_score":0.6108014358580507,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.471029143361449,"median_score":1,"q1_score":0.035158177837729454,"q3_score":1,"avg_token_efficiency":1.0000019078540328,"is_incomplete":false,"total_prompt_cost_bin":5.184772499999995,"total_cache_read_cost_bin":18.036436799999926,"total_completion_cost_bin":0.64407,"total_reasoning_cost_bin":5.984504999999999,"total_cost_bin":29.849784299999918,"total_on_demand_input_cost_bin":185.54914050000014,"ci_99_lower":0.5304435511618375,"ci_99_upper":0.6882655609122397,"ci_98_lower":0.5367551729183692,"ci_98_upper":0.6813910076654738,"ci_95_lower":0.548359948365803,"ci_95_upper":0.6708786158326233,"ci_90_lower":0.5564966151735476,"ci_90_upper":0.6621109451005487,"ci_75_lower":0.5745098201981726,"ci_75_upper":0.6452574961188828,"ci_50_lower":0.5900564739807169,"ci_50_upper":0.629818411006531},"1048576":{"avg_score":0.5825086278753494,"min_score":0.005855290684849024,"max_score":1,"n_tests":309,"stddev":0.47699023955716013,"median_score":1,"q1_score":0.03148780018091202,"q3_score":1,"avg_token_efficiency":1.0000009538490442,"is_incomplete":false,"total_prompt_cost_bin":11.52589949999998,"total_cache_read_cost_bin":47.44005119999987,"total_completion_cost_bin":0.9201299999999997,"total_reasoning_cost_bin":10.04813249999999,"total_cost_bin":69.93421319999985,"total_on_demand_input_cost_bin":485.9264115000017,"ci_99_lower":0.5133273504570451,"ci_99_upper":0.6482054756689533,"ci_98_lower":0.5223868192586908,"ci_98_upper":0.6411729784853738,"ci_95_lower":0.5312055910020858,"ci_95_upper":0.6348018821372957,"ci_90_lower":0.5380879712202149,"ci_90_upper":0.6262851411434535,"ci_75_lower":0.5504405575073144,"ci_75_upper":0.6133806712599169,"ci_50_lower":0.561953326490554,"ci_50_upper":0.599940074144329}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Google: Gemini 3.6 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.00000375,"reasoning":0.00000375,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:13.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-22T01:25:38.954Z"},{"model_slug":"google/gemini-3.6-flash","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.6150193500655474,"auc_128k":0.8174602413184965,"cum_avg_1m":0.7648989465942516,"cum_avg_128k":0.8688153620652113,"avg_score_overall":0.6990876003391379,"avg_response_time_ms":11666.183894898611,"total_prompt_cost":23.460964499999996,"total_cache_read_cost":73.5513676499998,"total_completion_cost":3.3703949999999994,"total_reasoning_cost":20.3759325,"total_prompt_cost_128k":8.6328435,"total_completion_cost_128k":1.3981500000000002,"total_on_demand_input_cost":758.9746410000018,"total_on_demand_input_cost_128k":36.82019999999998,"total_runs":1167,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000228268310598},"bin_metrics":{"8192":{"avg_score":0.9389504715334624,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.23794304108439773,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000155340474724,"is_incomplete":false,"total_prompt_cost_bin":0.7973160000000006,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.20928000000000002,"total_reasoning_cost_bin":0.9441075000000001,"total_cost_bin":1.9507035000000006,"total_on_demand_input_cost_bin":0.7973160000000006,"ci_99_lower":0.8538966656804551,"ci_99_upper":0.9880043294655625,"ci_98_lower":0.8656166019497905,"ci_98_upper":0.9879440497141332,"ci_95_lower":0.8775725160061847,"ci_95_upper":0.9877759197261184,"ci_90_lower":0.8899647669168189,"ci_90_upper":0.975777383055538,"ci_75_lower":0.9025037743151187,"ci_75_upper":0.9636711958562956,"ci_50_lower":0.9264033873274458,"ci_50_upper":0.9515562409767881},"16384":{"avg_score":0.9879087639041245,"min_score":0.03270111232995987,"max_score":1,"n_tests":80,"stddev":0.10814730336950713,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024474,"is_incomplete":false,"total_prompt_cost_bin":0.9785925000000004,"total_cache_read_cost_bin":0.06742154999999996,"total_completion_cost_bin":0.20843999999999996,"total_reasoning_cost_bin":1.0295625,"total_cost_bin":2.2840165500000005,"total_on_demand_input_cost_bin":1.6528079999999983,"ci_99_lower":0.951635055616498,"ci_99_upper":1,"ci_98_lower":0.951635055616498,"ci_98_upper":1,"ci_95_lower":0.9637262917123735,"ci_95_upper":1,"ci_90_lower":0.9637262917123735,"ci_90_upper":1,"ci_75_lower":0.975817527808249,"ci_75_upper":1,"ci_50_lower":0.975817527808249,"ci_50_upper":1},"32768":{"avg_score":0.9007134401283966,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.29433595219306513,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000359090949906,"is_incomplete":false,"total_prompt_cost_bin":2.054301000000001,"total_cache_read_cost_bin":0.37558230000000126,"total_completion_cost_bin":0.39938999999999997,"total_reasoning_cost_bin":2.2096725000000013,"total_cost_bin":5.038945800000004,"total_on_demand_input_cost_bin":5.810123999999995,"ci_99_lower":0.8279051322793854,"ci_99_upper":0.9578463720493378,"ci_98_lower":0.8362499866920853,"ci_98_upper":0.9568600521202931,"ci_95_lower":0.8504384682304911,"ci_95_upper":0.9451267271125449,"ci_90_lower":0.8571857973975163,"ci_90_upper":0.9376531575645511,"ci_75_lower":0.8709746713357162,"ci_75_upper":0.9290723273400492,"ci_50_lower":0.8808138906816192,"ci_50_upper":0.9209152721390744},"65536":{"avg_score":0.7712041893645244,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.4149499045850191,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000152878522828,"is_incomplete":false,"total_prompt_cost_bin":1.720199999999998,"total_cache_read_cost_bin":0.66198825,"total_completion_cost_bin":0.25656749999999995,"total_reasoning_cost_bin":1.5228899999999999,"total_cost_bin":4.161645749999998,"total_on_demand_input_cost_bin":8.340082499999996,"ci_99_lower":0.644562311750234,"ci_99_upper":0.8746171885605024,"ci_98_lower":0.6462694059904008,"ci_98_upper":0.8636045648834724,"ci_95_lower":0.6783124222803643,"ci_95_upper":0.8515619024020784,"ci_90_lower":0.6914291055851124,"ci_90_upper":0.8401480761455262,"ci_75_lower":0.7155256422483088,"ci_75_upper":0.8177223498522139,"ci_50_lower":0.7375596641891582,"ci_50_upper":0.8050914356430225},"131072":{"avg_score":0.7452999453955483,"min_score":0.007240755017846823,"max_score":1,"n_tests":103,"stddev":0.4282408593363364,"median_score":1,"q1_score":0.06620718352496624,"q3_score":1,"avg_token_efficiency":1.0000076410572563,"is_incomplete":false,"total_prompt_cost_bin":3.082433999999999,"total_cache_read_cost_bin":1.7137435500000013,"total_completion_cost_bin":0.32447250000000005,"total_reasoning_cost_bin":2.2233825000000005,"total_cost_bin":7.3440325500000005,"total_on_demand_input_cost_bin":20.219869499999987,"ci_99_lower":0.6312855318599399,"ci_99_upper":0.849804145986377,"ci_98_lower":0.6335538741171776,"ci_98_upper":0.8400270470994009,"ci_95_lower":0.6526691420740692,"ci_95_upper":0.8217775016869373,"ci_90_lower":0.6714514445264832,"ci_90_upper":0.8122274676873575,"ci_75_lower":0.6960162701677458,"ci_75_upper":0.7924421536860518,"ci_50_lower":0.7160928089677998,"ci_50_upper":0.7735075015581475},"262144":{"avg_score":0.6642058582824543,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.4619962634416087,"median_score":1,"q1_score":0.03718047961592674,"q3_score":1,"avg_token_efficiency":1.0000038179365236,"is_incomplete":false,"total_prompt_cost_bin":2.8351875000000013,"total_cache_read_cost_bin":5.256144000000008,"total_completion_cost_bin":0.4013775000000002,"total_reasoning_cost_bin":2.1971925000000003,"total_cost_bin":10.68990150000001,"total_on_demand_input_cost_bin":55.39662749999999,"ci_99_lower":0.5609671266734639,"ci_99_upper":0.7675932672328533,"ci_98_lower":0.5743155363549195,"ci_98_upper":0.7528191560835755,"ci_95_lower":0.5815304903803122,"ci_95_upper":0.7388364109300665,"ci_90_lower":0.5974559004593588,"ci_90_upper":0.7271805638970861,"ci_75_lower":0.6166894734537407,"ci_75_upper":0.7110987021926302,"ci_50_lower":0.6357112899356595,"ci_50_upper":0.691566642202067},"524288":{"avg_score":0.5245705464967305,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.48117086024717837,"median_score":0.9988167583942413,"q1_score":0.02912269812077284,"q3_score":1,"avg_token_efficiency":1.0000019078540328,"is_incomplete":false,"total_prompt_cost_bin":5.184772499999996,"total_cache_read_cost_bin":18.036436799999926,"total_completion_cost_bin":0.6575549999999992,"total_reasoning_cost_bin":3.810945000000001,"total_cost_bin":27.68970929999992,"total_on_demand_input_cost_bin":185.54914050000022,"ci_99_lower":0.4428455133104827,"ci_99_upper":0.5952044317730992,"ci_98_lower":0.44860792094531815,"ci_98_upper":0.5925275482544162,"ci_95_lower":0.4630593441593731,"ci_95_upper":0.5843176669957633,"ci_90_lower":0.47271634756130665,"ci_90_upper":0.5765520393155846,"ci_75_lower":0.48756280971947025,"ci_75_upper":0.5607758474144571,"ci_50_lower":0.5039042183069781,"ci_50_upper":0.5467554757261048},"1048576":{"avg_score":0.5863383576487726,"min_score":0,"max_score":1,"n_tests":306,"stddev":0.47903436817684986,"median_score":1,"q1_score":0.03129092184826732,"q3_score":1,"avg_token_efficiency":1.0000009538490566,"is_incomplete":false,"total_prompt_cost_bin":6.808160999999998,"total_cache_read_cost_bin":47.44005119999987,"total_completion_cost_bin":0.9133125,"total_reasoning_cost_bin":6.438179999999998,"total_cost_bin":61.59970469999987,"total_on_demand_input_cost_bin":481.2086730000016,"ci_99_lower":0.5203501724101507,"ci_99_upper":0.6567012351425289,"ci_98_lower":0.5259842494404737,"ci_98_upper":0.6527488207669087,"ci_95_lower":0.5316232470931616,"ci_95_upper":0.6428470263600972,"ci_90_lower":0.5424420230079776,"ci_90_upper":0.6334001016520416,"ci_75_lower":0.55434867172381,"ci_75_upper":0.6172342089600111,"ci_50_lower":0.5663436388654339,"ci_50_upper":0.6026919225690768}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Google: Gemini 3.6 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.00000375,"reasoning":0.00000375,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:13.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-21T16:44:28.014Z"},{"model_slug":"google/gemini-3.6-flash","reasoning_mode":"low","overall_metrics":{"auc_1m":0.5841954249544019,"auc_128k":0.6631222306389157,"cum_avg_1m":0.6742774431948548,"cum_avg_128k":0.7284543365189947,"avg_score_overall":0.6303615469859642,"avg_response_time_ms":7391.805370311248,"total_prompt_cost":37.68965399999997,"total_cache_read_cost":72.60027254999983,"total_completion_cost":3.3586800000000006,"total_reasoning_cost":9.681315000000005,"total_prompt_cost_128k":7.6503225,"total_completion_cost_128k":1.3971675000000001,"total_on_demand_input_cost":763.6923795000016,"total_on_demand_input_cost_128k":36.82019999999999,"total_runs":1170,"total_candidates_generated":null,"avg_token_efficiency_overall":1.000022770746487},"bin_metrics":{"8192":{"avg_score":0.8049025399959646,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.39270580005806627,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747239,"is_incomplete":false,"total_prompt_cost_bin":0.7973160000000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.20855249999999997,"total_reasoning_cost_bin":0.3573524999999999,"total_cost_bin":1.3632210000000002,"total_on_demand_input_cost_bin":0.7973160000000004,"ci_99_lower":0.6826738475312596,"ci_99_upper":0.9151680253028753,"ci_98_lower":0.6939907321481733,"ci_98_upper":0.9027446041733492,"ci_95_lower":0.7077243354910753,"ci_95_upper":0.8900425111857475,"ci_90_lower":0.7312877752579516,"ci_90_upper":0.8777907145681093,"ci_75_lower":0.7554584061232162,"ci_75_upper":0.8535576480600866,"ci_50_lower":0.7796052889723797,"ci_50_upper":0.8301446244760882},"16384":{"avg_score":0.9392752268235199,"min_score":0.013550135307013988,"max_score":1,"n_tests":80,"stddev":0.23668172198803003,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024478,"is_incomplete":false,"total_prompt_cost_bin":0.9663330000000004,"total_cache_read_cost_bin":0.06864749999999996,"total_completion_cost_bin":0.2076975,"total_reasoning_cost_bin":0.46689750000000024,"total_cost_bin":1.7095755000000006,"total_on_demand_input_cost_bin":1.6528079999999983,"ci_99_lower":0.8546062304808293,"ci_99_upper":0.9880237175414804,"ci_98_lower":0.8663208768911427,"ci_98_upper":0.9879635352175683,"ci_95_lower":0.8782835654146038,"ci_95_upper":0.9879212419502437,"ci_90_lower":0.8905487021076259,"ci_90_upper":0.9759203648194671,"ci_75_lower":0.9030542200867785,"ci_75_upper":0.9638780326233245,"ci_50_lower":0.9267071539739845,"ci_50_upper":0.9518340240232647},"32768":{"avg_score":0.7024304821166237,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4469973188425273,"median_score":1,"q1_score":0.0575403431430459,"q3_score":1,"avg_token_efficiency":1.0000359090949917,"is_incomplete":false,"total_prompt_cost_bin":1.8701204999999994,"total_cache_read_cost_bin":0.3940003500000005,"total_completion_cost_bin":0.39738749999999995,"total_reasoning_cost_bin":0.9579675,"total_cost_bin":3.6194758499999997,"total_on_demand_input_cost_bin":5.810123999999995,"ci_99_lower":0.6022085580125927,"ci_99_upper":0.7945192651782503,"ci_98_lower":0.6074181376961539,"ci_98_upper":0.7873503862139049,"ci_95_lower":0.6240561746232903,"ci_95_upper":0.7745674562493257,"ci_90_lower":0.6386336713427547,"ci_90_upper":0.7654362864482477,"ci_75_lower":0.6582933123692568,"ci_75_upper":0.7449514889025629,"ci_50_lower":0.6747331795687548,"ci_50_upper":0.7252232299185433},"65536":{"avg_score":0.622762084182571,"min_score":0.002102312631905079,"max_score":1,"n_tests":85,"stddev":0.47642432303975596,"median_score":1,"q1_score":0.03230403736233711,"q3_score":1,"avg_token_efficiency":1.0000152878522803,"is_incomplete":false,"total_prompt_cost_bin":1.1183925,"total_cache_read_cost_bin":0.722169,"total_completion_cost_bin":0.2557875,"total_reasoning_cost_bin":0.6919725000000001,"total_cost_bin":2.7883214999999995,"total_on_demand_input_cost_bin":8.340082499999998,"ci_99_lower":0.4841297818233314,"ci_99_upper":0.7378151480169857,"ci_98_lower":0.5058336428608745,"ci_98_upper":0.7369595392136014,"ci_95_lower":0.5200077910269337,"ci_95_upper":0.7251192685813808,"ci_90_lower":0.5408768139746698,"ci_90_upper":0.7133321097745178,"ci_75_lower":0.5651844410170965,"ci_75_upper":0.6804912941475564,"ci_50_lower":0.588036926964517,"ci_50_upper":0.6568873658338015},"131072":{"avg_score":0.5729013494762945,"min_score":0.003846153849735856,"max_score":1,"n_tests":103,"stddev":0.48615697204610653,"median_score":1,"q1_score":0.026465950533747673,"q3_score":1,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":2.8981605,"total_cache_read_cost_bin":1.7321709000000014,"total_completion_cost_bin":0.32774250000000016,"total_reasoning_cost_bin":0.7629300000000002,"total_cost_bin":5.721003900000001,"total_on_demand_input_cost_bin":20.2198695,"ci_99_lower":0.44958459597348815,"ci_99_upper":0.6964999396080102,"ci_98_lower":0.4584374683407515,"ci_98_upper":0.677996423681678,"ci_95_lower":0.47842369384581035,"ci_95_upper":0.6673883869927557,"ci_90_lower":0.4970770976392019,"ci_90_upper":0.6574049708053686,"ci_75_lower":0.5163991877761697,"ci_75_upper":0.6300309548209291,"ci_50_lower":0.5433662931900471,"ci_50_upper":0.6102690700429166},"262144":{"avg_score":0.6572627927051157,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.46437324709833155,"median_score":1,"q1_score":0.03801348805427551,"q3_score":1,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":3.2160675000000056,"total_cache_read_cost_bin":5.218056000000008,"total_completion_cost_bin":0.3974100000000002,"total_reasoning_cost_bin":1.3080750000000005,"total_cost_bin":10.139608500000014,"total_on_demand_input_cost_bin":55.396627499999994,"ci_99_lower":0.5374875519436235,"ci_99_upper":0.759070305910196,"ci_98_lower":0.5619091577332546,"ci_98_upper":0.7461006263597597,"ci_95_lower":0.5778786010355754,"ci_95_upper":0.7323313283690264,"ci_90_lower":0.5935462909811737,"ci_90_upper":0.7193417255918608,"ci_75_lower":0.6102589994069545,"ci_75_upper":0.7038541116713751,"ci_50_lower":0.6292133997021084,"ci_50_upper":0.68488304441339},"524288":{"avg_score":0.5536590900382626,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.47945721259970436,"median_score":1,"q1_score":0.03408948518335819,"q3_score":1,"avg_token_efficiency":1.0000019078540328,"is_incomplete":false,"total_prompt_cost_bin":9.05525249999999,"total_cache_read_cost_bin":17.64938879999993,"total_completion_cost_bin":0.647745,"total_reasoning_cost_bin":2.1909675,"total_cost_bin":29.54335379999992,"total_on_demand_input_cost_bin":185.54914050000014,"ci_99_lower":0.47266302999173154,"ci_99_upper":0.6280906647596393,"ci_98_lower":0.47977585584108745,"ci_98_upper":0.6227261160226278,"ci_95_lower":0.4924289367996823,"ci_95_upper":0.6148736211504803,"ci_90_lower":0.5020019336291827,"ci_90_upper":0.6047803101699081,"ci_75_lower":0.5187326657745625,"ci_75_upper":0.590928744955602,"ci_50_lower":0.5323752560736651,"ci_50_upper":0.5753549487937747},"1048576":{"avg_score":0.5410259802204868,"min_score":0.00522193219512701,"max_score":1,"n_tests":309,"stddev":0.4831342031081126,"median_score":1,"q1_score":0.03008849546313286,"q3_score":1,"avg_token_efficiency":1.000000953849044,"is_incomplete":false,"total_prompt_cost_bin":17.76801149999997,"total_cache_read_cost_bin":46.81583999999988,"total_completion_cost_bin":0.9163575,"total_reasoning_cost_bin":2.945152500000003,"total_cost_bin":68.44536149999986,"total_on_demand_input_cost_bin":485.92641150000145,"ci_99_lower":0.47097366262096185,"ci_99_upper":0.6124360450657242,"ci_98_lower":0.4765321064960591,"ci_98_upper":0.6034869787837515,"ci_95_lower":0.48999080525776734,"ci_95_upper":0.5945165240324123,"ci_90_lower":0.4978607677809388,"ci_90_upper":0.5845309628308585,"ci_75_lower":0.5086435691747757,"ci_75_upper":0.5718538319401165,"ci_50_lower":0.5212300770307973,"ci_50_upper":0.5591115199012868}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Google: Gemini 3.6 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.00000375,"reasoning":0.00000375,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:13.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-21T19:31:26.095Z"},{"model_slug":"openai/gpt-5.6-sol","reasoning_mode":"max","overall_metrics":{"auc_1m":0.5646008109341077,"auc_128k":0.9712010552603224,"cum_avg_1m":0.7933402925545212,"cum_avg_128k":0.9785610940351631,"avg_score_overall":0.8551447886787571,"avg_response_time_ms":65528.43801663673,"total_prompt_cost":17.21771999999997,"total_cache_read_cost":146.400031,"total_completion_cost":103.21803000000006,"total_reasoning_cost":93.72900000000001,"total_prompt_cost_128k":4.35158,"total_completion_cost_128k":23.570880000000002,"total_on_demand_input_cost":1482.3672350000024,"total_on_demand_input_cost_128k":120.27207000000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9805125337959009},"bin_metrics":{"8192":{"avg_score":0.9999939523637295,"min_score":0.9995161890983582,"max_score":1,"n_tests":80,"stddev":0.00005409170321633159,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598392,"is_incomplete":false,"total_prompt_cost_bin":0.29437499999999994,"total_cache_read_cost_bin":0.23015849999999985,"total_completion_cost_bin":1.8973800000000003,"total_reasoning_cost_bin":1.06515,"total_cost_bin":3.4870635,"total_on_demand_input_cost_bin":2.595959999999999,"ci_99_lower":0.9999758094549179,"ci_99_upper":1,"ci_98_lower":0.9999758094549179,"ci_98_upper":1,"ci_95_lower":0.9999818570911885,"ci_95_upper":1,"ci_90_lower":0.9999818570911885,"ci_90_upper":1,"ci_75_lower":0.9999879047274589,"ci_75_upper":1,"ci_50_lower":0.9999879047274589,"ci_50_upper":1},"16384":{"avg_score":0.9999858118593693,"min_score":0.9988649487495422,"max_score":1,"n_tests":80,"stddev":0.0001269025876984849,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188716,"is_incomplete":false,"total_prompt_cost_bin":0.39171999999999974,"total_cache_read_cost_bin":0.49367200000000017,"total_completion_cost_bin":2.3088600000000006,"total_reasoning_cost_bin":1.4920799999999994,"total_cost_bin":4.686332,"total_on_demand_input_cost_bin":5.328439999999997,"ci_99_lower":0.9999432474374771,"ci_99_upper":1,"ci_98_lower":0.9999432474374771,"ci_98_upper":1,"ci_95_lower":0.9999574355781078,"ci_95_upper":1,"ci_90_lower":0.9999574355781078,"ci_90_upper":1,"ci_75_lower":0.9999716237187386,"ci_75_upper":1,"ci_50_lower":0.9999716237187386,"ci_50_upper":1},"32768":{"avg_score":0.9800097513943911,"min_score":0.07382550090551376,"max_score":1,"n_tests":136,"stddev":0.13344301548285695,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.995363441697986,"is_incomplete":false,"total_prompt_cost_bin":1.3899850000000007,"total_cache_read_cost_bin":1.791212000000003,"total_completion_cost_bin":5.783429999999999,"total_reasoning_cost_bin":4.18728,"total_cost_bin":13.151907000000005,"total_on_demand_input_cost_bin":19.302105000000005,"ci_99_lower":0.9403375836785006,"ci_99_upper":0.9999883797545643,"ci_98_lower":0.9467334838956594,"ci_98_upper":0.9999832903462298,"ci_95_lower":0.9533599942388451,"ci_95_upper":0.9999721783487235,"ci_90_lower":0.9598756925339865,"ci_90_upper":0.9934611418558394,"ci_75_lower":0.9665860498104901,"ci_75_upper":0.9933378899141269,"ci_50_lower":0.973278277553618,"ci_50_upper":0.9868186338092475},"65536":{"avg_score":0.9883735312587199,"min_score":0.0146329952403903,"max_score":1,"n_tests":85,"stddev":0.10687453108039222,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9776435846759917,"is_incomplete":false,"total_prompt_cost_bin":0.9760650000000001,"total_cache_read_cost_bin":2.620227999999998,"total_completion_cost_bin":4.760879999999999,"total_reasoning_cost_bin":3.7422000000000004,"total_cost_bin":12.099372999999998,"total_on_demand_input_cost_bin":27.178345000000014,"ci_99_lower":0.9535773239468828,"ci_99_upper":0.9999977275098072,"ci_98_lower":0.9535952802037492,"ci_98_upper":0.9999977160902584,"ci_95_lower":0.9651463081665775,"ci_95_upper":0.9999932189373409,"ci_90_lower":0.9651907091741176,"ci_90_upper":0.9999891687841976,"ci_75_lower":0.9767696691567407,"ci_75_upper":0.9999761098447968,"ci_50_lower":0.976810723258292,"ci_50_upper":0.999958007475909},"131072":{"avg_score":0.924442423299606,"min_score":0.005767250433564186,"max_score":1,"n_tests":103,"stddev":0.2616551971873621,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9772722316188371,"is_incomplete":false,"total_prompt_cost_bin":1.2994350000000003,"total_cache_read_cost_bin":6.456778500000007,"total_completion_cost_bin":8.820330000000004,"total_reasoning_cost_bin":7.547520000000002,"total_cost_bin":24.124063500000013,"total_on_demand_input_cost_bin":65.86722,"ci_99_lower":0.8397816169737088,"ci_99_upper":0.9807573583798972,"ci_98_lower":0.8581794479797658,"ci_98_upper":0.9719635772766563,"ci_95_lower":0.867359737388211,"ci_95_upper":0.9715620676254474,"ci_90_lower":0.8772039927528646,"ci_90_upper":0.962388776329988,"ci_75_lower":0.8879231676680841,"ci_75_upper":0.952887468700053,"ci_50_lower":0.9057538509052427,"ci_50_upper":0.9432738166646036},"262144":{"avg_score":0.8353025832986261,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.3649051124759012,"median_score":1,"q1_score":0.9997991323471069,"q3_score":1,"avg_token_efficiency":0.9824198103380463,"is_incomplete":false,"total_prompt_cost_bin":1.3145549999999993,"total_cache_read_cost_bin":18.00938999999997,"total_completion_cost_bin":22.81317,"total_reasoning_cost_bin":21.21798,"total_cost_bin":63.35509499999996,"total_on_demand_input_cost_bin":181.40845500000012,"ci_99_lower":0.7539268010421755,"ci_99_upper":0.9041523861387044,"ci_98_lower":0.7599368649699348,"ci_98_upper":0.9037284972463859,"ci_95_lower":0.7674002422341839,"ci_95_upper":0.890816295035811,"ci_90_lower":0.7801746123615679,"ci_90_upper":0.8836092717661556,"ci_75_lower":0.7948693247381532,"ci_75_upper":0.869742243426046,"ci_50_lower":0.8139522767886651,"ci_50_upper":0.8562154002959925},"524288":{"avg_score":0.6186142869617285,"min_score":0.006074411328881979,"max_score":1,"n_tests":236,"stddev":0.4694362061696369,"median_score":1,"q1_score":0.0412564342841506,"q3_score":1,"avg_token_efficiency":0.9793816566660328,"is_incomplete":false,"total_prompt_cost_bin":11.551584999999971,"total_cache_read_cost_bin":116.79859200000004,"total_completion_cost_bin":56.83398000000006,"total_reasoning_cost_bin":54.47679000000001,"total_cost_bin":239.66094700000008,"total_on_demand_input_cost_bin":1180.6867100000024,"ci_99_lower":0.5373761974730612,"ci_99_upper":0.6931274504973987,"ci_98_lower":0.5446283496774228,"ci_98_upper":0.6878485787352863,"ci_95_lower":0.555950715498095,"ci_95_upper":0.6788838147337181,"ci_90_lower":0.5640495744757789,"ci_90_upper":0.6692581115990986,"ci_75_lower":0.5826030002356413,"ci_75_upper":0.6534156428267233,"ci_50_lower":0.597407449087645,"ci_50_upper":0.6367795489308171}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"OpenAI: GPT-5.6 Sol","provider_name":"Openai","run_provider_names":["OpenAI"],"pricing_metadata":{"prompt":0.000002,"completion":0.00001,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-09T09:54:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-10T17:41:19.440Z"},{"model_slug":"openai/gpt-5.5","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.5090571334306894,"auc_128k":0.8745130687517292,"cum_avg_1m":0.7318189789353764,"cum_avg_128k":0.9119833140787295,"avg_score_overall":0.7858981043365367,"avg_response_time_ms":18962.597836023702,"total_prompt_cost":31.303590000000042,"total_cache_read_cost":147.8810879999993,"total_completion_cost":43.729680000000016,"total_reasoning_cost":34.01745,"total_prompt_cost_128k":3.9854049999999988,"total_completion_cost_128k":17.79276,"total_on_demand_input_cost":1510.8110500000016,"total_on_demand_input_cost_128k":120.26956499999999,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778016},"bin_metrics":{"8192":{"avg_score":0.9999824523925781,"min_score":0.99859619140625,"max_score":1,"n_tests":80,"stddev":0.00015695057215116934,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598381,"is_incomplete":false,"total_prompt_cost_bin":0.37771999999999967,"total_cache_read_cost_bin":0.22182399999999985,"total_completion_cost_bin":1.7079600000000004,"total_reasoning_cost_bin":0.876,"total_cost_bin":3.1835039999999997,"total_on_demand_input_cost_bin":2.59596,"ci_99_lower":0.9999298095703125,"ci_99_upper":1,"ci_98_lower":0.9999298095703125,"ci_98_upper":1,"ci_95_lower":0.9999473571777344,"ci_95_upper":1,"ci_90_lower":0.9999473571777344,"ci_90_upper":1,"ci_75_lower":0.9999649047851562,"ci_75_upper":1,"ci_50_lower":0.9999649047851562,"ci_50_upper":1},"16384":{"avg_score":0.9634038755088113,"min_score":0.011840158142149448,"max_score":1,"n_tests":80,"stddev":0.18651326470445836,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188716,"is_incomplete":false,"total_prompt_cost_bin":0.3300399999999998,"total_cache_read_cost_bin":0.49984000000000006,"total_completion_cost_bin":2.0778900000000005,"total_reasoning_cost_bin":1.2630599999999998,"total_cost_bin":4.1708300000000005,"total_on_demand_input_cost_bin":5.328440000000003,"ci_99_lower":0.9021157281958149,"ci_99_upper":0.9999947031773627,"ci_98_lower":0.9025496294136391,"ci_98_upper":0.9999946765601635,"ci_95_lower":0.9146997029223712,"ci_95_upper":0.9999869845807552,"ci_90_lower":0.9267862889851677,"ci_90_upper":0.9999763377010822,"ci_75_lower":0.9389512779191137,"ci_75_upper":0.987860914110206,"ci_50_lower":0.9510786821367219,"ci_50_upper":0.9757484454195946},"32768":{"avg_score":0.9154918342640431,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.27279660996921073,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.995210076854963,"is_incomplete":false,"total_prompt_cost_bin":0.8778399999999995,"total_cache_read_cost_bin":1.842176000000007,"total_completion_cost_bin":5.08236,"total_reasoning_cost_bin":3.4667699999999995,"total_cost_bin":11.269146000000006,"total_on_demand_input_cost_bin":19.299599999999998,"ci_99_lower":0.8509015179163544,"ci_99_upper":0.966104698983057,"ci_98_lower":0.8524828930881888,"ci_98_upper":0.9648381274827134,"ci_95_lower":0.8661553354564497,"ci_95_upper":0.9584553557941141,"ci_90_lower":0.8732083748442271,"ci_90_upper":0.9510540036933142,"ci_75_lower":0.8873736534874564,"ci_75_upper":0.9434134577927847,"ci_50_lower":0.9004184709791683,"ci_50_upper":0.9308148848542581},"65536":{"avg_score":0.8509849280557211,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.35280513667083147,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":1.0433050000000001,"total_cache_read_cost_bin":2.6135039999999954,"total_completion_cost_bin":3.581999999999999,"total_reasoning_cost_bin":2.5694100000000004,"total_cost_bin":9.808218999999994,"total_on_demand_input_cost_bin":27.17834500000001,"ci_99_lower":0.7256464748186442,"ci_99_upper":0.9425534531483974,"ci_98_lower":0.7476962674369706,"ci_98_upper":0.9311319914157776,"ci_95_lower":0.7700816024976836,"ci_95_upper":0.9197812968180241,"ci_90_lower":0.7820981972620767,"ci_90_upper":0.9085573222290945,"ci_75_lower":0.8050383108832381,"ci_75_upper":0.8962404770831414,"ci_50_lower":0.8277289586963461,"ci_50_upper":0.8743239179637065},"131072":{"avg_score":0.8300534801724937,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.3711702968161405,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":1.3564999999999996,"total_cache_read_cost_bin":6.451071999999995,"total_completion_cost_bin":5.342550000000003,"total_reasoning_cost_bin":4.070940000000001,"total_cost_bin":17.221061999999996,"total_on_demand_input_cost_bin":65.86721999999997,"ci_99_lower":0.7245763521368652,"ci_99_upper":0.915237502761974,"ci_98_lower":0.7445298664048495,"ci_98_upper":0.9057850801548016,"ci_95_lower":0.7543637664385087,"ci_95_upper":0.8965909904625168,"ci_90_lower":0.7636110326509189,"ci_90_upper":0.8866611055204717,"ci_75_lower":0.7832948946774354,"ci_75_upper":0.8681299717002209,"ci_50_lower":0.8021092731505632,"ci_50_upper":0.8579116223003598},"262144":{"avg_score":0.7187650191353251,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.4408498563207154,"median_score":1,"q1_score":0.04997968301177025,"q3_score":1,"avg_token_efficiency":0.9824198103380463,"is_incomplete":false,"total_prompt_cost_bin":6.127815000000001,"total_cache_read_cost_bin":17.528064000000025,"total_completion_cost_bin":9.372119999999999,"total_reasoning_cost_bin":7.792440000000001,"total_cost_bin":40.82043900000002,"total_on_demand_input_cost_bin":181.40845499999995,"ci_99_lower":0.6158308063470264,"ci_99_upper":0.815297846779399,"ci_98_lower":0.6291843133495711,"ci_98_upper":0.807247894977728,"ci_95_lower":0.6417195853399087,"ci_95_upper":0.7936681346730031,"ci_90_lower":0.6555965154157648,"ci_90_upper":0.7803240205950586,"ci_75_lower":0.6759371955768396,"ci_75_upper":0.7601526115281204,"ci_50_lower":0.6914890035805259,"ci_50_upper":0.7454879600719833},"524288":{"avg_score":0.5758702419540387,"min_score":0.00469667324796319,"max_score":1,"n_tests":236,"stddev":0.4747215397139031,"median_score":1,"q1_score":0.03548464272171259,"q3_score":1,"avg_token_efficiency":0.9774785269226758,"is_incomplete":false,"total_prompt_cost_bin":21.19037000000004,"total_cache_read_cost_bin":118.72460799999928,"total_completion_cost_bin":16.564800000000016,"total_reasoning_cost_bin":13.978829999999993,"total_cost_bin":170.45860799999932,"total_on_demand_input_cost_bin":1209.1330300000016,"ci_99_lower":0.4920433459362761,"ci_99_upper":0.6540403857387177,"ci_98_lower":0.5000900533959917,"ci_98_upper":0.6464254523780971,"ci_95_lower":0.5153319908624171,"ci_95_upper":0.6375204746430381,"ci_90_lower":0.5241767684441707,"ci_90_upper":0.6274603611436451,"ci_75_lower":0.5409760437084469,"ci_75_upper":0.6120750689444969,"ci_50_lower":0.5549003294618776,"ci_50_upper":0.597708724518563}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.5","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.000005,"completion":0.00003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T17:31:33.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-25T05:10:17.386Z"},{"model_slug":"openai/gpt-5.5","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.5053625639639504,"auc_128k":0.9170978185200739,"cum_avg_1m":0.7496283076980224,"cum_avg_128k":0.9447192239165914,"avg_score_overall":0.7977035476138726,"avg_response_time_ms":54334.64167755517,"total_prompt_cost":34.85621000000005,"total_cache_read_cost":147.35961599999933,"total_completion_cost":129.68844,"total_reasoning_cost":119.94174000000001,"total_prompt_cost_128k":3.692285,"total_completion_cost_128k":41.094179999999994,"total_on_demand_input_cost":1511.0442600000013,"total_on_demand_input_cost_128k":120.26956499999997,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9800183372228054},"bin_metrics":{"8192":{"avg_score":0.9999982446432114,"min_score":0.9998595714569092,"max_score":1,"n_tests":80,"stddev":0.00001570038841661663,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598395,"is_incomplete":false,"total_prompt_cost_bin":0.3892399999999999,"total_cache_read_cost_bin":0.2206719999999998,"total_completion_cost_bin":3.6221099999999993,"total_reasoning_cost_bin":2.7898800000000006,"total_cost_bin":7.021901999999999,"total_on_demand_input_cost_bin":2.595960000000002,"ci_99_lower":0.9999929785728454,"ci_99_upper":1,"ci_98_lower":0.9999929785728454,"ci_98_upper":1,"ci_95_lower":0.9999947339296341,"ci_95_upper":1,"ci_90_lower":0.9999947339296341,"ci_90_upper":1,"ci_75_lower":0.9999964892864227,"ci_75_upper":1,"ci_50_lower":0.9999964892864227,"ci_50_upper":1},"16384":{"avg_score":0.9987007603049278,"min_score":0.8989971280097961,"max_score":1,"n_tests":80,"stddev":0.011289539738148679,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.46059999999999984,"total_cache_read_cost_bin":0.48678400000000027,"total_completion_cost_bin":4.2444299999999995,"total_reasoning_cost_bin":3.4279499999999983,"total_cost_bin":8.619763999999998,"total_on_demand_input_cost_bin":5.328439999999997,"ci_99_lower":0.994886942639947,"ci_99_upper":0.9999939654804767,"ci_98_lower":0.9949104204252363,"ci_98_upper":0.9999922408983112,"ci_95_lower":0.9961564755998552,"ci_95_upper":0.9999879185296594,"ci_90_lower":0.996181647144258,"ci_90_upper":0.9999851547181606,"ci_75_lower":0.9974281415343285,"ci_75_upper":0.9999721840955317,"ci_50_lower":0.9974608294665813,"ci_50_upper":0.9999556431546808},"32768":{"avg_score":0.9581411622793359,"min_score":0.022179193794727325,"max_score":1,"n_tests":136,"stddev":0.19538534393043638,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9952100768549628,"is_incomplete":false,"total_prompt_cost_bin":0.7613599999999996,"total_cache_read_cost_bin":1.8538240000000068,"total_completion_cost_bin":11.016270000000002,"total_reasoning_cost_bin":9.408390000000002,"total_cost_bin":23.03984400000001,"total_on_demand_input_cost_bin":19.2996,"ci_99_lower":0.9085359591603115,"ci_99_upper":0.9933863960272249,"ci_98_lower":0.9145929705535116,"ci_98_upper":0.9929443499829401,"ci_95_lower":0.9227994149461772,"ci_95_upper":0.9864431521478656,"ci_90_lower":0.9292994513737913,"ci_90_upper":0.9857570541009087,"ci_75_lower":0.9374344846377534,"ci_75_upper":0.9789623325238661,"ci_50_lower":0.9450352570685723,"ci_50_upper":0.9714223955045728},"65536":{"avg_score":0.9084848593810901,"min_score":0.014639905653893948,"max_score":1,"n_tests":85,"stddev":0.28545455700927036,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9776435846759913,"is_incomplete":false,"total_prompt_cost_bin":0.7245850000000007,"total_cache_read_cost_bin":2.6453759999999953,"total_completion_cost_bin":8.722859999999997,"total_reasoning_cost_bin":7.699589999999999,"total_cost_bin":19.792410999999994,"total_on_demand_input_cost_bin":27.178345000000007,"ci_99_lower":0.8168711060623584,"ci_99_upper":0.9770539903519128,"ci_98_lower":0.827770666238797,"ci_98_upper":0.9658897727521903,"ci_95_lower":0.8397873642747564,"ci_95_upper":0.9653989610210171,"ci_90_lower":0.8510327839226846,"ci_90_upper":0.9543654790980851,"ci_75_lower":0.8735741742790731,"ci_75_upper":0.9428078691228567,"ci_50_lower":0.8859116883154081,"ci_50_upper":0.9312222560675925},"131072":{"avg_score":0.858271092974391,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.34493766543797616,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":1.3564999999999998,"total_cache_read_cost_bin":6.451071999999995,"total_completion_cost_bin":13.48851,"total_reasoning_cost_bin":12.201539999999998,"total_cost_bin":33.49762199999999,"total_on_demand_input_cost_bin":65.86721999999996,"ci_99_lower":0.7637508962959848,"ci_99_upper":0.9341534646486105,"ci_98_lower":0.7727806568273243,"ci_98_upper":0.9338280856849634,"ci_95_lower":0.7829755600330474,"ci_95_upper":0.9152331605115092,"ci_90_lower":0.8002679631248506,"ci_90_upper":0.9059636885878966,"ci_75_lower":0.8200678090704492,"ci_75_upper":0.8959943315410395,"ci_50_lower":0.8304947971070221,"ci_50_upper":0.8775771909587773},"262144":{"avg_score":0.7319269505930497,"min_score":0.011488250456750393,"max_score":1,"n_tests":141,"stddev":0.4350371756397304,"median_score":1,"q1_score":0.05039525777101517,"q3_score":1,"avg_token_efficiency":0.9824198103380458,"is_incomplete":false,"total_prompt_cost_bin":2.9546949999999983,"total_cache_read_cost_bin":17.845376000000027,"total_completion_cost_bin":28.55259000000001,"total_reasoning_cost_bin":26.966850000000022,"total_cost_bin":76.31951100000006,"total_on_demand_input_cost_bin":181.40845499999986,"ci_99_lower":0.6233355166860823,"ci_99_upper":0.8275106162678265,"ci_98_lower":0.6427358014548052,"ci_98_upper":0.820249242088262,"ci_95_lower":0.6565966426824212,"ci_95_upper":0.8012279735549505,"ci_90_lower":0.668427496468186,"ci_90_upper":0.7937131781887315,"ci_75_lower":0.6901729741936272,"ci_75_upper":0.7736254303182759,"ci_50_lower":0.7044949457082395,"ci_50_upper":0.7589565658981496},"524288":{"avg_score":0.541503391408172,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4798989353074104,"median_score":0.9994923174381256,"q1_score":0.0330197187140584,"q3_score":1,"avg_token_efficiency":0.9776670565355918,"is_incomplete":false,"total_prompt_cost_bin":28.209230000000048,"total_cache_read_cost_bin":117.85651199999931,"total_completion_cost_bin":60.04167000000002,"total_reasoning_cost_bin":57.44754,"total_cost_bin":263.5549519999994,"total_on_demand_input_cost_bin":1209.3662400000014,"ci_99_lower":0.45992482244093325,"ci_99_upper":0.6145918963961816,"ci_98_lower":0.4665460185636963,"ci_98_upper":0.6078331205720019,"ci_95_lower":0.48161966044789223,"ci_95_upper":0.6014252325210137,"ci_90_lower":0.490821932814818,"ci_90_upper":0.5927666767833801,"ci_75_lower":0.504849609902614,"ci_75_upper":0.5780948716934998,"ci_50_lower":0.5206243148674375,"ci_50_upper":0.5630391531809374}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.5","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.000005,"completion":0.00003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T17:31:33.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-25T18:49:54.280Z"},{"model_slug":"openai/gpt-5.5","reasoning_mode":"high","overall_metrics":{"auc_1m":0.501908180980127,"auc_128k":0.8816367627837757,"cum_avg_1m":0.7382123132092969,"cum_avg_128k":0.9217762153737059,"avg_score_overall":0.7896332648646576,"avg_response_time_ms":35262.12984651749,"total_prompt_cost":40.834920000000025,"total_cache_read_cost":146.57190399999934,"total_completion_cost":82.02134999999998,"total_reasoning_cost":72.27156000000002,"total_prompt_cost_128k":4.178684999999998,"total_completion_cost_128k":30.089189999999995,"total_on_demand_input_cost":1510.8110500000016,"total_on_demand_input_cost_128k":120.269565,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778019},"bin_metrics":{"8192":{"avg_score":0.9999808728694916,"min_score":0.9988649487495422,"max_score":1,"n_tests":80,"stddev":0.00013384256507884164,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598395,"is_incomplete":false,"total_prompt_cost_bin":0.37771999999999967,"total_cache_read_cost_bin":0.22182400000000005,"total_completion_cost_bin":2.665560000000001,"total_reasoning_cost_bin":1.8333899999999996,"total_cost_bin":5.0984940000000005,"total_on_demand_input_cost_bin":2.5959599999999985,"ci_99_lower":0.9999333694577217,"ci_99_upper":1,"ci_98_lower":0.9999383021593095,"ci_98_upper":1,"ci_95_lower":0.9999475575983524,"ci_95_upper":1,"ci_90_lower":0.9999524965882302,"ci_90_upper":1,"ci_75_lower":0.9999617457389831,"ci_75_upper":1,"ci_50_lower":0.9999716237187386,"ci_50_upper":0.9999950610101223},"16384":{"avg_score":0.9879045006819069,"min_score":0.03270111232995987,"max_score":1,"n_tests":80,"stddev":0.10814682741460972,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.4055599999999998,"total_cache_read_cost_bin":0.4922880000000002,"total_completion_cost_bin":3.2136000000000005,"total_reasoning_cost_bin":2.39616,"total_cost_bin":6.507608,"total_on_demand_input_cost_bin":5.328439999999997,"ci_99_lower":0.9516265291720629,"ci_99_upper":1,"ci_98_lower":0.9516307923942804,"ci_98_upper":1,"ci_95_lower":0.9637177652679384,"ci_95_upper":1,"ci_90_lower":0.963722028490156,"ci_90_upper":1,"ci_75_lower":0.9758127316832542,"ci_75_upper":1,"ci_50_lower":0.975817527808249,"ci_50_upper":0.9999957367777824},"32768":{"avg_score":0.966078467119266,"min_score":0.039231546223163605,"max_score":1,"n_tests":136,"stddev":0.1742237024958105,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9952100768549639,"is_incomplete":false,"total_prompt_cost_bin":1.3143199999999982,"total_cache_read_cost_bin":1.7985280000000048,"total_completion_cost_bin":7.932929999999999,"total_reasoning_cost_bin":6.322680000000005,"total_cost_bin":17.368458000000008,"total_on_demand_input_cost_bin":19.299599999999987,"ci_99_lower":0.9184707613467403,"ci_99_upper":0.9999687911548159,"ci_98_lower":0.9250174995177589,"ci_98_upper":0.9934537180535057,"ci_95_lower":0.9322330301175551,"ci_95_upper":0.9931683115442009,"ci_90_lower":0.9388031274682898,"ci_90_upper":0.98682349211372,"ci_75_lower":0.946205048054895,"ci_75_upper":0.9860340459272265,"ci_50_lower":0.9582845912698437,"ci_50_upper":0.9788424981593647},"65536":{"avg_score":0.8623999540748841,"min_score":0.011402508243918419,"max_score":1,"n_tests":85,"stddev":0.34140661704128106,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9776435846759918,"is_incomplete":false,"total_prompt_cost_bin":0.7245850000000003,"total_cache_read_cost_bin":2.6453759999999953,"total_completion_cost_bin":6.330900000000001,"total_reasoning_cost_bin":5.29752,"total_cost_bin":14.998380999999995,"total_on_demand_input_cost_bin":27.17834500000002,"ci_99_lower":0.7479558210235966,"ci_99_upper":0.9428160356422338,"ci_98_lower":0.7693557483008241,"ci_98_upper":0.9425352448323632,"ci_95_lower":0.7815264397391173,"ci_95_upper":0.9311598283729023,"ci_90_lower":0.7936033616330037,"ci_90_upper":0.9197519937206936,"ci_75_lower":0.8167747533304945,"ci_75_upper":0.9075382090598235,"ci_50_lower":0.8392287916777765,"ci_50_upper":0.8855897151700713},"131072":{"avg_score":0.7925172821229812,"min_score":0.004621605854481459,"max_score":1,"n_tests":103,"stddev":0.4001130642523872,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":1.3565000000000005,"total_cache_read_cost_bin":6.451071999999995,"total_completion_cost_bin":9.946199999999996,"total_reasoning_cost_bin":8.67828,"total_cost_bin":26.43205199999999,"total_on_demand_input_cost_bin":65.86721999999999,"ci_99_lower":0.6789196971956034,"ci_99_upper":0.8865528663663076,"ci_98_lower":0.6880537992478032,"ci_98_upper":0.8778137627990863,"ci_95_lower":0.7081637613747712,"ci_95_upper":0.8680291889740598,"ci_90_lower":0.7257761751997819,"ci_90_upper":0.8578811141962945,"ci_75_lower":0.744891065048524,"ci_75_upper":0.8389753979812739,"ci_50_lower":0.7644186333992781,"ci_50_upper":0.8205823016780185},"262144":{"avg_score":0.752899470706041,"min_score":0.014084506779909134,"max_score":1,"n_tests":141,"stddev":0.42357937693736447,"median_score":1,"q1_score":0.08829787373542786,"q3_score":1,"avg_token_efficiency":0.9824198103380458,"is_incomplete":false,"total_prompt_cost_bin":6.781895000000008,"total_cache_read_cost_bin":17.462656000000024,"total_completion_cost_bin":15.844950000000003,"total_reasoning_cost_bin":14.249339999999993,"total_cost_bin":54.33884100000003,"total_on_demand_input_cost_bin":181.40845499999975,"ci_99_lower":0.6571470165221642,"ci_99_upper":0.8430407625231363,"ci_98_lower":0.6645779080341524,"ci_98_upper":0.8350104853963968,"ci_95_lower":0.6774206903631014,"ci_95_upper":0.8274327235002441,"ci_90_lower":0.6911953620592807,"ci_90_upper":0.8144790082991598,"ci_75_lower":0.7113761371117191,"ci_75_upper":0.794326454798975,"ci_50_lower":0.7256522490180607,"ci_50_upper":0.7799223679124463},"524288":{"avg_score":0.5439179580998055,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.48147170630941116,"median_score":1,"q1_score":0.031146734952926636,"q3_score":1,"avg_token_efficiency":0.977478526922676,"is_incomplete":false,"total_prompt_cost_bin":29.874340000000018,"total_cache_read_cost_bin":117.50015999999931,"total_completion_cost_bin":36.087209999999985,"total_reasoning_cost_bin":33.49419000000002,"total_cost_bin":216.9558999999993,"total_on_demand_input_cost_bin":1209.1330300000018,"ci_99_lower":0.46223869929856704,"ci_99_upper":0.6183368675164106,"ci_98_lower":0.4692734231849367,"ci_98_upper":0.6110101450035956,"ci_95_lower":0.48366376177261794,"ci_95_upper":0.6041786598594627,"ci_90_lower":0.49189878382761487,"ci_90_upper":0.5948263919569939,"ci_75_lower":0.5082710471548835,"ci_75_upper":0.5806077793008702,"ci_50_lower":0.5224645357150821,"ci_50_upper":0.5654916102550056}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.5","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.000005,"completion":0.00003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T17:31:33.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-25T13:49:27.307Z"},{"model_slug":"openai/gpt-5.6-terra","reasoning_mode":"max","overall_metrics":{"auc_1m":0.4727893669061518,"auc_128k":0.8942685088493842,"cum_avg_1m":0.725863173658628,"cum_avg_128k":0.9259391727286186,"avg_score_overall":0.7659157621428159,"avg_response_time_ms":84017.88948028954,"total_prompt_cost":752.8437975000008,"total_cache_read_cost":0.256172,"total_completion_cost":130.63205999999997,"total_reasoning_cost":125.73798000000004,"total_prompt_cost_128k":60.1347825,"total_completion_cost_128k":39.73706999999999,"total_on_demand_input_cost":755.4055250000008,"total_on_demand_input_cost_128k":60.1347825,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778019},"bin_metrics":{"8192":{"avg_score":0.9876667948672548,"min_score":0.021897809579968452,"max_score":1,"n_tests":80,"stddev":0.10934412926555884,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598393,"is_incomplete":false,"total_prompt_cost_bin":1.2979800000000008,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.4004000000000005,"total_reasoning_cost_bin":0.9850499999999996,"total_cost_bin":3.683430000000001,"total_on_demand_input_cost_bin":1.2979800000000008,"ci_99_lower":0.9509304741732776,"ci_99_upper":0.9999791426025331,"ci_98_lower":0.9509685651697218,"ci_98_upper":0.9999758625477553,"ci_95_lower":0.9631687707034872,"ci_95_upper":0.9999645054154097,"ci_90_lower":0.9632301054848359,"ci_90_upper":0.999950978755951,"ci_75_lower":0.975409617787227,"ci_75_upper":0.9999189207330346,"ci_50_lower":0.9755045418161898,"ci_50_upper":0.9998707339167596},"16384":{"avg_score":0.9999125652015209,"min_score":0.997533917427063,"max_score":1,"n_tests":80,"stddev":0.0003294855009678376,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188722,"is_incomplete":false,"total_prompt_cost_bin":2.6642199999999976,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.4646950000000007,"total_reasoning_cost_bin":2.05641,"total_cost_bin":7.185324999999999,"total_on_demand_input_cost_bin":2.6642199999999976,"ci_99_lower":0.9997974243834614,"ci_99_upper":0.999982512049377,"ci_98_lower":0.9998082805573939,"ci_98_upper":0.9999781121909619,"ci_95_lower":0.9998289788700641,"ci_95_upper":0.9999723094142974,"ci_90_lower":0.999845363907516,"ci_90_upper":0.9999655369296669,"ci_75_lower":0.9998676489107311,"ci_75_upper":0.9999530527740716,"ci_50_lower":0.9998902192339302,"ci_50_upper":0.9999374099075794},"32768":{"avg_score":0.9430140695562038,"min_score":0.020003637298941612,"max_score":1,"n_tests":136,"stddev":0.22268003939177025,"median_score":1,"q1_score":0.9996937066316605,"q3_score":1,"avg_token_efficiency":0.9952100768549625,"is_incomplete":false,"total_prompt_cost_bin":9.649799999999995,"total_cache_read_cost_bin":0,"total_completion_cost_bin":7.278254999999999,"total_reasoning_cost_bin":6.474554999999998,"total_cost_bin":23.402609999999992,"total_on_demand_input_cost_bin":9.649799999999995,"ci_99_lower":0.887707311352304,"ci_99_upper":0.9862166743159212,"ci_98_lower":0.8941670093675801,"ci_98_upper":0.9840845977272564,"ci_95_lower":0.9001861021470497,"ci_95_upper":0.9776346624465932,"ci_90_lower":0.9084848297650323,"ci_90_upper":0.97233414048644,"ci_75_lower":0.919643557329169,"ci_75_upper":0.9647556836305953,"ci_50_lower":0.9298067714450607,"ci_50_upper":0.9567950527416542},"65536":{"avg_score":0.8974367213818957,"min_score":0.008648648858070374,"max_score":1,"n_tests":85,"stddev":0.2994834039283396,"median_score":1,"q1_score":0.9997740983963013,"q3_score":1,"avg_token_efficiency":0.9776435846759918,"is_incomplete":false,"total_prompt_cost_bin":13.589172500000007,"total_cache_read_cost_bin":0,"total_completion_cost_bin":7.873770000000002,"total_reasoning_cost_bin":7.366620000000003,"total_cost_bin":28.829562500000016,"total_on_demand_input_cost_bin":13.589172500000007,"ci_99_lower":0.8051779240515522,"ci_99_upper":0.9656877610408426,"ci_98_lower":0.8061950353169266,"ci_98_upper":0.965442075851209,"ci_95_lower":0.8181465992101413,"ci_95_upper":0.9543986149970443,"ci_90_lower":0.839172250341405,"ci_90_upper":0.9435850746359895,"ci_75_lower":0.8621449437047191,"ci_75_upper":0.931825278461089,"ci_50_lower":0.8747507164638271,"ci_50_upper":0.9201294865678339},"131072":{"avg_score":0.8016657126362173,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.3936726546875098,"median_score":1,"q1_score":0.9993235468864441,"q3_score":1,"avg_token_efficiency":0.9772722316188369,"is_incomplete":false,"total_prompt_cost_bin":32.93361,"total_cache_read_cost_bin":0,"total_completion_cost_bin":20.719949999999987,"total_reasoning_cost_bin":20.082524999999993,"total_cost_bin":73.73608499999997,"total_on_demand_input_cost_bin":32.93361,"ci_99_lower":0.68765636935265,"ci_99_upper":0.8963321470447535,"ci_98_lower":0.6904552089237581,"ci_98_upper":0.8869618305640783,"ci_95_lower":0.7170407235111534,"ci_95_upper":0.8773405410283814,"ci_90_lower":0.734958043229739,"ci_90_upper":0.867110718798471,"ci_75_lower":0.7543350683482255,"ci_75_upper":0.8480117873659411,"ci_50_lower":0.7735656653982994,"ci_50_upper":0.8299401892412418},"262144":{"avg_score":0.6787150402830795,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.455681897783336,"median_score":0.9998052716255188,"q1_score":0.046250201761722565,"q3_score":1,"avg_token_efficiency":0.9824198103380463,"is_incomplete":false,"total_prompt_cost_bin":90.70422749999986,"total_cache_read_cost_bin":0,"total_completion_cost_bin":30.065624999999997,"total_reasoning_cost_bin":29.26357500000001,"total_cost_bin":150.03342749999987,"total_on_demand_input_cost_bin":90.70422749999986,"ci_99_lower":0.5747285742160751,"ci_99_upper":0.7755352233965048,"ci_98_lower":0.5830397068654973,"ci_98_upper":0.7672331754465634,"ci_95_lower":0.6022238229702768,"ci_95_upper":0.7539317596797065,"ci_90_lower":0.6159992369369055,"ci_90_upper":0.740767551421069,"ci_75_lower":0.6317126140104119,"ci_75_upper":0.7251660288080362,"ci_50_lower":0.6508373817303449,"ci_50_upper":0.7059464354046574},"524288":{"avg_score":0.49849448534285146,"min_score":0.0036479707341641188,"max_score":1,"n_tests":236,"stddev":0.48171186951973977,"median_score":0.10733075812458992,"q1_score":0.028894133400171995,"q3_score":1,"avg_token_efficiency":0.977478526922676,"is_incomplete":false,"total_prompt_cost_bin":602.0047875000009,"total_cache_read_cost_bin":0.256172,"total_completion_cost_bin":60.829364999999996,"total_reasoning_cost_bin":59.509245000000035,"total_cost_bin":722.5995695000009,"total_on_demand_input_cost_bin":604.5665150000009,"ci_99_lower":0.41748537076940523,"ci_99_upper":0.5722523693885738,"ci_98_lower":0.42044892498132197,"ci_98_upper":0.5665753741070167,"ci_95_lower":0.43507311663640186,"ci_95_upper":0.5600145078979935,"ci_90_lower":0.4482024457008712,"ci_90_upper":0.5521578478486814,"ci_75_lower":0.4624353870492965,"ci_75_upper":0.5342908261181792,"ci_50_lower":0.47699228850917463,"ci_50_upper":0.5197221922633432}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low","none"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.6 Terra","provider_name":"Openai","run_provider_names":["OpenAI"],"pricing_metadata":{"prompt":0.000002,"completion":0.000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-09T09:54:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-10T15:29:53.765Z"},{"model_slug":"openai/gpt-5.5","reasoning_mode":"low","overall_metrics":{"auc_1m":0.47256202631220656,"auc_128k":0.8326411365177632,"cum_avg_1m":0.7020739436578093,"cum_avg_128k":0.880613558988865,"avg_score_overall":0.7503102515595905,"avg_response_time_ms":8866.799941644436,"total_prompt_cost":37.62980000000002,"total_cache_read_cost":147.2487679999993,"total_completion_cost":17.267249999999997,"total_reasoning_cost":7.49676,"total_prompt_cost_128k":4.512764999999998,"total_completion_cost_128k":8.344470000000001,"total_on_demand_input_cost":1510.8110500000014,"total_on_demand_input_cost_128k":120.26956500000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778016},"bin_metrics":{"8192":{"avg_score":0.9999982044100761,"min_score":0.9998563528060913,"max_score":1,"n_tests":80,"stddev":0.000016060244517846382,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598395,"is_incomplete":false,"total_prompt_cost_bin":0.3405999999999998,"total_cache_read_cost_bin":0.22553600000000004,"total_completion_cost_bin":1.0665600000000002,"total_reasoning_cost_bin":0.2343300000000001,"total_cost_bin":1.8670260000000003,"total_on_demand_input_cost_bin":2.595959999999999,"ci_99_lower":0.9999928176403046,"ci_99_upper":1,"ci_98_lower":0.9999928176403046,"ci_98_upper":1,"ci_95_lower":0.9999946132302284,"ci_95_upper":1,"ci_90_lower":0.9999946132302284,"ci_90_upper":1,"ci_75_lower":0.9999964088201523,"ci_75_upper":1,"ci_50_lower":0.9999964088201523,"ci_50_upper":1},"16384":{"avg_score":0.9634485644288361,"min_score":0.019668737426400185,"max_score":1,"n_tests":80,"stddev":0.18615835644398843,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.4580399999999998,"total_cache_read_cost_bin":0.4870400000000003,"total_completion_cost_bin":1.11252,"total_reasoning_cost_bin":0.29703000000000007,"total_cost_bin":2.35463,"total_on_demand_input_cost_bin":5.328439999999998,"ci_99_lower":0.9024283964509378,"ci_99_upper":0.9999911699853838,"ci_98_lower":0.9025877113798634,"ci_98_upper":0.999991125613451,"ci_95_lower":0.9147810963046503,"ci_95_upper":0.9999639518558979,"ci_90_lower":0.9268866859865375,"ci_90_upper":0.9999193936958909,"ci_75_lower":0.939102078252472,"ci_75_upper":0.9877768200589344,"ci_50_lower":0.9512299227353652,"ci_50_upper":0.975675531453453},"32768":{"avg_score":0.9174749356530169,"min_score":0.014975041151046753,"max_score":1,"n_tests":136,"stddev":0.266732125241372,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9952100768549622,"is_incomplete":false,"total_prompt_cost_bin":0.9943199999999996,"total_cache_read_cost_bin":1.8305280000000037,"total_completion_cost_bin":2.2979400000000023,"total_reasoning_cost_bin":0.6829200000000002,"total_cost_bin":5.805708000000005,"total_on_demand_input_cost_bin":19.299600000000012,"ci_99_lower":0.854226496569155,"ci_99_upper":0.967876272977549,"ci_98_lower":0.8565352331444293,"ci_98_upper":0.9655252858076026,"ci_95_lower":0.8691941669272423,"ci_95_upper":0.9597293821519569,"ci_90_lower":0.8757584972307085,"ci_90_upper":0.9523132383727524,"ci_75_lower":0.8897881206392091,"ci_75_upper":0.9446888691095143,"ci_50_lower":0.9021113542155089,"ci_50_upper":0.9335461630184643},"65536":{"avg_score":0.85171796507476,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.3509995542279188,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":0.7245850000000007,"total_cache_read_cost_bin":2.6453759999999953,"total_completion_cost_bin":1.6449599999999998,"total_reasoning_cost_bin":0.6194699999999997,"total_cost_bin":5.6343909999999955,"total_on_demand_input_cost_bin":27.178345000000004,"ci_99_lower":0.7271912087300134,"ci_99_upper":0.9428036937991924,"ci_98_lower":0.7489122201492462,"ci_98_upper":0.9315078626845908,"ci_95_lower":0.7711388527193819,"ci_95_upper":0.9201866812374005,"ci_90_lower":0.7831445063354776,"ci_90_upper":0.9090272642283098,"ci_75_lower":0.8060093244893806,"ci_75_upper":0.8968122829847476,"ci_50_lower":0.828490338756648,"ci_50_upper":0.8749251303756063},"131072":{"avg_score":0.670428125377636,"min_score":0.010824313387274742,"max_score":1,"n_tests":103,"stddev":0.46167502551434453,"median_score":1,"q1_score":0.037863124161958694,"q3_score":1,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":1.9952199999999982,"total_cache_read_cost_bin":6.387199999999995,"total_completion_cost_bin":2.2224899999999987,"total_reasoning_cost_bin":0.9388799999999998,"total_cost_bin":11.54378999999999,"total_on_demand_input_cost_bin":65.86722,"ci_99_lower":0.5388776100059808,"ci_99_upper":0.7842434617691193,"ci_98_lower":0.56641182319017,"ci_98_upper":0.7732102302210495,"ci_95_lower":0.5836822114663325,"ci_95_upper":0.7553429331334344,"ci_90_lower":0.5943521250812427,"ci_90_upper":0.7454771714574359,"ci_75_lower":0.6142251989114758,"ci_75_upper":0.726399331850913,"ci_50_lower":0.6410267998631682,"ci_50_upper":0.6992669546890786},"262144":{"avg_score":0.6702835381836861,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.4606865321938275,"median_score":1,"q1_score":0.04102219268679619,"q3_score":1,"avg_token_efficiency":0.9824198103380458,"is_incomplete":false,"total_prompt_cost_bin":6.806214999999999,"total_cache_read_cost_bin":17.460224000000025,"total_completion_cost_bin":3.131519999999999,"total_reasoning_cost_bin":1.5387599999999992,"total_cost_bin":28.93671900000002,"total_on_demand_input_cost_bin":181.40845499999975,"ci_99_lower":0.5673152470851567,"ci_99_upper":0.7738336651263666,"ci_98_lower":0.5803104481733147,"ci_98_upper":0.7601487023511517,"ci_95_lower":0.588084351774206,"ci_95_upper":0.7457113547244023,"ci_90_lower":0.6078872436099416,"ci_90_upper":0.7334879454615004,"ci_75_lower":0.6225475170756922,"ci_75_upper":0.7175271661294929,"ci_50_lower":0.642664950844627,"ci_50_upper":0.6975705970742523},"524288":{"avg_score":0.5432402161344633,"min_score":0.003955174703150988,"max_score":1,"n_tests":236,"stddev":0.4822856138338036,"median_score":1,"q1_score":0.029317961540073156,"q3_score":1,"avg_token_efficiency":0.977478526922676,"is_incomplete":false,"total_prompt_cost_bin":26.310820000000025,"total_cache_read_cost_bin":118.21286399999929,"total_completion_cost_bin":5.7912599999999985,"total_reasoning_cost_bin":3.1853700000000003,"total_cost_bin":153.50031399999932,"total_on_demand_input_cost_bin":1209.1330300000016,"ci_99_lower":0.46146571165975775,"ci_99_upper":0.6180450892137951,"ci_98_lower":0.4685516634950675,"ci_98_upper":0.6100990980744788,"ci_95_lower":0.48316647050634887,"ci_95_upper":0.6036877501406707,"ci_90_lower":0.4909858762293509,"ci_90_upper":0.5939780806007763,"ci_75_lower":0.5071825778242043,"ci_75_upper":0.5799642875134526,"ci_50_lower":0.5216457282860913,"ci_50_upper":0.5646634876424194}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.5","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.000005,"completion":0.00003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T17:31:33.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-25T12:16:52.126Z"},{"model_slug":"anthropic/claude-opus-4.6","reasoning_mode":"high","overall_metrics":{"auc_1m":0.46946986589065726,"auc_128k":0.7683305880046378,"cum_avg_1m":0.6682532455490728,"cum_avg_128k":0.8224254643236476,"avg_score_overall":0.722628434536693,"avg_response_time_ms":44798.36121243467,"total_prompt_cost":7.9852300000000005,"total_cache_read_cost":103.72802849999987,"total_completion_cost":37.917049999999996,"total_reasoning_cost":12.587675000000003,"total_prompt_cost_128k":3.464639999999999,"total_completion_cost_128k":11.967699999999999,"total_on_demand_input_cost":1045.2655150000014,"total_on_demand_input_cost_128k":138.620425,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1311011541866511},"bin_metrics":{"8192":{"avg_score":0.9817371804267168,"min_score":0.09643349051475525,"max_score":1,"n_tests":80,"stddev":0.10088771968672706,"median_score":0.9971585273742676,"q1_score":0.9921955615282059,"q3_score":1,"avg_token_efficiency":1.1377505627360658,"is_incomplete":false,"total_prompt_cost_bin":1.1261749999999993,"total_cache_read_cost_bin":0.18940649999999998,"total_completion_cost_bin":1.4363999999999992,"total_reasoning_cost_bin":0.4789500000000002,"total_cost_bin":3.2309314999999987,"total_on_demand_input_cost_bin":3.0202400000000016,"ci_99_lower":0.9458770009763539,"ci_99_upper":0.9956937990710139,"ci_98_lower":0.9479235512986779,"ci_98_upper":0.9954639434888959,"ci_95_lower":0.9580525673553348,"ci_95_upper":0.9947078967653215,"ci_90_lower":0.9594840942323208,"ci_90_upper":0.9943040154501795,"ci_75_lower":0.9696617470122874,"ci_75_upper":0.9935107594355941,"ci_50_lower":0.9717765012755989,"ci_50_upper":0.9923915132880211},"16384":{"avg_score":0.8444463170890231,"min_score":0.005645730532705784,"max_score":1,"n_tests":80,"stddev":0.3463712554227494,"median_score":0.9943967759609222,"q1_score":0.9706265330314636,"q3_score":0.9986556768417358,"avg_token_efficiency":1.122757280298134,"is_incomplete":false,"total_prompt_cost_bin":0.4299300000000002,"total_cache_read_cost_bin":0.5764150000000002,"total_completion_cost_bin":1.6767,"total_reasoning_cost_bin":0.5850249999999999,"total_cost_bin":3.2680700000000007,"total_on_demand_input_cost_bin":6.194079999999996,"ci_99_lower":0.7347037617672759,"ci_99_upper":0.9408327831072384,"ci_98_lower":0.7482436046022922,"ci_98_upper":0.9282466010033386,"ci_95_lower":0.7630053389951355,"ci_95_upper":0.9162012133878306,"ci_90_lower":0.7745804852660513,"ci_90_upper":0.9041711859157658,"ci_75_lower":0.7965240364501369,"ci_75_upper":0.8894086398533546,"ci_50_lower":0.819807772434433,"ci_50_upper":0.8686105497123209},"32768":{"avg_score":0.8617721610747086,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.32461354891578753,"median_score":0.9936986565589905,"q1_score":0.9692837446928024,"q3_score":0.9998337030410767,"avg_token_efficiency":1.1449043855358678,"is_incomplete":false,"total_prompt_cost_bin":0.7607499999999998,"total_cache_read_cost_bin":2.1453210000000036,"total_completion_cost_bin":3.07515,"total_reasoning_cost_bin":1.0742000000000005,"total_cost_bin":7.0554210000000035,"total_on_demand_input_cost_bin":22.213959999999975,"ci_99_lower":0.7847958118643403,"ci_99_upper":0.9307515271680827,"ci_98_lower":0.795299624194843,"ci_98_upper":0.9239202830022516,"ci_95_lower":0.8044885065564491,"ci_95_upper":0.9136004027963174,"ci_90_lower":0.8143714255038716,"ci_90_upper":0.904753743621661,"ci_75_lower":0.8269273454508734,"ci_75_upper":0.893075339625605,"ci_50_lower":0.8413625990608982,"ci_50_upper":0.8814263817702714},"65536":{"avg_score":0.7427088094436947,"min_score":0.008833922445774078,"max_score":1,"n_tests":85,"stddev":0.4249544773805631,"median_score":0.9955320954322815,"q1_score":0.04968944191932678,"q3_score":0.9989455342292786,"avg_token_efficiency":1.1315791278603582,"is_incomplete":false,"total_prompt_cost_bin":0.3894899999999996,"total_cache_read_cost_bin":3.106824000000004,"total_completion_cost_bin":2.465925,"total_reasoning_cost_bin":0.8537500000000005,"total_cost_bin":6.8159890000000045,"total_on_demand_input_cost_bin":31.457729999999973,"ci_99_lower":0.6056403310239206,"ci_99_upper":0.8463921838866766,"ci_98_lower":0.6288652029872378,"ci_98_upper":0.8437403356172144,"ci_95_lower":0.6420688127189436,"ci_95_upper":0.8241024120220476,"ci_90_lower":0.6637590190012226,"ci_90_upper":0.8125540178510197,"ci_75_lower":0.6873591899159639,"ci_75_upper":0.7977217224974404,"ci_50_lower":0.7091094034997856,"ci_50_upper":0.775316979785395},"131072":{"avg_score":0.681462853584095,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.44806617698472556,"median_score":0.9824368953704834,"q1_score":0.03669251501560211,"q3_score":0.998795211315155,"avg_token_efficiency":1.1236718471186093,"is_incomplete":false,"total_prompt_cost_bin":0.758295,"total_cache_read_cost_bin":7.4976120000000055,"total_completion_cost_bin":3.3135250000000007,"total_reasoning_cost_bin":1.1867250000000005,"total_cost_bin":12.756157000000007,"total_on_demand_input_cost_bin":75.73441500000008,"ci_99_lower":0.559743557998117,"ci_99_upper":0.7929465956944582,"ci_98_lower":0.5772233637092744,"ci_98_upper":0.7750361383870217,"ci_95_lower":0.5905609809753409,"ci_95_upper":0.7625210589555191,"ci_90_lower":0.6065206737789886,"ci_90_upper":0.75473316653713,"ci_75_lower":0.6280557231794482,"ci_75_upper":0.7336634012047671,"ci_50_lower":0.6495420235860189,"ci_50_upper":0.7110254947402406},"262144":{"avg_score":0.690880147241558,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.44608023448799655,"median_score":0.992023229598999,"q1_score":0.0416666679084301,"q3_score":0.999459445476532,"avg_token_efficiency":1.1264169418592798,"is_incomplete":false,"total_prompt_cost_bin":1.5066050000000004,"total_cache_read_cost_bin":20.649159999999945,"total_completion_cost_bin":7.192849999999998,"total_reasoning_cost_bin":2.5331249999999996,"total_cost_bin":31.881739999999944,"total_on_demand_input_cost_bin":207.99820499999993,"ci_99_lower":0.5870313031977895,"ci_99_upper":0.7849488186758468,"ci_98_lower":0.5961271217749091,"ci_98_upper":0.7764025266704988,"ci_95_lower":0.6192562904838863,"ci_95_upper":0.7642124782604535,"ci_90_lower":0.6291631542316287,"ci_90_upper":0.7520547320248877,"ci_75_lower":0.6449296093613557,"ci_75_upper":0.7332501426382083,"ci_50_lower":0.6642302042294412,"ci_50_upper":0.7163581240877628},"524288":{"avg_score":0.5430184955327866,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.47131789011793623,"median_score":0.9407427906990051,"q1_score":0.03174971044063568,"q3_score":0.9979692548513412,"avg_token_efficiency":1.1295900633711309,"is_incomplete":false,"total_prompt_cost_bin":3.013985000000001,"total_cache_read_cost_bin":69.56328999999991,"total_completion_cost_bin":18.7565,"total_reasoning_cost_bin":5.8759000000000015,"total_cost_bin":97.20967499999992,"total_on_demand_input_cost_bin":698.6468850000015,"ci_99_lower":0.4634016889644889,"ci_99_upper":0.6154321540842254,"ci_98_lower":0.4693377531518256,"ci_98_upper":0.6108395903310305,"ci_95_lower":0.4827056842663728,"ci_95_upper":0.60359467843839,"ci_90_lower":0.49216627765183485,"ci_90_upper":0.5928065546135166,"ci_75_lower":0.5081592622775164,"ci_75_upper":0.5789885589506644,"ci_50_lower":0.522114650809287,"ci_50_upper":0.563652363840601}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-04T15:30:50.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T02:54:21.434Z"},{"model_slug":"anthropic/claude-opus-4.6","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.461984861990706,"auc_128k":0.8100559225665265,"cum_avg_1m":0.6863586871343588,"cum_avg_128k":0.8566578267830955,"avg_score_overall":0.7305917048755964,"avg_response_time_ms":38694.042232925465,"total_prompt_cost":8.409135,"total_cache_read_cost":103.68563799999987,"total_completion_cost":36.99060000000001,"total_reasoning_cost":12.465624999999998,"total_prompt_cost_128k":3.888544999999998,"total_completion_cost_128k":11.618875000000001,"total_on_demand_input_cost":1045.2655150000019,"total_on_demand_input_cost_128k":138.620425,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.131101154186651},"bin_metrics":{"8192":{"avg_score":0.9930317617952824,"min_score":0.9468473196029663,"max_score":1,"n_tests":80,"stddev":0.011503289814064335,"median_score":0.9974396526813507,"q1_score":0.99247607588768,"q3_score":1,"avg_token_efficiency":1.1377505627360658,"is_incomplete":false,"total_prompt_cost_bin":1.233494999999999,"total_cache_read_cost_bin":0.17867449999999999,"total_completion_cost_bin":1.4903249999999995,"total_reasoning_cost_bin":0.5243000000000002,"total_cost_bin":3.426794499999999,"total_on_demand_input_cost_bin":3.0202400000000016,"ci_99_lower":0.9888791979774834,"ci_99_upper":0.9960807051174343,"ci_98_lower":0.989218166694045,"ci_98_upper":0.9958184349909425,"ci_95_lower":0.9903490461781621,"ci_95_upper":0.9953484713099896,"ci_90_lower":0.9907836832478643,"ci_90_upper":0.9950132907554506,"ci_75_lower":0.9914189880713821,"ci_75_upper":0.9944017673842608,"ci_50_lower":0.9921715682372452,"ci_50_upper":0.993868718110025},"16384":{"avg_score":0.8919055021367968,"min_score":0.005645730532705784,"max_score":1,"n_tests":80,"stddev":0.2902056654976717,"median_score":0.9941722452640533,"q1_score":0.9735265970230103,"q3_score":0.9987180382013321,"avg_token_efficiency":1.122757280298134,"is_incomplete":false,"total_prompt_cost_bin":0.7465149999999997,"total_cache_read_cost_bin":0.5447565000000001,"total_completion_cost_bin":1.6249,"total_reasoning_cost_bin":0.5859999999999999,"total_cost_bin":3.5021714999999998,"total_on_demand_input_cost_bin":6.194079999999995,"ci_99_lower":0.8044229952284367,"ci_99_upper":0.9650086624628748,"ci_98_lower":0.8060850586941233,"ci_98_upper":0.9549282376063056,"ci_95_lower":0.8200920945301187,"ci_95_upper":0.9480970545826131,"ci_90_lower":0.8306372930551879,"ci_90_upper":0.9400202663487289,"ci_75_lower":0.854292702281964,"ci_75_upper":0.9272088982048444,"ci_50_lower":0.8685620298434514,"ci_50_upper":0.9144433953944827},"32768":{"avg_score":0.8744879168927369,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.31233568849001114,"median_score":0.9939269125461578,"q1_score":0.9707318246364594,"q3_score":1,"avg_token_efficiency":1.1449043855358674,"is_incomplete":false,"total_prompt_cost_bin":0.7607499999999999,"total_cache_read_cost_bin":2.145321000000003,"total_completion_cost_bin":3.0849499999999987,"total_reasoning_cost_bin":1.045925,"total_cost_bin":7.036946000000002,"total_on_demand_input_cost_bin":22.213959999999975,"ci_99_lower":0.8018586290696967,"ci_99_upper":0.9333118680014055,"ci_98_lower":0.804096925333673,"ci_98_upper":0.9318689238393735,"ci_95_lower":0.8178854474142152,"ci_95_upper":0.924869072908426,"ci_90_lower":0.8310905347251236,"ci_90_upper":0.9170633295105826,"ci_75_lower":0.8405775225694336,"ci_75_upper":0.9046204747886376,"ci_50_lower":0.8539984235403073,"ci_50_upper":0.8938915673193952},"65536":{"avg_score":0.7987725706771016,"min_score":0.008833922445774078,"max_score":1,"n_tests":85,"stddev":0.3889584878003877,"median_score":0.9968017339706421,"q1_score":0.9694610834121704,"q3_score":0.9992172122001648,"avg_token_efficiency":1.1315791278603584,"is_incomplete":false,"total_prompt_cost_bin":0.38948999999999956,"total_cache_read_cost_bin":3.106824000000004,"total_completion_cost_bin":2.1041000000000003,"total_reasoning_cost_bin":0.7832499999999997,"total_cost_bin":6.383664000000004,"total_on_demand_input_cost_bin":31.457729999999973,"ci_99_lower":0.6834604860733756,"ci_99_upper":0.8998108324333477,"ci_98_lower":0.685031243112486,"ci_98_upper":0.8907583585423144,"ci_95_lower":0.7073230715348001,"ci_95_upper":0.8780265471859671,"ci_90_lower":0.7208686257602975,"ci_90_upper":0.8668133320675835,"ci_75_lower":0.7434298153364045,"ci_75_upper":0.8446543489815668,"ci_50_lower":0.7736124270294301,"ci_50_upper":0.8309040610651102},"131072":{"avg_score":0.7250913824135601,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4299711515438539,"median_score":0.9888429641723633,"q1_score":0.04065926745533943,"q3_score":0.9990157186985016,"avg_token_efficiency":1.1236718471186091,"is_incomplete":false,"total_prompt_cost_bin":0.758295,"total_cache_read_cost_bin":7.4976120000000055,"total_completion_cost_bin":3.314600000000002,"total_reasoning_cost_bin":1.1682,"total_cost_bin":12.738707000000009,"total_on_demand_input_cost_bin":75.73441500000008,"ci_99_lower":0.6055989616599106,"ci_99_upper":0.8370716520411063,"ci_98_lower":0.6156159060903671,"ci_98_upper":0.8199837910498773,"ci_95_lower":0.6320491489554186,"ci_95_upper":0.8070639269534968,"ci_90_lower":0.64982800501132,"ci_90_upper":0.7936476015047287,"ci_75_lower":0.6755729994077679,"ci_75_upper":0.7734864739157503,"ci_50_lower":0.6913978908249421,"ci_50_upper":0.7537885747094937},"262144":{"avg_score":0.7184729106932324,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.43384476043081865,"median_score":0.9930986762046814,"q1_score":0.048882681876420975,"q3_score":0.9994535446166992,"avg_token_efficiency":1.1264169418592798,"is_incomplete":false,"total_prompt_cost_bin":1.5066050000000004,"total_cache_read_cost_bin":20.649159999999945,"total_completion_cost_bin":6.526349999999998,"total_reasoning_cost_bin":2.3281,"total_cost_bin":31.010214999999942,"total_on_demand_input_cost_bin":207.998205,"ci_99_lower":0.6105965251287491,"ci_99_upper":0.8144102458319633,"ci_98_lower":0.629483794379015,"ci_98_upper":0.8007008841294129,"ci_95_lower":0.6419813953268223,"ci_95_upper":0.788010887795331,"ci_90_lower":0.6527726572734329,"ci_90_upper":0.7789271358985137,"ci_75_lower":0.6765052381077706,"ci_75_upper":0.7610662132823602,"ci_50_lower":0.6916980201304983,"ci_50_upper":0.744032919714442},"524288":{"avg_score":0.4891074524661605,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.47691492341766284,"median_score":0.11445221677422523,"q1_score":0.0261663431301713,"q3_score":0.9970346689224243,"avg_token_efficiency":1.1295900633711315,"is_incomplete":false,"total_prompt_cost_bin":3.013985000000001,"total_cache_read_cost_bin":69.56328999999991,"total_completion_cost_bin":18.845375000000008,"total_reasoning_cost_bin":6.029849999999998,"total_cost_bin":97.45249999999992,"total_on_demand_input_cost_bin":698.6468850000018,"ci_99_lower":0.40778823950152326,"ci_99_upper":0.5609740251243327,"ci_98_lower":0.4129340064094177,"ci_98_upper":0.5558344256532994,"ci_95_lower":0.4280608143061557,"ci_95_upper":0.5512781065275549,"ci_90_lower":0.4378359663239049,"ci_90_upper":0.5418926359818247,"ci_75_lower":0.45335028599849037,"ci_75_upper":0.5249821191106803,"ci_50_lower":0.4680808230100488,"ci_50_upper":0.5097455472934013}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-04T15:30:50.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T04:56:58.349Z"},{"model_slug":"anthropic/claude-opus-5","reasoning_mode":"max","overall_metrics":{"auc_1m":0.4574079371517476,"auc_128k":0.9746580090990594,"cum_avg_1m":0.7467001368637713,"cum_avg_128k":0.9783899606394234,"avg_score_overall":0.773349898567685,"avg_response_time_ms":120730.69670588795,"total_prompt_cost":64.79708999999994,"total_cache_read_cost":145.0869094999997,"total_completion_cost":252.72735000000003,"total_reasoning_cost":47.40499999999999,"total_prompt_cost_128k":4.596204999999999,"total_completion_cost_128k":50.891000000000005,"total_on_demand_input_cost":1515.6661849999996,"total_on_demand_input_cost_128k":200.15950000000004,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.63760612874201},"bin_metrics":{"8192":{"avg_score":0.9873868629336358,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.11179104353552688,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.6722716647051485,"is_incomplete":false,"total_prompt_cost_bin":0.6736600000000003,"total_cache_read_cost_bin":0.3767439999999997,"total_completion_cost_bin":1.9863250000000006,"total_reasoning_cost_bin":0.11354999999999998,"total_cost_bin":3.1502790000000003,"total_on_demand_input_cost_bin":4.441099999999999,"ci_99_lower":0.9498332954794169,"ci_99_upper":0.9999565069638192,"ci_98_lower":0.9498725859299303,"ci_98_upper":0.999951165355742,"ci_95_lower":0.9623558555915952,"ci_95_upper":0.9999336748756468,"ci_90_lower":0.9623996278643607,"ci_90_upper":0.9999226385354996,"ci_75_lower":0.9748651464469731,"ci_75_upper":0.9999013640917838,"ci_50_lower":0.9749243635684252,"ci_50_upper":0.9998699428513647},"16384":{"avg_score":0.9993443578481674,"min_score":0.9815893769264221,"max_score":1,"n_tests":80,"stddev":0.0028130561606044,"median_score":1,"q1_score":0.9995357692241669,"q3_score":1,"avg_token_efficiency":1.5830137761150538,"is_incomplete":false,"total_prompt_cost_bin":0.5135900000000002,"total_cache_read_cost_bin":0.8197870000000005,"total_completion_cost_bin":2.652650000000001,"total_reasoning_cost_bin":0.18237499999999998,"total_cost_bin":4.168402000000002,"total_on_demand_input_cost_bin":8.711460000000006,"ci_99_lower":0.9983749145194888,"ci_99_upper":0.9998667763061821,"ci_98_lower":0.9984595460221172,"ci_98_upper":0.9998574936315416,"ci_95_lower":0.9986355397850275,"ci_95_upper":0.999824801683426,"ci_90_lower":0.9987163298204541,"ci_90_upper":0.999797315709293,"ci_75_lower":0.9989355562254787,"ci_75_upper":0.999715375341475,"ci_50_lower":0.9991402763873338,"ci_50_upper":0.9995670706033708},"32768":{"avg_score":0.9926172171664589,"min_score":0.019837692379951477,"max_score":1,"n_tests":136,"stddev":0.08403969094453936,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.660794921666312,"is_incomplete":false,"total_prompt_cost_bin":0.7020049999999989,"total_cache_read_cost_bin":3.1567765000000008,"total_completion_cost_bin":8.342900000000007,"total_reasoning_cost_bin":1.0588249999999997,"total_cost_bin":13.260506500000005,"total_on_demand_input_cost_bin":32.269769999999994,"ci_99_lower":0.9709007768354871,"ci_99_upper":0.9999519786304412,"ci_98_lower":0.9709802190345875,"ci_98_upper":0.9999422909932978,"ci_95_lower":0.9781149928512819,"ci_95_upper":0.9999307298594539,"ci_90_lower":0.978225568528561,"ci_90_upper":0.9999142050085699,"ci_75_lower":0.9853374909280854,"ci_75_upper":0.9998683907529887,"ci_50_lower":0.98551074470229,"ci_50_upper":0.9997972577591152},"65536":{"avg_score":0.9994515278760124,"min_score":0.9776711463928223,"max_score":1,"n_tests":85,"stddev":0.003203924460870609,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.6412164101683422,"is_incomplete":false,"total_prompt_cost_bin":0.56041,"total_cache_read_cost_bin":4.506516000000003,"total_completion_cost_bin":7.355549999999999,"total_reasoning_cost_bin":1.0497749999999995,"total_cost_bin":13.472251000000004,"total_on_demand_input_cost_bin":45.62557,"ci_99_lower":0.9984490899478689,"ci_99_upper":0.9999790985408952,"ci_98_lower":0.9985319129929823,"ci_98_upper":0.999972208941684,"ci_95_lower":0.9986801277013386,"ci_95_upper":0.9999636556997018,"ci_90_lower":0.9987400123301674,"ci_90_upper":0.9999496481348488,"ci_75_lower":0.9989758262739461,"ci_75_upper":0.9998983139500899,"ci_50_lower":0.999213158383089,"ci_50_upper":0.9997055383289561},"131072":{"avg_score":0.9131498373728426,"min_score":0.00709534389898181,"max_score":1,"n_tests":103,"stddev":0.27749864258702656,"median_score":1,"q1_score":0.9996056258678436,"q3_score":1,"avg_token_efficiency":1.6188892872031542,"is_incomplete":false,"total_prompt_cost_bin":2.1465399999999994,"total_cache_read_cost_bin":10.696505999999982,"total_completion_cost_bin":30.553575,"total_reasoning_cost_bin":5.751374999999999,"total_cost_bin":49.14799599999998,"total_on_demand_input_cost_bin":109.11160000000005,"ci_99_lower":0.8279660913259361,"ci_99_upper":0.970272581876658,"ci_98_lower":0.8370959302185338,"ci_98_upper":0.9696462251443091,"ci_95_lower":0.8557216205241981,"ci_95_upper":0.9607647611248732,"ci_90_lower":0.865270388782142,"ci_90_upper":0.9518939396065831,"ci_75_lower":0.8760656767786996,"ci_75_upper":0.942042192663473,"ci_50_lower":0.8943887129154743,"ci_50_upper":0.9321557686006362},"262144":{"avg_score":0.6563967841426372,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4614539294528857,"median_score":0.9995178580284119,"q1_score":0.03442136570811272,"q3_score":1,"avg_token_efficiency":1.6370585869996066,"is_incomplete":false,"total_prompt_cost_bin":4.326025,"total_cache_read_cost_bin":29.796456999999933,"total_completion_cost_bin":71.53994999999999,"total_reasoning_cost_bin":14.491575000000001,"total_cost_bin":120.15400699999992,"total_on_demand_input_cost_bin":302.2905949999998,"ci_99_lower":0.5450517862400119,"ci_99_upper":0.7558053842695687,"ci_98_lower":0.5641457027125879,"ci_98_upper":0.7443026052791532,"ci_95_lower":0.576852973134683,"ci_95_upper":0.7298508609126566,"ci_90_lower":0.591491407389812,"ci_90_upper":0.7195554042925252,"ci_75_lower":0.6103886230035472,"ci_75_upper":0.7019064552092872,"ci_50_lower":0.6280813182238489,"ci_50_upper":0.6847958383191681},"524288":{"avg_score":0.425254507570417,"min_score":0.003955174703150988,"max_score":1,"n_tests":236,"stddev":0.46833571107853683,"median_score":0.06056507118046284,"q1_score":0.02638146886602044,"q3_score":0.9997258484363556,"avg_token_efficiency":1.6381935594129562,"is_incomplete":false,"total_prompt_cost_bin":55.874859999999934,"total_cache_read_cost_bin":95.7341229999998,"total_completion_cost_bin":130.29640000000003,"total_reasoning_cost_bin":24.75752499999999,"total_cost_bin":306.6629079999998,"total_on_demand_input_cost_bin":1013.2160899999997,"ci_99_lower":0.33717157094441436,"ci_99_upper":0.4983278633325195,"ci_98_lower":0.35051718493414474,"ci_98_upper":0.4923946642798521,"ci_95_lower":0.3629638582029009,"ci_95_upper":0.48369706744654695,"ci_90_lower":0.37199449014615155,"ci_90_upper":0.47668884061898453,"ci_75_lower":0.3899906915368343,"ci_75_upper":0.4612378943832374,"ci_50_lower":0.4066404273133662,"ci_50_upper":0.4451989714047374}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Claude Opus 5","provider_name":"Anthropic","run_provider_names":["Claude Platform on AWS"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-24T17:02:24.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T00:25:09.571Z"},{"model_slug":"anthropic/claude-opus-4.6","reasoning_mode":"low","overall_metrics":{"auc_1m":0.45568872901502416,"auc_128k":0.8014280297221215,"cum_avg_1m":0.6795880317961295,"cum_avg_128k":0.850775545571594,"avg_score_overall":0.7243370033555062,"avg_response_time_ms":39992.6364312255,"total_prompt_cost":49.685335000000094,"total_cache_read_cost":99.55801799999988,"total_completion_cost":38.214950000000016,"total_reasoning_cost":12.546800000000001,"total_prompt_cost_128k":3.7227849999999982,"total_completion_cost_128k":12.012725,"total_on_demand_input_cost":1045.2655150000014,"total_on_demand_input_cost_128k":138.620425,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1311011541866514},"bin_metrics":{"8192":{"avg_score":0.9930317617952824,"min_score":0.9468473196029663,"max_score":1,"n_tests":80,"stddev":0.011503289814064335,"median_score":0.9974396526813507,"q1_score":0.99247607588768,"q3_score":1,"avg_token_efficiency":1.1377505627360658,"is_incomplete":false,"total_prompt_cost_bin":1.2695599999999987,"total_cache_read_cost_bin":0.175068,"total_completion_cost_bin":1.4653499999999997,"total_reasoning_cost_bin":0.5092750000000001,"total_cost_bin":3.4192529999999985,"total_on_demand_input_cost_bin":3.0202400000000016,"ci_99_lower":0.9888791979774834,"ci_99_upper":0.9960807051174343,"ci_98_lower":0.989218166694045,"ci_98_upper":0.9958184349909425,"ci_95_lower":0.9903490461781621,"ci_95_upper":0.9953484713099896,"ci_90_lower":0.9907836832478643,"ci_90_upper":0.9950132907554506,"ci_75_lower":0.9914189880713821,"ci_75_upper":0.9944017673842608,"ci_50_lower":0.9921715682372452,"ci_50_upper":0.993868718110025},"16384":{"avg_score":0.892796691285912,"min_score":0.005645730532705784,"max_score":1,"n_tests":80,"stddev":0.2902301417381284,"median_score":0.9950683414936066,"q1_score":0.9735265970230103,"q3_score":0.9987180382013321,"avg_token_efficiency":1.122757280298134,"is_incomplete":false,"total_prompt_cost_bin":0.6801199999999997,"total_cache_read_cost_bin":0.551396,"total_completion_cost_bin":1.6202000000000003,"total_reasoning_cost_bin":0.58535,"total_cost_bin":3.4370659999999997,"total_on_demand_input_cost_bin":6.194079999999995,"ci_99_lower":0.8053891911914689,"ci_99_upper":0.9659805254577076,"ci_98_lower":0.8071052014708986,"ci_98_upper":0.9555969596337527,"ci_95_lower":0.8209449894088903,"ci_95_upper":0.9493896296957973,"ci_90_lower":0.8316111071448541,"ci_90_upper":0.9408416681393282,"ci_75_lower":0.8550732947638607,"ci_75_upper":0.928188317657623,"ci_50_lower":0.869332088952069,"ci_50_upper":0.9154708275193115},"32768":{"avg_score":0.8823442611798836,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.3039038164614198,"median_score":0.9945562481880188,"q1_score":0.9731056243181229,"q3_score":1,"avg_token_efficiency":1.1449043855358678,"is_incomplete":false,"total_prompt_cost_bin":0.6253199999999999,"total_cache_read_cost_bin":2.1588640000000026,"total_completion_cost_bin":3.546049999999999,"total_reasoning_cost_bin":1.2222999999999995,"total_cost_bin":7.5525340000000005,"total_on_demand_input_cost_bin":22.21395999999997,"ci_99_lower":0.8115217757073697,"ci_99_upper":0.9440694559060006,"ci_98_lower":0.8184630021634033,"ci_98_upper":0.9383072142338599,"ci_95_lower":0.8263972036595292,"ci_95_upper":0.9318908359872533,"ci_90_lower":0.8382614957134458,"ci_90_upper":0.9245222650362206,"ci_75_lower":0.8519832079787996,"ci_75_upper":0.9115669974859938,"ci_50_lower":0.8628514121438082,"ci_50_upper":0.9021536314561245},"65536":{"avg_score":0.7979283450390486,"min_score":0.008833922445774078,"max_score":1,"n_tests":85,"stddev":0.3879682967336661,"median_score":0.9955320954322815,"q1_score":0.9629040956497192,"q3_score":0.9991452693939209,"avg_token_efficiency":1.1315791278603584,"is_incomplete":false,"total_prompt_cost_bin":0.3894899999999996,"total_cache_read_cost_bin":3.106824000000004,"total_completion_cost_bin":2.129150000000001,"total_reasoning_cost_bin":0.7697500000000005,"total_cost_bin":6.395214000000005,"total_on_demand_input_cost_bin":31.457729999999973,"ci_99_lower":0.6828055076502154,"ci_99_upper":0.8981874916964924,"ci_98_lower":0.6846806978299557,"ci_98_upper":0.8900682384009089,"ci_95_lower":0.7064025992357775,"ci_95_upper":0.8772069215747145,"ci_90_lower":0.7205735782763976,"ci_90_upper":0.8657654805382823,"ci_75_lower":0.7429729075266924,"ci_75_upper":0.8439062679337119,"ci_50_lower":0.7725358163992709,"ci_50_upper":0.8296473290606896},"131072":{"avg_score":0.6877766685578429,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4460301237531676,"median_score":0.9824368953704834,"q1_score":0.03743759170174599,"q3_score":0.9986220598220825,"avg_token_efficiency":1.1236718471186093,"is_incomplete":false,"total_prompt_cost_bin":0.7582950000000002,"total_cache_read_cost_bin":7.4976120000000055,"total_completion_cost_bin":3.2519750000000007,"total_reasoning_cost_bin":1.1694500000000003,"total_cost_bin":12.677332000000007,"total_on_demand_input_cost_bin":75.73441500000007,"ci_99_lower":0.5738749011612533,"ci_99_upper":0.7990805064019684,"ci_98_lower":0.5842923064630714,"ci_98_upper":0.7862901024488298,"ci_95_lower":0.5953753211715224,"ci_95_upper":0.7659458223048787,"ci_90_lower":0.6130916904719113,"ci_90_upper":0.7565631116552785,"ci_75_lower":0.6334938775703935,"ci_75_upper":0.7369816404726646,"ci_50_lower":0.6581535386803189,"ci_50_upper":0.7170189753122791},"262144":{"avg_score":0.6844516034132359,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4490521059825686,"median_score":0.9916805028915405,"q1_score":0.04037326201796532,"q3_score":0.9989423751831055,"avg_token_efficiency":1.12641694185928,"is_incomplete":false,"total_prompt_cost_bin":1.5066050000000009,"total_cache_read_cost_bin":20.649159999999945,"total_completion_cost_bin":6.770349999999999,"total_reasoning_cost_bin":2.315275,"total_cost_bin":31.241389999999946,"total_on_demand_input_cost_bin":207.99820499999996,"ci_99_lower":0.5781921755692075,"ci_99_upper":0.7783588562591219,"ci_98_lower":0.5888317444420862,"ci_98_upper":0.7720587030321473,"ci_95_lower":0.60805018930762,"ci_95_upper":0.7572717546978686,"ci_90_lower":0.622570690025364,"ci_90_upper":0.7465564342784523,"ci_75_lower":0.6404114006073148,"ci_75_upper":0.7273939531339837,"ci_50_lower":0.657201970282592,"ci_50_upper":0.7114389280038093},"524288":{"avg_score":0.49837492309783,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.47384021653347663,"median_score":0.1413515880703926,"q1_score":0.030454897321760654,"q3_score":0.9972477853298187,"avg_token_efficiency":1.1295900633711313,"is_incomplete":false,"total_prompt_cost_bin":44.45594500000009,"total_cache_read_cost_bin":65.41909399999992,"total_completion_cost_bin":19.431875000000016,"total_reasoning_cost_bin":5.9754,"total_cost_bin":135.28231400000004,"total_on_demand_input_cost_bin":698.6468850000014,"ci_99_lower":0.4177909515223316,"ci_99_upper":0.5708865306213926,"ci_98_lower":0.4238703170847108,"ci_98_upper":0.5625793659839763,"ci_95_lower":0.4370963269422646,"ci_95_upper":0.5581586015619874,"ci_90_lower":0.4481960535778514,"ci_90_upper":0.5515646023348751,"ci_75_lower":0.4627055479392411,"ci_75_upper":0.5345639167238821,"ci_50_lower":0.4773380668085045,"ci_50_upper":0.5192119664462259}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-04T15:30:50.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T03:46:47.703Z"},{"model_slug":"anthropic/claude-sonnet-4.6","reasoning_mode":"high","overall_metrics":{"auc_1m":0.4441323870639113,"auc_128k":0.795092171188418,"cum_avg_1m":0.6636338229096712,"cum_avg_128k":0.8313021080209234,"avg_score_overall":0.7049701870954515,"avg_response_time_ms":47562.98425704123,"total_prompt_cost":4.456484999999999,"total_cache_read_cost":62.27028239999993,"total_completion_cost":25.90803000000001,"total_reasoning_cost":8.026350000000003,"total_prompt_cost_128k":1.7441310000000005,"total_completion_cost_128k":7.418220000000001,"total_on_demand_input_cost":627.1593090000009,"total_on_demand_input_cost_128k":83.17225499999995,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.131101154186651},"bin_metrics":{"8192":{"avg_score":0.9672270056791603,"min_score":0.022883296012878418,"max_score":1,"n_tests":80,"stddev":0.148169252800693,"median_score":0.9968799948692322,"q1_score":0.9903403222560883,"q3_score":1,"avg_token_efficiency":1.1377505627360658,"is_incomplete":false,"total_prompt_cost_bin":0.41764800000000024,"total_cache_read_cost_bin":0.13944959999999998,"total_completion_cost_bin":0.9498450000000003,"total_reasoning_cost_bin":0.3143549999999999,"total_cost_bin":1.8212976000000003,"total_on_demand_input_cost_bin":1.8121439999999982,"ci_99_lower":0.9161194903282448,"ci_99_upper":0.9943964977189897,"ci_98_lower":0.9203680053045973,"ci_98_upper":0.9936122198551892,"ci_95_lower":0.9301988538424485,"ci_95_upper":0.9925989686883987,"ci_90_lower":0.9343037877324969,"ci_90_upper":0.9913841125369072,"ci_75_lower":0.945195158815477,"ci_75_upper":0.986158371064812,"ci_50_lower":0.956525512970984,"ci_50_upper":0.9790844222297891},"16384":{"avg_score":0.8201178086455911,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.3689684876242362,"median_score":0.9933691620826721,"q1_score":0.9697859734296799,"q3_score":0.9984362870454788,"avg_token_efficiency":1.122757280298134,"is_incomplete":false,"total_prompt_cost_bin":0.18136200000000002,"total_cache_read_cost_bin":0.35350860000000006,"total_completion_cost_bin":1.3950600000000004,"total_reasoning_cost_bin":0.4310550000000001,"total_cost_bin":2.3609856000000007,"total_on_demand_input_cost_bin":3.7164480000000055,"ci_99_lower":0.7002777986075961,"ci_99_upper":0.9193253299486823,"ci_98_lower":0.7110669809803367,"ci_98_upper":0.9076651599416509,"ci_95_lower":0.7347288184193894,"ci_95_upper":0.8944707507820566,"ci_90_lower":0.7482762170617934,"ci_90_upper":0.8821888483408838,"ci_75_lower":0.7718335760262562,"ci_75_upper":0.8661890286864946,"ci_50_lower":0.7944834788795562,"ci_50_upper":0.8456998372741509},"32768":{"avg_score":0.8515271310239126,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.3318795712388633,"median_score":0.9924105107784271,"q1_score":0.9656716585159302,"q3_score":0.9994787722826004,"avg_token_efficiency":1.1449043855358678,"is_incomplete":false,"total_prompt_cost_bin":0.4564500000000003,"total_cache_read_cost_bin":1.2871925999999994,"total_completion_cost_bin":1.9054499999999996,"total_reasoning_cost_bin":0.6782099999999999,"total_cost_bin":4.327302599999999,"total_on_demand_input_cost_bin":13.328376,"ci_99_lower":0.7774033211042971,"ci_99_upper":0.9201046606918852,"ci_98_lower":0.7828183515996653,"ci_98_upper":0.9182872368582963,"ci_95_lower":0.7914017464237986,"ci_95_upper":0.904813772163727,"ci_90_lower":0.8015633156675476,"ci_90_upper":0.896833854695176,"ci_75_lower":0.8161460440678638,"ci_75_upper":0.883541613281362,"ci_50_lower":0.8315076115755159,"ci_50_upper":0.8695089341982727},"65536":{"avg_score":0.7937282899613766,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.3894679768421117,"median_score":0.9935934543609619,"q1_score":0.9539285898208618,"q3_score":0.9989455342292786,"avg_token_efficiency":1.131579127860358,"is_incomplete":false,"total_prompt_cost_bin":0.233694,"total_cache_read_cost_bin":1.8640943999999986,"total_completion_cost_bin":1.2528,"total_reasoning_cost_bin":0.450585,"total_cost_bin":3.801173399999999,"total_on_demand_input_cost_bin":18.874637999999997,"ci_99_lower":0.6772226650845259,"ci_99_upper":0.8941509571587667,"ci_98_lower":0.6809474769840564,"ci_98_upper":0.886309825967559,"ci_95_lower":0.701065011232927,"ci_95_upper":0.872716797710232,"ci_90_lower":0.7165138126641293,"ci_90_upper":0.8611493063444162,"ci_75_lower":0.7393362926798599,"ci_75_upper":0.84013882021091,"ci_50_lower":0.7667828840176192,"ci_50_upper":0.8249677338966113},"131072":{"avg_score":0.7239103047945763,"min_score":0.0057299453765153885,"max_score":1,"n_tests":103,"stddev":0.43070091269038907,"median_score":0.9857470989227295,"q1_score":0.042588481679558754,"q3_score":0.9993852376937866,"avg_token_efficiency":1.1236718471186093,"is_incomplete":false,"total_prompt_cost_bin":0.454977,"total_cache_read_cost_bin":4.498567200000001,"total_completion_cost_bin":1.9150650000000005,"total_reasoning_cost_bin":0.6615000000000001,"total_cost_bin":7.530109200000002,"total_on_demand_input_cost_bin":45.440648999999944,"ci_99_lower":0.6040190404871234,"ci_99_upper":0.8359206007496064,"ci_98_lower":0.6141196176866793,"ci_98_upper":0.8186817448488573,"ci_95_lower":0.6305910954401078,"ci_95_upper":0.8057117526652292,"ci_90_lower":0.6485380728715506,"ci_90_upper":0.7926833723011671,"ci_75_lower":0.674335500580322,"ci_75_upper":0.7720605198952858,"ci_50_lower":0.6903017437915109,"ci_50_upper":0.7525434353302947},"262144":{"avg_score":0.6931555800565935,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.44743339585707065,"median_score":0.9937343597412109,"q1_score":0.04595300182700157,"q3_score":0.9996576309204102,"avg_token_efficiency":1.1264169418592793,"is_incomplete":false,"total_prompt_cost_bin":0.9039629999999998,"total_cache_read_cost_bin":12.389496000000031,"total_completion_cost_bin":2.8776750000000013,"total_reasoning_cost_bin":1.1087550000000004,"total_cost_bin":17.279889000000033,"total_on_demand_input_cost_bin":124.79892300000007,"ci_99_lower":0.5889620268384015,"ci_99_upper":0.7874378631578458,"ci_98_lower":0.5984200524509197,"ci_98_upper":0.7788351775509642,"ci_95_lower":0.6214287920438036,"ci_95_upper":0.7674784355093841,"ci_90_lower":0.6310815335987037,"ci_90_upper":0.7542848510930235,"ci_75_lower":0.6465976655287982,"ci_75_upper":0.7356808261362942,"ci_50_lower":0.6665648178239552,"ci_50_upper":0.7191451461279784},"524288":{"avg_score":0.4594044631161585,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4699627876107834,"median_score":0.11870478466153145,"q1_score":0.024878743570297956,"q3_score":0.9954227209091187,"avg_token_efficiency":1.129590063371131,"is_incomplete":false,"total_prompt_cost_bin":1.808390999999999,"total_cache_read_cost_bin":41.7379739999999,"total_completion_cost_bin":15.612135000000007,"total_reasoning_cost_bin":4.381890000000002,"total_cost_bin":63.54038999999992,"total_on_demand_input_cost_bin":419.1881310000009,"ci_99_lower":0.37943318199005244,"ci_99_upper":0.5317975414952587,"ci_98_lower":0.38662460686502803,"ci_98_upper":0.5271685370176407,"ci_95_lower":0.3966127807743575,"ci_95_upper":0.5202064326232776,"ci_90_lower":0.40730664715687825,"ci_90_upper":0.5104584009445793,"ci_75_lower":0.42292422189138495,"ci_75_upper":0.49394074827777934,"ci_50_lower":0.4398113049342794,"ci_50_upper":0.4776686841466117}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Anthropic: Claude Sonnet 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000003,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-17T15:43:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T03:02:28.477Z"},{"model_slug":"anthropic/claude-sonnet-4.6","reasoning_mode":"low","overall_metrics":{"auc_1m":0.4417150838667061,"auc_128k":0.7974800178455447,"cum_avg_1m":0.6640402244706173,"cum_avg_128k":0.830062035588192,"avg_score_overall":0.7037599304659589,"avg_response_time_ms":41963.64557841763,"total_prompt_cost":8.084475000000005,"total_cache_read_cost":61.72986059999994,"total_completion_cost":25.684215000000002,"total_reasoning_cost":8.270069999999997,"total_prompt_cost_128k":1.8200970000000005,"total_completion_cost_128k":7.313085,"total_on_demand_input_cost":625.383081000001,"total_on_demand_input_cost_128k":83.17225499999995,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1311011511418312},"bin_metrics":{"8192":{"avg_score":0.9477776915533468,"min_score":0.02108139730989933,"max_score":1,"n_tests":80,"stddev":0.2020683342442148,"median_score":0.9971585273742676,"q1_score":0.9921955615282059,"q3_score":1,"avg_token_efficiency":1.1377505627360662,"is_incomplete":false,"total_prompt_cost_bin":0.3849210000000002,"total_cache_read_cost_bin":0.1427223,"total_completion_cost_bin":0.9395400000000003,"total_reasoning_cost_bin":0.31123499999999993,"total_cost_bin":1.7784183000000002,"total_on_demand_input_cost_bin":1.8121439999999982,"ci_99_lower":0.8793635732174153,"ci_99_upper":0.9940605010762811,"ci_98_lower":0.8848591516022571,"ci_98_upper":0.9931782163381576,"ci_95_lower":0.8957577854051487,"ci_95_upper":0.9848041104711591,"ci_90_lower":0.9028670797741506,"ci_90_upper":0.982104624689091,"ci_75_lower":0.9220799681410426,"ci_75_upper":0.9719285645522178,"ci_50_lower":0.9336816434806677,"ci_50_upper":0.961588909913553},"16384":{"avg_score":0.8223612703557592,"min_score":0.003857280593365431,"max_score":1,"n_tests":80,"stddev":0.36857318395636357,"median_score":0.9937342703342438,"q1_score":0.9720740169286728,"q3_score":0.9986556768417358,"avg_token_efficiency":1.1227572802981338,"is_incomplete":false,"total_prompt_cost_bin":0.37131299999999995,"total_cache_read_cost_bin":0.3345135,"total_completion_cost_bin":1.0441050000000003,"total_reasoning_cost_bin":0.3777599999999998,"total_cost_bin":2.1276915,"total_on_demand_input_cost_bin":3.7164480000000055,"ci_99_lower":0.7024543901795114,"ci_99_upper":0.9205808358176145,"ci_98_lower":0.7133845008235657,"ci_98_upper":0.9088937232512979,"ci_95_lower":0.7375877035123994,"ci_95_upper":0.8963989835609392,"ci_90_lower":0.7504411820825772,"ci_90_upper":0.8837303904324654,"ci_75_lower":0.7740473594203648,"ci_75_upper":0.868940395291429,"ci_50_lower":0.7971016711264383,"ci_50_upper":0.8475354204783799},"32768":{"avg_score":0.8582990116367856,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.3249366736235707,"median_score":0.992753803730011,"q1_score":0.9660480916500092,"q3_score":0.9994666129350662,"avg_token_efficiency":1.1449043855358674,"is_incomplete":false,"total_prompt_cost_bin":0.37519200000000025,"total_cache_read_cost_bin":1.2953183999999993,"total_completion_cost_bin":1.9477350000000004,"total_reasoning_cost_bin":0.70278,"total_cost_bin":4.3210254,"total_on_demand_input_cost_bin":13.328376,"ci_99_lower":0.7808301062242518,"ci_99_upper":0.9248868980286694,"ci_98_lower":0.7921413239981527,"ci_98_upper":0.9225154957600684,"ci_95_lower":0.8023563510205456,"ci_95_upper":0.9106803938795672,"ci_90_lower":0.8110375005638857,"ci_90_upper":0.9028202008047682,"ci_75_lower":0.823878904522751,"ci_75_upper":0.8901564888260243,"ci_50_lower":0.8382841334536995,"ci_50_upper":0.877358486137596},"65536":{"avg_score":0.7961918319411138,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.3890258725525186,"median_score":0.9955320954322815,"q1_score":0.9587835073471069,"q3_score":0.9989455342292786,"avg_token_efficiency":1.1315791278603577,"is_incomplete":false,"total_prompt_cost_bin":0.233694,"total_cache_read_cost_bin":1.8640943999999986,"total_completion_cost_bin":1.5091049999999995,"total_reasoning_cost_bin":0.5406749999999998,"total_cost_bin":4.147568399999997,"total_on_demand_input_cost_bin":18.874637999999997,"ci_99_lower":0.6809525349162081,"ci_99_upper":0.8966673001641516,"ci_98_lower":0.6829155351337702,"ci_98_upper":0.8883027614175396,"ci_95_lower":0.7045281954624636,"ci_95_upper":0.8756966284903533,"ci_90_lower":0.7186967529060647,"ci_90_upper":0.8642519942331403,"ci_75_lower":0.7408761398428504,"ci_75_upper":0.8423198695157601,"ci_50_lower":0.7702947136781672,"ci_50_upper":0.8280853500425377},"131072":{"avg_score":0.7256803724539548,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4267330099321197,"median_score":0.9836065769195557,"q1_score":0.05986509844660759,"q3_score":0.9984652101993561,"avg_token_efficiency":1.1236718471186093,"is_incomplete":false,"total_prompt_cost_bin":0.4549770000000002,"total_cache_read_cost_bin":4.498567200000001,"total_completion_cost_bin":1.8725999999999992,"total_reasoning_cost_bin":0.6449100000000003,"total_cost_bin":7.471054200000001,"total_on_demand_input_cost_bin":45.440648999999944,"ci_99_lower":0.6079879275654009,"ci_99_upper":0.835102354842758,"ci_98_lower":0.6168671264381653,"ci_98_upper":0.8212773265061647,"ci_95_lower":0.6340723754009074,"ci_95_upper":0.8057842502373869,"ci_90_lower":0.6513656244010808,"ci_90_upper":0.7946764062658331,"ci_75_lower":0.6755081376007721,"ci_75_upper":0.7747935141141197,"ci_50_lower":0.6934167486287708,"ci_50_upper":0.7534336459882855},"262144":{"avg_score":0.7269327188247537,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4300334036393658,"median_score":0.9930986762046814,"q1_score":0.05886909365653992,"q3_score":0.999459445476532,"avg_token_efficiency":1.126416941859279,"is_incomplete":false,"total_prompt_cost_bin":0.9039629999999997,"total_cache_read_cost_bin":12.389496000000031,"total_completion_cost_bin":2.8645950000000004,"total_reasoning_cost_bin":1.128015,"total_cost_bin":17.286069000000033,"total_on_demand_input_cost_bin":124.79892300000004,"ci_99_lower":0.6208244261901218,"ci_99_upper":0.819079148067964,"ci_98_lower":0.638870988798473,"ci_98_upper":0.8127186402515343,"ci_95_lower":0.652693837291168,"ci_95_upper":0.795192910304643,"ci_90_lower":0.6620901069666609,"ci_90_upper":0.7870504748586447,"ci_75_lower":0.685072765421,"ci_75_upper":0.768740588268645,"ci_50_lower":0.7004232705217383,"ci_50_upper":0.7525610395035118},"524288":{"avg_score":0.4350788989992244,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.46537228360718264,"median_score":0.07458731159567833,"q1_score":0.027503293938934803,"q3_score":0.9916012287139893,"avg_token_efficiency":1.1295900522626992,"is_incomplete":false,"total_prompt_cost_bin":5.360415000000004,"total_cache_read_cost_bin":41.20514879999991,"total_completion_cost_bin":15.506535,"total_reasoning_cost_bin":4.564694999999998,"total_cost_bin":66.63679379999991,"total_on_demand_input_cost_bin":417.41190300000096,"ci_99_lower":0.3558169954466994,"ci_99_upper":0.5069314732696956,"ci_98_lower":0.36120076895602665,"ci_98_upper":0.5012693733975224,"ci_95_lower":0.3717814971319443,"ci_95_upper":0.49566328000076026,"ci_90_lower":0.3826108226057347,"ci_90_upper":0.48702683306537903,"ci_75_lower":0.39942687818934935,"ci_75_upper":0.4708426998272367,"ci_50_lower":0.4155131551479996,"ci_50_upper":0.45345505054939494}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Anthropic: Claude Sonnet 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000003,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-17T15:43:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:12:48.728Z"},{"model_slug":"anthropic/claude-sonnet-4.6","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.43437129702078064,"auc_128k":0.8000670701498181,"cum_avg_1m":0.6591666544523529,"cum_avg_128k":0.8302430984964572,"avg_score_overall":0.6961224262208356,"avg_response_time_ms":42180.424634838215,"total_prompt_cost":6.178599000000004,"total_cache_read_cost":62.09807099999993,"total_completion_cost":25.044734999999996,"total_reasoning_cost":8.326095,"total_prompt_cost_128k":1.6901610000000005,"total_completion_cost_128k":7.769894999999999,"total_on_demand_input_cost":627.1593090000007,"total_on_demand_input_cost_128k":83.17225499999995,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1311011541866514},"bin_metrics":{"8192":{"avg_score":0.9577131343539804,"min_score":0.022074338048696518,"max_score":1,"n_tests":80,"stddev":0.18227418065288437,"median_score":0.9970805644989014,"q1_score":0.9921955615282059,"q3_score":1,"avg_token_efficiency":1.1377505627360662,"is_incomplete":false,"total_prompt_cost_bin":0.44801400000000025,"total_cache_read_cost_bin":0.13641299999999998,"total_completion_cost_bin":1.1390550000000002,"total_reasoning_cost_bin":0.37287000000000003,"total_cost_bin":2.0963520000000004,"total_on_demand_input_cost_bin":1.812143999999998,"ci_99_lower":0.8961647008503787,"ci_99_upper":0.9951347343176603,"ci_98_lower":0.899062813829165,"ci_98_upper":0.9945597617477179,"ci_95_lower":0.9094119268399663,"ci_95_upper":0.993762936964631,"ci_90_lower":0.9215521461423486,"ci_90_upper":0.9920631681755185,"ci_75_lower":0.9333967546233907,"ci_75_upper":0.9813922099536285,"ci_50_lower":0.9455654623219744,"ci_50_upper":0.9701885098591447},"16384":{"avg_score":0.8203134896000848,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.36917253298513825,"median_score":0.9930905103683472,"q1_score":0.9706265330314636,"q3_score":0.9983714818954468,"avg_token_efficiency":1.122757280298134,"is_incomplete":false,"total_prompt_cost_bin":0.17828400000000005,"total_cache_read_cost_bin":0.35381640000000003,"total_completion_cost_bin":1.5306300000000002,"total_reasoning_cost_bin":0.5500050000000001,"total_cost_bin":2.6127354000000005,"total_on_demand_input_cost_bin":3.716448000000005,"ci_99_lower":0.7001800485833083,"ci_99_upper":0.9196685021016747,"ci_98_lower":0.7113579265144654,"ci_98_upper":0.9078826701636427,"ci_95_lower":0.7349789278930985,"ci_95_upper":0.8946763890644069,"ci_90_lower":0.7484860766667407,"ci_90_upper":0.8823405916930642,"ci_75_lower":0.7718685406871373,"ci_75_upper":0.8664793764211937,"ci_50_lower":0.7946921831928193,"ci_50_upper":0.8459186963678802},"32768":{"avg_score":0.837720130037804,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.34627151588263727,"median_score":0.992753803730011,"q1_score":0.9637879431247711,"q3_score":0.9994260519742966,"avg_token_efficiency":1.144904385535868,"is_incomplete":false,"total_prompt_cost_bin":0.3751920000000002,"total_cache_read_cost_bin":1.2953183999999993,"total_completion_cost_bin":1.8974249999999995,"total_reasoning_cost_bin":0.6868500000000001,"total_cost_bin":4.2547853999999985,"total_on_demand_input_cost_bin":13.328376000000002,"ci_99_lower":0.7610439545827188,"ci_99_upper":0.9084718415594184,"ci_98_lower":0.7674250381953179,"ci_98_upper":0.9036645317687618,"ci_95_lower":0.7745925193717567,"ci_95_upper":0.8942322501325072,"ci_90_lower":0.7834715682835154,"ci_90_upper":0.8869167283750893,"ci_75_lower":0.7998803774971748,"ci_75_upper":0.8715173399607867,"ci_50_lower":0.8159208981044631,"ci_50_upper":0.8573154249482767},"65536":{"avg_score":0.8183219532975379,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.37094959522433846,"median_score":0.9955320954322815,"q1_score":0.9629040956497192,"q3_score":0.9989455342292786,"avg_token_efficiency":1.1315791278603577,"is_incomplete":false,"total_prompt_cost_bin":0.233694,"total_cache_read_cost_bin":1.8640943999999986,"total_completion_cost_bin":1.462154999999999,"total_reasoning_cost_bin":0.49465500000000007,"total_cost_bin":4.054598399999997,"total_on_demand_input_cost_bin":18.874637999999997,"ci_99_lower":0.7043644670266658,"ci_99_upper":0.9114827492156559,"ci_98_lower":0.7140875035832253,"ci_98_upper":0.8997711576882969,"ci_95_lower":0.7273690776088658,"ci_95_upper":0.8885497115976999,"ci_90_lower":0.746376003538094,"ci_90_upper":0.8777652051223114,"ci_75_lower":0.7710913703653632,"ci_75_upper":0.8637418524088228,"ci_50_lower":0.7936939306146301,"ci_50_upper":0.8492232514079661},"131072":{"avg_score":0.7171467851928788,"min_score":0.0057299453765153885,"max_score":1,"n_tests":103,"stddev":0.4305850900780958,"median_score":0.9801771640777588,"q1_score":0.048849523067474365,"q3_score":0.9990395903587341,"avg_token_efficiency":1.1236718471186096,"is_incomplete":false,"total_prompt_cost_bin":0.454977,"total_cache_read_cost_bin":4.498567200000001,"total_completion_cost_bin":1.7406300000000003,"total_reasoning_cost_bin":0.625215,"total_cost_bin":7.319389200000002,"total_on_demand_input_cost_bin":45.440648999999944,"ci_99_lower":0.5978121038565403,"ci_99_upper":0.8285773933953741,"ci_98_lower":0.6091662491373808,"ci_98_upper":0.8128172358530076,"ci_95_lower":0.6243723985102725,"ci_95_upper":0.7984184388994711,"ci_90_lower":0.6421322725857925,"ci_90_upper":0.7868070441691511,"ci_75_lower":0.6660026053065045,"ci_75_upper":0.7663990906117601,"ci_50_lower":0.6853685113917856,"ci_50_upper":0.7456958922749675},"262144":{"avg_score":0.6847782201127063,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4479002593843822,"median_score":0.9933082461357117,"q1_score":0.04939006268978119,"q3_score":0.9994535446166992,"avg_token_efficiency":1.1264169418592793,"is_incomplete":false,"total_prompt_cost_bin":0.9039629999999998,"total_cache_read_cost_bin":12.389496000000031,"total_completion_cost_bin":3.0367649999999977,"total_reasoning_cost_bin":1.1799300000000001,"total_cost_bin":17.51015400000003,"total_on_demand_input_cost_bin":124.79892300000006,"ci_99_lower":0.5795233512345985,"ci_99_upper":0.7787091873565867,"ci_98_lower":0.5898929611732853,"ci_98_upper":0.7716187462942503,"ci_95_lower":0.6077283167456119,"ci_95_upper":0.7560355932573154,"ci_90_lower":0.6229408684891735,"ci_90_upper":0.7471893908267722,"ci_75_lower":0.640305749671154,"ci_75_upper":0.7283854684024613,"ci_50_lower":0.6577944359354053,"ci_50_upper":0.7118178048354395},"524288":{"avg_score":0.437339523023831,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.465935304084005,"median_score":0.08082112297415733,"q1_score":0.02633730275556445,"q3_score":0.9918380379676819,"avg_token_efficiency":1.1295900633711318,"is_incomplete":false,"total_prompt_cost_bin":3.5844750000000034,"total_cache_read_cost_bin":41.560365599999905,"total_completion_cost_bin":14.238074999999997,"total_reasoning_cost_bin":4.41657,"total_cost_bin":63.799485599999905,"total_on_demand_input_cost_bin":419.1881310000007,"ci_99_lower":0.35683577982578585,"ci_99_upper":0.5091863178888284,"ci_98_lower":0.3627959751086191,"ci_98_upper":0.5040668768804103,"ci_95_lower":0.37417005699594563,"ci_95_upper":0.49795546854705103,"ci_90_lower":0.3842276186331065,"ci_90_upper":0.48993774124636247,"ci_75_lower":0.40175164740096575,"ci_75_upper":0.473204601658666,"ci_50_lower":0.4179755568487562,"ci_50_upper":0.45537203236116036}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Anthropic: Claude Sonnet 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000003,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-17T15:43:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T04:09:58.837Z"},{"model_slug":"google/gemini-3.5-flash","reasoning_mode":"high","overall_metrics":{"auc_1m":0.43272730179841884,"auc_128k":0.6866629673795913,"cum_avg_1m":0.6379459923028088,"cum_avg_128k":0.7818547806448927,"avg_score_overall":0.5318659676885492,"avg_response_time_ms":49243.11609439215,"total_prompt_cost":32.600615999999974,"total_cache_read_cost":73.2664342499998,"total_completion_cost":5.241294,"total_reasoning_cost":133.44232499999998,"total_prompt_cost_128k":8.017249499999995,"total_completion_cost_128k":1.6791659999999997,"total_on_demand_input_cost":765.2649585000015,"total_on_demand_input_cost_128k":36.8202,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.000022752115489},"bin_metrics":{"8192":{"avg_score":0.9506966145010665,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.21351607554062757,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000155340474723,"is_incomplete":false,"total_prompt_cost_bin":0.7973159999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.25221600000000005,"total_reasoning_cost_bin":1.230075,"total_cost_bin":2.279607,"total_on_demand_input_cost_bin":0.7973159999999998,"ci_99_lower":0.8779626687174896,"ci_99_upper":0.9993879185989499,"ci_98_lower":0.8883356860780623,"ci_98_upper":0.999384842813015,"ci_95_lower":0.8905213827174155,"ci_95_upper":0.9878333333414048,"ci_90_lower":0.9023225315997843,"ci_90_upper":0.9874833380337804,"ci_75_lower":0.9254947015026118,"ci_75_upper":0.975510698207654,"ci_50_lower":0.9380412816011812,"ci_50_upper":0.9634784383466467},"16384":{"avg_score":0.9754592092474923,"min_score":0.02010050229728222,"max_score":1,"n_tests":80,"stddev":0.15253729541513236,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000074513502448,"is_incomplete":false,"total_prompt_cost_bin":1.6528079999999985,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.249768,"total_reasoning_cost_bin":1.1929680000000005,"total_cost_bin":3.095543999999999,"total_on_demand_input_cost_bin":1.6528079999999985,"ci_99_lower":0.9264247838434531,"ci_99_upper":1,"ci_98_lower":0.9269510291223415,"ci_98_upper":1,"ci_95_lower":0.9388499764335575,"ci_95_upper":0.9999950610101223,"ci_90_lower":0.9396378922706935,"ci_90_upper":0.9999854299798607,"ci_75_lower":0.9514444436412305,"ci_75_upper":0.9992084927856922,"ci_50_lower":0.963437583274208,"ci_50_upper":0.9877079529687762},"32768":{"avg_score":0.7957280917462948,"min_score":0.00585774052888155,"max_score":1,"n_tests":136,"stddev":0.39407692864396415,"median_score":1,"q1_score":0.999755859375,"q3_score":1,"avg_token_efficiency":1.0000359090949906,"is_incomplete":false,"total_prompt_cost_bin":1.9561050000000002,"total_cache_read_cost_bin":0.3854019000000012,"total_completion_cost_bin":0.473913,"total_reasoning_cost_bin":3.2909670000000006,"total_cost_bin":6.106386900000002,"total_on_demand_input_cost_bin":5.810123999999996,"ci_99_lower":0.6978682362596738,"ci_99_upper":0.8801451397970018,"ci_98_lower":0.7103051453202908,"ci_98_upper":0.872556405709714,"ci_95_lower":0.7243688185435637,"ci_95_upper":0.8599370977567399,"ci_90_lower":0.7374734914943795,"ci_90_upper":0.8509856425513348,"ci_75_lower":0.7534499677216553,"ci_75_upper":0.8362115119945135,"ci_50_lower":0.7727716788718039,"ci_50_upper":0.8187211235858473},"65536":{"avg_score":0.6123315817073864,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4778027550873331,"median_score":1,"q1_score":0.03145589679479599,"q3_score":1,"avg_token_efficiency":1.0000152878522814,"is_incomplete":false,"total_prompt_cost_bin":1.6342274999999993,"total_cache_read_cost_bin":0.6705855,"total_completion_cost_bin":0.31046399999999996,"total_reasoning_cost_bin":3.0864149999999997,"total_cost_bin":5.7016919999999995,"total_on_demand_input_cost_bin":8.340082499999998,"ci_99_lower":0.47609132170896323,"ci_99_upper":0.737169421763488,"ci_98_lower":0.4952690730349106,"ci_98_upper":0.72633092707286,"ci_95_lower":0.5093143366707269,"ci_95_upper":0.7148432624540019,"ci_90_lower":0.5309969399173688,"ci_90_upper":0.7021305546350778,"ci_75_lower":0.5543732011512689,"ci_75_upper":0.671009009343791,"ci_50_lower":0.5769084451418809,"ci_50_upper":0.6462017927976216},"131072":{"avg_score":0.5750584060222236,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.48452106036152437,"median_score":0.9996020793914795,"q1_score":0.02830175869166851,"q3_score":1,"avg_token_efficiency":1.0000076410572556,"is_incomplete":false,"total_prompt_cost_bin":1.976792999999997,"total_cache_read_cost_bin":1.8243076500000017,"total_completion_cost_bin":0.3928049999999998,"total_reasoning_cost_bin":5.093388000000001,"total_cost_bin":9.287293649999999,"total_on_demand_input_cost_bin":20.219869500000012,"ci_99_lower":0.4521049067687958,"ci_99_upper":0.697769115579787,"ci_98_lower":0.4615013548506433,"ci_98_upper":0.6795119000710682,"ci_95_lower":0.4807951111267366,"ci_95_upper":0.6691433135573233,"ci_90_lower":0.499013539316422,"ci_90_upper":0.6596663848933249,"ci_75_lower":0.518578265639343,"ci_75_upper":0.6318802174700094,"ci_50_lower":0.5453410609184484,"ci_50_upper":0.612078400166},"262144":{"avg_score":0.5461335538404314,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4827062306511305,"median_score":0.9985775351524353,"q1_score":0.03147163242101669,"q3_score":1,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":3.9778275000000054,"total_cache_read_cost_bin":5.1418800000000076,"total_completion_cost_bin":0.5108400000000001,"total_reasoning_cost_bin":13.430510999999996,"total_cost_bin":23.06105850000001,"total_on_demand_input_cost_bin":55.3966275,"ci_99_lower":0.43182405589701583,"ci_99_upper":0.6490795905342167,"ci_98_lower":0.4527096482133493,"ci_98_upper":0.6413690064552551,"ci_95_lower":0.46722973025460063,"ci_95_upper":0.6270707584156304,"ci_90_lower":0.4834152856417304,"ci_90_upper":0.6150987247318849,"ci_75_lower":0.5027204429204839,"ci_75_upper":0.5943917507447751,"ci_50_lower":0.5192622893212482,"ci_50_upper":0.5758864529718188},"524288":{"avg_score":0.38797592961967337,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4634983706307982,"median_score":0.04657421074807644,"q1_score":0.023732191417366266,"q3_score":1,"avg_token_efficiency":1.0000019078540323,"is_incomplete":false,"total_prompt_cost_bin":7.507060499999993,"total_cache_read_cost_bin":17.804207999999928,"total_completion_cost_bin":1.2018149999999992,"total_reasoning_cost_bin":39.53741400000001,"total_cost_bin":66.05049749999993,"total_on_demand_input_cost_bin":185.54914050000014,"ci_99_lower":0.30544524531824796,"ci_99_upper":0.462067573646116,"ci_98_lower":0.3130011618640313,"ci_98_upper":0.4570580366701953,"ci_95_lower":0.327938887084562,"ci_95_upper":0.44645220817760284,"ci_90_lower":0.33671159228799324,"ci_90_upper":0.43962512153868716,"ci_75_lower":0.3526359777336484,"ci_75_upper":0.42312985747251997,"ci_50_lower":0.3682570026033672,"ci_50_upper":0.4074491460218639},"1048576":{"avg_score":0.26018455173790217,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.41299484686103954,"median_score":0.030819187872111797,"q1_score":0.017143280245363712,"q3_score":0.18891668692231178,"avg_token_efficiency":1.0000009538490409,"is_incomplete":false,"total_prompt_cost_bin":13.098478499999981,"total_cache_read_cost_bin":47.44005119999987,"total_completion_cost_bin":1.8494730000000004,"total_reasoning_cost_bin":66.580587,"total_cost_bin":128.96858969999985,"total_on_demand_input_cost_bin":487.4989905000013,"ci_99_lower":0.19977911262914178,"ci_99_upper":0.3180138065411273,"ci_98_lower":0.20685848306171267,"ci_98_upper":0.31326288240836303,"ci_95_lower":0.21300365742913732,"ci_95_upper":0.3072464551535367,"ci_90_lower":0.22148238690149397,"ci_90_upper":0.3001338970951826,"ci_75_lower":0.2319714293565466,"ci_75_upper":0.2872289964706719,"ci_50_lower":0.24387737639556306,"ci_50_upper":0.2753912590457232}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":0.0000015,"completion":0.000009,"reasoning":0.000009,"has_tiered_pricing":true},"api_created_at":"2026-05-19T12:30:00.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"anthropic/claude-opus-4.8","reasoning_mode":"max","overall_metrics":{"auc_1m":0.41835007031936344,"auc_128k":0.8777900186673483,"cum_avg_1m":0.6925514005179141,"cum_avg_128k":0.9047730474115336,"avg_score_overall":0.7262115683479263,"avg_response_time_ms":75096.1037937636,"total_prompt_cost":22.567225000000022,"total_cache_read_cost":138.1484404999998,"total_completion_cost":144.509325,"total_reasoning_cost":13.887800000000006,"total_prompt_cost_128k":3.160605000000001,"total_completion_cost_128k":49.352225,"total_on_demand_input_cost":1404.051629999999,"total_on_demand_input_cost_128k":200.15950000000004,"total_runs":833,"total_candidates_generated":null,"avg_token_efficiency_overall":1.637591811808624},"bin_metrics":{"8192":{"avg_score":0.9740119151771068,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.1570688873884934,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.6722716647051477,"is_incomplete":false,"total_prompt_cost_bin":0.3998550000000004,"total_cache_read_cost_bin":0.40412450000000016,"total_completion_cost_bin":4.4578250000000015,"total_reasoning_cost_bin":0.3895749999999999,"total_cost_bin":5.651379500000002,"total_on_demand_input_cost_bin":4.4411,"ci_99_lower":0.9230656426809728,"ci_99_upper":0.9998418617732823,"ci_98_lower":0.9240555458664895,"ci_98_upper":0.9998146156519652,"ci_95_lower":0.935688812918961,"ci_95_upper":0.9996827430464328,"ci_90_lower":0.9372379502654076,"ci_90_upper":0.9994589298218488,"ci_75_lower":0.949665882345289,"ci_75_upper":0.9975517032667994,"ci_50_lower":0.9618220662698149,"ci_50_upper":0.9865786347538232},"16384":{"avg_score":0.9613693707040512,"min_score":0.005645730532705784,"max_score":1,"n_tests":80,"stddev":0.1862783413320043,"median_score":1,"q1_score":0.9980840981006622,"q3_score":1,"avg_token_efficiency":1.5830137761150542,"is_incomplete":false,"total_prompt_cost_bin":0.33049,"total_cache_read_cost_bin":0.8380969999999999,"total_completion_cost_bin":6.945474999999998,"total_reasoning_cost_bin":0.6077499999999998,"total_cost_bin":8.721811999999998,"total_on_demand_input_cost_bin":8.711460000000002,"ci_99_lower":0.899778959292802,"ci_99_upper":0.9985917912721634,"ci_98_lower":0.9007768830823479,"ci_98_upper":0.9983704643622042,"ci_95_lower":0.9123013682704185,"ci_95_upper":0.997978908251971,"ci_90_lower":0.9244895647792146,"ci_90_upper":0.997173996604979,"ci_75_lower":0.9367310421614093,"ci_75_upper":0.9856571079944841,"ci_50_lower":0.9491536315355916,"ci_50_upper":0.9738196458667516},"32768":{"avg_score":0.9300222400565604,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.2438127313085668,"median_score":1,"q1_score":0.9990289062261581,"q3_score":1,"avg_token_efficiency":1.6607949216663114,"is_incomplete":false,"total_prompt_cost_bin":0.7823700000000002,"total_cache_read_cost_bin":3.14874,"total_completion_cost_bin":13.215524999999994,"total_reasoning_cost_bin":1.2317250000000008,"total_cost_bin":18.378359999999997,"total_on_demand_input_cost_bin":32.269769999999994,"ci_99_lower":0.8714807111898484,"ci_99_upper":0.9780467926500612,"ci_98_lower":0.8760410774612682,"ci_98_upper":0.9720793586907743,"ci_95_lower":0.8832238463397302,"ci_95_upper":0.9698094977843164,"ci_90_lower":0.8923642777392695,"ci_90_upper":0.9636480482455226,"ci_75_lower":0.9040256776876477,"ci_75_upper":0.9557738329526996,"ci_50_lower":0.9156454919498441,"ci_50_upper":0.9443998483685017},"65536":{"avg_score":0.9069383508580573,"min_score":0.015236567705869675,"max_score":1,"n_tests":85,"stddev":0.2830306346284175,"median_score":1,"q1_score":0.9987834692001343,"q3_score":1,"avg_token_efficiency":1.6412164101683422,"is_incomplete":false,"total_prompt_cost_bin":0.5604099999999999,"total_cache_read_cost_bin":4.506516000000003,"total_completion_cost_bin":10.276100000000008,"total_reasoning_cost_bin":0.9842499999999997,"total_cost_bin":16.327276000000012,"total_on_demand_input_cost_bin":45.625569999999975,"ci_99_lower":0.8160352844225572,"ci_99_upper":0.9732015499883496,"ci_98_lower":0.826194081966053,"ci_98_upper":0.9643796762690386,"ci_95_lower":0.8377970068058108,"ci_95_upper":0.9632267274439116,"ci_90_lower":0.8497169247740771,"ci_90_upper":0.9527567468331578,"ci_75_lower":0.8715640958179445,"ci_75_upper":0.9410739441977485,"ci_50_lower":0.8851036249550389,"ci_50_upper":0.9293506757191876},"131072":{"avg_score":0.7515233602618925,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4213170631117181,"median_score":0.9996055364608765,"q1_score":0.5968111157417297,"q3_score":1,"avg_token_efficiency":1.6188892872031542,"is_incomplete":false,"total_prompt_cost_bin":1.0874800000000004,"total_cache_read_cost_bin":10.80241199999998,"total_completion_cost_bin":14.457299999999998,"total_reasoning_cost_bin":1.3380000000000007,"total_cost_bin":27.68519199999998,"total_on_demand_input_cost_bin":109.11160000000007,"ci_99_lower":0.6389198298416173,"ci_99_upper":0.8555843086045426,"ci_98_lower":0.6460240221283968,"ci_98_upper":0.848267350392823,"ci_95_lower":0.6629954557544868,"ci_95_upper":0.8278218363681854,"ci_90_lower":0.6815641113339914,"ci_90_upper":0.8174932444124545,"ci_75_lower":0.7016376417426357,"ci_75_upper":0.7986278881966498,"ci_50_lower":0.7223494950957705,"ci_50_upper":0.779097126187639},"262144":{"avg_score":0.6184493604857121,"min_score":0.005767250433564186,"max_score":1,"n_tests":137,"stddev":0.4723669515477762,"median_score":0.9986187815666199,"q1_score":0.033415406942367554,"q3_score":1,"avg_token_efficiency":1.6370583919172534,"is_incomplete":false,"total_prompt_cost_bin":6.468515000000004,"total_cache_read_cost_bin":28.72464199999994,"total_completion_cost_bin":29.487,"total_reasoning_cost_bin":2.7671249999999996,"total_cost_bin":67.44728199999993,"total_on_demand_input_cost_bin":293.71493499999974,"ci_99_lower":0.5095991802709449,"ci_99_upper":0.7154285711728131,"ci_98_lower":0.5242124138994102,"ci_98_upper":0.7078264544795464,"ci_95_lower":0.5348225456727057,"ci_95_upper":0.6960933806644888,"ci_90_lower":0.5529251868979339,"ci_90_upper":0.6863134805092683,"ci_75_lower":0.5747876657949705,"ci_75_upper":0.6666535582940646,"ci_50_lower":0.5905357500058973,"ci_50_upper":0.6457298042126897},"524288":{"avg_score":0.3980966065999314,"min_score":0,"max_score":1,"n_tests":212,"stddev":0.4594245178473069,"median_score":0.052879855036735535,"q1_score":0.025334641337394714,"q3_score":0.994745522737503,"avg_token_efficiency":1.6381936013464689,"is_incomplete":false,"total_prompt_cost_bin":12.938105000000016,"total_cache_read_cost_bin":89.72390899999986,"total_completion_cost_bin":65.6701,"total_reasoning_cost_bin":6.569375000000004,"total_cost_bin":174.90148899999988,"total_on_demand_input_cost_bin":910.1771949999993,"ci_99_lower":0.3109672861281324,"ci_99_upper":0.47595560069586906,"ci_98_lower":0.32518055690667713,"ci_98_upper":0.472590896817559,"ci_95_lower":0.3385775547844187,"ci_95_upper":0.46168477296975297,"ci_90_lower":0.34670065001475864,"ci_90_upper":0.45183623441827775,"ci_75_lower":0.36341584362141804,"ci_75_upper":0.4346263128139775,"ci_50_lower":0.37624902054222675,"ci_50_upper":0.419306160177583}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.8","provider_name":"Anthropic","run_provider_names":["Claude Platform on AWS"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-27T18:04:51.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"qwen/qwen3.8-max","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.4119693108121391,"auc_128k":0.9610840864738284,"cum_avg_1m":0.7271718809209768,"cum_avg_128k":0.9685662199342744,"avg_score_overall":0.737184181615535,"avg_response_time_ms":66197.51673540034,"total_prompt_cost":6.216167999999994,"total_cache_read_cost":46.39174400000002,"total_completion_cost":19.155504000000004,"total_reasoning_cost":17.14107,"total_prompt_cost_128k":1.903288,"total_completion_cost_128k":14.297748000000002,"total_on_demand_input_cost":377.3501200000006,"total_on_demand_input_cost_128k":50.06815199999998,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0209236185915027},"bin_metrics":{"8192":{"avg_score":0.9867187290685251,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.11046772176975414,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0284002083617256,"is_incomplete":false,"total_prompt_cost_bin":0.21350400000000025,"total_cache_read_cost_bin":0.10992000000000005,"total_completion_cost_bin":1.0719840000000003,"total_reasoning_cost_bin":0.898404,"total_cost_bin":2.293812000000001,"total_on_demand_input_cost_bin":1.092864,"ci_99_lower":0.948783611971885,"ci_99_upper":1,"ci_98_lower":0.9497237716801464,"ci_98_upper":1,"ci_95_lower":0.9611247231950983,"ci_95_upper":1,"ci_90_lower":0.9620790710439906,"ci_90_upper":1,"ci_75_lower":0.9742512302938848,"ci_75_upper":0.9999574355781078,"ci_50_lower":0.9753461538348347,"ci_50_upper":0.999031464010477},"16384":{"avg_score":0.9753734395140782,"min_score":0.011054813861846924,"max_score":1,"n_tests":80,"stddev":0.154265787599074,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0188019619265731,"is_incomplete":false,"total_prompt_cost_bin":0.17465600000000026,"total_cache_read_cost_bin":0.2591039999999998,"total_completion_cost_bin":1.5740039999999993,"total_reasoning_cost_bin":1.4021039999999998,"total_cost_bin":3.4098679999999995,"total_on_demand_input_cost_bin":2.247488000000002,"ci_99_lower":0.9260256525221048,"ci_99_upper":0.9999932877719402,"ci_98_lower":0.9263492410811596,"ci_98_upper":0.9999872460961342,"ci_95_lower":0.9383672648016363,"ci_95_upper":0.9999734826385975,"ci_90_lower":0.9388639417849481,"ci_90_upper":0.9999523868784308,"ci_75_lower":0.9510080528212711,"ci_75_upper":0.9998071282170713,"ci_50_lower":0.9631223725620657,"ci_50_upper":0.9876670893514528},"32768":{"avg_score":0.9928445065229693,"min_score":0.03251568600535393,"max_score":1,"n_tests":136,"stddev":0.08295829848647143,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.027047586677628,"is_incomplete":false,"total_prompt_cost_bin":0.42588800000000016,"total_cache_read_cost_bin":0.9422080000000025,"total_completion_cost_bin":3.2646180000000014,"total_reasoning_cost_bin":2.933237999999999,"total_cost_bin":7.565952000000003,"total_on_demand_input_cost_bin":7.963552000000002,"ci_99_lower":0.9714612930815886,"ci_99_upper":1,"ci_98_lower":0.9714896923259777,"ci_98_upper":1,"ci_95_lower":0.9785751577959779,"ci_95_upper":1,"ci_90_lower":0.9786300444701577,"ci_90_upper":0.9999867515528903,"ci_75_lower":0.9857041543787893,"ci_75_upper":0.9999716102200396,"ci_50_lower":0.9857722895000788,"ci_50_upper":0.9999451133258203},"65536":{"avg_score":0.9648652777349686,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.18319360022846728,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0215407639042384,"is_incomplete":false,"total_prompt_cost_bin":0.37092799999999954,"total_cache_read_cost_bin":1.373567999999999,"total_completion_cost_bin":2.46273,"total_reasoning_cost_bin":2.2523820000000008,"total_cost_bin":6.459607999999999,"total_on_demand_input_cost_bin":11.359471999999988,"ci_99_lower":0.9057589269955916,"ci_99_upper":0.9999944111634703,"ci_98_lower":0.9068801805109662,"ci_98_upper":0.9999839340378257,"ci_95_lower":0.9182875471873044,"ci_95_upper":0.9997030103907866,"ci_90_lower":0.9298591792147936,"ci_90_upper":0.9884508105849519,"ci_75_lower":0.9415123715524233,"ci_75_upper":0.9880112421479733,"ci_50_lower":0.9532075700007708,"ci_50_upper":0.9766946936190567},"131072":{"avg_score":0.9230291468308316,"min_score":0.005760921631008387,"max_score":1,"n_tests":103,"stddev":0.2618126933279601,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0165119538245984,"is_incomplete":false,"total_prompt_cost_bin":0.7183119999999998,"total_cache_read_cost_bin":3.335807999999995,"total_completion_cost_bin":5.924412000000001,"total_reasoning_cost_bin":5.6601,"total_cost_bin":15.638631999999996,"total_on_demand_input_cost_bin":27.404775999999984,"ci_99_lower":0.8384223090752607,"ci_99_upper":0.978913054112232,"ci_98_lower":0.8564927937419524,"ci_98_upper":0.9709323461252509,"ci_95_lower":0.8659335172318094,"ci_95_upper":0.9700614054418323,"ci_90_lower":0.8757922007467199,"ci_90_upper":0.96128507864015,"ci_75_lower":0.886572581032083,"ci_75_upper":0.9514654067676125,"ci_50_lower":0.9045955502163637,"ci_50_upper":0.9417127776038787},"262144":{"avg_score":0.677437562001737,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4525587377833918,"median_score":0.992023229598999,"q1_score":0.0416666679084301,"q3_score":0.9990415573120117,"avg_token_efficiency":1.0182111239853666,"is_incomplete":false,"total_prompt_cost_bin":0.7314760000000006,"total_cache_read_cost_bin":9.309440000000004,"total_completion_cost_bin":0.9722279999999998,"total_reasoning_cost_bin":0.6424200000000003,"total_cost_bin":11.655564000000004,"total_on_demand_input_cost_bin":75.20699599999999,"ci_99_lower":0.575529214784285,"ci_99_upper":0.7732679844044186,"ci_98_lower":0.5816242705001694,"ci_98_upper":0.7659948082058196,"ci_95_lower":0.6002065408936493,"ci_95_upper":0.7513930086057604,"ci_90_lower":0.6157809572060697,"ci_90_upper":0.7391426537968615,"ci_75_lower":0.6311150184792882,"ci_75_upper":0.7209239940424054,"ci_50_lower":0.6500118738296277,"ci_50_upper":0.7047855014725364},"524288":{"avg_score":0.2971063856947062,"min_score":0.0021798366215080023,"max_score":1,"n_tests":236,"stddev":0.42305732288008485,"median_score":0.03800036944448948,"q1_score":0.021581947337836027,"q3_score":0.9407444447278976,"avg_token_efficiency":1.0189030743149656,"is_incomplete":false,"total_prompt_cost_bin":3.5814039999999943,"total_cache_read_cost_bin":31.06169600000002,"total_completion_cost_bin":3.8855280000000016,"total_reasoning_cost_bin":3.3524220000000016,"total_cost_bin":41.881050000000016,"total_on_demand_input_cost_bin":252.0749720000006,"ci_99_lower":0.22738811775361745,"ci_99_upper":0.3636578898379476,"ci_98_lower":0.2334105655849197,"ci_98_upper":0.360457503209305,"ci_95_lower":0.24237992662659716,"ci_95_upper":0.34824316509135095,"ci_90_lower":0.2534492443521878,"ci_90_upper":0.3421395022840874,"ci_75_lower":0.2666537683152286,"ci_75_upper":0.32904137833272323,"ci_50_lower":0.2794041216609727,"ci_50_upper":0.316270898494098}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","enabled"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"Qwen: Qwen3.8 Max","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":0.000002,"completion":0.000006,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-03T04:33:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T14:38:14.772Z"},{"model_slug":"meta/muse-spark-1.2","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.4092673189851644,"auc_128k":0.7252452946502114,"cum_avg_1m":0.6331270205453597,"cum_avg_128k":0.8057990398110487,"avg_score_overall":0.6755747295892824,"avg_response_time_ms":19439.046922070087,"total_prompt_cost":16.45325500000001,"total_cache_read_cost":25.615507800000096,"total_completion_cost":13.596871999999998,"total_reasoning_cost":12.1737595,"total_prompt_cost_128k":4.594544999999999,"total_completion_cost_128k":6.981007499999999,"total_on_demand_input_cost":229.9158199999998,"total_on_demand_input_cost_128k":30.528581250000006,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.99488296777093},"bin_metrics":{"8192":{"avg_score":0.9267345839412883,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.25892754840720805,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9856975537391577,"is_incomplete":false,"total_prompt_cost_bin":0.35717124999999994,"total_cache_read_cost_bin":0.035819849999999986,"total_completion_cost_bin":0.7672992499999998,"total_reasoning_cost_bin":0.6482949999999998,"total_cost_bin":1.8085853499999995,"total_on_demand_input_cost_bin":0.6556700000000003,"ci_99_lower":0.8410168462131404,"ci_99_upper":0.9878381234128029,"ci_98_lower":0.8527258800202981,"ci_98_upper":0.9877280265325681,"ci_95_lower":0.8653746027220041,"ci_95_upper":0.9756715765059926,"ci_90_lower":0.8768772072158754,"ci_90_upper":0.9754011034499854,"ci_75_lower":0.8901022170990472,"ci_75_upper":0.9632486836286261,"ci_50_lower":0.9025475455331616,"ci_50_upper":0.9509672533953563},"16384":{"avg_score":0.9514814352616667,"min_score":0.01194353960454464,"max_score":1,"n_tests":80,"stddev":0.21285046426984305,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9858622248642389,"is_incomplete":false,"total_prompt_cost_bin":0.43497125000000014,"total_cache_read_cost_bin":0.1108318500000001,"total_completion_cost_bin":0.8122302499999995,"total_reasoning_cost_bin":0.6921804999999999,"total_cost_bin":2.0502138499999996,"total_on_demand_input_cost_bin":1.35857,"ci_99_lower":0.8786593867825577,"ci_99_upper":1,"ci_98_lower":0.889946574265603,"ci_98_upper":1,"ci_95_lower":0.8915170643315651,"ci_95_upper":0.988113317778334,"ci_90_lower":0.9031862291379367,"ci_90_upper":0.9880047821439802,"ci_75_lower":0.9267800237517804,"ci_75_upper":0.9758273588726297,"ci_50_lower":0.9389047353062778,"ci_50_upper":0.9638759298017249},"32768":{"avg_score":0.909399231455823,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.27836988465762036,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0122430424432116,"is_incomplete":false,"total_prompt_cost_bin":0.8674162499999997,"total_cache_read_cost_bin":0.48463245000000027,"total_completion_cost_bin":1.894514,"total_reasoning_cost_bin":1.65933175,"total_cost_bin":4.90589445,"total_on_demand_input_cost_bin":4.906019999999998,"ci_99_lower":0.837774652915036,"ci_99_upper":0.9652551628382848,"ci_98_lower":0.8522539025543067,"ci_98_upper":0.9590598778832046,"ci_95_lower":0.8590741792497283,"ci_95_upper":0.9546606101055873,"ci_90_lower":0.8660509273638394,"ci_90_upper":0.9460671933274545,"ci_75_lower":0.8809471367209935,"ci_75_upper":0.936727516421396,"ci_50_lower":0.8929913140463588,"ci_50_upper":0.925619068377059},"65536":{"avg_score":0.6471862434683477,"min_score":0.007328691892325878,"max_score":1,"n_tests":85,"stddev":0.4686531994297227,"median_score":1,"q1_score":0.032487887889146805,"q3_score":1,"avg_token_efficiency":0.9925325139268658,"is_incomplete":false,"total_prompt_cost_bin":0.9789499999999997,"total_cache_read_cost_bin":0.7102936500000007,"total_completion_cost_bin":1.4936752500000001,"total_reasoning_cost_bin":1.3433697499999995,"total_cost_bin":4.52628865,"total_on_demand_input_cost_bin":6.898063750000002,"ci_99_lower":0.5086006879424677,"ci_99_upper":0.7713690290662515,"ci_98_lower":0.5197585718051038,"ci_98_upper":0.7591631941590458,"ci_95_lower":0.5448820177312284,"ci_95_upper":0.7407665404271992,"ci_90_lower":0.5660276035252302,"ci_90_upper":0.7273521764680524,"ci_75_lower":0.588766625923488,"ci_75_upper":0.7039562881472247,"ci_50_lower":0.6126423635246123,"ci_50_upper":0.6811545752673683},"131072":{"avg_score":0.5941937049281177,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.48180927700441706,"median_score":1,"q1_score":0.03186064958572388,"q3_score":1,"avg_token_efficiency":0.991720654856731,"is_incomplete":false,"total_prompt_cost_bin":1.9560362500000001,"total_cache_read_cost_bin":1.770506550000002,"total_completion_cost_bin":2.013288749999999,"total_reasoning_cost_bin":1.8297227500000004,"total_cost_bin":7.569554300000002,"total_on_demand_input_cost_bin":16.710257500000004,"ci_99_lower":0.47189426803909407,"ci_99_upper":0.7158479689708594,"ci_98_lower":0.4880843954422122,"ci_98_upper":0.7063480945180615,"ci_95_lower":0.5077481307357197,"ci_95_upper":0.6881756516797377,"ci_90_lower":0.5178697058286753,"ci_90_upper":0.6773583237827729,"ci_75_lower":0.5375545885327987,"ci_75_upper":0.6509810212645816,"ci_50_lower":0.5645335034288701,"ci_50_upper":0.6301013493347569},"262144":{"avg_score":0.5576785784511966,"min_score":0.006257822271436453,"max_score":1,"n_tests":141,"stddev":0.4803403710261095,"median_score":1,"q1_score":0.0342567153275013,"q3_score":1,"avg_token_efficiency":0.9975809987442881,"is_incomplete":false,"total_prompt_cost_bin":4.008401249999999,"total_cache_read_cost_bin":5.045233050000011,"total_completion_cost_bin":2.270354249999999,"total_reasoning_cost_bin":2.0387505000000004,"total_cost_bin":13.362739050000009,"total_on_demand_input_cost_bin":46.05200999999997,"ci_99_lower":0.4475598826656508,"ci_99_upper":0.660073073991287,"ci_98_lower":0.46711775338719763,"ci_98_upper":0.653932568932857,"ci_95_lower":0.47656863935296717,"ci_95_upper":0.6387973485743225,"ci_90_lower":0.4900863022056362,"ci_90_upper":0.6251951179853541,"ci_75_lower":0.5123092361828245,"ci_75_upper":0.6058063728434318,"ci_50_lower":0.5299984865973816,"ci_50_upper":0.587378007813905},"524288":{"avg_score":0.4783423868564384,"min_score":0.003955174703150988,"max_score":1,"n_tests":236,"stddev":0.4806447893367864,"median_score":0.08636957034468651,"q1_score":0.030532084871083498,"q3_score":1,"avg_token_efficiency":0.9916652001408923,"is_incomplete":false,"total_prompt_cost_bin":7.850308750000011,"total_cache_read_cost_bin":17.45819040000008,"total_completion_cost_bin":4.34551025,"total_reasoning_cost_bin":3.96210925,"total_cost_bin":33.61611865000009,"total_on_demand_input_cost_bin":153.33522874999983,"ci_99_lower":0.39997363182538487,"ci_99_upper":0.5536071095507104,"ci_98_lower":0.40341528370573926,"ci_98_upper":0.5497270501442293,"ci_95_lower":0.4133526896838379,"ci_95_upper":0.5402386078433927,"ci_90_lower":0.42405657755897647,"ci_90_upper":0.5287738412914608,"ci_75_lower":0.44315198898316416,"ci_75_upper":0.5152786358462359,"ci_50_lower":0.45817307128533485,"ci_50_upper":0.49893937415001377}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.2","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-05T19:48:07.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T19:47:10.622Z"},{"model_slug":"z-ai/glm-5.3","reasoning_mode":"high","overall_metrics":{"auc_1m":0.4089485897277999,"auc_128k":0.8587471756716032,"cum_avg_1m":0.6772290390417013,"cum_avg_128k":0.8881636718545961,"avg_score_overall":0.7039559853920858,"avg_response_time_ms":22113.261153908974,"total_prompt_cost":1.6962722000000001,"total_cache_read_cost":47.417876479999954,"total_completion_cost":6.740452400000001,"total_reasoning_cost":5.3052692000000015,"total_prompt_cost_128k":0.5736724000000001,"total_completion_cost_128k":2.3737868000000004,"total_on_demand_input_cost":257.02329939999987,"total_on_demand_input_cost_128k":34.07457479999999,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9922152709249368},"bin_metrics":{"8192":{"avg_score":0.9595715187373571,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.18805126963523522,"median_score":1,"q1_score":0.996298760175705,"q3_score":1,"avg_token_efficiency":0.9871057389459204,"is_incomplete":false,"total_prompt_cost_bin":0.12183360000000008,"total_cache_read_cost_bin":0.11390080000000004,"total_completion_cost_bin":0.1606176,"total_reasoning_cost_bin":0.038332800000000014,"total_cost_bin":0.43468480000000015,"total_on_demand_input_cost_bin":0.7351456000000001,"ci_99_lower":0.8968335845827824,"ci_99_upper":0.997603007517755,"ci_98_lower":0.8986264881074895,"ci_98_upper":0.9972775740846992,"ci_95_lower":0.9100242221943334,"ci_95_upper":0.9965652412362397,"ci_90_lower":0.9222496010642499,"ci_90_upper":0.9953031720966101,"ci_75_lower":0.9345830742269755,"ci_75_upper":0.9840896812238498,"ci_50_lower":0.9471781733474928,"ci_50_upper":0.9722290210862411},"16384":{"avg_score":0.9489695644704625,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.21374218692611402,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.978922123160325,"is_incomplete":false,"total_prompt_cost_bin":0.07296800000000003,"total_cache_read_cost_bin":0.26687232000000005,"total_completion_cost_bin":0.18104240000000002,"total_reasoning_cost_bin":0.0605616,"total_cost_bin":0.5814443200000001,"total_on_demand_input_cost_bin":1.5099728000000001,"ci_99_lower":0.8762617382468888,"ci_99_upper":0.997892154417932,"ci_98_lower":0.885734608204104,"ci_98_upper":0.9974796852171421,"ci_95_lower":0.8888386247341987,"ci_95_upper":0.9864723656128627,"ci_90_lower":0.900744955720147,"ci_90_upper":0.9855759348277934,"ci_75_lower":0.9239116181968712,"ci_75_upper":0.9739501715434017,"ci_50_lower":0.9362425509956667,"ci_50_upper":0.961893820867408},"32768":{"avg_score":0.9433582227211446,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.21698475966353856,"median_score":1,"q1_score":0.9926232844591141,"q3_score":1,"avg_token_efficiency":1.0076444612558,"is_incomplete":false,"total_prompt_cost_bin":0.09083199999999989,"total_cache_read_cost_bin":0.998882560000002,"total_completion_cost_bin":0.3921984000000002,"total_reasoning_cost_bin":0.15543880000000002,"total_cost_bin":1.637351760000002,"total_on_demand_input_cost_bin":5.469430399999997,"ci_99_lower":0.8863700494329746,"ci_99_upper":0.9853153130377861,"ci_98_lower":0.8935776282699011,"ci_98_upper":0.9809934811065357,"ci_95_lower":0.9010916856168906,"ci_95_upper":0.978019534575972,"ci_90_lower":0.9099206938730169,"ci_90_upper":0.9725708948133295,"ci_75_lower":0.9211844241499956,"ci_75_upper":0.965101358732414,"ci_50_lower":0.9297526987091475,"ci_50_upper":0.9572332234614912},"65536":{"avg_score":0.8961083237300901,"min_score":0.021984733641147614,"max_score":1,"n_tests":85,"stddev":0.2984912820777626,"median_score":1,"q1_score":0.9985548853874207,"q3_score":1,"avg_token_efficiency":0.9893262016400359,"is_incomplete":false,"total_prompt_cost_bin":0.09920959999999998,"total_cache_read_cost_bin":1.4117376000000026,"total_completion_cost_bin":0.7471464000000001,"total_reasoning_cost_bin":0.5972032,"total_cost_bin":2.8552968000000027,"total_on_demand_input_cost_bin":7.7008736,"ci_99_lower":0.8038666384323117,"ci_99_upper":0.9647143615589422,"ci_98_lower":0.8052542452014545,"ci_98_upper":0.9635227626009899,"ci_95_lower":0.8170892584115705,"ci_95_upper":0.95329162558212,"ci_90_lower":0.8377736005647218,"ci_90_upper":0.9425650918834351,"ci_75_lower":0.8608853315112783,"ci_75_upper":0.9306782116291715,"ci_50_lower":0.8735151178915711,"ci_50_upper":0.9186364047229291},"131072":{"avg_score":0.6928107296139255,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.45121715528702977,"median_score":0.9986486434936523,"q1_score":0.0337465014308691,"q3_score":1,"avg_token_efficiency":0.9887354495866274,"is_incomplete":false,"total_prompt_cost_bin":0.1888292,"total_cache_read_cost_bin":3.430202879999993,"total_completion_cost_bin":0.8927819999999999,"total_reasoning_cost_bin":0.7043432000000005,"total_cost_bin":5.216157279999993,"total_on_demand_input_cost_bin":18.6591524,"ci_99_lower":0.5785593391056467,"ci_99_upper":0.8065546513462746,"ci_98_lower":0.5882464629013172,"ci_98_upper":0.7930406823094925,"ci_95_lower":0.5992017644968956,"ci_95_upper":0.7706083624268818,"ci_90_lower":0.6174652251313233,"ci_90_upper":0.7616387688750356,"ci_75_lower":0.6379308851553891,"ci_75_upper":0.7420197284336256,"ci_50_lower":0.6629462415116243,"ci_50_upper":0.7225718316294307},"262144":{"avg_score":0.5576621801443794,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.48213984735293625,"median_score":0.9884792566299438,"q1_score":0.027256567031145096,"q3_score":1,"avg_token_efficiency":0.9954328543594119,"is_incomplete":false,"total_prompt_cost_bin":0.3754730000000002,"total_cache_read_cost_bin":9.488460799999988,"total_completion_cost_bin":1.4785363999999996,"total_reasoning_cost_bin":1.2410596,"total_cost_bin":12.583529799999987,"total_on_demand_input_cost_bin":51.46718499999996,"ci_99_lower":0.44575989172327435,"ci_99_upper":0.6607707099316573,"ci_98_lower":0.46083374143739936,"ci_98_upper":0.6552366248289031,"ci_95_lower":0.4796493792334235,"ci_95_upper":0.6383710967431321,"ci_90_lower":0.4894960492912924,"ci_90_upper":0.6251624132405821,"ci_75_lower":0.5121664137475057,"ci_75_upper":0.6060431139369855,"ci_50_lower":0.5304920076217555,"ci_50_upper":0.5882224691722127},"524288":{"avg_score":0.4193517729162507,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.46933348226789845,"median_score":0.047594040632247925,"q1_score":0.02353279385715723,"q3_score":0.9979110509157181,"avg_token_efficiency":0.990198990066015,"is_incomplete":false,"total_prompt_cost_bin":0.7471267999999998,"total_cache_read_cost_bin":31.70781951999997,"total_completion_cost_bin":2.888129200000001,"total_reasoning_cost_bin":2.508330000000001,"total_cost_bin":37.85140551999997,"total_on_demand_input_cost_bin":171.4815395999999,"ci_99_lower":0.33014369902873836,"ci_99_upper":0.49205407892457426,"ci_98_lower":0.3447351354317566,"ci_98_upper":0.4854650450100672,"ci_95_lower":0.3567984658144454,"ci_95_upper":0.4783806438659421,"ci_90_lower":0.3668194855242707,"ci_90_upper":0.4713562003862689,"ci_75_lower":0.38384264049374783,"ci_75_upper":0.45540920156966774,"ci_50_lower":0.40038129292829044,"ci_50_upper":0.43924404592578836}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":true,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"Z.ai: GLM 5.3","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":0.0000014,"completion":0.0000044,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-18T20:57:35.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-23T19:19:33.246Z"},{"model_slug":"meta/muse-spark-1.2","reasoning_mode":"high","overall_metrics":{"auc_1m":0.40284840685293605,"auc_128k":0.7457958370195527,"cum_avg_1m":0.640372827518129,"cum_avg_128k":0.8175855165253234,"avg_score_overall":0.6770612995278709,"avg_response_time_ms":36158.55258749365,"total_prompt_cost":21.625297499999974,"total_cache_read_cost":24.99486270000009,"total_completion_cost":19.59705599999999,"total_reasoning_cost":18.183047000000002,"total_prompt_cost_128k":6.217727499999999,"total_completion_cost_128k":9.710485,"total_on_demand_input_cost":229.91581999999997,"total_on_demand_input_cost_128k":30.52858125000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.99488296777093},"bin_metrics":{"8192":{"avg_score":0.9757715721381828,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.15228517944086195,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9856975537391586,"is_incomplete":false,"total_prompt_cost_bin":0.42208124999999985,"total_cache_read_cost_bin":0.028030649999999994,"total_completion_cost_bin":0.8961975000000001,"total_reasoning_cost_bin":0.7759395000000001,"total_cost_bin":2.1222489,"total_on_demand_input_cost_bin":0.6556700000000002,"ci_99_lower":0.9268871710196837,"ci_99_upper":1,"ci_98_lower":0.9277316263518296,"ci_98_upper":1,"ci_95_lower":0.939216221193783,"ci_95_upper":1,"ci_90_lower":0.9400670578004793,"ci_90_upper":1,"ci_75_lower":0.9519685625797137,"ci_75_upper":1,"ci_50_lower":0.9638700673589483,"ci_50_upper":0.9880984952207654},"16384":{"avg_score":0.9147787257330492,"min_score":0.01270289346575737,"max_score":1,"n_tests":80,"stddev":0.2769541625587172,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9858622248642381,"is_incomplete":false,"total_prompt_cost_bin":0.5931249999999997,"total_cache_read_cost_bin":0.09185340000000004,"total_completion_cost_bin":1.07383475,"total_reasoning_cost_bin":0.9537850000000004,"total_cost_bin":2.7125981500000003,"total_on_demand_input_cost_bin":1.3585699999999987,"ci_99_lower":0.8178442538056988,"ci_99_upper":0.9876593134542927,"ci_98_lower":0.8294622282872443,"ci_98_upper":0.975809821090661,"ci_95_lower":0.842064645861974,"ci_95_upper":0.9753202087664976,"ci_90_lower":0.8540285269950982,"ci_90_upper":0.9634808074217289,"ci_75_lower":0.8781090604024939,"ci_75_upper":0.951238137180917,"ci_50_lower":0.8906151922652498,"ci_50_upper":0.9389397575694601},"32768":{"avg_score":0.9152628571438768,"min_score":0.01512605044990778,"max_score":1,"n_tests":136,"stddev":0.27345831820507843,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.012243042443212,"is_incomplete":false,"total_prompt_cost_bin":1.0932650000000002,"total_cache_read_cost_bin":0.45753060000000084,"total_completion_cost_bin":2.612151999999999,"total_reasoning_cost_bin":2.3787887499999996,"total_cost_bin":6.541736349999999,"total_on_demand_input_cost_bin":4.906020000000004,"ci_99_lower":0.8509359023050296,"ci_99_upper":0.9655293801439555,"ci_98_lower":0.8519948642954284,"ci_98_upper":0.9647110969985506,"ci_95_lower":0.8658088496093541,"ci_95_upper":0.9580684862887103,"ci_90_lower":0.8728286968283903,"ci_90_upper":0.9508026129822932,"ci_75_lower":0.8869519860488841,"ci_75_upper":0.9433878984615577,"ci_50_lower":0.9005306090522722,"ci_50_upper":0.9301015850180513},"65536":{"avg_score":0.7263186993734801,"min_score":0.007328691892325878,"max_score":1,"n_tests":85,"stddev":0.4389696350729161,"median_score":1,"q1_score":0.04968944191932678,"q3_score":1,"avg_token_efficiency":0.992532513926866,"is_incomplete":false,"total_prompt_cost_bin":1.2623612499999994,"total_cache_read_cost_bin":0.6762843000000005,"total_completion_cost_bin":2.2658832499999995,"total_reasoning_cost_bin":2.1198659999999996,"total_cost_bin":6.3243947999999985,"total_on_demand_input_cost_bin":6.898063750000002,"ci_99_lower":0.5896682491811978,"ci_99_upper":0.8410575384383143,"ci_98_lower":0.6116297897801679,"ci_98_upper":0.8286571202785434,"ci_95_lower":0.6233649565148002,"ci_95_upper":0.8172587641906541,"ci_90_lower":0.6454851634236163,"ci_90_upper":0.7968750961787776,"ci_75_lower":0.6696011282873394,"ci_75_upper":0.7825284551240175,"ci_50_lower":0.6924091471770011,"ci_50_upper":0.7597855121883399},"131072":{"avg_score":0.5557957282380283,"min_score":0.007238883059471846,"max_score":1,"n_tests":103,"stddev":0.4872769829896328,"median_score":1,"q1_score":0.026361236348748207,"q3_score":1,"avg_token_efficiency":0.9917206548567312,"is_incomplete":false,"total_prompt_cost_bin":2.8468950000000004,"total_cache_read_cost_bin":1.6636035000000013,"total_completion_cost_bin":2.8624175000000016,"total_reasoning_cost_bin":2.67694325,"total_cost_bin":10.049859250000004,"total_on_demand_input_cost_bin":16.710257500000004,"ci_99_lower":0.4326809878119955,"ci_99_upper":0.6701200996374584,"ci_98_lower":0.4413844197454582,"ci_98_upper":0.6600841781932825,"ci_95_lower":0.4702497891657896,"ci_95_upper":0.6503512889895028,"ci_90_lower":0.479848341394478,"ci_90_upper":0.6397893457675751,"ci_75_lower":0.4992078329601497,"ci_75_upper":0.6129612959344936,"ci_50_lower":0.5255491961736358,"ci_50_upper":0.5931325405915222},"262144":{"avg_score":0.5897582296566073,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4787083193161519,"median_score":1,"q1_score":0.03763066232204437,"q3_score":1,"avg_token_efficiency":0.9975809987442873,"is_incomplete":false,"total_prompt_cost_bin":5.643609999999992,"total_cache_read_cost_bin":4.849008000000008,"total_completion_cost_bin":3.459066499999998,"total_reasoning_cost_bin":3.2280195000000003,"total_cost_bin":17.179704,"total_on_demand_input_cost_bin":46.052009999999996,"ci_99_lower":0.47834944386855893,"ci_99_upper":0.6917202094295359,"ci_98_lower":0.48844959865733717,"ci_98_upper":0.6856764130901004,"ci_95_lower":0.5071269859899172,"ci_95_upper":0.6713291118877562,"ci_90_lower":0.5216510423845513,"ci_90_upper":0.6575050123458636,"ci_75_lower":0.5444059727142947,"ci_75_upper":0.6376747203171781,"ci_50_lower":0.5636093473509416,"ci_50_upper":0.6179467055002056},"524288":{"avg_score":0.445296807861808,"min_score":0.005519144702702761,"max_score":1,"n_tests":236,"stddev":0.4752704042003465,"median_score":0.06356992200016975,"q1_score":0.02646801806986332,"q3_score":1,"avg_token_efficiency":0.9916652001408927,"is_incomplete":false,"total_prompt_cost_bin":9.763959999999985,"total_cache_read_cost_bin":17.228552250000078,"total_completion_cost_bin":6.427504499999994,"total_reasoning_cost_bin":6.049705000000001,"total_cost_bin":39.46972175000006,"total_on_demand_input_cost_bin":153.33522874999997,"ci_99_lower":0.3619237632064397,"ci_99_upper":0.5169496442340865,"ci_98_lower":0.3695521750189837,"ci_98_upper":0.5141763182364254,"ci_95_lower":0.3804599587651665,"ci_95_upper":0.5072208766786034,"ci_90_lower":0.3917786980682187,"ci_90_upper":0.4984862029318642,"ci_75_lower":0.4094908134138677,"ci_75_upper":0.48177842096863627,"ci_50_lower":0.42593695255888275,"ci_50_upper":0.4639194903592382}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.2","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-05T19:48:07.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T18:39:33.555Z"},{"model_slug":"qwen/qwen3.8-max","reasoning_mode":"low","overall_metrics":{"auc_1m":0.40093908885069834,"auc_128k":0.9098778190193819,"cum_avg_1m":0.7065374152381962,"cum_avg_128k":0.9311855899929707,"avg_score_overall":0.7159278689818673,"avg_response_time_ms":22979.562435274347,"total_prompt_cost":4.632060000000002,"total_cache_read_cost":46.55392000000002,"total_completion_cost":6.597636000000003,"total_reasoning_cost":4.583412,"total_prompt_cost_128k":1.9292520000000009,"total_completion_cost_128k":2.1877200000000006,"total_on_demand_input_cost":377.0634200000002,"total_on_demand_input_cost_128k":49.79050000000001,"total_runs":860,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0205612301911153},"bin_metrics":{"8192":{"avg_score":0.9930317617952824,"min_score":0.9468473196029663,"max_score":1,"n_tests":80,"stddev":0.011503289814064335,"median_score":0.9974396526813507,"q1_score":0.99247607588768,"q3_score":1,"avg_token_efficiency":1.0265361226650422,"is_incomplete":false,"total_prompt_cost_bin":0.21158399999999997,"total_cache_read_cost_bin":0.10992000000000002,"total_completion_cost_bin":0.3226920000000001,"total_reasoning_cost_bin":0.148872,"total_cost_bin":0.7930680000000001,"total_on_demand_input_cost_bin":1.0909439999999992,"ci_99_lower":0.9888791979774834,"ci_99_upper":0.9960807051174343,"ci_98_lower":0.989218166694045,"ci_98_upper":0.9958184349909425,"ci_95_lower":0.9903490461781621,"ci_95_upper":0.9953484713099896,"ci_90_lower":0.9907836832478643,"ci_90_upper":0.9950132907554506,"ci_75_lower":0.9914189880713821,"ci_75_upper":0.9944017673842608,"ci_50_lower":0.9921715682372452,"ci_50_upper":0.993868718110025},"16384":{"avg_score":0.9669909844640643,"min_score":0.011840158142149448,"max_score":1,"n_tests":80,"stddev":0.1543694682694298,"median_score":0.9964780211448669,"q1_score":0.9902336895465851,"q3_score":0.9989913254976273,"avg_token_efficiency":1.0179077998972046,"is_incomplete":false,"total_prompt_cost_bin":0.21472,"total_cache_read_cost_bin":0.2538559999999997,"total_completion_cost_bin":0.33769800000000005,"total_reasoning_cost_bin":0.16621799999999995,"total_cost_bin":0.9724919999999997,"total_on_demand_input_cost_bin":2.245568000000001,"ci_99_lower":0.9152248953906819,"ci_99_upper":0.9941090223528445,"ci_98_lower":0.917493471911177,"ci_98_upper":0.9937781243100763,"ci_95_lower":0.9290003349847393,"ci_95_upper":0.9927824380621313,"ci_90_lower":0.9316547903302126,"ci_90_upper":0.9919062993302942,"ci_75_lower":0.9433770731789992,"ci_75_upper":0.9887079572305083,"ci_50_lower":0.9553372494643554,"ci_50_upper":0.9794041053508408},"32768":{"avg_score":0.9478216813603306,"min_score":0.03251568600535393,"max_score":1,"n_tests":136,"stddev":0.18678728366784147,"median_score":0.9945185780525208,"q1_score":0.9752895534038544,"q3_score":0.9996630251407623,"avg_token_efficiency":1.0266166775377403,"is_incomplete":false,"total_prompt_cost_bin":0.42262400000000006,"total_cache_read_cost_bin":0.9422080000000027,"total_completion_cost_bin":0.6367020000000005,"total_reasoning_cost_bin":0.30531600000000003,"total_cost_bin":2.306850000000003,"total_on_demand_input_cost_bin":7.960287999999999,"ci_99_lower":0.9001301465277999,"ci_99_upper":0.9832142623544068,"ci_98_lower":0.9046407044254354,"ci_98_upper":0.981023770711823,"ci_95_lower":0.9126574503922122,"ci_95_upper":0.9764667916885468,"ci_90_lower":0.9185168435016428,"ci_90_upper":0.9733561898937777,"ci_75_lower":0.9284557467326522,"ci_75_upper":0.9668956295320053,"ci_50_lower":0.9359131732558393,"ci_50_upper":0.9591048991458271},"65536":{"avg_score":0.9326828971176463,"min_score":0.010611956007778645,"max_score":1,"n_tests":85,"stddev":0.2292080607668388,"median_score":0.9966216087341309,"q1_score":0.9803921580314636,"q3_score":0.9990800619125366,"avg_token_efficiency":1.0213573096768616,"is_incomplete":false,"total_prompt_cost_bin":0.3688880000000004,"total_cache_read_cost_bin":1.373567999999999,"total_completion_cost_bin":0.3948479999999999,"total_reasoning_cost_bin":0.182862,"total_cost_bin":2.3201659999999995,"total_on_demand_input_cost_bin":11.357432,"ci_99_lower":0.8531559883044924,"ci_99_upper":0.9812480840636965,"ci_98_lower":0.8634414972123854,"ci_98_upper":0.9794401653325733,"ci_95_lower":0.8752427404316362,"ci_95_upper":0.9764156158665633,"ci_90_lower":0.8858590361382812,"ci_90_upper":0.968701601402615,"ci_75_lower":0.9066058625864303,"ci_75_upper":0.9589288722819593,"ci_50_lower":0.9188493427875288,"ci_50_upper":0.9530322900041939},"131072":{"avg_score":0.8154006252275305,"min_score":0.005540166050195694,"max_score":1,"n_tests":102,"stddev":0.3686525104984757,"median_score":0.9914496541023254,"q1_score":0.9530569761991501,"q3_score":0.999455913901329,"avg_token_efficiency":1.0164203300260524,"is_incomplete":false,"total_prompt_cost_bin":0.7114360000000003,"total_cache_read_cost_bin":3.3031039999999954,"total_completion_cost_bin":0.4957799999999999,"total_reasoning_cost_bin":0.23442000000000002,"total_cost_bin":4.744739999999996,"total_on_demand_input_cost_bin":27.13626800000001,"ci_99_lower":0.71286790307141,"ci_99_upper":0.9013297184166161,"ci_98_lower":0.7288107433161406,"ci_98_upper":0.8922875648532428,"ci_95_lower":0.7389171879727157,"ci_95_upper":0.8814697675226602,"ci_90_lower":0.7490730497808031,"ci_90_upper":0.8722080288537066,"ci_75_lower":0.7696534327388851,"ci_75_upper":0.8558641992550453,"ci_50_lower":0.7901555900404905,"ci_50_upper":0.8421629203800752},"262144":{"avg_score":0.7115805773427776,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.43691568135783143,"median_score":0.9916805028915405,"q1_score":0.05013129487633705,"q3_score":0.9991645812988281,"avg_token_efficiency":1.0181653087470655,"is_incomplete":false,"total_prompt_cost_bin":1.26006,"total_cache_read_cost_bin":9.242944000000003,"total_completion_cost_bin":0.7573799999999996,"total_reasoning_cost_bin":0.42736799999999997,"total_cost_bin":11.687752000000001,"total_on_demand_input_cost_bin":75.20361200000002,"ci_99_lower":0.6075386413215852,"ci_99_upper":0.8074367305805497,"ci_98_lower":0.6227259880230359,"ci_98_upper":0.7973309623687496,"ci_95_lower":0.6332039330974518,"ci_95_upper":0.7858925964852551,"ci_90_lower":0.6492434003659531,"ci_90_upper":0.7724615098591497,"ci_75_lower":0.6676396055610236,"ci_75_upper":0.7528470733788373,"ci_50_lower":0.6844412927195456,"ci_50_upper":0.7370208452425977},"524288":{"avg_score":0.28479079459793866,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4168968947868397,"median_score":0.036370085552334785,"q1_score":0.021547731012105942,"q3_score":0.9180001467466354,"avg_token_efficiency":1.0188801800666105,"is_incomplete":false,"total_prompt_cost_bin":1.4427480000000006,"total_cache_read_cost_bin":31.32832000000002,"total_completion_cost_bin":3.6525360000000036,"total_reasoning_cost_bin":3.1183560000000003,"total_cost_bin":39.541960000000024,"total_on_demand_input_cost_bin":252.06930800000012,"ci_99_lower":0.2152802978806598,"ci_99_upper":0.35362806924986284,"ci_98_lower":0.2209903316387129,"ci_98_upper":0.345696355516527,"ci_95_lower":0.23143437699717173,"ci_95_upper":0.3356607540443508,"ci_90_lower":0.24170705480866087,"ci_90_upper":0.32923769254333113,"ci_75_lower":0.25416080523575796,"ci_75_upper":0.31632922978832984,"ci_50_lower":0.267776191504169,"ci_50_upper":0.3028005334823412}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","enabled"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"Qwen: Qwen3.8 Max","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":0.000002,"completion":0.000006,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-03T04:33:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T19:44:25.553Z"},{"model_slug":"google/gemini-3.1-pro-preview","reasoning_mode":"high","overall_metrics":{"auc_1m":0.399809327515913,"auc_128k":0.7735264277305169,"cum_avg_1m":0.6622077447603189,"cum_avg_128k":0.8508440315831896,"avg_score_overall":0.5383676394832541,"avg_response_time_ms":74201.28464378402,"total_prompt_cost":56.05472,"total_cache_read_cost":192.47590079999958,"total_completion_cost":5.392715999999999,"total_reasoning_cost":113.86232400000002,"total_prompt_cost_128k":6.469375999999997,"total_completion_cost_128k":2.2466880000000002,"total_on_demand_input_cost":1991.6129559999974,"total_on_demand_input_cost_128k":49.093599999999995,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154894},"bin_metrics":{"8192":{"avg_score":1,"min_score":1,"max_score":1,"n_tests":80,"stddev":0,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747248,"is_incomplete":false,"total_prompt_cost_bin":1.0549320000000004,"total_cache_read_cost_bin":0.0008156,"total_completion_cost_bin":0.3357,"total_reasoning_cost_bin":1.9077840000000004,"total_cost_bin":3.2992316000000006,"total_on_demand_input_cost_bin":1.0630880000000005,"ci_99_lower":1,"ci_99_upper":1,"ci_98_lower":1,"ci_98_upper":1,"ci_95_lower":1,"ci_95_upper":1,"ci_90_lower":1,"ci_90_upper":1,"ci_75_lower":1,"ci_75_upper":1,"ci_50_lower":1,"ci_50_upper":1},"16384":{"avg_score":0.9876080380287021,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.11083707736965724,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024465,"is_incomplete":false,"total_prompt_cost_bin":0.7642180000000003,"total_cache_read_cost_bin":0.14395260000000001,"total_completion_cost_bin":0.3326039999999999,"total_reasoning_cost_bin":2.2919519999999998,"total_cost_bin":3.5327266,"total_on_demand_input_cost_bin":2.203743999999997,"ci_99_lower":0.9504321521148086,"ci_99_upper":1,"ci_98_lower":0.9504321521148086,"ci_98_upper":1,"ci_95_lower":0.9628241140861065,"ci_95_upper":1,"ci_90_lower":0.9628241140861065,"ci_90_upper":1,"ci_75_lower":0.9752160760574042,"ci_75_upper":1,"ci_50_lower":0.9752160760574042,"ci_50_upper":1},"32768":{"avg_score":0.9422144837992485,"min_score":0.008746355772018433,"max_score":1,"n_tests":136,"stddev":0.22690368094949823,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000035909094991,"is_incomplete":false,"total_prompt_cost_bin":1.2638499999999986,"total_cache_read_cost_bin":0.6482982000000002,"total_completion_cost_bin":0.635364,"total_reasoning_cost_bin":4.840008000000003,"total_cost_bin":7.387520200000002,"total_on_demand_input_cost_bin":7.746831999999992,"ci_99_lower":0.8859000174788183,"ci_99_upper":0.9860006300531848,"ci_98_lower":0.892096563718172,"ci_98_upper":0.9841859986909035,"ci_95_lower":0.898696075339788,"ci_95_upper":0.9775372409605873,"ci_90_lower":0.9072585789038226,"ci_90_upper":0.9718208082804375,"ci_75_lower":0.9187007290747715,"ci_75_upper":0.9641674250657054,"ci_50_lower":0.9286580034702852,"ci_50_upper":0.9563680754714262},"65536":{"avg_score":0.7486251815824824,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4279174646622142,"median_score":1,"q1_score":0.051848702132701874,"q3_score":1,"avg_token_efficiency":1.000015287852282,"is_incomplete":false,"total_prompt_cost_bin":0.8017180000000002,"total_cache_read_cost_bin":1.0318391999999987,"total_completion_cost_bin":0.4129560000000001,"total_reasoning_cost_bin":4.311672000000001,"total_cost_bin":6.5581852,"total_on_demand_input_cost_bin":11.12011,"ci_99_lower":0.6113835644980564,"ci_99_upper":0.8522750662429915,"ci_98_lower":0.6344151558231782,"ci_98_upper":0.8512295889004189,"ci_95_lower":0.6465515254547491,"ci_95_upper":0.8293221844423234,"ci_90_lower":0.6692226309849716,"ci_90_upper":0.8177156808336868,"ci_75_lower":0.6922425357227707,"ci_75_upper":0.8051532488357386,"ci_50_lower":0.7146332438076464,"ci_50_upper":0.7822530794669601},"131072":{"avg_score":0.5757724545055151,"min_score":0.005760921631008387,"max_score":1,"n_tests":103,"stddev":0.48403842095525995,"median_score":1,"q1_score":0.031457237899303436,"q3_score":1,"avg_token_efficiency":1.0000076410572558,"is_incomplete":false,"total_prompt_cost_bin":2.584657999999998,"total_cache_read_cost_bin":2.4375167999999996,"total_completion_cost_bin":0.5300640000000001,"total_reasoning_cost_bin":7.762404000000003,"total_cost_bin":13.314642800000001,"total_on_demand_input_cost_bin":26.95982600000001,"ci_99_lower":0.4530970777912535,"ci_99_upper":0.6981533827900533,"ci_98_lower":0.4623341337794402,"ci_98_upper":0.6800699093142634,"ci_95_lower":0.48157309807669735,"ci_95_upper":0.6698996576417273,"ci_90_lower":0.4998420144178118,"ci_90_upper":0.6602079782013338,"ci_75_lower":0.5194312137731315,"ci_75_upper":0.6325183067989573,"ci_50_lower":0.5461574993322341,"ci_50_upper":0.6128080257732258},"262144":{"avg_score":0.4741957656302694,"min_score":0.002326934365555644,"max_score":1,"n_tests":141,"stddev":0.4814099604840455,"median_score":0.06868044286966324,"q1_score":0.030090270563960075,"q3_score":1,"avg_token_efficiency":1.000003817936524,"is_incomplete":false,"total_prompt_cost_bin":7.412441999999998,"total_cache_read_cost_bin":13.81379200000005,"total_completion_cost_bin":0.6515520000000002,"total_reasoning_cost_bin":11.641368000000003,"total_cost_bin":33.51915400000005,"total_on_demand_input_cost_bin":147.72433999999996,"ci_99_lower":0.35906371740924015,"ci_99_upper":0.5895905851842407,"ci_98_lower":0.3707297222929716,"ci_98_upper":0.5759853809991032,"ci_95_lower":0.3935701922002886,"ci_95_upper":0.5563358532503438,"ci_90_lower":0.4070558741687077,"ci_90_upper":0.5426136341246492,"ci_75_lower":0.42801340792434633,"ci_75_upper":0.5226339212732707,"ci_50_lower":0.44695566780122115,"ci_50_upper":0.5017966695174456},"524288":{"avg_score":0.31063356638937334,"min_score":0.003484320593997836,"max_score":1,"n_tests":236,"stddev":0.4340267413371076,"median_score":0.03719854727387428,"q1_score":0.023190067149698734,"q3_score":1,"avg_token_efficiency":1.0000019078540323,"is_incomplete":false,"total_prompt_cost_bin":12.133727999999984,"total_cache_read_cost_bin":47.89166719999998,"total_completion_cost_bin":1.0307639999999998,"total_reasoning_cost_bin":32.89466399999999,"total_cost_bin":93.95082319999995,"total_on_demand_input_cost_bin":494.79770800000045,"ci_99_lower":0.24271141232522128,"ci_99_upper":0.3778932372917731,"ci_98_lower":0.24768059614954036,"ci_98_upper":0.37237183871017016,"ci_95_lower":0.2562093343668601,"ci_95_upper":0.3666191365756538,"ci_90_lower":0.26509685120643983,"ci_90_upper":0.35768560284765444,"ci_75_lower":0.2791719896055638,"ci_75_upper":0.3436761214937995,"ci_50_lower":0.29376380601444013,"ci_50_upper":0.33001430771314594},"1048576":{"avg_score":0.2586124681469594,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.40652874192445937,"median_score":0.03417593613266945,"q1_score":0.022400172892957926,"q3_score":0.12813044898211956,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":30.03917400000002,"total_cache_read_cost_bin":126.50801919999957,"total_completion_cost_bin":1.4637119999999995,"total_reasoning_cost_bin":48.212472000000005,"total_cost_bin":206.22337719999956,"total_on_demand_input_cost_bin":1299.997307999997,"ci_99_lower":0.20127663147562155,"ci_99_upper":0.3183763047485357,"ci_98_lower":0.205493502883879,"ci_98_upper":0.31213858963045926,"ci_95_lower":0.2115384982722894,"ci_95_upper":0.30572607956509357,"ci_90_lower":0.22151176458536348,"ci_90_upper":0.2987156909079321,"ci_75_lower":0.23193598344871372,"ci_75_upper":0.28555416032482284,"ci_50_lower":0.2426240902771467,"ci_50_upper":0.27406303155754186}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.1 Pro Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":0.000002,"completion":0.000012,"reasoning":0.000012,"has_tiered_pricing":true},"api_created_at":"2026-02-19T14:00:27.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:21:10.070Z"},{"model_slug":"meta/muse-spark-1.2","reasoning_mode":"low","overall_metrics":{"auc_1m":0.3992565836856498,"auc_128k":0.7349391643415406,"cum_avg_1m":0.6350913331634801,"cum_avg_128k":0.8186654209721732,"avg_score_overall":0.6715186090396802,"avg_response_time_ms":10769.59363430967,"total_prompt_cost":6.763350000000001,"total_cache_read_cost":26.7782964000001,"total_completion_cost":7.99376125,"total_reasoning_cost":6.581771,"total_prompt_cost_128k":1.9204662499999998,"total_completion_cost_128k":4.059527750000001,"total_on_demand_input_cost":229.91582000000017,"total_on_demand_input_cost_128k":30.52858125000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9948829677709302},"bin_metrics":{"8192":{"avg_score":0.9754532845923677,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.15426262918077935,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9856975537391582,"is_incomplete":false,"total_prompt_cost_bin":0.22415125,"total_cache_read_cost_bin":0.05178224999999999,"total_completion_cost_bin":0.4935355000000001,"total_reasoning_cost_bin":0.37333275000000005,"total_cost_bin":1.1428017500000003,"total_on_demand_input_cost_bin":0.6556700000000004,"ci_99_lower":0.9262608061429346,"ci_99_upper":1,"ci_98_lower":0.9264564375397749,"ci_98_upper":1,"ci_95_lower":0.9385839340509847,"ci_95_upper":1,"ci_90_lower":0.938781043770723,"ci_90_upper":1,"ci_75_lower":0.9510051240446046,"ci_75_upper":1,"ci_50_lower":0.9632292043184861,"ci_50_upper":0.9877759197261184},"16384":{"avg_score":0.9514278262853623,"min_score":0.01270289346575737,"max_score":1,"n_tests":80,"stddev":0.2130772741640587,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9858622248642387,"is_incomplete":false,"total_prompt_cost_bin":0.21555875000000002,"total_cache_read_cost_bin":0.13716135000000018,"total_completion_cost_bin":0.5674684999999999,"total_reasoning_cost_bin":0.44894025000000004,"total_cost_bin":1.36912885,"total_on_demand_input_cost_bin":1.3585700000000003,"ci_99_lower":0.8785068868034286,"ci_99_upper":1,"ci_98_lower":0.8897907797172665,"ci_98_upper":1,"ci_95_lower":0.8911388030898525,"ci_95_upper":0.988113317778334,"ci_90_lower":0.9030308995884843,"ci_90_upper":0.9878299894509837,"ci_75_lower":0.9267453986220062,"ci_75_upper":0.9757929722865811,"ci_50_lower":0.9389734232681802,"ci_50_upper":0.9637732966803014},"32768":{"avg_score":0.8946002338420572,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.30075071086802946,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0122430424432125,"is_incomplete":false,"total_prompt_cost_bin":0.29671000000000003,"total_cache_read_cost_bin":0.5531172000000004,"total_completion_cost_bin":1.1643385,"total_reasoning_cost_bin":0.9290627499999997,"total_cost_bin":2.9432284500000003,"total_on_demand_input_cost_bin":4.906020000000001,"ci_99_lower":0.8228386430345778,"ci_99_upper":0.9568664768424231,"ci_98_lower":0.8304182743475951,"ci_98_upper":0.9504668748633498,"ci_95_lower":0.8394830101332285,"ci_95_upper":0.9436659708152173,"ci_90_lower":0.8506515667430529,"ci_90_upper":0.9365084119255805,"ci_75_lower":0.8644761560599898,"ci_75_upper":0.9230326291371431,"ci_50_lower":0.8760767189797822,"ci_50_upper":0.9146276766191209},"65536":{"avg_score":0.6690176706541987,"min_score":0.008358662948012352,"max_score":1,"n_tests":85,"stddev":0.46274160594600605,"median_score":1,"q1_score":0.03955841809511185,"q3_score":1,"avg_token_efficiency":0.9925325139268661,"is_incomplete":false,"total_prompt_cost_bin":0.37044000000000005,"total_cache_read_cost_bin":0.783314850000001,"total_completion_cost_bin":0.7826417500000001,"total_reasoning_cost_bin":0.6363610000000001,"total_cost_bin":2.572757600000001,"total_on_demand_input_cost_bin":6.8980637499999995,"ci_99_lower":0.532379862123936,"ci_99_upper":0.7828175531518372,"ci_98_lower":0.5526305444024723,"ci_98_upper":0.7734935841104563,"ci_95_lower":0.5753657870326081,"ci_95_upper":0.7613541131521411,"ci_90_lower":0.5878522304007237,"ci_90_upper":0.7487382358241388,"ci_75_lower":0.6118261569535689,"ci_75_upper":0.7265493150964818,"ci_50_lower":0.634510869702653,"ci_50_upper":0.7030127175398828},"131072":{"avg_score":0.6028280894868799,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.48104740169069293,"median_score":1,"q1_score":0.028407370671629906,"q3_score":1,"avg_token_efficiency":0.9917206548567309,"is_incomplete":false,"total_prompt_cost_bin":0.8136062499999996,"total_cache_read_cost_bin":1.9075981500000017,"total_completion_cost_bin":1.0515435000000004,"total_reasoning_cost_bin":0.8665112500000002,"total_cost_bin":4.639259150000002,"total_on_demand_input_cost_bin":16.710257500000008,"ci_99_lower":0.48833483898887886,"ci_99_upper":0.7164311035608565,"ci_98_lower":0.49781484720640756,"ci_98_upper":0.7072544726327026,"ci_95_lower":0.5164268743832261,"ci_95_upper":0.6973026974955167,"ci_90_lower":0.5265018314044037,"ci_90_upper":0.6792777412858095,"ci_75_lower":0.546165961089535,"ci_75_upper":0.6595210049491267,"ci_50_lower":0.565823931900278,"ci_50_upper":0.6398720800451645},"262144":{"avg_score":0.5223540397664757,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.48354172102789567,"median_score":1,"q1_score":0.02876869961619377,"q3_score":1,"avg_token_efficiency":0.9975809987442883,"is_incomplete":false,"total_prompt_cost_bin":1.4750412499999999,"total_cache_read_cost_bin":5.349236250000013,"total_completion_cost_bin":1.3367865000000003,"total_reasoning_cost_bin":1.1070824999999995,"total_cost_bin":9.268146500000013,"total_on_demand_input_cost_bin":46.05201000000002,"ci_99_lower":0.41137917896405735,"ci_99_upper":0.6249426895877717,"ci_98_lower":0.42382340001180135,"ci_98_upper":0.6170044000644289,"ci_95_lower":0.4384563885827331,"ci_95_upper":0.6024407323558835,"ci_90_lower":0.45792008221895825,"ci_90_upper":0.5909990434832079,"ci_75_lower":0.4783628377749697,"ci_75_upper":0.5710144634342073,"ci_50_lower":0.4951289767607492,"ci_50_upper":0.549029143679068},"524288":{"avg_score":0.4650495206804993,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.47669756557559884,"median_score":0.08863307908177376,"q1_score":0.030423573218286037,"q3_score":1,"avg_token_efficiency":0.9916652001408925,"is_incomplete":false,"total_prompt_cost_bin":3.3678425000000014,"total_cache_read_cost_bin":17.996086350000084,"total_completion_cost_bin":2.5974469999999994,"total_reasoning_cost_bin":2.2204805000000003,"total_cost_bin":26.181856350000086,"total_on_demand_input_cost_bin":153.33522875000014,"ci_99_lower":0.3841231837020733,"ci_99_upper":0.5407238086463115,"ci_98_lower":0.3894461049846772,"ci_98_upper":0.5335688242930682,"ci_95_lower":0.40075095458113064,"ci_95_upper":0.5271230845632213,"ci_90_lower":0.41269953702808304,"ci_90_upper":0.5165635720897226,"ci_75_lower":0.42782772323291773,"ci_75_upper":0.499661590152354,"ci_50_lower":0.4451542559434498,"ci_50_upper":0.4837544614022945}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.2","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-05T19:48:07.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T20:56:27.416Z"},{"model_slug":"qwen/qwen3.8-max","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.3956254294218814,"auc_128k":0.9332084526009006,"cum_avg_1m":0.7152344932080571,"cum_avg_128k":0.9519030783080733,"avg_score_overall":0.7210882644332204,"avg_response_time_ms":30814.754122253347,"total_prompt_cost":8.817988000000005,"total_cache_read_cost":45.89158400000002,"total_completion_cost":8.646132000000001,"total_reasoning_cost":6.632862000000003,"total_prompt_cost_128k":1.8703439999999993,"total_completion_cost_128k":2.4637740000000004,"total_on_demand_input_cost":375.9506600000002,"total_on_demand_input_cost_128k":49.765384000000005,"total_runs":859,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0197660085199718},"bin_metrics":{"8192":{"avg_score":0.9930317617952824,"min_score":0.9468473196029663,"max_score":1,"n_tests":80,"stddev":0.011503289814064335,"median_score":0.9974396526813507,"q1_score":0.99247607588768,"q3_score":1,"avg_token_efficiency":1.0224972703222286,"is_incomplete":false,"total_prompt_cost_bin":0.21561599999999997,"total_cache_read_cost_bin":0.10889600000000002,"total_completion_cost_bin":0.37144800000000006,"total_reasoning_cost_bin":0.19762799999999997,"total_cost_bin":0.893588,"total_on_demand_input_cost_bin":1.0867839999999998,"ci_99_lower":0.9888791979774834,"ci_99_upper":0.9960807051174343,"ci_98_lower":0.989218166694045,"ci_98_upper":0.9958184349909425,"ci_95_lower":0.9903490461781621,"ci_95_upper":0.9953484713099896,"ci_90_lower":0.9907836832478643,"ci_90_upper":0.9950132907554506,"ci_75_lower":0.9914189880713821,"ci_75_upper":0.9944017673842608,"ci_50_lower":0.9921715682372452,"ci_50_upper":0.993868718110025},"16384":{"avg_score":0.979461366776377,"min_score":0.030832476913928986,"max_score":1,"n_tests":80,"stddev":0.10814829788354222,"median_score":0.9964780211448669,"q1_score":0.9907990992069244,"q3_score":0.9989913254976273,"avg_token_efficiency":1.0159704488335681,"is_incomplete":false,"total_prompt_cost_bin":0.168576,"total_cache_read_cost_bin":0.2591039999999997,"total_completion_cost_bin":0.38459400000000005,"total_reasoning_cost_bin":0.21324599999999996,"total_cost_bin":1.0255199999999998,"total_on_demand_input_cost_bin":2.241408,"ci_99_lower":0.9411497062183917,"ci_99_upper":0.9946020936071872,"ci_98_lower":0.9426797887310385,"ci_98_upper":0.9942227103561163,"ci_95_lower":0.9540390652790665,"ci_95_upper":0.9934972191043198,"ci_90_lower":0.9557310349680483,"ci_90_upper":0.9929720577597618,"ci_75_lower":0.9666594206355512,"ci_75_upper":0.9920620236545802,"ci_50_lower":0.968793911114335,"ci_50_upper":0.9908802317455412},"32768":{"avg_score":0.9811374739853337,"min_score":0.050490882247686386,"max_score":1,"n_tests":136,"stddev":0.08209959920263633,"median_score":0.9960004985332489,"q1_score":0.9789050668478012,"q3_score":1,"avg_token_efficiency":1.0256830410679834,"is_incomplete":false,"total_prompt_cost_bin":0.4155519999999998,"total_cache_read_cost_bin":0.9422080000000027,"total_completion_cost_bin":0.695922,"total_reasoning_cost_bin":0.36504,"total_cost_bin":2.4187220000000025,"total_on_demand_input_cost_bin":7.9532160000000065,"ci_99_lower":0.9589000090386938,"ci_99_upper":0.9910939133978066,"ci_98_lower":0.9596974153597566,"ci_98_upper":0.9907293202070628,"ci_95_lower":0.9657163293210461,"ci_95_upper":0.9901644439929549,"ci_90_lower":0.9679851174847607,"ci_90_upper":0.9895906153189785,"ci_75_lower":0.9732315784648937,"ci_75_upper":0.9885880991487819,"ci_50_lower":0.9761104807805489,"ci_50_upper":0.9872040832963059},"65536":{"avg_score":0.932733640030903,"min_score":0.004449388012290001,"max_score":1,"n_tests":85,"stddev":0.22945161491544872,"median_score":0.9968017339706421,"q1_score":0.9803921580314636,"q3_score":0.9990800619125366,"avg_token_efficiency":1.0209598255175438,"is_incomplete":false,"total_prompt_cost_bin":0.3644679999999995,"total_cache_read_cost_bin":1.373567999999999,"total_completion_cost_bin":0.4437659999999999,"total_reasoning_cost_bin":0.23215799999999998,"total_cost_bin":2.4139599999999986,"total_on_demand_input_cost_bin":11.353011999999998,"ci_99_lower":0.8530172869874274,"ci_99_upper":0.981294863903128,"ci_98_lower":0.8634931248178815,"ci_98_upper":0.9795397482665584,"ci_95_lower":0.8751979258179884,"ci_95_upper":0.9764866836182773,"ci_90_lower":0.885860152596279,"ci_90_upper":0.9687117562530673,"ci_75_lower":0.9066135606043699,"ci_75_upper":0.9589928139192874,"ci_50_lower":0.9188946400604705,"ci_50_upper":0.9530404725057238},"131072":{"avg_score":0.8731511489524707,"min_score":0.00738688837736845,"max_score":1,"n_tests":102,"stddev":0.31128590174642057,"median_score":0.9930680096149445,"q1_score":0.9634176194667816,"q3_score":0.999437540769577,"avg_token_efficiency":1.0162216625662874,"is_incomplete":false,"total_prompt_cost_bin":0.7061319999999999,"total_cache_read_cost_bin":3.3031039999999954,"total_completion_cost_bin":0.5680440000000001,"total_reasoning_cost_bin":0.30633000000000005,"total_cost_bin":4.883609999999996,"total_on_demand_input_cost_bin":27.130964,"ci_99_lower":0.7885840184121009,"ci_99_upper":0.9406014390515309,"ci_98_lower":0.7969091802045667,"ci_98_upper":0.9322335346583641,"ci_95_lower":0.806213548935621,"ci_95_upper":0.9286013224376731,"ci_90_lower":0.8165805338283855,"ci_90_upper":0.9197916197841621,"ci_75_lower":0.8360692991352384,"ci_75_upper":0.9069227236665456,"ci_50_lower":0.8530374367894861,"ci_50_upper":0.8933964741105834},"262144":{"avg_score":0.7055088767504438,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.44095892273191434,"median_score":0.9934512376785278,"q1_score":0.04939006268978119,"q3_score":0.9994992613792419,"avg_token_efficiency":1.018066042397412,"is_incomplete":false,"total_prompt_cost_bin":1.2527279999999994,"total_cache_read_cost_bin":9.242944000000003,"total_completion_cost_bin":1.0031879999999995,"total_reasoning_cost_bin":0.6694139999999998,"total_cost_bin":12.168274000000002,"total_on_demand_input_cost_bin":75.19628000000006,"ci_99_lower":0.5961599875776616,"ci_99_upper":0.8029558305483202,"ci_98_lower":0.6115717484097737,"ci_98_upper":0.7929824261629874,"ci_95_lower":0.6296384046886675,"ci_95_upper":0.7795466591937612,"ci_90_lower":0.6433745045299107,"ci_90_upper":0.7671751559702911,"ci_75_lower":0.659056916591424,"ci_75_upper":0.7478396075987009,"ci_50_lower":0.6781957512779214,"ci_50_upper":0.7317006841396041},"524288":{"avg_score":0.2568516773736461,"min_score":0,"max_score":1,"n_tests":235,"stddev":0.4023433800846824,"median_score":0.03294573724269867,"q1_score":0.02109225932508707,"q3_score":0.15315596014261246,"avg_token_efficiency":1.0188305679666327,"is_incomplete":false,"total_prompt_cost_bin":5.694916000000006,"total_cache_read_cost_bin":30.66176000000002,"total_completion_cost_bin":5.179170000000001,"total_reasoning_cost_bin":4.649046000000003,"total_cost_bin":46.184892000000026,"total_on_demand_input_cost_bin":250.98899600000013,"ci_99_lower":0.19397755649497614,"ci_99_upper":0.3223722941925888,"ci_98_lower":0.19847020465020646,"ci_98_upper":0.3178282168326167,"ci_95_lower":0.2061613519110053,"ci_95_upper":0.3075974936244831,"ci_90_lower":0.21423715334935511,"ci_90_upper":0.30029958446886634,"ci_75_lower":0.2273012992498673,"ci_75_upper":0.287440569272037,"ci_50_lower":0.23920145777742716,"ci_50_upper":0.2742457216925067}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","enabled"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"Qwen: Qwen3.8 Max","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":0.000002,"completion":0.000006,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-03T04:33:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T17:28:33.393Z"},{"model_slug":"z-ai/glm-5.3","reasoning_mode":"max","overall_metrics":{"auc_1m":0.3944086146736884,"auc_128k":0.807799967617093,"cum_avg_1m":0.6591293001078363,"cum_avg_128k":0.8603967044249515,"avg_score_overall":0.6836536747660882,"avg_response_time_ms":43659.09633885471,"total_prompt_cost":1.6606113999999996,"total_cache_read_cost":47.42449919999996,"total_completion_cost":13.042453600000005,"total_reasoning_cost":11.597414400000002,"total_prompt_cost_128k":0.5382804,"total_completion_cost_128k":4.371672799999999,"total_on_demand_input_cost":257.0232993999999,"total_on_demand_input_cost_128k":34.074574799999986,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9922152709249371},"bin_metrics":{"8192":{"avg_score":0.9966563999652862,"min_score":0.9538077116012573,"max_score":1,"n_tests":80,"stddev":0.007862686485893215,"median_score":1,"q1_score":0.9966350048780441,"q3_score":1,"avg_token_efficiency":0.9871057389459196,"is_incomplete":false,"total_prompt_cost_bin":0.10230080000000004,"total_cache_read_cost_bin":0.11752832000000002,"total_completion_cost_bin":0.5181088,"total_reasoning_cost_bin":0.3956348000000003,"total_cost_bin":1.1335727200000003,"total_on_demand_input_cost_bin":0.7351456000000003,"ci_99_lower":0.9938202617429196,"ci_99_upper":0.9986454977989198,"ci_98_lower":0.9942644175216555,"ci_98_upper":0.9985074808821083,"ci_95_lower":0.994708428196609,"ci_95_upper":0.9981904524564743,"ci_90_lower":0.9950227381661535,"ci_90_upper":0.9979293430596591,"ci_75_lower":0.995576662849635,"ci_75_upper":0.9976278682239355,"ci_50_lower":0.9960875025019049,"ci_50_upper":0.9972410514950751},"16384":{"avg_score":0.9265416666457895,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.24360869650341702,"median_score":0.9980654716491699,"q1_score":0.9917350858449936,"q3_score":1,"avg_token_efficiency":0.978922123160325,"is_incomplete":false,"total_prompt_cost_bin":0.05791520000000002,"total_cache_read_cost_bin":0.26966783999999994,"total_completion_cost_bin":0.38510120000000003,"total_reasoning_cost_bin":0.26494599999999996,"total_cost_bin":0.9776302399999999,"total_on_demand_input_cost_bin":1.5099728,"ci_99_lower":0.8451370420535386,"ci_99_upper":0.9833832422752166,"ci_98_lower":0.8512704229143564,"ci_98_upper":0.9815477139339782,"ci_95_lower":0.8645304812048562,"ci_95_upper":0.975603754921467,"ci_90_lower":0.8761256546972436,"ci_90_upper":0.96857005182188,"ci_75_lower":0.893101191965252,"ci_75_upper":0.9573312784501469,"ci_50_lower":0.9080992327842978,"ci_50_upper":0.9448997019339003},"32768":{"avg_score":0.9122866017449482,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.2652218731479715,"median_score":0.9970544576644897,"q1_score":0.9789050668478012,"q3_score":1,"avg_token_efficiency":1.0076444612558006,"is_incomplete":false,"total_prompt_cost_bin":0.09002559999999989,"total_cache_read_cost_bin":0.9990323200000022,"total_completion_cost_bin":1.0729443999999995,"total_reasoning_cost_bin":0.8350144000000004,"total_cost_bin":2.997016720000002,"total_on_demand_input_cost_bin":5.469430399999991,"ci_99_lower":0.8480473906724536,"ci_99_upper":0.9625811584788949,"ci_98_lower":0.8557296388130635,"ci_98_upper":0.9588732035748442,"ci_95_lower":0.863619715824076,"ci_95_upper":0.9550332396797946,"ci_90_lower":0.8703815825790276,"ci_90_upper":0.9481246402656646,"ci_75_lower":0.883970203846676,"ci_75_upper":0.9399205397746888,"ci_50_lower":0.8972318928675069,"ci_50_upper":0.927712660844383},"65536":{"avg_score":0.8130017969976453,"min_score":0.010600706562399864,"max_score":1,"n_tests":85,"stddev":0.37894154003932834,"median_score":0.9985548853874207,"q1_score":0.9770603179931641,"q3_score":1,"avg_token_efficiency":0.9893262016400356,"is_incomplete":false,"total_prompt_cost_bin":0.0992096,"total_cache_read_cost_bin":1.4117376000000026,"total_completion_cost_bin":0.8877308000000002,"total_reasoning_cost_bin":0.7377172000000001,"total_cost_bin":3.136395200000003,"total_on_demand_input_cost_bin":7.700873599999997,"ci_99_lower":0.6988780851010232,"ci_99_upper":0.9055614596253589,"ci_98_lower":0.7093534428915558,"ci_98_upper":0.9029514339263824,"ci_95_lower":0.7219337272748132,"ci_95_upper":0.8909828108779209,"ci_90_lower":0.7430217372812331,"ci_90_upper":0.8723480918817222,"ci_75_lower":0.7649825060767506,"ci_75_upper":0.8585643186507856,"ci_50_lower":0.7886362089063315,"ci_50_upper":0.845135346520692},"131072":{"avg_score":0.6534970567710878,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.46464466735978704,"median_score":0.992023229598999,"q1_score":0.030463471077382565,"q3_score":1,"avg_token_efficiency":0.9887354495866282,"is_incomplete":false,"total_prompt_cost_bin":0.18882920000000003,"total_cache_read_cost_bin":3.430202879999993,"total_completion_cost_bin":1.5077875999999995,"total_reasoning_cost_bin":1.3192344,"total_cost_bin":6.446054079999993,"total_on_demand_input_cost_bin":18.659152399999996,"ci_99_lower":0.5238277034712074,"ci_99_upper":0.7735150211008075,"ci_98_lower":0.5417843752622026,"ci_98_upper":0.757205795620758,"ci_95_lower":0.5604786856659435,"ci_95_upper":0.7387796638608068,"ci_90_lower":0.5774743635512581,"ci_90_upper":0.7287970430502505,"ci_75_lower":0.5967525323724049,"ci_75_upper":0.7089857128527902,"ci_50_lower":0.618024765728584,"ci_50_upper":0.6831714926367779},"262144":{"avg_score":0.5777301701786433,"min_score":0.0019743337761610746,"max_score":1,"n_tests":141,"stddev":0.4775774605475047,"median_score":0.9842022061347961,"q1_score":0.03351955488324165,"q3_score":1,"avg_token_efficiency":0.9954328543594119,"is_incomplete":false,"total_prompt_cost_bin":0.3754730000000001,"total_cache_read_cost_bin":9.488460799999988,"total_completion_cost_bin":2.528455600000001,"total_reasoning_cost_bin":2.2907984000000012,"total_cost_bin":14.68318779999999,"total_on_demand_input_cost_bin":51.46718499999996,"ci_99_lower":0.46828524536944444,"ci_99_upper":0.6797538988644931,"ci_98_lower":0.4758786920942399,"ci_98_upper":0.6747750133044146,"ci_95_lower":0.49679189119293976,"ci_95_upper":0.6589542575579728,"ci_90_lower":0.5108423846938106,"ci_90_upper":0.6460700820525448,"ci_75_lower":0.5358647233224474,"ci_75_upper":0.6259962584524488,"ci_50_lower":0.5508351228216412,"ci_50_upper":0.6064019814795808},"524288":{"avg_score":0.3933207085592893,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4620284060025003,"median_score":0.04936020448803902,"q1_score":0.02518113749101758,"q3_score":0.9935440123081207,"avg_token_efficiency":0.990198990066015,"is_incomplete":false,"total_prompt_cost_bin":0.7468579999999995,"total_cache_read_cost_bin":31.707869439999968,"total_completion_cost_bin":6.142325200000006,"total_reasoning_cost_bin":5.754069199999999,"total_cost_bin":44.35112183999997,"total_on_demand_input_cost_bin":171.48153960000002,"ci_99_lower":0.3079442783050985,"ci_99_upper":0.4668331978520106,"ci_98_lower":0.31715804481393484,"ci_98_upper":0.4626446162256357,"ci_95_lower":0.3344258427161672,"ci_95_upper":0.453151635455966,"ci_90_lower":0.34159195145465815,"ci_90_upper":0.444195509553883,"ci_75_lower":0.3588648835059957,"ci_75_upper":0.4285238256874806,"ci_50_lower":0.37324929500730725,"ci_50_upper":0.41265911788391585}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":true,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"Z.ai: GLM 5.3","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":0.0000014,"completion":0.0000044,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-18T20:57:35.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-23T16:24:17.090Z"},{"model_slug":"meta/muse-spark-1.2","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.3830387512155856,"auc_128k":0.7086503758983156,"cum_avg_1m":0.6137636035888665,"cum_avg_128k":0.7864917538662934,"avg_score_overall":0.6514005923218339,"avg_response_time_ms":62476.14017528219,"total_prompt_cost":23.678065000000007,"total_cache_read_cost":24.74853060000009,"total_completion_cost":37.29550950000001,"total_reasoning_cost":35.87843199999999,"total_prompt_cost_128k":9.95083,"total_completion_cost_128k":17.2079525,"total_on_demand_input_cost":229.9158200000001,"total_on_demand_input_cost_128k":30.528581250000006,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9948829677709297},"bin_metrics":{"8192":{"avg_score":0.9268556962022558,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.2585048336705615,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9856975537391586,"is_incomplete":false,"total_prompt_cost_bin":0.37401500000000004,"total_cache_read_cost_bin":0.033798599999999984,"total_completion_cost_bin":1.17668475,"total_reasoning_cost_bin":1.0571109999999997,"total_cost_bin":2.6416093499999995,"total_on_demand_input_cost_bin":0.6556700000000003,"ci_99_lower":0.841176509415498,"ci_99_upper":0.9878545288229361,"ci_98_lower":0.8526600891018752,"ci_98_upper":0.987702578259632,"ci_95_lower":0.8654836558108218,"ci_95_upper":0.9758014650084078,"ci_90_lower":0.8770185050112196,"ci_90_upper":0.9753756551770494,"ci_75_lower":0.8903216021077242,"ci_75_upper":0.9633093044394627,"ci_50_lower":0.9028138916764875,"ci_50_upper":0.9509327622596174},"16384":{"avg_score":0.9271610809490085,"min_score":0.01270289346575737,"max_score":1,"n_tests":80,"stddev":0.257437907490539,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9858622248642368,"is_incomplete":false,"total_prompt_cost_bin":0.5760412499999998,"total_cache_read_cost_bin":0.09390345000000003,"total_completion_cost_bin":1.4634407500000006,"total_reasoning_cost_bin":1.3437564999999998,"total_cost_bin":3.47714195,"total_on_demand_input_cost_bin":1.35857,"ci_99_lower":0.8414980257654097,"ci_99_upper":0.9880047821439802,"ci_98_lower":0.8527733234255575,"ci_98_upper":0.9877642433624715,"ci_95_lower":0.8657244457420893,"ci_95_upper":0.9759042805084027,"ci_90_lower":0.8771121440222487,"ci_90_upper":0.9754230295307934,"ci_75_lower":0.8907758663932328,"ci_75_upper":0.9632612372428412,"ci_50_lower":0.9034139680850785,"ci_50_upper":0.9510429810266942},"32768":{"avg_score":0.9183784025885603,"min_score":0.03454894572496414,"max_score":1,"n_tests":136,"stddev":0.26361611452128403,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.012243042443211,"is_incomplete":false,"total_prompt_cost_bin":1.6139725,"total_cache_read_cost_bin":0.39504570000000006,"total_completion_cost_bin":3.7942639999999996,"total_reasoning_cost_bin":3.5591667499999997,"total_cost_bin":9.36244895,"total_on_demand_input_cost_bin":4.906019999999998,"ci_99_lower":0.855589560073773,"ci_99_upper":0.9676726407857665,"ci_98_lower":0.8579220161751351,"ci_98_upper":0.9660767677382511,"ci_95_lower":0.8705730706583434,"ci_95_upper":0.959909057249452,"ci_90_lower":0.87737005231902,"ci_90_upper":0.9528257590211818,"ci_75_lower":0.8910015897777425,"ci_75_upper":0.9453317135202643,"ci_50_lower":0.9034717554870225,"ci_50_upper":0.9334245014743989},"65536":{"avg_score":0.6900983014089219,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.45502183864942486,"median_score":1,"q1_score":0.03859250992536545,"q3_score":1,"avg_token_efficiency":0.9925325139268667,"is_incomplete":false,"total_prompt_cost_bin":2.5581812499999996,"total_cache_read_cost_bin":0.5207858999999998,"total_completion_cost_bin":3.7492565,"total_reasoning_cost_bin":3.6019302499999997,"total_cost_bin":10.4301539,"total_on_demand_input_cost_bin":6.898063750000005,"ci_99_lower":0.5633325277652372,"ci_99_upper":0.8036210453806115,"ci_98_lower":0.5742747319050353,"ci_98_upper":0.7939603898737798,"ci_95_lower":0.597371951492832,"ci_95_upper":0.7822155758919305,"ci_90_lower":0.6008710447003079,"ci_90_upper":0.770030226645553,"ci_75_lower":0.6326752450878677,"ci_75_upper":0.7473676790332641,"ci_50_lower":0.655787880792666,"ci_50_upper":0.7237167641219191},"131072":{"avg_score":0.46996528818272054,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4880165670439809,"median_score":0.04372842237353325,"q1_score":0.023030445910990238,"q3_score":1,"avg_token_efficiency":0.9917206548567309,"is_incomplete":false,"total_prompt_cost_bin":4.82862,"total_cache_read_cost_bin":1.4257965000000012,"total_completion_cost_bin":7.0243065000000025,"total_reasoning_cost_bin":6.839996749999999,"total_cost_bin":20.118719750000004,"total_on_demand_input_cost_bin":16.7102575,"ci_99_lower":0.3455751643406829,"ci_99_upper":0.5759470287505853,"ci_98_lower":0.35507344741433033,"ci_98_upper":0.5737982244901269,"ci_95_lower":0.374486086430487,"ci_95_upper":0.5642994831545612,"ci_90_lower":0.3934807985544078,"ci_90_upper":0.5544173270150865,"ci_75_lower":0.41299852973824475,"ci_75_upper":0.5272033724496458,"ci_50_lower":0.43813778726424735,"ci_50_upper":0.5069734067644798},"262144":{"avg_score":0.5279483079071428,"min_score":0.006973500829190016,"max_score":1,"n_tests":141,"stddev":0.48408426960781314,"median_score":1,"q1_score":0.03327907249331474,"q3_score":1,"avg_token_efficiency":0.9975809987442875,"is_incomplete":false,"total_prompt_cost_bin":4.58331875,"total_cache_read_cost_bin":4.97624295000001,"total_completion_cost_bin":7.903410500000004,"total_reasoning_cost_bin":7.6748157499999925,"total_cost_bin":25.137787950000007,"total_on_demand_input_cost_bin":46.052009999999974,"ci_99_lower":0.41759272898476346,"ci_99_upper":0.6311406420603236,"ci_98_lower":0.4272224932510392,"ci_98_upper":0.6240220203350714,"ci_95_lower":0.4447863999655939,"ci_95_upper":0.609219957508084,"ci_90_lower":0.46470435933698734,"ci_90_upper":0.5963310613769193,"ci_75_lower":0.48098789438017453,"ci_75_upper":0.5776670256923514,"ci_50_lower":0.5007987375030994,"ci_50_upper":0.5556426442596494},"524288":{"avg_score":0.4497017514723219,"min_score":0.0013020833721384406,"max_score":1,"n_tests":236,"stddev":0.47531686237606463,"median_score":0.07594693452119827,"q1_score":0.026766119059175253,"q3_score":1,"avg_token_efficiency":0.9916652001408925,"is_incomplete":false,"total_prompt_cost_bin":9.143916250000007,"total_cache_read_cost_bin":17.30295750000008,"total_completion_cost_bin":12.184146500000002,"total_reasoning_cost_bin":11.801655,"total_cost_bin":50.43267525000009,"total_on_demand_input_cost_bin":153.33522875000014,"ci_99_lower":0.3680788446850956,"ci_99_upper":0.523527839983482,"ci_98_lower":0.3747661380341742,"ci_98_upper":0.5202265547129318,"ci_95_lower":0.3860297749238779,"ci_95_upper":0.5110242565792406,"ci_90_lower":0.39579406190723754,"ci_90_upper":0.5031130650668243,"ci_75_lower":0.41304597496670437,"ci_75_upper":0.48494617185096395,"ci_50_lower":0.43003648650495563,"ci_50_upper":0.46797625689682953}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.2","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-05T19:48:07.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T15:24:53.893Z"},{"model_slug":"openai/gpt-5.4","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.3823141244570734,"auc_128k":0.8057622112308847,"cum_avg_1m":0.6541910255738237,"cum_avg_128k":0.8679627088630394,"avg_score_overall":0.676483040265025,"avg_response_time_ms":158975.41651638265,"total_prompt_cost":19.98729750000004,"total_cache_read_cost":73.52937600000016,"total_completion_cost":100.48243500000001,"total_reasoning_cost":95.60430000000002,"total_prompt_cost_128k":3.038782500000001,"total_completion_cost_128k":42.16068,"total_on_demand_input_cost":755.4055250000006,"total_on_demand_input_cost_128k":60.13478250000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778014},"bin_metrics":{"8192":{"avg_score":0.9999649509787559,"min_score":0.9988221526145935,"max_score":1,"n_tests":80,"stddev":0.0001615147662275131,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598381,"is_incomplete":false,"total_prompt_cost_bin":0.13542000000000012,"total_cache_read_cost_bin":0.11625600000000008,"total_completion_cost_bin":2.9862600000000006,"total_reasoning_cost_bin":2.570189999999999,"total_cost_bin":5.808126,"total_on_demand_input_cost_bin":1.2979800000000008,"ci_99_lower":0.9999067075885832,"ci_99_upper":0.9999981081187724,"ci_98_lower":0.9999138641804458,"ci_98_upper":0.9999980986118316,"ci_95_lower":0.9999238483607769,"ci_95_upper":0.999993946403265,"ci_90_lower":0.9999309964478015,"ci_90_upper":0.9999920644238591,"ci_75_lower":0.9999433130025863,"ci_75_upper":0.9999848324805497,"ci_50_lower":0.9999539695680142,"ci_50_upper":0.9999781608581543},"16384":{"avg_score":0.987756414199248,"min_score":0.027475375682115555,"max_score":1,"n_tests":80,"stddev":0.10872198972619432,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188713,"is_incomplete":false,"total_prompt_cost_bin":0.15894000000000022,"total_cache_read_cost_bin":0.25052800000000003,"total_completion_cost_bin":4.736084999999999,"total_reasoning_cost_bin":4.326299999999998,"total_cost_bin":9.471852999999998,"total_on_demand_input_cost_bin":2.664219999999997,"ci_99_lower":0.9512419759482145,"ci_99_upper":0.999967234235257,"ci_98_lower":0.9512908496782183,"ci_98_upper":0.9999595485329629,"ci_95_lower":0.9634158280771227,"ci_95_upper":0.9999488797411322,"ci_90_lower":0.963451312026009,"ci_90_upper":0.9999408444017172,"ci_75_lower":0.9755799472332001,"ci_75_upper":0.9999244546517729,"ci_50_lower":0.9756319492124021,"ci_50_upper":0.9998983366414904},"32768":{"avg_score":0.9151233102893457,"min_score":0.014577259309589863,"max_score":1,"n_tests":136,"stddev":0.27382257150040085,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9952100768549632,"is_incomplete":false,"total_prompt_cost_bin":0.4126799999999996,"total_cache_read_cost_bin":0.9237120000000003,"total_completion_cost_bin":11.526329999999994,"total_reasoning_cost_bin":10.71798,"total_cost_bin":23.580701999999995,"total_on_demand_input_cost_bin":9.649799999999994,"ci_99_lower":0.8505280567617293,"ci_99_upper":0.9658649438829504,"ci_98_lower":0.8520302072239985,"ci_98_upper":0.9646879370464012,"ci_95_lower":0.8655709421830526,"ci_95_upper":0.9580971802776569,"ci_90_lower":0.8725367003541631,"ci_90_upper":0.9507980105521925,"ci_75_lower":0.8868280591362717,"ci_75_upper":0.9431657200629535,"ci_50_lower":0.9001005154020866,"ci_50_upper":0.9304370220069198},"65536":{"avg_score":0.8057853327077977,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.39070715672070017,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.977643584675992,"is_incomplete":false,"total_prompt_cost_bin":0.36229250000000046,"total_cache_read_cost_bin":1.3226879999999976,"total_completion_cost_bin":8.896754999999999,"total_reasoning_cost_bin":8.39085,"total_cost_bin":18.972585499999997,"total_on_demand_input_cost_bin":13.589172500000007,"ci_99_lower":0.6904609594816232,"ci_99_upper":0.9084853882750168,"ci_98_lower":0.6918636486727525,"ci_98_upper":0.8972740217800088,"ci_95_lower":0.7136946080153916,"ci_95_upper":0.8856247711926698,"ci_90_lower":0.7263631380042609,"ci_90_upper":0.8743339459181708,"ci_75_lower":0.7491035411490455,"ci_75_upper":0.8515791148522539,"ci_50_lower":0.7822509912116562,"ci_50_upper":0.8394152657293221},"131072":{"avg_score":0.6311835361400493,"min_score":0.0057299453765153885,"max_score":1,"n_tests":103,"stddev":0.4747605744187245,"median_score":1,"q1_score":0.028277432546019554,"q3_score":1,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":1.9694500000000006,"total_cache_read_cost_bin":3.0964159999999974,"total_completion_cost_bin":14.015250000000002,"total_reasoning_cost_bin":13.370189999999997,"total_cost_bin":32.451306,"total_on_demand_input_cost_bin":32.93361000000001,"ci_99_lower":0.5075720540368567,"ci_99_upper":0.7531451286865145,"ci_98_lower":0.5163528295838301,"ci_98_upper":0.7355766778189725,"ci_95_lower":0.5384663420632138,"ci_95_upper":0.7251501429012245,"ci_90_lower":0.554610467962785,"ci_90_upper":0.7075051824164404,"ci_75_lower":0.5741065057775557,"ci_75_upper":0.6872453735824856,"ci_50_lower":0.5996377369845558,"ci_50_upper":0.660906140099153},"262144":{"avg_score":0.47834630603142453,"min_score":0.006257822271436453,"max_score":1,"n_tests":141,"stddev":0.4832249260539682,"median_score":0.0553586483001709,"q1_score":0.02866130694746971,"q3_score":1,"avg_token_efficiency":0.9824198103380458,"is_incomplete":false,"total_prompt_cost_bin":1.4536675,"total_cache_read_cost_bin":8.925056000000016,"total_completion_cost_bin":19.481564999999996,"total_reasoning_cost_bin":18.679590000000005,"total_cost_bin":48.539878500000015,"total_on_demand_input_cost_bin":90.70422749999993,"ci_99_lower":0.3633902039653655,"ci_99_upper":0.5912046972915186,"ci_98_lower":0.37535648958483403,"ci_98_upper":0.5794789712706659,"ci_95_lower":0.3977719203008386,"ci_95_upper":0.5611431017146504,"ci_90_lower":0.41119267483264976,"ci_90_upper":0.5468847836286873,"ci_75_lower":0.4318618592674058,"ci_75_upper":0.526380307763957,"ci_50_lower":0.45062912408847045,"ci_50_upper":0.5057027930656369},"524288":{"avg_score":0.41536835424396823,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.46534054191516405,"median_score":0.06173054128885269,"q1_score":0.02866526460275054,"q3_score":1,"avg_token_efficiency":0.9774785269226756,"is_incomplete":false,"total_prompt_cost_bin":15.494847500000038,"total_cache_read_cost_bin":58.89472000000014,"total_completion_cost_bin":38.840190000000014,"total_reasoning_cost_bin":37.54920000000003,"total_cost_bin":150.77895750000022,"total_on_demand_input_cost_bin":604.5665150000007,"ci_99_lower":0.3280635060373915,"ci_99_upper":0.4893468190487514,"ci_98_lower":0.3420399020302753,"ci_98_upper":0.48369555350335836,"ci_95_lower":0.35441685490938146,"ci_95_upper":0.4739676447012028,"ci_90_lower":0.3637023443183013,"ci_90_upper":0.4658206513537768,"ci_75_lower":0.380606848470915,"ci_75_upper":0.4503531238274803,"ci_50_lower":0.39620168988052284,"ci_50_upper":0.43508517892189696}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.0000025,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-05T18:12:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T09:47:26.670Z"},{"model_slug":"openai/gpt-5.4","reasoning_mode":"high","overall_metrics":{"auc_1m":0.3803994325963543,"auc_128k":0.7687324170419354,"cum_avg_1m":0.6372151348093316,"cum_avg_128k":0.8360971751657521,"avg_score_overall":0.6614570973138676,"avg_response_time_ms":45209.67686924951,"total_prompt_cost":20.882984999999994,"total_cache_read_cost":73.43980800000014,"total_completion_cost":45.164129999999986,"total_reasoning_cost":40.297035,"total_prompt_cost_128k":2.100542500000001,"total_completion_cost_128k":18.16845,"total_on_demand_input_cost":755.4055250000006,"total_on_demand_input_cost_128k":60.1347825,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778017},"bin_metrics":{"8192":{"avg_score":0.999991125613451,"min_score":0.9996156692504883,"max_score":1,"n_tests":80,"stddev":0.00005596543282954074,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598395,"is_incomplete":false,"total_prompt_cost_bin":0.14950000000000005,"total_cache_read_cost_bin":0.11484800000000009,"total_completion_cost_bin":1.5259200000000002,"total_reasoning_cost_bin":1.1097899999999998,"total_cost_bin":2.900058,"total_on_demand_input_cost_bin":1.2979799999999995,"ci_99_lower":0.9999726392887532,"ci_99_upper":1,"ci_98_lower":0.9999740960448981,"ci_98_upper":1,"ci_95_lower":0.9999774470925331,"ci_95_upper":1,"ci_90_lower":0.9999789148569107,"ci_90_upper":1,"ci_75_lower":0.9999829851090908,"ci_75_upper":1,"ci_50_lower":0.9999870553612709,"ci_50_upper":0.9999959297478199},"16384":{"avg_score":0.9259453479549847,"min_score":0.009598080068826675,"max_score":1,"n_tests":80,"stddev":0.26041265176863926,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188716,"is_incomplete":false,"total_prompt_cost_bin":0.2920600000000001,"total_cache_read_cost_bin":0.23721600000000023,"total_completion_cost_bin":1.9972500000000002,"total_reasoning_cost_bin":1.5926700000000003,"total_cost_bin":4.1191960000000005,"total_on_demand_input_cost_bin":2.664219999999998,"ci_99_lower":0.8395575358706991,"ci_99_upper":0.9876240568798966,"ci_98_lower":0.8510328761520796,"ci_98_upper":0.9872965448895702,"ci_95_lower":0.8635969858817408,"ci_95_upper":0.9753537870800938,"ci_90_lower":0.8751998136035399,"ci_90_upper":0.9746665562433191,"ci_75_lower":0.889082750568923,"ci_75_upper":0.9622638853412354,"ci_50_lower":0.9019187835976481,"ci_50_upper":0.9500112243666081},"32768":{"avg_score":0.8751423188334551,"min_score":0.015374760143458843,"max_score":1,"n_tests":136,"stddev":0.32120853303148694,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9952100768549625,"is_incomplete":false,"total_prompt_cost_bin":0.45907999999999954,"total_cache_read_cost_bin":0.9190719999999998,"total_completion_cost_bin":5.027624999999999,"total_reasoning_cost_bin":4.219305000000003,"total_cost_bin":10.625082,"total_on_demand_input_cost_bin":9.6498,"ci_99_lower":0.7997804855933646,"ci_99_upper":0.9437303508761997,"ci_98_lower":0.807473894086365,"ci_98_upper":0.9369043180323892,"ci_95_lower":0.8182589552902807,"ci_95_upper":0.9258322984238688,"ci_90_lower":0.8286926387525025,"ci_90_upper":0.9183121221407098,"ci_75_lower":0.8409567989737672,"ci_75_upper":0.9057202032152631,"ci_50_lower":0.8544261515455539,"ci_50_upper":0.8951569692289922},"65536":{"avg_score":0.7495086903409923,"min_score":0.006941766478121281,"max_score":1,"n_tests":85,"stddev":0.4263538450810548,"median_score":1,"q1_score":0.05736137554049492,"q3_score":1,"avg_token_efficiency":0.9776435846759917,"is_incomplete":false,"total_prompt_cost_bin":0.521652500000001,"total_cache_read_cost_bin":1.3067519999999977,"total_completion_cost_bin":4.115219999999999,"total_reasoning_cost_bin":3.60582,"total_cost_bin":9.549444499999998,"total_on_demand_input_cost_bin":13.589172500000004,"ci_99_lower":0.6127930492203244,"ci_99_upper":0.8529761029781554,"ci_98_lower":0.6360298212172792,"ci_98_upper":0.8518467872311525,"ci_95_lower":0.6480312908473699,"ci_95_upper":0.8301454552189064,"ci_90_lower":0.6704026175235563,"ci_90_upper":0.8183878934673746,"ci_75_lower":0.6934456311234766,"ci_75_upper":0.8057148239041185,"ci_50_lower":0.7156428531661411,"ci_50_upper":0.7830465585020754},"131072":{"avg_score":0.6298983930858775,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4762932761530856,"median_score":1,"q1_score":0.026625921949744225,"q3_score":1,"avg_token_efficiency":0.9772722316188369,"is_incomplete":false,"total_prompt_cost_bin":0.6782500000000004,"total_cache_read_cost_bin":3.2255359999999973,"total_completion_cost_bin":5.502435,"total_reasoning_cost_bin":4.866854999999999,"total_cost_bin":14.273075999999998,"total_on_demand_input_cost_bin":32.93361,"ci_99_lower":0.5058103708786329,"ci_99_upper":0.7521873695162369,"ci_98_lower":0.5148808870941075,"ci_98_upper":0.7344535932926126,"ci_95_lower":0.536889391707921,"ci_95_upper":0.7241789574100234,"ci_90_lower":0.5530918580326206,"ci_90_upper":0.7065536420508649,"ci_75_lower":0.572590271360806,"ci_75_upper":0.6861351893263081,"ci_50_lower":0.5982478547731677,"ci_50_upper":0.6596722016127969},"262144":{"avg_score":0.5119472994825783,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.48585051702511245,"median_score":0.08280254900455475,"q1_score":0.02964254654943943,"q3_score":1,"avg_token_efficiency":0.982419810338046,"is_incomplete":false,"total_prompt_cost_bin":3.287587499999999,"total_cache_read_cost_bin":8.74166400000001,"total_completion_cost_bin":8.770454999999995,"total_reasoning_cost_bin":7.958744999999999,"total_cost_bin":28.758451500000007,"total_on_demand_input_cost_bin":90.70422749999999,"ci_99_lower":0.3951324843587552,"ci_99_upper":0.6198353021388795,"ci_98_lower":0.4090769377442949,"ci_98_upper":0.6084748876617283,"ci_95_lower":0.42934993224306006,"ci_95_upper":0.5944175103272258,"ci_90_lower":0.4489617437305893,"ci_90_upper":0.5807108403307365,"ci_75_lower":0.4654822294428575,"ci_75_upper":0.5608000420252727,"ci_50_lower":0.48474814442765785,"ci_50_upper":0.5392328608089212},"524288":{"avg_score":0.40528790316331376,"min_score":0.00469667324796319,"max_score":1,"n_tests":236,"stddev":0.4643207837137985,"median_score":0.052944980561733246,"q1_score":0.02542388252913952,"q3_score":1,"avg_token_efficiency":0.9774785269226757,"is_incomplete":false,"total_prompt_cost_bin":15.494854999999994,"total_cache_read_cost_bin":58.89472000000014,"total_completion_cost_bin":18.225224999999995,"total_reasoning_cost_bin":16.943849999999998,"total_cost_bin":109.55865000000013,"total_on_demand_input_cost_bin":604.5665150000007,"ci_99_lower":0.31893234194026493,"ci_99_upper":0.4801107999187464,"ci_98_lower":0.33139341805033556,"ci_98_upper":0.4737808262303835,"ci_95_lower":0.34469947413648916,"ci_95_upper":0.4647846381751213,"ci_90_lower":0.35365014048283366,"ci_90_upper":0.4559938687061638,"ci_75_lower":0.3710506256577022,"ci_75_upper":0.44061151887359234,"ci_50_lower":0.3860830258591874,"ci_50_upper":0.424771342633225}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.0000025,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-05T18:12:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:11:14.989Z"},{"model_slug":"openai/gpt-5.4","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.36876690432141723,"auc_128k":0.6919650214498761,"cum_avg_1m":0.6001846252105031,"cum_avg_128k":0.7782475312382208,"avg_score_overall":0.6288533162595499,"avg_response_time_ms":27265.438843765878,"total_prompt_cost":30.708617500000056,"total_cache_read_cost":72.45404800000014,"total_completion_cost":26.93832000000001,"total_reasoning_cost":22.049100000000003,"total_prompt_cost_128k":2.9383025000000003,"total_completion_cost_128k":10.7388,"total_on_demand_input_cost":755.4055250000013,"total_on_demand_input_cost_128k":60.1347825,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778016},"bin_metrics":{"8192":{"avg_score":0.9645440356340259,"min_score":0.006989952176809311,"max_score":1,"n_tests":80,"stddev":0.18091503472887316,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598384,"is_incomplete":false,"total_prompt_cost_bin":0.13286000000000012,"total_cache_read_cost_bin":0.11651200000000005,"total_completion_cost_bin":0.9595799999999998,"total_reasoning_cost_bin":0.54564,"total_cost_bin":1.7545920000000002,"total_on_demand_input_cost_bin":1.2979800000000008,"ci_99_lower":0.9042805244736373,"ci_99_upper":0.9999959500990808,"ci_98_lower":0.9055843567852863,"ci_98_upper":0.9999959297478199,"ci_95_lower":0.9171592688560486,"ci_95_upper":0.999991125613451,"ci_90_lower":0.9286271929088981,"ci_90_upper":0.9999829851090908,"ci_75_lower":0.9406639896798878,"ci_75_upper":0.9880480125546456,"ci_50_lower":0.9522234605625273,"ci_50_upper":0.9769513066858053},"16384":{"avg_score":0.8420790599426254,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.36082119222218956,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.18742000000000042,"total_cache_read_cost_bin":0.24767999999999987,"total_completion_cost_bin":1.3517700000000004,"total_reasoning_cost_bin":0.9418949999999999,"total_cost_bin":2.7287650000000006,"total_on_demand_input_cost_bin":2.66422,"ci_99_lower":0.7309176594230812,"ci_99_upper":0.9396652186224237,"ci_98_lower":0.732755839566933,"ci_98_upper":0.9273741429543589,"ci_95_lower":0.7570350461493945,"ci_95_upper":0.9150331213069148,"ci_90_lower":0.7700107058451977,"ci_90_upper":0.9028645580541342,"ci_75_lower":0.7934968650894006,"ci_75_upper":0.890235678353929,"ci_50_lower":0.817320474673761,"ci_50_upper":0.8663883571105544},"32768":{"avg_score":0.8394749021424692,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.3572680434544465,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9952100768549628,"is_incomplete":false,"total_prompt_cost_bin":0.6200399999999995,"total_cache_read_cost_bin":0.9029760000000001,"total_completion_cost_bin":3.073665000000001,"total_reasoning_cost_bin":2.267475000000001,"total_cost_bin":6.864156000000001,"total_on_demand_input_cost_bin":9.649799999999997,"ci_99_lower":0.7590015080160811,"ci_99_upper":0.910107426764358,"ci_98_lower":0.7667908236354946,"ci_98_upper":0.9060359754111372,"ci_95_lower":0.7766866477550459,"ci_95_upper":0.8958318086263127,"ci_90_lower":0.7832605575389393,"ci_90_upper":0.8894958591401947,"ci_75_lower":0.8001914018348222,"ci_75_upper":0.8744956566913649,"ci_50_lower":0.817888441392878,"ci_50_upper":0.8602967733270763},"65536":{"avg_score":0.5675506868509247,"min_score":0.00469667324796319,"max_score":1,"n_tests":85,"stddev":0.4840680626139874,"median_score":1,"q1_score":0.03145589679479599,"q3_score":1,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":0.6810125000000008,"total_cache_read_cost_bin":1.2908159999999977,"total_completion_cost_bin":2.2699799999999994,"total_reasoning_cost_bin":1.7622000000000004,"total_cost_bin":6.004008499999998,"total_on_demand_input_cost_bin":13.589172500000002,"ci_99_lower":0.4289138534352567,"ci_99_upper":0.6936223098729902,"ci_98_lower":0.4418215557231623,"ci_98_upper":0.6912762482154786,"ci_95_lower":0.4663127387236968,"ci_95_upper":0.670651289555792,"ci_90_lower":0.485230011400931,"ci_90_upper":0.6585854671161403,"ci_75_lower":0.5092409615493927,"ci_75_upper":0.6283485134960809,"ci_50_lower":0.5323569963206811,"ci_50_upper":0.6020973834770676},"131072":{"avg_score":0.6775889716210585,"min_score":0.005760921631008387,"max_score":1,"n_tests":103,"stddev":0.4609792110931609,"median_score":1,"q1_score":0.03203001245856285,"q3_score":1,"avg_token_efficiency":0.9772722316188368,"is_incomplete":false,"total_prompt_cost_bin":1.3169699999999995,"total_cache_read_cost_bin":3.1616639999999974,"total_completion_cost_bin":3.083804999999999,"total_reasoning_cost_bin":2.447340000000001,"total_cost_bin":10.009778999999996,"total_on_demand_input_cost_bin":32.93361,"ci_99_lower":0.5544705769577,"ci_99_upper":0.792004576038967,"ci_98_lower":0.5641875835462898,"ci_98_upper":0.7736985239271272,"ci_95_lower":0.5831367579555519,"ci_95_upper":0.7625196871108113,"ci_90_lower":0.6013126644785108,"ci_90_upper":0.7532877187674087,"ci_75_lower":0.6206905826003951,"ci_75_upper":0.733506743323727,"ci_50_lower":0.6461221836925511,"ci_50_upper":0.7067468017362311},"262144":{"avg_score":0.5274282845648679,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4837581077522586,"median_score":0.9963000416755676,"q1_score":0.03321622312068939,"q3_score":1,"avg_token_efficiency":0.9824198103380458,"is_incomplete":false,"total_prompt_cost_bin":4.5583075,"total_cache_read_cost_bin":8.614592000000012,"total_completion_cost_bin":4.59378,"total_reasoning_cost_bin":3.7885500000000008,"total_cost_bin":21.555229500000014,"total_on_demand_input_cost_bin":90.70422749999993,"ci_99_lower":0.4169941602426033,"ci_99_upper":0.6298903936968707,"ci_98_lower":0.42563291678713727,"ci_98_upper":0.6233143644630644,"ci_95_lower":0.44410818955276476,"ci_95_upper":0.6086108076954194,"ci_90_lower":0.46447621201799116,"ci_90_upper":0.5958336625803025,"ci_75_lower":0.48062728518943787,"ci_75_upper":0.5774707500013714,"ci_50_lower":0.5002078335907257,"ci_50_upper":0.5551257801377578},"524288":{"avg_score":0.38281106092805295,"min_score":0.002119392389431596,"max_score":1,"n_tests":236,"stddev":0.45885657132164204,"median_score":0.046371206641197205,"q1_score":0.025468704756349325,"q3_score":1,"avg_token_efficiency":0.9774785269226757,"is_incomplete":false,"total_prompt_cost_bin":23.212007500000055,"total_cache_read_cost_bin":58.119808000000134,"total_completion_cost_bin":11.605740000000011,"total_reasoning_cost_bin":10.295999999999998,"total_cost_bin":103.2335555000002,"total_on_demand_input_cost_bin":604.5665150000013,"ci_99_lower":0.30460449424794267,"ci_99_upper":0.4552344536628097,"ci_98_lower":0.3104738210841729,"ci_98_upper":0.45063278238626464,"ci_95_lower":0.323638084066649,"ci_95_upper":0.44098397807550604,"ci_90_lower":0.3313381380483437,"ci_90_upper":0.4333496255602814,"ci_75_lower":0.3485218214520531,"ci_75_upper":0.41759305219676535,"ci_50_lower":0.36356146099717723,"ci_50_upper":0.40183431578216033}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.0000025,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-05T18:12:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T03:21:12.674Z"},{"model_slug":"z-ai/glm-5.3-flash","reasoning_mode":"high","overall_metrics":{"auc_1m":0.368073464719938,"auc_128k":0.7533698513679468,"cum_avg_1m":0.6067908029042485,"cum_avg_128k":0.7954843634612345,"avg_score_overall":0.6343124996275231,"avg_response_time_ms":41591.83160663903,"total_prompt_cost":0.6621045750000003,"total_cache_read_cost":2.5950979200000006,"total_completion_cost":0.43856650000000014,"total_reasoning_cost":0,"total_prompt_cost_128k":0.17866455000000003,"total_completion_cost_128k":0.15477024999999997,"total_on_demand_input_cost":13.637594174999984,"total_on_demand_input_cost_128k":1.82814615,"total_runs":857,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9945235004327082},"bin_metrics":{"8192":{"avg_score":0.9385191911365837,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.23903743009713976,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9987562745501892,"is_incomplete":false,"total_prompt_cost_bin":0.01580400000000001,"total_cache_read_cost_bin":0.004805760000000004,"total_completion_cost_bin":0.01047025,"total_reasoning_cost_bin":0,"total_cost_bin":0.031080010000000012,"total_on_demand_input_cost_bin":0.03983279999999996,"ci_99_lower":0.8535012072848621,"ci_99_upper":0.9880117036716547,"ci_98_lower":0.8647599352532997,"ci_98_upper":0.9878408710677177,"ci_95_lower":0.8766260078479535,"ci_95_upper":0.9875045791640877,"ci_90_lower":0.8891829360835254,"ci_90_upper":0.9757273266790435,"ci_75_lower":0.9020390645193401,"ci_75_upper":0.9636905197170563,"ci_50_lower":0.9254988408647478,"ci_50_upper":0.9516094835940747},"16384":{"avg_score":0.7930041916377377,"min_score":0.006335797253996134,"max_score":1,"n_tests":80,"stddev":0.40063095137873217,"median_score":1,"q1_score":0.9994918704032898,"q3_score":1,"avg_token_efficiency":0.9845106358438895,"is_incomplete":false,"total_prompt_cost_bin":0.010325400000000012,"total_cache_read_cost_bin":0.014203199999999985,"total_completion_cost_bin":0.014488500000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.0390171,"total_on_demand_input_cost_bin":0.08134140000000002,"ci_99_lower":0.6708810624132167,"ci_99_upper":0.8916546508095343,"ci_98_lower":0.6823284854658995,"ci_98_upper":0.8906917508933111,"ci_95_lower":0.6954727793893836,"ci_95_upper":0.8785000791680068,"ci_90_lower":0.7195398475517868,"ci_90_upper":0.8656917176846765,"ci_75_lower":0.7439181479334365,"ci_75_upper":0.8416721665489604,"ci_50_lower":0.7649020342651057,"ci_50_upper":0.8185415682732127},"32768":{"avg_score":0.8586948029577102,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.33499362376841024,"median_score":1,"q1_score":0.9960471242666245,"q3_score":1,"avg_token_efficiency":1.0103376433801001,"is_incomplete":false,"total_prompt_cost_bin":0.015975000000000007,"total_cache_read_cost_bin":0.055559040000000004,"total_completion_cost_bin":0.029764999999999982,"total_reasoning_cost_bin":0,"total_cost_bin":0.10129903999999999,"total_on_demand_input_cost_bin":0.2937701999999999,"ci_99_lower":0.7842079242095712,"ci_99_upper":0.9278175628234577,"ci_98_lower":0.789884095313096,"ci_98_upper":0.9234514520956915,"ci_95_lower":0.7987554026004493,"ci_95_upper":0.914034940259615,"ci_90_lower":0.8080874872657553,"ci_90_upper":0.9058045676590102,"ci_75_lower":0.8229743749462239,"ci_75_upper":0.8920507708541717,"ci_50_lower":0.8376712223736618,"ci_50_upper":0.8779054168194039},"65536":{"avg_score":0.758441476162304,"min_score":0.009786988608539104,"max_score":1,"n_tests":85,"stddev":0.4180870199007373,"median_score":1,"q1_score":0.9228426218032837,"q3_score":1,"avg_token_efficiency":0.9904727905611433,"is_incomplete":false,"total_prompt_cost_bin":0.028823324999999997,"total_cache_read_cost_bin":0.07684031999999999,"total_completion_cost_bin":0.042791,"total_reasoning_cost_bin":0,"total_cost_bin":0.148454645,"total_on_demand_input_cost_bin":0.4130249250000001,"ci_99_lower":0.6310177184856114,"ci_99_upper":0.8618138124407543,"ci_98_lower":0.635286832169794,"ci_98_upper":0.8505159230403164,"ci_95_lower":0.6650121730459198,"ci_95_upper":0.8390756190990042,"ci_90_lower":0.679756637904793,"ci_90_upper":0.8281116046553807,"ci_75_lower":0.7052104147493511,"ci_75_upper":0.8063720073838554,"ci_50_lower":0.7260504170183969,"ci_50_upper":0.7913366566028665},"131072":{"avg_score":0.6287621554118373,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.47030566024438797,"median_score":1,"q1_score":0.03466278873383999,"q3_score":1,"avg_token_efficiency":0.9893085288808494,"is_incomplete":false,"total_prompt_cost_bin":0.10773682500000002,"total_cache_read_cost_bin":0.17848799999999984,"total_completion_cost_bin":0.057255499999999994,"total_reasoning_cost_bin":0,"total_cost_bin":0.34348032499999986,"total_on_demand_input_cost_bin":1.000176825,"ci_99_lower":0.5056959161656068,"ci_99_upper":0.7479798351814384,"ci_98_lower":0.5156015729010829,"ci_98_upper":0.7323281292477848,"ci_95_lower":0.5381132468090674,"ci_95_upper":0.7223754238373447,"ci_90_lower":0.5530379387929818,"ci_90_upper":0.7049321973109753,"ci_75_lower":0.5721293933578264,"ci_75_upper":0.683906468775268,"ci_50_lower":0.5969702236968074,"ci_50_upper":0.6600675830155276},"262144":{"avg_score":0.4865306699548715,"min_score":0.005610098131000996,"max_score":1,"n_tests":140,"stddev":0.48244773411233666,"median_score":0.07309429720044136,"q1_score":0.028317831456661224,"q3_score":1,"avg_token_efficiency":0.9957192474762526,"is_incomplete":false,"total_prompt_cost_bin":0.052390874999999955,"total_cache_read_cost_bin":0.5372025599999986,"total_completion_cost_bin":0.11645625000000007,"total_reasoning_cost_bin":0,"total_cost_bin":0.7060496849999985,"total_on_demand_input_cost_bin":2.7384036749999976,"ci_99_lower":0.37113845207869806,"ci_99_upper":0.5895210605998457,"ci_98_lower":0.3896550928744421,"ci_98_upper":0.5827602919966447,"ci_95_lower":0.40445659813354723,"ci_95_upper":0.5668759295930587,"ci_90_lower":0.42254998550151607,"ci_90_upper":0.5545409631243507,"ci_75_lower":0.4399659868758005,"ci_75_upper":0.5337112508809826,"ci_50_lower":0.45969166165034814,"ci_50_upper":0.5134552871783463},"524288":{"avg_score":0.39037393597294395,"min_score":0,"max_score":1,"n_tests":233,"stddev":0.46368030324720416,"median_score":0.050526317209005356,"q1_score":0.0260743610560894,"q3_score":1,"avg_token_efficiency":0.9903420910751545,"is_incomplete":false,"total_prompt_cost_bin":0.4310491500000003,"total_cache_read_cost_bin":1.7279990400000023,"total_completion_cost_bin":0.1673400000000001,"total_reasoning_cost_bin":0,"total_cost_bin":2.326388190000003,"total_on_demand_input_cost_bin":9.071044349999987,"ci_99_lower":0.31156688946559064,"ci_99_upper":0.4708692901461613,"ci_98_lower":0.32183751180068526,"ci_98_upper":0.4628959815402799,"ci_95_lower":0.3301892877642821,"ci_95_upper":0.45048491717281236,"ci_90_lower":0.34155303782557617,"ci_90_upper":0.4398708968526232,"ci_75_lower":0.35634135557668356,"ci_75_upper":0.42424792174775505,"ci_50_lower":0.36892982496122545,"ci_50_upper":0.4109248849228544}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":true,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"Z.ai: GLM 5.3 Flash","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":7.5e-8,"completion":2.5e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-26T13:59:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"meta/muse-spark-1.1","reasoning_mode":"minimal","overall_metrics":{"auc_1m":0.3661916835623741,"auc_128k":0.6306628689898928,"cum_avg_1m":0.5619453795326286,"cum_avg_128k":0.7086671466071488,"avg_score_overall":0.6005556959048797,"avg_response_time_ms":28270.411566934796,"total_prompt_cost":0.6544612500000039,"total_cache_read_cost":27.531639599999913,"total_completion_cost":6.6462520000000005,"total_reasoning_cost":5.239446750000001,"total_prompt_cost_128k":0.002420000000000005,"total_completion_cost_128k":3.47241575,"total_on_demand_input_cost":230.08479125000022,"total_on_demand_input_cost_128k":30.623566250000014,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9996872469154788},"bin_metrics":{"8192":{"avg_score":0.9024844398139976,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.2944141248440178,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.010086008270763,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.08051640000000004,"total_completion_cost_bin":0.48500574999999996,"total_reasoning_cost_bin":0.36416975000000007,"total_cost_bin":0.9300919000000001,"total_on_demand_input_cost_bin":0.6713699999999998,"ci_99_lower":0.8163287645705277,"ci_99_upper":0.975469152672391,"ci_98_lower":0.816797381366021,"ci_98_upper":0.963874607441714,"ci_95_lower":0.8289614701998653,"ci_95_upper":0.9630340756379884,"ci_90_lower":0.8415352716506458,"ci_90_upper":0.9512685798224993,"ci_75_lower":0.86518437093182,"ci_75_upper":0.9389761002195882,"ci_50_lower":0.8782527374685742,"ci_50_upper":0.9266389870259444},"16384":{"avg_score":0.7668965581571683,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4203707919232108,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9975608447485003,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.16486440000000008,"total_completion_cost_bin":0.45303725000000017,"total_reasoning_cost_bin":0.33821075,"total_cost_bin":0.9565124000000003,"total_on_demand_input_cost_bin":1.3742700000000005,"ci_99_lower":0.6433236833150732,"ci_99_upper":0.8780090204799781,"ci_98_lower":0.655354284272762,"ci_98_upper":0.8654480025303782,"ci_95_lower":0.6698131935545825,"ci_95_upper":0.8531885934606545,"ci_90_lower":0.692292277640081,"ci_90_upper":0.8405996078299359,"ci_75_lower":0.7171163569903001,"ci_75_upper":0.81638848605362,"ci_50_lower":0.7311270766600501,"ci_50_upper":0.8029056477651466},"32768":{"avg_score":0.7962341415336328,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.39282033441745406,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.017880770356745,"is_incomplete":false,"total_prompt_cost_bin":0.0006800000000000015,"total_cache_read_cost_bin":0.5918436000000005,"total_completion_cost_bin":0.9880824999999999,"total_reasoning_cost_bin":0.7567847499999999,"total_cost_bin":2.3373908500000002,"total_on_demand_input_cost_bin":4.932710000000007,"ci_99_lower":0.6990426510230977,"ci_99_upper":0.8805684954456313,"ci_98_lower":0.7114867146349828,"ci_98_upper":0.8731060416183455,"ci_95_lower":0.7248729047353074,"ci_95_upper":0.8605126643206699,"ci_90_lower":0.7381840038130416,"ci_90_upper":0.8514611372277346,"ci_75_lower":0.7541716291598278,"ci_75_upper":0.8366413291294199,"ci_50_lower":0.7733737571115318,"ci_50_upper":0.8188850102451397},"65536":{"avg_score":0.5793855894904802,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4821845818919602,"median_score":1,"q1_score":0.032487887889146805,"q3_score":1,"avg_token_efficiency":0.9949327067350517,"is_incomplete":false,"total_prompt_cost_bin":0.0004250000000000008,"total_cache_read_cost_bin":0.8297184000000002,"total_completion_cost_bin":0.7065795,"total_reasoning_cost_bin":0.5579145,"total_cost_bin":2.0946374000000003,"total_on_demand_input_cost_bin":6.9147449999999955,"ci_99_lower":0.4395388061872081,"ci_99_upper":0.7049078527125804,"ci_98_lower":0.45254175382490985,"ci_98_upper":0.7023488397015368,"ci_95_lower":0.47741976612001,"ci_95_upper":0.681313361665234,"ci_90_lower":0.49764891565119956,"ci_90_upper":0.6699060409329832,"ci_75_lower":0.5212480336126378,"ci_75_upper":0.6452757531037445,"ci_50_lower":0.5452479941167814,"ci_50_upper":0.6135472623713534},"131072":{"avg_score":0.49833500404046505,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.48970135302090484,"median_score":0.0555555559694767,"q1_score":0.024666928686201572,"q3_score":1,"avg_token_efficiency":0.9929203008459693,"is_incomplete":false,"total_prompt_cost_bin":0.000515000000000001,"total_cache_read_cost_bin":2.0075947499999995,"total_completion_cost_bin":0.83971075,"total_reasoning_cost_bin":0.6549760000000006,"total_cost_bin":3.5027965,"total_on_demand_input_cost_bin":16.73047125000001,"ci_99_lower":0.3744036191517094,"ci_99_upper":0.6215178514593869,"ci_98_lower":0.38542351022199955,"ci_98_upper":0.6115450313186626,"ci_95_lower":0.4034744502158239,"ci_95_upper":0.5929981600085173,"ci_90_lower":0.4216496084269715,"ci_90_upper":0.5752482583808477,"ci_75_lower":0.44166294062703404,"ci_75_upper":0.5554812486561522,"ci_50_lower":0.4683631924167502,"ci_50_upper":0.534971995784978},"262144":{"avg_score":0.5269662420651776,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.48512426429374583,"median_score":1,"q1_score":0.029645541682839394,"q3_score":1,"avg_token_efficiency":0.9981804147787295,"is_incomplete":false,"total_prompt_cost_bin":0.0007050000000000015,"total_cache_read_cost_bin":5.529477149999993,"total_completion_cost_bin":1.14133325,"total_reasoning_cost_bin":0.9112807500000003,"total_cost_bin":7.582796149999993,"total_on_demand_input_cost_bin":46.079681249999986,"ci_99_lower":0.4164173716823785,"ci_99_upper":0.6304251549483428,"ci_98_lower":0.4260806451680251,"ci_98_upper":0.623096499475799,"ci_95_lower":0.4434854034109434,"ci_95_upper":0.6083942400286195,"ci_90_lower":0.4632770751568271,"ci_90_upper":0.5954392775679369,"ci_75_lower":0.48001668419575827,"ci_75_upper":0.577004263832247,"ci_50_lower":0.49969951636223986,"ci_50_upper":0.5548420292069369},"524288":{"avg_score":0.425261061160107,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.47333224715324845,"median_score":0.04954055696725845,"q1_score":0.028103931806981564,"q3_score":1,"avg_token_efficiency":0.9919647332235331,"is_incomplete":false,"total_prompt_cost_bin":0.6513362500000038,"total_cache_read_cost_bin":18.327624899999922,"total_completion_cost_bin":2.032503,"total_reasoning_cost_bin":1.65611025,"total_cost_bin":22.667574399999925,"total_on_demand_input_cost_bin":153.38154375000022,"ci_99_lower":0.33794492964802997,"ci_99_upper":0.4990368843083053,"ci_98_lower":0.35007877331949233,"ci_98_upper":0.49290893236345856,"ci_95_lower":0.36187960619927717,"ci_95_upper":0.48313038116187784,"ci_90_lower":0.37102957486320026,"ci_90_upper":0.47759533213487015,"ci_75_lower":0.3893246469190256,"ci_75_upper":0.46205945337517784,"ci_50_lower":0.40650982150790216,"ci_50_upper":0.4450351381102198}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.1","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:29:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-17T05:25:49.251Z"},{"model_slug":"meta/muse-spark-1.2","reasoning_mode":"minimal","overall_metrics":{"auc_1m":0.36592188805074116,"auc_128k":0.6952375876541715,"cum_avg_1m":0.6034503022078139,"cum_avg_128k":0.7801223823524697,"avg_score_overall":0.6282715974056511,"avg_response_time_ms":7668.9727940576,"total_prompt_cost":3.288808750000002,"total_cache_read_cost":27.195241350000106,"total_completion_cost":4.93071825,"total_reasoning_cost":3.511961999999999,"total_prompt_cost_128k":0.8602525000000001,"total_completion_cost_128k":2.6316170000000003,"total_on_demand_input_cost":229.91582,"total_on_demand_input_cost_128k":30.528581250000006,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.99488296777093},"bin_metrics":{"8192":{"avg_score":0.9512822703458369,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.21371813888068653,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.985697553739158,"is_incomplete":false,"total_prompt_cost_bin":0.09113125000000002,"total_cache_read_cost_bin":0.06774465,"total_completion_cost_bin":0.35007250000000006,"total_reasoning_cost_bin":0.229925,"total_cost_bin":0.7388734000000001,"total_on_demand_input_cost_bin":0.6556700000000003,"ci_99_lower":0.8780787255660398,"ci_99_upper":1,"ci_98_lower":0.8895616216058844,"ci_98_upper":1,"ci_95_lower":0.8909736962872556,"ci_95_upper":0.9880984952207654,"ci_90_lower":0.9027247971156612,"ci_90_upper":0.9878333333414048,"ci_75_lower":0.9264847835584077,"ci_75_upper":0.9757721081317867,"ci_50_lower":0.9387993787880987,"ci_50_upper":0.9637651619035751},"16384":{"avg_score":0.9640007246518507,"min_score":0.026058631017804146,"max_score":1,"n_tests":80,"stddev":0.18354159169365183,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9858622248642366,"is_incomplete":false,"total_prompt_cost_bin":0.11497625000000002,"total_cache_read_cost_bin":0.14923125000000015,"total_completion_cost_bin":0.37253375,"total_reasoning_cost_bin":0.25300674999999995,"total_cost_bin":0.8897480000000001,"total_on_demand_input_cost_bin":1.3585699999999992,"ci_99_lower":0.9036519935926189,"ci_99_upper":1,"ci_98_lower":0.9041229543022345,"ci_98_upper":1,"ci_95_lower":0.9160631232894957,"ci_95_upper":1,"ci_90_lower":0.9278941112325991,"ci_90_upper":1,"ci_75_lower":0.9399397753179073,"ci_75_upper":0.9880616739857941,"ci_50_lower":0.9518781013321131,"ci_50_upper":0.9761749917641283},"32768":{"avg_score":0.7960879520139903,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.39357689184360733,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.012243042443213,"is_incomplete":false,"total_prompt_cost_bin":0.15315374999999992,"total_cache_read_cost_bin":0.5703439500000008,"total_completion_cost_bin":0.7810650000000001,"total_reasoning_cost_bin":0.54613775,"total_cost_bin":2.050700450000001,"total_on_demand_input_cost_bin":4.906020000000003,"ci_99_lower":0.699231575659531,"ci_99_upper":0.880731037444519,"ci_98_lower":0.7109890576134956,"ci_98_upper":0.8729975291157334,"ci_95_lower":0.7242547119259304,"ci_95_upper":0.8604430457591662,"ci_90_lower":0.7376907868413528,"ci_90_upper":0.8513636421028744,"ci_75_lower":0.7540675633298473,"ci_75_upper":0.8363842151599836,"ci_50_lower":0.7728854062015136,"ci_50_upper":0.818979237061713},"65536":{"avg_score":0.6808469388086129,"min_score":0.011319420300424099,"max_score":1,"n_tests":85,"stddev":0.45810848486442546,"median_score":1,"q1_score":0.039603959769010544,"q3_score":1,"avg_token_efficiency":0.9925325139268658,"is_incomplete":false,"total_prompt_cost_bin":0.1681125000000001,"total_cache_read_cost_bin":0.8075941500000011,"total_completion_cost_bin":0.5127582499999999,"total_reasoning_cost_bin":0.3624995,"total_cost_bin":1.8509644000000012,"total_on_demand_input_cost_bin":6.898063750000002,"ci_99_lower":0.5446988826328341,"ci_99_upper":0.7846988492152819,"ci_98_lower":0.5649389007190132,"ci_98_upper":0.7833173238991157,"ci_95_lower":0.5879881337860271,"ci_95_upper":0.7721895542236812,"ci_90_lower":0.600364893317661,"ci_90_upper":0.7606867118552327,"ci_75_lower":0.623516510685851,"ci_75_upper":0.7378837647765656,"ci_50_lower":0.6464815480168908,"ci_50_upper":0.7145231430227047},"131072":{"avg_score":0.5083940259420575,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4886316670708644,"median_score":0.07685881108045578,"q1_score":0.025868408381938934,"q3_score":1,"avg_token_efficiency":0.9917206548567307,"is_incomplete":false,"total_prompt_cost_bin":0.33287874999999995,"total_cache_read_cost_bin":1.9652854500000023,"total_completion_cost_bin":0.6151875000000001,"total_reasoning_cost_bin":0.429488,"total_cost_bin":3.3428397000000025,"total_on_demand_input_cost_bin":16.7102575,"ci_99_lower":0.3838731762125198,"ci_99_upper":0.6240164924991538,"ci_98_lower":0.40283023968829496,"ci_98_upper":0.6207041044032554,"ci_95_lower":0.4143787952150146,"ci_95_upper":0.6023944805985434,"ci_90_lower":0.4321576758201948,"ci_90_upper":0.5848053207463874,"ci_75_lower":0.4517473689547753,"ci_75_upper":0.5658519328761172,"ci_50_lower":0.477949153445435,"ci_50_upper":0.5457369364816154},"262144":{"avg_score":0.5216325217963639,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.4839131383146696,"median_score":1,"q1_score":0.030031200498342514,"q3_score":1,"avg_token_efficiency":0.9975809987442877,"is_incomplete":false,"total_prompt_cost_bin":1.109965,"total_cache_read_cost_bin":5.393045400000015,"total_completion_cost_bin":0.8450785000000001,"total_reasoning_cost_bin":0.6136362500000001,"total_cost_bin":7.961725150000015,"total_on_demand_input_cost_bin":46.05200999999996,"ci_99_lower":0.40994939204067626,"ci_99_upper":0.6251210443096908,"ci_98_lower":0.4237209618907316,"ci_98_upper":0.6173449690505228,"ci_95_lower":0.4380168464799339,"ci_95_upper":0.6023822779265225,"ci_90_lower":0.4577224672170924,"ci_90_upper":0.5902092051546994,"ci_75_lower":0.47754141947185186,"ci_75_upper":0.5701031988455273,"ci_50_lower":0.494223266768976,"ci_50_upper":0.548573865550273},"524288":{"avg_score":0.40535798410379914,"min_score":0.003955174703150988,"max_score":1,"n_tests":236,"stddev":0.4684446685139675,"median_score":0.047594040632247925,"q1_score":0.025762570556253195,"q3_score":1,"avg_token_efficiency":0.9916652001408931,"is_incomplete":false,"total_prompt_cost_bin":1.318591250000002,"total_cache_read_cost_bin":18.241996500000088,"total_completion_cost_bin":1.45402275,"total_reasoning_cost_bin":1.0772687499999996,"total_cost_bin":22.091879250000087,"total_on_demand_input_cost_bin":153.33522875000003,"ci_99_lower":0.321538594488165,"ci_99_upper":0.4806133498139265,"ci_98_lower":0.3335542413363257,"ci_98_upper":0.4773479596770692,"ci_95_lower":0.34470225489536555,"ci_95_upper":0.4653578515419583,"ci_90_lower":0.35367417161262166,"ci_90_upper":0.45642921563249744,"ci_75_lower":0.37007339623504143,"ci_75_upper":0.4411081524713308,"ci_50_lower":0.3857222254791542,"ci_50_upper":0.4250002817529001}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.2","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-05T19:48:07.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T23:00:49.413Z"},{"model_slug":"z-ai/glm-5.3","reasoning_mode":"low","overall_metrics":{"auc_1m":0.36520277727726275,"auc_128k":0.7499385452655578,"cum_avg_1m":0.6082766212856932,"cum_avg_128k":0.7959115829119819,"avg_score_overall":0.6270474426934558,"avg_response_time_ms":14092.48984675568,"total_prompt_cost":1.7768225999999991,"total_cache_read_cost":47.40291711999994,"total_completion_cost":4.931722399999999,"total_reasoning_cost":3.5077811999999984,"total_prompt_cost_128k":0.6547603999999998,"total_completion_cost_128k":1.1765467999999997,"total_on_demand_input_cost":257.02329940000004,"total_on_demand_input_cost_128k":34.074574799999986,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9922152709249368},"bin_metrics":{"8192":{"avg_score":0.9001491959672421,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.29305268632825776,"median_score":1,"q1_score":0.9958013147115707,"q3_score":1,"avg_token_efficiency":0.9871057389459198,"is_incomplete":false,"total_prompt_cost_bin":0.13634880000000005,"total_cache_read_cost_bin":0.11120512000000005,"total_completion_cost_bin":0.14006520000000003,"total_reasoning_cost_bin":0.01879680000000001,"total_cost_bin":0.4064159200000001,"total_on_demand_input_cost_bin":0.7351456000000005,"ci_99_lower":0.8133844280514166,"ci_99_upper":0.9735340920608724,"ci_98_lower":0.8143808974167333,"ci_98_upper":0.9619257497307844,"ci_95_lower":0.8271801208268154,"ci_95_upper":0.9595771039871034,"ci_90_lower":0.8390499440964777,"ci_90_upper":0.9487762405048125,"ci_75_lower":0.8624119308398803,"ci_75_upper":0.9362773566244869,"ci_50_lower":0.8762742034974509,"ci_50_upper":0.9237788434373215},"16384":{"avg_score":0.9018094968982041,"min_score":0.005645730532705784,"max_score":1,"n_tests":80,"stddev":0.2931443270499698,"median_score":1,"q1_score":0.9990370571613312,"q3_score":1,"avg_token_efficiency":0.9789221231603247,"is_incomplete":false,"total_prompt_cost_bin":0.13918240000000007,"total_cache_read_cost_bin":0.25457536000000003,"total_completion_cost_bin":0.14748360000000005,"total_reasoning_cost_bin":0.02731520000000001,"total_cost_bin":0.5685565600000001,"total_on_demand_input_cost_bin":1.5099727999999994,"ci_99_lower":0.8156536269186763,"ci_99_upper":0.974980504249339,"ci_98_lower":0.816406187913497,"ci_98_upper":0.9631795624352526,"ci_95_lower":0.8287170204712311,"ci_95_upper":0.9617579132132231,"ci_90_lower":0.8409198703954461,"ci_90_upper":0.950424294853583,"ci_75_lower":0.8643152257835027,"ci_75_upper":0.9380904072750126,"ci_50_lower":0.8778116475150455,"ci_50_upper":0.925641656210064},"32768":{"avg_score":0.77647737412104,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4022226461487959,"median_score":0.9993734061717987,"q1_score":0.9594105631113052,"q3_score":1,"avg_token_efficiency":1.0076444612557998,"is_incomplete":false,"total_prompt_cost_bin":0.09119039999999992,"total_cache_read_cost_bin":0.9988160000000019,"total_completion_cost_bin":0.30382439999999994,"total_reasoning_cost_bin":0.06788759999999999,"total_cost_bin":1.4617184000000016,"total_on_demand_input_cost_bin":5.469430399999995,"ci_99_lower":0.681988431154074,"ci_99_upper":0.8659868084570831,"ci_98_lower":0.6926253552609021,"ci_98_upper":0.8553754741910619,"ci_95_lower":0.7040526018543032,"ci_95_upper":0.8450936174458012,"ci_90_lower":0.7179551537562956,"ci_90_upper":0.8315334252243595,"ci_75_lower":0.736195549599121,"ci_75_upper":0.8163220843497959,"ci_50_lower":0.7517666434890571,"ci_50_upper":0.798723279775432},"65536":{"avg_score":0.7561849114978139,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4207464912797303,"median_score":1,"q1_score":0.9398028254508972,"q3_score":1,"avg_token_efficiency":0.9893262016400347,"is_incomplete":false,"total_prompt_cost_bin":0.09920959999999983,"total_cache_read_cost_bin":1.4117376000000026,"total_completion_cost_bin":0.23921479999999995,"total_reasoning_cost_bin":0.0895312,"total_cost_bin":1.8396932000000021,"total_on_demand_input_cost_bin":7.700873600000001,"ci_99_lower":0.6286174785739139,"ci_99_upper":0.860094921276028,"ci_98_lower":0.6318740480419468,"ci_98_upper":0.8484878297816304,"ci_95_lower":0.6628492922634435,"ci_95_upper":0.8373507265552111,"ci_90_lower":0.6769102197638986,"ci_90_upper":0.8263253554652499,"ci_75_lower":0.7022967605125707,"ci_75_upper":0.8049173202571075,"ci_50_lower":0.7235596708213801,"ci_50_upper":0.7896166018758188},"131072":{"avg_score":0.6449369360756093,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4684622541324285,"median_score":0.9991374015808105,"q1_score":0.026762044988572598,"q3_score":1,"avg_token_efficiency":0.9887354495866274,"is_incomplete":false,"total_prompt_cost_bin":0.18882920000000003,"total_cache_read_cost_bin":3.430202879999993,"total_completion_cost_bin":0.34595879999999973,"total_reasoning_cost_bin":0.1596364,"total_cost_bin":4.124627279999992,"total_on_demand_input_cost_bin":18.65915239999999,"ci_99_lower":0.5229042106583823,"ci_99_upper":0.7595075665370027,"ci_98_lower":0.5309342420445664,"ci_98_upper":0.7492233735451348,"ci_95_lower":0.551678082824219,"ci_95_upper":0.7363528189964783,"ci_90_lower":0.5690068974815201,"ci_90_upper":0.7210477714180421,"ci_75_lower":0.5880447395430715,"ci_75_upper":0.7002588146150301,"ci_50_lower":0.609032180780789,"ci_50_upper":0.6759487381781388},"262144":{"avg_score":0.5244693178419314,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4828578351161233,"median_score":0.96147620677948,"q1_score":0.029756536707282066,"q3_score":1,"avg_token_efficiency":0.9954328543594122,"is_incomplete":false,"total_prompt_cost_bin":0.37547299999999995,"total_cache_read_cost_bin":9.488460799999988,"total_completion_cost_bin":0.6100952,"total_reasoning_cost_bin":0.3761911999999999,"total_cost_bin":10.850220199999987,"total_on_demand_input_cost_bin":51.46718499999995,"ci_99_lower":0.41413708629431734,"ci_99_upper":0.6269238234031449,"ci_98_lower":0.42251939969725166,"ci_98_upper":0.620178014713848,"ci_95_lower":0.44102916955800914,"ci_95_upper":0.6047393637263473,"ci_90_lower":0.46199648992310394,"ci_90_upper":0.5924445529308183,"ci_75_lower":0.4787039701250574,"ci_75_upper":0.5744146691228211,"ci_50_lower":0.49759409995667025,"ci_50_upper":0.5518237240075841},"524288":{"avg_score":0.3621857378837037,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.45295825980579174,"median_score":0.04438845254480839,"q1_score":0.02504134038463235,"q3_score":0.9949668049812317,"avg_token_efficiency":0.990198990066015,"is_incomplete":false,"total_prompt_cost_bin":0.7465891999999995,"total_cache_read_cost_bin":31.70791935999996,"total_completion_cost_bin":3.1450803999999994,"total_reasoning_cost_bin":2.7684227999999984,"total_cost_bin":38.36801175999995,"total_on_demand_input_cost_bin":171.48153960000008,"ci_99_lower":0.2885009003814973,"ci_99_upper":0.4332139420721056,"ci_98_lower":0.29337951161304754,"ci_98_upper":0.42937209828403555,"ci_95_lower":0.3036417602072075,"ci_95_upper":0.419403450682333,"ci_90_lower":0.31233991518975784,"ci_90_upper":0.41226866328378475,"ci_75_lower":0.32806654982401995,"ci_75_upper":0.39596093629740403,"ci_50_lower":0.34350009367236933,"ci_50_upper":0.382106131869092}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":true,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"Z.ai: GLM 5.3","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":0.0000014,"completion":0.0000044,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-18T20:57:35.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-23T21:37:08.455Z"},{"model_slug":"z-ai/glm-5.3-flash","reasoning_mode":"low","overall_metrics":{"auc_1m":0.361753069514185,"auc_128k":0.6906748483619307,"cum_avg_1m":0.5630824356347758,"cum_avg_128k":0.7110216965184264,"avg_score_overall":0.5910009580282602,"avg_response_time_ms":55615.38515085619,"total_prompt_cost":1.0484249250000004,"total_cache_read_cost":2.5373183999999998,"total_completion_cost":0.27655825,"total_reasoning_cost":0,"total_prompt_cost_128k":0.11740214999999998,"total_completion_cost_128k":0.0651275,"total_on_demand_input_cost":13.735016924999982,"total_on_demand_input_cost_128k":1.8281461499999996,"total_runs":860,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9945151550909103},"bin_metrics":{"8192":{"avg_score":0.8171749247063417,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.3828808566487316,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9987562745501893,"is_incomplete":false,"total_prompt_cost_bin":0.022365599999999996,"total_cache_read_cost_bin":0.0034934399999999974,"total_completion_cost_bin":0.00787125,"total_reasoning_cost_bin":0,"total_cost_bin":0.033730289999999996,"total_on_demand_input_cost_bin":0.03983279999999999,"ci_99_lower":0.6947939241544809,"ci_99_upper":0.9270313242083648,"ci_98_lower":0.7072100591234048,"ci_98_upper":0.9145967807226698,"ci_95_lower":0.7297877164903912,"ci_95_upper":0.8911455232743174,"ci_90_lower":0.7434525065569323,"ci_90_upper":0.8802668753601024,"ci_75_lower":0.7676218628948845,"ci_75_upper":0.865483423931437,"ci_50_lower":0.7921192032576074,"ci_50_upper":0.8423057355394122},"16384":{"avg_score":0.7081363200908527,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.44866097155490947,"median_score":1,"q1_score":0.0448969230055809,"q3_score":1,"avg_token_efficiency":0.9845106358438892,"is_incomplete":false,"total_prompt_cost_bin":0.011491799999999995,"total_cache_read_cost_bin":0.013969919999999995,"total_completion_cost_bin":0.0089275,"total_reasoning_cost_bin":0,"total_cost_bin":0.03438921999999999,"total_on_demand_input_cost_bin":0.08134140000000009,"ci_99_lower":0.5750919867071207,"ci_99_upper":0.8295366848924315,"ci_98_lower":0.5854691863714251,"ci_98_upper":0.8163221286305924,"ci_95_lower":0.6101504905012552,"ci_95_upper":0.7947442475313438,"ci_90_lower":0.622853781504673,"ci_90_upper":0.7914818012184696,"ci_75_lower":0.6478956075356108,"ci_75_upper":0.757771015483013,"ci_50_lower":0.6721475120459218,"ci_50_upper":0.743661948290537},"32768":{"avg_score":0.7273110933858655,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4322429221578146,"median_score":1,"q1_score":0.08075995557010174,"q3_score":1,"avg_token_efficiency":1.0103376433801003,"is_incomplete":false,"total_prompt_cost_bin":0.014117399999999995,"total_cache_read_cost_bin":0.05593056000000002,"total_completion_cost_bin":0.01684150000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.08688946000000003,"total_on_demand_input_cost_bin":0.29377019999999976,"ci_99_lower":0.6251320231014517,"ci_99_upper":0.8248747066149801,"ci_98_lower":0.6318317748425077,"ci_98_upper":0.8099837477430051,"ci_95_lower":0.6527272560652123,"ci_95_upper":0.7978773508989594,"ci_90_lower":0.6651660610560451,"ci_90_upper":0.7844424163841347,"ci_75_lower":0.6830060846753921,"ci_75_upper":0.769344610572101,"ci_50_lower":0.7013652482723153,"ci_50_upper":0.7500642264503232},"65536":{"avg_score":0.7487264625728131,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.42718368144791496,"median_score":1,"q1_score":0.058749210089445114,"q3_score":1,"avg_token_efficiency":0.990472790561143,"is_incomplete":false,"total_prompt_cost_bin":0.03519292500000003,"total_cache_read_cost_bin":0.0755664,"total_completion_cost_bin":0.011611000000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.12237032500000003,"total_on_demand_input_cost_bin":0.4130249249999997,"ci_99_lower":0.6113573638473583,"ci_99_upper":0.8527808672786853,"ci_98_lower":0.634823254104375,"ci_98_upper":0.8513187134581454,"ci_95_lower":0.6472806079773341,"ci_95_upper":0.8295746674646131,"ci_90_lower":0.6694296751143959,"ci_90_upper":0.8179860976836918,"ci_75_lower":0.6926714992150664,"ci_75_upper":0.8048338126856833,"ci_50_lower":0.7148679047329899,"ci_50_upper":0.7822210525902098},"131072":{"avg_score":0.5537596818362589,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4860046453692376,"median_score":0.9954792261123657,"q1_score":0.024928688071668148,"q3_score":1,"avg_token_efficiency":0.9893085288808491,"is_incomplete":false,"total_prompt_cost_bin":0.034234424999999964,"total_cache_read_cost_bin":0.19318847999999977,"total_completion_cost_bin":0.019876249999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.24729915499999972,"total_on_demand_input_cost_bin":1.000176825,"ci_99_lower":0.43031949964805183,"ci_99_upper":0.6688157292509234,"ci_98_lower":0.4389154550629822,"ci_98_upper":0.6592373207607307,"ci_95_lower":0.4682144954396543,"ci_95_upper":0.6478116792010694,"ci_90_lower":0.47777410501849304,"ci_90_upper":0.6363117617740323,"ci_75_lower":0.49771380819520433,"ci_75_upper":0.6113817903187208,"ci_50_lower":0.5230649873395044,"ci_50_upper":0.590279304012159},"262144":{"avg_score":0.6010836862978783,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4772005096824153,"median_score":1,"q1_score":0.033339373767375946,"q3_score":1,"avg_token_efficiency":0.9957191995987953,"is_incomplete":false,"total_prompt_cost_bin":0.13803735,"total_cache_read_cost_bin":0.5239852799999987,"total_completion_cost_bin":0.0223265,"total_reasoning_cost_bin":0,"total_cost_bin":0.6843491299999988,"total_on_demand_input_cost_bin":2.7579637499999974,"ci_99_lower":0.47850525098449903,"ci_99_upper":0.7035226223286214,"ci_98_lower":0.5057454665015776,"ci_98_upper":0.696872221537476,"ci_95_lower":0.5208073127562352,"ci_95_upper":0.6829531160480481,"ci_90_lower":0.5377940624400788,"ci_90_upper":0.6691357155469186,"ci_75_lower":0.5548874890243516,"ci_75_upper":0.6495826981865953,"ci_50_lower":0.5735060298511534,"ci_50_upper":0.6286826937390055},"524288":{"avg_score":0.3484673161881956,"min_score":0,"max_score":1,"n_tests":235,"stddev":0.4520079941366356,"median_score":0.039436619728803635,"q1_score":0.02272031642496586,"q3_score":1,"avg_token_efficiency":0.9903420775791832,"is_incomplete":false,"total_prompt_cost_bin":0.7929854250000004,"total_cache_read_cost_bin":1.6711843200000014,"total_completion_cost_bin":0.18910424999999997,"total_reasoning_cost_bin":0,"total_cost_bin":2.653273995000002,"total_on_demand_input_cost_bin":9.148907024999986,"ci_99_lower":0.279976520567419,"ci_99_upper":0.42044677357709503,"ci_98_lower":0.28365246459788224,"ci_98_upper":0.41669948433572424,"ci_95_lower":0.28926806454952964,"ci_95_upper":0.407459515659089,"ci_90_lower":0.29966173498732773,"ci_90_upper":0.39781579649165033,"ci_75_lower":0.31540431601807795,"ci_75_upper":0.38332830987130095,"ci_50_lower":0.3288724464262796,"ci_50_upper":0.3681210559768048}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":true,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"Z.ai: GLM 5.3 Flash","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":7.5e-8,"completion":2.5e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-26T13:59:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"z-ai/glm-5.3-flash","reasoning_mode":"max","overall_metrics":{"auc_1m":0.36061971763188766,"auc_128k":0.7162429505862863,"cum_avg_1m":0.6008375316335297,"cum_avg_128k":0.7793282888671291,"avg_score_overall":0.6233917389706665,"avg_response_time_ms":40786.24332928325,"total_prompt_cost":0.8809947750000005,"total_cache_read_cost":2.5766486400000006,"total_completion_cost":0.48955525,"total_reasoning_cost":0,"total_prompt_cost_128k":0.12473167499999999,"total_completion_cost_128k":0.207094,"total_on_demand_input_cost":13.764237974999986,"total_on_demand_input_cost_128k":1.8184356749999997,"total_runs":860,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9945163554108363},"bin_metrics":{"8192":{"avg_score":0.9601235747570172,"min_score":0.01620626077055931,"max_score":1,"n_tests":80,"stddev":0.18682866994738556,"median_score":1,"q1_score":0.9966235160827637,"q3_score":1,"avg_token_efficiency":0.9987562745501896,"is_incomplete":false,"total_prompt_cost_bin":0.016701600000000004,"total_cache_read_cost_bin":0.00462624,"total_completion_cost_bin":0.018853749999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.04018159,"total_on_demand_input_cost_bin":0.039832799999999995,"ci_99_lower":0.8979160691173748,"ci_99_upper":0.9978770200237632,"ci_98_lower":0.8997482616240159,"ci_98_upper":0.9975714391618967,"ci_95_lower":0.9106528044352309,"ci_95_upper":0.9969093377329409,"ci_90_lower":0.9230385856505018,"ci_90_upper":0.9954837490618229,"ci_75_lower":0.9351955954974983,"ci_75_upper":0.9844621803291375,"ci_50_lower":0.9477931088244076,"ci_50_upper":0.9727193853585049},"16384":{"avg_score":0.8257012848916929,"min_score":0.006335797253996134,"max_score":1,"n_tests":80,"stddev":0.37144588087934993,"median_score":0.9991774260997772,"q1_score":0.9887671172618866,"q3_score":1,"avg_token_efficiency":0.9845106358438889,"is_incomplete":false,"total_prompt_cost_bin":0.009518999999999994,"total_cache_read_cost_bin":0.014364479999999999,"total_completion_cost_bin":0.019241000000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.04312447999999999,"total_on_demand_input_cost_bin":0.08134139999999997,"ci_99_lower":0.7048814953371475,"ci_99_upper":0.924574941925588,"ci_98_lower":0.715724873839994,"ci_98_upper":0.9132076670498935,"ci_95_lower":0.7403076893559773,"ci_95_upper":0.8999710373210837,"ci_90_lower":0.7536924178228946,"ci_90_upper":0.8877047185460105,"ci_75_lower":0.7769561829598387,"ci_75_upper":0.873194074370258,"ci_50_lower":0.8004968973284121,"ci_50_upper":0.8509050267035492},"32768":{"avg_score":0.8205692084790345,"min_score":0.002053388161584735,"max_score":1,"n_tests":136,"stddev":0.36750769603613653,"median_score":1,"q1_score":0.9683390110731125,"q3_score":1,"avg_token_efficiency":1.0103376433801001,"is_incomplete":false,"total_prompt_cost_bin":0.014107800000000005,"total_cache_read_cost_bin":0.055932480000000014,"total_completion_cost_bin":0.045191750000000024,"total_reasoning_cost_bin":0,"total_cost_bin":0.11523203000000004,"total_on_demand_input_cost_bin":0.2937701999999996,"ci_99_lower":0.7363610284033689,"ci_99_upper":0.8922047273007218,"ci_98_lower":0.7477594242249924,"ci_98_upper":0.8901335033059449,"ci_95_lower":0.7554504428765487,"ci_95_upper":0.8799955942691561,"ci_90_lower":0.7630082865224241,"ci_90_upper":0.8715212565246689,"ci_75_lower":0.7815245376088356,"ci_75_upper":0.8556157843811568,"ci_50_lower":0.7993092006787157,"ci_50_upper":0.8430165975257885},"65536":{"avg_score":0.7011667069047689,"min_score":0.004449388012290001,"max_score":1,"n_tests":85,"stddev":0.4493840520842911,"median_score":0.9990800619125366,"q1_score":0.0401785708963871,"q3_score":1,"avg_token_efficiency":0.9904727905611436,"is_incomplete":false,"total_prompt_cost_bin":0.025319324999999986,"total_cache_read_cost_bin":0.07754112,"total_completion_cost_bin":0.06056124999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.16342169499999998,"total_on_demand_input_cost_bin":0.41302492500000015,"ci_99_lower":0.5745831903741312,"ci_99_upper":0.8141913953845334,"ci_98_lower":0.5867622150308727,"ci_98_upper":0.804129402154728,"ci_95_lower":0.6086100135949056,"ci_95_upper":0.7925572430169867,"ci_90_lower":0.6210959868522032,"ci_90_upper":0.7804451469173108,"ci_75_lower":0.6437667696010869,"ci_75_upper":0.758015899457421,"ci_50_lower":0.6672089196440271,"ci_50_upper":0.7348245646366302},"131072":{"avg_score":0.5890806693031325,"min_score":0.0028476507868617773,"max_score":1,"n_tests":102,"stddev":0.48306067988832696,"median_score":1,"q1_score":0.026830146554857492,"q3_score":1,"avg_token_efficiency":0.9893085128340142,"is_incomplete":false,"total_prompt_cost_bin":0.05908394999999999,"total_cache_read_cost_bin":0.18627647999999988,"total_completion_cost_bin":0.06324624999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.30860667999999986,"total_on_demand_input_cost_bin":0.9904663499999999,"ci_99_lower":0.46474364934978757,"ci_99_upper":0.7041409261805672,"ci_98_lower":0.482802666806132,"ci_98_upper":0.7025341252480432,"ci_95_lower":0.5009902573471395,"ci_95_upper":0.6857583548929588,"ci_90_lower":0.5116146940944543,"ci_90_upper":0.668642458877316,"ci_75_lower":0.5396964767595124,"ci_75_upper":0.6383961386078121,"ci_50_lower":0.5598274599663758,"ci_50_upper":0.6185951207372972},"262144":{"avg_score":0.5558503455472515,"min_score":0.0037854888942092657,"max_score":1,"n_tests":141,"stddev":0.4815379946002929,"median_score":0.9974597692489624,"q1_score":0.03351955488324165,"q3_score":1,"avg_token_efficiency":0.995719199598795,"is_incomplete":false,"total_prompt_cost_bin":0.1109845499999999,"total_cache_read_cost_bin":0.5293958399999986,"total_completion_cost_bin":0.11122475000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.7516051399999985,"total_on_demand_input_cost_bin":2.757963749999997,"ci_99_lower":0.44588735574665683,"ci_99_upper":0.6585010916835724,"ci_98_lower":0.4652836959816661,"ci_98_upper":0.6521195023721547,"ci_95_lower":0.4739127358119086,"ci_95_upper":0.6374573387330685,"ci_90_lower":0.4881079831770754,"ci_90_upper":0.6236740814338617,"ci_75_lower":0.5098141766395188,"ci_75_upper":0.6039418463189822,"ci_50_lower":0.528046941444532,"ci_50_upper":0.5852474186467073},"524288":{"avg_score":0.35420846318534055,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4534188007964863,"median_score":0.03876280412077904,"q1_score":0.023266945965588093,"q3_score":0.9998786300420761,"avg_token_efficiency":0.9903420791182314,"is_incomplete":false,"total_prompt_cost_bin":0.6452785500000006,"total_cache_read_cost_bin":1.708512000000002,"total_completion_cost_bin":0.17123649999999999,"total_reasoning_cost_bin":0,"total_cost_bin":2.5250270500000025,"total_on_demand_input_cost_bin":9.18783854999999,"ci_99_lower":0.28061445009789604,"ci_99_upper":0.4281515406412112,"ci_98_lower":0.28666136444928086,"ci_98_upper":0.4212456152787219,"ci_95_lower":0.29532467752877106,"ci_95_upper":0.4112008445931724,"ci_90_lower":0.3043530146883883,"ci_90_upper":0.40592950685640355,"ci_75_lower":0.3195578033912047,"ci_75_upper":0.38830274576049933,"ci_50_lower":0.33540371459611823,"ci_50_upper":0.37449815102914047}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":true,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"Z.ai: GLM 5.3 Flash","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":7.5e-8,"completion":2.5e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-26T13:59:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"google/gemini-3-flash-preview","reasoning_mode":"high","overall_metrics":{"auc_1m":0.3578038478035309,"auc_128k":0.6071270897450527,"cum_avg_1m":0.5738767463892087,"cum_avg_128k":0.7197355327200994,"avg_score_overall":0.46787695031983734,"avg_response_time_ms":70866.07577115913,"total_prompt_cost":5.992118999999992,"total_cache_read_cost":24.909920499999952,"total_completion_cost":1.3761539999999999,"total_reasoning_cost":64.57828800000004,"total_prompt_cost_128k":1.9460284999999993,"total_completion_cost_128k":0.5556329999999999,"total_on_demand_input_cost":255.09132399999964,"total_on_demand_input_cost_128k":12.276404499999998,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0008273041147895},"bin_metrics":{"8192":{"avg_score":0.975603310065344,"min_score":0.0181695818901062,"max_score":1,"n_tests":80,"stddev":0.15263788625390293,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0119319703644796,"is_incomplete":false,"total_prompt_cost_bin":0.12174449999999996,"total_cache_read_cost_bin":0.014703200000000003,"total_completion_cost_bin":0.083895,"total_reasoning_cost_bin":0.8735579999999998,"total_cost_bin":1.0939006999999998,"total_on_demand_input_cost_bin":0.2687764999999999,"ci_99_lower":0.9266818807271775,"ci_99_upper":0.9999841254837811,"ci_98_lower":0.92714042636659,"ci_98_upper":0.9999815162345767,"ci_95_lower":0.9389042906649411,"ci_95_upper":0.9999649768695236,"ci_90_lower":0.939621944911778,"ci_90_upper":0.9999345131218433,"ci_75_lower":0.9515870951581746,"ci_75_upper":0.9997415372170508,"ci_50_lower":0.9635048703290522,"ci_50_upper":0.9877850627992302},"16384":{"avg_score":0.9260473232483492,"min_score":0.01270289346575737,"max_score":1,"n_tests":80,"stddev":0.25738616912411344,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024471,"is_incomplete":false,"total_prompt_cost_bin":0.26469550000000014,"total_cache_read_cost_bin":0.028624049999999984,"total_completion_cost_bin":0.08286,"total_reasoning_cost_bin":0.9668969999999998,"total_cost_bin":1.34307655,"total_on_demand_input_cost_bin":0.5509359999999995,"ci_99_lower":0.839965666251257,"ci_99_upper":0.987533052529674,"ci_98_lower":0.850726168547757,"ci_98_upper":0.9866077912102919,"ci_95_lower":0.8628412329434649,"ci_95_upper":0.9755616004671902,"ci_90_lower":0.8747778090054634,"ci_90_upper":0.973544886795571,"ci_75_lower":0.8895856857066974,"ci_75_upper":0.9610625008732313,"ci_50_lower":0.9029733799747193,"ci_50_upper":0.949060960143106},"32768":{"avg_score":0.751957214814063,"min_score":0.004637680947780609,"max_score":1,"n_tests":136,"stddev":0.42256368782417714,"median_score":1,"q1_score":0.11066229455173016,"q3_score":1,"avg_token_efficiency":1.000035909094991,"is_incomplete":false,"total_prompt_cost_bin":0.5901624999999993,"total_cache_read_cost_bin":0.13465455000000007,"total_completion_cost_bin":0.159048,"total_reasoning_cost_bin":2.3057489999999996,"total_cost_bin":3.189614049999999,"total_on_demand_input_cost_bin":1.936707999999999,"ci_99_lower":0.6597907861286212,"ci_99_upper":0.8440905687013669,"ci_98_lower":0.6664620778420666,"ci_98_upper":0.8374419237146674,"ci_95_lower":0.6805919154439299,"ci_95_upper":0.8220397063664516,"ci_90_lower":0.6886782225245363,"ci_90_upper":0.8082422771600678,"ci_75_lower":0.7088585280426367,"ci_75_upper":0.7936185373503523,"ci_50_lower":0.7253203684573665,"ci_50_upper":0.7745195713274264},"65536":{"avg_score":0.5969413999687223,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4770472299971867,"median_score":0.9993585348129272,"q1_score":0.03230403736233711,"q3_score":1,"avg_token_efficiency":1.000015287852281,"is_incomplete":false,"total_prompt_cost_bin":0.3724194999999999,"total_cache_read_cost_bin":0.24076080000000052,"total_completion_cost_bin":0.102474,"total_reasoning_cost_bin":3.0091199999999994,"total_cost_bin":3.7247743,"total_on_demand_input_cost_bin":2.780027499999999,"ci_99_lower":0.4604391091919559,"ci_99_upper":0.7159364305923638,"ci_98_lower":0.46706693249601217,"ci_98_upper":0.7107490869957734,"ci_95_lower":0.4967145695320933,"ci_95_upper":0.6998936677877516,"ci_90_lower":0.5137956394167508,"ci_90_upper":0.6846249444585512,"ci_75_lower":0.539279904202832,"ci_75_upper":0.6574038335779572,"ci_50_lower":0.5631032167139518,"ci_50_upper":0.6317675504176056},"131072":{"avg_score":0.34812841550401835,"min_score":0.007390300277620554,"max_score":1,"n_tests":103,"stddev":0.4597231181556534,"median_score":0.03600514307618141,"q1_score":0.020892522297799587,"q3_score":0.9998141527175903,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":0.5970065,"total_cache_read_cost_bin":0.6142949999999989,"total_completion_cost_bin":0.12735599999999997,"total_reasoning_cost_bin":4.987878000000001,"total_cost_bin":6.3265355,"total_on_demand_input_cost_bin":6.739956500000002,"ci_99_lower":0.24171026209726462,"ci_99_upper":0.4614011775701334,"ci_98_lower":0.2441949270497465,"ci_98_upper":0.4527212193386756,"ci_95_lower":0.26159582900344536,"ci_95_upper":0.4341883672742643,"ci_90_lower":0.27265098188208076,"ci_90_upper":0.42346537201642737,"ci_75_lower":0.2922332720442232,"ci_75_upper":0.40342283246527855,"ci_50_lower":0.3197592740609464,"ci_50_upper":0.3770396301638136},"262144":{"avg_score":0.44872211559067965,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.47900784919783984,"median_score":0.05730833858251572,"q1_score":0.026810724288225174,"q3_score":1,"avg_token_efficiency":1.0000038179365243,"is_incomplete":false,"total_prompt_cost_bin":1.0713374999999996,"total_cache_read_cost_bin":1.7394205000000063,"total_completion_cost_bin":0.16035900000000006,"total_reasoning_cost_bin":9.112328999999999,"total_cost_bin":12.083446000000006,"total_on_demand_input_cost_bin":18.465542499999987,"ci_99_lower":0.33390255665054036,"ci_99_upper":0.5607252189139126,"ci_98_lower":0.35695211492587836,"ci_98_upper":0.5488433742763784,"ci_95_lower":0.36739461610551116,"ci_95_upper":0.5314596849911869,"ci_90_lower":0.3800589638629248,"ci_90_upper":0.5162039111875301,"ci_75_lower":0.4016949483164559,"ci_75_upper":0.49685677842559883,"ci_50_lower":0.42085565196173524,"ci_50_upper":0.4757255767587678},"524288":{"avg_score":0.33651063032152323,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4457500843581148,"median_score":0.040442535653710365,"q1_score":0.023462104611098766,"q3_score":0.9997444599866867,"avg_token_efficiency":1.0000019078540325,"is_incomplete":false,"total_prompt_cost_bin":1.2110184999999982,"total_cache_read_cost_bin":6.063869499999997,"total_completion_cost_bin":0.273903,"total_reasoning_cost_bin":16.524258000000007,"total_cost_bin":24.073049,"total_on_demand_input_cost_bin":61.849713500000064,"ci_99_lower":0.2635778411606798,"ci_99_upper":0.4084933348385552,"ci_98_lower":0.2710627154785433,"ci_98_upper":0.40343991458416967,"ci_95_lower":0.2792860431850679,"ci_95_upper":0.39391665476439935,"ci_90_lower":0.28768780095452423,"ci_90_upper":0.386294795768296,"ci_75_lower":0.30284855839383484,"ci_75_upper":0.3703786433695113,"ci_50_lower":0.3180777571505425,"ci_50_upper":0.35597450853058066},"1048576":{"avg_score":0.20710356160097065,"min_score":0.004716981202363968,"max_score":1,"n_tests":310,"stddev":0.37082760339930093,"median_score":0.031931474804878235,"q1_score":0.02050809608772397,"q3_score":0.06267048884183168,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":1.7637344999999944,"total_cache_read_cost_bin":16.073592899999948,"total_completion_cost_bin":0.3862589999999999,"total_reasoning_cost_bin":26.798499000000035,"total_cost_bin":45.02208539999998,"total_on_demand_input_cost_bin":162.4996634999996,"ci_99_lower":0.1583780059964817,"ci_99_upper":0.2600361101437179,"ci_98_lower":0.15962628617987878,"ci_98_upper":0.2574767637378686,"ci_95_lower":0.16721819584317987,"ci_95_upper":0.25211373557614525,"ci_90_lower":0.17215616144480242,"ci_90_upper":0.2427342456051959,"ci_75_lower":0.18163677983646911,"ci_75_upper":0.2312701599957842,"ci_50_lower":0.19176289297324878,"ci_50_upper":0.22071857826273528}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3 Flash Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":5e-7,"completion":0.000003,"reasoning":0.000003,"has_tiered_pricing":true},"api_created_at":"2025-12-17T15:57:58.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T02:47:52.893Z"},{"model_slug":"google/gemini-3.1-pro-preview","reasoning_mode":"low","overall_metrics":{"auc_1m":0.3548685282101936,"auc_128k":0.650302436125342,"cum_avg_1m":0.5970788556402405,"cum_avg_128k":0.761702974957079,"avg_score_overall":0.48689898479741883,"avg_response_time_ms":72102.90103188707,"total_prompt_cost":57.39021600000003,"total_cache_read_cost":192.01835979999933,"total_completion_cost":5.366591999999999,"total_reasoning_cost":32.707932000000014,"total_prompt_cost_128k":6.8436819999999985,"total_completion_cost_128k":2.2278720000000005,"total_on_demand_input_cost":1988.2905719999974,"total_on_demand_input_cost_128k":47.86206400000003,"total_runs":1165,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0008722894953521},"bin_metrics":{"8192":{"avg_score":0.9877240918343887,"min_score":0.017927346751093864,"max_score":1,"n_tests":80,"stddev":0.10979906057540668,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0011172215348227,"is_incomplete":false,"total_prompt_cost_bin":1.0561320000000005,"total_cache_read_cost_bin":0.0008148,"total_completion_cost_bin":0.33608399999999994,"total_reasoning_cost_bin":0.9785880000000002,"total_cost_bin":2.3716188000000007,"total_on_demand_input_cost_bin":1.0642800000000003,"ci_99_lower":0.9508963673375547,"ci_99_upper":1,"ci_98_lower":0.9508963673375547,"ci_98_upper":1,"ci_95_lower":0.963172275503166,"ci_95_upper":1,"ci_90_lower":0.963172275503166,"ci_90_upper":1,"ci_75_lower":0.9754481836687774,"ci_75_upper":1,"ci_50_lower":0.9754481836687774,"ci_50_upper":1},"16384":{"avg_score":0.9515464016934857,"min_score":0.026944274082779884,"max_score":1,"n_tests":80,"stddev":0.2125385597151244,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0022256134002931,"is_incomplete":false,"total_prompt_cost_bin":0.6620640000000001,"total_cache_read_cost_bin":0.1546311999999999,"total_completion_cost_bin":0.3350999999999999,"total_reasoning_cost_bin":1.1155679999999997,"total_cost_bin":2.2673631999999997,"total_on_demand_input_cost_bin":2.208375999999999,"ci_99_lower":0.8788614491611952,"ci_99_upper":1,"ci_98_lower":0.8906887595818844,"ci_98_upper":1,"ci_95_lower":0.8911256321863038,"ci_95_upper":0.9879602840635926,"ci_90_lower":0.9031488986918703,"ci_90_upper":0.9878928987309337,"ci_75_lower":0.9272189398965566,"ci_75_upper":0.9757995389954887,"ci_50_lower":0.9393467218615115,"ci_50_upper":0.9637460815254599},"32768":{"avg_score":0.8440644131264622,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.3532127724164476,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0038576033044286,"is_incomplete":false,"total_prompt_cost_bin":1.3052979999999985,"total_cache_read_cost_bin":0.6468653999999981,"total_completion_cost_bin":0.6354120000000004,"total_reasoning_cost_bin":2.31828,"total_cost_bin":4.905855399999997,"total_on_demand_input_cost_bin":7.773952000000009,"ci_99_lower":0.7660663945130192,"ci_99_upper":0.9195049550619083,"ci_98_lower":0.7719501274078228,"ci_98_upper":0.9091568121438681,"ci_95_lower":0.7806893049227416,"ci_95_upper":0.9006652832209296,"ci_90_lower":0.7881530401177931,"ci_90_upper":0.8943197751780698,"ci_75_lower":0.8050843360872344,"ci_75_upper":0.8793418384816236,"ci_50_lower":0.8227081601079964,"ci_50_upper":0.8647504668252248},"65536":{"avg_score":0.6002208907595452,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4807366773203132,"median_score":1,"q1_score":0.030576243996620178,"q3_score":1,"avg_token_efficiency":1.001168002060588,"is_incomplete":false,"total_prompt_cost_bin":0.9324559999999994,"total_cache_read_cost_bin":1.0200472000000005,"total_completion_cost_bin":0.4121760000000001,"total_reasoning_cost_bin":1.9586639999999995,"total_cost_bin":4.323343199999999,"total_on_demand_input_cost_bin":11.132928000000001,"ci_99_lower":0.46273400856203895,"ci_99_upper":0.7157350383024553,"ci_98_lower":0.47366806465212036,"ci_98_upper":0.7145113704860869,"ci_95_lower":0.49805247265434655,"ci_95_upper":0.7031297459321862,"ci_90_lower":0.5192697979822097,"ci_90_upper":0.6908953616932472,"ci_75_lower":0.5424167854793589,"ci_75_upper":0.6581825477180673,"ci_50_lower":0.5654856638407664,"ci_50_upper":0.6344328718780375},"131072":{"avg_score":0.42495907737151245,"min_score":0.007238883059471846,"max_score":1,"n_tests":98,"stddev":0.4800969146022057,"median_score":0.04056696593761444,"q1_score":0.024763562250882387,"q3_score":1,"avg_token_efficiency":1.0012331671436776,"is_incomplete":false,"total_prompt_cost_bin":2.8877319999999997,"total_cache_read_cost_bin":2.2794795999999997,"total_completion_cost_bin":0.5090999999999999,"total_reasoning_cost_bin":2.6785920000000014,"total_cost_bin":8.354903600000002,"total_on_demand_input_cost_bin":25.682528000000016,"ci_99_lower":0.30513532030978247,"ci_99_upper":0.5450699840602464,"ci_98_lower":0.30748150631438526,"ci_98_upper":0.5344858295437215,"ci_95_lower":0.3262034990518278,"ci_95_upper":0.5238725179043713,"ci_90_lower":0.3460369091331294,"ci_90_upper":0.5049448350050049,"ci_75_lower":0.3661223367505175,"ci_75_upper":0.4838119084716357,"ci_50_lower":0.3936273840194264,"ci_50_upper":0.45554178810383816},"262144":{"avg_score":0.44115195531028173,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.47598575082718275,"median_score":0.060400791466236115,"q1_score":0.029629629105329514,"q3_score":1,"avg_token_efficiency":1.000003817936524,"is_incomplete":false,"total_prompt_cost_bin":14.63695600000001,"total_cache_read_cost_bin":13.102272000000026,"total_completion_cost_bin":0.6402720000000003,"total_reasoning_cost_bin":3.7357680000000006,"total_cost_bin":32.115268000000036,"total_on_demand_input_cost_bin":147.72433999999996,"ci_99_lower":0.3288738278304028,"ci_99_upper":0.5508717764794483,"ci_98_lower":0.34623448916099936,"ci_98_upper":0.542137324717061,"ci_95_lower":0.3596406892354844,"ci_95_upper":0.5236869843154881,"ci_90_lower":0.37597141521487465,"ci_90_upper":0.5090295043184442,"ci_75_lower":0.39481067500929246,"ci_75_upper":0.4890570009774562,"ci_50_lower":0.41356654245908375,"ci_50_upper":0.46872020915979584},"524288":{"avg_score":0.27897490709722833,"min_score":0.0029607696924358606,"max_score":1,"n_tests":235,"stddev":0.4192143749617748,"median_score":0.03718047961592674,"q1_score":0.02392047829926014,"q3_score":0.6278307735919952,"avg_token_efficiency":1.0000135822995342,"is_incomplete":false,"total_prompt_cost_bin":10.041806000000001,"total_cache_read_cost_bin":47.891305600000145,"total_completion_cost_bin":1.0373519999999996,"total_reasoning_cost_bin":7.416696000000002,"total_cost_bin":66.38715960000015,"total_on_demand_input_cost_bin":492.7068600000008,"ci_99_lower":0.20958486494573547,"ci_99_upper":0.34675006408685755,"ci_98_lower":0.21693199496046503,"ci_98_upper":0.34165136451802236,"ci_95_lower":0.2275056431856888,"ci_95_upper":0.3318547362583798,"ci_90_lower":0.23610282673857155,"ci_90_upper":0.3234706124022821,"ci_75_lower":0.2483520334758647,"ci_75_upper":0.31032617004598156,"ci_50_lower":0.26041952300732557,"ci_50_upper":0.29814745234746604},"1048576":{"avg_score":0.24798910792901993,"min_score":0.00273348530754447,"max_score":1,"n_tests":310,"stddev":0.39953959975501174,"median_score":0.03369317762553692,"q1_score":0.02233324758708477,"q3_score":0.11583800427615643,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":25.86777200000002,"total_cache_read_cost_bin":126.92294399999916,"total_completion_cost_bin":1.461095999999999,"total_reasoning_cost_bin":12.505776000000013,"total_cost_bin":166.7575879999992,"total_on_demand_input_cost_bin":1299.9973079999966,"ci_99_lower":0.18848628464589393,"ci_99_upper":0.30659259799798766,"ci_98_lower":0.19814482038085832,"ci_98_upper":0.3019918439141099,"ci_95_lower":0.20312903366122964,"ci_95_upper":0.29627002217806875,"ci_90_lower":0.21069554302439605,"ci_90_upper":0.28660790933562713,"ci_75_lower":0.22162057665653617,"ci_75_upper":0.27394738703620647,"ci_50_lower":0.23167454597649106,"ci_50_upper":0.26297805904872684}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.1 Pro Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":0.000002,"completion":0.000012,"reasoning":0.000012,"has_tiered_pricing":true},"api_created_at":"2026-02-19T14:00:27.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T17:59:39.862Z"},{"model_slug":"meta/muse-spark-1.1","reasoning_mode":"low","overall_metrics":{"auc_1m":0.3541513136532905,"auc_128k":0.6406317463316074,"cum_avg_1m":0.5576780293120615,"cum_avg_128k":0.7114845118777448,"avg_score_overall":0.588874949150179,"avg_response_time_ms":25721.21972911449,"total_prompt_cost":0.6551700000000058,"total_cache_read_cost":27.530571449999908,"total_completion_cost":6.622970500000001,"total_reasoning_cost":5.224975499999999,"total_prompt_cost_128k":0.002415000000000005,"total_completion_cost_128k":3.5710200000000003,"total_on_demand_input_cost":230.0765987500003,"total_on_demand_input_cost_128k":30.61537375000003,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9996926134650075},"bin_metrics":{"8192":{"avg_score":0.8776479136897251,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.32578775668451426,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0101437657600698,"is_incomplete":false,"total_prompt_cost_bin":0.0003950000000000007,"total_cache_read_cost_bin":0.07953390000000005,"total_completion_cost_bin":0.47164799999999996,"total_reasoning_cost_bin":0.3526862499999999,"total_cost_bin":0.90426315,"total_on_demand_input_cost_bin":0.6631774999999998,"ci_99_lower":0.7800705124458763,"ci_99_upper":0.9633189300158993,"ci_98_lower":0.791093241341412,"ci_98_upper":0.951323302994715,"ci_95_lower":0.8034090817213292,"ci_95_upper":0.9394186247297329,"ci_90_lower":0.8156382388505153,"ci_90_upper":0.9270108261192217,"ci_75_lower":0.83998245356197,"ci_75_upper":0.9146935273340204,"ci_50_lower":0.8532147326797713,"ci_50_upper":0.9020065448479726},"16384":{"avg_score":0.8288073807838373,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.37409712215639046,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9975608447484994,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.16486439999999997,"total_completion_cost_bin":0.44867675000000007,"total_reasoning_cost_bin":0.32953224999999975,"total_cost_bin":0.9434733999999998,"total_on_demand_input_cost_bin":1.3742700000000005,"ci_99_lower":0.7064783819339937,"ci_99_upper":0.9271890339808306,"ci_98_lower":0.7181673845306505,"ci_98_upper":0.9154278456018073,"ci_95_lower":0.7427544611276244,"ci_95_upper":0.9028096956430818,"ci_90_lower":0.7560106682381593,"ci_90_upper":0.8902105090208352,"ci_75_lower":0.7797386058897245,"ci_75_upper":0.8772510776208946,"ci_50_lower":0.803834936421481,"ci_50_upper":0.8537138731742743},"32768":{"avg_score":0.7455960318740622,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.42521396715849274,"median_score":1,"q1_score":0.1487259790301323,"q3_score":1,"avg_token_efficiency":1.0178807703567447,"is_incomplete":false,"total_prompt_cost_bin":0.0006800000000000015,"total_cache_read_cost_bin":0.5918436000000004,"total_completion_cost_bin":1.0423040000000001,"total_reasoning_cost_bin":0.81198375,"total_cost_bin":2.4468113500000004,"total_on_demand_input_cost_bin":4.932710000000005,"ci_99_lower":0.6529654688290526,"ci_99_upper":0.8393167717745768,"ci_98_lower":0.6605852777372895,"ci_98_upper":0.8288711594208485,"ci_95_lower":0.6748043173171668,"ci_95_upper":0.8111362582163352,"ci_90_lower":0.6835232387113672,"ci_90_upper":0.8024145383689114,"ci_75_lower":0.7030007229293593,"ci_75_upper":0.787412512818478,"ci_50_lower":0.7184216723202125,"ci_50_upper":0.7696031775558367},"65536":{"avg_score":0.6345840695834555,"min_score":0.0038729666266590357,"max_score":1,"n_tests":85,"stddev":0.47310701197190763,"median_score":1,"q1_score":0.03799768537282944,"q3_score":1,"avg_token_efficiency":0.9949327067350517,"is_incomplete":false,"total_prompt_cost_bin":0.0004250000000000008,"total_cache_read_cost_bin":0.8297183999999993,"total_completion_cost_bin":0.6600759999999999,"total_reasoning_cost_bin":0.51099875,"total_cost_bin":2.001218149999999,"total_on_demand_input_cost_bin":6.914745000000004,"ci_99_lower":0.49626667421965803,"ci_99_upper":0.7591484245396877,"ci_98_lower":0.5076286134309519,"ci_98_upper":0.7491494916129913,"ci_95_lower":0.5316378064348143,"ci_95_upper":0.7275490915385651,"ci_90_lower":0.5540790596178404,"ci_90_upper":0.71596370305729,"ci_75_lower":0.5769930349799859,"ci_75_upper":0.6918892479604384,"ci_50_lower":0.5999025638133069,"ci_50_upper":0.6690134682432365},"131072":{"avg_score":0.4707871634576434,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4873103338866417,"median_score":0.04285417124629021,"q1_score":0.02587046939879656,"q3_score":1,"avg_token_efficiency":0.9929203008459689,"is_incomplete":false,"total_prompt_cost_bin":0.000515000000000001,"total_cache_read_cost_bin":2.0075947500000004,"total_completion_cost_bin":0.9483152500000004,"total_reasoning_cost_bin":0.7642009999999998,"total_cost_bin":3.7206260000000007,"total_on_demand_input_cost_bin":16.73047125000002,"ci_99_lower":0.3463413773635425,"ci_99_upper":0.5768079617283605,"ci_98_lower":0.35563257479560967,"ci_98_upper":0.5746487961789295,"ci_95_lower":0.3750116619771432,"ci_95_upper":0.5649343355523776,"ci_90_lower":0.39427978517521556,"ci_90_upper":0.5549643039417416,"ci_75_lower":0.4139980758614929,"ci_75_upper":0.5278821474843144,"ci_50_lower":0.4388624133719629,"ci_50_upper":0.5078411971533092},"262144":{"avg_score":0.49127654474334304,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4859781171598366,"median_score":0.07464607805013657,"q1_score":0.02633085288107395,"q3_score":1,"avg_token_efficiency":0.99818041477873,"is_incomplete":false,"total_prompt_cost_bin":0.0007050000000000015,"total_cache_read_cost_bin":5.529477149999992,"total_completion_cost_bin":1.1092712500000002,"total_reasoning_cost_bin":0.8853345000000004,"total_cost_bin":7.524787899999993,"total_on_demand_input_cost_bin":46.079681250000036,"ci_99_lower":0.38026725981539733,"ci_99_upper":0.6008391171447431,"ci_98_lower":0.38814810707131464,"ci_98_upper":0.5946283100763046,"ci_95_lower":0.41082849903395463,"ci_95_upper":0.5746686859366793,"ci_90_lower":0.42653382861019107,"ci_90_upper":0.5601769812855909,"ci_75_lower":0.4479222849662851,"ci_75_upper":0.5395798865472072,"ci_50_lower":0.4639867032689855,"ci_50_upper":0.5185513045178092},"524288":{"avg_score":0.4127251303644251,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.47093834465323253,"median_score":0.048153601586818695,"q1_score":0.026426057796925306,"q3_score":1,"avg_token_efficiency":0.9919647332235325,"is_incomplete":false,"total_prompt_cost_bin":0.6520500000000058,"total_cache_read_cost_bin":18.327539249999916,"total_completion_cost_bin":1.9426792499999999,"total_reasoning_cost_bin":1.570239,"total_cost_bin":22.492507499999924,"total_on_demand_input_cost_bin":153.38154375000025,"ci_99_lower":0.326817852375512,"ci_99_upper":0.4861000153331788,"ci_98_lower":0.3388407712119189,"ci_98_upper":0.4812407928412401,"ci_95_lower":0.3528854751561855,"ci_95_upper":0.4728333298965373,"ci_90_lower":0.35914967265907455,"ci_90_upper":0.46482955557017946,"ci_75_lower":0.37741858958487673,"ci_75_upper":0.44777474813932805,"ci_50_lower":0.3930668729757969,"ci_50_upper":0.4329917699353122}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.1","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:29:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-17T05:33:45.240Z"},{"model_slug":"openai/gpt-5.6-luna","reasoning_mode":"max","overall_metrics":{"auc_1m":0.35263200845493736,"auc_128k":0.7482454661148853,"cum_avg_1m":0.6165198313317533,"cum_avg_128k":0.8209301680200088,"avg_score_overall":0.6368529564580955,"avg_response_time_ms":81073.19985235305,"total_prompt_cost":293.76064499999967,"total_cache_read_cost":0.8455109,"total_completion_cost":77.59223400000002,"total_reasoning_cost":75.63785399999999,"total_prompt_cost_128k":23.54233700000001,"total_completion_cost_128k":22.703370000000003,"total_on_demand_input_cost":302.27053399999966,"total_on_demand_input_cost_128k":24.05391300000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9800866768523053},"bin_metrics":{"8192":{"avg_score":0.9643171447096393,"min_score":0.028362305834889412,"max_score":1,"n_tests":80,"stddev":0.18176852864657222,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598399,"is_incomplete":false,"total_prompt_cost_bin":0.5191920000000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.582804,"total_reasoning_cost_bin":0.4154699999999998,"total_cost_bin":1.5174660000000002,"total_on_demand_input_cost_bin":0.5191920000000004,"ci_99_lower":0.9043542426320491,"ci_99_upper":0.9999771004095674,"ci_98_lower":0.9050339482785202,"ci_98_upper":0.9999720679596066,"ci_95_lower":0.9166565522737801,"ci_95_upper":0.9999639934301378,"ci_90_lower":0.9285359466494992,"ci_90_upper":0.9999383786693216,"ci_75_lower":0.9404017706401647,"ci_75_upper":0.9879412147682161,"ci_50_lower":0.9521927761845291,"ci_50_upper":0.9764438378158957},"16384":{"avg_score":0.9388627332868055,"min_score":0.009598080068826675,"max_score":1,"n_tests":80,"stddev":0.2372255197946599,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.967068739318871,"is_incomplete":false,"total_prompt_cost_bin":1.0656880000000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.2983519999999997,"total_reasoning_cost_bin":1.1359080000000006,"total_cost_bin":3.4999480000000007,"total_on_demand_input_cost_bin":1.0656880000000002,"ci_99_lower":0.8543904377963628,"ci_99_upper":0.9880362978035118,"ci_98_lower":0.8655910806716419,"ci_98_upper":0.9877979794046842,"ci_95_lower":0.877508029746241,"ci_95_upper":0.9874229557067157,"ci_90_lower":0.8899568739603273,"ci_90_upper":0.9757983366027474,"ci_75_lower":0.9027069572155596,"ci_75_upper":0.9638745651405771,"ci_50_lower":0.9259219296916853,"ci_50_upper":0.9519230288919062},"32768":{"avg_score":0.902445901212666,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.2884571254994887,"median_score":1,"q1_score":0.9997977763414383,"q3_score":1,"avg_token_efficiency":0.9952100768549625,"is_incomplete":false,"total_prompt_cost_bin":3.8599200000000025,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.840808,"total_reasoning_cost_bin":2.5177739999999997,"total_cost_bin":9.218502000000003,"total_on_demand_input_cost_bin":3.8599200000000025,"ci_99_lower":0.8301716892347288,"ci_99_upper":0.958835731255411,"ci_98_lower":0.8389213830151814,"ci_98_upper":0.9573385189717416,"ci_95_lower":0.8526311604655348,"ci_95_upper":0.9466804120689629,"ci_90_lower":0.8590443908702582,"ci_90_upper":0.939541430169415,"ci_75_lower":0.8723886539570659,"ci_75_upper":0.9302697352232987,"ci_50_lower":0.883834091035704,"ci_50_upper":0.9217032745980439},"65536":{"avg_score":0.7148956861182609,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4439492750489713,"median_score":1,"q1_score":0.042375244200229645,"q3_score":1,"avg_token_efficiency":0.9776435846759917,"is_incomplete":false,"total_prompt_cost_bin":5.307775,"total_cache_read_cost_bin":0.0127894,"total_completion_cost_bin":6.439080000000002,"total_reasoning_cost_bin":6.231708000000001,"total_cost_bin":17.991352400000004,"total_on_demand_input_cost_bin":5.435669,"ci_99_lower":0.5877370384257452,"ci_99_upper":0.8293088628943078,"ci_98_lower":0.5897976480825421,"ci_98_upper":0.8180092985090963,"ci_95_lower":0.6218451108505456,"ci_95_upper":0.8053923632189467,"ci_90_lower":0.6341171094898463,"ci_90_upper":0.7938151457555154,"ci_75_lower":0.6572685413692585,"ci_75_upper":0.7714667058440253,"ci_50_lower":0.6804511344389004,"ci_50_upper":0.7478715519023621},"131072":{"avg_score":0.5841293747726719,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4836849752919028,"median_score":0.9994416236877441,"q1_score":0.030605671927332878,"q3_score":1,"avg_token_efficiency":0.9772722316188371,"is_incomplete":false,"total_prompt_cost_bin":12.789762000000009,"total_cache_read_cost_bin":0.0383682,"total_completion_cost_bin":11.542326000000001,"total_reasoning_cost_bin":11.286917999999998,"total_cost_bin":35.65737420000001,"total_on_demand_input_cost_bin":13.173444000000009,"ci_99_lower":0.46122518082795827,"ci_99_upper":0.7060044287338635,"ci_98_lower":0.4717489612579472,"ci_98_upper":0.688478629029157,"ci_95_lower":0.49734067662711734,"ci_95_upper":0.6788410143267807,"ci_90_lower":0.5084795557394244,"ci_90_upper":0.6683575007693787,"ci_75_lower":0.5276610215496299,"ci_75_upper":0.6407835037742506,"ci_50_lower":0.5545162462812477,"ci_50_upper":0.6213252623226422},"262144":{"avg_score":0.45136778427994684,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4805374181601482,"median_score":0.05122143402695656,"q1_score":0.02876869961619377,"q3_score":1,"avg_token_efficiency":0.982419810338046,"is_incomplete":false,"total_prompt_cost_bin":35.509745999999986,"total_cache_read_cost_bin":0.0771945,"total_completion_cost_bin":18.244632,"total_reasoning_cost_bin":17.923055999999995,"total_cost_bin":71.75462849999997,"total_on_demand_input_cost_bin":36.28169099999998,"ci_99_lower":0.3349022916543829,"ci_99_upper":0.5615091049040631,"ci_98_lower":0.360012264450903,"ci_98_upper":0.5529862174637622,"ci_95_lower":0.36963356626550303,"ci_95_upper":0.5345378980829449,"ci_90_lower":0.38187481304564586,"ci_90_upper":0.5186112764923081,"ci_75_lower":0.40369428935190765,"ci_75_upper":0.49922554659308754,"ci_50_lower":0.4236499166362985,"ci_50_upper":0.4786583916494682},"524288":{"avg_score":0.3761400262740346,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4602503359602323,"median_score":0.04120427370071411,"q1_score":0.024370830040425062,"q3_score":1,"avg_token_efficiency":0.9779163803533865,"is_incomplete":false,"total_prompt_cost_bin":234.7085619999997,"total_cache_read_cost_bin":0.7171588,"total_completion_cost_bin":36.64423200000001,"total_reasoning_cost_bin":36.127019999999995,"total_cost_bin":308.1969727999997,"total_on_demand_input_cost_bin":241.9349299999997,"ci_99_lower":0.29852695227288384,"ci_99_upper":0.450557533241177,"ci_98_lower":0.30482128005828,"ci_98_upper":0.4444411339257425,"ci_95_lower":0.3170169108358835,"ci_95_upper":0.4342120369859789,"ci_90_lower":0.3254280401298074,"ci_90_upper":0.42740488392547543,"ci_75_lower":0.3420466126241944,"ci_75_upper":0.4111008156584013,"ci_50_lower":0.35640919400340587,"ci_50_upper":0.3957889897855378}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low","none"],"reasoning_default":"low","sponsors":[],"openrouter_name":"OpenAI: GPT-5.6 Luna","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":2e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-09T09:54:24.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-09T20:14:27.240Z"},{"model_slug":"meta/muse-spark-1.1","reasoning_mode":"high","overall_metrics":{"auc_1m":0.3506184800115724,"auc_128k":0.6019873976123402,"cum_avg_1m":0.5490688025573909,"cum_avg_128k":0.6999004505153952,"avg_score_overall":0.5845717754632529,"avg_response_time_ms":71130.37273269636,"total_prompt_cost":0.004305000000000009,"total_cache_read_cost":27.609658349999915,"total_completion_cost":21.136082999999996,"total_reasoning_cost":19.71847,"total_prompt_cost_128k":0.002420000000000005,"total_completion_cost_128k":11.722545499999997,"total_on_demand_input_cost":230.08479125000025,"total_on_demand_input_cost_128k":30.623566250000017,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9996872469154785},"bin_metrics":{"8192":{"avg_score":0.8914562556077726,"min_score":0.015384615398943424,"max_score":1,"n_tests":80,"stddev":0.3069488862033714,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0100860082707628,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.08051639999999997,"total_completion_cost_bin":1.0798017500000003,"total_reasoning_cost_bin":0.9598284999999998,"total_cost_bin":2.12054665,"total_on_demand_input_cost_bin":0.6713700000000007,"ci_99_lower":0.7943732170344447,"ci_99_upper":0.9646878194930032,"ci_98_lower":0.8055248662766535,"ci_98_upper":0.9634796772734261,"ci_95_lower":0.8178013720797025,"ci_95_upper":0.9514818096079398,"ci_90_lower":0.8300236475188285,"ci_90_upper":0.9401219334796769,"ci_75_lower":0.8535499122343027,"ci_75_upper":0.9275662593485322,"ci_50_lower":0.8674070267035858,"ci_50_upper":0.9152080411993666},"16384":{"avg_score":0.818141320545692,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.3810184931673544,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9975608447485003,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.16486440000000008,"total_completion_cost_bin":1.17555425,"total_reasoning_cost_bin":1.0556489999999996,"total_cost_bin":2.39646765,"total_on_demand_input_cost_bin":1.37427,"ci_99_lower":0.6968765657392214,"ci_99_upper":0.9277500453843969,"ci_98_lower":0.7095747725863476,"ci_98_upper":0.9151377182134893,"ci_95_lower":0.7314695548042074,"ci_95_upper":0.8916517679483513,"ci_90_lower":0.7446993229439249,"ci_90_upper":0.8805582583107749,"ci_75_lower":0.7690930877462961,"ci_75_upper":0.8663114707538625,"ci_50_lower":0.7933675861015217,"ci_50_upper":0.8429348095989553},"32768":{"avg_score":0.7821635001506794,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.40241779707120423,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0178807703567443,"is_incomplete":false,"total_prompt_cost_bin":0.0006800000000000015,"total_cache_read_cost_bin":0.5918436000000006,"total_completion_cost_bin":3.36951475,"total_reasoning_cost_bin":3.1332147500000014,"total_cost_bin":7.0952531000000025,"total_on_demand_input_cost_bin":4.9327100000000055,"ci_99_lower":0.6891050012762187,"ci_99_upper":0.8732789280330283,"ci_98_lower":0.6976657610231675,"ci_98_upper":0.8604837062449694,"ci_95_lower":0.7102055895854017,"ci_95_upper":0.8516468155368142,"ci_90_lower":0.7244349250243807,"ci_90_upper":0.8378994430819362,"ci_75_lower":0.741130284681918,"ci_75_upper":0.8228094944767208,"ci_50_lower":0.7581003788004369,"ci_50_upper":0.8038918435374988},"65536":{"avg_score":0.4897078250096563,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.48512176384339867,"median_score":0.06942317634820938,"q1_score":0.024761518463492393,"q3_score":1,"avg_token_efficiency":0.9949327067350514,"is_incomplete":false,"total_prompt_cost_bin":0.0004250000000000008,"total_cache_read_cost_bin":0.8297183999999996,"total_completion_cost_bin":2.7754285000000003,"total_reasoning_cost_bin":2.6255395000000004,"total_cost_bin":6.2311114000000005,"total_on_demand_input_cost_bin":6.914744999999999,"ci_99_lower":0.3477258725750041,"ci_99_upper":0.6246608723343404,"ci_98_lower":0.3709276812788208,"ci_98_upper":0.6146173847889089,"ci_95_lower":0.3845905058270813,"ci_95_upper":0.5952002636862316,"ci_90_lower":0.4003470438962583,"ci_90_upper":0.5806149548305856,"ci_75_lower":0.4298418370424769,"ci_75_upper":0.5508482549640843,"ci_50_lower":0.45500060854589236,"ci_50_upper":0.5241544073981725},"131072":{"avg_score":0.5180333512631761,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.48908628127209214,"median_score":0.9985427260398865,"q1_score":0.02649853751063347,"q3_score":1,"avg_token_efficiency":0.992920300845969,"is_incomplete":false,"total_prompt_cost_bin":0.000515000000000001,"total_cache_read_cost_bin":2.00759475,"total_completion_cost_bin":3.3222462499999974,"total_reasoning_cost_bin":3.1340647500000003,"total_cost_bin":8.464420749999999,"total_on_demand_input_cost_bin":16.73047125000001,"ci_99_lower":0.3848092137004398,"ci_99_upper":0.641073554855448,"ci_98_lower":0.40466274829938537,"ci_98_upper":0.6311960292716501,"ci_95_lower":0.42385447444428137,"ci_95_upper":0.6118018893288217,"ci_90_lower":0.4417104941608556,"ci_90_upper":0.601941673951427,"ci_75_lower":0.4611969465811995,"ci_75_upper":0.5749992992922786,"ci_50_lower":0.48848459756203017,"ci_50_upper":0.5548565950463933},"262144":{"avg_score":0.4796601196318363,"min_score":0.002918642945587635,"max_score":1,"n_tests":141,"stddev":0.4831791657955898,"median_score":0.060400791466236115,"q1_score":0.029099488630890846,"q3_score":1,"avg_token_efficiency":0.9981804147787298,"is_incomplete":false,"total_prompt_cost_bin":0.0007050000000000015,"total_cache_read_cost_bin":5.529477149999992,"total_completion_cost_bin":3.6275662500000005,"total_reasoning_cost_bin":3.3998002500000006,"total_cost_bin":12.557548649999992,"total_on_demand_input_cost_bin":46.07968125000003,"ci_99_lower":0.36422360328792314,"ci_99_upper":0.5934906839325542,"ci_98_lower":0.3760254528158813,"ci_98_upper":0.5805539270501471,"ci_95_lower":0.39877599743463216,"ci_95_upper":0.562220322926973,"ci_90_lower":0.4120609481461472,"ci_90_upper":0.5481481642511026,"ci_75_lower":0.4326853006145197,"ci_75_upper":0.5279370917390012,"ci_50_lower":0.45201138360136855,"ci_50_upper":0.5071305295614301},"524288":{"avg_score":0.413388048250315,"min_score":0.0020505809225142,"max_score":1,"n_tests":236,"stddev":0.4705307767945825,"median_score":0.05244271457195282,"q1_score":0.024268270004540682,"q3_score":1,"avg_token_efficiency":0.9919647332235327,"is_incomplete":false,"total_prompt_cost_bin":0.0011800000000000027,"total_cache_read_cost_bin":18.405643649999924,"total_completion_cost_bin":5.7859712499999985,"total_reasoning_cost_bin":5.4103732499999975,"total_cost_bin":29.603168149999924,"total_on_demand_input_cost_bin":153.3815437500002,"ci_99_lower":0.3268546050893719,"ci_99_upper":0.48672009555149875,"ci_98_lower":0.339183320100508,"ci_98_upper":0.4817787481888919,"ci_95_lower":0.35357898818166195,"ci_95_upper":0.4730781711156527,"ci_90_lower":0.3601176984374356,"ci_90_upper":0.46526984720258047,"ci_75_lower":0.37812695933887697,"ci_75_upper":0.44858602496020406,"ci_50_lower":0.39377932869646726,"ci_50_upper":0.43368832590314954}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.1","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:29:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-17T02:53:07.696Z"},{"model_slug":"meta/muse-spark-1.1","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.3492442993188922,"auc_128k":0.6679176648959294,"cum_avg_1m":0.5776811008233254,"cum_avg_128k":0.745694703664644,"avg_score_overall":0.6063359434476813,"avg_response_time_ms":42360.29661242832,"total_prompt_cost":0.004305000000000009,"total_cache_read_cost":27.60965834999992,"total_completion_cost":10.62029525,"total_reasoning_cost":9.199218499999999,"total_prompt_cost_128k":0.002420000000000005,"total_completion_cost_128k":5.19894,"total_on_demand_input_cost":230.08479125000025,"total_on_demand_input_cost_128k":30.62356625,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9996872469154787},"bin_metrics":{"8192":{"avg_score":0.8914571266504936,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.30686461486150407,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0100860082707634,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.08051640000000002,"total_completion_cost_bin":0.5680635000000002,"total_reasoning_cost_bin":0.44760574999999997,"total_cost_bin":1.0965856500000002,"total_on_demand_input_cost_bin":0.6713700000000004,"ci_99_lower":0.7942004714468494,"ci_99_upper":0.9643960526930169,"ci_98_lower":0.8058955269093858,"ci_98_upper":0.9637253804125357,"ci_95_lower":0.8183946734655183,"ci_95_upper":0.9516971970209853,"ci_90_lower":0.8304047928540967,"ci_90_upper":0.9399266867141705,"ci_75_lower":0.8540912926386226,"ci_75_upper":0.9276016817282653,"ci_50_lower":0.8675622244976693,"ci_50_upper":0.9152888111333595},"16384":{"avg_score":0.8779606369207613,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.3243673253383902,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9975608447484994,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.16486439999999997,"total_completion_cost_bin":0.6335092499999999,"total_reasoning_cost_bin":0.5150915,"total_cost_bin":1.3138651499999998,"total_on_demand_input_cost_bin":1.3742700000000003,"ci_99_lower":0.7809102404554724,"ci_99_upper":0.9632739242383976,"ci_98_lower":0.791594551497139,"ci_98_upper":0.9512905188675503,"ci_95_lower":0.8037665359018138,"ci_95_upper":0.9394419149204623,"ci_90_lower":0.8158471938199363,"ci_90_upper":0.9271309752593516,"ci_75_lower":0.8403060084383469,"ci_75_upper":0.9147710669509251,"ci_50_lower":0.8535200095386244,"ci_50_upper":0.9023056362522766},"32768":{"avg_score":0.851740548195427,"min_score":0.008541532792150974,"max_score":1,"n_tests":136,"stddev":0.3484689217418091,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0178807703567456,"is_incomplete":false,"total_prompt_cost_bin":0.0006800000000000015,"total_cache_read_cost_bin":0.5918435999999991,"total_completion_cost_bin":1.37907825,"total_reasoning_cost_bin":1.1448055000000006,"total_cost_bin":3.11640735,"total_on_demand_input_cost_bin":4.9327099999999975,"ci_99_lower":0.7762146551367443,"ci_99_upper":0.9230649075533524,"ci_98_lower":0.7802686470259419,"ci_98_upper":0.9170017786532202,"ci_95_lower":0.7885095062693033,"ci_95_upper":0.9091194783500054,"ci_90_lower":0.797924811901826,"ci_90_upper":0.9007417105908698,"ci_75_lower":0.8153959518522778,"ci_75_upper":0.8865094460009405,"ci_50_lower":0.8295975048009389,"ci_50_upper":0.8722925505939844},"65536":{"avg_score":0.6358064919600592,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.47171854433061317,"median_score":1,"q1_score":0.03554007038474083,"q3_score":1,"avg_token_efficiency":0.9949327067350526,"is_incomplete":false,"total_prompt_cost_bin":0.0004250000000000008,"total_cache_read_cost_bin":0.8297184000000004,"total_completion_cost_bin":1.2098559999999996,"total_reasoning_cost_bin":1.0598607499999997,"total_cost_bin":3.0998601499999996,"total_on_demand_input_cost_bin":6.914744999999998,"ci_99_lower":0.49715600586025155,"ci_99_upper":0.7602969978745096,"ci_98_lower":0.5089337858672198,"ci_98_upper":0.7492846856636051,"ci_95_lower":0.5332831438455512,"ci_95_upper":0.7291604644692886,"ci_90_lower":0.5553998414490043,"ci_90_upper":0.7175425674547165,"ci_75_lower":0.5779371099683511,"ci_75_upper":0.6931857650072369,"ci_50_lower":0.6006527148063897,"ci_50_upper":0.6699953650439378},"131072":{"avg_score":0.4715087145964787,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.4865929364084747,"median_score":0.0428713858127594,"q1_score":0.026781778782606125,"q3_score":1,"avg_token_efficiency":0.992920300845969,"is_incomplete":false,"total_prompt_cost_bin":0.000515000000000001,"total_cache_read_cost_bin":2.007594750000001,"total_completion_cost_bin":1.4084330000000007,"total_reasoning_cost_bin":1.2225167499999996,"total_cost_bin":4.639059500000001,"total_on_demand_input_cost_bin":16.730471250000004,"ci_99_lower":0.3473965167431658,"ci_99_upper":0.5773561101499718,"ci_98_lower":0.35684718255473535,"ci_98_upper":0.5751288239743777,"ci_95_lower":0.3760655150126023,"ci_95_upper":0.5655260461415345,"ci_90_lower":0.3951951560950883,"ci_90_upper":0.5556533441135154,"ci_75_lower":0.4147106978784596,"ci_75_upper":0.5285273630960926,"ci_50_lower":0.4397455793330717,"ci_50_upper":0.5084507458298513},"262144":{"avg_score":0.5124842713897427,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4859064038334097,"median_score":0.08734602481126785,"q1_score":0.028867505490779877,"q3_score":1,"avg_token_efficiency":0.9981804147787295,"is_incomplete":false,"total_prompt_cost_bin":0.0007050000000000015,"total_cache_read_cost_bin":5.529477149999992,"total_completion_cost_bin":2.1208010000000006,"total_reasoning_cost_bin":1.8890654999999994,"total_cost_bin":9.540048649999992,"total_on_demand_input_cost_bin":46.07968125000002,"ci_99_lower":0.3958573677207522,"ci_99_upper":0.620160572930117,"ci_98_lower":0.4092252658783451,"ci_98_upper":0.609316747121493,"ci_95_lower":0.42954217538919554,"ci_95_upper":0.5948613410416812,"ci_90_lower":0.44920963125818586,"ci_90_upper":0.581474573846662,"ci_75_lower":0.46601222966862044,"ci_75_upper":0.5611125361756422,"ci_50_lower":0.48556120428900684,"ci_50_upper":0.5397916229033896},"524288":{"avg_score":0.38049101687363995,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.46181707844948716,"median_score":0.043815262615680695,"q1_score":0.02441494120284915,"q3_score":1,"avg_token_efficiency":0.991964733223532,"is_incomplete":false,"total_prompt_cost_bin":0.0011800000000000027,"total_cache_read_cost_bin":18.405643649999927,"total_completion_cost_bin":3.3005542499999994,"total_reasoning_cost_bin":2.920272749999999,"total_cost_bin":24.627650649999925,"total_on_demand_input_cost_bin":153.38154375000025,"ci_99_lower":0.30298337456047575,"ci_99_upper":0.45407630812622884,"ci_98_lower":0.30756754909726886,"ci_98_upper":0.4490496072225406,"ci_95_lower":0.32107836843704535,"ci_95_upper":0.43879186672230563,"ci_90_lower":0.328890531472477,"ci_90_upper":0.4316411967841628,"ci_75_lower":0.3460562545016913,"ci_75_upper":0.4151405771084128,"ci_50_lower":0.36055168020366335,"ci_50_upper":0.39970561678909056}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.1","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:29:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-17T01:49:06.894Z"},{"model_slug":"google/gemini-3.5-flash","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.3476693806710958,"auc_128k":0.6408195322955204,"cum_avg_1m":0.575943320961411,"cum_avg_128k":0.7414520289660262,"avg_score_overall":0.4618837236238677,"avg_response_time_ms":21169.019883909154,"total_prompt_cost":35.00898749999996,"total_cache_read_cost":73.02559709999981,"total_completion_cost":4.102515,"total_reasoning_cost":23.342472000000004,"total_prompt_cost_128k":7.710116999999994,"total_completion_cost_128k":1.6889129999999999,"total_on_demand_input_cost":765.2649585000015,"total_on_demand_input_cost_128k":36.8202,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154892},"bin_metrics":{"8192":{"avg_score":0.9037671981379389,"min_score":0.018300654366612434,"max_score":1,"n_tests":80,"stddev":0.29054969119561797,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747232,"is_incomplete":false,"total_prompt_cost_bin":0.7973159999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.25533900000000004,"total_reasoning_cost_bin":0.7638210000000002,"total_cost_bin":1.8164759999999998,"total_on_demand_input_cost_bin":0.7973159999999996,"ci_99_lower":0.8186581392138033,"ci_99_upper":0.9757997319959104,"ci_98_lower":0.8192090592118911,"ci_98_upper":0.9643130804838147,"ci_95_lower":0.8312715956754982,"ci_95_upper":0.9631955179647775,"ci_90_lower":0.8434876955254003,"ci_90_upper":0.951960686162347,"ci_75_lower":0.8669283824594458,"ci_75_upper":0.9397954990941798,"ci_50_lower":0.8799485728959553,"ci_50_upper":0.927556431753328},"16384":{"avg_score":0.9511898074415512,"min_score":0.011840158142149448,"max_score":1,"n_tests":80,"stddev":0.21377107379066276,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024467,"is_incomplete":false,"total_prompt_cost_bin":1.6528079999999985,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.24986700000000003,"total_reasoning_cost_bin":0.700047,"total_cost_bin":2.6027219999999986,"total_on_demand_input_cost_bin":1.6528079999999985,"ci_99_lower":0.8781247841542935,"ci_99_upper":0.9999239674285054,"ci_98_lower":0.8894893401244189,"ci_98_upper":0.9999235853552818,"ci_95_lower":0.8908086900206399,"ci_95_upper":0.988016470707953,"ci_90_lower":0.9026234624977224,"ci_90_upper":0.9877365458803251,"ci_75_lower":0.9264093967503868,"ci_75_upper":0.9757326656719669,"ci_50_lower":0.9387152016744948,"ci_50_upper":0.9636347502935678},"32768":{"avg_score":0.7812810653036543,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4039209020541221,"median_score":1,"q1_score":0.9996998906135559,"q3_score":1,"avg_token_efficiency":1.0000359090949913,"is_incomplete":false,"total_prompt_cost_bin":2.005191,"total_cache_read_cost_bin":0.3804933000000002,"total_completion_cost_bin":0.478719,"total_reasoning_cost_bin":1.719189,"total_cost_bin":4.5835923,"total_on_demand_input_cost_bin":5.810123999999993,"ci_99_lower":0.687980805170647,"ci_99_upper":0.8730601632164838,"ci_98_lower":0.6966635710956283,"ci_98_upper":0.8600804014782154,"ci_95_lower":0.709567826554335,"ci_95_upper":0.8514443022194695,"ci_90_lower":0.7235568604677716,"ci_90_upper":0.8375287878358745,"ci_75_lower":0.7396412116260549,"ci_75_upper":0.8223051229917259,"ci_50_lower":0.7574550913088709,"ci_50_upper":0.8030627914366264},"65536":{"avg_score":0.5528465912512996,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4848583088939673,"median_score":0.9987484216690063,"q1_score":0.028731433674693108,"q3_score":1,"avg_token_efficiency":1.0000152878522814,"is_incomplete":false,"total_prompt_cost_bin":1.462282499999999,"total_cache_read_cost_bin":0.68778,"total_completion_cost_bin":0.31106699999999987,"total_reasoning_cost_bin":1.1554289999999996,"total_cost_bin":3.6165584999999982,"total_on_demand_input_cost_bin":8.3400825,"ci_99_lower":0.40590372756271464,"ci_99_upper":0.6804822611403926,"ci_98_lower":0.4259014122745789,"ci_98_upper":0.6768410472785505,"ci_95_lower":0.45204730588778413,"ci_95_upper":0.6577655852262806,"ci_90_lower":0.47194764542722084,"ci_90_upper":0.6445519096508402,"ci_75_lower":0.49452011365279114,"ci_75_upper":0.6175172857803237,"ci_50_lower":0.5184232454664786,"ci_50_upper":0.5876124531797626},"131072":{"avg_score":0.5181754826956871,"min_score":0.006819288711994886,"max_score":1,"n_tests":103,"stddev":0.4888198632614881,"median_score":0.9949331879615784,"q1_score":0.026603619568049908,"q3_score":1,"avg_token_efficiency":1.0000076410572556,"is_incomplete":false,"total_prompt_cost_bin":1.7925194999999976,"total_cache_read_cost_bin":1.8427350000000018,"total_completion_cost_bin":0.3939209999999999,"total_reasoning_cost_bin":2.035494,"total_cost_bin":6.064669499999999,"total_on_demand_input_cost_bin":20.219869500000012,"ci_99_lower":0.384948466679226,"ci_99_upper":0.6411702178332059,"ci_98_lower":0.4050315842533809,"ci_98_upper":0.6310355443248833,"ci_95_lower":0.42411327785518455,"ci_95_upper":0.6119895076216425,"ci_90_lower":0.441853830153213,"ci_90_upper":0.601816508686396,"ci_75_lower":0.4612967443293033,"ci_75_upper":0.5750549817627331,"ci_50_lower":0.4885578153765697,"ci_50_upper":0.5549107544142211},"262144":{"avg_score":0.41697673325043777,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.47211295400420894,"median_score":0.04574209079146385,"q1_score":0.028974739834666252,"q3_score":1,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":4.358707500000002,"total_cache_read_cost_bin":5.103792000000007,"total_completion_cost_bin":0.48230099999999987,"total_reasoning_cost_bin":3.5764199999999984,"total_cost_bin":13.521220500000009,"total_on_demand_input_cost_bin":55.39662749999999,"ci_99_lower":0.30834674981355015,"ci_99_upper":0.5244588719429993,"ci_98_lower":0.31949949036505904,"ci_98_upper":0.5145754429226715,"ci_95_lower":0.33579071824049167,"ci_95_upper":0.4999529692836449,"ci_90_lower":0.3499289795815786,"ci_90_upper":0.48558370338344997,"ci_75_lower":0.36937924776765424,"ci_75_upper":0.4634092662166408,"ci_50_lower":0.3897337600892634,"ci_50_upper":0.44432784226749433},"524288":{"avg_score":0.30768517710483567,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.43196649643691754,"median_score":0.040997063741087914,"q1_score":0.0243167276494205,"q3_score":0.9995011240243912,"avg_token_efficiency":1.0000019078540325,"is_incomplete":false,"total_prompt_cost_bin":8.281156499999993,"total_cache_read_cost_bin":17.72679839999993,"total_completion_cost_bin":0.8025660000000001,"total_reasoning_cost_bin":9.845640000000008,"total_cost_bin":36.65616089999993,"total_on_demand_input_cost_bin":185.54914050000016,"ci_99_lower":0.2408317767722178,"ci_99_upper":0.3754416658471228,"ci_98_lower":0.24355948421368137,"ci_98_upper":0.3686761812099317,"ci_95_lower":0.25246944788900616,"ci_95_upper":0.36195302647386796,"ci_90_lower":0.26289187047790374,"ci_90_upper":0.3534670817972826,"ci_75_lower":0.2764589119392671,"ci_75_upper":0.34058168130829114,"ci_50_lower":0.2906179043478668,"ci_50_upper":0.32713612103691697},"1048576":{"avg_score":0.17562451250588282,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.3385989981983521,"median_score":0.03358699381351471,"q1_score":0.02150846691802144,"q3_score":0.05454786401242018,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":14.659006499999974,"total_cache_read_cost_bin":47.283998399999874,"total_completion_cost_bin":1.1287349999999998,"total_reasoning_cost_bin":3.5464320000000007,"total_cost_bin":66.61817189999985,"total_on_demand_input_cost_bin":487.4989905000014,"ci_99_lower":0.128818110834441,"ci_99_upper":0.2240035070301627,"ci_98_lower":0.1323993049067888,"ci_98_upper":0.22132372066019582,"ci_95_lower":0.13868919065443708,"ci_95_upper":0.2156982494469717,"ci_90_lower":0.14400504335090744,"ci_90_upper":0.2084998145407524,"ci_75_lower":0.1515967745556883,"ci_75_upper":0.19855540247165576,"ci_50_lower":0.16141173199998335,"ci_50_upper":0.18824086822137506}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":0.0000015,"completion":0.000009,"reasoning":0.000009,"has_tiered_pricing":true},"api_created_at":"2026-05-19T12:30:00.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"google/gemini-3-flash-preview","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.34741998616509034,"auc_128k":0.5766669487177244,"cum_avg_1m":0.560714930993686,"cum_avg_128k":0.7025054216480879,"avg_score_overall":0.46238393017570434,"avg_response_time_ms":60733.77350254857,"total_prompt_cost":8.388214000000001,"total_cache_read_cost":24.670486299999922,"total_completion_cost":1.3933440000000004,"total_reasoning_cost":50.59461000000001,"total_prompt_cost_128k":1.860686499999999,"total_completion_cost_128k":0.5616690000000001,"total_on_demand_input_cost":255.09307699999965,"total_on_demand_input_cost_128k":12.278157500000002,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0011380420774558},"bin_metrics":{"8192":{"avg_score":0.9750627648551017,"min_score":0.0181695818901062,"max_score":1,"n_tests":80,"stddev":0.15353780245697116,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.016480397293012,"is_incomplete":false,"total_prompt_cost_bin":0.13317549999999992,"total_cache_read_cost_bin":0.013735399999999984,"total_completion_cost_bin":0.08356199999999997,"total_reasoning_cost_bin":0.5544209999999999,"total_cost_bin":0.7848938999999998,"total_on_demand_input_cost_bin":0.2705295,"ci_99_lower":0.9255979055592325,"ci_99_upper":0.999989426676184,"ci_98_lower":0.9263022175198421,"ci_98_upper":0.999985711582005,"ci_95_lower":0.9378801349818241,"ci_95_upper":0.9999767076037824,"ci_90_lower":0.939047761799302,"ci_90_upper":0.9999531193822622,"ci_75_lower":0.9511010594666005,"ci_75_upper":0.9985344470478594,"ci_50_lower":0.96312029393157,"ci_50_upper":0.9873315677512438},"16384":{"avg_score":0.9565687391674146,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.1928414318087503,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024474,"is_incomplete":false,"total_prompt_cost_bin":0.28940650000000007,"total_cache_read_cost_bin":0.026152949999999973,"total_completion_cost_bin":0.08301299999999999,"total_reasoning_cost_bin":0.5954340000000002,"total_cost_bin":0.9940064500000003,"total_on_demand_input_cost_bin":0.5509359999999999,"ci_99_lower":0.8909352408072446,"ci_99_upper":0.9989755420014262,"ci_98_lower":0.8954172782972456,"ci_98_upper":0.998941907517612,"ci_95_lower":0.9042317358613946,"ci_95_upper":0.993322444818914,"ci_90_lower":0.9152695651259273,"ci_90_upper":0.9878382504172623,"ci_75_lower":0.9310964230884564,"ci_75_upper":0.9812171404249966,"ci_50_lower":0.9442421527870466,"ci_50_upper":0.9699848629767075},"32768":{"avg_score":0.7302579649908961,"min_score":0.005860192701220512,"max_score":1,"n_tests":136,"stddev":0.4348234633573025,"median_score":1,"q1_score":0.06143832951784134,"q3_score":1,"avg_token_efficiency":1.0000359090949904,"is_incomplete":false,"total_prompt_cost_bin":0.5906494999999994,"total_cache_read_cost_bin":0.13460584999999986,"total_completion_cost_bin":0.1591200000000001,"total_reasoning_cost_bin":1.5058649999999996,"total_cost_bin":2.390240349999999,"total_on_demand_input_cost_bin":1.9367080000000008,"ci_99_lower":0.6303416331383221,"ci_99_upper":0.8294732187132466,"ci_98_lower":0.6381703273167231,"ci_98_upper":0.8152496625124203,"ci_95_lower":0.654017343829807,"ci_95_upper":0.801496606694983,"ci_90_lower":0.6671502730458194,"ci_90_upper":0.7874631084001843,"ci_75_lower":0.686838987402392,"ci_75_upper":0.7731136946647037,"ci_50_lower":0.7024841635327046,"ci_50_upper":0.7571451916429557},"65536":{"avg_score":0.567147643853198,"min_score":0.007581501267850399,"max_score":1,"n_tests":85,"stddev":0.4826840816102933,"median_score":0.9982497096061707,"q1_score":0.029518619179725647,"q3_score":1,"avg_token_efficiency":1.0000152878522812,"is_incomplete":false,"total_prompt_cost_bin":0.37279749999999995,"total_cache_read_cost_bin":0.240723,"total_completion_cost_bin":0.10313100000000004,"total_reasoning_cost_bin":1.6138800000000004,"total_cost_bin":2.3305315,"total_on_demand_input_cost_bin":2.7800274999999997,"ci_99_lower":0.42813445072286926,"ci_99_upper":0.6919500542812269,"ci_98_lower":0.44321523737085655,"ci_98_upper":0.6900084257612333,"ci_95_lower":0.4669253805372864,"ci_95_upper":0.6704126000571449,"ci_90_lower":0.4856298572581042,"ci_90_upper":0.6578136833401068,"ci_75_lower":0.5089757136480115,"ci_75_upper":0.6270624264047536,"ci_50_lower":0.5323594547479469,"ci_50_upper":0.6017101805629756},"131072":{"avg_score":0.2834899953738289,"min_score":0.004605642054229975,"max_score":1,"n_tests":103,"stddev":0.42850772534152715,"median_score":0.03448275849223137,"q1_score":0.02147552650421858,"q3_score":0.9853911697864532,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":0.4746574999999996,"total_cache_read_cost_bin":0.6265299000000011,"total_completion_cost_bin":0.13284300000000002,"total_reasoning_cost_bin":4.592844000000001,"total_cost_bin":5.826874400000002,"total_on_demand_input_cost_bin":6.739956500000003,"ci_99_lower":0.17964091551307027,"ci_99_upper":0.38708568535629856,"ci_98_lower":0.18843739101862156,"ci_98_upper":0.3853958920719446,"ci_95_lower":0.20736334245319668,"ci_95_upper":0.36707279793522546,"ci_90_lower":0.21746569596845836,"ci_90_upper":0.3505386946024373,"ci_75_lower":0.23649114414708217,"ci_75_upper":0.3306400534963615,"ci_50_lower":0.2551209002312779,"ci_50_upper":0.3112627298953192},"262144":{"avg_score":0.3978287241989681,"min_score":0.004376367665827274,"max_score":1,"n_tests":141,"stddev":0.46924787336428475,"median_score":0.04512276127934456,"q1_score":0.026101142168045044,"q3_score":1,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":0.6911424999999993,"total_cache_read_cost_bin":1.7774400000000037,"total_completion_cost_bin":0.167328,"total_reasoning_cost_bin":6.136614000000001,"total_cost_bin":8.772524500000003,"total_on_demand_input_cost_bin":18.465542499999994,"ci_99_lower":0.29338162370751025,"ci_99_upper":0.50134065170419,"ci_98_lower":0.3016947020736279,"ci_98_upper":0.49858916748249044,"ci_95_lower":0.3219566633917411,"ci_95_upper":0.4793143329551569,"ci_90_lower":0.3302952962661664,"ci_90_upper":0.46638167983297496,"ci_75_lower":0.3505794533911485,"ci_75_upper":0.4428291909536182,"ci_50_lower":0.3723131269191768,"ci_50_upper":0.4253518790805858},"524288":{"avg_score":0.32780556075701145,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.44278844659264294,"median_score":0.03893323615193367,"q1_score":0.023088951129466295,"q3_score":0.9997870922088623,"avg_token_efficiency":1.0000019078540323,"is_incomplete":false,"total_prompt_cost_bin":1.4702255000000002,"total_cache_read_cost_bin":6.037948800000019,"total_completion_cost_bin":0.2739690000000002,"total_reasoning_cost_bin":13.571631000000002,"total_cost_bin":21.353774300000023,"total_on_demand_input_cost_bin":61.84971350000003,"ci_99_lower":0.2594195941486988,"ci_99_upper":0.39892256117118047,"ci_98_lower":0.2641897916171889,"ci_98_upper":0.3952203083318911,"ci_95_lower":0.2704673005863373,"ci_95_upper":0.385110901956993,"ci_90_lower":0.2799966615629659,"ci_90_upper":0.37766724909464355,"ci_75_lower":0.2963025028848432,"ci_75_upper":0.36038327315339813,"ci_50_lower":0.3100782308068474,"ci_50_upper":0.3475312947545208},"1048576":{"avg_score":0.24755805475306847,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.3996954101356587,"median_score":0.03510240279138088,"q1_score":0.022748759016394615,"q3_score":0.09664640761911869,"avg_token_efficiency":1.0000009538490409,"is_incomplete":false,"total_prompt_cost_bin":4.366159500000004,"total_cache_read_cost_bin":15.813350399999896,"total_completion_cost_bin":0.3903780000000001,"total_reasoning_cost_bin":22.02392100000001,"total_cost_bin":42.59380889999991,"total_on_demand_input_cost_bin":162.49966349999963,"ci_99_lower":0.18824745338572368,"ci_99_upper":0.30588330845909584,"ci_98_lower":0.19778287893766536,"ci_98_upper":0.3011179622341878,"ci_95_lower":0.20242283561101723,"ci_95_upper":0.29554232152122556,"ci_90_lower":0.2101985851650487,"ci_90_upper":0.2861912823804173,"ci_75_lower":0.2212908227899818,"ci_75_upper":0.27347141999841457,"ci_50_lower":0.2312636984966605,"ci_50_upper":0.2624903134674945}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3 Flash Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":5e-7,"completion":0.000003,"reasoning":0.000003,"has_tiered_pricing":true},"api_created_at":"2025-12-17T15:57:58.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:35:10.140Z"},{"model_slug":"meta/muse-spark-1.1","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.33936062369337566,"auc_128k":0.5943216469178202,"cum_avg_1m":0.5452941971663366,"cum_avg_128k":0.6971100106972384,"avg_score_overall":0.5830072640915703,"avg_response_time_ms":84196.2032445094,"total_prompt_cost":0.004290000000000009,"total_cache_read_cost":27.375688199999907,"total_completion_cost":25.548734750000005,"total_reasoning_cost":24.142260999999994,"total_prompt_cost_128k":0.002420000000000005,"total_completion_cost_128k":14.004247250000002,"total_on_demand_input_cost":228.13502500000027,"total_on_demand_input_cost_128k":30.62356625000001,"total_runs":858,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9997142552416887},"bin_metrics":{"8192":{"avg_score":0.8644687640015036,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.3353343508728598,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0100860082707628,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.08051639999999996,"total_completion_cost_bin":1.49344575,"total_reasoning_cost_bin":1.3718447499999997,"total_cost_bin":2.9462069,"total_on_demand_input_cost_bin":0.6713700000000004,"ci_99_lower":0.7572861711599399,"ci_99_upper":0.9520716645267093,"ci_98_lower":0.7755827626872343,"ci_98_upper":0.9396239414443261,"ci_95_lower":0.782346610724926,"ci_95_upper":0.9271762695885263,"ci_90_lower":0.7947583285730797,"ci_90_upper":0.9225139807874803,"ci_75_lower":0.818518433504505,"ci_75_upper":0.9030047195410588,"ci_50_lower":0.8407047106011305,"ci_50_upper":0.8887790411012246},"16384":{"avg_score":0.8544673106982372,"min_score":0.013550135307013988,"max_score":1,"n_tests":80,"stddev":0.3481023115733669,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9975608447484998,"is_incomplete":false,"total_prompt_cost_bin":0.0004000000000000007,"total_cache_read_cost_bin":0.1648644,"total_completion_cost_bin":1.5011935000000003,"total_reasoning_cost_bin":1.3813732500000004,"total_cost_bin":3.047831150000001,"total_on_demand_input_cost_bin":1.3742699999999997,"ci_99_lower":0.7448849313817336,"ci_99_upper":0.9514652662454754,"ci_98_lower":0.7579108120813035,"ci_98_upper":0.9389787504208508,"ci_95_lower":0.7704644623285277,"ci_95_upper":0.9268875517285778,"ci_90_lower":0.7823941661341814,"ci_90_upper":0.9148926365934313,"ci_75_lower":0.8059735091519542,"ci_75_upper":0.9021765440906165,"ci_50_lower":0.8300438376958483,"ci_50_upper":0.8786691043205791},"32768":{"avg_score":0.8580987917153192,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.34310776607473353,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0178807703567456,"is_incomplete":false,"total_prompt_cost_bin":0.0006800000000000015,"total_cache_read_cost_bin":0.5918435999999997,"total_completion_cost_bin":3.586579250000002,"total_reasoning_cost_bin":3.3540659999999978,"total_cost_bin":7.533168849999999,"total_on_demand_input_cost_bin":4.932709999999999,"ci_99_lower":0.782035158579974,"ci_99_upper":0.9292933014000643,"ci_98_lower":0.7868667028379894,"ci_98_upper":0.9229162879544994,"ci_95_lower":0.7952485187455108,"ci_95_upper":0.9149657789169664,"ci_90_lower":0.8077017132316969,"ci_90_upper":0.90735317644456,"ci_75_lower":0.8222022164225243,"ci_75_upper":0.8931768490418857,"ci_50_lower":0.8367255039642366,"ci_50_upper":0.8788481155343641},"65536":{"avg_score":0.49626611614435473,"min_score":0.0037678976077586412,"max_score":1,"n_tests":85,"stddev":0.48919003596843763,"median_score":0.052147239446640015,"q1_score":0.023540489375591278,"q3_score":1,"avg_token_efficiency":0.9949327067350516,"is_incomplete":false,"total_prompt_cost_bin":0.0004250000000000008,"total_cache_read_cost_bin":0.8297183999999995,"total_completion_cost_bin":3.1942915,"total_reasoning_cost_bin":3.045571249999999,"total_cost_bin":7.070006149999998,"total_on_demand_input_cost_bin":6.914744999999998,"ci_99_lower":0.3579277518646764,"ci_99_upper":0.6340987922140562,"ci_98_lower":0.3694034419163568,"ci_98_upper":0.6216108847952667,"ci_95_lower":0.3926618494828204,"ci_95_upper":0.6014465862129785,"ci_90_lower":0.40527081827961786,"ci_90_upper":0.587845647172345,"ci_75_lower":0.43788867055999986,"ci_75_upper":0.5546650159597288,"ci_50_lower":0.4619489528546931,"ci_50_upper":0.5308904865708695},"131072":{"avg_score":0.4122490709267774,"min_score":0.005760921631008387,"max_score":1,"n_tests":103,"stddev":0.48036018876815406,"median_score":0.034924332052469254,"q1_score":0.01981703843921423,"q3_score":1,"avg_token_efficiency":0.992920300845969,"is_incomplete":false,"total_prompt_cost_bin":0.000515000000000001,"total_cache_read_cost_bin":2.00759475,"total_completion_cost_bin":4.22873725,"total_reasoning_cost_bin":4.044725,"total_cost_bin":10.281572,"total_on_demand_input_cost_bin":16.73047125000001,"ci_99_lower":0.28790938453355497,"ci_99_upper":0.525674357338406,"ci_98_lower":0.29930244962802355,"ci_98_upper":0.5169374029229856,"ci_95_lower":0.3171729988890962,"ci_95_upper":0.5061888110887223,"ci_90_lower":0.3351441981304817,"ci_90_upper":0.4885947531251942,"ci_75_lower":0.35548027005109895,"ci_75_upper":0.46915140899816,"ci_50_lower":0.38313258254055155,"ci_50_upper":0.44912764432419594},"262144":{"avg_score":0.46669113361078535,"min_score":0.003799392143264413,"max_score":1,"n_tests":141,"stddev":0.48130256677238037,"median_score":0.057162072509527206,"q1_score":0.030577776953577995,"q3_score":1,"avg_token_efficiency":0.99818041477873,"is_incomplete":false,"total_prompt_cost_bin":0.0007050000000000015,"total_cache_read_cost_bin":5.529477149999992,"total_completion_cost_bin":4.769987499999999,"total_reasoning_cost_bin":4.540389749999998,"total_cost_bin":14.84055939999999,"total_on_demand_input_cost_bin":46.07968125000003,"ci_99_lower":0.35087926083525106,"ci_99_upper":0.5821382215448593,"ci_98_lower":0.3686845762487699,"ci_98_upper":0.56901685624925,"ci_95_lower":0.38544407139969034,"ci_95_upper":0.5487075101686576,"ci_90_lower":0.3989975531810914,"ci_90_upper":0.5344950109389147,"ci_75_lower":0.4184078781936542,"ci_75_upper":0.5151268221980377,"ci_50_lower":0.4394560400425483,"ci_50_upper":0.49364014865388},"524288":{"avg_score":0.4101123902337154,"min_score":0.0029607696924358606,"max_score":1,"n_tests":233,"stddev":0.4690462867842618,"median_score":0.048701297491788864,"q1_score":0.024981897324323654,"q3_score":1,"avg_token_efficiency":0.9919647572699368,"is_incomplete":false,"total_prompt_cost_bin":0.0011650000000000026,"total_cache_read_cost_bin":18.171673499999915,"total_completion_cost_bin":6.774500000000003,"total_reasoning_cost_bin":6.404290999999997,"total_cost_bin":31.351629499999916,"total_on_demand_input_cost_bin":151.43177750000024,"ci_99_lower":0.3322294326007462,"ci_99_upper":0.4945905295556237,"ci_98_lower":0.3446910501658005,"ci_98_upper":0.4850082115935014,"ci_95_lower":0.3533708620646593,"ci_95_upper":0.4716253489981377,"ci_90_lower":0.36042911245376713,"ci_90_upper":0.4607204247901592,"ci_75_lower":0.3741472968979051,"ci_75_upper":0.4449861615481776,"ci_50_lower":0.38864746132127087,"ci_50_upper":0.43067267673043136}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","minimal","enabled"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Meta: Muse Spark 1.1","provider_name":"Meta","run_provider_names":["Meta"],"pricing_metadata":{"prompt":0.00000125,"completion":0.00000425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:29:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-17T00:19:45.552Z"},{"model_slug":"openai/gpt-5.4","reasoning_mode":"low","overall_metrics":{"auc_1m":0.3392869809405123,"auc_128k":0.6587364285711454,"cum_avg_1m":0.5715521633307228,"cum_avg_128k":0.750936245868309,"avg_score_overall":0.5931672448592269,"avg_response_time_ms":15024.325792053436,"total_prompt_cost":48.58645500000007,"total_cache_read_cost":70.66771200000014,"total_completion_cost":11.925405000000005,"total_reasoning_cost":7.074120000000002,"total_prompt_cost_128k":2.8698224999999997,"total_completion_cost_128k":4.9170300000000005,"total_on_demand_input_cost":755.4055250000008,"total_on_demand_input_cost_128k":60.134782499999986,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778016},"bin_metrics":{"8192":{"avg_score":0.9637066692113876,"min_score":0.006989952176809311,"max_score":1,"n_tests":80,"stddev":0.18506291766857336,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9759044449598393,"is_incomplete":false,"total_prompt_cost_bin":0.19366000000000008,"total_cache_read_cost_bin":0.11043200000000011,"total_completion_cost_bin":0.5855550000000003,"total_reasoning_cost_bin":0.17063999999999996,"total_cost_bin":1.0602870000000004,"total_on_demand_input_cost_bin":1.2979800000000004,"ci_99_lower":0.9025857562609017,"ci_99_upper":1,"ci_98_lower":0.9035239181369543,"ci_98_upper":1,"ci_95_lower":0.9154709592461586,"ci_95_upper":1,"ci_90_lower":0.9273823277512566,"ci_90_upper":1,"ci_75_lower":0.939355717599392,"ci_75_upper":0.9880576208233833,"ci_50_lower":0.9512980967760086,"ci_50_upper":0.9761192948091775},"16384":{"avg_score":0.8780640561482869,"min_score":0.009598080068826675,"max_score":1,"n_tests":80,"stddev":0.3246778475589409,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.967068739318871,"is_incomplete":false,"total_prompt_cost_bin":0.28022000000000036,"total_cache_read_cost_bin":0.23840000000000006,"total_completion_cost_bin":0.6463050000000001,"total_reasoning_cost_bin":0.24250500000000005,"total_cost_bin":1.4074300000000006,"total_on_demand_input_cost_bin":2.664219999999998,"ci_99_lower":0.7807303473852225,"ci_99_upper":0.9633573712172802,"ci_98_lower":0.7918797288197092,"ci_98_upper":0.9515229744112585,"ci_95_lower":0.8040003079987945,"ci_95_upper":0.9395389687240823,"ci_90_lower":0.8162198923842515,"ci_90_upper":0.9271643724606837,"ci_75_lower":0.8405095632435405,"ci_75_upper":0.9149561871003243,"ci_50_lower":0.8536870370706311,"ci_50_upper":0.9023713782575215},"32768":{"avg_score":0.7680257048957762,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4118095420840482,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9952100768549628,"is_incomplete":false,"total_prompt_cost_bin":0.5579599999999999,"total_cache_read_cost_bin":0.909184,"total_completion_cost_bin":1.389135,"total_reasoning_cost_bin":0.5928599999999999,"total_cost_bin":3.4491389999999997,"total_on_demand_input_cost_bin":9.6498,"ci_99_lower":0.6761257621057517,"ci_99_upper":0.8593898671102258,"ci_98_lower":0.6849188091690817,"ci_98_upper":0.8521345901430023,"ci_95_lower":0.6983072760577402,"ci_95_upper":0.8395583559705889,"ci_90_lower":0.7084080429948003,"ci_90_upper":0.8235922823562493,"ci_75_lower":0.7277638701912913,"ci_75_upper":0.8083697409884361,"ci_50_lower":0.743771344264935,"ci_50_upper":0.791907461502192},"65536":{"avg_score":0.5992403493536745,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.48188647920461164,"median_score":1,"q1_score":0.028325745835900307,"q3_score":1,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":0.8403725000000005,"total_cache_read_cost_bin":1.2748799999999978,"total_completion_cost_bin":1.0171799999999998,"total_reasoning_cost_bin":0.5099550000000002,"total_cost_bin":3.642387499999998,"total_on_demand_input_cost_bin":13.589172500000002,"ci_99_lower":0.4615139606980169,"ci_99_upper":0.7146648576956461,"ci_98_lower":0.472761304754216,"ci_98_upper":0.7137448333999252,"ci_95_lower":0.4966092537472243,"ci_95_upper":0.7023580822000718,"ci_90_lower":0.5182041016149829,"ci_90_upper":0.6902217745046844,"ci_75_lower":0.5414130761164844,"ci_75_upper":0.6571300999494268,"ci_50_lower":0.5645316791282419,"ci_50_upper":0.6334660780068269},"131072":{"avg_score":0.5456444497324201,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4883397651720651,"median_score":1,"q1_score":0.025222646072506905,"q3_score":1,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":0.9976099999999988,"total_cache_read_cost_bin":3.1935999999999973,"total_completion_cost_bin":1.278855,"total_reasoning_cost_bin":0.6493200000000002,"total_cost_bin":6.119384999999997,"total_on_demand_input_cost_bin":32.93360999999999,"ci_99_lower":0.42330281550219523,"ci_99_upper":0.6600218673019547,"ci_98_lower":0.43191881177282343,"ci_98_upper":0.6585306773369101,"ci_95_lower":0.4527144299928004,"ci_95_upper":0.6391793365636614,"ci_90_lower":0.46946778372031844,"ci_90_upper":0.6232238834983784,"ci_75_lower":0.4893074496844173,"ci_75_upper":0.6029705499268919,"ci_50_lower":0.5153165313113454,"ci_50_upper":0.5823223128224253},"262144":{"avg_score":0.43367029823270353,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4747472252748649,"median_score":0.055525895208120346,"q1_score":0.030070163309574127,"q3_score":1,"avg_token_efficiency":0.982419810338046,"is_incomplete":false,"total_prompt_cost_bin":7.148707499999997,"total_cache_read_cost_bin":8.355552000000014,"total_completion_cost_bin":2.050349999999999,"total_reasoning_cost_bin":1.2486600000000003,"total_cost_bin":18.80326950000001,"total_on_demand_input_cost_bin":90.70422749999996,"ci_99_lower":0.31773835366325953,"ci_99_upper":0.5405632950022572,"ci_98_lower":0.33965588480045905,"ci_98_upper":0.5352893019299491,"ci_95_lower":0.3530157976791243,"ci_95_upper":0.5155592254808021,"ci_90_lower":0.3699926129136076,"ci_90_upper":0.5005409018539474,"ci_75_lower":0.38669956345416623,"ci_75_upper":0.48153288453381426,"ci_50_lower":0.4060591104925868,"ci_50_upper":0.46033132712254365},"524288":{"avg_score":0.38406577907153405,"min_score":0.0013020833721384406,"max_score":1,"n_tests":236,"stddev":0.4611098659017385,"median_score":0.04635872319340706,"q1_score":0.027408099733293056,"q3_score":1,"avg_token_efficiency":0.9774785269226759,"is_incomplete":false,"total_prompt_cost_bin":38.56792500000007,"total_cache_read_cost_bin":56.58566400000012,"total_completion_cost_bin":4.9580250000000055,"total_reasoning_cost_bin":3.6601800000000013,"total_cost_bin":103.7717940000002,"total_on_demand_input_cost_bin":604.5665150000009,"ci_99_lower":0.30369052824340576,"ci_99_upper":0.4553357063660741,"ci_98_lower":0.3103054600957706,"ci_98_upper":0.45358464518131325,"ci_95_lower":0.3239052233390532,"ci_95_upper":0.4437341884146563,"ci_90_lower":0.3330873314467095,"ci_90_upper":0.4346803218550029,"ci_75_lower":0.34870496831434267,"ci_75_upper":0.4188097778247014,"ci_50_lower":0.3641491248683019,"ci_50_upper":0.4037209995683151}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.0000025,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-05T18:12:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T02:25:56.147Z"},{"model_slug":"google/gemini-3.5-flash-lite","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.3378812601957188,"auc_128k":0.5680512318787176,"cum_avg_1m":0.5312271832582738,"cum_avg_128k":0.6677991934468981,"avg_score_overall":0.4356741679354748,"avg_response_time_ms":11209.678363386329,"total_prompt_cost":5.846579100000002,"total_cache_read_cost":14.720641259999919,"total_completion_cost":1.1078774999999998,"total_reasoning_cost":9.651140000000002,"total_prompt_cost_128k":1.9645239,"total_completion_cost_128k":0.46592999999999996,"total_on_demand_input_cost":153.05299170000006,"total_on_demand_input_cost_128k":7.364040000000003,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154894},"bin_metrics":{"8192":{"avg_score":0.9631410761270672,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.1871614616083451,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747239,"is_incomplete":false,"total_prompt_cost_bin":0.15946320000000003,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06978000000000001,"total_reasoning_cost_bin":0.28587,"total_cost_bin":0.5151132,"total_on_demand_input_cost_bin":0.15946320000000003,"ci_99_lower":0.9017800599950133,"ci_99_upper":0.99999660840258,"ci_98_lower":0.9019361610757188,"ci_98_upper":0.9999965913593769,"ci_95_lower":0.9141924022929743,"ci_95_upper":0.9998566552996635,"ci_90_lower":0.9263009621214587,"ci_90_upper":0.9995870181918144,"ci_75_lower":0.9386353158392012,"ci_75_upper":0.987630803999491,"ci_50_lower":0.9508574261155446,"ci_50_upper":0.975463711260818},"16384":{"avg_score":0.734039966450655,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.43464812423147586,"median_score":1,"q1_score":0.06086102873086929,"q3_score":1,"avg_token_efficiency":1.000074513502447,"is_incomplete":false,"total_prompt_cost_bin":0.3305616,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06929749999999998,"total_reasoning_cost_bin":0.35669999999999996,"total_cost_bin":0.7565591,"total_on_demand_input_cost_bin":0.3305616,"ci_99_lower":0.6023471278093638,"ci_99_upper":0.8425741673512239,"ci_98_lower":0.6128459381069697,"ci_98_upper":0.8317310667380224,"ci_95_lower":0.6365383124304935,"ci_95_upper":0.8291853499125136,"ci_90_lower":0.6501065598736749,"ci_90_upper":0.807855835823575,"ci_75_lower":0.6748639328121498,"ci_75_upper":0.7834688228933374,"ci_50_lower":0.6982950963967596,"ci_50_upper":0.769004075635894},"32768":{"avg_score":0.6455733018057586,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4667951857651454,"median_score":1,"q1_score":0.04116049315780401,"q3_score":1,"avg_token_efficiency":1.0000359090949906,"is_incomplete":false,"total_prompt_cost_bin":0.5090745000000003,"total_cache_read_cost_bin":0.06529503000000005,"total_completion_cost_bin":0.13165,"total_reasoning_cost_bin":0.6796200000000004,"total_cost_bin":1.3856395300000006,"total_on_demand_input_cost_bin":1.1620247999999984,"ci_99_lower":0.5396892256954144,"ci_99_upper":0.7442700394497576,"ci_98_lower":0.5530986050539992,"ci_98_upper":0.737386026194641,"ci_95_lower":0.5659086659260485,"ci_95_upper":0.723466377585753,"ci_90_lower":0.5800669287254363,"ci_90_upper":0.7121153479501919,"ci_75_lower":0.5972887132434955,"ci_75_upper":0.6894446507285465,"ci_50_lower":0.6181178437109338,"ci_50_upper":0.6686697396465152},"65536":{"avg_score":0.5082257968099678,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4889590630512672,"median_score":0.10603290796279907,"q1_score":0.02598169445991516,"q3_score":1,"avg_token_efficiency":1.0000152878522819,"is_incomplete":false,"total_prompt_cost_bin":0.43001609999999996,"total_cache_read_cost_bin":0.12380004000000021,"total_completion_cost_bin":0.08544500000000002,"total_reasoning_cost_bin":0.6605824999999999,"total_cost_bin":1.29984364,"total_on_demand_input_cost_bin":1.6680164999999993,"ci_99_lower":0.36270364282587,"ci_99_upper":0.6362191586592508,"ci_98_lower":0.38030721110423266,"ci_98_upper":0.6341013004158569,"ci_95_lower":0.4037017488759011,"ci_95_upper":0.6208521667889813,"ci_90_lower":0.4173730286895571,"ci_90_upper":0.600026092857999,"ci_75_lower":0.45007104688214467,"ci_75_upper":0.5679287896477891,"ci_50_lower":0.47380637358600164,"ci_50_upper":0.5423545288853346},"131072":{"avg_score":0.48801582604104166,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4851357903774127,"median_score":0.05499781668186188,"q1_score":0.028524868190288544,"q3_score":1,"avg_token_efficiency":1.0000076410572558,"is_incomplete":false,"total_prompt_cost_bin":0.5354084999999996,"total_cache_read_cost_bin":0.35085654000000044,"total_completion_cost_bin":0.10975749999999995,"total_reasoning_cost_bin":0.7376675000000004,"total_cost_bin":1.7336900400000004,"total_on_demand_input_cost_bin":4.043973900000005,"ci_99_lower":0.36402273253942347,"ci_99_upper":0.6094066410632443,"ci_98_lower":0.37591416812999157,"ci_98_upper":0.5973979058993745,"ci_95_lower":0.3963802284312444,"ci_95_upper":0.5760488809210709,"ci_90_lower":0.4109267645913348,"ci_90_upper":0.5680684236719191,"ci_75_lower":0.43250469826116655,"ci_75_upper":0.5457497935886811,"ci_50_lower":0.4562720547959172,"ci_50_upper":0.5216774414101326},"262144":{"avg_score":0.40301556137377226,"min_score":0.0019753086380660534,"max_score":1,"n_tests":141,"stddev":0.4716811316630924,"median_score":0.044936709105968475,"q1_score":0.024060150608420372,"q3_score":1,"avg_token_efficiency":1.000003817936524,"is_incomplete":false,"total_prompt_cost_bin":0.7587075000000006,"total_cache_read_cost_bin":1.0320617999999968,"total_completion_cost_bin":0.13495500000000005,"total_reasoning_cost_bin":1.2150925000000004,"total_cost_bin":3.140816799999998,"total_on_demand_input_cost_bin":11.079325499999985,"ci_99_lower":0.2981270459668169,"ci_99_upper":0.5066145859602822,"ci_98_lower":0.30707234789222704,"ci_98_upper":0.5006117931064482,"ci_95_lower":0.32473713279782684,"ci_95_upper":0.4861505420821973,"ci_90_lower":0.33489692582540764,"ci_90_upper":0.4743054015814583,"ci_75_lower":0.35599359780003426,"ci_75_upper":0.4500115779624816,"ci_50_lower":0.37615186405774365,"ci_50_upper":0.4305843668919658},"524288":{"avg_score":0.2852413429201865,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.42109812385852735,"median_score":0.036610398441553116,"q1_score":0.023739496245980263,"q3_score":0.9137639403343201,"avg_token_efficiency":1.0000019078540323,"is_incomplete":false,"total_prompt_cost_bin":1.3047816000000012,"total_cache_read_cost_bin":3.5805046499999964,"total_completion_cost_bin":0.2104550000000001,"total_reasoning_cost_bin":2.272835,"total_cost_bin":7.3685762499999985,"total_on_demand_input_cost_bin":37.109828099999994,"ci_99_lower":0.2154368686432293,"ci_99_upper":0.35478493000012706,"ci_98_lower":0.22017814544832603,"ci_98_upper":0.34518230127989535,"ci_95_lower":0.2317748863295646,"ci_95_upper":0.3372770246676952,"ci_90_lower":0.24157018060519725,"ci_90_upper":0.32885308261626295,"ci_75_lower":0.25435603797444145,"ci_75_upper":0.3168762433062696,"ci_50_lower":0.26869052135280996,"ci_50_upper":0.3033152403750445},"1048576":{"avg_score":0.22256459453774075,"min_score":0.002482416108250618,"max_score":1,"n_tests":310,"stddev":0.3810991806144748,"median_score":0.034076469019055367,"q1_score":0.01998480223119259,"q3_score":0.0804575253278017,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":1.8185661000000002,"total_cache_read_cost_bin":9.568123199999924,"total_completion_cost_bin":0.2965374999999999,"total_reasoning_cost_bin":3.4427725,"total_cost_bin":15.125999299999924,"total_on_demand_input_cost_bin":97.49979810000008,"ci_99_lower":0.17070909857809274,"ci_99_upper":0.2768452018353367,"ci_98_lower":0.17652701639202273,"ci_98_upper":0.27424952903152594,"ci_95_lower":0.18178646906305315,"ci_95_upper":0.2673255557922886,"ci_90_lower":0.18684869340583382,"ci_90_upper":0.26026521761773996,"ci_75_lower":0.19719963261722437,"ci_75_upper":0.24688612346541167,"ci_50_lower":0.20677600670568347,"ci_50_upper":0.2373944777094068}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"minimal","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash Lite","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":3e-7,"completion":0.0000025,"reasoning":0.0000025,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:06.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"anthropic/claude-sonnet-5","reasoning_mode":"max","overall_metrics":{"auc_1m":0.33758082865081646,"auc_128k":0.7160874645101685,"cum_avg_1m":0.6054984587763718,"cum_avg_128k":0.8003364117199719,"avg_score_overall":0.6203257904334467,"avg_response_time_ms":102496.89285459087,"total_prompt_cost":8.672003999999998,"total_cache_read_cost":59.75944700000004,"total_completion_cost":102.23700000000002,"total_reasoning_cost":10.480419999999999,"total_prompt_cost_128k":4.340646,"total_completion_cost_128k":42.952389999999994,"total_on_demand_input_cost":606.2664739999994,"total_on_demand_input_cost_128k":80.06380000000004,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6376061287420098},"bin_metrics":{"8192":{"avg_score":0.9642024754080921,"min_score":0.03722627833485603,"max_score":1,"n_tests":80,"stddev":0.18130931402903835,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.6722716647051485,"is_incomplete":false,"total_prompt_cost_bin":0.16630799999999987,"total_cache_read_cost_bin":0.16101319999999988,"total_completion_cost_bin":1.8012700000000001,"total_reasoning_cost_bin":0.19394999999999996,"total_cost_bin":2.3225412,"total_on_demand_input_cost_bin":1.776439999999999,"ci_99_lower":0.9045499330393504,"ci_99_upper":0.9999770622812212,"ci_98_lower":0.9048877542708068,"ci_98_upper":0.9999702652916312,"ci_95_lower":0.9167805928364396,"ci_95_upper":0.9997741617262363,"ci_90_lower":0.9284582003206016,"ci_90_upper":0.9994031882286071,"ci_75_lower":0.9404312279948499,"ci_75_upper":0.9879278185311705,"ci_50_lower":0.9522793011623435,"ci_50_upper":0.9762198427226394},"16384":{"avg_score":0.9490601768367923,"min_score":0.013550135307013988,"max_score":1,"n_tests":80,"stddev":0.21381890586260588,"median_score":1,"q1_score":0.9990460127592087,"q3_score":1,"avg_token_efficiency":1.583013776115054,"is_incomplete":false,"total_prompt_cost_bin":0.13219599999999998,"total_cache_read_cost_bin":0.3352387999999997,"total_completion_cost_bin":4.217729999999998,"total_reasoning_cost_bin":0.45339000000000007,"total_cost_bin":5.138554799999997,"total_on_demand_input_cost_bin":3.4845839999999986,"ci_99_lower":0.8764918784367037,"ci_99_upper":0.9981149387881161,"ci_98_lower":0.8858262621870963,"ci_98_upper":0.9978447476550937,"ci_95_lower":0.8889794415759388,"ci_95_upper":0.9866945425383165,"ci_90_lower":0.9007227595860604,"ci_90_upper":0.985546964716632,"ci_75_lower":0.9239008616103093,"ci_75_upper":0.9741274494153913,"ci_50_lower":0.9362145056627924,"ci_50_upper":0.9619782183144707},"32768":{"avg_score":0.8788704754791551,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.3168307808478457,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.660794921666311,"is_incomplete":false,"total_prompt_cost_bin":0.20302200000000004,"total_cache_read_cost_bin":1.2704885999999993,"total_completion_cost_bin":11.906070000000001,"total_reasoning_cost_bin":1.1782800000000002,"total_cost_bin":14.557860600000001,"total_on_demand_input_cost_bin":12.907908000000015,"ci_99_lower":0.8080207729914829,"ci_99_upper":0.941758840319971,"ci_98_lower":0.8127797802421949,"ci_98_upper":0.9364410385731196,"ci_95_lower":0.8227913659802973,"ci_95_upper":0.9291863486491766,"ci_90_lower":0.8322087276129477,"ci_90_upper":0.9217399372534333,"ci_75_lower":0.8452518624989727,"ci_75_upper":0.9119590862088337,"ci_50_lower":0.8584604892025441,"ci_50_upper":0.8985790388696306},"65536":{"avg_score":0.6804066573762718,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.4576954212984886,"median_score":1,"q1_score":0.039603959769010544,"q3_score":1,"avg_token_efficiency":1.6412164101683422,"is_incomplete":false,"total_prompt_cost_bin":0.43876000000000004,"total_cache_read_cost_bin":1.7811468000000015,"total_completion_cost_bin":11.932369999999999,"total_reasoning_cost_bin":1.2022000000000002,"total_cost_bin":15.354476799999999,"total_on_demand_input_cost_bin":18.250228000000007,"ci_99_lower":0.5444138969259117,"ci_99_upper":0.7844790460800424,"ci_98_lower":0.5645219805587959,"ci_98_upper":0.7829769752306096,"ci_95_lower":0.5880171035671168,"ci_95_upper":0.771757515313049,"ci_90_lower":0.6001024634999169,"ci_90_upper":0.7601102066281087,"ci_75_lower":0.62315171104308,"ci_75_upper":0.7376117486753226,"ci_50_lower":0.6460682831266347,"ci_50_upper":0.7141603532438988},"131072":{"avg_score":0.529142273499549,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4868854218445591,"median_score":0.9739508032798767,"q1_score":0.02430454082787037,"q3_score":1,"avg_token_efficiency":1.6188892872031537,"is_incomplete":false,"total_prompt_cost_bin":3.40036,"total_cache_read_cost_bin":4.024428000000001,"total_completion_cost_bin":13.09495,"total_reasoning_cost_bin":1.4489799999999997,"total_cost_bin":21.968718000000003,"total_on_demand_input_cost_bin":43.64464000000002,"ci_99_lower":0.40000314788519886,"ci_99_upper":0.6508997717722618,"ci_98_lower":0.4189725254556996,"ci_98_upper":0.6395362224698392,"ci_95_lower":0.4393704526572832,"ci_95_upper":0.6195585621152416,"ci_90_lower":0.45269497932811964,"ci_90_upper":0.609638003359811,"ci_75_lower":0.4741098806103782,"ci_75_upper":0.586543797838583,"ci_50_lower":0.4961250379536916,"ci_50_upper":0.5631371262942179},"262144":{"avg_score":0.5221814280970299,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4828363932187716,"median_score":0.9567966461181641,"q1_score":0.028870871290564537,"q3_score":1,"avg_token_efficiency":1.6370585869996057,"is_incomplete":false,"total_prompt_cost_bin":2.5878620000000008,"total_cache_read_cost_bin":11.832837599999985,"total_completion_cost_bin":23.782030000000006,"total_reasoning_cost_bin":2.423489999999998,"total_cost_bin":40.62621959999999,"total_on_demand_input_cost_bin":120.9162380000001,"ci_99_lower":0.4118274671021281,"ci_99_upper":0.6225643718542188,"ci_98_lower":0.41970316701462984,"ci_98_upper":0.6178977814695223,"ci_95_lower":0.4378341434181648,"ci_95_upper":0.6024579625315831,"ci_90_lower":0.4592953044412565,"ci_90_upper":0.5910631513999155,"ci_75_lower":0.4772019390403194,"ci_75_upper":0.5716330893395521,"ci_50_lower":0.49570402453475176,"ci_50_upper":0.5496203041245752},"524288":{"avg_score":0.3201241835140844,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.434829640564579,"median_score":0.04087517224252224,"q1_score":0.02330105658620596,"q3_score":0.9635952562093735,"avg_token_efficiency":1.6381935594129555,"is_incomplete":false,"total_prompt_cost_bin":1.7434959999999977,"total_cache_read_cost_bin":40.35429400000005,"total_completion_cost_bin":35.502580000000016,"total_reasoning_cost_bin":3.5801299999999996,"total_cost_bin":81.18050000000007,"total_on_demand_input_cost_bin":405.2864359999993,"ci_99_lower":0.25217086995168986,"ci_99_upper":0.38760096889142126,"ci_98_lower":0.2573535140570632,"ci_98_upper":0.38406253549420294,"ci_95_lower":0.26391910725030004,"ci_95_upper":0.3750827817514171,"ci_90_lower":0.2742337757468981,"ci_90_upper":0.3684155015721799,"ci_75_lower":0.28904549566468013,"ci_75_upper":0.3530856035462328,"ci_50_lower":0.30342376172311714,"ci_50_upper":0.3394657544334423}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Anthropic: Claude Sonnet 5","provider_name":"Anthropic","run_provider_names":["Claude Platform on AWS"],"pricing_metadata":{"prompt":0.000002,"completion":0.00001,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-06-30T18:11:23.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-09T20:34:23.060Z"},{"model_slug":"google/gemini-3-flash-preview","reasoning_mode":"low","overall_metrics":{"auc_1m":0.33729727341387156,"auc_128k":0.40405132407731786,"cum_avg_1m":0.4581834264984271,"cum_avg_128k":0.5324842415331323,"avg_score_overall":0.3958157448242812,"avg_response_time_ms":29404.166742000692,"total_prompt_cost":6.491102999999999,"total_cache_read_cost":24.860017299999917,"total_completion_cost":1.360338,"total_reasoning_cost":14.701676999999991,"total_prompt_cost_128k":2.0607275,"total_completion_cost_128k":0.55938,"total_on_demand_input_cost":255.09127599999965,"total_on_demand_input_cost_128k":12.276356500000006,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000996733437817},"bin_metrics":{"8192":{"avg_score":0.8445173800340854,"min_score":0.012427506037056446,"max_score":1,"n_tests":80,"stddev":0.35530019010848135,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000155340474723,"is_incomplete":false,"total_prompt_cost_bin":0.18966450000000004,"total_cache_read_cost_bin":0.00761075,"total_completion_cost_bin":0.08384999999999997,"total_reasoning_cost_bin":0.193752,"total_cost_bin":0.47487725000000003,"total_on_demand_input_cost_bin":0.265772,"ci_99_lower":0.73421969314554,"ci_99_upper":0.9410907984083169,"ci_98_lower":0.7365608086496359,"ci_98_upper":0.9288360723191872,"ci_95_lower":0.7606745414831676,"ci_95_upper":0.9166158347323653,"ci_90_lower":0.7742688299168367,"ci_90_upper":0.9045663506089476,"ci_75_lower":0.7967032890373957,"ci_75_upper":0.8914653323197854,"ci_50_lower":0.8195473472063896,"ci_50_upper":0.8685458000429208},"16384":{"avg_score":0.7566271857009269,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.4242051501332105,"median_score":1,"q1_score":0.761372271925211,"q3_score":1,"avg_token_efficiency":1.0000962307665302,"is_incomplete":false,"total_prompt_cost_bin":0.28124800000000016,"total_cache_read_cost_bin":0.026969749999999973,"total_completion_cost_bin":0.08290799999999998,"total_reasoning_cost_bin":0.229482,"total_cost_bin":0.6206077500000001,"total_on_demand_input_cost_bin":0.5509455000000004,"ci_99_lower":0.6347305358677986,"ci_99_upper":0.8660793833884527,"ci_98_lower":0.6450423980440245,"ci_98_upper":0.8544471571766771,"ci_95_lower":0.6602550906006946,"ci_95_upper":0.8422330640105065,"ci_90_lower":0.6824873500829562,"ci_90_upper":0.8297728321899194,"ci_75_lower":0.7071887027894262,"ci_75_upper":0.8058312396198744,"ci_50_lower":0.7207762165751773,"ci_50_upper":0.7822602344444022},"32768":{"avg_score":0.43436867165249116,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4753523096913851,"median_score":0.059214070439338684,"q1_score":0.02719556773081422,"q3_score":1,"avg_token_efficiency":1.0004592503017251,"is_incomplete":false,"total_prompt_cost_bin":0.5568209999999999,"total_cache_read_cost_bin":0.13808209999999982,"total_completion_cost_bin":0.15897600000000006,"total_reasoning_cost_bin":0.28322699999999995,"total_cost_bin":1.1371060999999996,"total_on_demand_input_cost_bin":1.9376419999999999,"ci_99_lower":0.3222969378953156,"ci_99_upper":0.5337429847580685,"ci_98_lower":0.33719248159254406,"ci_98_upper":0.5309176693802274,"ci_95_lower":0.355594457649054,"ci_95_upper":0.5124247823686079,"ci_90_lower":0.3671444780908881,"ci_90_upper":0.5045078840643434,"ci_75_lower":0.3871899666211153,"ci_75_upper":0.48381339066113205,"ci_50_lower":0.4055416961662773,"ci_50_upper":0.46330390664404736},"65536":{"avg_score":0.34641624826061373,"min_score":0.004494382068514824,"max_score":1,"n_tests":85,"stddev":0.4539362903267949,"median_score":0.03732394427061081,"q1_score":0.0260743610560894,"q3_score":1,"avg_token_efficiency":1.0000152878522817,"is_incomplete":false,"total_prompt_cost_bin":0.3727975000000001,"total_cache_read_cost_bin":0.240723,"total_completion_cost_bin":0.09966600000000003,"total_reasoning_cost_bin":0.09774900000000002,"total_cost_bin":0.8109355000000001,"total_on_demand_input_cost_bin":2.7800275000000005,"ci_99_lower":0.23672984683100026,"ci_99_upper":0.476510478278896,"ci_98_lower":0.2427683566797196,"ci_98_upper":0.46416588739618003,"ci_95_lower":0.2529781553539557,"ci_95_upper":0.4483825040475794,"ci_90_lower":0.26550703175968543,"ci_90_upper":0.4310905280709267,"ci_75_lower":0.288387337342069,"ci_75_upper":0.4021322180934743,"ci_50_lower":0.3123506149071652,"ci_50_upper":0.3776248335537008},"131072":{"avg_score":0.2804917220175447,"min_score":0.003092384897172451,"max_score":1,"n_tests":103,"stddev":0.4301114476919643,"median_score":0.032562125474214554,"q1_score":0.018815181218087673,"q3_score":0.9799483418464661,"avg_token_efficiency":1.0003063106988845,"is_incomplete":false,"total_prompt_cost_bin":0.6601964999999999,"total_cache_read_cost_bin":0.608177300000001,"total_completion_cost_bin":0.13398,"total_reasoning_cost_bin":0.6353340000000001,"total_cost_bin":2.037687800000001,"total_on_demand_input_cost_bin":6.741969500000004,"ci_99_lower":0.17600653598835528,"ci_99_upper":0.3842185635269321,"ci_98_lower":0.18528079042532164,"ci_98_upper":0.3830899132724661,"ci_95_lower":0.20451344486288683,"ci_95_upper":0.36438070067438494,"ci_90_lower":0.21401918139647763,"ci_90_upper":0.3473701953280997,"ci_75_lower":0.23329893222504786,"ci_75_upper":0.3275999681737604,"ci_50_lower":0.25208482268838284,"ci_50_upper":0.3086989536125312},"262144":{"avg_score":0.3965336204595626,"min_score":0.002959674457088113,"max_score":1,"n_tests":141,"stddev":0.47019855200018285,"median_score":0.044936709105968475,"q1_score":0.027808988466858864,"q3_score":1,"avg_token_efficiency":1.000003817936525,"is_incomplete":false,"total_prompt_cost_bin":1.452902499999999,"total_cache_read_cost_bin":1.7012640000000034,"total_completion_cost_bin":0.162264,"total_reasoning_cost_bin":1.3088729999999997,"total_cost_bin":4.625303500000002,"total_on_demand_input_cost_bin":18.46554249999999,"ci_99_lower":0.2919510248254618,"ci_99_upper":0.5005066907495682,"ci_98_lower":0.3003472317607598,"ci_98_upper":0.49840778979448047,"ci_95_lower":0.32045813699559217,"ci_95_upper":0.47906466191977176,"ci_90_lower":0.3285482630131978,"ci_90_upper":0.46530813363781987,"ci_75_lower":0.34901411393335324,"ci_75_upper":0.4409858323012376,"ci_50_lower":0.3703549697063037,"ci_50_upper":0.4241354191074716},"524288":{"avg_score":0.3504275319776182,"min_score":0.0008779631461948156,"max_score":1,"n_tests":236,"stddev":0.4498055495535535,"median_score":0.03985871933400631,"q1_score":0.024427499156445265,"q3_score":1,"avg_token_efficiency":1.0000019078540323,"is_incomplete":false,"total_prompt_cost_bin":1.2121934999999988,"total_cache_read_cost_bin":6.06375200000002,"total_completion_cost_bin":0.26021700000000014,"total_reasoning_cost_bin":3.611000999999997,"total_cost_bin":11.147163500000016,"total_on_demand_input_cost_bin":61.84971350000005,"ci_99_lower":0.279957730018073,"ci_99_upper":0.42384163100748934,"ci_98_lower":0.28476780306804383,"ci_98_upper":0.4172588103512791,"ci_95_lower":0.29064964795017995,"ci_95_upper":0.40762403002741326,"ci_90_lower":0.3021120421202672,"ci_90_upper":0.40157754587347944,"ci_75_lower":0.3160568594836859,"ci_75_upper":0.38399811100976355,"ci_50_lower":0.3309667473229761,"ci_50_upper":0.37034822896619174},"1048576":{"avg_score":0.25608505188457426,"min_score":0.005247813183814287,"max_score":1,"n_tests":310,"stddev":0.4047299758139245,"median_score":0.03528047353029251,"q1_score":0.02332566911354661,"q3_score":0.11360211670398712,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":1.7652795000000014,"total_cache_read_cost_bin":16.073438399999894,"total_completion_cost_bin":0.37847699999999995,"total_reasoning_cost_bin":8.342258999999995,"total_cost_bin":26.55945389999989,"total_on_demand_input_cost_bin":162.4996634999996,"ci_99_lower":0.19892743452708236,"ci_99_upper":0.3165573761993794,"ci_98_lower":0.2041999237557813,"ci_98_upper":0.30977736905927683,"ci_95_lower":0.20936477081737512,"ci_95_upper":0.30400738269575084,"ci_90_lower":0.21897161954751,"ci_90_upper":0.29605521337564794,"ci_75_lower":0.2292333365125101,"ci_75_upper":0.2830661181567027,"ci_50_lower":0.24010265597758154,"ci_50_upper":0.2712045161598062}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3 Flash Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":5e-7,"completion":0.000003,"reasoning":0.000003,"has_tiered_pricing":true},"api_created_at":"2025-12-17T15:57:58.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T06:59:54.256Z"},{"model_slug":"google/gemini-3.5-flash-lite","reasoning_mode":"high","overall_metrics":{"auc_1m":0.333392282499969,"auc_128k":0.643711160100752,"cum_avg_1m":0.5631163383533597,"cum_avg_128k":0.7324459646367106,"avg_score_overall":0.4517600475965556,"avg_response_time_ms":15200.690395703792,"total_prompt_cost":5.841670500000002,"total_cache_read_cost":14.721132119999918,"total_completion_cost":1.1233775000000001,"total_reasoning_cost":15.419267500000005,"total_prompt_cost_128k":1.9596153000000012,"total_completion_cost_128k":0.4658050000000001,"total_on_demand_input_cost":153.0529917000001,"total_on_demand_input_cost_128k":7.364040000000003,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154892},"bin_metrics":{"8192":{"avg_score":0.9626836560666561,"min_score":0.02666666731238365,"max_score":1,"n_tests":80,"stddev":0.18313041510623856,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747245,"is_incomplete":false,"total_prompt_cost_bin":0.15946320000000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.07001,"total_reasoning_cost_bin":0.3977275,"total_cost_bin":0.6272007,"total_on_demand_input_cost_bin":0.15946320000000005,"ci_99_lower":0.9019116328696255,"ci_99_upper":1,"ci_98_lower":0.903172507208772,"ci_98_upper":1,"ci_95_lower":0.9147022181970533,"ci_95_upper":0.998569418489933,"ci_90_lower":0.9257967769354581,"ci_90_upper":0.9957797845453022,"ci_75_lower":0.9386154846288264,"ci_75_upper":0.9866679137106985,"ci_50_lower":0.9505799247534015,"ci_50_upper":0.9756666666828095},"16384":{"avg_score":0.8066518233041279,"min_score":0.011840158142149448,"max_score":1,"n_tests":80,"stddev":0.3891913490754102,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0000745135024474,"is_incomplete":false,"total_prompt_cost_bin":0.33056160000000007,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06960250000000001,"total_reasoning_cost_bin":0.4679225000000002,"total_cost_bin":0.8680866000000003,"total_on_demand_input_cost_bin":0.33056160000000007,"ci_99_lower":0.6852585860271355,"ci_99_upper":0.9159455507516395,"ci_98_lower":0.6967489375369623,"ci_98_upper":0.9037642999445089,"ci_95_lower":0.7103545541563654,"ci_95_upper":0.8910381136005162,"ci_90_lower":0.7335459040448769,"ci_90_upper":0.8787099924322683,"ci_75_lower":0.7575234049989377,"ci_75_upper":0.8547496132450759,"ci_50_lower":0.7813667464040919,"ci_50_upper":0.8318883303145413},"32768":{"avg_score":0.7611220300211298,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4161199292648659,"median_score":1,"q1_score":0.8002918995916843,"q3_score":1,"avg_token_efficiency":1.0000359090949908,"is_incomplete":false,"total_prompt_cost_bin":0.5041659000000007,"total_cache_read_cost_bin":0.0657858900000001,"total_completion_cost_bin":0.13239500000000007,"total_reasoning_cost_bin":0.8763525,"total_cost_bin":1.5786992900000008,"total_on_demand_input_cost_bin":1.1620247999999995,"ci_99_lower":0.6706028936905033,"ci_99_upper":0.8525855241936412,"ci_98_lower":0.6800993768213769,"ci_98_upper":0.8454975255399683,"ci_95_lower":0.6903058977648684,"ci_95_upper":0.8353721830797867,"ci_90_lower":0.6986611795089523,"ci_90_upper":0.8159071085721199,"ci_75_lower":0.717923844421902,"ci_75_upper":0.8014670046031824,"ci_50_lower":0.7365482617666096,"ci_50_upper":0.7861428686489542},"65536":{"avg_score":0.5769462466459064,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.4846862979148287,"median_score":1,"q1_score":0.02740362286567688,"q3_score":1,"avg_token_efficiency":1.0000152878522812,"is_incomplete":false,"total_prompt_cost_bin":0.43001609999999996,"total_cache_read_cost_bin":0.12380004000000021,"total_completion_cost_bin":0.08526500000000004,"total_reasoning_cost_bin":0.7735700000000002,"total_cost_bin":1.4126511400000004,"total_on_demand_input_cost_bin":1.668016499999999,"ci_99_lower":0.4380352724083864,"ci_99_upper":0.7028193549593253,"ci_98_lower":0.45051454844753097,"ci_98_upper":0.7014996366935837,"ci_95_lower":0.4742718868894393,"ci_95_upper":0.6799756001798875,"ci_90_lower":0.4957670417400625,"ci_90_upper":0.6682261237923932,"ci_75_lower":0.5191151428639013,"ci_75_upper":0.643996233965599,"ci_50_lower":0.5426451051224241,"ci_50_upper":0.6113781105453039},"131072":{"avg_score":0.554826067145733,"min_score":0.0047505940310657024,"max_score":1,"n_tests":103,"stddev":0.4865010341854257,"median_score":0.9982832670211792,"q1_score":0.02593260630965233,"q3_score":1,"avg_token_efficiency":1.0000076410572558,"is_incomplete":false,"total_prompt_cost_bin":0.5354085000000003,"total_cache_read_cost_bin":0.35085654000000044,"total_completion_cost_bin":0.10853249999999999,"total_reasoning_cost_bin":1.3456875000000004,"total_cost_bin":2.3404850400000012,"total_on_demand_input_cost_bin":4.043973900000005,"ci_99_lower":0.43254331505018767,"ci_99_upper":0.6694964028725139,"ci_98_lower":0.4392378599014477,"ci_98_upper":0.6598480795872697,"ci_95_lower":0.4694335235089506,"ci_95_upper":0.6495256641814123,"ci_90_lower":0.4794391649619233,"ci_90_upper":0.6377238408786706,"ci_75_lower":0.4987433786267749,"ci_75_upper":0.6121163342393329,"ci_50_lower":0.5240638155096835,"ci_50_upper":0.5916502770630968},"262144":{"avg_score":0.376469289172269,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.46358507607518035,"median_score":0.045004501938819885,"q1_score":0.028744326904416084,"q3_score":1,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":0.7587075000000002,"total_cache_read_cost_bin":1.0320617999999968,"total_completion_cost_bin":0.13483250000000005,"total_reasoning_cost_bin":1.9443024999999998,"total_cost_bin":3.869904299999997,"total_on_demand_input_cost_bin":11.079325499999985,"ci_99_lower":0.285877476656679,"ci_99_upper":0.4801054259017687,"ci_98_lower":0.28834208159709135,"ci_98_upper":0.4685890895571123,"ci_95_lower":0.30034564353074833,"ci_95_upper":0.45854508640159564,"ci_90_lower":0.3084540008458289,"ci_90_upper":0.4435352379101137,"ci_75_lower":0.33132963634780543,"ci_75_upper":0.42283360670252157,"ci_50_lower":0.3496618470101096,"ci_50_upper":0.40376533190876657},"524288":{"avg_score":0.26824187347566775,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.40893253876701036,"median_score":0.03632296994328499,"q1_score":0.020409791264683008,"q3_score":0.19498023390769958,"avg_token_efficiency":1.0000019078540325,"is_incomplete":false,"total_prompt_cost_bin":1.3047816000000008,"total_cache_read_cost_bin":3.5805046499999964,"total_completion_cost_bin":0.2119525,"total_reasoning_cost_bin":3.9955800000000035,"total_cost_bin":9.092818750000001,"total_on_demand_input_cost_bin":37.1098281,"ci_99_lower":0.19962184356239926,"ci_99_upper":0.3308299745964991,"ci_98_lower":0.2093170130882961,"ci_98_upper":0.32690695440769585,"ci_95_lower":0.2176872406761612,"ci_95_upper":0.31937629260340805,"ci_90_lower":0.22597552850261388,"ci_90_upper":0.3125441667854675,"ci_75_lower":0.23772124826816363,"ci_75_upper":0.29981379695293414,"ci_50_lower":0.25162294017416387,"ci_50_upper":0.2857219744190057},"1048576":{"avg_score":0.19798972099538772,"min_score":0.0018489984795451164,"max_score":1,"n_tests":310,"stddev":0.360107083506259,"median_score":0.03268558345735073,"q1_score":0.020980249159038067,"q3_score":0.06364927254617214,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":1.8185661,"total_cache_read_cost_bin":9.568123199999924,"total_completion_cost_bin":0.3107875000000001,"total_reasoning_cost_bin":5.618125000000001,"total_cost_bin":17.315601799999925,"total_on_demand_input_cost_bin":97.4997981000001,"ci_99_lower":0.15175419943121773,"ci_99_upper":0.25040024364695346,"ci_98_lower":0.15291950448142785,"ci_98_upper":0.24575207522202042,"ci_95_lower":0.15900944324386784,"ci_95_upper":0.2426870954629304,"ci_90_lower":0.1653703304691871,"ci_90_upper":0.23353861107701254,"ci_75_lower":0.17298065885007682,"ci_75_upper":0.22151674431459528,"ci_50_lower":0.18309437922472435,"ci_50_upper":0.21082704100727795}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"minimal","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash Lite","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":3e-7,"completion":0.0000025,"reasoning":0.0000025,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:06.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"deepseek/deepseek-v4-pro-0813","reasoning_mode":"high","overall_metrics":{"auc_1m":0.3318839464160683,"auc_128k":0.8112841540096557,"cum_avg_1m":0.6338354531191699,"cum_avg_128k":0.8694664215212257,"avg_score_overall":0.6409443415415828,"avg_response_time_ms":81247.86927956682,"total_prompt_cost":0.5487725099999997,"total_cache_read_cost":0.6643069439999989,"total_completion_cost":4.912909140000003,"total_reasoning_cost":4.614898470000002,"total_prompt_cost_128k":0.18908927999999992,"total_completion_cost_128k":2.29222902,"total_on_demand_input_cost":80.26560578999997,"total_on_demand_input_cost_128k":10.657263359999995,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9995516953065471},"bin_metrics":{"8192":{"avg_score":0.9877504605334252,"min_score":0.02666666731238365,"max_score":1,"n_tests":80,"stddev":0.10881511632494327,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.003855716867561,"is_incomplete":false,"total_prompt_cost_bin":0.020333639999999976,"total_cache_read_cost_bin":0.0017650559999999994,"total_completion_cost_bin":0.21013719,"total_reasoning_cost_bin":0.18599120999999996,"total_cost_bin":0.41822709599999996,"total_on_demand_input_cost_bin":0.23214036000000027,"ci_99_lower":0.95116758774966,"ci_99_upper":1,"ci_98_lower":0.9512504605576396,"ci_98_upper":1,"ci_95_lower":0.9633342544082553,"ci_95_upper":1,"ci_90_lower":0.9634171272162348,"ci_90_upper":1,"ci_75_lower":0.9755734347738325,"ci_75_upper":1,"ci_50_lower":0.9756666666828095,"ci_50_upper":0.9999171271920204},"16384":{"avg_score":0.999869991093874,"min_score":0.9925925731658936,"max_score":1,"n_tests":80,"stddev":0.0008523172625865453,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9949113004154972,"is_incomplete":false,"total_prompt_cost_bin":0.027321479999999992,"total_cache_read_cost_bin":0.003743088,"total_completion_cost_bin":0.23476254000000002,"total_reasoning_cost_bin":0.21092019000000006,"total_cost_bin":0.4767472980000001,"total_on_demand_input_cost_bin":0.4764920400000003,"ci_99_lower":0.9995627956055104,"ci_99_upper":1,"ci_98_lower":0.9995883747935295,"ci_98_upper":1,"ci_95_lower":0.9996641784906387,"ci_95_upper":0.9999959923326969,"ci_90_lower":0.9996935792267323,"ci_90_upper":0.9999881773814558,"ci_75_lower":0.9997646167874337,"ci_75_upper":0.99997453186661,"ci_50_lower":0.9998020328581333,"ci_50_upper":0.9999538101255894},"32768":{"avg_score":0.9445872175342896,"min_score":0.0078125,"max_score":1,"n_tests":136,"stddev":0.2226778725629244,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0131260836713822,"is_incomplete":false,"total_prompt_cost_bin":0.042783120000000015,"total_cache_read_cost_bin":0.013870816000000013,"total_completion_cost_bin":0.56443251,"total_reasoning_cost_bin":0.51782052,"total_cost_bin":1.138906966,"total_on_demand_input_cost_bin":1.7072810399999976,"ci_99_lower":0.8884102302187068,"ci_99_upper":0.9863783870148473,"ci_98_lower":0.8953854937873342,"ci_98_upper":0.9856705852148726,"ci_95_lower":0.9022176504073501,"ci_95_upper":0.979188231004895,"ci_90_lower":0.9098222163341501,"ci_90_upper":0.9729488579556346,"ci_75_lower":0.9225935953128261,"ci_75_upper":0.965776340210783,"ci_50_lower":0.9310653010136722,"ci_50_upper":0.9581807717639843},"65536":{"avg_score":0.8406613434281419,"min_score":0.017337461933493614,"max_score":1,"n_tests":85,"stddev":0.3608405653887166,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9959528561071479,"is_incomplete":false,"total_prompt_cost_bin":0.03282162000000001,"total_cache_read_cost_bin":0.019799808000000006,"total_completion_cost_bin":0.5162249400000003,"total_reasoning_cost_bin":0.48667365000000007,"total_cost_bin":1.0555200180000004,"total_on_demand_input_cost_bin":2.4087985799999987,"ci_99_lower":0.7263023259309942,"ci_99_upper":0.9322584236962173,"ci_98_lower":0.7375427669586943,"ci_98_upper":0.9207089096542228,"ci_95_lower":0.7496898901889867,"ci_95_upper":0.9092339941021055,"ci_90_lower":0.7713113717526636,"ci_90_upper":0.8979793222347165,"ci_75_lower":0.7949223597424433,"ci_75_upper":0.8859609925297692,"ci_50_lower":0.817259737914976,"ci_50_upper":0.8637424281195683},"131072":{"avg_score":0.5744630950163978,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4837649422014318,"median_score":1,"q1_score":0.028282858431339264,"q3_score":1,"avg_token_efficiency":0.9946849399707403,"is_incomplete":false,"total_prompt_cost_bin":0.06582941999999993,"total_cache_read_cost_bin":0.04805601599999998,"total_completion_cost_bin":0.7666718399999999,"total_reasoning_cost_bin":0.7280038200000002,"total_cost_bin":1.6085610959999999,"total_on_demand_input_cost_bin":5.832551339999999,"ci_99_lower":0.4514930593896797,"ci_99_upper":0.6974978373204788,"ci_98_lower":0.46066894069596565,"ci_98_upper":0.6787813018291321,"ci_95_lower":0.4806766749653531,"ci_95_upper":0.6683211918420144,"ci_90_lower":0.4987371937549239,"ci_90_upper":0.6581785380754402,"ci_75_lower":0.5182590572204716,"ci_75_upper":0.6312833675101412,"ci_50_lower":0.5448673930967416,"ci_50_upper":0.6114890292321911},"262144":{"avg_score":0.38902411283760047,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.46853292139263597,"median_score":0.04479283466935158,"q1_score":0.02678571455180645,"q3_score":1,"avg_token_efficiency":0.9997813486782479,"is_incomplete":false,"total_prompt_cost_bin":0.12026401499999995,"total_cache_read_cost_bin":0.13284320000000002,"total_completion_cost_bin":1.0317782400000004,"total_reasoning_cost_bin":0.9853489500000007,"total_cost_bin":2.270234405000001,"total_on_demand_input_cost_bin":16.061448015,"ci_99_lower":0.2907298913783582,"ci_99_upper":0.49273914980622835,"ci_98_lower":0.29833230205192324,"ci_98_upper":0.4852236391790046,"ci_95_lower":0.31282382324558405,"ci_95_upper":0.4718163378002512,"ci_90_lower":0.3211030298710226,"ci_90_upper":0.4581518420831512,"ci_75_lower":0.3431877126520523,"ci_75_upper":0.4358871598950511,"ci_50_lower":0.3627434651873484,"ci_50_upper":0.41638573309892113},"524288":{"avg_score":0.33432740450963,"min_score":0.004887585528194904,"max_score":1,"n_tests":236,"stddev":0.44624146227015377,"median_score":0.04160479083657265,"q1_score":0.02375116292387247,"q3_score":1,"avg_token_efficiency":0.9951262252002445,"is_incomplete":false,"total_prompt_cost_bin":0.23941921499999982,"total_cache_read_cost_bin":0.44422895999999895,"total_completion_cost_bin":1.5889018800000019,"total_reasoning_cost_bin":1.5001401300000006,"total_cost_bin":3.7726901850000014,"total_on_demand_input_cost_bin":53.546894414999976,"ci_99_lower":0.2615683460698396,"ci_99_upper":0.40648472829656757,"ci_98_lower":0.2688989941304021,"ci_98_upper":0.4018249753836771,"ci_95_lower":0.2770272717223664,"ci_95_upper":0.3919897836180684,"ci_90_lower":0.2855032675009679,"ci_90_upper":0.38388879176108875,"ci_75_lower":0.300702236695373,"ci_75_upper":0.36808131285573126,"ci_50_lower":0.31589418889804727,"ci_50_upper":0.3538887854561928}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Pro 0813","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":0.00000132,"completion":0.00000396,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:42:44.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T13:10:53.353Z"},{"model_slug":"z-ai/glm-5.2","reasoning_mode":"max","overall_metrics":{"auc_1m":0.3296808631123966,"auc_128k":0.6540647665498102,"cum_avg_1m":0.5599831471047912,"cum_avg_128k":0.7331386910307981,"avg_score_overall":0.578535634199802,"avg_response_time_ms":57575.42781603876,"total_prompt_cost":24.36225838619999,"total_cache_read_cost":28.26808182744,"total_completion_cost":10.090045944,"total_reasoning_cost":8.0109913092,"total_prompt_cost_128k":5.822998809600001,"total_completion_cost_128k":5.66570961,"total_on_demand_input_cost":176.57500668780014,"total_on_demand_input_cost_128k":23.409232887600012,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9922152709249364},"bin_metrics":{"8192":{"avg_score":0.9508078781771474,"min_score":0.010457987897098064,"max_score":1,"n_tests":80,"stddev":0.21529549632588688,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9871057389459195,"is_incomplete":false,"total_prompt_cost_bin":0.3204284789999998,"total_cache_read_cost_bin":0.034285930379999996,"total_completion_cost_bin":0.5839565952,"total_reasoning_cost_bin":0.488454252,"total_cost_bin":1.4271252565799997,"total_on_demand_input_cost_bin":0.5050450271999999,"ci_99_lower":0.87719627521222,"ci_99_upper":0.9999026480838656,"ci_98_lower":0.8889605625917903,"ci_98_upper":0.9998946636915207,"ci_95_lower":0.8896542815273278,"ci_95_upper":0.9877354092895985,"ci_90_lower":0.9018185924179852,"ci_90_upper":0.9876402083667927,"ci_75_lower":0.9260598477485472,"ci_75_upper":0.975447049504146,"ci_50_lower":0.9384380365721883,"ci_50_upper":0.9632519795442931},"16384":{"avg_score":0.8178864801069722,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.38148748826817275,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9789221231603251,"is_incomplete":false,"total_prompt_cost_bin":0.3369685536000002,"total_cache_read_cost_bin":0.13007108399999984,"total_completion_cost_bin":0.5359363943999998,"total_reasoning_cost_bin":0.20276337839999997,"total_cost_bin":1.2057394103999999,"total_on_demand_input_cost_bin":1.0373513136,"ci_99_lower":0.6961698892766144,"ci_99_upper":0.9273979256141465,"ci_98_lower":0.708671289751539,"ci_98_upper":0.91504843330523,"ci_95_lower":0.7316294821852353,"ci_95_upper":0.891212541674031,"ci_90_lower":0.7445979025377892,"ci_90_upper":0.8801148803753308,"ci_75_lower":0.7689495323371376,"ci_75_upper":0.866203951931675,"ci_50_lower":0.7932323365646881,"ci_50_upper":0.8425071657344233},"32768":{"avg_score":0.7467950020115549,"min_score":0.007011393550783396,"max_score":1,"n_tests":136,"stddev":0.42376333324796356,"median_score":1,"q1_score":0.11861924454569817,"q3_score":1,"avg_token_efficiency":1.0076444612557995,"is_incomplete":false,"total_prompt_cost_bin":0.8635954110000013,"total_cache_read_cost_bin":0.5374391794199997,"total_completion_cost_bin":2.3264949971999997,"total_reasoning_cost_bin":2.2280514696000018,"total_cost_bin":5.9555810572200025,"total_on_demand_input_cost_bin":3.757498684800002,"ci_99_lower":0.6548564447925309,"ci_99_upper":0.8398443170097274,"ci_98_lower":0.6618666033996338,"ci_98_upper":0.8297255136023672,"ci_95_lower":0.6763579461512136,"ci_95_upper":0.811242917365213,"ci_90_lower":0.6846522297941299,"ci_90_upper":0.8033726593529161,"ci_75_lower":0.7045726643811898,"ci_75_upper":0.7886599710284773,"ci_50_lower":0.7197029996168567,"ci_50_upper":0.7701216145326137},"65536":{"avg_score":0.6337682267650961,"min_score":0.011319420300424099,"max_score":1,"n_tests":85,"stddev":0.473936366419611,"median_score":1,"q1_score":0.03301799297332764,"q3_score":1,"avg_token_efficiency":0.9893262016400347,"is_incomplete":false,"total_prompt_cost_bin":1.8089380511999997,"total_cache_read_cost_bin":0.6465758208000005,"total_completion_cost_bin":0.9928447056000002,"total_reasoning_cost_bin":0.8882709395999998,"total_cost_bin":4.3366295172,"total_on_demand_input_cost_bin":5.290500163199999,"ci_99_lower":0.4951810050757931,"ci_99_upper":0.7585960129171282,"ci_98_lower":0.507089862644782,"ci_98_upper":0.748374928348643,"ci_95_lower":0.5307440614092218,"ci_95_upper":0.7268935831266401,"ci_90_lower":0.5531647777891554,"ci_90_upper":0.7152633557890483,"ci_75_lower":0.5762398026144022,"ci_75_upper":0.6912218625438126,"ci_50_lower":0.5991779233411174,"ci_50_upper":0.668136204480577},"131072":{"avg_score":0.5164358680932195,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.48963486724817196,"median_score":0.9484291672706604,"q1_score":0.022174709476530552,"q3_score":1,"avg_token_efficiency":0.9887354495866274,"is_incomplete":false,"total_prompt_cost_bin":2.4930683148,"total_cache_read_cost_bin":1.9176428855999985,"total_completion_cost_bin":1.2264769176000008,"total_reasoning_cost_bin":0.5839989143999998,"total_cost_bin":6.221187032399999,"total_on_demand_input_cost_bin":12.818837698800012,"ci_99_lower":0.38327736720747935,"ci_99_upper":0.6390048966714118,"ci_98_lower":0.4034284096906975,"ci_98_upper":0.6292437733072805,"ci_95_lower":0.42237325799809683,"ci_95_upper":0.6103929886962874,"ci_90_lower":0.4400183144730778,"ci_90_upper":0.6001518206033422,"ci_75_lower":0.45943940807138434,"ci_75_upper":0.5735787492605984,"ci_50_lower":0.4868306790374545,"ci_50_upper":0.553299352867999},"262144":{"avg_score":0.464717977856953,"min_score":0.002959674457088113,"max_score":1,"n_tests":141,"stddev":0.4828928415058173,"median_score":0.04812249541282654,"q1_score":0.026079868897795677,"q3_score":1,"avg_token_efficiency":0.9954328543594114,"is_incomplete":false,"total_prompt_cost_bin":5.5528945919999995,"total_cache_read_cost_bin":5.535225707699998,"total_completion_cost_bin":1.2423738228000003,"total_reasoning_cost_bin":0.39010140840000007,"total_cost_bin":12.720595530899999,"total_on_demand_input_cost_bin":35.35795609500002,"ci_99_lower":0.3484482057805996,"ci_99_upper":0.5802623613186078,"ci_98_lower":0.3663132984033317,"ci_98_upper":0.5674703768896706,"ci_95_lower":0.38330520081138963,"ci_95_upper":0.5472439809152461,"ci_90_lower":0.3967093995252974,"ci_90_upper":0.532637048668413,"ci_75_lower":0.41643215762162317,"ci_75_upper":0.5133701037443805,"ci_50_lower":0.43720894205509453,"ci_50_upper":0.49179067322530234},"524288":{"avg_score":0.3494537438273872,"min_score":0.004977600648999214,"max_score":1,"n_tests":236,"stddev":0.45335155846505076,"median_score":0.03932320885360241,"q1_score":0.02263179887086153,"q3_score":1,"avg_token_efficiency":0.9901989900660149,"is_incomplete":false,"total_prompt_cost_bin":12.986364984599987,"total_cache_read_cost_bin":19.46684121954,"total_completion_cost_bin":3.1819625111999996,"total_reasoning_cost_bin":3.229350946799998,"total_cost_bin":38.86451966213998,"total_on_demand_input_cost_bin":117.80781770520011,"ci_99_lower":0.278062454499837,"ci_99_upper":0.42168389120954086,"ci_98_lower":0.28290473555149587,"ci_98_upper":0.41629517373524333,"ci_95_lower":0.28918505370990827,"ci_95_upper":0.406958909526268,"ci_90_lower":0.3006314261567826,"ci_90_upper":0.40196950611277577,"ci_75_lower":0.3150865268699679,"ci_75_upper":0.38327459444911455,"ci_50_lower":0.3298513872417724,"ci_50_upper":0.3691623184812192}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high"],"reasoning_default":"max","sponsors":[],"openrouter_name":"Z.ai: GLM 5.2","provider_name":"Z-ai","run_provider_names":[],"pricing_metadata":{"prompt":0.00000119,"completion":0.00000374,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-06-16T17:45:30.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-22T21:49:31.642Z"},{"model_slug":"deepseek/deepseek-v4-pro-0813","reasoning_mode":"max","overall_metrics":{"auc_1m":0.3286685512908284,"auc_128k":0.7874056805534696,"cum_avg_1m":0.6132058126661923,"cum_avg_128k":0.8404723515949634,"avg_score_overall":0.6272772296178225,"avg_response_time_ms":94267.0193251057,"total_prompt_cost":0.554309625,"total_cache_read_cost":0.664301375999999,"total_completion_cost":5.742121800000001,"total_reasoning_cost":5.457774480000001,"total_prompt_cost_128k":0.18926502000000012,"total_completion_cost_128k":2.7042984300000006,"total_on_demand_input_cost":80.27047474500004,"total_on_demand_input_cost_128k":10.66000038,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9999495018599178},"bin_metrics":{"8192":{"avg_score":0.9389050394529477,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.23679240248015332,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0058751430389692,"is_incomplete":false,"total_prompt_cost_bin":0.020284920000000005,"total_cache_read_cost_bin":0.0017692320000000004,"total_completion_cost_bin":0.22038317999999998,"total_reasoning_cost_bin":0.19640685000000005,"total_cost_bin":0.438844182,"total_on_demand_input_cost_bin":0.2325927600000002,"ci_99_lower":0.8546434988698456,"ci_99_upper":0.9880320089310407,"ci_98_lower":0.8658653727960773,"ci_98_upper":0.9878125227987766,"ci_95_lower":0.8779282567044719,"ci_95_upper":0.987466186331585,"ci_90_lower":0.8901410846598445,"ci_90_upper":0.975747937427368,"ci_75_lower":0.902829234590172,"ci_75_upper":0.963846956170164,"ci_50_lower":0.9260185449908023,"ci_50_upper":0.9518745582085103},"16384":{"avg_score":0.9145013366243802,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.27643006760934774,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9958799759473143,"is_incomplete":false,"total_prompt_cost_bin":0.032840760000000024,"total_cache_read_cost_bin":0.003700863999999998,"total_completion_cost_bin":0.2711537699999999,"total_reasoning_cost_bin":0.24781862999999987,"total_cost_bin":0.5555140239999998,"total_on_demand_input_cost_bin":0.47694444000000014,"ci_99_lower":0.8182679840938072,"ci_99_upper":0.9863324243581738,"ci_98_lower":0.8291695174091729,"ci_98_upper":0.9755374598518246,"ci_95_lower":0.8421073836978759,"ci_95_upper":0.9739385881359339,"ci_90_lower":0.8542605208914028,"ci_90_upper":0.963087622281164,"ci_75_lower":0.8777512157335877,"ci_75_upper":0.9508629663207102,"ci_50_lower":0.8906715245830127,"ci_50_upper":0.9383875726867701},"32768":{"avg_score":0.9721443951951668,"min_score":0.035393256694078445,"max_score":1,"n_tests":136,"stddev":0.1602828758699988,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0135929019062608,"is_incomplete":false,"total_prompt_cost_bin":0.03302868000000002,"total_cache_read_cost_bin":0.01395851200000001,"total_completion_cost_bin":0.6485919600000003,"total_reasoning_cost_bin":0.6015562800000003,"total_cost_bin":1.2971354320000006,"total_on_demand_input_cost_bin":1.708050119999999,"ci_99_lower":0.9297883053223932,"ci_99_upper":0.9999582883014398,"ci_98_lower":0.9305924577663158,"ci_98_upper":0.9999453472828165,"ci_95_lower":0.9435462051947766,"ci_95_upper":0.9931850176961984,"ci_90_lower":0.9448453751159832,"ci_90_upper":0.9929394264924615,"ci_75_lower":0.9578591064988252,"ci_75_upper":0.9861365346621502,"ci_50_lower":0.9649748771506197,"ci_50_upper":0.9793743942042483},"65536":{"avg_score":0.7731017756221049,"min_score":0.010026996023952961,"max_score":1,"n_tests":85,"stddev":0.4114144256066381,"median_score":1,"q1_score":0.997723400592804,"q3_score":1,"avg_token_efficiency":0.9961515981868075,"is_incomplete":false,"total_prompt_cost_bin":0.03681013500000005,"total_cache_read_cost_bin":0.019770576000000005,"total_completion_cost_bin":0.57380763,"total_reasoning_cost_bin":0.54379263,"total_cost_bin":1.1741809710000002,"total_on_demand_input_cost_bin":2.4092792550000017,"ci_99_lower":0.6471534937636597,"ci_99_upper":0.8771119764764409,"ci_98_lower":0.651002954559072,"ci_98_upper":0.8654892689916578,"ci_95_lower":0.6797828078727403,"ci_95_upper":0.8528399353419595,"ci_90_lower":0.6968289756999516,"ci_90_upper":0.8417731961794197,"ci_75_lower":0.7218164966345819,"ci_75_upper":0.8202120163847747,"ci_50_lower":0.7407352212350815,"ci_50_upper":0.805818485811024},"131072":{"avg_score":0.6037092110802176,"min_score":0.007238883059471846,"max_score":1,"n_tests":103,"stddev":0.47924405443900286,"median_score":1,"q1_score":0.02948626410216093,"q3_score":1,"avg_token_efficiency":0.9947842737150728,"is_incomplete":false,"total_prompt_cost_bin":0.06630052500000001,"total_cache_read_cost_bin":0.048056943999999976,"total_completion_cost_bin":0.9903618900000003,"total_reasoning_cost_bin":0.9523107000000002,"total_cost_bin":2.0570300590000006,"total_on_demand_input_cost_bin":5.833133804999998,"ci_99_lower":0.4891173745448095,"ci_99_upper":0.7171300953740788,"ci_98_lower":0.4989068099711253,"ci_98_upper":0.7079537013200681,"ci_95_lower":0.5174918617035976,"ci_95_upper":0.6980021382263835,"ci_90_lower":0.5276283460416045,"ci_90_upper":0.6801227228173499,"ci_75_lower":0.5473139390837654,"ci_75_upper":0.6602065268597673,"ci_50_lower":0.5671386433182085,"ci_50_upper":0.640269208116258},"262144":{"avg_score":0.3607300235014673,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4599964881710665,"median_score":0.04000762104988098,"q1_score":0.023225806653499603,"q3_score":1,"avg_token_efficiency":0.9998309818530743,"is_incomplete":false,"total_prompt_cost_bin":0.12106136999999985,"total_cache_read_cost_bin":0.13284320000000002,"total_completion_cost_bin":1.2964748699999995,"total_reasoning_cost_bin":1.2501543300000006,"total_cost_bin":2.8005337700000004,"total_on_demand_input_cost_bin":16.062245370000007,"ci_99_lower":0.26491234838869254,"ci_99_upper":0.45878724048295594,"ci_98_lower":0.27034557992541264,"ci_98_upper":0.4563194410951894,"ci_95_lower":0.28553718757998947,"ci_95_upper":0.44186112471139516,"ci_90_lower":0.29685430173268407,"ci_90_upper":0.4252553743004785,"ci_75_lower":0.31832458359455185,"ci_75_upper":0.4066198154819917,"ci_50_lower":0.33373015015617224,"ci_50_upper":0.387467404849355},"524288":{"avg_score":0.34255471985325436,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4469344546574435,"median_score":0.04159051179885864,"q1_score":0.024248037487268448,"q3_score":1,"avg_token_efficiency":0.9951510273026298,"is_incomplete":false,"total_prompt_cost_bin":0.2439832350000001,"total_cache_read_cost_bin":0.4442020479999991,"total_completion_cost_bin":1.7413485000000006,"total_reasoning_cost_bin":1.6657350600000003,"total_cost_bin":4.0952688429999995,"total_on_demand_input_cost_bin":53.54822899500004,"ci_99_lower":0.2730220807732184,"ci_99_upper":0.41677677745699504,"ci_98_lower":0.2765690525874034,"ci_98_upper":0.4088935372941551,"ci_95_lower":0.28524457961972016,"ci_95_upper":0.40125882725028605,"ci_90_lower":0.29530924592814334,"ci_90_upper":0.3927238007741033,"ci_75_lower":0.30891191198833884,"ci_75_upper":0.37644127931103494,"ci_50_lower":0.32394213302054053,"ci_50_upper":0.36132910532084633}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Pro 0813","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":0.00000132,"completion":0.00000396,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:42:44.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T04:23:20.975Z"},{"model_slug":"deepseek/deepseek-v4-flash-0731","reasoning_mode":"max","overall_metrics":{"auc_1m":0.3195964237489163,"auc_128k":0.7037004702966082,"cum_avg_1m":0.5796295364012869,"cum_avg_128k":0.7787059918156211,"avg_score_overall":0.5892329360569344,"avg_response_time_ms":26349.03688905297,"total_prompt_cost":0.11731131000000003,"total_cache_read_cost":3.2980746240000034,"total_completion_cost":0.5900238,"total_reasoning_cost":0.5313591,"total_prompt_cost_128k":0.041969160000000026,"total_completion_cost_128k":0.33371586,"total_on_demand_input_cost":16.607684430000013,"total_on_demand_input_cost_128k":2.20551732,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9999495018599177},"bin_metrics":{"8192":{"avg_score":0.9629369889036752,"min_score":0.008075370453298092,"max_score":1,"n_tests":80,"stddev":0.18733292694065048,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0058751430389687,"is_incomplete":false,"total_prompt_cost_bin":0.00659304,"total_cache_read_cost_bin":0.008305920000000003,"total_completion_cost_bin":0.025632359999999993,"total_reasoning_cost_bin":0.020663640000000004,"total_cost_bin":0.06119496,"total_on_demand_input_cost_bin":0.04812263999999999,"ci_99_lower":0.9012851143796579,"ci_99_upper":0.9998041999191045,"ci_98_lower":0.9017447510459461,"ci_98_upper":0.9997705636247992,"ci_95_lower":0.9138711422029883,"ci_95_upper":0.9996973606571555,"ci_90_lower":0.9260387127130524,"ci_90_upper":0.9995230849832296,"ci_75_lower":0.9383503827819367,"ci_75_upper":0.9873647233398515,"ci_50_lower":0.9505957514775217,"ci_50_upper":0.975302048647427},"16384":{"avg_score":0.8538301997876261,"min_score":0.0044469148851931095,"max_score":1,"n_tests":80,"stddev":0.3495743271063266,"median_score":1,"q1_score":0.9996949285268784,"q3_score":1,"avg_token_efficiency":0.9958799759473143,"is_incomplete":false,"total_prompt_cost_bin":0.005435280000000002,"total_cache_read_cost_bin":0.018648575999999993,"total_completion_cost_bin":0.037604700000000005,"total_reasoning_cost_bin":0.03268782,"total_cost_bin":0.094376376,"total_on_demand_input_cost_bin":0.09867815999999995,"ci_99_lower":0.743448065978213,"ci_99_upper":0.9513021575037564,"ci_98_lower":0.7570630740992492,"ci_98_upper":0.938757694745611,"ci_95_lower":0.7697099452260591,"ci_95_upper":0.9263069848669693,"ci_90_lower":0.7821116484352387,"ci_90_upper":0.914335613079602,"ci_75_lower":0.8052710524141731,"ci_75_upper":0.9016517776995897,"ci_50_lower":0.8292575318890157,"ci_50_upper":0.8781783946891665},"32768":{"avg_score":0.8033602186136276,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.38668724284054085,"median_score":1,"q1_score":0.9966581612825394,"q3_score":1,"avg_token_efficiency":1.0135929019062615,"is_incomplete":false,"total_prompt_cost_bin":0.009010800000000018,"total_cache_read_cost_bin":0.06887577600000014,"total_completion_cost_bin":0.08517798000000001,"total_reasoning_cost_bin":0.07553412000000001,"total_cost_bin":0.23859867600000018,"total_on_demand_input_cost_bin":0.35338967999999976,"ci_99_lower":0.7105918403327643,"ci_99_upper":0.8879132484135505,"ci_98_lower":0.7185151485846077,"ci_98_upper":0.8806188324779984,"ci_95_lower":0.7344846059076886,"ci_95_upper":0.8666320375502384,"ci_90_lower":0.7455714031232192,"ci_90_upper":0.8583869311082012,"ci_75_lower":0.7621284056119346,"ci_75_upper":0.843165916211814,"ci_50_lower":0.7801222714564681,"ci_50_upper":0.8261664278774441},"65536":{"avg_score":0.6448011986911297,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.47084866229857925,"median_score":0.9992284178733826,"q1_score":0.03414282202720642,"q3_score":1,"avg_token_efficiency":0.9961515981868065,"is_incomplete":false,"total_prompt_cost_bin":0.007408530000000002,"total_cache_read_cost_bin":0.09821260799999994,"total_completion_cost_bin":0.07594685999999999,"total_reasoning_cost_bin":0.06969924000000001,"total_cost_bin":0.25126723799999995,"total_on_demand_input_cost_bin":0.4984715699999996,"ci_99_lower":0.5058921141216312,"ci_99_upper":0.7707069291887148,"ci_98_lower":0.5173848147518276,"ci_98_upper":0.7583611577614484,"ci_95_lower":0.5420677794759874,"ci_95_upper":0.7375491650649073,"ci_90_lower":0.5640144526125754,"ci_90_upper":0.7254785873225944,"ci_75_lower":0.5870134915937395,"ci_75_upper":0.7021908204660148,"ci_50_lower":0.610132682758986,"ci_50_upper":0.6792467998340725},"131072":{"avg_score":0.6286013530820463,"min_score":0.0057299453765153885,"max_score":1,"n_tests":103,"stddev":0.472052549742346,"median_score":0.9982206225395203,"q1_score":0.03344130329787731,"q3_score":1,"avg_token_efficiency":0.9947842737150724,"is_incomplete":false,"total_prompt_cost_bin":0.01352151,"total_cache_read_cost_bin":0.23866675199999984,"total_completion_cost_bin":0.10935395999999997,"total_reasoning_cost_bin":0.10150632,"total_cost_bin":0.4630485419999998,"total_on_demand_input_cost_bin":1.2068552700000006,"ci_99_lower":0.5053984664338506,"ci_99_upper":0.7484227308864226,"ci_98_lower":0.5150953427069251,"ci_98_upper":0.7327538114646902,"ci_95_lower":0.5374430465602398,"ci_95_upper":0.7223273615895328,"ci_90_lower":0.5522863102471003,"ci_90_upper":0.704871660868357,"ci_75_lower":0.5717372440414286,"ci_75_upper":0.6843344185725126,"ci_50_lower":0.5969188808061145,"ci_50_upper":0.6595388320432886},"262144":{"avg_score":0.4451615002224261,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4781115601805329,"median_score":0.05745946988463402,"q1_score":0.030031200498342514,"q3_score":1,"avg_token_efficiency":0.9998309818530743,"is_incomplete":false,"total_prompt_cost_bin":0.025047179999999992,"total_cache_read_cost_bin":0.6596351999999995,"total_completion_cost_bin":0.09558197999999998,"total_reasoning_cost_bin":0.08608860000000002,"total_cost_bin":0.8663529599999995,"total_on_demand_input_cost_bin":3.3232231800000003,"ci_99_lower":0.3293372717152799,"ci_99_upper":0.5570480592605815,"ci_98_lower":0.34634695345565597,"ci_98_upper":0.5464477429714001,"ci_95_lower":0.3673829493183867,"ci_95_upper":0.527469837054914,"ci_90_lower":0.37712300242492863,"ci_90_upper":0.513280782731851,"ci_75_lower":0.3978640709657731,"ci_75_upper":0.4930287954750048,"ci_50_lower":0.4174200301752129,"ci_50_upper":0.4723887582902354},"524288":{"avg_score":0.29834483190976335,"min_score":0.004477611742913723,"max_score":1,"n_tests":236,"stddev":0.42748032699706406,"median_score":0.03757714480161667,"q1_score":0.02275283820927143,"q3_score":0.9837007373571396,"avg_token_efficiency":0.9951510273026295,"is_incomplete":false,"total_prompt_cost_bin":0.05029497000000001,"total_cache_read_cost_bin":2.205729792000004,"total_completion_cost_bin":0.16072596000000003,"total_reasoning_cost_bin":0.14517936000000004,"total_cost_bin":2.5619300820000044,"total_on_demand_input_cost_bin":11.078943930000012,"ci_99_lower":0.2284733230335701,"ci_99_upper":0.36600430680666773,"ci_98_lower":0.23432862314306815,"ci_98_upper":0.36208159263389805,"ci_95_lower":0.24305014421520127,"ci_95_upper":0.3499170880991634,"ci_90_lower":0.2539426300259331,"ci_90_upper":0.3433940272416201,"ci_75_lower":0.2682726332117208,"ci_75_upper":0.3307213685329771,"ci_50_lower":0.28122065351413705,"ci_50_upper":0.31770822880651517}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1310720,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Flash 0731","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":6e-8,"completion":1.2e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-31T06:21:48.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-01T22:55:39.945Z"},{"model_slug":"google/gemini-3.5-flash-lite","reasoning_mode":"low","overall_metrics":{"auc_1m":0.3084232455227491,"auc_128k":0.5257220064167618,"cum_avg_1m":0.49823378799066814,"cum_avg_128k":0.6303759571574643,"avg_score_overall":0.40870415778587277,"avg_response_time_ms":6854.063593368465,"total_prompt_cost":5.846579100000001,"total_cache_read_cost":14.720641259999919,"total_completion_cost":1.1150650000000004,"total_reasoning_cost":5.159675000000001,"total_prompt_cost_128k":1.9645239000000003,"total_completion_cost_128k":0.46689500000000006,"total_on_demand_input_cost":153.05299170000006,"total_on_demand_input_cost_128k":7.364040000000003,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154896},"bin_metrics":{"8192":{"avg_score":0.8765334342722781,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.32464104562104423,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747248,"is_incomplete":false,"total_prompt_cost_bin":0.15946319999999992,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06986750000000001,"total_reasoning_cost_bin":0.21422500000000008,"total_cost_bin":0.4435557,"total_on_demand_input_cost_bin":0.15946319999999992,"ci_99_lower":0.7788745759292505,"ci_99_upper":0.9624124038456356,"ci_98_lower":0.7894945760850096,"ci_98_upper":0.9497115136366338,"ci_95_lower":0.8014647426491138,"ci_95_upper":0.9378767574109952,"ci_90_lower":0.8131085854396224,"ci_90_upper":0.9270408129354474,"ci_75_lower":0.8374127659393708,"ci_75_upper":0.913807662104955,"ci_50_lower":0.8524294394243043,"ci_50_upper":0.9009829204936977},"16384":{"avg_score":0.7693042392289499,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.41590695600723065,"median_score":1,"q1_score":0.9991826266050339,"q3_score":1,"avg_token_efficiency":1.0000745135024471,"is_incomplete":false,"total_prompt_cost_bin":0.3305616,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0693,"total_reasoning_cost_bin":0.23532749999999997,"total_cost_bin":0.6351891000000001,"total_on_demand_input_cost_bin":0.3305616,"ci_99_lower":0.6470557421879203,"ci_99_upper":0.8788358441263409,"ci_98_lower":0.6590838237885328,"ci_98_upper":0.8667894733974244,"ci_95_lower":0.6728384730297694,"ci_95_upper":0.8546621772181242,"ci_90_lower":0.6956966812472092,"ci_90_upper":0.8421020556487202,"ci_75_lower":0.7202359168248221,"ci_75_upper":0.8182453618941509,"ci_50_lower":0.7336984731795383,"ci_50_upper":0.8052434646422626},"32768":{"avg_score":0.6336779338958235,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4677355084180635,"median_score":1,"q1_score":0.04362505115568638,"q3_score":1,"avg_token_efficiency":1.000035909094992,"is_incomplete":false,"total_prompt_cost_bin":0.5090745000000001,"total_cache_read_cost_bin":0.06529503000000007,"total_completion_cost_bin":0.13344750000000002,"total_reasoning_cost_bin":0.5104225000000001,"total_cost_bin":1.2182395300000004,"total_on_demand_input_cost_bin":1.1620247999999995,"ci_99_lower":0.522453701636028,"ci_99_upper":0.7305121251860264,"ci_98_lower":0.5394076032655807,"ci_98_upper":0.7235081854204798,"ci_95_lower":0.5556328768756323,"ci_95_upper":0.7154786548321307,"ci_90_lower":0.5687078508832629,"ci_90_upper":0.6984987973973414,"ci_75_lower":0.5861149009861573,"ci_75_upper":0.6792281574953984,"ci_50_lower":0.6072885746500962,"ci_50_upper":0.6594972817110829},"65536":{"avg_score":0.4515582525116556,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4844647816479542,"median_score":0.04197901114821434,"q1_score":0.025459688156843185,"q3_score":1,"avg_token_efficiency":1.000015287852282,"is_incomplete":false,"total_prompt_cost_bin":0.4300160999999999,"total_cache_read_cost_bin":0.12380004000000021,"total_completion_cost_bin":0.08443999999999999,"total_reasoning_cost_bin":0.37592000000000014,"total_cost_bin":1.0141761400000002,"total_on_demand_input_cost_bin":1.6680164999999987,"ci_99_lower":0.3138026327496285,"ci_99_upper":0.5881923426304232,"ci_98_lower":0.3257840315656846,"ci_98_upper":0.5681128929614582,"ci_95_lower":0.34669558198958195,"ci_95_upper":0.5549901863686084,"ci_90_lower":0.3602431448620251,"ci_90_upper":0.5416142792112248,"ci_75_lower":0.3930864193545216,"ci_75_upper":0.5092870718347566,"ci_50_lower":0.4173044868334032,"ci_50_upper":0.48642172269961415},"131072":{"avg_score":0.42080592587861476,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4781527001601576,"median_score":0.03697334602475166,"q1_score":0.02365003526210785,"q3_score":1,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":0.5354085000000004,"total_cache_read_cost_bin":0.35085654000000044,"total_completion_cost_bin":0.10984000000000003,"total_reasoning_cost_bin":0.5133500000000001,"total_cost_bin":1.509455040000001,"total_on_demand_input_cost_bin":4.043973900000005,"ci_99_lower":0.296575405819328,"ci_99_upper":0.5361868107606412,"ci_98_lower":0.3111161210146977,"ci_98_upper":0.5240901379587436,"ci_95_lower":0.3261414533951184,"ci_95_upper":0.5100814539986636,"ci_90_lower":0.3413288822228053,"ci_90_upper":0.49866131254566776,"ci_75_lower":0.3650473085581325,"ci_75_upper":0.47781741406284817,"ci_50_lower":0.38740874573167916,"ci_50_upper":0.45580652993606424},"262144":{"avg_score":0.35839535375059656,"min_score":0.003799392143264413,"max_score":1,"n_tests":141,"stddev":0.4554522945655455,"median_score":0.04401480779051781,"q1_score":0.0260506272315979,"q3_score":1,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":0.7587075000000004,"total_cache_read_cost_bin":1.0320617999999968,"total_completion_cost_bin":0.13353250000000005,"total_reasoning_cost_bin":0.7058275,"total_cost_bin":2.6301292999999974,"total_on_demand_input_cost_bin":11.079325499999984,"ci_99_lower":0.26785456406263014,"ci_99_upper":0.4575749707499152,"ci_98_lower":0.27173107664591256,"ci_98_upper":0.45101577604529675,"ci_95_lower":0.28448715252778656,"ci_95_upper":0.4370497840800956,"ci_90_lower":0.2946603823308701,"ci_90_upper":0.4229641739558098,"ci_75_lower":0.3153441591408757,"ci_75_upper":0.4019376553864638,"ci_50_lower":0.33209678121546865,"ci_50_upper":0.3843825647379569},"524288":{"avg_score":0.2560588079810506,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4055434049201448,"median_score":0.03517480380833149,"q1_score":0.020707974210381508,"q3_score":0.14960088953375816,"avg_token_efficiency":1.0000019078540328,"is_incomplete":false,"total_prompt_cost_bin":1.3047816000000003,"total_cache_read_cost_bin":3.5805046499999964,"total_completion_cost_bin":0.21518250000000017,"total_reasoning_cost_bin":1.0991400000000002,"total_cost_bin":6.199608749999997,"total_on_demand_input_cost_bin":37.1098281,"ci_99_lower":0.18981116165150985,"ci_99_upper":0.32074960423404075,"ci_98_lower":0.19475344727831,"ci_98_upper":0.3158123665084565,"ci_95_lower":0.20580691964897305,"ci_95_upper":0.30879511817361394,"ci_90_lower":0.21472205916751114,"ci_90_upper":0.30202374885320443,"ci_75_lower":0.22605078465932235,"ci_75_upper":0.2865236169684252,"ci_50_lower":0.23917257165046946,"ci_50_upper":0.27348961358666973},"1048576":{"avg_score":0.21953635640637648,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.38067713721341134,"median_score":0.030013572424650192,"q1_score":0.019342525862157345,"q3_score":0.07969384454190731,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":1.8185660999999997,"total_cache_read_cost_bin":9.568123199999924,"total_completion_cost_bin":0.299455,"total_reasoning_cost_bin":1.5054625000000001,"total_cost_bin":13.191606799999924,"total_on_demand_input_cost_bin":97.49979810000008,"ci_99_lower":0.16934860219499623,"ci_99_upper":0.27474887510763496,"ci_98_lower":0.1746577464641267,"ci_98_upper":0.2705811185852339,"ci_95_lower":0.17757607345782866,"ci_95_upper":0.2655083914077465,"ci_90_lower":0.18439543761189034,"ci_90_upper":0.256784335729974,"ci_75_lower":0.1942435963064312,"ci_75_upper":0.2442109667256685,"ci_50_lower":0.20364112714981478,"ci_50_upper":0.23352504063438953}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"minimal","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash Lite","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":3e-7,"completion":0.0000025,"reasoning":0.0000025,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:06.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"google/gemini-3.6-flash","reasoning_mode":"minimal","overall_metrics":{"auc_1m":0.29405516971200873,"auc_128k":0.25899486158651824,"cum_avg_1m":0.31556252113777333,"cum_avg_128k":0.3194179140661996,"avg_score_overall":0.30520026650519366,"avg_response_time_ms":4059.649936937306,"total_prompt_cost":30.29274749999996,"total_cache_read_cost":73.18271849999981,"total_completion_cost":3.3907049999999987,"total_reasoning_cost":0,"total_prompt_cost_128k":8.448855,"total_completion_cost_128k":1.4162249999999998,"total_on_demand_input_cost":762.1199325000014,"total_on_demand_input_cost_128k":36.8202,"total_runs":1168,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000228064706111},"bin_metrics":{"8192":{"avg_score":0.4947330749113462,"min_score":0.0011554014636203647,"max_score":1,"n_tests":80,"stddev":0.48408258493626816,"median_score":0.11386319994926453,"q1_score":0.029806291218847036,"q3_score":1,"avg_token_efficiency":1.0001553404747234,"is_incomplete":false,"total_prompt_cost_bin":0.7973160000000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.2111175,"total_reasoning_cost_bin":0,"total_cost_bin":1.0084335000000004,"total_on_demand_input_cost_bin":0.7973160000000004,"ci_99_lower":0.3527622325366828,"ci_99_upper":0.6253581697007903,"ci_98_lower":0.3701033666658914,"ci_98_upper":0.6144132099158742,"ci_95_lower":0.3839601081746878,"ci_95_upper":0.5970647265607112,"ci_90_lower":0.4041680676882242,"ci_90_upper":0.5817884248949119,"ci_75_lower":0.4352097182791112,"ci_75_upper":0.5556990881854289,"ci_50_lower":0.4589314806118637,"ci_50_upper":0.5308665678618127},"16384":{"avg_score":0.3808543058228679,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.46996644759501427,"median_score":0.03638450428843498,"q1_score":0.02466225018724799,"q3_score":1,"avg_token_efficiency":1.0000745135024471,"is_incomplete":false,"total_prompt_cost_bin":0.8927429999999997,"total_cache_read_cost_bin":0.07600649999999994,"total_completion_cost_bin":0.21041249999999997,"total_reasoning_cost_bin":0,"total_cost_bin":1.1791619999999996,"total_on_demand_input_cost_bin":1.6528079999999987,"ci_99_lower":0.24696996687765932,"ci_99_upper":0.5142075828660744,"ci_98_lower":0.25733723713364454,"ci_98_upper":0.5025742820589802,"ci_95_lower":0.2725051393522881,"ci_95_upper":0.4894963148163515,"ci_90_lower":0.2937726459454279,"ci_90_upper":0.46629980361496565,"ci_75_lower":0.3207594930223422,"ci_75_upper":0.4420035217975965,"ci_50_lower":0.3444988701201509,"ci_50_upper":0.41670721745758776},"32768":{"avg_score":0.2651425009672119,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.41008552192097203,"median_score":0.037810616195201874,"q1_score":0.025264735333621502,"q3_score":0.16304322332143784,"avg_token_efficiency":1.0000359090949906,"is_incomplete":false,"total_prompt_cost_bin":1.857861,"total_cache_read_cost_bin":0.39522630000000075,"total_completion_cost_bin":0.40286249999999973,"total_reasoning_cost_bin":0,"total_cost_bin":2.6559498000000006,"total_on_demand_input_cost_bin":5.810123999999996,"ci_99_lower":0.18013948171568164,"ci_99_upper":0.3491254107497533,"ci_98_lower":0.18490292443072096,"ci_98_upper":0.3450408646555426,"ci_95_lower":0.19859255006000132,"ci_95_upper":0.3336080293313631,"ci_90_lower":0.2070971417278462,"ci_90_upper":0.32054760778393004,"ci_75_lower":0.22343297207579088,"ci_75_upper":0.30457354480099375,"ci_50_lower":0.24115694540010094,"ci_50_upper":0.2871923587800291},"65536":{"avg_score":0.2227043350939365,"min_score":0.008178100921213627,"max_score":1,"n_tests":85,"stddev":0.39120079244839917,"median_score":0.029749831184744835,"q1_score":0.0202312134206295,"q3_score":0.042375244200229645,"avg_token_efficiency":1.0000152878522814,"is_incomplete":false,"total_prompt_cost_bin":1.6342274999999977,"total_cache_read_cost_bin":0.6705855,"total_completion_cost_bin":0.26340750000000007,"total_reasoning_cost_bin":0,"total_cost_bin":2.5682204999999976,"total_on_demand_input_cost_bin":8.340082500000001,"ci_99_lower":0.13159963786832113,"ci_99_upper":0.3370527052863994,"ci_98_lower":0.13421498899194687,"ci_98_upper":0.3273858459073612,"ci_95_lower":0.15180596253505962,"ci_95_upper":0.30389975645250694,"ci_90_lower":0.15600713181966808,"ci_90_upper":0.29213309511049274,"ci_75_lower":0.17475329200490652,"ci_75_upper":0.26806075041565824,"ci_50_lower":0.18941531430875116,"ci_50_upper":0.2461545099696035},"131072":{"avg_score":0.23365535353563582,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.401188469569623,"median_score":0.029182879254221916,"q1_score":0.019556314684450626,"q3_score":0.04771394468843937,"avg_token_efficiency":1.0000076410572558,"is_incomplete":false,"total_prompt_cost_bin":3.266707500000001,"total_cache_read_cost_bin":1.6953162000000013,"total_completion_cost_bin":0.32842500000000024,"total_reasoning_cost_bin":0,"total_cost_bin":5.2904487000000024,"total_on_demand_input_cost_bin":20.219869500000005,"ci_99_lower":0.1386465526328906,"ci_99_upper":0.3361310296334005,"ci_98_lower":0.1475030432392626,"ci_98_upper":0.32734461643052676,"ci_95_lower":0.16472878738145968,"ci_95_upper":0.3088147557955416,"ci_90_lower":0.17492542356677643,"ci_90_upper":0.29900516095272833,"ci_75_lower":0.1867593445993725,"ci_75_upper":0.2787861656572769,"ci_50_lower":0.20597174845113714,"ci_50_upper":0.2561424147372713},"262144":{"avg_score":0.3417432009199542,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.45240706434002065,"median_score":0.03996983543038368,"q1_score":0.02271866798400879,"q3_score":1,"avg_token_efficiency":1.0000038179365247,"is_incomplete":false,"total_prompt_cost_bin":2.8351874999999986,"total_cache_read_cost_bin":5.256144000000008,"total_completion_cost_bin":0.39924750000000025,"total_reasoning_cost_bin":0,"total_cost_bin":8.490579000000006,"total_on_demand_input_cost_bin":55.39662750000001,"ci_99_lower":0.24845086598171068,"ci_99_upper":0.43823747247493006,"ci_98_lower":0.25759953318928297,"ci_98_upper":0.4332597836118574,"ci_95_lower":0.26794062991501066,"ci_95_upper":0.4164332464283682,"ci_90_lower":0.27897517353862666,"ci_90_upper":0.40454310632520524,"ci_75_lower":0.3002041899607391,"ci_75_upper":0.385551791827414,"ci_50_lower":0.3145727686219063,"ci_50_upper":0.3682093746920532},"524288":{"avg_score":0.29047795300149065,"min_score":0.0021798366215080023,"max_score":1,"n_tests":234,"stddev":0.42320804382229366,"median_score":0.0401480458676815,"q1_score":0.024826818611472845,"q3_score":1,"avg_token_efficiency":1.0000019078539977,"is_incomplete":false,"total_prompt_cost_bin":7.482805499999988,"total_cache_read_cost_bin":17.64938879999993,"total_completion_cost_bin":0.6488399999999995,"total_reasoning_cost_bin":0,"total_cost_bin":25.781034299999916,"total_on_demand_input_cost_bin":183.97669350000018,"ci_99_lower":0.22079699502481767,"ci_99_upper":0.3531671731237324,"ci_98_lower":0.22744605974638515,"ci_98_upper":0.34930553935121933,"ci_95_lower":0.23695329065120213,"ci_95_upper":0.3415920950324597,"ci_90_lower":0.24727729029135587,"ci_90_upper":0.33275595090489307,"ci_75_lower":0.25867447715352637,"ci_75_upper":0.32345977738677745,"ci_50_lower":0.27268483530721055,"ci_50_upper":0.31044708535325927},"1048576":{"avg_score":0.2951894448497438,"min_score":0.0021008404437452555,"max_score":1,"n_tests":309,"stddev":0.4289150845277642,"median_score":0.03373249992728233,"q1_score":0.021564161404967308,"q3_score":1,"avg_token_efficiency":1.0000009538490442,"is_incomplete":false,"total_prompt_cost_bin":11.525899499999976,"total_cache_read_cost_bin":47.44005119999987,"total_completion_cost_bin":0.9263924999999991,"total_reasoning_cost_bin":0,"total_cost_bin":59.89234319999985,"total_on_demand_input_cost_bin":485.9264115000012,"ci_99_lower":0.22903553350834996,"ci_99_upper":0.357883845180044,"ci_98_lower":0.23440851022488612,"ci_98_upper":0.35012231888254036,"ci_95_lower":0.24406840513393654,"ci_95_upper":0.34223438400455336,"ci_90_lower":0.2527094396844681,"ci_90_upper":0.3341072504319368,"ci_75_lower":0.26787945419682935,"ci_75_upper":0.32256321887870715,"ci_50_lower":0.27933724823804923,"ci_50_upper":0.3117081646758138}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"Google: Gemini 3.6 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.00000375,"reasoning":0.00000375,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:13.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-21T17:46:07.356Z"},{"model_slug":"anthropic/claude-sonnet-4.6","reasoning_mode":null,"overall_metrics":{"auc_1m":0.2929703540863144,"auc_128k":0.5282787649040911,"cum_avg_1m":0.44707573280439783,"cum_avg_128k":0.5635794030079244,"avg_score_overall":0.46729149765434536,"avg_response_time_ms":39591.61297472734,"total_prompt_cost":4.2333359999999995,"total_cache_read_cost":62.29259729999993,"total_completion_cost":5.553660000000001,"total_reasoning_cost":0,"total_prompt_cost_128k":1.5209820000000003,"total_completion_cost_128k":3.17103,"total_on_demand_input_cost":627.159309000001,"total_on_demand_input_cost_128k":83.17225499999998,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1311011541866511},"bin_metrics":{"8192":{"avg_score":0.7434459097741637,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.42052994206006766,"median_score":0.9929593503475189,"q1_score":0.1255621649324894,"q3_score":0.9989095777273178,"avg_token_efficiency":1.1377505627360658,"is_incomplete":false,"total_prompt_cost_bin":0.36475500000000016,"total_cache_read_cost_bin":0.14473889999999995,"total_completion_cost_bin":0.4932450000000002,"total_reasoning_cost_bin":0,"total_cost_bin":1.0027389000000002,"total_on_demand_input_cost_bin":1.8121439999999982,"ci_99_lower":0.620571741970256,"ci_99_upper":0.8465567530401749,"ci_98_lower":0.6321125028717798,"ci_98_upper":0.8394319333722233,"ci_95_lower":0.6495854832729675,"ci_95_upper":0.8292729857124503,"ci_90_lower":0.6636608543634065,"ci_90_upper":0.8226468543396913,"ci_75_lower":0.6876755628167303,"ci_75_upper":0.7943637992480944,"ci_50_lower":0.710485132553731,"ci_50_upper":0.7766695769678337},"16384":{"avg_score":0.5412828032654943,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.47533736701864726,"median_score":0.9201827645301819,"q1_score":0.026323718018829823,"q3_score":0.9962773472070694,"avg_token_efficiency":1.1227572802981338,"is_incomplete":false,"total_prompt_cost_bin":0.2548799999999999,"total_cache_read_cost_bin":0.34615680000000004,"total_completion_cost_bin":0.44610000000000005,"total_reasoning_cost_bin":0,"total_cost_bin":1.0471368,"total_on_demand_input_cost_bin":3.7164480000000055,"ci_99_lower":0.407655097195864,"ci_99_upper":0.6797879075938835,"ci_98_lower":0.4170519945961132,"ci_98_upper":0.6651673679361702,"ci_95_lower":0.4333188424958644,"ci_95_upper":0.6429271713040362,"ci_90_lower":0.4549668162265152,"ci_90_upper":0.6297842786509136,"ci_75_lower":0.4829133652150631,"ci_75_upper":0.603570123260215,"ci_50_lower":0.5059144895676582,"ci_50_upper":0.5774224421104008},"32768":{"avg_score":0.4926890294488711,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.4753979824927881,"median_score":0.13771099597215652,"q1_score":0.0297760465182364,"q3_score":0.9948182106018066,"avg_token_efficiency":1.1449043855358674,"is_incomplete":false,"total_prompt_cost_bin":0.21267599999999995,"total_cache_read_cost_bin":1.3115699999999992,"total_completion_cost_bin":0.9161849999999996,"total_reasoning_cost_bin":0,"total_cost_bin":2.4404309999999985,"total_on_demand_input_cost_bin":13.328376000000008,"ci_99_lower":0.39127715761569565,"ci_99_upper":0.6014682925530017,"ci_98_lower":0.40046285315771135,"ci_98_upper":0.5859209353158845,"ci_95_lower":0.41383125292114153,"ci_95_upper":0.5710078771490182,"ci_90_lower":0.42436935987686464,"ci_90_upper":0.5585367437862956,"ci_75_lower":0.4488578539173485,"ci_75_upper":0.5406955969519913,"ci_50_lower":0.46582171174326076,"ci_50_upper":0.5213476082589135},"65536":{"avg_score":0.5502750676125288,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.482582730995945,"median_score":0.9753334522247314,"q1_score":0.02942449226975441,"q3_score":0.9979954957962036,"avg_token_efficiency":1.1315791278603577,"is_incomplete":false,"total_prompt_cost_bin":0.233694,"total_cache_read_cost_bin":1.8640943999999986,"total_completion_cost_bin":0.5881949999999998,"total_reasoning_cost_bin":0,"total_cost_bin":2.6859833999999982,"total_on_demand_input_cost_bin":18.874638,"ci_99_lower":0.4042422778335235,"ci_99_upper":0.6765566483653315,"ci_98_lower":0.4245831644981004,"ci_98_upper":0.6736020204327343,"ci_95_lower":0.44861759751295566,"ci_95_upper":0.654352641732377,"ci_90_lower":0.46962970998850373,"ci_90_upper":0.6414824592376895,"ci_75_lower":0.49224765604389287,"ci_75_upper":0.6152375753147199,"ci_50_lower":0.5157899817451834,"ci_50_upper":0.5846225140287595},"131072":{"avg_score":0.4902042049385643,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.48053452915374056,"median_score":0.04639101028442383,"q1_score":0.02597795519977808,"q3_score":0.992059737443924,"avg_token_efficiency":1.1236718471186093,"is_incomplete":false,"total_prompt_cost_bin":0.45497700000000013,"total_cache_read_cost_bin":4.498567200000001,"total_completion_cost_bin":0.7273050000000003,"total_reasoning_cost_bin":0,"total_cost_bin":5.680849200000002,"total_on_demand_input_cost_bin":45.44064899999996,"ci_99_lower":0.36801535874919383,"ci_99_upper":0.6105081916356446,"ci_98_lower":0.37946144706962937,"ci_98_upper":0.6019588789685847,"ci_95_lower":0.3974136109999042,"ci_95_upper":0.5828274035152134,"ci_90_lower":0.4138028142634707,"ci_90_upper":0.5672182331131816,"ci_75_lower":0.435168232242362,"ci_75_upper":0.5467400049744966,"ci_50_lower":0.4597377844354888,"ci_50_upper":0.5250595943523046},"262144":{"avg_score":0.4606918674622867,"min_score":0.006430868059396744,"max_score":1,"n_tests":141,"stddev":0.47869469583355717,"median_score":0.052988048642873764,"q1_score":0.025599999353289604,"q3_score":0.9967663884162903,"avg_token_efficiency":1.1264169418592793,"is_incomplete":false,"total_prompt_cost_bin":0.9039629999999996,"total_cache_read_cost_bin":12.389496000000031,"total_completion_cost_bin":0.9327450000000004,"total_reasoning_cost_bin":0,"total_cost_bin":14.226204000000031,"total_on_demand_input_cost_bin":124.79892300000004,"ci_99_lower":0.3452098278960887,"ci_99_upper":0.5763433671777686,"ci_98_lower":0.3648006880235799,"ci_98_upper":0.5618126458089456,"ci_95_lower":0.37954429160631825,"ci_95_upper":0.5415048963472855,"ci_90_lower":0.3931839013234415,"ci_90_upper":0.5275258646901458,"ci_75_lower":0.41277864909192535,"ci_75_upper":0.5096280280911819,"ci_50_lower":0.433612524057864,"ci_50_upper":0.4876597599664716},"524288":{"avg_score":0.29801697993327436,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.42755312153256736,"median_score":0.031787531450390816,"q1_score":0.02137674205005169,"q3_score":0.9482669830322266,"avg_token_efficiency":1.1295900633711318,"is_incomplete":false,"total_prompt_cost_bin":1.8083909999999992,"total_cache_read_cost_bin":41.7379739999999,"total_completion_cost_bin":1.4498850000000003,"total_reasoning_cost_bin":0,"total_cost_bin":44.996249999999904,"total_on_demand_input_cost_bin":419.1881310000011,"ci_99_lower":0.2290369571824913,"ci_99_upper":0.36464140960444513,"ci_98_lower":0.23448438326486826,"ci_98_upper":0.3594185852148901,"ci_95_lower":0.24328110527308358,"ci_95_upper":0.3503083492349537,"ci_90_lower":0.25337361601236574,"ci_90_upper":0.34330514363512415,"ci_75_lower":0.267392932169753,"ci_75_upper":0.3305862686442413,"ci_50_lower":0.28102055835582995,"ci_50_upper":0.31724098206929846}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Anthropic: Claude Sonnet 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000003,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-17T15:43:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T01:24:42.892Z"},{"model_slug":"deepseek/deepseek-v4-flash-0731","reasoning_mode":"high","overall_metrics":{"auc_1m":0.2925354504954971,"auc_128k":0.6919254830755546,"cum_avg_1m":0.5631488592486662,"cum_avg_128k":0.7655626420116851,"avg_score_overall":0.5645771179456524,"avg_response_time_ms":25030.109706577034,"total_prompt_cost":0.11367774000000014,"total_cache_read_cost":3.289211136000003,"total_completion_cost":0.5271955199999999,"total_reasoning_cost":0.4684741199999999,"total_prompt_cost_128k":0.039271679999999996,"total_completion_cost_128k":0.2784140999999999,"total_on_demand_input_cost":16.55973341999998,"total_on_demand_input_cost_128k":2.20495104,"total_runs":860,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9995568398952276},"bin_metrics":{"8192":{"avg_score":0.9514718527672812,"min_score":0.02108139730989933,"max_score":1,"n_tests":80,"stddev":0.21286851757739947,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0038557168675617,"is_incomplete":false,"total_prompt_cost_bin":0.007156079999999998,"total_cache_read_cost_bin":0.008174592000000005,"total_completion_cost_bin":0.022899959999999994,"total_reasoning_cost_bin":0.0178488,"total_cost_bin":0.056079432,"total_on_demand_input_cost_bin":0.04802903999999999,"ci_99_lower":0.8786524971789913,"ci_99_upper":1,"ci_98_lower":0.890298354686238,"ci_98_upper":1,"ci_95_lower":0.8910321174224373,"ci_95_upper":0.9879833512473851,"ci_90_lower":0.903056303638732,"ci_90_upper":0.9878704552305863,"ci_75_lower":0.9269988877000287,"ci_75_upper":0.9757582451333292,"ci_50_lower":0.9392179542628583,"ci_50_upper":0.9637242617085576},"16384":{"avg_score":0.9139793683891184,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.2780310954150751,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9949113004154974,"is_incomplete":false,"total_prompt_cost_bin":0.00524952,"total_cache_read_cost_bin":0.018667007999999992,"total_completion_cost_bin":0.0261873,"total_reasoning_cost_bin":0.021257820000000007,"total_cost_bin":0.071361648,"total_on_demand_input_cost_bin":0.09858455999999997,"ci_99_lower":0.8171120973116486,"ci_99_upper":0.9861015299435821,"ci_98_lower":0.828145395221538,"ci_98_upper":0.9753672034669434,"ci_95_lower":0.841030801810557,"ci_95_upper":0.9740321373971528,"ci_90_lower":0.8531895234755938,"ci_90_upper":0.9628392421943136,"ci_75_lower":0.8769924583612011,"ci_75_upper":0.9504173846333288,"ci_50_lower":0.8899008376058191,"ci_50_upper":0.9379977960110408},"32768":{"avg_score":0.7664030382772634,"min_score":0.007011393550783396,"max_score":1,"n_tests":136,"stddev":0.41400919633325794,"median_score":1,"q1_score":0.9939576387405396,"q3_score":1,"avg_token_efficiency":1.0131260836713811,"is_incomplete":false,"total_prompt_cost_bin":0.0066744,"total_cache_read_cost_bin":0.06931123200000014,"total_completion_cost_bin":0.06547985999999997,"total_reasoning_cost_bin":0.05592276000000002,"total_cost_bin":0.19738825200000013,"total_on_demand_input_cost_bin":0.35323055999999986,"ci_99_lower":0.6747330257648889,"ci_99_upper":0.8584110257575658,"ci_98_lower":0.6822416922049803,"ci_98_upper":0.8511802392111927,"ci_95_lower":0.6959732801534462,"ci_95_upper":0.8379550178239108,"ci_90_lower":0.7077454635527615,"ci_90_upper":0.8224402730663047,"ci_75_lower":0.7249461807783946,"ci_75_upper":0.807741550164578,"ci_50_lower":0.7407623788937652,"ci_50_upper":0.7893740817277883},"65536":{"avg_score":0.7245661749175805,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.44059784038864064,"median_score":1,"q1_score":0.04961089417338371,"q3_score":1,"avg_token_efficiency":0.9959528561071482,"is_incomplete":false,"total_prompt_cost_bin":0.006790680000000003,"total_cache_read_cost_bin":0.09831628799999993,"total_completion_cost_bin":0.04175154000000001,"total_reasoning_cost_bin":0.035686980000000035,"total_cost_bin":0.18254548799999998,"total_on_demand_input_cost_bin":0.4983721199999996,"ci_99_lower":0.5868135449845782,"ci_99_upper":0.8399364151776055,"ci_98_lower":0.6099590735262916,"ci_98_upper":0.8275923302611337,"ci_95_lower":0.6215431619445136,"ci_95_upper":0.8158626906332723,"ci_90_lower":0.6437347214093759,"ci_90_upper":0.7953573735206219,"ci_75_lower":0.6676669853855857,"ci_75_upper":0.78107433840427,"ci_50_lower":0.6905324503669844,"ci_50_upper":0.7580502205377664},"131072":{"avg_score":0.47139277570718197,"min_score":0.003610108280554414,"max_score":1,"n_tests":103,"stddev":0.4866324939839305,"median_score":0.04515080526471138,"q1_score":0.026253627613186836,"q3_score":1,"avg_token_efficiency":0.99468493997074,"is_incomplete":false,"total_prompt_cost_bin":0.013400999999999996,"total_cache_read_cost_bin":0.23866675199999984,"total_completion_cost_bin":0.12209543999999994,"total_reasoning_cost_bin":0.11417382000000002,"total_cost_bin":0.48833701199999985,"total_on_demand_input_cost_bin":1.2067347600000005,"ci_99_lower":0.34712633494139666,"ci_99_upper":0.5771475037700016,"ci_98_lower":0.356620653727752,"ci_98_upper":0.575123824405945,"ci_95_lower":0.37620585465355033,"ci_95_upper":0.56534110955573,"ci_90_lower":0.39516318482346835,"ci_90_upper":0.5555423947343902,"ci_75_lower":0.41461130210624564,"ci_75_upper":0.5283620813561974,"ci_50_lower":0.43970870565469333,"ci_50_upper":0.50821948866925},"262144":{"avg_score":0.39633958781441897,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4688204261708255,"median_score":0.04479283466935158,"q1_score":0.028678303584456444,"q3_score":1,"avg_token_efficiency":0.9997813486782475,"is_incomplete":false,"total_prompt_cost_bin":0.024882210000000033,"total_cache_read_cost_bin":0.6596351999999995,"total_completion_cost_bin":0.11862558,"total_reasoning_cost_bin":0.10899809999999995,"total_cost_bin":0.9121410899999995,"total_on_demand_input_cost_bin":3.3230582100000006,"ci_99_lower":0.29206817537130036,"ci_99_upper":0.4997533869221542,"ci_98_lower":0.3007594856491365,"ci_98_upper":0.49731677181229444,"ci_95_lower":0.3204205414848336,"ci_95_upper":0.47879629318828604,"ci_90_lower":0.3283824580292224,"ci_90_upper":0.4651509598533273,"ci_75_lower":0.34928173208625735,"ci_75_upper":0.44096121306247105,"ci_50_lower":0.3704155014558689,"ci_50_upper":0.42395368866136973},"524288":{"avg_score":0.28103807611648546,"min_score":0.0029607696924358606,"max_score":1,"n_tests":235,"stddev":0.4213417630159445,"median_score":0.034629981964826584,"q1_score":0.02154580783098936,"q3_score":0.9703054428100586,"avg_token_efficiency":0.9951262204179429,"is_incomplete":false,"total_prompt_cost_bin":0.0495238500000001,"total_cache_read_cost_bin":2.196440064000004,"total_completion_cost_bin":0.13015584,"total_reasoning_cost_bin":0.11458583999999991,"total_cost_bin":2.4907055940000036,"total_on_demand_input_cost_bin":11.031724169999979,"ci_99_lower":0.21067825869245432,"ci_99_upper":0.35079989539625295,"ci_98_lower":0.21906772504799424,"ci_98_upper":0.3415240952044408,"ci_95_lower":0.228907648626705,"ci_95_upper":0.3342392514949971,"ci_90_lower":0.23775004070341352,"ci_90_upper":0.326710392052278,"ci_75_lower":0.25030516032897376,"ci_75_upper":0.3126892591702772,"ci_50_lower":0.26296328331759955,"ci_50_upper":0.30035882585810775}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1310720,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Flash 0731","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":6e-8,"completion":1.2e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-31T06:21:48.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-02T01:06:34.217Z"},{"model_slug":"qwen/qwen3.7-flash","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.2868909957129761,"auc_128k":0.7418999159301672,"cum_avg_1m":0.5876748390233342,"cum_avg_128k":0.8115287530649431,"avg_score_overall":0.5813151806845512,"avg_response_time_ms":30899.707528605002,"total_prompt_cost":33.36913968,"total_cache_read_cost":0.02133208,"total_completion_cost":0.45688552000000004,"total_reasoning_cost":0.4133272000000001,"total_prompt_cost_128k":0.7504705199999997,"total_completion_cost_128k":0.2640027,"total_on_demand_input_cost":33.475802120000004,"total_on_demand_input_cost_128k":0.7504705199999997,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0197607994354971},"bin_metrics":{"8192":{"avg_score":0.9755576208233834,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.15364665606030914,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222286,"is_incomplete":false,"total_prompt_cost_bin":0.016301760000000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.03944901999999999,"total_reasoning_cost_bin":0.03569461999999999,"total_cost_bin":0.09144539999999998,"total_on_demand_input_cost_bin":0.016301760000000002,"ci_99_lower":0.9261124535426497,"ci_99_upper":1,"ci_98_lower":0.9272193308770657,"ci_98_upper":1,"ci_95_lower":0.9386152416467667,"ci_95_upper":1,"ci_90_lower":0.9397304832935334,"ci_90_upper":1,"ci_75_lower":0.95167286247015,"ci_75_upper":1,"ci_50_lower":0.9636152416467667,"ci_50_upper":0.9880576208233833},"16384":{"avg_score":0.9139740278245881,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.2795138333717257,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0159704488335684,"is_incomplete":false,"total_prompt_cost_bin":0.03362112000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.043371119999999985,"total_reasoning_cost_bin":0.039666639999999996,"total_cost_bin":0.11665887999999999,"total_on_demand_input_cost_bin":0.03362112000000001,"ci_99_lower":0.8162952242875472,"ci_99_upper":0.9874426151961089,"ci_98_lower":0.8277181501216256,"ci_98_upper":0.9757269369056447,"ci_95_lower":0.8406447688775369,"ci_95_upper":0.9749457830004393,"ci_90_lower":0.8528019813611173,"ci_90_upper":0.9630715578363742,"ci_75_lower":0.8769262354675447,"ci_75_upper":0.9507053678738885,"ci_50_lower":0.8897724205919075,"ci_50_upper":0.9382472280994988},"32768":{"avg_score":0.8506942489526838,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.35066501389070975,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0256830410679836,"is_incomplete":false,"total_prompt_cost_bin":0.11929823999999972,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08004281999999997,"total_reasoning_cost_bin":0.07284836,"total_cost_bin":0.2721894199999997,"total_on_demand_input_cost_bin":0.11929823999999972,"ci_99_lower":0.7738191611455371,"ci_99_upper":0.9221328128917101,"ci_98_lower":0.7791938199012143,"ci_98_upper":0.9156812970534391,"ci_95_lower":0.7874482932824106,"ci_95_upper":0.908036074874861,"ci_90_lower":0.7953526575279231,"ci_90_upper":0.9004189486789982,"ci_75_lower":0.814357816676099,"ci_75_upper":0.8858246583429933,"ci_50_lower":0.8290208408186172,"ci_50_upper":0.8718343885069343},"65536":{"avg_score":0.7240023345248225,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4411477478435282,"median_score":1,"q1_score":0.04207920655608177,"q3_score":1,"avg_token_efficiency":1.0209598255175436,"is_incomplete":false,"total_prompt_cost_bin":0.1702951799999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.04609813000000001,"total_reasoning_cost_bin":0.04145063000000001,"total_cost_bin":0.25784393999999994,"total_on_demand_input_cost_bin":0.1702951799999999,"ci_99_lower":0.5873026915781528,"ci_99_upper":0.8395126426316907,"ci_98_lower":0.6092486026548946,"ci_98_upper":0.8273935649955974,"ci_95_lower":0.6208794488783871,"ci_95_upper":0.8153236733261934,"ci_90_lower":0.6428185741523937,"ci_90_upper":0.7951131576705063,"ci_75_lower":0.666823484161047,"ci_75_upper":0.7803110619198859,"ci_50_lower":0.6898859953302342,"ci_50_upper":0.757107059126172},"131072":{"avg_score":0.5934155331992372,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.48267829267273665,"median_score":1,"q1_score":0.027350514195859432,"q3_score":1,"avg_token_efficiency":1.0162215936488592,"is_incomplete":false,"total_prompt_cost_bin":0.41095422,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.05504161000000002,"total_reasoning_cost_bin":0.04941898000000002,"total_cost_bin":0.51541481,"total_on_demand_input_cost_bin":0.41095422,"ci_99_lower":0.4706019052582481,"ci_99_upper":0.7152598960940767,"ci_98_lower":0.4872519759345773,"ci_98_upper":0.705789537317124,"ci_95_lower":0.5068252644987632,"ci_95_upper":0.6875693290430523,"ci_90_lower":0.5170398129028205,"ci_90_upper":0.6768113072604411,"ci_75_lower":0.5365806041772532,"ci_75_upper":0.6503430972897529,"ci_50_lower":0.5639254989343225,"ci_50_upper":0.6296994165767174},"262144":{"avg_score":0.4308709999321834,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.47735853820408447,"median_score":0.05071248859167099,"q1_score":0.02633085288107395,"q3_score":1,"avg_token_efficiency":1.018066042397412,"is_incomplete":false,"total_prompt_cost_bin":7.412965559999994,"total_cache_read_cost_bin":0.02133208,"total_completion_cost_bin":0.07206264000000003,"total_reasoning_cost_bin":0.06481722000000002,"total_cost_bin":7.571177499999994,"total_on_demand_input_cost_bin":7.519627999999993,"ci_99_lower":0.3141954095336668,"ci_99_upper":0.5389991994575657,"ci_98_lower":0.3356051769152495,"ci_98_upper":0.5331588228702122,"ci_95_lower":0.3493521250333598,"ci_95_upper":0.5133137291497121,"ci_90_lower":0.36715585048383115,"ci_90_upper":0.4985971753733193,"ci_75_lower":0.3834397206735196,"ci_75_upper":0.47902985342533877,"ci_50_lower":0.4030689478055937,"ci_50_upper":0.4578710937961698},"524288":{"avg_score":0.21288394692977416,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.3750206402977327,"median_score":0.03221469558775425,"q1_score":0.02026958903297782,"q3_score":0.07411767914891243,"avg_token_efficiency":1.018830575861842,"is_incomplete":false,"total_prompt_cost_bin":25.20570360000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.12082018000000007,"total_reasoning_cost_bin":0.10943075000000005,"total_cost_bin":25.43595453000001,"total_on_demand_input_cost_bin":25.20570360000001,"ci_99_lower":0.1540529766337161,"ci_99_upper":0.27329302351907,"ci_98_lower":0.15541411252573953,"ci_98_upper":0.26970785021734567,"ci_95_lower":0.16495763330253718,"ci_95_upper":0.2626125560741441,"ci_90_lower":0.17319960285360939,"ci_90_upper":0.25410794256367497,"ci_75_lower":0.1853747378851749,"ci_75_upper":0.24040799210059174,"ci_50_lower":0.1971355249663278,"ci_50_upper":0.22867833666544865}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.7 Flash","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":3e-8,"completion":1.3e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-27T22:16:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-02T00:23:17.010Z"},{"model_slug":"deepseek/deepseek-v4-pro","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.2826442813607889,"auc_128k":0.6823669658051469,"cum_avg_1m":0.5610413992242442,"cum_avg_128k":0.7737276306344697,"avg_score_overall":0.5599068957818191,"avg_response_time_ms":73250.04831118874,"total_prompt_cost":5.9362796999999965,"total_cache_read_cost":0.6161507039999989,"total_completion_cost":2.0976752700000008,"total_reasoning_cost":1.8202226999999995,"total_prompt_cost_128k":0.12313458000000019,"total_completion_cost_128k":0.88762533,"total_on_demand_input_cost":79.87436418000003,"total_on_demand_input_cost_128k":10.506730740000002,"total_runs":852,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971181068932661},"bin_metrics":{"8192":{"avg_score":0.963538823230192,"min_score":0.02207357808947563,"max_score":1,"n_tests":80,"stddev":0.18561621495406772,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9915838193643983,"is_incomplete":false,"total_prompt_cost_bin":0.049934520000000045,"total_cache_read_cost_bin":0.0014954720000000012,"total_completion_cost_bin":0.10353087000000005,"total_reasoning_cost_bin":0.07944666,"total_cost_bin":0.2344075220000001,"total_on_demand_input_cost_bin":0.2293911600000001,"ci_99_lower":0.9026564739323222,"ci_99_upper":0.9999722210988402,"ci_98_lower":0.9029220414282755,"ci_98_upper":0.9999636046662926,"ci_95_lower":0.9149286559445319,"ci_95_upper":0.999949481189251,"ci_90_lower":0.9270942580956034,"ci_90_upper":0.9999142114073037,"ci_75_lower":0.9391464440792333,"ci_75_upper":0.9877429374668282,"ci_50_lower":0.9513286189991048,"ci_50_upper":0.975756258436013},"16384":{"avg_score":0.9613703629240776,"min_score":0.008643042296171188,"max_score":1,"n_tests":78,"stddev":0.19071075805906196,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9891138747762229,"is_incomplete":false,"total_prompt_cost_bin":0.03509145000000006,"total_cache_read_cost_bin":0.0035417120000000002,"total_completion_cost_bin":0.09838046999999998,"total_reasoning_cost_bin":0.07515929999999998,"total_cost_bin":0.21217293200000004,"total_on_demand_input_cost_bin":0.4600968899999997,"ci_99_lower":0.897771868436263,"ci_99_upper":0.9996596285624381,"ci_98_lower":0.8987291325332645,"ci_98_upper":0.9996257798793988,"ci_95_lower":0.9114404308472355,"ci_95_upper":0.9992855128569481,"ci_90_lower":0.9234032983831966,"ci_90_upper":0.9985458061481134,"ci_75_lower":0.9359549054762539,"ci_75_upper":0.9866843808215494,"ci_50_lower":0.9485252559579049,"ci_50_upper":0.9744106924089675},"32768":{"avg_score":0.7939459871609268,"min_score":0.0021052630618214607,"max_score":1,"n_tests":133,"stddev":0.3931141144038136,"median_score":1,"q1_score":0.9996712803840637,"q3_score":1,"avg_token_efficiency":1.010302690531389,"is_incomplete":false,"total_prompt_cost_bin":0.03432237000000008,"total_cache_read_cost_bin":0.013607728000000008,"total_completion_cost_bin":0.26955123000000003,"total_reasoning_cost_bin":0.22350213,"total_cost_bin":0.5409834580000001,"total_on_demand_input_cost_bin":1.667249730000003,"ci_99_lower":0.6949614111755805,"ci_99_upper":0.8796351564679332,"ci_98_lower":0.7029965819542198,"ci_98_upper":0.8741290237116313,"ci_95_lower":0.718649232743616,"ci_95_upper":0.8584399001898693,"ci_90_lower":0.7321102063109418,"ci_90_upper":0.8487781805704047,"ci_75_lower":0.7509441078877728,"ci_75_upper":0.833235052890841,"ci_50_lower":0.7686892237113369,"ci_50_upper":0.8185867673662296},"65536":{"avg_score":0.6653548250823016,"min_score":0.0037678976077586412,"max_score":1,"n_tests":84,"stddev":0.4631164133236847,"median_score":1,"q1_score":0.039862739853560925,"q3_score":1,"avg_token_efficiency":0.9947451654205393,"is_incomplete":false,"total_prompt_cost_bin":0.0015964499999999986,"total_cache_read_cost_bin":0.019799808000000006,"total_completion_cost_bin":0.16873737,"total_reasoning_cost_bin":0.13986033,"total_cost_bin":0.329993958,"total_on_demand_input_cost_bin":2.377573409999997,"ci_99_lower":0.5371701201787662,"ci_99_upper":0.7806959070871207,"ci_98_lower":0.5485943519456555,"ci_98_upper":0.7702382161563618,"ci_95_lower":0.5609118309688971,"ci_95_upper":0.7574053492823927,"ci_90_lower":0.5841697577292854,"ci_90_upper":0.7448920477144253,"ci_75_lower":0.6071949309684681,"ci_75_upper":0.7220599857260129,"ci_50_lower":0.6310535122742987,"ci_50_upper":0.6992887320624765},"131072":{"avg_score":0.4844281547748502,"min_score":0.005760921631008387,"max_score":1,"n_tests":102,"stddev":0.4867631961111715,"median_score":0.04865792207419872,"q1_score":0.026354107074439526,"q3_score":1,"avg_token_efficiency":0.9940811780292832,"is_incomplete":false,"total_prompt_cost_bin":0.0021897900000000027,"total_cache_read_cost_bin":0.048085247999999976,"total_completion_cost_bin":0.24742538999999994,"total_reasoning_cost_bin":0.20987097000000005,"total_cost_bin":0.507571398,"total_on_demand_input_cost_bin":5.772419550000001,"ci_99_lower":0.3617271839595381,"ci_99_upper":0.6069557650040779,"ci_98_lower":0.37099044465564884,"ci_98_upper":0.5981161678626258,"ci_95_lower":0.3892479084054346,"ci_95_upper":0.5769387656789702,"ci_90_lower":0.4062337877452556,"ci_90_upper":0.561485255526963,"ci_75_lower":0.4276565942820847,"ci_75_upper":0.5419689218054854,"ci_50_lower":0.455002741736369,"ci_50_upper":0.51997337948677},"262144":{"avg_score":0.331682164262203,"min_score":0,"max_score":1,"n_tests":139,"stddev":0.4486874480963375,"median_score":0.039540816098451614,"q1_score":0.022072562016546726,"q3_score":0.9999013841152191,"avg_token_efficiency":0.9994797717960258,"is_incomplete":false,"total_prompt_cost_bin":5.808676364999996,"total_cache_read_cost_bin":0.08350143999999994,"total_completion_cost_bin":0.4878812100000002,"total_reasoning_cost_bin":0.44308316999999975,"total_cost_bin":6.823142184999996,"total_on_demand_input_cost_bin":15.828849164999998,"ci_99_lower":0.2443491529776018,"ci_99_upper":0.43762438009917676,"ci_98_lower":0.24718993126416391,"ci_98_upper":0.4361159005662654,"ci_95_lower":0.25854249592872985,"ci_95_upper":0.41505860191570565,"ci_90_lower":0.26929911714015364,"ci_90_upper":0.39722401000695,"ci_75_lower":0.2846869753647291,"ci_75_upper":0.3772419259509511,"ci_50_lower":0.30426462787570324,"ci_50_upper":0.3582123625297401},"524288":{"avg_score":0.28801087635940226,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4258976313112416,"median_score":0.03734792210161686,"q1_score":0.02025854028761387,"q3_score":0.9906888902187347,"avg_token_efficiency":0.9949755047319108,"is_incomplete":false,"total_prompt_cost_bin":0.004468754999999991,"total_cache_read_cost_bin":0.44611929599999894,"total_completion_cost_bin":0.7221687300000005,"total_reasoning_cost_bin":0.6493001399999997,"total_cost_bin":1.8220569209999993,"total_on_demand_input_cost_bin":53.538784275000026,"ci_99_lower":0.21781578373402688,"ci_99_upper":0.35747897217639174,"ci_98_lower":0.22344982373208536,"ci_98_upper":0.3495362547815035,"ci_95_lower":0.23194699004796626,"ci_95_upper":0.3402746760043694,"ci_90_lower":0.24454299391620257,"ci_90_upper":0.33347701764403115,"ci_75_lower":0.2566810948273534,"ci_75_upper":0.32025820787668513,"ci_50_lower":0.27051656617994135,"ci_50_upper":0.307336012532903}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Pro 0423","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":8.7e-7,"completion":0.00000174,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T03:17:59.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-24T17:14:14.237Z"},{"model_slug":"deepseek/deepseek-v4-pro-0813","reasoning_mode":"low","overall_metrics":{"auc_1m":0.27741493264345457,"auc_128k":0.6878641988764467,"cum_avg_1m":0.5713403134463152,"cum_avg_128k":0.7894255387594645,"avg_score_overall":0.5727489043719975,"avg_response_time_ms":36646.76214464921,"total_prompt_cost":0.5408994449999998,"total_cache_read_cost":0.6641259839999987,"total_completion_cost":2.3679955800000005,"total_reasoning_cost":2.084020620000001,"total_prompt_cost_128k":0.17479518000000008,"total_completion_cost_128k":1.1863302599999999,"total_on_demand_input_cost":80.23601752500002,"total_on_demand_input_cost_128k":10.640630700000003,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971342554822208},"bin_metrics":{"8192":{"avg_score":0.999994621425867,"min_score":0.9995697140693665,"max_score":1,"n_tests":80,"stddev":0.00004810742953291828,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.991583819364398,"is_incomplete":false,"total_prompt_cost_bin":0.03139307999999995,"total_cache_read_cost_bin":0.001649984000000002,"total_completion_cost_bin":0.12784910999999996,"total_reasoning_cost_bin":0.10369355999999999,"total_cost_bin":0.2645857339999999,"total_on_demand_input_cost_bin":0.22939116000000015,"ci_99_lower":0.9999784857034684,"ci_99_upper":1,"ci_98_lower":0.9999784857034684,"ci_98_upper":1,"ci_95_lower":0.9999838642776012,"ci_95_upper":1,"ci_90_lower":0.9999838642776012,"ci_90_upper":1,"ci_75_lower":0.9999892428517342,"ci_75_upper":1,"ci_50_lower":0.9999892428517342,"ci_50_upper":1},"16384":{"avg_score":0.9756143150269054,"min_score":0.014419610612094402,"max_score":1,"n_tests":80,"stddev":0.15300720222198613,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9890247337221416,"is_incomplete":false,"total_prompt_cost_bin":0.02306892000000004,"total_cache_read_cost_bin":0.0037556160000000002,"total_completion_cost_bin":0.17424620999999998,"total_reasoning_cost_bin":0.15042126,"total_cost_bin":0.351492006,"total_on_demand_input_cost_bin":0.47374284,"ci_99_lower":0.9265812120127375,"ci_99_upper":0.9999974661245943,"ci_98_lower":0.9271408599370624,"ci_98_upper":0.999997453391552,"ci_95_lower":0.9389043841627426,"ci_95_upper":0.9999812617897987,"ci_90_lower":0.9395576663478278,"ci_90_upper":0.9999761685729027,"ci_75_lower":0.9515636494732462,"ci_75_upper":0.9999062505550682,"ci_50_lower":0.9635534781380557,"ci_50_upper":0.9878967579454184},"32768":{"avg_score":0.8944008112268742,"min_score":0.014577259309589863,"max_score":1,"n_tests":136,"stddev":0.3006979433278657,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0102892651671214,"is_incomplete":false,"total_prompt_cost_bin":0.03165060000000006,"total_cache_read_cost_bin":0.013924640000000009,"total_completion_cost_bin":0.36341030999999985,"total_reasoning_cost_bin":0.31639725000000013,"total_cost_bin":0.7253828,"total_on_demand_input_cost_bin":1.7026074000000038,"ci_99_lower":0.8228931435108569,"ci_99_upper":0.9569679731533601,"ci_98_lower":0.8302076684916392,"ci_98_upper":0.9502422145625357,"ci_95_lower":0.8386542134289972,"ci_95_upper":0.9437411265180665,"ci_90_lower":0.8511367734966745,"ci_90_upper":0.9365561032195246,"ci_75_lower":0.8647794719187968,"ci_75_upper":0.9227687635481262,"ci_50_lower":0.8747530695383081,"ci_50_upper":0.9148973450116019},"65536":{"avg_score":0.6814349908710403,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4568564572871818,"median_score":1,"q1_score":0.039603959769010544,"q3_score":1,"avg_token_efficiency":0.9947451157769154,"is_incomplete":false,"total_prompt_cost_bin":0.029900595000000006,"total_cache_read_cost_bin":0.019799808000000006,"total_completion_cost_bin":0.20393235000000004,"total_reasoning_cost_bin":0.17384253,"total_cost_bin":0.42747528300000004,"total_on_demand_input_cost_bin":2.405877554999997,"ci_99_lower":0.5460976497049936,"ci_99_upper":0.7869432812485405,"ci_98_lower":0.5642009643047828,"ci_98_upper":0.7844304393631789,"ci_95_lower":0.5878155105336405,"ci_95_upper":0.7725556832146557,"ci_90_lower":0.6001353146700079,"ci_90_upper":0.7604967912045472,"ci_75_lower":0.6238187318558203,"ci_75_upper":0.7383471628185361,"ci_50_lower":0.6467426184537437,"ci_50_upper":0.7144012951100354},"131072":{"avg_score":0.3956829552466357,"min_score":0.003470214083790779,"max_score":1,"n_tests":103,"stddev":0.4741408074147034,"median_score":0.03290755674242973,"q1_score":0.024564924649894238,"q3_score":1,"avg_token_efficiency":0.9940812964474945,"is_incomplete":false,"total_prompt_cost_bin":0.05878198500000003,"total_cache_read_cost_bin":0.048085247999999976,"total_completion_cost_bin":0.3168922800000001,"total_reasoning_cost_bin":0.27956058,"total_cost_bin":0.7033200930000001,"total_on_demand_input_cost_bin":5.829011745000002,"ci_99_lower":0.2736614459532412,"ci_99_upper":0.5096685215760728,"ci_98_lower":0.2895270473572416,"ci_98_upper":0.4996996873548905,"ci_95_lower":0.30179500640923773,"ci_95_upper":0.4898133014811392,"ci_90_lower":0.31913519673166996,"ci_90_upper":0.47120677343958506,"ci_75_lower":0.34140710978321187,"ci_75_upper":0.45236977460473166,"ci_50_lower":0.3661477879555534,"ci_50_upper":0.42588000962040523},"262144":{"avg_score":0.3410117183547758,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.4524168710608003,"median_score":0.0400702990591526,"q1_score":0.02379349060356617,"q3_score":1,"avg_token_efficiency":0.9994797316927638,"is_incomplete":false,"total_prompt_cost_bin":0.13479518999999998,"total_cache_read_cost_bin":0.13268172799999978,"total_completion_cost_bin":0.3771067200000003,"total_reasoning_cost_bin":0.3304764600000001,"total_cost_bin":0.9750600980000002,"total_on_demand_input_cost_bin":16.056602549999994,"ci_99_lower":0.24761598105899038,"ci_99_upper":0.43815555997954103,"ci_98_lower":0.25764656128000185,"ci_98_upper":0.43190313469136377,"ci_95_lower":0.26725220988262677,"ci_95_upper":0.4156836645176109,"ci_90_lower":0.2784043563696628,"ci_90_upper":0.4035061483357286,"ci_75_lower":0.29936498161615865,"ci_75_upper":0.38464314963545565,"ci_50_lower":0.31375286040385086,"ci_50_upper":0.36733870687878006},"524288":{"avg_score":0.28258309541842347,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.41907551950040545,"median_score":0.037240808829665184,"q1_score":0.021009267773479223,"q3_score":0.9452862441539764,"avg_token_efficiency":0.9949755047319111,"is_incomplete":false,"total_prompt_cost_bin":0.23130907499999975,"total_cache_read_cost_bin":0.44422895999999895,"total_completion_cost_bin":0.8045586000000002,"total_reasoning_cost_bin":0.7296289800000005,"total_cost_bin":2.2097256149999995,"total_on_demand_input_cost_bin":53.53878427500002,"ci_99_lower":0.21351829338493714,"ci_99_upper":0.3522210208647855,"ci_98_lower":0.2169390520526854,"ci_98_upper":0.34248097610468503,"ci_95_lower":0.23062979115905258,"ci_95_upper":0.3348695003227291,"ci_90_lower":0.2403607991949746,"ci_90_upper":0.3262727088088237,"ci_75_lower":0.2517842709267378,"ci_75_upper":0.3140219552803763,"ci_50_lower":0.2654832231886665,"ci_50_upper":0.30081431860927005}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Pro 0813","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":0.00000132,"completion":0.00000396,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:42:44.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T14:29:39.892Z"},{"model_slug":"qwen/qwen3.6-plus","reasoning_mode":null,"overall_metrics":{"auc_1m":0.2765440425462077,"auc_128k":0.5813697849491529,"cum_avg_1m":0.4712692722702153,"cum_avg_128k":0.6192937722216316,"avg_score_overall":0.47896268790258956,"avg_response_time_ms":17195.910557479947,"total_prompt_cost":220.84604749999986,"total_cache_read_cost":0,"total_completion_cost":0.64933635,"total_reasoning_cost":0,"total_prompt_cost_128k":8.1304119,"total_completion_cost_128k":0.37094849999999996,"total_on_demand_input_cost":220.84604749999986,"total_on_demand_input_cost_128k":8.1304119,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0198220004437082},"bin_metrics":{"8192":{"avg_score":0.7594957993191201,"min_score":0.006942034233361483,"max_score":1,"n_tests":80,"stddev":0.4193424232748701,"median_score":1,"q1_score":0.7712443098425865,"q3_score":1,"avg_token_efficiency":1.0228079512716755,"is_incomplete":false,"total_prompt_cost_bin":0.17665439999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.055588649999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.23224304999999995,"total_on_demand_input_cost_bin":0.17665439999999996,"ci_99_lower":0.6386535257631623,"ci_99_upper":0.8679640182394359,"ci_98_lower":0.648375498608395,"ci_98_upper":0.8562657693841028,"ci_95_lower":0.6650215147141716,"ci_95_upper":0.8442743422708008,"ci_90_lower":0.6847970228805207,"ci_90_upper":0.8318646459165029,"ci_75_lower":0.7098291411974059,"ci_75_upper":0.8084162406608811,"ci_50_lower":0.7245329950936139,"ci_50_upper":0.786233460000949},"16384":{"avg_score":0.6843754859524779,"min_score":0.008738393895328045,"max_score":1,"n_tests":80,"stddev":0.45778070253768016,"median_score":1,"q1_score":0.037593740969896317,"q3_score":1,"avg_token_efficiency":1.0161194758384642,"is_incomplete":false,"total_prompt_cost_bin":0.3642807999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.05477939999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.41906019999999977,"total_on_demand_input_cost_bin":0.3642807999999998,"ci_99_lower":0.5390548683568486,"ci_99_upper":0.8063589165828888,"ci_98_lower":0.5622073733827564,"ci_98_upper":0.7940261001634644,"ci_95_lower":0.5868108991492772,"ci_95_upper":0.7813745187697351,"ci_90_lower":0.5994757487438619,"ci_90_upper":0.757784192148829,"ci_75_lower":0.6241024737944826,"ci_75_upper":0.7440426615386969,"ci_50_lower":0.64823491795687,"ci_50_upper":0.7206061936740297},"32768":{"avg_score":0.5411269867909141,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.48184127939587734,"median_score":1,"q1_score":0.03472774662077427,"q3_score":1,"avg_token_efficiency":1.0257548592579657,"is_incomplete":false,"total_prompt_cost_bin":1.2924860000000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.10672935,"total_reasoning_cost_bin":0,"total_cost_bin":1.3992153500000004,"total_on_demand_input_cost_bin":1.2924860000000005,"ci_99_lower":0.43547333396714427,"ci_99_upper":0.6471557842574069,"ci_98_lower":0.4408295852325596,"ci_98_upper":0.6345986777274212,"ci_95_lower":0.46044134531437203,"ci_95_upper":0.6223284604179378,"ci_90_lower":0.47644244473091985,"ci_90_upper":0.6061388472115675,"ci_75_lower":0.49680756560487416,"ci_75_upper":0.5906717487703323,"ci_50_lower":0.515647194289212,"ci_50_upper":0.5691770351635675},"65536":{"avg_score":0.6224861645468456,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.47678930287438576,"median_score":1,"q1_score":0.029972752556204796,"q3_score":1,"avg_token_efficiency":1.0209904012221072,"is_incomplete":false,"total_prompt_cost_bin":1.8449196999999988,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06770400000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.9126236999999988,"total_on_demand_input_cost_bin":1.8449196999999988,"ci_99_lower":0.48351521580764917,"ci_99_upper":0.7376622951492009,"ci_98_lower":0.5054233011050588,"ci_98_upper":0.7367662201963694,"ci_95_lower":0.5196119570975904,"ci_95_upper":0.7248121718447382,"ci_90_lower":0.5404881668403086,"ci_90_upper":0.7130126390871865,"ci_75_lower":0.5647701307781972,"ci_75_upper":0.6803331521908095,"ci_50_lower":0.587831601051285,"ci_50_upper":0.6566577604738995},"131072":{"avg_score":0.48898442449880025,"min_score":0.006770480889827013,"max_score":1,"n_tests":103,"stddev":0.4892197291093428,"median_score":0.04328072816133499,"q1_score":0.024522414430975914,"q3_score":1,"avg_token_efficiency":1.016236875763371,"is_incomplete":false,"total_prompt_cost_bin":4.452071000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08614709999999996,"total_reasoning_cost_bin":0,"total_cost_bin":4.538218100000001,"total_on_demand_input_cost_bin":4.452071000000001,"ci_99_lower":0.3652352117948478,"ci_99_upper":0.6122304987835752,"ci_98_lower":0.3753142890160071,"ci_98_upper":0.602225404215018,"ci_95_lower":0.39500405484227263,"ci_95_upper":0.5747872030615663,"ci_90_lower":0.412643782598717,"ci_90_upper":0.5729406388223316,"ci_75_lower":0.43251260961756305,"ci_75_upper":0.5459009800552734,"ci_50_lower":0.45989132610384775,"ci_50_upper":0.5259408770305403},"262144":{"avg_score":0.41034266605960035,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.473479696327771,"median_score":0.04856229946017265,"q1_score":0.02633085288107395,"q3_score":1,"avg_token_efficiency":1.0180736782704627,"is_incomplete":false,"total_prompt_cost_bin":48.87794860000003,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.10684050000000005,"total_reasoning_cost_bin":0,"total_cost_bin":48.98478910000003,"total_on_demand_input_cost_bin":48.87794860000003,"ci_99_lower":0.30361269720658285,"ci_99_upper":0.5140668610249008,"ci_98_lower":0.31431690165604603,"ci_98_upper":0.5055063775724712,"ci_95_lower":0.33332992093129815,"ci_95_upper":0.4963925507774986,"ci_90_lower":0.3423533414564003,"ci_90_upper":0.4791607969220899,"ci_75_lower":0.36278366814334445,"ci_75_upper":0.45823395714194326,"ci_50_lower":0.3827315950072668,"ci_50_upper":0.4380950267019629},"524288":{"avg_score":0.2633426509939639,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4080007045066611,"median_score":0.03477182611823082,"q1_score":0.023506958037614822,"q3_score":0.14867932349443436,"avg_token_efficiency":1.018834391569901,"is_incomplete":false,"total_prompt_cost_bin":163.83768699999982,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.17154734999999996,"total_reasoning_cost_bin":0,"total_cost_bin":164.00923434999982,"total_on_demand_input_cost_bin":163.83768699999982,"ci_99_lower":0.19699163034736863,"ci_99_upper":0.3265077441534712,"ci_98_lower":0.2004240306988319,"ci_98_upper":0.32372446688642853,"ci_95_lower":0.21310056633909402,"ci_95_upper":0.3154045854766668,"ci_90_lower":0.22125752001183926,"ci_90_upper":0.307267895360207,"ci_75_lower":0.23280457731916457,"ci_75_upper":0.2943792774183409,"ci_50_lower":0.2463176790082243,"ci_50_upper":0.28126177191315016}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 Plus","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":3.25e-7,"completion":0.00000195,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-02T12:39:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T06:15:25.595Z"},{"model_slug":"anthropic/claude-opus-4.6","reasoning_mode":null,"overall_metrics":{"auc_1m":0.27095099823021906,"auc_128k":0.5522262475393137,"cum_avg_1m":0.4700421802669266,"cum_avg_128k":0.6156760021967631,"avg_score_overall":0.48192821082044673,"avg_response_time_ms":37637.6735136628,"total_prompt_cost":8.242309999999998,"total_cache_read_cost":103.70232049999987,"total_completion_cost":9.626700000000003,"total_reasoning_cost":0,"total_prompt_cost_128k":3.721719999999998,"total_completion_cost_128k":5.734925000000002,"total_on_demand_input_cost":1045.2655150000014,"total_on_demand_input_cost_128k":138.620425,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1311011541866511},"bin_metrics":{"8192":{"avg_score":0.8160142089880538,"min_score":0.006900716107338667,"max_score":1,"n_tests":80,"stddev":0.3635358799358795,"median_score":0.9947320520877838,"q1_score":0.9581753760576248,"q3_score":0.9997385889291763,"avg_token_efficiency":1.1377505627360662,"is_incomplete":false,"total_prompt_cost_bin":1.0640299999999996,"total_cache_read_cost_bin":0.19562100000000002,"total_completion_cost_bin":0.9887250000000002,"total_reasoning_cost_bin":0,"total_cost_bin":2.248376,"total_on_demand_input_cost_bin":3.0202400000000007,"ci_99_lower":0.6993862559412664,"ci_99_upper":0.9197364262784832,"ci_98_lower":0.7136452509539085,"ci_98_upper":0.9083924188172094,"ci_95_lower":0.7294613372405001,"ci_95_upper":0.8906737361580599,"ci_90_lower":0.7447696191971772,"ci_90_upper":0.8804447928306763,"ci_75_lower":0.7666402584785827,"ci_75_upper":0.8599664629044127,"ci_50_lower":0.789723015241907,"ci_50_upper":0.8426841876877005},"16384":{"avg_score":0.6927279114839621,"min_score":0.005360647104680538,"max_score":1,"n_tests":80,"stddev":0.4428577207527429,"median_score":0.990612268447876,"q1_score":0.042129188776016235,"q3_score":0.997787669301033,"avg_token_efficiency":1.1227572802981336,"is_incomplete":false,"total_prompt_cost_bin":0.6137249999999997,"total_cache_read_cost_bin":0.5580355,"total_completion_cost_bin":0.9783999999999998,"total_reasoning_cost_bin":0,"total_cost_bin":2.1501604999999997,"total_on_demand_input_cost_bin":6.194079999999995,"ci_99_lower":0.5574943624599837,"ci_99_upper":0.8063010815763264,"ci_98_lower":0.5682084457194432,"ci_98_upper":0.7962810911975103,"ci_95_lower":0.5957708784652641,"ci_95_upper":0.7832877446120255,"ci_90_lower":0.6097181803092827,"ci_90_upper":0.7701037522614934,"ci_75_lower":0.6353540713709663,"ci_75_upper":0.7453820401176927,"ci_50_lower":0.6591987443360268,"ci_50_upper":0.7248427987156901},"32768":{"avg_score":0.6107851586403901,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.46332336569544236,"median_score":0.9734238386154175,"q1_score":0.03832455072551966,"q3_score":0.9989349842071533,"avg_token_efficiency":1.1449043855358672,"is_incomplete":false,"total_prompt_cost_bin":0.8961799999999991,"total_cache_read_cost_bin":2.1317780000000046,"total_completion_cost_bin":1.6045000000000007,"total_reasoning_cost_bin":0,"total_cost_bin":4.632458000000004,"total_on_demand_input_cost_bin":22.21395999999999,"ci_99_lower":0.5102785251152646,"ci_99_upper":0.7111930757059428,"ci_98_lower":0.5172717763258398,"ci_98_upper":0.7019375768090729,"ci_95_lower":0.5329264076310638,"ci_95_upper":0.6881986918452159,"ci_90_lower":0.5460541000518682,"ci_90_upper":0.6781029199987909,"ci_75_lower":0.5641304102301707,"ci_75_upper":0.6566816647008475,"ci_50_lower":0.5854959701194756,"ci_50_upper":0.635881344712021},"65536":{"avg_score":0.5842641689814627,"min_score":0.00469667324796319,"max_score":1,"n_tests":85,"stddev":0.4803356339665814,"median_score":0.981246829032898,"q1_score":0.03105897828936577,"q3_score":0.9986085295677185,"avg_token_efficiency":1.1315791278603582,"is_incomplete":false,"total_prompt_cost_bin":0.3894899999999994,"total_cache_read_cost_bin":3.106824000000004,"total_completion_cost_bin":0.9573750000000003,"total_reasoning_cost_bin":0,"total_cost_bin":4.453689000000004,"total_on_demand_input_cost_bin":31.457729999999973,"ci_99_lower":0.4365234821046538,"ci_99_upper":0.7088385379385641,"ci_98_lower":0.45845551385155275,"ci_98_upper":0.6990496919570381,"ci_95_lower":0.4899836404516143,"ci_95_upper":0.6873315486933707,"ci_90_lower":0.4963816388688215,"ci_90_upper":0.674706112984294,"ci_75_lower":0.5269529374015025,"ci_75_upper":0.6421431913971901,"ci_50_lower":0.5502826492754085,"ci_50_upper":0.6185454915195484},"131072":{"avg_score":0.3745885628899469,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4618493215262632,"median_score":0.03959965333342552,"q1_score":0.023743238300085068,"q3_score":0.9905925095081329,"avg_token_efficiency":1.1236718471186091,"is_incomplete":false,"total_prompt_cost_bin":0.758295,"total_cache_read_cost_bin":7.4976120000000055,"total_completion_cost_bin":1.2059250000000001,"total_reasoning_cost_bin":0,"total_cost_bin":9.461832000000006,"total_on_demand_input_cost_bin":75.73441500000007,"ci_99_lower":0.2689846849351335,"ci_99_upper":0.493122369047069,"ci_98_lower":0.27091077931247903,"ci_98_upper":0.47713308091284434,"ci_95_lower":0.28192417444627355,"ci_95_upper":0.4636179509917409,"ci_90_lower":0.29799338872408837,"ci_90_upper":0.44888799102995,"ci_75_lower":0.32028834635021114,"ci_75_upper":0.4274284781829354,"ci_50_lower":0.3457376444068041,"ci_50_upper":0.40303893797935975},"262144":{"avg_score":0.40013675567700613,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.466094986493596,"median_score":0.04641350358724594,"q1_score":0.027411891147494316,"q3_score":0.9943060278892517,"avg_token_efficiency":1.1264169418592795,"is_incomplete":false,"total_prompt_cost_bin":1.5066050000000002,"total_cache_read_cost_bin":20.649159999999945,"total_completion_cost_bin":1.5274250000000007,"total_reasoning_cost_bin":0,"total_cost_bin":23.683189999999946,"total_on_demand_input_cost_bin":207.99820499999996,"ci_99_lower":0.2961969066229616,"ci_99_upper":0.5025191355751437,"ci_98_lower":0.3054971064642032,"ci_98_upper":0.4967055794316615,"ci_95_lower":0.32246980186938207,"ci_95_upper":0.4830161963082546,"ci_90_lower":0.3323890823279739,"ci_90_upper":0.47108299765184336,"ci_75_lower":0.3543022125042935,"ci_75_upper":0.44536911428646775,"ci_50_lower":0.37386095386695334,"ci_50_upper":0.4280605579194685},"524288":{"avg_score":0.28182067547459155,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.4154177771388649,"median_score":0.03902701288461685,"q1_score":0.02314022369682789,"q3_score":0.842176765203476,"avg_token_efficiency":1.1295900633711324,"is_incomplete":false,"total_prompt_cost_bin":3.0139850000000004,"total_cache_read_cost_bin":69.56328999999991,"total_completion_cost_bin":2.3643499999999995,"total_reasoning_cost_bin":0,"total_cost_bin":74.94162499999992,"total_on_demand_input_cost_bin":698.6468850000015,"ci_99_lower":0.21352252536097155,"ci_99_upper":0.3506081989999383,"ci_98_lower":0.21709471670188546,"ci_98_upper":0.34074443358489925,"ci_95_lower":0.22964316635793153,"ci_95_upper":0.33330156315937914,"ci_90_lower":0.23902471757580732,"ci_90_upper":0.32538394583724595,"ci_75_lower":0.25125854195641173,"ci_75_upper":0.31293923720289263,"ci_50_lower":0.265203996017374,"ci_50_upper":0.2999233676893124}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.6","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-04T15:30:50.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T01:26:41.555Z"},{"model_slug":"qwen/qwen3.6-plus","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.2660283817427902,"auc_128k":0.7092814701308616,"cum_avg_1m":0.5710502735038412,"cum_avg_128k":0.7970393569647582,"avg_score_overall":0.5609362836198126,"avg_response_time_ms":92451.77942390752,"total_prompt_cost":216.3170317750001,"total_cache_read_cost":1.1437182250000002,"total_completion_cost":7.01830935,"total_reasoning_cost":6.3592776,"total_prompt_cost_128k":8.114366324999999,"total_completion_cost_128k":4.014426,"total_on_demand_input_cost":220.84475270000007,"total_on_demand_input_cost_128k":8.1300973,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0197607994354971},"bin_metrics":{"8192":{"avg_score":0.987702578259632,"min_score":0.01620626077055931,"max_score":1,"n_tests":80,"stddev":0.10999148383778654,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222288,"is_incomplete":false,"total_prompt_cost_bin":0.17243232499999991,"total_cache_read_cost_bin":0.0041700750000000005,"total_completion_cost_bin":0.5985934500000001,"total_reasoning_cost_bin":0.5420843999999998,"total_cost_bin":1.3172802499999998,"total_on_demand_input_cost_bin":0.1766023999999999,"ci_99_lower":0.9508103130385279,"ci_99_upper":1,"ci_98_lower":0.9508103130385279,"ci_98_upper":1,"ci_95_lower":0.963107734778896,"ci_95_upper":1,"ci_90_lower":0.963107734778896,"ci_90_upper":1,"ci_75_lower":0.975405156519264,"ci_75_upper":1,"ci_50_lower":0.975405156519264,"ci_50_upper":1},"16384":{"avg_score":0.9276068823877722,"min_score":0.01199815422296524,"max_score":1,"n_tests":80,"stddev":0.25585656173967625,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0159704488335684,"is_incomplete":false,"total_prompt_cost_bin":0.3606177249999999,"total_cache_read_cost_bin":0.003611075,"total_completion_cost_bin":0.6858988500000001,"total_reasoning_cost_bin":0.6303589500000001,"total_cost_bin":1.6804866,"total_on_demand_input_cost_bin":0.3642287999999999,"ci_99_lower":0.8425244720506017,"ci_99_upper":0.9880985635649414,"ci_98_lower":0.8534531135826837,"ci_98_upper":0.9877909919014201,"ci_95_lower":0.8665590516850351,"ci_95_upper":0.9761482963489835,"ci_90_lower":0.8775109725643415,"ci_90_upper":0.9754409688292072,"ci_75_lower":0.8913590404612478,"ci_75_upper":0.9634593360446161,"ci_50_lower":0.9041037658927963,"ci_50_upper":0.9512194619339425},"32768":{"avg_score":0.8447421783091746,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.35499309830725834,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0256830410679838,"is_incomplete":false,"total_prompt_cost_bin":1.2844477750000005,"total_cache_read_cost_bin":0.007949825,"total_completion_cost_bin":1.18452555,"total_reasoning_cost_bin":1.07695575,"total_cost_bin":3.553878900000001,"total_on_demand_input_cost_bin":1.2923976000000006,"ci_99_lower":0.7667002950840107,"ci_99_upper":0.9215901598985468,"ci_98_lower":0.7725967692251503,"ci_98_upper":0.9100098141462213,"ci_95_lower":0.7809982648988165,"ci_95_upper":0.9017126811390934,"ci_90_lower":0.7892394389469273,"ci_90_upper":0.8940845143352121,"ci_75_lower":0.8074013032438576,"ci_75_upper":0.879626849976322,"ci_50_lower":0.8229508100587682,"ci_50_upper":0.8654508751115987},"65536":{"avg_score":0.6595766604275388,"min_score":0.010600706562399864,"max_score":1,"n_tests":85,"stddev":0.4639118786723146,"median_score":1,"q1_score":0.039948999881744385,"q3_score":1,"avg_token_efficiency":1.0209598255175443,"is_incomplete":false,"total_prompt_cost_bin":1.8448644499999969,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.7158469500000001,"total_reasoning_cost_bin":0.6460076999999995,"total_cost_bin":3.2067190999999964,"total_on_demand_input_cost_bin":1.8448644499999969,"ci_99_lower":0.5240725330529077,"ci_99_upper":0.7727096551445806,"ci_98_lower":0.5385036889251102,"ci_98_upper":0.7711147659233607,"ci_95_lower":0.5587720746127889,"ci_95_upper":0.7516089020310627,"ci_90_lower":0.5788662890700952,"ci_90_upper":0.7391114685180432,"ci_75_lower":0.6013349531163625,"ci_75_upper":0.7167699383421089,"ci_50_lower":0.624891527317574,"ci_50_upper":0.6929833260775707},"131072":{"avg_score":0.5655684854396736,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4859497228442129,"median_score":1,"q1_score":0.02766021154820919,"q3_score":1,"avg_token_efficiency":1.0162215936488588,"is_incomplete":false,"total_prompt_cost_bin":4.452004050000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.8295611999999999,"total_reasoning_cost_bin":0.7428505500000003,"total_cost_bin":6.024415800000002,"total_on_demand_input_cost_bin":4.452004050000002,"ci_99_lower":0.4427783133813352,"ci_99_upper":0.6972475757841043,"ci_98_lower":0.45236108024862315,"ci_98_upper":0.6699467810352776,"ci_95_lower":0.4714832316526863,"ci_95_upper":0.6601279997715536,"ci_90_lower":0.4897255295512348,"ci_90_upper":0.6427282384045202,"ci_75_lower":0.5093719263875278,"ci_75_upper":0.6224099630778158,"ci_50_lower":0.529958133268152,"ci_50_upper":0.6031182679360126},"262144":{"avg_score":0.39050103366995853,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.46786446798560605,"median_score":0.04574209079146385,"q1_score":0.02805999107658863,"q3_score":1,"avg_token_efficiency":1.0180660423974117,"is_incomplete":false,"total_prompt_cost_bin":48.53093765000003,"total_cache_read_cost_bin":0.086659625,"total_completion_cost_bin":1.0932928500000003,"total_reasoning_cost_bin":0.9855553499999996,"total_cost_bin":50.696445475000026,"total_on_demand_input_cost_bin":48.87758200000003,"ci_99_lower":0.29172363840956705,"ci_99_upper":0.4942035806115802,"ci_98_lower":0.29980044389001576,"ci_98_upper":0.4869662241650587,"ci_95_lower":0.3144143576314342,"ci_95_upper":0.4738108327228358,"ci_90_lower":0.322435906120074,"ci_90_upper":0.4600648529492385,"ci_75_lower":0.3453992138994884,"ci_75_upper":0.43676733379318305,"ci_50_lower":0.3646926808605261,"ci_50_upper":0.4180860443342913},"524288":{"avg_score":0.19270436953697986,"min_score":0.002560819499194622,"max_score":1,"n_tests":236,"stddev":0.3551995326445792,"median_score":0.03223351761698723,"q1_score":0.02178717590868473,"q3_score":0.06926291063427925,"avg_token_efficiency":1.0188305758618423,"is_incomplete":false,"total_prompt_cost_bin":159.67172780000007,"total_cache_read_cost_bin":1.041327625,"total_completion_cost_bin":1.9105904999999996,"total_reasoning_cost_bin":1.7354649000000006,"total_cost_bin":164.3591108250001,"total_on_demand_input_cost_bin":163.83707340000004,"ci_99_lower":0.13915350576442906,"ci_99_upper":0.25085994106466325,"ci_98_lower":0.14146124509893265,"ci_98_upper":0.24571581381482616,"ci_95_lower":0.1485640216119325,"ci_95_upper":0.23917851783914817,"ci_90_lower":0.15435801796453327,"ci_90_upper":0.2305922746875864,"ci_75_lower":0.16795858707522157,"ci_75_upper":0.21974792754424585,"ci_50_lower":0.17629712118897428,"ci_50_upper":0.20722479262400095}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 Plus","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":3.25e-7,"completion":0.00000195,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-02T12:39:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-22T19:09:01.245Z"},{"model_slug":"moonshotai/kimi-k3","reasoning_mode":"max","overall_metrics":{"auc_1m":0.26119261434764457,"auc_128k":0.6395104484035606,"cum_avg_1m":0.5335751319265754,"cum_avg_128k":0.7334475842857517,"avg_score_overall":0.5297603328565177,"avg_response_time_ms":154168.13693652002,"total_prompt_cost":4.5791369999999985,"total_cache_read_cost":54.68674559999997,"total_completion_cost":81.42027,"total_reasoning_cost":76.416405,"total_prompt_cost_128k":1.2593639999999997,"total_completion_cost_128k":36.453705,"total_on_demand_input_cost":551.4465929999993,"total_on_demand_input_cost_128k":73.25936399999995,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9960202770664935},"bin_metrics":{"8192":{"avg_score":0.9750987019855529,"min_score":0.020156774669885635,"max_score":1,"n_tests":80,"stddev":0.15049570817015434,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0022382136202694,"is_incomplete":false,"total_prompt_cost_bin":0.1774079999999999,"total_cache_read_cost_bin":0.14215679999999992,"total_completion_cost_bin":1.7345250000000005,"total_reasoning_cost_bin":1.3014450000000002,"total_cost_bin":3.3555348000000005,"total_on_demand_input_cost_bin":1.5989759999999995,"ci_99_lower":0.9260008167475462,"ci_99_upper":0.9996934165023268,"ci_98_lower":0.9271638260846957,"ci_98_upper":0.9996289953514933,"ci_95_lower":0.9386733047489543,"ci_95_upper":0.9995038189366461,"ci_90_lower":0.9403308960259892,"ci_90_upper":0.9992585999891163,"ci_75_lower":0.9518336136359722,"ci_75_upper":0.9980054125189781,"ci_50_lower":0.9633152441470884,"ci_50_upper":0.9871161396964452},"16384":{"avg_score":0.898998312902404,"min_score":0.005645730532705784,"max_score":1,"n_tests":80,"stddev":0.29479671316796396,"median_score":1,"q1_score":0.9976973980665207,"q3_score":1,"avg_token_efficiency":0.9862665450897008,"is_incomplete":false,"total_prompt_cost_bin":0.22339200000000017,"total_cache_read_cost_bin":0.3036671999999996,"total_completion_cost_bin":3.6418649999999997,"total_reasoning_cost_bin":3.2206049999999995,"total_cost_bin":7.389529199999998,"total_on_demand_input_cost_bin":3.260063999999997,"ci_99_lower":0.8118080630023615,"ci_99_upper":0.9732884101537639,"ci_98_lower":0.812693431428459,"ci_98_upper":0.9612643013489433,"ci_95_lower":0.8259153198420244,"ci_95_upper":0.9580763081145415,"ci_90_lower":0.837550294715329,"ci_90_upper":0.9480298697971739,"ci_75_lower":0.861084574679262,"ci_75_upper":0.9351725932421687,"ci_50_lower":0.8751138823587097,"ci_50_upper":0.9227039328674438},"32768":{"avg_score":0.7454364945639528,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.4201712733156234,"median_score":1,"q1_score":0.16150888428092003,"q3_score":1,"avg_token_efficiency":1.012972494300967,"is_incomplete":false,"total_prompt_cost_bin":0.20205599999999985,"total_cache_read_cost_bin":1.1582207999999994,"total_completion_cost_bin":9.272219999999995,"total_reasoning_cost_bin":8.439824999999997,"total_cost_bin":19.07232179999999,"total_on_demand_input_cost_bin":11.784264000000002,"ci_99_lower":0.6525137128235826,"ci_99_upper":0.8363085430537296,"ci_98_lower":0.6605791031416534,"ci_98_upper":0.8268795290955697,"ci_95_lower":0.6754029955795405,"ci_95_upper":0.8098828123559703,"ci_90_lower":0.684170349465464,"ci_90_upper":0.8028516890362988,"ci_75_lower":0.7033170745100292,"ci_75_upper":0.7865894917464602,"ci_50_lower":0.7191232342139462,"ci_50_upper":0.7694595946480096},"65536":{"avg_score":0.6647413870538859,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4617204401112394,"median_score":0.9995623826980591,"q1_score":0.034629981964826584,"q3_score":1,"avg_token_efficiency":0.9919594891780263,"is_incomplete":false,"total_prompt_cost_bin":0.22425899999999988,"total_cache_read_cost_bin":1.6321536000000014,"total_completion_cost_bin":7.730805,"total_reasoning_cost_bin":7.201845000000003,"total_cost_bin":16.789062600000005,"total_on_demand_input_cost_bin":16.54579499999999,"ci_99_lower":0.527749657977208,"ci_99_upper":0.7783050909054959,"ci_98_lower":0.548955961557653,"ci_98_upper":0.7694702076119754,"ci_95_lower":0.568526710773106,"ci_95_upper":0.7571268426290001,"ci_90_lower":0.5838158160519293,"ci_90_upper":0.744272741759196,"ci_75_lower":0.6075580786174054,"ci_75_upper":0.7230277691666475,"ci_50_lower":0.6301889463738702,"ci_50_upper":0.6984324040877469},"131072":{"avg_score":0.382963024922963,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.46814689754087596,"median_score":0.03654765337705612,"q1_score":0.023328067734837532,"q3_score":0.9995285272598267,"avg_token_efficiency":0.9908711621601468,"is_incomplete":false,"total_prompt_cost_bin":0.43224899999999994,"total_cache_read_cost_bin":3.9638016000000054,"total_completion_cost_bin":14.074290000000005,"total_reasoning_cost_bin":13.43157,"total_cost_bin":31.90191060000001,"total_on_demand_input_cost_bin":40.070264999999964,"ci_99_lower":0.2700207383698418,"ci_99_upper":0.5022003287549845,"ci_98_lower":0.2783324344443085,"ci_98_upper":0.4889461041261551,"ci_95_lower":0.2901202603200907,"ci_95_upper":0.47442773904457,"ci_90_lower":0.3073530535982217,"ci_90_upper":0.4588054586472034,"ci_75_lower":0.3281658835682534,"ci_75_upper":0.43962266041532166,"ci_50_lower":0.3531981846978843,"ci_50_upper":0.41247941858815573},"262144":{"avg_score":0.34793038893794864,"min_score":0.0063157896511256695,"max_score":1,"n_tests":141,"stddev":0.4520799396620754,"median_score":0.03854239732027054,"q1_score":0.026058631017804146,"q3_score":0.9995007514953613,"avg_token_efficiency":0.995854506186401,"is_incomplete":false,"total_prompt_cost_bin":1.6600049999999986,"total_cache_read_cost_bin":10.86735359999999,"total_completion_cost_bin":19.317600000000006,"total_reasoning_cost_bin":18.499095,"total_cost_bin":50.344053599999995,"total_on_demand_input_cost_bin":110.33354099999995,"ci_99_lower":0.2589610700104353,"ci_99_upper":0.44211162365469353,"ci_98_lower":0.26504532911309453,"ci_98_upper":0.43798509898096827,"ci_95_lower":0.27272826445095755,"ci_95_upper":0.423913900854383,"ci_90_lower":0.28506463383050334,"ci_90_upper":0.412012521004476,"ci_75_lower":0.30560653407047406,"ci_75_upper":0.3917213431675093,"ci_50_lower":0.32133765685527277,"ci_50_upper":0.37426705462759},"524288":{"avg_score":0.25343274504589547,"min_score":0.003484320593997836,"max_score":1,"n_tests":236,"stddev":0.4030913811992304,"median_score":0.03430435620248318,"q1_score":0.021642340812832117,"q3_score":0.1549982689321041,"avg_token_efficiency":0.9912586736339484,"is_incomplete":false,"total_prompt_cost_bin":1.6597680000000001,"total_cache_read_cost_bin":36.619391999999976,"total_completion_cost_bin":25.648965,"total_reasoning_cost_bin":24.32202,"total_cost_bin":88.25014499999997,"total_on_demand_input_cost_bin":367.8536879999994,"ci_99_lower":0.18780099858325155,"ci_99_upper":0.31867303952781906,"ci_98_lower":0.19042415509735416,"ci_98_upper":0.3131398225355384,"ci_95_lower":0.20296610827943493,"ci_95_upper":0.3063271025643128,"ci_90_lower":0.2117321147692441,"ci_90_upper":0.29888459459584504,"ci_75_lower":0.22403142045467095,"ci_75_upper":0.28314836266175825,"ci_50_lower":0.23628035742099895,"ci_50_upper":0.2707440668627742}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"MoonshotAI: Kimi K3","provider_name":"Moonshotai","run_provider_names":["Moonshot AI"],"pricing_metadata":{"prompt":0.000003,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:30:58.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-20T22:28:21.060Z"},{"model_slug":"qwen/qwen3.7-flash","reasoning_mode":null,"overall_metrics":{"auc_1m":0.26016065393683724,"auc_128k":0.5254061286556339,"cum_avg_1m":0.451291640024241,"cum_avg_128k":0.5944388326220137,"avg_score_overall":0.4574798042766975,"avg_response_time_ms":7858.2464805186555,"total_prompt_cost":33.475981960000006,"total_cache_read_cost":0,"total_completion_cost":0.04298319999999999,"total_reasoning_cost":0,"total_prompt_cost_128k":0.75049956,"total_completion_cost_128k":0.024734449999999998,"total_on_demand_input_cost":33.475981960000006,"total_on_demand_input_cost_128k":0.75049956,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0198220004437082},"bin_metrics":{"8192":{"avg_score":0.7952705209376291,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.39668105662984265,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0228079512716761,"is_incomplete":false,"total_prompt_cost_bin":0.01630656000000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0037790999999999988,"total_reasoning_cost_bin":0,"total_cost_bin":0.02008566000000002,"total_on_demand_input_cost_bin":0.01630656000000002,"ci_99_lower":0.6740702070661355,"ci_99_upper":0.8935414223128465,"ci_98_lower":0.6850557989659719,"ci_98_upper":0.8918318903702311,"ci_95_lower":0.6984733258077176,"ci_95_upper":0.8801817202667007,"ci_90_lower":0.7223106116382406,"ci_90_upper":0.8670248175261077,"ci_75_lower":0.7464268071373226,"ci_75_upper":0.8434075818659039,"ci_50_lower":0.7668292791291605,"ci_50_upper":0.8213751389936079},"16384":{"avg_score":0.6603201462537982,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4658928733275914,"median_score":1,"q1_score":0.03873698692768812,"q3_score":1,"avg_token_efficiency":1.0161194758384635,"is_incomplete":false,"total_prompt_cost_bin":0.033625919999999976,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0036680799999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.03729399999999997,"total_on_demand_input_cost_bin":0.033625919999999976,"ci_99_lower":0.514927210863796,"ci_99_upper":0.7829401649338542,"ci_98_lower":0.5382296060851076,"ci_98_upper":0.7704215922877191,"ci_95_lower":0.5625513320302706,"ci_95_upper":0.7568625413364498,"ci_90_lower":0.5751044174999697,"ci_90_upper":0.7441089284868213,"ci_75_lower":0.599578367594222,"ci_75_upper":0.7205308420190704,"ci_50_lower":0.6236126086034347,"ci_50_upper":0.6965201654675183},"32768":{"avg_score":0.5230811189069167,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.47949803766923976,"median_score":0.6080613285303116,"q1_score":0.03860486764460802,"q3_score":1,"avg_token_efficiency":1.0257548592579657,"is_incomplete":false,"total_prompt_cost_bin":0.11930639999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.007177040000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.12648344,"total_on_demand_input_cost_bin":0.11930639999999999,"ci_99_lower":0.4185022198909599,"ci_99_upper":0.6285293052586648,"ci_98_lower":0.42835600175632865,"ci_98_upper":0.620805692768892,"ci_95_lower":0.4430820093335677,"ci_95_upper":0.599217221152503,"ci_90_lower":0.4566511612833197,"ci_90_upper":0.5882919056911144,"ci_75_lower":0.4752181689997515,"ci_75_upper":0.5714237589743468,"ci_50_lower":0.49596590543788965,"ci_50_upper":0.5512423181956847},"65536":{"avg_score":0.47482179260867485,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4867888433922377,"median_score":0.06127138063311577,"q1_score":0.02454473450779915,"q3_score":1,"avg_token_efficiency":1.0209904012221076,"is_incomplete":false,"total_prompt_cost_bin":0.17030028000000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0045136,"total_reasoning_cost_bin":0,"total_cost_bin":0.17481388000000006,"total_on_demand_input_cost_bin":0.17030028000000005,"ci_99_lower":0.33690215601273543,"ci_99_upper":0.6105924943941371,"ci_98_lower":0.3494267288791563,"ci_98_upper":0.5990134001162779,"ci_95_lower":0.36853544582585424,"ci_95_upper":0.5809664261146613,"ci_90_lower":0.3841902240316438,"ci_90_upper":0.5663052849761923,"ci_75_lower":0.414985370697618,"ci_75_upper":0.5410005744984921,"ci_50_lower":0.44030827625709423,"ci_50_upper":0.5088940585634726},"131072":{"avg_score":0.5187005844030493,"min_score":0.00709534389898181,"max_score":1,"n_tests":103,"stddev":0.4884150896334104,"median_score":0.9952413439750671,"q1_score":0.025608915835618973,"q3_score":1,"avg_token_efficiency":1.0162368757633715,"is_incomplete":false,"total_prompt_cost_bin":0.4109603999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.00559663,"total_reasoning_cost_bin":0,"total_cost_bin":0.4165570299999999,"total_on_demand_input_cost_bin":0.4109603999999999,"ci_99_lower":0.385486532928517,"ci_99_upper":0.6416930933116068,"ci_98_lower":0.40519060176806254,"ci_98_upper":0.6312621321737567,"ci_95_lower":0.42591266213810003,"ci_95_upper":0.6120445084910686,"ci_90_lower":0.44221347022950835,"ci_90_upper":0.6020880590464233,"ci_75_lower":0.46184865221408317,"ci_75_upper":0.5757842626274582,"ci_50_lower":0.48866331665460205,"ci_50_upper":0.5552029856548473},"262144":{"avg_score":0.40087347896274256,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4738886212619655,"median_score":0.03993154689669609,"q1_score":0.023225806653499603,"q3_score":1,"avg_token_efficiency":1.0180736782704627,"is_incomplete":false,"total_prompt_cost_bin":7.5196844,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.007054709999999997,"total_reasoning_cost_bin":0,"total_cost_bin":7.52673911,"total_on_demand_input_cost_bin":7.5196844,"ci_99_lower":0.2956040484819508,"ci_99_upper":0.5044753871067074,"ci_98_lower":0.30375907429338744,"ci_98_upper":0.49801175856416224,"ci_95_lower":0.3218198947240235,"ci_95_upper":0.48509059416023675,"ci_90_lower":0.3319915322509284,"ci_90_upper":0.4729285323731468,"ci_75_lower":0.3535004344980186,"ci_75_upper":0.4481722260693234,"ci_50_lower":0.3740669848832487,"ci_50_upper":0.4285662077286393},"524288":{"avg_score":0.23726547812111676,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.39188955099080525,"median_score":0.036425692960619926,"q1_score":0.0222817687317729,"q3_score":0.1090652234852314,"avg_token_efficiency":1.018834391569901,"is_incomplete":false,"total_prompt_cost_bin":25.205798000000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.011194039999999995,"total_reasoning_cost_bin":0,"total_cost_bin":25.216992040000004,"total_on_demand_input_cost_bin":25.205798000000005,"ci_99_lower":0.17422862762528063,"ci_99_upper":0.30873656791877063,"ci_98_lower":0.17698294658864003,"ci_98_upper":0.29762068632465266,"ci_95_lower":0.18770053005220874,"ci_95_upper":0.2867795439457503,"ci_90_lower":0.19860841966982956,"ci_90_upper":0.28095826058608075,"ci_75_lower":0.20747237629047496,"ci_75_upper":0.26630858599538304,"ci_50_lower":0.22081235448447042,"ci_50_upper":0.2535340656593967}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.7 Flash","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":3e-8,"completion":1.3e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-27T22:16:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-02T01:33:52.038Z"},{"model_slug":"qwen/qwen3.7-max","reasoning_mode":null,"overall_metrics":{"auc_1m":0.25965743945683434,"auc_128k":0.5362512019225546,"cum_avg_1m":0.43546561612087287,"cum_avg_128k":0.562116632599102,"avg_score_overall":0.45021204208159055,"avg_response_time_ms":7109.69461963401,"total_prompt_cost":278.2499944000001,"total_cache_read_cost":0,"total_completion_cost":1.4841627,"total_reasoning_cost":0,"total_prompt_cost_128k":36.89956170000001,"total_completion_cost_128k":0.8486840249999998,"total_on_demand_input_cost":278.2499944000001,"total_on_demand_input_cost_128k":36.89956170000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0198220004437082},"bin_metrics":{"8192":{"avg_score":0.6612023613473866,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.46488552721000664,"median_score":1,"q1_score":0.03415457718074322,"q3_score":1,"avg_token_efficiency":1.0228079512716763,"is_incomplete":false,"total_prompt_cost_bin":0.8017392000000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.12755062499999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.9292898250000001,"total_on_demand_input_cost_bin":0.8017392000000002,"ci_99_lower":0.5153970657763711,"ci_99_upper":0.7837649510337504,"ci_98_lower":0.538449706494459,"ci_98_upper":0.7714148461135919,"ci_95_lower":0.5621420071809552,"ci_95_upper":0.7572723913914524,"ci_90_lower":0.5759275891919969,"ci_90_upper":0.7440521498527959,"ci_75_lower":0.6007360534109466,"ci_75_upper":0.7205870938581939,"ci_50_lower":0.624245908537705,"ci_50_upper":0.6969839270968805},"16384":{"avg_score":0.5643234518938698,"min_score":0.008738393895328045,"max_score":1,"n_tests":80,"stddev":0.48479383754031374,"median_score":1,"q1_score":0.03129185410216451,"q3_score":1,"avg_token_efficiency":1.0161194758384635,"is_incomplete":false,"total_prompt_cost_bin":1.6532744,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.1251921,"total_reasoning_cost_bin":0,"total_cost_bin":1.7784665,"total_on_demand_input_cost_bin":1.6532744,"ci_99_lower":0.42936785260279425,"ci_99_upper":0.7087877942146733,"ci_98_lower":0.43151969959808045,"ci_98_upper":0.6967224601430352,"ci_95_lower":0.4580861349243787,"ci_95_upper":0.6723179046341102,"ci_90_lower":0.4785205937514547,"ci_90_upper":0.6497909469268052,"ci_75_lower":0.5037740644271254,"ci_75_upper":0.6257782256274368,"ci_50_lower":0.5278490921220509,"ci_50_upper":0.6007367294427239},"32768":{"avg_score":0.6051621019449906,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.47407677160350564,"median_score":1,"q1_score":0.03911831323057413,"q3_score":1,"avg_token_efficiency":1.0257548592579655,"is_incomplete":false,"total_prompt_cost_bin":5.865898000000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.24391485000000002,"total_reasoning_cost_bin":0,"total_cost_bin":6.109812850000005,"total_on_demand_input_cost_bin":5.865898000000005,"ci_99_lower":0.49247772835225884,"ci_99_upper":0.7102823421590353,"ci_98_lower":0.5145991130825818,"ci_98_upper":0.6972830835880834,"ci_95_lower":0.5273220202775145,"ci_95_upper":0.6881475740683313,"ci_90_lower":0.5384346967487706,"ci_90_upper":0.6739414541059352,"ci_75_lower":0.5581753846691915,"ci_75_upper":0.6521456662457615,"ci_50_lower":0.5782372473014916,"ci_50_upper":0.6311905015985865},"65536":{"avg_score":0.5658071866258979,"min_score":0.00757862813770771,"max_score":1,"n_tests":85,"stddev":0.48584911382880086,"median_score":1,"q1_score":0.030199915170669556,"q3_score":1,"avg_token_efficiency":1.020990401222107,"is_incomplete":false,"total_prompt_cost_bin":8.3730971,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.15881767499999994,"total_reasoning_cost_bin":0,"total_cost_bin":8.531914775,"total_on_demand_input_cost_bin":8.3730971,"ci_99_lower":0.42709863624299926,"ci_99_upper":0.6919509834471871,"ci_98_lower":0.4407714936985251,"ci_98_upper":0.6904871427874794,"ci_95_lower":0.46448309691951556,"ci_95_upper":0.6690872353131828,"ci_90_lower":0.48400997999507717,"ci_90_upper":0.6571667019606513,"ci_75_lower":0.5077801636543453,"ci_75_upper":0.62560514814823,"ci_50_lower":0.5310228737098549,"ci_50_upper":0.6004468112280045},"131072":{"avg_score":0.4140880611833655,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4785527731134048,"median_score":0.03952569141983986,"q1_score":0.020613654516637325,"q3_score":1,"avg_token_efficiency":1.0162368757633713,"is_incomplete":false,"total_prompt_cost_bin":20.205553000000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.19320877500000003,"total_reasoning_cost_bin":0,"total_cost_bin":20.398761775,"total_on_demand_input_cost_bin":20.205553000000002,"ci_99_lower":0.2896495922731498,"ci_99_upper":0.5266625276343554,"ci_98_lower":0.30180921494137963,"ci_98_upper":0.5180879153199133,"ci_95_lower":0.3193803871461886,"ci_95_upper":0.5071330712370131,"ci_90_lower":0.336844196853463,"ci_90_upper":0.4908279468691953,"ci_75_lower":0.35751665068012706,"ci_75_upper":0.4706767807450164,"ci_50_lower":0.38469998698423896,"ci_50_upper":0.45011450462404967},"262144":{"avg_score":0.4454492697465821,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.4791594281023036,"median_score":0.05163797736167908,"q1_score":0.02887844108045101,"q3_score":1,"avg_token_efficiency":1.0180736782704618,"is_incomplete":false,"total_prompt_cost_bin":55.45767245000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.23978632499999983,"total_reasoning_cost_bin":0,"total_cost_bin":55.697458775000044,"total_on_demand_input_cost_bin":55.45767245000005,"ci_99_lower":0.32822228224508804,"ci_99_upper":0.5568760791134605,"ci_98_lower":0.3446731600289544,"ci_98_upper":0.5470077506978567,"ci_95_lower":0.3681366906534378,"ci_95_upper":0.5281847823510303,"ci_90_lower":0.3770779455493449,"ci_90_upper":0.5138873228793369,"ci_75_lower":0.39797067790166707,"ci_75_upper":0.493534509578046,"ci_50_lower":0.4179169739999458,"ci_50_upper":0.47307027780767924},"524288":{"avg_score":0.22769249622489057,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.3841003483685903,"median_score":0.03381909243762493,"q1_score":0.020432095043361187,"q3_score":0.08762103877961636,"avg_token_efficiency":1.0188343915699012,"is_incomplete":false,"total_prompt_cost_bin":185.89276025000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.39569235000000025,"total_reasoning_cost_bin":0,"total_cost_bin":186.28845260000003,"total_on_demand_input_cost_bin":185.89276025000004,"ci_99_lower":0.16730156719408854,"ci_99_upper":0.2932455529074847,"ci_98_lower":0.1695321976638624,"ci_98_upper":0.2864702554252907,"ci_95_lower":0.1799267220800699,"ci_95_upper":0.2801708458850537,"ci_90_lower":0.18892057066836584,"ci_90_upper":0.26957410697354656,"ci_75_lower":0.19891685322918576,"ci_75_upper":0.255808148805276,"ci_50_lower":0.21152115289010384,"ci_50_upper":0.24347404919794596}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.7 Max","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":0.000001475,"completion":0.000004425,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-21T15:21:01.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T13:23:14.128Z"},{"model_slug":"deepseek/deepseek-v4-flash","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.2541180650009069,"auc_128k":0.6254563527462699,"cum_avg_1m":0.515532875134038,"cum_avg_128k":0.712819246239786,"avg_score_overall":0.5092819261691305,"avg_response_time_ms":21650.402923902984,"total_prompt_cost":1.1747174600000017,"total_cache_read_cost":4.929673727999993,"total_completion_cost":0.3470012,"total_reasoning_cost":0.25601912,"total_prompt_cost_128k":0.03674104,"total_completion_cost_128k":0.17602928,"total_on_demand_input_cost":25.823086099999998,"total_on_demand_input_cost_128k":3.4245708000000024,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971342554822209},"bin_metrics":{"8192":{"avg_score":0.9631940849823877,"min_score":0.028362305834889412,"max_score":1,"n_tests":80,"stddev":0.18484616233152568,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.991583819364398,"is_incomplete":false,"total_prompt_cost_bin":0.014153439999999998,"total_cache_read_cost_bin":0.01193472,"total_completion_cost_bin":0.023915639999999995,"total_reasoning_cost_bin":0.016145359999999997,"total_cost_bin":0.06614915999999998,"total_on_demand_input_cost_bin":0.07382703999999997,"ci_99_lower":0.902607443594141,"ci_99_upper":0.9996839203350246,"ci_98_lower":0.9029020745977759,"ci_98_upper":0.9996615053415299,"ci_95_lower":0.9146891530399444,"ci_95_upper":0.9995129360631109,"ci_90_lower":0.9268198269221466,"ci_90_upper":0.9991539649292827,"ci_75_lower":0.9389093791134656,"ci_75_upper":0.9873585547175027,"ci_50_lower":0.9510816797905136,"ci_50_upper":0.9753967952507081},"16384":{"avg_score":0.847864743892569,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.3493742612411855,"median_score":1,"q1_score":0.9955441206693649,"q3_score":1,"avg_token_efficiency":0.9890247337221423,"is_incomplete":false,"total_prompt_cost_bin":0.02045232000000001,"total_cache_read_cost_bin":0.02640332799999998,"total_completion_cost_bin":0.025860800000000003,"total_reasoning_cost_bin":0.01821708,"total_cost_bin":0.09093352799999999,"total_on_demand_input_cost_bin":0.15246896000000001,"ci_99_lower":0.7377415741533042,"ci_99_upper":0.9453503618715331,"ci_98_lower":0.7478288322975858,"ci_98_upper":0.9291727923373692,"ci_95_lower":0.7629069215230992,"ci_95_upper":0.9206396897972444,"ci_90_lower":0.7799958470556885,"ci_90_upper":0.9078184279851848,"ci_75_lower":0.8004433218637131,"ci_75_upper":0.8899708023323911,"ci_50_lower":0.8233377782744355,"ci_50_upper":0.8724019395711365},"32768":{"avg_score":0.6708694771343076,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.4584438791333291,"median_score":0.9996112883090973,"q1_score":0.035478307865560055,"q3_score":1,"avg_token_efficiency":1.0102892651671211,"is_incomplete":false,"total_prompt_cost_bin":0.000903839999999999,"total_cache_read_cost_bin":0.10941235199999975,"total_completion_cost_bin":0.05486124000000001,"total_reasoning_cost_bin":0.03994228,"total_cost_bin":0.20511971199999976,"total_on_demand_input_cost_bin":0.5479656000000003,"ci_99_lower":0.5688948871743152,"ci_99_upper":0.7690521196370108,"ci_98_lower":0.577312814021579,"ci_98_upper":0.7565830078466986,"ci_95_lower":0.5864661170831396,"ci_95_upper":0.7446011771843426,"ci_90_lower":0.6072269057072631,"ci_90_upper":0.7347603612332878,"ci_75_lower":0.6264204531963675,"ci_75_upper":0.7145103185916121,"ci_50_lower":0.6440333174164955,"ci_50_upper":0.6945416613041957},"65536":{"avg_score":0.6435855003512081,"min_score":0.008358662948012352,"max_score":1,"n_tests":85,"stddev":0.46988928593157553,"median_score":0.9992172122001648,"q1_score":0.03000682033598423,"q3_score":1,"avg_token_efficiency":0.994745115776915,"is_incomplete":false,"total_prompt_cost_bin":0.0005198200000000006,"total_cache_read_cost_bin":0.1547571199999998,"total_completion_cost_bin":0.032335520000000006,"total_reasoning_cost_bin":0.022744960000000005,"total_cost_bin":0.21035741999999982,"total_on_demand_input_cost_bin":0.7743054200000004,"ci_99_lower":0.5052290803751196,"ci_99_upper":0.7687592712363557,"ci_98_lower":0.5171021640735733,"ci_98_upper":0.7565656989480861,"ci_95_lower":0.5409043115593821,"ci_95_upper":0.7367569065011819,"ci_90_lower":0.5625340531739023,"ci_90_upper":0.7244329012020984,"ci_75_lower":0.5859201354358126,"ci_75_upper":0.7008726034806493,"ci_50_lower":0.6088234798489687,"ci_50_upper":0.6779412834631169},"131072":{"avg_score":0.4385824248384576,"min_score":0.0057299453765153885,"max_score":1,"n_tests":103,"stddev":0.4809814458330595,"median_score":0.03768844157457352,"q1_score":0.023100536316633224,"q3_score":1,"avg_token_efficiency":0.9940812964474944,"is_incomplete":false,"total_prompt_cost_bin":0.0007116199999999996,"total_cache_read_cost_bin":0.37505843199999944,"total_completion_cost_bin":0.03905608,"total_reasoning_cost_bin":0.026769680000000004,"total_cost_bin":0.4415958119999995,"total_on_demand_input_cost_bin":1.8760037800000018,"ci_99_lower":0.30675033176932665,"ci_99_upper":0.5511342356643115,"ci_98_lower":0.32589191718772953,"ci_98_upper":0.5429031201626074,"ci_95_lower":0.34445341904670257,"ci_95_upper":0.5322027116721197,"ci_90_lower":0.3556066816769884,"ci_90_upper":0.5201065073515406,"ci_75_lower":0.3826273356307777,"ci_75_upper":0.49499041011423617,"ci_50_lower":0.4087053002660769,"ci_50_upper":0.4747526208130932},"262144":{"avg_score":0.3127298078628534,"min_score":0.0024676125030964613,"max_score":1,"n_tests":141,"stddev":0.44082852305564413,"median_score":0.0342567153275013,"q1_score":0.023255813866853714,"q3_score":0.9991670250892639,"avg_token_efficiency":0.9994797316927643,"is_incomplete":false,"total_prompt_cost_bin":1.1365382000000015,"total_cache_read_cost_bin":0.8062207999999994,"total_completion_cost_bin":0.06348272,"total_reasoning_cost_bin":0.04870907999999999,"total_cost_bin":2.054950800000001,"total_on_demand_input_cost_bin":5.1676422,"ci_99_lower":0.22365701364253396,"ci_99_upper":0.4084216722175941,"ci_98_lower":0.22983534647325213,"ci_98_upper":0.3961643044857294,"ci_95_lower":0.24280236231820354,"ci_95_upper":0.3861957200422242,"ci_90_lower":0.2515144396640182,"ci_90_upper":0.3736407833665127,"ci_75_lower":0.2716737175374771,"ci_75_upper":0.3550108997027409,"ci_50_lower":0.2899189203620908,"ci_50_upper":0.33523034724631473},"524288":{"avg_score":0.24743696201052043,"min_score":0.001411432633176446,"max_score":1,"n_tests":236,"stddev":0.3982306675979535,"median_score":0.03421211428940296,"q1_score":0.023106290493160486,"q3_score":0.12692151963710785,"avg_token_efficiency":0.9949755047319113,"is_incomplete":false,"total_prompt_cost_bin":0.001438220000000002,"total_cache_read_cost_bin":3.4458869759999944,"total_completion_cost_bin":0.10748919999999998,"total_reasoning_cost_bin":0.08349067999999998,"total_cost_bin":3.638305075999994,"total_on_demand_input_cost_bin":17.230873099999997,"ci_99_lower":0.18175750267388047,"ci_99_upper":0.31564160469927827,"ci_98_lower":0.18615741476380085,"ci_98_upper":0.30749980754290024,"ci_95_lower":0.19699425100471224,"ci_95_upper":0.3011723942365063,"ci_90_lower":0.20640349302054145,"ci_90_upper":0.29325711603584237,"ci_75_lower":0.21786653341910944,"ci_75_upper":0.27737878013332673,"ci_50_lower":0.2303313438872357,"ci_50_upper":0.2645204013770434}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Flash 0423","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":8.106e-8,"completion":1.6212e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T03:17:46.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-24T14:13:25.610Z"},{"model_slug":"qwen/qwen3.6-flash","reasoning_mode":null,"overall_metrics":{"auc_1m":0.23706265373201277,"auc_128k":0.50448344214062,"cum_avg_1m":0.41925207024032557,"cum_avg_128k":0.5538135645312877,"avg_score_overall":0.42437277140926555,"avg_response_time_ms":13703.95431748943,"total_prompt_cost":127.41118124999973,"total_cache_read_cost":0,"total_completion_cost":0.376390125,"total_reasoning_cost":0,"total_prompt_cost_128k":4.69062225,"total_completion_cost_128k":0.21414037499999994,"total_on_demand_input_cost":127.41118124999973,"total_on_demand_input_cost_128k":4.69062225,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0198220004437082},"bin_metrics":{"8192":{"avg_score":0.6851635661034379,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4567339101892933,"median_score":1,"q1_score":0.037784758023917675,"q3_score":1,"avg_token_efficiency":1.0228079512716757,"is_incomplete":false,"total_prompt_cost_bin":0.10191599999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.032205374999999994,"total_reasoning_cost_bin":0,"total_cost_bin":0.13412137499999996,"total_on_demand_input_cost_bin":0.10191599999999996,"ci_99_lower":0.5391875018435239,"ci_99_upper":0.8065558835129486,"ci_98_lower":0.562477135547495,"ci_98_upper":0.7946783675476908,"ci_95_lower":0.5874739708623383,"ci_95_upper":0.7811935784925299,"ci_90_lower":0.6001130352396286,"ci_90_upper":0.7587055731075816,"ci_75_lower":0.625263841571723,"ci_75_upper":0.743866201533092,"ci_50_lower":0.6492526970760082,"ci_50_upper":0.7211390050157205},"16384":{"avg_score":0.6244263979024254,"min_score":0.00935307890176773,"max_score":1,"n_tests":80,"stddev":0.47523855830523876,"median_score":1,"q1_score":0.037135208025574684,"q3_score":1,"avg_token_efficiency":1.0161194758384626,"is_incomplete":false,"total_prompt_cost_bin":0.2101619999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.031716000000000015,"total_reasoning_cost_bin":0,"total_cost_bin":0.24187799999999993,"total_on_demand_input_cost_bin":0.2101619999999999,"ci_99_lower":0.4781946389140212,"ci_99_upper":0.759334510988323,"ci_98_lower":0.5008484717373504,"ci_98_upper":0.7349984443651045,"ci_95_lower":0.5151067975355546,"ci_95_upper":0.7216218897138607,"ci_90_lower":0.5389890878181904,"ci_90_upper":0.7086478162487038,"ci_75_lower":0.5636268206610112,"ci_75_upper":0.6853681372274878,"ci_50_lower":0.5882983722200152,"ci_50_upper":0.6601746914820978},"32768":{"avg_score":0.5217512919796932,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4812348732498998,"median_score":0.6081266850233078,"q1_score":0.03215197520330548,"q3_score":1,"avg_token_efficiency":1.0257548592579653,"is_incomplete":false,"total_prompt_cost_bin":0.7456649999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06157012499999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.8072351249999996,"total_on_demand_input_cost_bin":0.7456649999999996,"ci_99_lower":0.4171744174884738,"ci_99_upper":0.6265569765423339,"ci_98_lower":0.42619242739636004,"ci_98_upper":0.6193269710059908,"ci_95_lower":0.44068544551820893,"ci_95_upper":0.5983080614440363,"ci_90_lower":0.4554268439884147,"ci_90_upper":0.5870542120084042,"ci_75_lower":0.47389196192136085,"ci_75_upper":0.5702919616121696,"ci_50_lower":0.49482060974611053,"ci_50_upper":0.5505513428048717},"65536":{"avg_score":0.4859350547906669,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.48790006801545605,"median_score":0.06381647288799286,"q1_score":0.02402501180768013,"q3_score":1,"avg_token_efficiency":1.020990401222107,"is_incomplete":false,"total_prompt_cost_bin":1.0643767499999994,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.039471749999999986,"total_reasoning_cost_bin":0,"total_cost_bin":1.1038484999999993,"total_on_demand_input_cost_bin":1.0643767499999994,"ci_99_lower":0.34521979371359685,"ci_99_upper":0.6232729215569904,"ci_98_lower":0.3680662940708811,"ci_98_upper":0.6111625125362153,"ci_95_lower":0.38216298268135523,"ci_95_upper":0.5907766587125577,"ci_90_lower":0.39564725468291295,"ci_90_upper":0.5772412832675722,"ci_75_lower":0.4276394113610663,"ci_75_upper":0.5456030779381228,"ci_50_lower":0.4513292987557018,"ci_50_upper":0.5202783666085452},"131072":{"avg_score":0.4517915118802158,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.485389705823116,"median_score":0.04372842237353325,"q1_score":0.02211037464439869,"q3_score":1,"avg_token_efficiency":1.0162368757633713,"is_incomplete":false,"total_prompt_cost_bin":2.568502500000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.04917712499999998,"total_reasoning_cost_bin":0,"total_cost_bin":2.617679625000001,"total_on_demand_input_cost_bin":2.568502500000001,"ci_99_lower":0.318881040978476,"ci_99_upper":0.5659558764159074,"ci_98_lower":0.33763466954181526,"ci_98_upper":0.5552962898203053,"ci_95_lower":0.35648914383378477,"ci_95_upper":0.5457858125804652,"ci_90_lower":0.37343488508604866,"ci_90_upper":0.5286896173670455,"ci_75_lower":0.3947613857758855,"ci_75_upper":0.5091523984763591,"ci_50_lower":0.4221439831381555,"ci_50_upper":0.48869283457887946},"262144":{"avg_score":0.38134025783172365,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.46787415768204205,"median_score":0.041763342916965485,"q1_score":0.02277039922773838,"q3_score":1,"avg_token_efficiency":1.0180736782704625,"is_incomplete":false,"total_prompt_cost_bin":28.1988165,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.063423,"total_reasoning_cost_bin":0,"total_cost_bin":28.2622395,"total_on_demand_input_cost_bin":28.1988165,"ci_99_lower":0.28341377118171407,"ci_99_upper":0.4899793171008111,"ci_98_lower":0.2905139726921189,"ci_98_upper":0.4772699927556467,"ci_95_lower":0.30118111467021125,"ci_95_upper":0.4618433492247871,"ci_90_lower":0.3142407112769743,"ci_90_upper":0.44860405064846803,"ci_75_lower":0.3353036667989378,"ci_75_upper":0.42643189949317045,"ci_50_lower":0.35473180017376266,"ci_50_upper":0.4083306208638481},"524288":{"avg_score":0.20360848143444224,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.36542749871550173,"median_score":0.03236061707139015,"q1_score":0.020526780746877193,"q3_score":0.08225353248417377,"avg_token_efficiency":1.0188343915699014,"is_incomplete":false,"total_prompt_cost_bin":94.52174249999973,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.09882675000000006,"total_reasoning_cost_bin":0,"total_cost_bin":94.62056924999973,"total_on_demand_input_cost_bin":94.52174249999973,"ci_99_lower":0.14319408214071933,"ci_99_upper":0.2650579149882266,"ci_98_lower":0.1496273553871818,"ci_98_upper":0.25912354780860264,"ci_95_lower":0.15831515967443396,"ci_95_upper":0.25099805911092316,"ci_90_lower":0.16538243596343388,"ci_90_upper":0.24258323517850977,"ci_75_lower":0.1784998753280241,"ci_75_upper":0.2313748324606607,"ci_50_lower":0.18869671006726446,"ci_50_upper":0.21848046590260356}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 Flash","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":1.875e-7,"completion":0.000001125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-27T03:42:42.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T04:16:41.774Z"},{"model_slug":"google/gemini-3.5-flash","reasoning_mode":"low","overall_metrics":{"auc_1m":0.23621802959505553,"auc_128k":0.37496931409635564,"cum_avg_1m":0.40649671301590273,"cum_avg_128k":0.516457049356566,"avg_score_overall":0.33383181334865286,"avg_response_time_ms":19993.43512202653,"total_prompt_cost":37.994744999999966,"total_cache_read_cost":72.72702134999982,"total_completion_cost":4.080456000000001,"total_reasoning_cost":2.934603,"total_prompt_cost_128k":7.992706499999992,"total_completion_cost_128k":1.6886700000000001,"total_on_demand_input_cost":765.2649585000015,"total_on_demand_input_cost_128k":36.8202,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154894},"bin_metrics":{"8192":{"avg_score":0.8787720720050857,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.32278199811028546,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747237,"is_incomplete":false,"total_prompt_cost_bin":0.7973159999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.2511359999999999,"total_reasoning_cost_bin":0.5805359999999999,"total_cost_bin":1.6289879999999997,"total_on_demand_input_cost_bin":0.7973159999999998,"ci_99_lower":0.7822802800944774,"ci_99_upper":0.9635477802802342,"ci_98_lower":0.7930057097019162,"ci_98_upper":0.9519297138256952,"ci_95_lower":0.8049137231800705,"ci_95_upper":0.940136439667549,"ci_90_lower":0.8171254335273989,"ci_90_upper":0.9276597349368967,"ci_75_lower":0.8412502727325774,"ci_75_upper":0.915477839842788,"ci_50_lower":0.8544954226701521,"ci_50_upper":0.9029251153639053},"16384":{"avg_score":0.6342966050491669,"min_score":0.010909090749919415,"max_score":1,"n_tests":80,"stddev":0.47397650710482564,"median_score":1,"q1_score":0.031288267113268375,"q3_score":1,"avg_token_efficiency":1.000074513502448,"is_incomplete":false,"total_prompt_cost_bin":1.652807999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.24903000000000008,"total_reasoning_cost_bin":0.29737800000000003,"total_cost_bin":2.199215999999998,"total_on_demand_input_cost_bin":1.652807999999998,"ci_99_lower":0.48787606507388404,"ci_99_upper":0.7682565835529239,"ci_98_lower":0.5004239690667018,"ci_98_upper":0.7447914635204943,"ci_95_lower":0.5257129951450042,"ci_95_upper":0.731654254627938,"ci_90_lower":0.5479209065629402,"ci_90_upper":0.7189688817824935,"ci_75_lower":0.5730808554711984,"ci_75_upper":0.6952294422226261,"ci_50_lower":0.5975410794577328,"ci_50_upper":0.6697120418917621},"32768":{"avg_score":0.5835467717826011,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.4780195229108617,"median_score":1,"q1_score":0.034234208054840565,"q3_score":1,"avg_token_efficiency":1.0000359090949908,"is_incomplete":false,"total_prompt_cost_bin":1.9315619999999998,"total_cache_read_cost_bin":0.38785620000000104,"total_completion_cost_bin":0.48223800000000017,"total_reasoning_cost_bin":1.123758,"total_cost_bin":3.925414200000001,"total_on_demand_input_cost_bin":5.810123999999995,"ci_99_lower":0.4740633260014532,"ci_99_upper":0.6888500750226804,"ci_98_lower":0.4898014735351727,"ci_98_upper":0.6761990224941196,"ci_95_lower":0.5063250162312589,"ci_95_upper":0.6617536409558584,"ci_90_lower":0.515311038002341,"ci_90_upper":0.6524620174335295,"ci_75_lower":0.5380875253198182,"ci_75_upper":0.6319945327281055,"ci_50_lower":0.556806133076915,"ci_50_upper":0.6100989330071196},"65536":{"avg_score":0.2701946303686675,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4207389514085017,"median_score":0.03496503457427025,"q1_score":0.02222222276031971,"q3_score":0.08787943422794342,"avg_token_efficiency":1.0000152878522812,"is_incomplete":false,"total_prompt_cost_bin":1.6342274999999988,"total_cache_read_cost_bin":0.6705855,"total_completion_cost_bin":0.3152250000000001,"total_reasoning_cost_bin":0.09602100000000001,"total_cost_bin":2.716058999999999,"total_on_demand_input_cost_bin":8.340082499999996,"ci_99_lower":0.16682516379457185,"ci_99_upper":0.38757231076215115,"ci_98_lower":0.1712596760193872,"ci_98_upper":0.3825377093992908,"ci_95_lower":0.18964569985893936,"ci_95_upper":0.3616431176629575,"ci_90_lower":0.20116867775645325,"ci_90_upper":0.3409488015621897,"ci_75_lower":0.21365082649760608,"ci_75_upper":0.3165615817436072,"ci_50_lower":0.2362801309209317,"ci_50_upper":0.29551658090204},"131072":{"avg_score":0.21547516757730864,"min_score":0.003092384897172451,"max_score":1,"n_tests":103,"stddev":0.3872130284378599,"median_score":0.027210883796215057,"q1_score":0.01641554106026888,"q3_score":0.048751652240753174,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":1.9767929999999958,"total_cache_read_cost_bin":1.8243076500000017,"total_completion_cost_bin":0.3910409999999999,"total_reasoning_cost_bin":0.216981,"total_cost_bin":4.409122649999997,"total_on_demand_input_cost_bin":20.219869500000012,"ci_99_lower":0.12065383528415748,"ci_99_upper":0.31765727847719616,"ci_98_lower":0.1312862784891543,"ci_98_upper":0.30124827390432674,"ci_95_lower":0.14904571329808602,"ci_95_upper":0.2829669648550848,"ci_90_lower":0.15831607171058548,"ci_90_upper":0.27276164970419464,"ci_75_lower":0.1755451504485817,"ci_75_upper":0.2545199443872123,"ci_50_lower":0.18808017618438788,"ci_50_upper":0.2388266966727933},"262144":{"avg_score":0.2747869574438437,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.41866331189625355,"median_score":0.037837836891412735,"q1_score":0.027486437931656837,"q3_score":0.2101576179265976,"avg_token_efficiency":1.000003817936524,"is_incomplete":false,"total_prompt_cost_bin":4.7395875,"total_cache_read_cost_bin":5.065704000000007,"total_completion_cost_bin":0.4952880000000002,"total_reasoning_cost_bin":0.335502,"total_cost_bin":10.636081500000007,"total_on_demand_input_cost_bin":55.39662749999999,"ci_99_lower":0.18664463483538574,"ci_99_upper":0.36539408188312417,"ci_98_lower":0.19554778346096674,"ci_98_upper":0.35173293359607377,"ci_95_lower":0.20542240795359043,"ci_95_upper":0.34490303153622937,"ci_90_lower":0.21789776167282437,"ci_90_upper":0.3302469330173521,"ci_75_lower":0.23444564285496555,"ci_75_upper":0.31454063054229503,"ci_50_lower":0.25119471708799757,"ci_50_upper":0.29727972760517485},"524288":{"avg_score":0.21372585818990356,"min_score":0.0013020833721384406,"max_score":1,"n_tests":236,"stddev":0.3731072381333667,"median_score":0.03362136334180832,"q1_score":0.022293691523373127,"q3_score":0.08457018062472343,"avg_token_efficiency":1.0000019078540328,"is_incomplete":false,"total_prompt_cost_bin":10.603444499999995,"total_cache_read_cost_bin":17.49456959999993,"total_completion_cost_bin":0.7759890000000004,"total_reasoning_cost_bin":0.18439200000000003,"total_cost_bin":29.058395099999924,"total_on_demand_input_cost_bin":185.54914050000016,"ci_99_lower":0.15263362456591587,"ci_99_upper":0.2749639133660723,"ci_98_lower":0.15685810906304462,"ci_98_upper":0.2695056011210545,"ci_95_lower":0.1656930960973792,"ci_95_upper":0.2633812161830527,"ci_90_lower":0.17522881325179912,"ci_90_upper":0.25389827834960205,"ci_75_lower":0.18670057014624342,"ci_75_upper":0.24151328326516414,"ci_50_lower":0.19809748850068792,"ci_50_upper":0.22918427345415446},"1048576":{"avg_score":0.18117564171064465,"min_score":0.0012024048482999206,"max_score":1,"n_tests":310,"stddev":0.3463296934370596,"median_score":0.03304317034780979,"q1_score":0.02112810965627432,"q3_score":0.05815728288143873,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":14.65900649999998,"total_cache_read_cost_bin":47.283998399999874,"total_completion_cost_bin":1.120509,"total_reasoning_cost_bin":0.100035,"total_cost_bin":63.16354889999985,"total_on_demand_input_cost_bin":487.4989905000014,"ci_99_lower":0.13419111800205427,"ci_99_upper":0.23242551787021276,"ci_98_lower":0.13643368471301567,"ci_98_upper":0.22767773793557763,"ci_95_lower":0.14398998643938196,"ci_95_upper":0.22110031172408792,"ci_90_lower":0.14979763115955247,"ci_90_upper":0.21387667978783287,"ci_75_lower":0.15791042650089365,"ci_75_upper":0.20469325795997828,"ci_50_lower":0.16640018526588626,"ci_50_upper":0.1939092296614094}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":0.0000015,"completion":0.000009,"reasoning":0.000009,"has_tiered_pricing":true},"api_created_at":"2026-05-19T12:30:00.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"deepseek/deepseek-v4-flash-0731","reasoning_mode":"low","overall_metrics":{"auc_1m":0.23617609434261508,"auc_128k":0.5028040012334745,"cum_avg_1m":0.45758740050408003,"cum_avg_128k":0.6248940471785798,"avg_score_overall":0.45731168628346314,"avg_response_time_ms":13757.707440703034,"total_prompt_cost":0.11283183000000009,"total_cache_read_cost":3.2975447040000025,"total_completion_cost":0.2540826,"total_reasoning_cost":0.19527174,"total_prompt_cost_128k":0.0383994,"total_completion_cost_128k":0.14629914,"total_on_demand_input_cost":16.600555350000008,"total_on_demand_input_cost_128k":2.2015097999999997,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971342554822209},"bin_metrics":{"8192":{"avg_score":0.861810504947789,"min_score":0.01392451487481594,"max_score":1,"n_tests":80,"stddev":0.3375803156009393,"median_score":1,"q1_score":0.9953998178243637,"q3_score":1,"avg_token_efficiency":0.9915838193643985,"is_incomplete":false,"total_prompt_cost_bin":0.007243920000000004,"total_cache_read_cost_bin":0.008043264000000005,"total_completion_cost_bin":0.019818719999999998,"total_reasoning_cost_bin":0.014794200000000002,"total_cost_bin":0.04990010400000001,"total_on_demand_input_cost_bin":0.047460240000000035,"ci_99_lower":0.7543086188912858,"ci_99_upper":0.9503076602246147,"ci_98_lower":0.7726432287395,"ci_98_upper":0.9373742445742246,"ci_95_lower":0.779060465565417,"ci_95_upper":0.9248626107856398,"ci_90_lower":0.791795021276921,"ci_90_upper":0.9195532791293226,"ci_75_lower":0.8153485374146839,"ci_75_upper":0.9006290189456194,"ci_50_lower":0.8379744747304358,"ci_50_upper":0.8864636424463241},"16384":{"avg_score":0.8388115736190229,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.3620112991042192,"median_score":0.9993297159671783,"q1_score":0.9939576387405396,"q3_score":1,"avg_token_efficiency":0.9890247337221421,"is_incomplete":false,"total_prompt_cost_bin":0.006500880000000001,"total_cache_read_cost_bin":0.01830297600000002,"total_completion_cost_bin":0.027007919999999987,"total_reasoning_cost_bin":0.022069619999999998,"total_cost_bin":0.073881396,"total_on_demand_input_cost_bin":0.09801576,"ci_99_lower":0.7273146087391069,"ci_99_upper":0.9371415711658192,"ci_98_lower":0.728798882844043,"ci_98_upper":0.9250178615606856,"ci_95_lower":0.7536708495253697,"ci_95_upper":0.9126257759719738,"ci_90_lower":0.7663285870023537,"ci_90_upper":0.9000383449206129,"ci_75_lower":0.7900199835043167,"ci_75_upper":0.8867993449879578,"ci_50_lower":0.8138624251936563,"ci_50_upper":0.8634854728676147},"32768":{"avg_score":0.5805729886633344,"min_score":0.003955174703150988,"max_score":1,"n_tests":136,"stddev":0.4769779409850481,"median_score":0.9888665974140167,"q1_score":0.030776889994740486,"q3_score":1,"avg_token_efficiency":1.0102892651671218,"is_incomplete":false,"total_prompt_cost_bin":0.006306480000000002,"total_cache_read_cost_bin":0.06919142400000014,"total_completion_cost_bin":0.047870279999999994,"total_reasoning_cost_bin":0.03824172,"total_cost_bin":0.16160990400000014,"total_on_demand_input_cost_bin":0.3522636000000004,"ci_99_lower":0.47132015034287605,"ci_99_upper":0.6864023286524153,"ci_98_lower":0.4873395622255596,"ci_98_upper":0.6733694495643859,"ci_95_lower":0.5037472342283924,"ci_95_upper":0.6584488805630826,"ci_90_lower":0.5120953986602699,"ci_90_upper":0.6498641180198106,"ci_75_lower":0.5356928341228495,"ci_75_upper":0.6298097721472218,"ci_50_lower":0.5540740144522929,"ci_50_upper":0.6072343776639386},"65536":{"avg_score":0.36905888247933677,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4676222189611436,"median_score":0.032592590898275375,"q1_score":0.023259468376636505,"q3_score":0.999404788017273,"avg_token_efficiency":0.994745115776915,"is_incomplete":false,"total_prompt_cost_bin":0.006186329999999993,"total_cache_read_cost_bin":0.09831628799999993,"total_completion_cost_bin":0.024534900000000002,"total_reasoning_cost_bin":0.01853208,"total_cost_bin":0.14756959799999994,"total_on_demand_input_cost_bin":0.4977677700000003,"ci_99_lower":0.2424329311146935,"ci_99_upper":0.5062718857946353,"ci_98_lower":0.2549389604444287,"ci_98_upper":0.4847712306751842,"ci_95_lower":0.2751750092796283,"ci_95_upper":0.4717043970526292,"ci_90_lower":0.28831378309425953,"ci_90_upper":0.45055356865885726,"ci_75_lower":0.3101932639295496,"ci_75_upper":0.42659680938728234,"ci_50_lower":0.334378578771344,"ci_50_upper":0.4033067303856917},"131072":{"avg_score":0.47421628618341627,"min_score":0.0057299453765153885,"max_score":1,"n_tests":103,"stddev":0.48349398413196426,"median_score":0.04909983649849892,"q1_score":0.022255144082009792,"q3_score":0.9998122453689575,"avg_token_efficiency":0.994081296447494,"is_incomplete":false,"total_prompt_cost_bin":0.012161789999999999,"total_cache_read_cost_bin":0.23876812799999983,"total_completion_cost_bin":0.02706732000000001,"total_reasoning_cost_bin":0.019090620000000003,"total_cost_bin":0.2970878579999998,"total_on_demand_input_cost_bin":1.2060024299999992,"ci_99_lower":0.35034199460850374,"ci_99_upper":0.5901121111658348,"ci_98_lower":0.36276890072438106,"ci_98_upper":0.5721957799678361,"ci_95_lower":0.3809186345673827,"ci_95_upper":0.5650765653362824,"ci_90_lower":0.3962681965660575,"ci_90_upper":0.5563535865488942,"ci_75_lower":0.4184262537616712,"ci_75_upper":0.5311260350140092,"ci_50_lower":0.4447590812455266,"ci_50_upper":0.5096288943138806},"262144":{"avg_score":0.2950173665485378,"min_score":0.005610098131000996,"max_score":1,"n_tests":141,"stddev":0.42828046843396544,"median_score":0.039036400616168976,"q1_score":0.028678303584456444,"q3_score":0.9874405860900879,"avg_token_efficiency":0.999479731692764,"is_incomplete":false,"total_prompt_cost_bin":0.026575379999999982,"total_cache_read_cost_bin":0.659096063999999,"total_completion_cost_bin":0.034747560000000004,"total_reasoning_cost_bin":0.02502432,"total_cost_bin":0.7454433239999989,"total_on_demand_input_cost_bin":3.3220557000000026,"ci_99_lower":0.2007951721166267,"ci_99_upper":0.38088025977607176,"ci_98_lower":0.21167991030730096,"ci_98_upper":0.3740359831219605,"ci_95_lower":0.22464523343006082,"ci_95_upper":0.3649072164331121,"ci_90_lower":0.23617885039874903,"ci_90_upper":0.3548586087324491,"ci_75_lower":0.2537126112266952,"ci_75_upper":0.3350007249420535,"ci_50_lower":0.2717279909304112,"ci_50_upper":0.3179391823900559},"524288":{"avg_score":0.24121160159120336,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.39210303676980596,"median_score":0.03386112302541733,"q1_score":0.022587877232581377,"q3_score":0.12428604997694492,"avg_token_efficiency":0.9949755047319113,"is_incomplete":false,"total_prompt_cost_bin":0.04785705000000011,"total_cache_read_cost_bin":2.2058265600000038,"total_completion_cost_bin":0.0730359,"total_reasoning_cost_bin":0.05751918000000001,"total_cost_bin":2.3842386900000037,"total_on_demand_input_cost_bin":11.076989850000006,"ci_99_lower":0.17854987977813513,"ci_99_upper":0.31161675972053565,"ci_98_lower":0.18168615406556687,"ci_98_upper":0.3007292033888901,"ci_95_lower":0.19089843321656236,"ci_95_upper":0.29362707295108753,"ci_90_lower":0.20104473025081243,"ci_90_upper":0.2840216688632716,"ci_75_lower":0.2114710766093024,"ci_75_upper":0.2694733739221099,"ci_50_lower":0.22474352046832546,"ci_50_upper":0.2579758329414644}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1310720,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Flash 0731","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":6e-8,"completion":1.2e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-31T06:21:48.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-02T01:43:22.976Z"},{"model_slug":"pokee/pokee-isaac","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.22161483731935872,"auc_128k":0.7508260377734094,"cum_avg_1m":0.5618487117522505,"cum_avg_128k":0.8250668107742017,"avg_score_overall":0.40337937220346093,"avg_response_time_ms":56147.28925764571,"total_prompt_cost":0,"total_cache_read_cost":0,"total_completion_cost":0,"total_reasoning_cost":0,"total_prompt_cost_128k":0,"total_completion_cost_128k":0,"total_on_demand_input_cost":0,"total_on_demand_input_cost_128k":0,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0190568226852843},"bin_metrics":{"8192":{"avg_score":0.9877196756191552,"min_score":0.01769087463617325,"max_score":1,"n_tests":80,"stddev":0.10982533440343323,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222286,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.9508816231042146,"ci_99_upper":1,"ci_98_lower":0.9508830834180116,"ci_98_upper":1,"ci_95_lower":0.9631604871712625,"ci_95_upper":1,"ci_90_lower":0.9631619474850595,"ci_90_upper":1,"ci_75_lower":0.9754406290128828,"ci_75_upper":1,"ci_50_lower":0.9754422718659044,"ci_50_upper":0.999998539686203},"16384":{"avg_score":0.9273334811907261,"min_score":0.01199815422296524,"max_score":1,"n_tests":80,"stddev":0.2568362710506895,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0159704488335681,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.8420046390057542,"ci_99_upper":0.9879734350368381,"ci_98_lower":0.8530576213782187,"ci_98_upper":0.9877374999923632,"ci_95_lower":0.8660404997796286,"ci_95_upper":0.9760196706978604,"ci_90_lower":0.8773483125551138,"ci_90_upper":0.9753874769201503,"ci_75_lower":0.8910161633451935,"ci_75_upper":0.9633513158390997,"ci_50_lower":0.9037037640868221,"ci_50_upper":0.9510780786396935},"32768":{"avg_score":0.8934908485115396,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.30364892539304306,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0256830410679856,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.8214803963877014,"ci_99_upper":0.9570242393892155,"ci_98_lower":0.8291263637111714,"ci_98_upper":0.950016248243046,"ci_95_lower":0.8371201502243126,"ci_95_upper":0.943348487359232,"ci_90_lower":0.850024041424074,"ci_90_upper":0.9359609656373162,"ci_75_lower":0.8639130341338495,"ci_75_upper":0.922065260956622,"ci_50_lower":0.8732860073770794,"ci_50_upper":0.9142627402333721},"65536":{"avg_score":0.714948883636252,"min_score":0.00469667324796319,"max_score":1,"n_tests":85,"stddev":0.4442766186827195,"median_score":1,"q1_score":0.048283666372299194,"q3_score":1,"avg_token_efficiency":1.020959825517544,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.587597551349934,"ci_99_upper":0.8297492068310488,"ci_98_lower":0.5901221817988245,"ci_98_upper":0.8180530035858425,"ci_95_lower":0.6218105550118558,"ci_95_upper":0.8055784413821119,"ci_90_lower":0.6339049061527475,"ci_90_upper":0.794125558633686,"ci_75_lower":0.6572287562344333,"ci_75_upper":0.771393137882628,"ci_50_lower":0.6804072839344906,"ci_50_upper":0.7479828515870716},"131072":{"avg_score":0.601841164913336,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4822166385093393,"median_score":1,"q1_score":0.026625921949744225,"q3_score":1,"avg_token_efficiency":1.0162215936488592,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.48687027856959614,"ci_99_upper":0.7158059028147277,"ci_98_lower":0.49648958218525824,"ci_98_upper":0.7065050820138269,"ci_95_lower":0.5155200017148232,"ci_95_upper":0.6967769282957024,"ci_90_lower":0.5254496523396122,"ci_90_upper":0.6784437656879552,"ci_75_lower":0.5451802023366045,"ci_75_upper":0.6587233134543691,"ci_50_lower":0.5647058318831362,"ci_50_upper":0.6390362237297559},"262144":{"avg_score":0.18698255231286934,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.35203763644975067,"median_score":0.03321569412946701,"q1_score":0.021164020523428917,"q3_score":0.0569319985806942,"avg_token_efficiency":1.0180660423974124,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.11371577838666949,"ci_99_upper":0.25667518697363645,"ci_98_lower":0.12097080732601635,"ci_98_upper":0.25422247856756314,"ci_95_lower":0.13032830483135635,"ci_95_upper":0.24735668247300596,"ci_90_lower":0.13952578176968858,"ci_90_upper":0.237279109469638,"ci_75_lower":0.1526344235111705,"ci_75_upper":0.22051006544431742,"ci_50_lower":0.16634914277043816,"ci_50_upper":0.20511064077157806},"524288":{"avg_score":0.1088277742156159,"min_score":0.002106002066284418,"max_score":1,"n_tests":236,"stddev":0.25796717331885616,"median_score":0.02558222785592079,"q1_score":0.016106729861348867,"q3_score":0.045053064823150635,"avg_token_efficiency":1.0188305758618428,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.07173809221593991,"ci_99_upper":0.15251852128875215,"ci_98_lower":0.07397054768173356,"ci_98_upper":0.1488107642220625,"ci_95_lower":0.07853043896670973,"ci_95_upper":0.1435426801656446,"ci_90_lower":0.08179046312140861,"ci_90_upper":0.1390732839551628,"ci_75_lower":0.08998139055706737,"ci_75_upper":0.12854970036574215,"ci_50_lower":0.09681081253309118,"ci_50_upper":0.11911307421800982},"1048576":{"avg_score":0.0736453136185094,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.19860880317459595,"median_score":0.023275471292436123,"q1_score":0.012688389513641596,"q3_score":0.03973911236971617,"avg_token_efficiency":1.0171015840338855,"is_incomplete":false,"total_prompt_cost_bin":0,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0,"total_reasoning_cost_bin":0,"total_cost_bin":0,"total_on_demand_input_cost_bin":0,"ci_99_lower":0.04742574005544509,"ci_99_upper":0.10756998274354225,"ci_98_lower":0.049982152951911336,"ci_98_upper":0.10003623835717311,"ci_95_lower":0.05260249498803469,"ci_95_upper":0.09624303775270575,"ci_90_lower":0.055727242914576744,"ci_90_upper":0.09279993584452782,"ci_75_lower":0.06052496500638119,"ci_75_upper":0.08597101367242078,"ci_50_lower":0.06498193575168824,"ci_50_upper":0.08047899575397041}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":10000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Pokee-Isaac 28B v0","provider_name":"Pokee","run_provider_names":["Pokee AI"],"pricing_metadata":{"prompt":1.5e-7,"completion":0.000001,"reasoning":null,"has_tiered_pricing":false},"api_created_at":null,"api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-15T21:47:53.171Z"},{"model_slug":"moonshotai/kimi-k3","reasoning_mode":"high","overall_metrics":{"auc_1m":0.2209163290451213,"auc_128k":0.5851273527310392,"cum_avg_1m":0.4831275822938699,"cum_avg_128k":0.67644304562624,"avg_score_overall":0.47073401930811193,"avg_response_time_ms":111913.23516611771,"total_prompt_cost":11.25256800000001,"total_cache_read_cost":54.01966080000008,"total_completion_cost":53.85164999999999,"total_reasoning_cost":48.805710000000005,"total_prompt_cost_128k":1.7162400000000004,"total_completion_cost_128k":21.383475,"total_on_demand_input_cost":551.4491760000008,"total_on_demand_input_cost_128k":73.260816,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9960508775705984},"bin_metrics":{"8192":{"avg_score":0.9150065772584639,"min_score":0.010457987897098064,"max_score":1,"n_tests":80,"stddev":0.2758272783882228,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.002393554094994,"is_incomplete":false,"total_prompt_cost_bin":0.2567520000000002,"total_cache_read_cost_bin":0.1342464,"total_completion_cost_bin":1.6244699999999999,"total_reasoning_cost_bin":1.18398,"total_cost_bin":3.1994484,"total_on_demand_input_cost_bin":1.5992159999999997,"ci_99_lower":0.8184902747289743,"ci_99_upper":0.9874304856302333,"ci_98_lower":0.8298989838620182,"ci_98_upper":0.9757960404707119,"ci_95_lower":0.8425765226501971,"ci_95_upper":0.9750164370247512,"ci_90_lower":0.8547188722190913,"ci_90_upper":0.9635440796456534,"ci_75_lower":0.8784134658490076,"ci_75_upper":0.951336717241793,"ci_50_lower":0.8910685849928995,"ci_50_upper":0.9390145900455535},"16384":{"avg_score":0.8286311586911325,"min_score":0.0044469148851931095,"max_score":1,"n_tests":80,"stddev":0.3731200169991366,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9863410585921487,"is_incomplete":false,"total_prompt_cost_bin":0.18907200000000013,"total_cache_read_cost_bin":0.30712319999999954,"total_completion_cost_bin":2.396624999999999,"total_reasoning_cost_bin":1.9704449999999996,"total_cost_bin":4.863265199999998,"total_on_demand_input_cost_bin":3.2603039999999988,"ci_99_lower":0.7067326022323396,"ci_99_upper":0.9271837296800222,"ci_98_lower":0.7183495437117525,"ci_98_upper":0.9147687323386781,"ci_95_lower":0.7431178947980516,"ci_95_upper":0.9023341837363842,"ci_90_lower":0.7558020026236772,"ci_90_upper":0.8901678790804,"ci_75_lower":0.7796933300902311,"ci_75_upper":0.8769722552562598,"ci_50_lower":0.8038147668761668,"ci_50_upper":0.8534483033479774},"32768":{"avg_score":0.6550231815969764,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4617632140316022,"median_score":1,"q1_score":0.048043339513242245,"q3_score":1,"avg_token_efficiency":1.013008403395958,"is_incomplete":false,"total_prompt_cost_bin":0.4190400000000001,"total_cache_read_cost_bin":1.1365631999999994,"total_completion_cost_bin":5.0460150000000015,"total_reasoning_cost_bin":4.215885000000002,"total_cost_bin":10.817503200000003,"total_on_demand_input_cost_bin":11.784672000000002,"ci_99_lower":0.5548653595330216,"ci_99_upper":0.7516879072788459,"ci_98_lower":0.5582749316624455,"ci_98_upper":0.7453847603912136,"ci_95_lower":0.5757710307565183,"ci_95_upper":0.7330885198181718,"ci_90_lower":0.5904613233842504,"ci_90_upper":0.7198674811799292,"ci_75_lower":0.6069011657134122,"ci_75_upper":0.6986826785909858,"ci_50_lower":0.6283271202731104,"ci_50_upper":0.6778832464196431},"65536":{"avg_score":0.588586738585111,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.48305060751863105,"median_score":1,"q1_score":0.02671118453145027,"q3_score":1,"avg_token_efficiency":0.9919747770303085,"is_incomplete":false,"total_prompt_cost_bin":0.41881800000000013,"total_cache_read_cost_bin":1.6127232000000014,"total_completion_cost_bin":4.497464999999999,"total_reasoning_cost_bin":3.967844999999999,"total_cost_bin":10.4968512,"total_on_demand_input_cost_bin":16.546049999999997,"ci_99_lower":0.4424573443587869,"ci_99_upper":0.7131316268813302,"ci_98_lower":0.4605125094576574,"ci_98_upper":0.7035668075985331,"ci_95_lower":0.4934293453455629,"ci_95_upper":0.6920349887155873,"ci_90_lower":0.5006374270677127,"ci_90_upper":0.6787959072530708,"ci_75_lower":0.5305394790368155,"ci_75_upper":0.6476158232167911,"ci_50_lower":0.5540322652661844,"ci_50_upper":0.6226552558761529},"131072":{"avg_score":0.394967571999516,"min_score":0.005760921631008387,"max_score":1,"n_tests":103,"stddev":0.4744842266519795,"median_score":0.03600514307618141,"q1_score":0.021757670678198338,"q3_score":1,"avg_token_efficiency":0.9908788032174033,"is_incomplete":false,"total_prompt_cost_bin":0.43255799999999966,"total_cache_read_cost_bin":3.9638016000000054,"total_completion_cost_bin":7.818900000000002,"total_reasoning_cost_bin":7.158510000000001,"total_cost_bin":19.373769600000006,"total_on_demand_input_cost_bin":40.07057400000001,"ci_99_lower":0.27323410444205753,"ci_99_upper":0.5088427015666136,"ci_98_lower":0.2885043710130222,"ci_98_upper":0.49916685899262975,"ci_95_lower":0.30102537322746487,"ci_95_upper":0.4888781902888898,"ci_90_lower":0.3185251356079921,"ci_90_upper":0.4705977435580916,"ci_75_lower":0.34059963637829566,"ci_75_upper":0.45174083532426434,"ci_50_lower":0.3652949838116946,"ci_50_upper":0.4252685425176909},"262144":{"avg_score":0.29395573869915614,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.4306869527990792,"median_score":0.03907204046845436,"q1_score":0.022673964500427246,"q3_score":1,"avg_token_efficiency":0.9958583241229254,"is_incomplete":false,"total_prompt_cost_bin":0.8786040000000003,"total_cache_read_cost_bin":10.94553599999999,"total_completion_cost_bin":10.151385000000003,"total_reasoning_cost_bin":9.323129999999999,"total_cost_bin":31.298654999999993,"total_on_demand_input_cost_bin":110.3339640000001,"ci_99_lower":0.19974578955412203,"ci_99_upper":0.3800612710892876,"ci_98_lower":0.20963570499416204,"ci_98_upper":0.37341281950282235,"ci_95_lower":0.22344149626892207,"ci_95_upper":0.364343868411833,"ci_90_lower":0.23414329984869334,"ci_90_upper":0.35393647885574225,"ci_75_lower":0.2526029812994393,"ci_75_upper":0.33454063487922814,"ci_50_lower":0.2704446838825202,"ci_50_upper":0.3165009758213918},"524288":{"avg_score":0.1888496915206029,"min_score":0.004279600456357002,"max_score":1,"n_tests":236,"stddev":0.35055904312100516,"median_score":0.03427689149975777,"q1_score":0.023565943352878094,"q3_score":0.06432518362998962,"avg_token_efficiency":0.9912605814879752,"is_incomplete":false,"total_prompt_cost_bin":8.65772400000001,"total_cache_read_cost_bin":35.919667200000085,"total_completion_cost_bin":22.316789999999994,"total_reasoning_cost_bin":20.985915000000002,"total_cost_bin":87.8800962000001,"total_on_demand_input_cost_bin":367.8543960000007,"ci_99_lower":0.13565482760434264,"ci_99_upper":0.2454337880639971,"ci_98_lower":0.138221886513589,"ci_98_upper":0.24037289625992683,"ci_95_lower":0.14662896120349553,"ci_95_upper":0.23481807146414765,"ci_90_lower":0.15192952053798994,"ci_90_upper":0.22587373555120932,"ci_75_lower":0.1644830974992917,"ci_75_upper":0.21539176611468527,"ci_50_lower":0.17326305264220335,"ci_50_upper":0.20342251226272418}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"MoonshotAI: Kimi K3","provider_name":"Moonshotai","run_provider_names":["Moonshot AI"],"pricing_metadata":{"prompt":0.000003,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:30:58.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-12T20:24:53.542Z"},{"model_slug":"google/gemini-3-flash-preview","reasoning_mode":null,"overall_metrics":{"auc_1m":0.2202816917109535,"auc_128k":0.26163530406383934,"cum_avg_1m":0.29210065189551015,"cum_avg_128k":0.33422664256137624,"avg_score_overall":0.256015062191083,"avg_response_time_ms":9968.661168099265,"total_prompt_cost":6.170183000000007,"total_cache_read_cost":24.891813650000085,"total_completion_cost":1.3435919999999997,"total_reasoning_cost":0,"total_prompt_cost_128k":1.9861855000000004,"total_completion_cost_128k":0.5520630000000001,"total_on_demand_input_cost":255.0883194999997,"total_on_demand_input_cost_128k":12.2734,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154894},"bin_metrics":{"8192":{"avg_score":0.5866923093679361,"min_score":0.006989952176809311,"max_score":1,"n_tests":80,"stddev":0.48129538937647937,"median_score":1,"q1_score":0.028082501608878374,"q3_score":1,"avg_token_efficiency":1.0001553404747237,"is_incomplete":false,"total_prompt_cost_bin":0.1321329999999999,"total_cache_read_cost_bin":0.01336390000000001,"total_completion_cost_bin":0.08428200000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.22977889999999992,"total_on_demand_input_cost_bin":0.26577199999999995,"ci_99_lower":0.44143021881423194,"ci_99_upper":0.7316310217891586,"ci_98_lower":0.455170385196805,"ci_98_upper":0.7085504486967111,"ci_95_lower":0.48604775749525286,"ci_95_upper":0.6926694496037089,"ci_90_lower":0.5017486082029063,"ci_90_upper":0.6719823015911971,"ci_75_lower":0.5267950527588254,"ci_75_upper":0.648633137725119,"ci_50_lower":0.5500998188508675,"ci_50_upper":0.6225411757797701},"16384":{"avg_score":0.3572702852427028,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.46149393422152957,"median_score":0.03843617998063564,"q1_score":0.02556890295818448,"q3_score":1,"avg_token_efficiency":1.0000745135024471,"is_incomplete":false,"total_prompt_cost_bin":0.2893825000000002,"total_cache_read_cost_bin":0.02615535000000003,"total_completion_cost_bin":0.08149200000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.39702985000000024,"total_on_demand_input_cost_bin":0.5509359999999998,"ci_99_lower":0.22371326007024617,"ci_99_upper":0.49061722859652945,"ci_98_lower":0.23470777833822648,"ci_98_upper":0.4782799135986715,"ci_95_lower":0.2591836306184996,"ci_95_upper":0.4567943588097112,"ci_90_lower":0.270810562071274,"ci_90_upper":0.44148036382277495,"ci_75_lower":0.29733630895643726,"ci_75_upper":0.4174433416716056,"ci_50_lower":0.32114392222429156,"ci_50_upper":0.3929732574906666},"32768":{"avg_score":0.28381707857239663,"min_score":0.005333333276212215,"max_score":1,"n_tests":136,"stddev":0.4236408857075922,"median_score":0.03771180100739002,"q1_score":0.023628925438970327,"q3_score":0.9989051520824432,"avg_token_efficiency":1.0000359090949913,"is_incomplete":false,"total_prompt_cost_bin":0.7174010000000001,"total_cache_read_cost_bin":0.12193069999999989,"total_completion_cost_bin":0.1577100000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.9970417000000001,"total_on_demand_input_cost_bin":1.9367079999999997,"ci_99_lower":0.1966487687452943,"ci_99_upper":0.37117777466419605,"ci_98_lower":0.20065495269078595,"ci_98_upper":0.36599261465311217,"ci_95_lower":0.21278601830691674,"ci_95_upper":0.35369811862243183,"ci_90_lower":0.22549181845097338,"ci_90_upper":0.3410424734981087,"ci_75_lower":0.24187691330735106,"ci_75_upper":0.3264836122616543,"ci_50_lower":0.2604191649682916,"ci_50_upper":0.3062367624233422},"65536":{"avg_score":0.23520629209849764,"min_score":0.004494382068514824,"max_score":1,"n_tests":85,"stddev":0.399057384495775,"median_score":0.029972752556204796,"q1_score":0.022002199664711952,"q3_score":0.06127138063311577,"avg_token_efficiency":1.0000152878522814,"is_incomplete":false,"total_prompt_cost_bin":0.3727135000000003,"total_cache_read_cost_bin":0.2407314000000002,"total_completion_cost_bin":0.09783599999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.7112809000000004,"total_on_demand_input_cost_bin":2.7800275,"ci_99_lower":0.14181341403980247,"ci_99_upper":0.3507059402211624,"ci_98_lower":0.14414712385734654,"ci_98_upper":0.3382621692846365,"ci_95_lower":0.15638280778137198,"ci_95_upper":0.3197860642958103,"ci_90_lower":0.16750618416158594,"ci_90_upper":0.3092900757529936,"ci_75_lower":0.1866107542779954,"ci_75_upper":0.2810570309623418,"ci_50_lower":0.2023079116938307,"ci_50_upper":0.2590665692983962},"131072":{"avg_score":0.20814724752534797,"min_score":0.006864007096737623,"max_score":1,"n_tests":103,"stddev":0.3775973678957647,"median_score":0.031110163778066635,"q1_score":0.019098056480288506,"q3_score":0.05527668632566929,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":0.47455549999999996,"total_cache_read_cost_bin":0.6265401000000003,"total_completion_cost_bin":0.13074300000000003,"total_reasoning_cost_bin":0,"total_cost_bin":1.2318386000000003,"total_on_demand_input_cost_bin":6.739956500000001,"ci_99_lower":0.12159297973921215,"ci_99_upper":0.30276655278812853,"ci_98_lower":0.1308666001900452,"ci_98_upper":0.2927095014941019,"ci_95_lower":0.1415264033043341,"ci_95_upper":0.2812483451591388,"ci_90_lower":0.15129859242972565,"ci_90_upper":0.26645700144558887,"ci_75_lower":0.16857047853889667,"ci_75_upper":0.24641055137583606,"ci_50_lower":0.18106145545889996,"ci_50_upper":0.22874490493790978},"262144":{"avg_score":0.27716703660759734,"min_score":0.002102312631905079,"max_score":1,"n_tests":141,"stddev":0.41894927272607624,"median_score":0.03726708143949509,"q1_score":0.024609245359897614,"q3_score":0.5106551051139832,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":0.6910025000000001,"total_cache_read_cost_bin":1.7774540000000052,"total_completion_cost_bin":0.16261799999999993,"total_reasoning_cost_bin":0,"total_cost_bin":2.631074500000005,"total_on_demand_input_cost_bin":18.465542499999987,"ci_99_lower":0.18998504882030262,"ci_99_upper":0.3692739131258708,"ci_98_lower":0.19645244248113958,"ci_98_upper":0.35395185584043887,"ci_95_lower":0.2070341339536302,"ci_95_upper":0.3464199631719302,"ci_90_lower":0.22034492477071174,"ci_90_upper":0.3349330940802337,"ci_75_lower":0.23696648034102827,"ci_75_upper":0.3159076375655619,"ci_50_lower":0.2537685835606861,"ci_50_upper":0.3002508296565225},"524288":{"avg_score":0.20636872022256839,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.3660452900712681,"median_score":0.03541246056556702,"q1_score":0.021813183557242155,"q3_score":0.08379341475665569,"avg_token_efficiency":1.000001907854033,"is_incomplete":false,"total_prompt_cost_bin":1.7280245000000007,"total_cache_read_cost_bin":6.0121688999999945,"total_completion_cost_bin":0.26034899999999983,"total_reasoning_cost_bin":0,"total_cost_bin":8.000542399999995,"total_on_demand_input_cost_bin":61.84971350000008,"ci_99_lower":0.14524460635856098,"ci_99_upper":0.26779151601866097,"ci_98_lower":0.15133263845750927,"ci_98_upper":0.26124599772516316,"ci_95_lower":0.16094022567799043,"ci_95_upper":0.25457360209847585,"ci_90_lower":0.16789597672019627,"ci_90_upper":0.24573681649872323,"ci_75_lower":0.1809525943206578,"ci_75_upper":0.2339673544102278,"ci_50_lower":0.19132014451488275,"ci_50_upper":0.22105323508632432},"1048576":{"avg_score":0.18213624552703434,"min_score":0.003673769300803542,"max_score":1,"n_tests":310,"stddev":0.34639245258025914,"median_score":0.03346114419400692,"q1_score":0.02206515707075596,"q3_score":0.06252746004611254,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":1.7649705000000058,"total_cache_read_cost_bin":16.073469300000085,"total_completion_cost_bin":0.36856199999999995,"total_reasoning_cost_bin":0,"total_cost_bin":18.20700180000009,"total_on_demand_input_cost_bin":162.49966349999963,"ci_99_lower":0.13522974154631037,"ci_99_upper":0.2336655630907532,"ci_98_lower":0.13693301831613175,"ci_98_upper":0.2287888556177908,"ci_95_lower":0.14495632810450548,"ci_95_upper":0.22191423464863141,"ci_90_lower":0.15068452950259068,"ci_90_upper":0.21488098184048618,"ci_75_lower":0.1587254956572114,"ci_75_upper":0.20562273716899537,"ci_50_lower":0.1672762526355068,"ci_50_upper":0.1950489255939535}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3 Flash Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":5e-7,"completion":0.000003,"reasoning":0.000003,"has_tiered_pricing":true},"api_created_at":"2025-12-17T15:57:58.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-19T23:17:36.452Z"},{"model_slug":"google/gemini-3.1-flash-lite-preview","reasoning_mode":"high","overall_metrics":{"auc_1m":0.2120776789734172,"auc_128k":0.4802372855470132,"cum_avg_1m":0.4599517045545305,"cum_avg_128k":0.6322775121561033,"avg_score_overall":0.3490238978177143,"avg_response_time_ms":41635.349510421685,"total_prompt_cost":5.628908749999999,"total_cache_read_cost":12.176852675000042,"total_completion_cost":0.657135,"total_reasoning_cost":19.529800500000007,"total_prompt_cost_128k":1.3132472499999999,"total_completion_cost_128k":0.276597,"total_on_demand_input_cost":127.39743549999986,"total_on_demand_input_cost_128k":6.114042999999999,"total_runs":1167,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0093729661451534},"bin_metrics":{"8192":{"avg_score":0.9879343241918832,"min_score":0.03866809979081154,"max_score":1,"n_tests":80,"stddev":0.10747472368916239,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.043355475945916,"is_incomplete":false,"total_prompt_cost_bin":0.13275175,"total_cache_read_cost_bin":0.00061235,"total_completion_cost_bin":0.042247500000000014,"total_reasoning_cost_bin":0.3166005,"total_cost_bin":0.49221210000000004,"total_on_demand_input_cost_bin":0.13887525000000003,"ci_99_lower":0.9518618181794881,"ci_99_upper":0.999981058485806,"ci_98_lower":0.9518787501379847,"ci_98_upper":0.9999790152385831,"ci_95_lower":0.9638890801463276,"ci_95_upper":0.9999726836010814,"ci_90_lower":0.9639083481486886,"ci_90_upper":0.9999673775210978,"ci_75_lower":0.9759096634574235,"ci_75_upper":0.9999578860588373,"ci_50_lower":0.9759359681047499,"ci_50_upper":0.9999437723308802},"16384":{"avg_score":0.8423559367423877,"min_score":0.0166898462921381,"max_score":1,"n_tests":80,"stddev":0.35980646688502593,"median_score":1,"q1_score":0.9994354844093323,"q3_score":1,"avg_token_efficiency":1.0398201343425864,"is_incomplete":false,"total_prompt_cost_bin":0.12806499999999998,"total_cache_read_cost_bin":0.015894149999999996,"total_completion_cost_bin":0.04156049999999999,"total_reasoning_cost_bin":0.4110795,"total_cost_bin":0.59659915,"total_on_demand_input_cost_bin":0.2870064999999998,"ci_99_lower":0.7319372335895896,"ci_99_upper":0.9396878986548399,"ci_98_lower":0.7332951871862169,"ci_98_upper":0.9273596651079132,"ci_95_lower":0.7574558610352687,"ci_95_upper":0.9150815754523501,"ci_90_lower":0.7703248619625811,"ci_90_upper":0.9029618496599141,"ci_75_lower":0.7938131348812021,"ci_75_upper":0.8904323522641789,"ci_50_lower":0.8177275899215601,"ci_50_upper":0.8666487749607767},"32768":{"avg_score":0.6592520896708319,"min_score":0.005958292167633772,"max_score":1,"n_tests":136,"stddev":0.4623101362718979,"median_score":0.9997783601284027,"q1_score":0.04248280543833971,"q3_score":1,"avg_token_efficiency":1.0201346601255257,"is_incomplete":false,"total_prompt_cost_bin":0.2314887500000001,"total_cache_read_cost_bin":0.07566005,"total_completion_cost_bin":0.079386,"total_reasoning_cost_bin":1.0222934999999995,"total_cost_bin":1.4088282999999997,"total_on_demand_input_cost_bin":0.9880892499999994,"ci_99_lower":0.5592804824121412,"ci_99_upper":0.7523340521363185,"ci_98_lower":0.5671760501109598,"ci_98_upper":0.7464701880211941,"ci_95_lower":0.5802567001951769,"ci_95_upper":0.7366240464668783,"ci_90_lower":0.5937322604947599,"ci_90_upper":0.724811253518462,"ci_75_lower":0.6138827596219284,"ci_75_upper":0.7028689879923203,"ci_50_lower":0.6320683602021103,"ci_50_upper":0.6860724184268346},"65536":{"avg_score":0.390460553140763,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4692251705422632,"median_score":0.04197901114821434,"q1_score":0.023379383608698845,"q3_score":0.9997711181640625,"avg_token_efficiency":1.0075590163506454,"is_incomplete":false,"total_prompt_cost_bin":0.24523899999999996,"total_cache_read_cost_bin":0.11552604999999987,"total_completion_cost_bin":0.05057250000000001,"total_reasoning_cost_bin":0.6412889999999999,"total_cost_bin":1.0526265499999998,"total_on_demand_input_cost_bin":1.4004994999999996,"ci_99_lower":0.25982650908099153,"ci_99_upper":0.5304337609651334,"ci_98_lower":0.27491487225933986,"ci_98_upper":0.5178248155080878,"ci_95_lower":0.2900530093338559,"ci_95_upper":0.49037900924353917,"ci_90_lower":0.3057261685605216,"ci_90_upper":0.47720155597719205,"ci_75_lower":0.33142469959908766,"ci_75_upper":0.4496188098770183,"ci_50_lower":0.35580367906402577,"ci_50_upper":0.4256484635420801},"131072":{"avg_score":0.28138465703465043,"min_score":0.004446420818567276,"max_score":1,"n_tests":100,"stddev":0.4262166879161047,"median_score":0.03397811017930508,"q1_score":0.020419847685843706,"q3_score":0.92765873670578,"avg_token_efficiency":1.0084889900680698,"is_incomplete":false,"total_prompt_cost_bin":0.5757027499999998,"total_cache_read_cost_bin":0.27238697500000014,"total_completion_cost_bin":0.06283049999999998,"total_reasoning_cost_bin":1.1878784999999996,"total_cost_bin":2.0987987249999995,"total_on_demand_input_cost_bin":3.2995725,"ci_99_lower":0.18327931610783563,"ci_99_upper":0.39059065717472696,"ci_98_lower":0.1931812480590306,"ci_98_upper":0.3829709547352977,"ci_95_lower":0.20205314627359622,"ci_95_upper":0.36711675017909146,"ci_90_lower":0.21409484451962635,"ci_90_upper":0.35012920253770424,"ci_75_lower":0.23388086085557008,"ci_75_upper":0.3302418168575969,"ci_50_lower":0.25228416216326877,"ci_50_upper":0.3090782928280532},"262144":{"avg_score":0.22541568694299363,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.37900205008269433,"median_score":0.03435399383306503,"q1_score":0.02206287905573845,"q3_score":0.08876560628414154,"avg_token_efficiency":1.0000038179365243,"is_incomplete":false,"total_prompt_cost_bin":0.8162104999999996,"total_cache_read_cost_bin":0.8416560750000011,"total_completion_cost_bin":0.07811850000000006,"total_reasoning_cost_bin":2.839617000000001,"total_cost_bin":4.575602075000002,"total_on_demand_input_cost_bin":9.232771249999995,"ci_99_lower":0.15175561643419425,"ci_99_upper":0.3054193133159527,"ci_98_lower":0.1567858785917282,"ci_98_upper":0.2989737677421579,"ci_95_lower":0.16429858879963968,"ci_95_upper":0.28694562450346883,"ci_90_lower":0.17503300675394093,"ci_90_upper":0.2757740290495336,"ci_75_lower":0.18885581158613457,"ci_75_upper":0.26187487890909206,"ci_50_lower":0.20267016440588678,"ci_50_upper":0.24414345597499876},"524288":{"avg_score":0.16870748529389026,"min_score":0,"max_score":1,"n_tests":235,"stddev":0.32399992388286486,"median_score":0.030886195600032806,"q1_score":0.019100663252174854,"q3_score":0.06463385000824928,"avg_token_efficiency":1.0002282530739481,"is_incomplete":false,"total_prompt_cost_bin":1.4686667500000028,"total_cache_read_cost_bin":2.933212274999998,"total_completion_cost_bin":0.12460950000000008,"total_reasoning_cost_bin":5.869430999999998,"total_cost_bin":10.395919525,"total_on_demand_input_cost_bin":30.800789500000032,"ci_99_lower":0.11974389935897146,"ci_99_upper":0.22597662124754067,"ci_98_lower":0.12359295030673371,"ci_98_upper":0.21923962116414838,"ci_95_lower":0.12840320254472934,"ci_95_upper":0.2132112623088339,"ci_90_lower":0.13494566255125234,"ci_90_upper":0.20413419776819047,"ci_75_lower":0.14450241287394944,"ci_75_upper":0.19286128474780895,"ci_50_lower":0.15368748437712326,"ci_50_upper":0.18295993608833747},"1048576":{"avg_score":0.12410290341884378,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.2630274296625434,"median_score":0.031019440852105618,"q1_score":0.019262225832790136,"q3_score":0.055141400545835495,"avg_token_efficiency":1.0000009538490409,"is_incomplete":false,"total_prompt_cost_bin":2.030784249999997,"total_cache_read_cost_bin":7.921904750000043,"total_completion_cost_bin":0.1778099999999999,"total_reasoning_cost_bin":7.241611500000008,"total_cost_bin":17.372110500000048,"total_on_demand_input_cost_bin":81.24983174999983,"ci_99_lower":0.09028549980438286,"ci_99_upper":0.16554776039425917,"ci_98_lower":0.09329234833645844,"ci_98_upper":0.1614262221620358,"ci_95_lower":0.09584876091380964,"ci_95_upper":0.1561288399949685,"ci_90_lower":0.09907430448238888,"ci_90_upper":0.1506964890124847,"ci_75_lower":0.10663574132711566,"ci_75_upper":0.1420203391841853,"ci_50_lower":0.1134716909716747,"ci_50_upper":0.13340863281778328}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.1 Flash Lite Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":2.5e-7,"completion":0.0000015,"reasoning":0.0000015,"has_tiered_pricing":true},"api_created_at":"2026-03-03T04:37:53.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T00:45:53.816Z"},{"model_slug":"qwen/qwen3.5-plus","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.2100989051429703,"auc_128k":0.6444024437690761,"cum_avg_1m":0.5227760322287035,"cum_avg_128k":0.7505840808736041,"avg_score_overall":0.5007543015784858,"avg_response_time_ms":139176.37068543117,"total_prompt_cost":59.68274169000003,"total_cache_read_cost":0,"total_completion_cost":13.24610196,"total_reasoning_cost":12.581309520000001,"total_prompt_cost_128k":6.504077840000002,"total_completion_cost_128k":5.477670120000001,"total_on_demand_input_cost":59.68274169000003,"total_on_demand_input_cost_128k":6.504077840000002,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0197607994354974},"bin_metrics":{"8192":{"avg_score":0.9689227155758999,"min_score":0.01399631705135107,"max_score":1,"n_tests":80,"stddev":0.1516059441656658,"median_score":0.9974396526813507,"q1_score":0.9915053844451904,"q3_score":1,"avg_token_efficiency":1.0224972703222284,"is_incomplete":false,"total_prompt_cost_bin":0.14128191999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.5753935199999999,"total_reasoning_cost_bin":0.5301379200000003,"total_cost_bin":1.24681336,"total_on_demand_input_cost_bin":0.14128191999999998,"ci_99_lower":0.918075832781964,"ci_99_upper":0.995227483022958,"ci_98_lower":0.9206287696901709,"ci_98_upper":0.9950213753283024,"ci_95_lower":0.9317309789778665,"ci_95_upper":0.9941558449715375,"ci_90_lower":0.9338276265910826,"ci_90_upper":0.9933400378748773,"ci_75_lower":0.9458234137622639,"ci_75_upper":0.9906202781014144,"ci_50_lower":0.9573752372409218,"ci_50_upper":0.9811127404478611},"16384":{"avg_score":0.94318091761088,"min_score":0.011840158142149448,"max_score":1,"n_tests":80,"stddev":0.2128952656171393,"median_score":0.9964780211448669,"q1_score":0.9901001006364822,"q3_score":0.9988732784986496,"avg_token_efficiency":1.0159704488335681,"is_incomplete":false,"total_prompt_cost_bin":0.29138304000000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.6225273600000001,"total_reasoning_cost_bin":0.5778926400000001,"total_cost_bin":1.4918030400000002,"total_on_demand_input_cost_bin":0.29138304000000004,"ci_99_lower":0.8707703135015327,"ci_99_upper":0.9925489674434066,"ci_98_lower":0.8800981646761066,"ci_98_upper":0.9915454002544284,"ci_95_lower":0.8832912530889735,"ci_95_upper":0.9813793924299534,"ci_90_lower":0.895241897610249,"ci_90_upper":0.9794268291402841,"ci_75_lower":0.9177758482575883,"ci_75_upper":0.9681738111859886,"ci_50_lower":0.9299315477517666,"ci_50_upper":0.9565957907150733},"32768":{"avg_score":0.7973111655412461,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.374110076084905,"median_score":0.9880377352237701,"q1_score":0.9561671614646912,"q3_score":0.9993285238742828,"avg_token_efficiency":1.0256830410679836,"is_incomplete":false,"total_prompt_cost_bin":1.0339180799999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.3879179600000007,"total_reasoning_cost_bin":1.3020134400000003,"total_cost_bin":3.723849480000001,"total_on_demand_input_cost_bin":1.0339180799999996,"ci_99_lower":0.7075924369234332,"ci_99_upper":0.8799170444836936,"ci_98_lower":0.716216425340116,"ci_98_upper":0.872265758166391,"ci_95_lower":0.7312766143110554,"ci_95_upper":0.8576174296734869,"ci_90_lower":0.7394298775056514,"ci_90_upper":0.8488436981405625,"ci_75_lower":0.7577229402616948,"ci_75_upper":0.8348087714522776,"ci_50_lower":0.77437696534702,"ci_50_upper":0.8205803507435655},"65536":{"avg_score":0.600424001574078,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.47340298189096014,"median_score":0.9803921580314636,"q1_score":0.028731433674693108,"q3_score":0.9981395602226257,"avg_token_efficiency":1.0209598255175443,"is_incomplete":false,"total_prompt_cost_bin":1.4758915600000015,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.1747907599999998,"total_reasoning_cost_bin":1.1168710800000003,"total_cost_bin":3.7675534000000015,"total_on_demand_input_cost_bin":1.4758915600000015,"ci_99_lower":0.46369856976597185,"ci_99_upper":0.7213451044750127,"ci_98_lower":0.47989102827297414,"ci_98_upper":0.7152072722225943,"ci_95_lower":0.5006507135168923,"ci_95_upper":0.7030157668713262,"ci_90_lower":0.5190944110103609,"ci_90_upper":0.6864254120036084,"ci_75_lower":0.5432144360356581,"ci_75_upper":0.6609048124635593,"ci_50_lower":0.5650544177603852,"ci_50_upper":0.6341891169931521},"131072":{"avg_score":0.44308160406591607,"min_score":0.0010498687624931335,"max_score":1,"n_tests":103,"stddev":0.47776876292978543,"median_score":0.04006030410528183,"q1_score":0.0218181274831295,"q3_score":0.9881417751312256,"avg_token_efficiency":1.0162215936488594,"is_incomplete":false,"total_prompt_cost_bin":3.56160324,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.71704052,"total_reasoning_cost_bin":1.64726952,"total_cost_bin":6.9259132800000005,"total_on_demand_input_cost_bin":3.56160324,"ci_99_lower":0.31272984200076975,"ci_99_upper":0.5546910923095736,"ci_98_lower":0.3308516998097834,"ci_98_upper":0.5458416040833967,"ci_95_lower":0.3487808931835936,"ci_95_upper":0.5353811753428884,"ci_90_lower":0.36587035796668343,"ci_90_upper":0.519389163866838,"ci_75_lower":0.3869763230356768,"ci_75_upper":0.49945456553216644,"ci_50_lower":0.4135532167822674,"ci_50_upper":0.47901670677300306},"262144":{"avg_score":0.29724772925264076,"min_score":0.002102312631905079,"max_score":1,"n_tests":141,"stddev":0.430721620523732,"median_score":0.03849855810403824,"q1_score":0.023127200081944466,"q3_score":0.9661687016487122,"avg_token_efficiency":1.018066042397412,"is_incomplete":false,"total_prompt_cost_bin":12.2193955,"total_cache_read_cost_bin":0,"total_completion_cost_bin":3.33929076,"total_reasoning_cost_bin":3.2279020799999993,"total_cost_bin":18.786588339999998,"total_on_demand_input_cost_bin":12.2193955,"ci_99_lower":0.2081233231416418,"ci_99_upper":0.3877835928581586,"ci_98_lower":0.2143610905686638,"ci_98_upper":0.37402444122854533,"ci_95_lower":0.2280091428028664,"ci_95_upper":0.36733036620558096,"ci_90_lower":0.2376654602977751,"ci_90_upper":0.3576195994398642,"ci_75_lower":0.25596454966807547,"ci_75_upper":0.33753932194027997,"ci_50_lower":0.2751028493991293,"ci_50_upper":0.3211718058970704},"524288":{"avg_score":0.1320401242089663,"min_score":0.002326934365555644,"max_score":1,"n_tests":236,"stddev":0.2868602990992516,"median_score":0.029866889119148254,"q1_score":0.018884653691202402,"q3_score":0.049731483682990074,"avg_token_efficiency":1.0188305758618426,"is_incomplete":false,"total_prompt_cost_bin":40.95926835000003,"total_cache_read_cost_bin":0,"total_completion_cost_bin":4.42914108,"total_reasoning_cost_bin":4.17922284,"total_cost_bin":49.56763227000003,"total_on_demand_input_cost_bin":40.95926835000003,"ci_99_lower":0.08912165184538333,"ci_99_upper":0.17927164457054212,"ci_98_lower":0.09174792429429313,"ci_98_upper":0.17593695209323137,"ci_95_lower":0.09783927371520233,"ci_95_upper":0.17059202695761658,"ci_90_lower":0.10304969902100018,"ci_90_upper":0.16448529831472175,"ci_75_lower":0.11102061457988723,"ci_75_upper":0.15361655516225858,"ci_50_lower":0.11874558616422662,"ci_50_upper":0.14361804174466486}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5 Plus 2026-02-15","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":2.6e-7,"completion":0.00000156,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-16T08:10:16.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T16:25:11.886Z"},{"model_slug":"x-ai/grok-4.6","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.21007328405764816,"auc_128k":0.7341246526580131,"cum_avg_1m":0.5688925192468711,"cum_avg_128k":0.8147446934182383,"avg_score_overall":0.7345051431819797,"avg_response_time_ms":54521.0740265625,"total_prompt_cost":62.35125800000001,"total_cache_read_cost":31.83967999999998,"total_completion_cost":14.846628000000004,"total_reasoning_cost":13.469933999999999,"total_prompt_cost_128k":20.923304,"total_completion_cost_128k":11.667114000000005,"total_on_demand_input_cost":189.77283199999994,"total_on_demand_input_cost_128k":47.43568799999999,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9708195439504925},"bin_metrics":{"8192":{"avg_score":0.9757435087580234,"min_score":0.017316017299890518,"max_score":1,"n_tests":80,"stddev":0.1522562084931854,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9944309981382429,"is_incomplete":false,"total_prompt_cost_bin":0.9272480000000002,"total_cache_read_cost_bin":0.032448000000000046,"total_completion_cost_bin":1.1436300000000006,"total_reasoning_cost_bin":0.98259,"total_cost_bin":3.085916000000001,"total_on_demand_input_cost_bin":1.0570399999999993,"ci_99_lower":0.9269182120955083,"ci_99_upper":1,"ci_98_lower":0.9275960469963029,"ci_98_upper":1,"ci_95_lower":0.9391729054506868,"ci_95_upper":1,"ci_90_lower":0.939946933509782,"ci_90_upper":1,"ci_75_lower":0.9518587504047901,"ci_75_upper":0.9999083131551743,"ci_50_lower":0.9637705672997982,"ci_50_upper":0.9879964962601662},"16384":{"avg_score":0.9607967002084479,"min_score":0.030150754377245903,"max_score":1,"n_tests":80,"stddev":0.16739588908342481,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9657422790751816,"is_incomplete":false,"total_prompt_cost_bin":1.5053280000000004,"total_cache_read_cost_bin":0.155456,"total_completion_cost_bin":1.8134700000000001,"total_reasoning_cost_bin":1.6552020000000003,"total_cost_bin":5.129456000000001,"total_on_demand_input_cost_bin":2.1271519999999993,"ci_99_lower":0.9035911201458656,"ci_99_upper":0.9964378572404385,"ci_98_lower":0.9077042712890543,"ci_98_upper":0.9961433706507087,"ci_95_lower":0.9148613922530785,"ci_95_upper":0.992075298698619,"ci_90_lower":0.9252219989732839,"ci_90_upper":0.9885165726952254,"ci_75_lower":0.9376444489229471,"ci_75_upper":0.981262886011973,"ci_50_lower":0.9497057249012869,"ci_50_upper":0.9730107511393726},"32768":{"avg_score":0.8959791211698971,"min_score":0.014925372786819935,"max_score":1,"n_tests":136,"stddev":0.2982411584014983,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9792742209463953,"is_incomplete":false,"total_prompt_cost_bin":3.5573920000000023,"total_cache_read_cost_bin":1.0099839999999993,"total_completion_cost_bin":3.2888640000000007,"total_reasoning_cost_bin":2.981843999999998,"total_cost_bin":10.838083999999998,"total_on_demand_input_cost_bin":7.597327999999995,"ci_99_lower":0.8238679506467751,"ci_99_upper":0.9568420305243104,"ci_98_lower":0.8315693395072595,"ci_98_upper":0.9500026580944713,"ci_95_lower":0.8441408557538285,"ci_95_upper":0.9435048043931601,"ci_90_lower":0.8502218197397752,"ci_90_upper":0.9359234862303472,"ci_75_lower":0.8641017846282065,"ci_75_upper":0.9246065655683837,"ci_50_lower":0.8782809804405962,"ci_50_upper":0.9142550072608763},"65536":{"avg_score":0.7150245369247654,"min_score":0.008384146727621555,"max_score":1,"n_tests":85,"stddev":0.44406730335198336,"median_score":1,"q1_score":0.04187099263072014,"q3_score":1,"avg_token_efficiency":0.9612913381648316,"is_incomplete":false,"total_prompt_cost_bin":5.831902000000002,"total_cache_read_cost_bin":1.214400000000001,"total_completion_cost_bin":2.4652920000000007,"total_reasoning_cost_bin":2.267106,"total_cost_bin":11.778700000000004,"total_on_demand_input_cost_bin":10.689501999999997,"ci_99_lower":0.587051652695381,"ci_99_upper":0.8301838332784417,"ci_98_lower":0.5909680572647381,"ci_98_upper":0.8183516378467354,"ci_95_lower":0.6210046333400533,"ci_95_upper":0.8057306848937537,"ci_90_lower":0.6333357958035434,"ci_90_upper":0.7935117623363347,"ci_75_lower":0.6569372493417605,"ci_75_upper":0.7708874430863516,"ci_50_lower":0.6805153942119111,"ci_50_upper":0.747263782720684},"131072":{"avg_score":0.5261796000300573,"min_score":0.0034822982270270586,"max_score":1,"n_tests":103,"stddev":0.489989906060912,"median_score":0.9971181750297546,"q1_score":0.02378109749406576,"q3_score":1,"avg_token_efficiency":0.9630946579012022,"is_incomplete":false,"total_prompt_cost_bin":9.101433999999998,"total_cache_read_cost_bin":4.215807999999997,"total_completion_cost_bin":2.955858000000002,"total_reasoning_cost_bin":2.7113879999999995,"total_cost_bin":18.984487999999995,"total_on_demand_input_cost_bin":25.964666,"ci_99_lower":0.3946072989909738,"ci_99_upper":0.649438355075392,"ci_98_lower":0.420032062005654,"ci_98_upper":0.6387977820799404,"ci_95_lower":0.432511726790786,"ci_95_upper":0.6205984699148775,"ci_90_lower":0.44976007881861413,"ci_90_upper":0.6102835721241364,"ci_75_lower":0.4694858715941257,"ci_75_upper":0.5831870445781003,"ci_50_lower":0.4942313472003432,"ci_50_upper":0.5633856910314791},"262144":{"avg_score":0.4774166868837775,"min_score":0.002326934365555644,"max_score":1,"n_tests":141,"stddev":0.4799445512780843,"median_score":0.0666959211230278,"q1_score":0.026810724288225174,"q3_score":1,"avg_token_efficiency":0.9635357818116278,"is_incomplete":false,"total_prompt_cost_bin":41.42795400000001,"total_cache_read_cost_bin":25.211583999999984,"total_completion_cost_bin":3.1795139999999997,"total_reasoning_cost_bin":2.871804,"total_cost_bin":72.69085599999998,"total_on_demand_input_cost_bin":142.33714399999994,"ci_99_lower":0.365005195536692,"ci_99_upper":0.5920101867801449,"ci_98_lower":0.3730054869873991,"ci_98_upper":0.5761331198662056,"ci_95_lower":0.39755083262604163,"ci_95_upper":0.5600253133547447,"ci_90_lower":0.41133832124719094,"ci_90_upper":0.5448008595701598,"ci_75_lower":0.4323050102469981,"ci_75_upper":0.5260344325238261,"ci_50_lower":0.45045875207808345,"ci_50_upper":0.5048563873093139}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":500000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","enabled"],"reasoning_default":"high","sponsors":[],"openrouter_name":"SpaceXAI: Grok 4.6","provider_name":"X-ai","run_provider_names":["SpaceXAI"],"pricing_metadata":{"prompt":0.000002,"completion":0.000006,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:35:57.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T20:52:13.528Z"},{"model_slug":"openai/gpt-5.5","reasoning_mode":null,"overall_metrics":{"auc_1m":0.20756501103361866,"auc_128k":0.36930334842064394,"cum_avg_1m":0.33886694486307806,"cum_avg_128k":0.4373637996180603,"avg_score_overall":0.34900587593485427,"avg_response_time_ms":5612.409940748403,"total_prompt_cost":27.89838000000001,"total_cache_read_cost":148.2213119999993,"total_completion_cost":9.80613,"total_reasoning_cost":0,"total_prompt_cost_128k":5.022205,"total_completion_cost_128k":5.562179999999999,"total_on_demand_input_cost":1510.811050000002,"total_on_demand_input_cost_128k":120.269565,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778016},"bin_metrics":{"8192":{"avg_score":0.6637943080218974,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.46134781471777847,"median_score":1,"q1_score":0.04150762688368559,"q3_score":1,"avg_token_efficiency":0.9759044449598389,"is_incomplete":false,"total_prompt_cost_bin":0.37771999999999983,"total_cache_read_cost_bin":0.22182399999999994,"total_completion_cost_bin":0.84711,"total_reasoning_cost_bin":0,"total_cost_bin":1.4466539999999997,"total_on_demand_input_cost_bin":2.59596,"ci_99_lower":0.5192743666168245,"ci_99_upper":0.7866662066945573,"ci_98_lower":0.5424028099554706,"ci_98_upper":0.7730970278345047,"ci_95_lower":0.5656256348901662,"ci_95_upper":0.7587122143607121,"ci_90_lower":0.5791445131343789,"ci_90_upper":0.7457553225159063,"ci_75_lower":0.6038749731618737,"ci_75_upper":0.7227624696628482,"ci_50_lower":0.6269891965930583,"ci_50_upper":0.6992344655445776},"16384":{"avg_score":0.514287691534264,"min_score":0.0044469148851931095,"max_score":1,"n_tests":80,"stddev":0.4888539085336498,"median_score":0.5301212146878242,"q1_score":0.026397607754915953,"q3_score":1,"avg_token_efficiency":0.9670687393188713,"is_incomplete":false,"total_prompt_cost_bin":0.5681200000000001,"total_cache_read_cost_bin":0.476032,"total_completion_cost_bin":0.7925999999999999,"total_reasoning_cost_bin":0,"total_cost_bin":1.8367519999999997,"total_on_demand_input_cost_bin":5.328439999999994,"ci_99_lower":0.3692208723412187,"ci_99_upper":0.6487765623916639,"ci_98_lower":0.37998605029116156,"ci_98_upper":0.6355369796575979,"ci_95_lower":0.4021909050912655,"ci_95_upper":0.6229053927185305,"ci_90_lower":0.41742980812094177,"ci_90_upper":0.6003544544917531,"ci_75_lower":0.45365453215199525,"ci_75_upper":0.5758729902714549,"ci_50_lower":0.47772355942870487,"ci_50_upper":0.5509480502150836},"32768":{"avg_score":0.34996160210849825,"min_score":0.003944773226976395,"max_score":1,"n_tests":136,"stddev":0.45184972018015995,"median_score":0.04722846485674381,"q1_score":0.02696936996653676,"q3_score":1,"avg_token_efficiency":0.9952100768549628,"is_incomplete":false,"total_prompt_cost_bin":1.0378399999999992,"total_cache_read_cost_bin":1.8261760000000045,"total_completion_cost_bin":1.606979999999998,"total_reasoning_cost_bin":0,"total_cost_bin":4.470996000000002,"total_on_demand_input_cost_bin":19.299600000000005,"ci_99_lower":0.2527436416704714,"ci_99_upper":0.4493937973186257,"ci_98_lower":0.25825467010593617,"ci_98_upper":0.4414565980425173,"ci_95_lower":0.27252843679803734,"ci_95_upper":0.42601440341788793,"ci_90_lower":0.28422454800011643,"ci_90_upper":0.41449592324326445,"ci_75_lower":0.3050511046479668,"ci_75_upper":0.3974673331014107,"ci_50_lower":0.3226043458288426,"ci_50_upper":0.3756671346422723},"65536":{"avg_score":0.3756175714846262,"min_score":0.0038729666266590357,"max_score":1,"n_tests":85,"stddev":0.4640899889979661,"median_score":0.048272356390953064,"q1_score":0.02659202180802822,"q3_score":1,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":1.0433050000000013,"total_cache_read_cost_bin":2.6135039999999954,"total_completion_cost_bin":1.0188300000000001,"total_reasoning_cost_bin":0,"total_cost_bin":4.675638999999997,"total_on_demand_input_cost_bin":27.178345000000018,"ci_99_lower":0.2489502910483656,"ci_99_upper":0.5120883570424117,"ci_98_lower":0.2622425764440011,"ci_98_upper":0.490703902073314,"ci_95_lower":0.2809474599146394,"ci_95_upper":0.4764783553110764,"ci_90_lower":0.2946010187362759,"ci_90_upper":0.45730945509105153,"ci_75_lower":0.316463543473647,"ci_75_upper":0.4329843016902861,"ci_50_lower":0.34097568440689324,"ci_50_upper":0.40966214301639836},"131072":{"avg_score":0.28315782494101566,"min_score":0.004666355438530445,"max_score":1,"n_tests":103,"stddev":0.42722137982009145,"median_score":0.033032454550266266,"q1_score":0.02273651771247387,"q3_score":0.9437693357467651,"avg_token_efficiency":0.9772722316188368,"is_incomplete":false,"total_prompt_cost_bin":1.995219999999999,"total_cache_read_cost_bin":6.387199999999995,"total_completion_cost_bin":1.2966600000000004,"total_reasoning_cost_bin":0,"total_cost_bin":9.679079999999995,"total_on_demand_input_cost_bin":65.86721999999997,"ci_99_lower":0.17875228061869708,"ci_99_upper":0.3868101048287348,"ci_98_lower":0.18860325999318361,"ci_98_upper":0.3844731105767751,"ci_95_lower":0.20768220302133453,"ci_95_upper":0.36533843523910886,"ci_90_lower":0.2168389294181218,"ci_90_upper":0.3499231723128157,"ci_75_lower":0.23633182593293542,"ci_75_upper":0.3297512678817159,"ci_50_lower":0.2549236037698348,"ci_50_upper":0.3109741204755055},"262144":{"avg_score":0.2750688059856547,"min_score":0.002102312631905079,"max_score":1,"n_tests":141,"stddev":0.4188068261381019,"median_score":0.03812117129564285,"q1_score":0.022857142612338066,"q3_score":0.2101576179265976,"avg_token_efficiency":0.9824198103380463,"is_incomplete":false,"total_prompt_cost_bin":6.806215000000005,"total_cache_read_cost_bin":17.460224000000025,"total_completion_cost_bin":1.6205099999999992,"total_reasoning_cost_bin":0,"total_cost_bin":25.88694900000003,"total_on_demand_input_cost_bin":181.4084549999998,"ci_99_lower":0.1877583443906145,"ci_99_upper":0.3662284744008616,"ci_98_lower":0.19561581507642217,"ci_98_upper":0.35203394927028303,"ci_95_lower":0.20550670124464898,"ci_95_upper":0.3455557705829554,"ci_90_lower":0.21784215950215538,"ci_90_upper":0.33035559043443447,"ci_75_lower":0.23468917657964672,"ci_75_upper":0.3145491781649477,"ci_50_lower":0.251287052535835,"ci_50_upper":0.297972883808704},"524288":{"avg_score":0.24904775482866817,"min_score":0.0021798366215080023,"max_score":1,"n_tests":236,"stddev":0.4010178475531242,"median_score":0.03453824855387211,"q1_score":0.02044420689344406,"q3_score":0.12107458896934986,"avg_token_efficiency":0.977478526922676,"is_incomplete":false,"total_prompt_cost_bin":16.069960000000005,"total_cache_read_cost_bin":119.23635199999927,"total_completion_cost_bin":2.6234400000000004,"total_reasoning_cost_bin":0,"total_cost_bin":137.92975199999927,"total_on_demand_input_cost_bin":1209.1330300000022,"ci_99_lower":0.1824676605938713,"ci_99_upper":0.317414749503699,"ci_98_lower":0.1872828369031333,"ci_98_upper":0.3094193229996263,"ci_95_lower":0.19790649324733475,"ci_95_upper":0.3029889583517107,"ci_90_lower":0.2075793547724508,"ci_90_upper":0.29479734325816975,"ci_75_lower":0.219508432034996,"ci_75_upper":0.2796316197593608,"ci_50_lower":0.2316294064514943,"ci_50_upper":0.26613752105888927}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.5","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.000005,"completion":0.00003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T17:31:33.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-25T06:56:43.223Z"},{"model_slug":"x-ai/grok-4.6","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.20550183677031625,"auc_128k":0.7441578889159068,"cum_avg_1m":0.5710755194471351,"cum_avg_128k":0.8200792706699757,"avg_score_overall":0.7356418636098504,"avg_response_time_ms":44130.3944859375,"total_prompt_cost":70.456096,"total_cache_read_cost":29.796415999999994,"total_completion_cost":12.328326000000002,"total_reasoning_cost":10.952826,"total_prompt_cost_128k":22.91632,"total_completion_cost_128k":9.620592000000002,"total_on_demand_input_cost":189.77130000000005,"total_on_demand_input_cost_128k":47.43472000000003,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9707781091017181},"bin_metrics":{"8192":{"avg_score":0.9833870843052864,"min_score":0.044609665870666504,"max_score":1,"n_tests":80,"stddev":0.11420061217472086,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9942756576635192,"is_incomplete":false,"total_prompt_cost_bin":0.9378399999999999,"total_cache_read_cost_bin":0.02976000000000005,"total_completion_cost_bin":1.02471,"total_reasoning_cost_bin":0.86373,"total_cost_bin":2.8560399999999997,"total_on_demand_input_cost_bin":1.0568799999999994,"ci_99_lower":0.9428894102573395,"ci_99_upper":1,"ci_98_lower":0.9475392544716598,"ci_98_upper":1,"ci_95_lower":0.9548317894339562,"ci_95_upper":1,"ci_90_lower":0.9595023259520532,"ci_90_upper":1,"ci_75_lower":0.9667741686105729,"ci_75_upper":1,"ci_50_lower":0.9761152416467667,"ci_50_upper":0.9953294634819031},"16384":{"avg_score":0.9692344612441957,"min_score":0.07425742596387863,"max_score":1,"n_tests":80,"stddev":0.141132173769735,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9656677655727351,"is_incomplete":false,"total_prompt_cost_bin":1.7637280000000002,"total_cache_read_cost_bin":0.09081599999999995,"total_completion_cost_bin":1.5012600000000005,"total_reasoning_cost_bin":1.344726,"total_cost_bin":4.7005300000000005,"total_on_demand_input_cost_bin":2.126991999999998,"ci_99_lower":0.9188076772722416,"ci_99_upper":0.9997022420950235,"ci_98_lower":0.9245663721654565,"ci_98_upper":0.9996105999872088,"ci_95_lower":0.9328607902745716,"ci_95_upper":0.995231241043657,"ci_90_lower":0.9391483963560313,"ci_90_upper":0.9926292429864406,"ci_75_lower":0.9504230168298818,"ci_75_upper":0.9867126057390124,"ci_50_lower":0.9587366891093552,"ci_50_upper":0.9806473745731638},"32768":{"avg_score":0.9149937754743459,"min_score":0.020003637298941612,"max_score":1,"n_tests":136,"stddev":0.2687643631849811,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9792383118514026,"is_incomplete":false,"total_prompt_cost_bin":4.307967999999998,"total_cache_read_cost_bin":0.8222719999999997,"total_completion_cost_bin":2.7146220000000008,"total_reasoning_cost_bin":2.405742,"total_cost_bin":10.250604,"total_on_demand_input_cost_bin":7.597056000000001,"ci_99_lower":0.848162365951233,"ci_99_upper":0.9687669231969288,"ci_98_lower":0.8568035400161684,"ci_98_upper":0.9644827914852923,"ci_95_lower":0.8670600845899623,"ci_95_upper":0.957819475504734,"ci_90_lower":0.8722826693775407,"ci_90_upper":0.9506924542844952,"ci_75_lower":0.8866685296662207,"ci_75_upper":0.9416046297804261,"ci_50_lower":0.8989926621468518,"ci_50_upper":0.9315988195834078},"65536":{"avg_score":0.7703588219962137,"min_score":0.004705882165580988,"max_score":1,"n_tests":85,"stddev":0.412151392989762,"median_score":1,"q1_score":0.9947534203529358,"q3_score":1,"avg_token_efficiency":0.9612760503125503,"is_incomplete":false,"total_prompt_cost_bin":5.526324000000001,"total_cache_read_cost_bin":1.2907520000000006,"total_completion_cost_bin":2.0787000000000004,"total_reasoning_cost_bin":1.8829560000000003,"total_cost_bin":10.778732000000002,"total_on_demand_input_cost_bin":10.68933200000001,"ci_99_lower":0.6409950370895128,"ci_99_upper":0.875651854401372,"ci_98_lower":0.6479641451620881,"ci_98_upper":0.8631919412858345,"ci_95_lower":0.6758866319605423,"ci_95_upper":0.8510162043807042,"ci_90_lower":0.6941390535638065,"ci_90_upper":0.8405639381498536,"ci_75_lower":0.7185804954702583,"ci_75_upper":0.8189404255354448,"ci_50_lower":0.738458944023039,"ci_50_upper":0.8021019481396412},"131072":{"avg_score":0.46242221032983616,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.4851894344714862,"median_score":0.051020409911870956,"q1_score":0.026049822568893433,"q3_score":1,"avg_token_efficiency":0.9630870168439464,"is_incomplete":false,"total_prompt_cost_bin":10.380459999999998,"total_cache_read_cost_bin":3.895999999999998,"total_completion_cost_bin":2.3013,"total_reasoning_cost_bin":2.0543879999999994,"total_cost_bin":18.632147999999997,"total_on_demand_input_cost_bin":25.964460000000013,"ci_99_lower":0.3379996353230728,"ci_99_upper":0.5758037958317888,"ci_98_lower":0.3497231434529322,"ci_98_upper":0.5654301645413258,"ci_95_lower":0.3624308446846062,"ci_95_upper":0.5552764596747578,"ci_90_lower":0.38612154753951045,"ci_90_upper":0.5455416647466442,"ci_75_lower":0.40594014239630855,"ci_75_upper":0.5198557462251646,"ci_50_lower":0.426654939762389,"ci_50_upper":0.49894740603846105},"262144":{"avg_score":0.4682078022272029,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.48035048269914404,"median_score":0.05714285746216774,"q1_score":0.029720280319452286,"q3_score":1,"avg_token_efficiency":0.9635319638751024,"is_incomplete":false,"total_prompt_cost_bin":47.53977600000001,"total_cache_read_cost_bin":23.666815999999994,"total_completion_cost_bin":2.7077340000000008,"total_reasoning_cost_bin":2.401284,"total_cost_bin":76.31561,"total_on_demand_input_cost_bin":142.33658000000003,"ci_99_lower":0.3526149963645978,"ci_99_upper":0.5823651054169625,"ci_98_lower":0.3689999559369757,"ci_98_upper":0.5699653956800012,"ci_95_lower":0.3875419272892573,"ci_95_upper":0.5503454022751857,"ci_90_lower":0.40044567614054,"ci_90_upper":0.5356067685337724,"ci_75_lower":0.4203784805648227,"ci_75_upper":0.5171374793095078,"ci_50_lower":0.4409687004352636,"ci_50_upper":0.4949342737978337}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":500000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","enabled"],"reasoning_default":"high","sponsors":[],"openrouter_name":"SpaceXAI: Grok 4.6","provider_name":"X-ai","run_provider_names":["SpaceXAI"],"pricing_metadata":{"prompt":0.000002,"completion":0.000006,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:35:57.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T01:01:44.664Z"},{"model_slug":"x-ai/grok-4.6","reasoning_mode":"high","overall_metrics":{"auc_1m":0.20242549264004722,"auc_128k":0.7191689207557711,"cum_avg_1m":0.5537895901246171,"cum_avg_128k":0.7943938289690182,"avg_score_overall":0.7154291842989624,"avg_response_time_ms":48847.239571875,"total_prompt_cost":69.610006,"total_cache_read_cost":29.992895999999988,"total_completion_cost":13.432265999999998,"total_reasoning_cost":12.053898000000004,"total_prompt_cost_128k":22.30576,"total_completion_cost_128k":10.342434,"total_on_demand_input_cost":189.7713,"total_on_demand_input_cost_128k":47.43472000000003,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9707781091017185},"bin_metrics":{"8192":{"avg_score":0.9586967571871355,"min_score":0.017316017299890518,"max_score":1,"n_tests":80,"stddev":0.1899992821256898,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9942756576635194,"is_incomplete":false,"total_prompt_cost_bin":0.9066079999999999,"total_cache_read_cost_bin":0.03756800000000005,"total_completion_cost_bin":1.2176519999999997,"total_reasoning_cost_bin":1.0546679999999997,"total_cost_bin":3.2164959999999994,"total_on_demand_input_cost_bin":1.0568799999999996,"ci_99_lower":0.8933949312698096,"ci_99_upper":0.9999289788603782,"ci_98_lower":0.8978135491218417,"ci_98_upper":0.9999079841226339,"ci_95_lower":0.9083769463654608,"ci_95_upper":0.9952563091740012,"ci_90_lower":0.9176903335866519,"ci_90_upper":0.9880426976527087,"ci_75_lower":0.9342238268465735,"ci_75_upper":0.9830728292698041,"ci_50_lower":0.9464314713375642,"ci_50_upper":0.9735816622094717},"16384":{"avg_score":0.9151161829242482,"min_score":0.02924695983529091,"max_score":1,"n_tests":80,"stddev":0.27307822592561437,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9656677655727351,"is_incomplete":false,"total_prompt_cost_bin":1.6731040000000013,"total_cache_read_cost_bin":0.11347199999999984,"total_completion_cost_bin":1.5066060000000001,"total_reasoning_cost_bin":1.3477979999999996,"total_cost_bin":4.640980000000001,"total_on_demand_input_cost_bin":2.126991999999999,"ci_99_lower":0.8201038165973732,"ci_99_upper":0.9852523305154173,"ci_98_lower":0.8307950187865644,"ci_98_upper":0.9755681142201647,"ci_95_lower":0.8435369735799031,"ci_95_upper":0.9733723889582325,"ci_90_lower":0.8556327551719732,"ci_90_upper":0.9631064835190773,"ci_75_lower":0.8785864971607226,"ci_75_upper":0.9507532793213612,"ci_50_lower":0.8915696555923205,"ci_50_upper":0.9385217120172455},"32768":{"avg_score":0.8911285294897799,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.30691894598353237,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9792383118514034,"is_incomplete":false,"total_prompt_cost_bin":3.944704000000001,"total_cache_read_cost_bin":0.9130879999999992,"total_completion_cost_bin":3.1109940000000016,"total_reasoning_cost_bin":2.8066500000000008,"total_cost_bin":10.775436000000003,"total_on_demand_input_cost_bin":7.597056000000007,"ci_99_lower":0.8188497373978745,"ci_99_upper":0.9543439562510707,"ci_98_lower":0.8267326322667,"ci_98_upper":0.9484625700679744,"ci_95_lower":0.8346667070569204,"ci_95_upper":0.9413553541016881,"ci_90_lower":0.8476799148299238,"ci_90_upper":0.9341401042224949,"ci_75_lower":0.8609800971132833,"ci_75_upper":0.9202940069299076,"ci_50_lower":0.8705959768755266,"ci_50_upper":0.9115963722228224},"65536":{"avg_score":0.7170074344557874,"min_score":0.004436229355633259,"max_score":1,"n_tests":85,"stddev":0.44165744989244166,"median_score":1,"q1_score":0.039948999881744385,"q3_score":1,"avg_token_efficiency":0.9612760503125503,"is_incomplete":false,"total_prompt_cost_bin":5.400884,"total_cache_read_cost_bin":1.3221119999999995,"total_completion_cost_bin":2.1161820000000002,"total_reasoning_cost_bin":1.9205700000000006,"total_cost_bin":10.759748000000002,"total_on_demand_input_cost_bin":10.689332000000002,"ci_99_lower":0.5861388526118415,"ci_99_upper":0.8309473804892314,"ci_98_lower":0.5951593725976041,"ci_98_upper":0.8218386250951273,"ci_95_lower":0.6213693229980111,"ci_95_upper":0.8069378782359554,"ci_90_lower":0.6350570858017925,"ci_90_upper":0.7940649417037254,"ci_75_lower":0.6590631477260852,"ci_75_upper":0.7717190688147264,"ci_50_lower":0.6826364547753816,"ci_50_upper":0.747277535287225},"131072":{"avg_score":0.49002024078814005,"min_score":0.005772005766630173,"max_score":1,"n_tests":103,"stddev":0.48735575103378415,"median_score":0.058394160121679306,"q1_score":0.026095355860888958,"q3_score":1,"avg_token_efficiency":0.9630870168439468,"is_incomplete":false,"total_prompt_cost_bin":10.380459999999998,"total_cache_read_cost_bin":3.8959999999999977,"total_completion_cost_bin":2.390999999999999,"total_reasoning_cost_bin":2.1419280000000005,"total_cost_bin":18.809387999999995,"total_on_demand_input_cost_bin":25.964460000000013,"ci_99_lower":0.36674051287577225,"ci_99_upper":0.6134196583364041,"ci_98_lower":0.377137272027502,"ci_98_upper":0.6027035557701099,"ci_95_lower":0.39639415776874704,"ci_95_upper":0.5759021439773789,"ci_90_lower":0.41377105093741956,"ci_90_upper":0.5729740625993321,"ci_75_lower":0.4337867518278851,"ci_75_upper":0.5470119297365404,"ci_50_lower":0.46074395134222235,"ci_50_upper":0.5264105283162439},"262144":{"avg_score":0.45834757615184635,"min_score":0.002918642945587635,"max_score":1,"n_tests":141,"stddev":0.47858184467596226,"median_score":0.055525895208120346,"q1_score":0.026058631017804146,"q3_score":1,"avg_token_efficiency":0.9635319638751029,"is_incomplete":false,"total_prompt_cost_bin":47.304246,"total_cache_read_cost_bin":23.71065599999999,"total_completion_cost_bin":3.0898319999999986,"total_reasoning_cost_bin":2.782284000000002,"total_cost_bin":76.887018,"total_on_demand_input_cost_bin":142.33657999999997,"ci_99_lower":0.34383560802273994,"ci_99_upper":0.5697869547158612,"ci_98_lower":0.36386242642968974,"ci_98_upper":0.5593963835156581,"ci_95_lower":0.3793998989365757,"ci_95_upper":0.5395653297765518,"ci_90_lower":0.3893381236519017,"ci_90_upper":0.5246104887192936,"ci_75_lower":0.41079154526542355,"ci_75_upper":0.5062749630471044,"ci_50_lower":0.43146139811658374,"ci_50_upper":0.4854273670650226}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":500000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","enabled"],"reasoning_default":"high","sponsors":[],"openrouter_name":"SpaceXAI: Grok 4.6","provider_name":"X-ai","run_provider_names":["SpaceXAI"],"pricing_metadata":{"prompt":0.000002,"completion":0.000006,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:35:57.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T04:17:32.371Z"},{"model_slug":"google/gemini-3.1-flash-lite-preview","reasoning_mode":"low","overall_metrics":{"auc_1m":0.19897277353733098,"auc_128k":0.3702684947091592,"cum_avg_1m":0.3456195993973497,"cum_avg_128k":0.4502598054148086,"avg_score_overall":0.2693530133920403,"avg_response_time_ms":31830.072103177583,"total_prompt_cost":6.401831000000003,"total_cache_read_cost":12.111519800000007,"total_completion_cost":0.6590565000000002,"total_reasoning_cost":9.031628999999999,"total_prompt_cost_128k":1.2500612500000003,"total_completion_cost_128k":0.27819900000000003,"total_on_demand_input_cost":127.51702899999984,"total_on_demand_input_cost_128k":6.109569250000002,"total_runs":1170,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0004796231398423},"bin_metrics":{"8192":{"avg_score":0.6519510847618222,"min_score":0.0011554014636203647,"max_score":1,"n_tests":80,"stddev":0.46399641455733814,"median_score":1,"q1_score":0.0370838800445199,"q3_score":1,"avg_token_efficiency":1.0003607687615759,"is_incomplete":false,"total_prompt_cost_bin":0.12781750000000003,"total_cache_read_cost_bin":0.000509325,"total_completion_cost_bin":0.041781,"total_reasoning_cost_bin":0.05588550000000002,"total_cost_bin":0.22599332500000008,"total_on_demand_input_cost_bin":0.13291075,"ci_99_lower":0.5153522209630027,"ci_99_upper":0.7751777450331719,"ci_98_lower":0.5207925036255329,"ci_98_upper":0.7600187979611365,"ci_95_lower":0.5522071588985272,"ci_95_upper":0.7470270012815309,"ci_90_lower":0.5671624940292531,"ci_90_upper":0.734911333490163,"ci_75_lower":0.5920667754215174,"ci_75_upper":0.711153401012416,"ci_50_lower":0.6152393412870879,"ci_50_upper":0.6872563608332711},"16384":{"avg_score":0.6238218299578875,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4759916001773727,"median_score":1,"q1_score":0.03510065842419863,"q3_score":1,"avg_token_efficiency":1.0009898312709977,"is_incomplete":false,"total_prompt_cost_bin":0.12346249999999996,"total_cache_read_cost_bin":0.015224575000000008,"total_completion_cost_bin":0.041699999999999994,"total_reasoning_cost_bin":0.0905535,"total_cost_bin":0.27094057499999996,"total_on_demand_input_cost_bin":0.27570825,"ci_99_lower":0.47757173378870355,"ci_99_upper":0.759025606188341,"ci_98_lower":0.4996871589701623,"ci_98_upper":0.7348690559496172,"ci_95_lower":0.5144089778151828,"ci_95_upper":0.7212913715816104,"ci_90_lower":0.5382942045293748,"ci_90_upper":0.7082301815191749,"ci_75_lower":0.562920812991797,"ci_75_upper":0.6849737921002088,"ci_50_lower":0.5874917335051578,"ci_50_upper":0.6594045127683784},"32768":{"avg_score":0.3471785278901841,"min_score":0.003955174703150988,"max_score":1,"n_tests":136,"stddev":0.4535259605602428,"median_score":0.042087290436029434,"q1_score":0.025035765022039413,"q3_score":1,"avg_token_efficiency":1.0025202770862707,"is_incomplete":false,"total_prompt_cost_bin":0.26023275000000023,"total_cache_read_cost_bin":0.071066675,"total_completion_cost_bin":0.07995150000000001,"total_reasoning_cost_bin":0.24027150000000008,"total_cost_bin":0.6515224250000002,"total_on_demand_input_cost_bin":0.9708994999999994,"ci_99_lower":0.24894106813596892,"ci_99_upper":0.44693425279064447,"ci_98_lower":0.2549645567035152,"ci_98_upper":0.4397580734575304,"ci_95_lower":0.2692252206571592,"ci_95_upper":0.4237502489298041,"ci_90_lower":0.28123907560965405,"ci_90_upper":0.41183979073402416,"ci_75_lower":0.3026400047447704,"ci_75_upper":0.39517578405673914,"ci_50_lower":0.31948489991876106,"ci_50_upper":0.37352334163476963},"65536":{"avg_score":0.36869660589624853,"min_score":0.004449388012290001,"max_score":1,"n_tests":85,"stddev":0.4600290923198026,"median_score":0.03776751831173897,"q1_score":0.024508407339453697,"q3_score":1,"avg_token_efficiency":1.0005350788304659,"is_incomplete":false,"total_prompt_cost_bin":0.22376025000000005,"total_cache_read_cost_bin":0.11669760000000008,"total_completion_cost_bin":0.05069700000000003,"total_reasoning_cost_bin":0.6252870000000005,"total_cost_bin":1.0164418500000005,"total_on_demand_input_cost_bin":1.3907362499999998,"ci_99_lower":0.246434568197576,"ci_99_upper":0.49981009630196016,"ci_98_lower":0.26039928738783824,"ci_98_upper":0.48564806658106247,"ci_95_lower":0.2747552502188174,"ci_95_upper":0.47007859052569767,"ci_90_lower":0.28777845466564245,"ci_90_upper":0.45109993786824976,"ci_75_lower":0.30855721761472527,"ci_75_upper":0.42614601402242175,"ci_50_lower":0.33440600619103544,"ci_50_upper":0.4031826323911767},"131072":{"avg_score":0.2596509785679005,"min_score":0.003610108280554414,"max_score":1,"n_tests":102,"stddev":0.4046140497061305,"median_score":0.03394061140716076,"q1_score":0.018940288107842207,"q3_score":0.23104560375213623,"avg_token_efficiency":1.000623405989048,"is_incomplete":false,"total_prompt_cost_bin":0.5147882500000002,"total_cache_read_cost_bin":0.2824526250000006,"total_completion_cost_bin":0.06406949999999999,"total_reasoning_cost_bin":0.6509940000000001,"total_cost_bin":1.512304375000001,"total_on_demand_input_cost_bin":3.3393145000000026,"ci_99_lower":0.16684042926583276,"ci_99_upper":0.37440563163047835,"ci_98_lower":0.17260211350583451,"ci_98_upper":0.36504807080845253,"ci_95_lower":0.18462670932588962,"ci_95_upper":0.3438568410500724,"ci_90_lower":0.1954396212496338,"ci_90_upper":0.325368017061755,"ci_75_lower":0.21345596712490078,"ci_75_upper":0.30460550132756553,"ci_50_lower":0.2301855925464884,"ci_50_upper":0.28563069128808016},"262144":{"avg_score":0.2215377989320864,"min_score":0.0032573288772255182,"max_score":1,"n_tests":141,"stddev":0.3780092427518603,"median_score":0.036023713648319244,"q1_score":0.0263671875,"q3_score":0.08342132717370987,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":0.8779512500000002,"total_cache_read_cost_bin":0.8354820000000018,"total_completion_cost_bin":0.078642,"total_reasoning_cost_bin":0.9312630000000003,"total_cost_bin":2.7233382500000025,"total_on_demand_input_cost_bin":9.232771249999995,"ci_99_lower":0.14723959292990244,"ci_99_upper":0.304760334776295,"ci_98_lower":0.1531934327968555,"ci_98_upper":0.29477798799602706,"ci_95_lower":0.1602467438601146,"ci_95_upper":0.28370155302464223,"ci_90_lower":0.1705416562810131,"ci_90_upper":0.27384469779373427,"ci_75_lower":0.18572780557809865,"ci_75_upper":0.2575773015826386,"ci_50_lower":0.19849108568523144,"ci_50_upper":0.2409973605977642},"524288":{"avg_score":0.18584755013531107,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.34197720767375595,"median_score":0.03457636199891567,"q1_score":0.021857681218534708,"q3_score":0.07385746389627457,"avg_token_efficiency":1.0000019078540325,"is_incomplete":false,"total_prompt_cost_bin":1.4682227499999996,"total_cache_read_cost_bin":2.945663399999988,"total_completion_cost_bin":0.12798450000000006,"total_reasoning_cost_bin":1.6632780000000005,"total_cost_bin":6.2051486499999875,"total_on_demand_input_cost_bin":30.924856750000043,"ci_99_lower":0.13368553054300963,"ci_99_upper":0.23762412291638735,"ci_98_lower":0.1361827044278557,"ci_98_upper":0.2367191797025478,"ci_95_lower":0.14390887508683806,"ci_95_upper":0.2298343391567113,"ci_90_lower":0.14984800563474043,"ci_90_upper":0.2221005716758693,"ci_75_lower":0.1622216325081585,"ci_75_upper":0.21153420335080442,"ci_50_lower":0.1705431810499295,"ci_50_upper":0.19976137562504953},"1048576":{"avg_score":0.10627241903735746,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.24378870653677337,"median_score":0.030259926803410053,"q1_score":0.018777637742459774,"q3_score":0.0490287970751524,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":2.8055957500000033,"total_cache_read_cost_bin":7.844423600000018,"total_completion_cost_bin":0.17423100000000002,"total_reasoning_cost_bin":4.774096499999996,"total_cost_bin":15.598346850000016,"total_on_demand_input_cost_bin":81.2498317499998,"ci_99_lower":0.07322074489817099,"ci_99_upper":0.14513517113501626,"ci_98_lower":0.07614110714019157,"ci_98_upper":0.14244773391343563,"ci_95_lower":0.08118021634660827,"ci_95_upper":0.13621904199139304,"ci_90_lower":0.0839117473437481,"ci_90_upper":0.13062090299723128,"ci_75_lower":0.09002305494223331,"ci_75_upper":0.1226792781231662,"ci_50_lower":0.09659175280549173,"ci_50_upper":0.11431582190956349}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.1 Flash Lite Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":2.5e-7,"completion":0.0000015,"reasoning":0.0000015,"has_tiered_pricing":true},"api_created_at":"2026-03-03T04:37:53.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T02:50:16.238Z"},{"model_slug":"google/gemini-3.5-flash","reasoning_mode":null,"overall_metrics":{"auc_1m":0.19765490416081358,"auc_128k":0.27844828949882416,"cum_avg_1m":0.2921201674747633,"cum_avg_128k":0.3517795773829227,"avg_score_overall":0.24571170448534346,"avg_response_time_ms":14036.552079563156,"total_prompt_cost":38.04382349999997,"total_cache_read_cost":72.72211349999981,"total_completion_cost":4.118715,"total_reasoning_cost":0,"total_prompt_cost_128k":8.029448999999996,"total_completion_cost_128k":1.7014859999999998,"total_on_demand_input_cost":765.2649585000015,"total_on_demand_input_cost_128k":36.8202,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154894},"bin_metrics":{"8192":{"avg_score":0.601243806432467,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4790768042792507,"median_score":1,"q1_score":0.02967744879424572,"q3_score":1,"avg_token_efficiency":1.0001553404747239,"is_incomplete":false,"total_prompt_cost_bin":0.7973159999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.252135,"total_reasoning_cost_bin":0,"total_cost_bin":1.049451,"total_on_demand_input_cost_bin":0.7973159999999999,"ci_99_lower":0.45652776119398186,"ci_99_upper":0.7349482678080675,"ci_98_lower":0.4676364419773454,"ci_98_upper":0.7221828282598872,"ci_95_lower":0.4939047326595755,"ci_95_upper":0.6997501025395467,"ci_90_lower":0.5144565940805478,"ci_90_upper":0.6865156974224373,"ci_75_lower":0.5409283261862583,"ci_75_upper":0.6624084436785779,"ci_50_lower":0.5652661426312989,"ci_50_upper":0.636279317777371},"16384":{"avg_score":0.4036845922120847,"min_score":0.006989952176809311,"max_score":1,"n_tests":80,"stddev":0.47739851032741815,"median_score":0.03638450428843498,"q1_score":0.023981403093785048,"q3_score":1,"avg_token_efficiency":1.0000745135024474,"is_incomplete":false,"total_prompt_cost_bin":1.652807999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.2478420000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.900649999999999,"total_on_demand_input_cost_bin":1.652807999999999,"ci_99_lower":0.2683745979948435,"ci_99_upper":0.5376363721837407,"ci_98_lower":0.2806911983437604,"ci_98_upper":0.536246590740746,"ci_95_lower":0.29273358831967927,"ci_95_upper":0.5122921247719205,"ci_90_lower":0.30704673573141916,"ci_90_upper":0.4903487246180884,"ci_75_lower":0.3428587628208334,"ci_75_upper":0.46473749447759477,"ci_50_lower":0.36700694978062526,"ci_50_upper":0.43988333007146135},"32768":{"avg_score":0.26287637257893737,"min_score":0.005333333276212215,"max_score":1,"n_tests":136,"stddev":0.4110454591794533,"median_score":0.03554010018706322,"q1_score":0.023659516125917435,"q3_score":0.1220997292548418,"avg_token_efficiency":1.000035909094991,"is_incomplete":false,"total_prompt_cost_bin":2.054277000000001,"total_cache_read_cost_bin":0.3755847000000006,"total_completion_cost_bin":0.4893839999999998,"total_reasoning_cost_bin":0,"total_cost_bin":2.919245700000001,"total_on_demand_input_cost_bin":5.8101239999999965,"ci_99_lower":0.17751368563913278,"ci_99_upper":0.34739335919791997,"ci_98_lower":0.1826020837038317,"ci_98_upper":0.34279298089615834,"ci_95_lower":0.19673523405097726,"ci_95_upper":0.33133597169276874,"ci_90_lower":0.20438100815962945,"ci_90_upper":0.31886759019997796,"ci_75_lower":0.22097312326556728,"ci_75_upper":0.3022854338735616,"ci_50_lower":0.23916886753902553,"ci_50_upper":0.285035657340332},"65536":{"avg_score":0.2587869852233459,"min_score":0.004494382068514824,"max_score":1,"n_tests":85,"stddev":0.413540286726816,"median_score":0.032674118876457214,"q1_score":0.024004174396395683,"q3_score":0.08787943422794342,"avg_token_efficiency":1.0000152878522814,"is_incomplete":false,"total_prompt_cost_bin":1.5482549999999988,"total_cache_read_cost_bin":0.67918275,"total_completion_cost_bin":0.3166560000000001,"total_reasoning_cost_bin":0,"total_cost_bin":2.5440937499999987,"total_on_demand_input_cost_bin":8.3400825,"ci_99_lower":0.1569855317116222,"ci_99_upper":0.3747952316230272,"ci_98_lower":0.1678017789989929,"ci_98_upper":0.3650825767661718,"ci_95_lower":0.17890693572021146,"ci_95_upper":0.3492237103382564,"ci_90_lower":0.1896294103272478,"ci_90_upper":0.3363467588587938,"ci_75_lower":0.20343064426800564,"ci_75_upper":0.30462368769872494,"ci_50_lower":0.22519108880694735,"ci_50_upper":0.28414230321205275},"131072":{"avg_score":0.23230613046777843,"min_score":0.003092384897172451,"max_score":1,"n_tests":103,"stddev":0.40101836881709546,"median_score":0.027210883796215057,"q1_score":0.018681971356272697,"q3_score":0.04554123803973198,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":1.9767929999999967,"total_cache_read_cost_bin":1.8243076500000017,"total_completion_cost_bin":0.3954689999999999,"total_reasoning_cost_bin":0,"total_cost_bin":4.196569649999999,"total_on_demand_input_cost_bin":20.219869500000005,"ci_99_lower":0.13764867564176156,"ci_99_upper":0.3345798771210281,"ci_98_lower":0.14644846438934336,"ci_98_upper":0.3258687227563164,"ci_95_lower":0.16356624422614122,"ci_95_upper":0.30671293052507886,"ci_90_lower":0.17375826426698077,"ci_90_upper":0.2974821683836567,"ci_75_lower":0.18553658661537678,"ci_75_upper":0.2773435665088258,"ci_50_lower":0.20483628868462192,"ci_50_upper":0.25465378458882926},"262144":{"avg_score":0.24713430481843968,"min_score":0.002326934365555644,"max_score":1,"n_tests":141,"stddev":0.4004758315070455,"median_score":0.035330262035131454,"q1_score":0.02413793094456196,"q3_score":0.07840771973133087,"avg_token_efficiency":1.0000038179365236,"is_incomplete":false,"total_prompt_cost_bin":4.7395875000000025,"total_cache_read_cost_bin":5.065704000000007,"total_completion_cost_bin":0.4954770000000003,"total_reasoning_cost_bin":0,"total_cost_bin":10.30076850000001,"total_on_demand_input_cost_bin":55.39662749999998,"ci_99_lower":0.1698462108159702,"ci_99_upper":0.33465974531048936,"ci_98_lower":0.17172702991471342,"ci_98_upper":0.3217185472944712,"ci_95_lower":0.18125443963985396,"ci_95_upper":0.3138013064273854,"ci_90_lower":0.19185998274891883,"ci_90_upper":0.3013508227657104,"ci_75_lower":0.20775120754739443,"ci_75_upper":0.28405384491102814,"ci_50_lower":0.22348894961120508,"ci_50_upper":0.2684262212979317},"524288":{"avg_score":0.159127711779655,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.3227670591858789,"median_score":0.0318718645721674,"q1_score":0.021210188511759043,"q3_score":0.05599107686430216,"avg_token_efficiency":1.000001907854033,"is_incomplete":false,"total_prompt_cost_bin":9.05525249999999,"total_cache_read_cost_bin":17.64938879999993,"total_completion_cost_bin":0.7832969999999995,"total_reasoning_cost_bin":0,"total_cost_bin":27.48793829999992,"total_on_demand_input_cost_bin":185.54914050000016,"ci_99_lower":0.1070123068581141,"ci_99_upper":0.21079189407058607,"ci_98_lower":0.11172416639440275,"ci_98_upper":0.2066407453379213,"ci_95_lower":0.11901427601942333,"ci_95_upper":0.20108086760448127,"ci_90_lower":0.1251830725911393,"ci_90_upper":0.19241558509556697,"ci_75_lower":0.13628385216540312,"ci_75_upper":0.18326156713528227,"ci_50_lower":0.14577868475193467,"ci_50_upper":0.17174230504845628},"1048576":{"avg_score":0.17180143628539818,"min_score":0.0020491802133619785,"max_score":1,"n_tests":310,"stddev":0.3381290629759933,"median_score":0.029755404219031334,"q1_score":0.020010410342365503,"q3_score":0.053290569223463535,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":16.21953449999998,"total_cache_read_cost_bin":47.127945599999876,"total_completion_cost_bin":1.1384550000000004,"total_reasoning_cost_bin":0,"total_cost_bin":64.48593509999986,"total_on_demand_input_cost_bin":487.49899050000136,"ci_99_lower":0.12425402777862825,"ci_99_upper":0.22176488342398643,"ci_98_lower":0.12941766190223183,"ci_98_upper":0.21811710144486277,"ci_95_lower":0.13486968531701413,"ci_95_upper":0.2118324372771707,"ci_90_lower":0.13978473206039427,"ci_90_upper":0.20454981085543908,"ci_75_lower":0.14821180921130545,"ci_75_upper":0.19432578611754692,"ci_50_lower":0.15792935180173617,"ci_50_upper":0.18441025544307194}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":0.0000015,"completion":0.000009,"reasoning":0.000009,"has_tiered_pricing":true},"api_created_at":"2026-05-19T12:30:00.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"google/gemini-3.1-flash-lite-preview","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.19559463528200263,"auc_128k":0.4040474958601922,"cum_avg_1m":0.4182677088293198,"cum_avg_128k":0.5704339816388058,"avg_score_overall":0.31819314128265547,"avg_response_time_ms":23199.16230009682,"total_prompt_cost":5.773285499999997,"total_cache_read_cost":12.174280599999971,"total_completion_cost":0.6631214999999998,"total_reasoning_cost":9.346136999999999,"total_prompt_cost_128k":1.3258862499999995,"total_completion_cost_128k":0.27777450000000004,"total_on_demand_input_cost":127.51609149999982,"total_on_demand_input_cost_128k":6.108631750000002,"total_runs":1170,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0002151973800355},"bin_metrics":{"8192":{"avg_score":0.9631616907892748,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.1872967063317279,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0001553404747237,"is_incomplete":false,"total_prompt_cost_bin":0.125754,"total_cache_read_cost_bin":0.0007132000000000001,"total_completion_cost_bin":0.041721,"total_reasoning_cost_bin":0.11549400000000001,"total_cost_bin":0.2836822,"total_on_demand_input_cost_bin":0.13288599999999995,"ci_99_lower":0.90178277607169,"ci_99_upper":0.9999972335994244,"ci_98_lower":0.9019494304494001,"ci_98_upper":0.9999955415725708,"ci_95_lower":0.9141779104014859,"ci_95_upper":0.9999033689498902,"ci_90_lower":0.9263302245060914,"ci_90_upper":0.9997246138378978,"ci_75_lower":0.9386183106573299,"ci_75_upper":0.9876635764027014,"ci_50_lower":0.9508662907406689,"ci_50_upper":0.9754843259230256},"16384":{"avg_score":0.8012364083289867,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.390882954050703,"median_score":1,"q1_score":0.9993467777967453,"q3_score":1,"avg_token_efficiency":1.0000745135024478,"is_incomplete":false,"total_prompt_cost_bin":0.15797774999999994,"total_cache_read_cost_bin":0.011749024999999995,"total_completion_cost_bin":0.04168799999999999,"total_reasoning_cost_bin":0.11871600000000002,"total_cost_bin":0.330130775,"total_on_demand_input_cost_bin":0.2754680000000001,"ci_99_lower":0.6811726740370214,"ci_99_upper":0.9071004526568285,"ci_98_lower":0.6917709369122168,"ci_98_upper":0.8986729158672969,"ci_95_lower":0.7044184160117584,"ci_95_upper":0.8859231014447141,"ci_90_lower":0.725061452210939,"ci_90_upper":0.8698845912076649,"ci_75_lower":0.7520541392212181,"ci_75_upper":0.8495599648438655,"ci_50_lower":0.7709715850272916,"ci_50_upper":0.8296317596657901},"32768":{"avg_score":0.5607252588105278,"min_score":0.003957783803343773,"max_score":1,"n_tests":136,"stddev":0.48188789020454514,"median_score":0.9996411800384521,"q1_score":0.029960133600980043,"q3_score":1,"avg_token_efficiency":1.0000359090949908,"is_incomplete":false,"total_prompt_cost_bin":0.20020799999999989,"total_cache_read_cost_bin":0.07681459999999991,"total_completion_cost_bin":0.08037450000000004,"total_reasoning_cost_bin":0.21093749999999997,"total_cost_bin":0.5683345999999998,"total_on_demand_input_cost_bin":0.9683539999999996,"ci_99_lower":0.4540364867448122,"ci_99_upper":0.6650541237517438,"ci_98_lower":0.46156472251223235,"ci_98_upper":0.6539371592560714,"ci_95_lower":0.4842248324610409,"ci_95_upper":0.6441518699835903,"ci_90_lower":0.4957336668181918,"ci_90_upper":0.6292022666477886,"ci_75_lower":0.5148258411641377,"ci_75_upper":0.6103389135335902,"ci_50_lower":0.5335889654311091,"ci_50_upper":0.5887935678953963},"65536":{"avg_score":0.2934575012611116,"min_score":0.004716981202363968,"max_score":1,"n_tests":85,"stddev":0.4299250063828207,"median_score":0.03207184001803398,"q1_score":0.02166847325861454,"q3_score":0.942148745059967,"avg_token_efficiency":1.001968370079096,"is_incomplete":false,"total_prompt_cost_bin":0.2556117499999998,"total_cache_read_cost_bin":0.11371167500000007,"total_completion_cost_bin":0.04966050000000001,"total_reasoning_cost_bin":0.2549925,"total_cost_bin":0.6739764249999999,"total_on_demand_input_cost_bin":1.3927285000000003,"ci_99_lower":0.18095338706113398,"ci_99_upper":0.4243813596396871,"ci_98_lower":0.19351928509700608,"ci_98_upper":0.4090519642786616,"ci_95_lower":0.20533813291599098,"ci_95_upper":0.392732036503251,"ci_90_lower":0.21766866682069924,"ci_90_upper":0.37047603623859365,"ci_75_lower":0.2378500028362717,"ci_75_upper":0.34402350687956,"ci_50_lower":0.25900357875918206,"ci_50_upper":0.3231616206152146},"131072":{"avg_score":0.23358904900412789,"min_score":0,"max_score":1,"n_tests":102,"stddev":0.39360137071997753,"median_score":0.03138869255781174,"q1_score":0.021917224396020174,"q3_score":0.056293134577572346,"avg_token_efficiency":1.0005873847747206,"is_incomplete":false,"total_prompt_cost_bin":0.5863347499999999,"total_cache_read_cost_bin":0.27528604999999995,"total_completion_cost_bin":0.0643305,"total_reasoning_cost_bin":0.20447549999999992,"total_cost_bin":1.1304267999999997,"total_on_demand_input_cost_bin":3.3391952500000017,"ci_99_lower":0.138787291888353,"ci_99_upper":0.3420893129460769,"ci_98_lower":0.14996815786391096,"ci_98_upper":0.33628625499248943,"ci_95_lower":0.16033984140114058,"ci_95_upper":0.3132352993824729,"ci_90_lower":0.17067436246114656,"ci_90_upper":0.2971276604636645,"ci_75_lower":0.18838412120114265,"ci_75_upper":0.2768977084904726,"ci_50_lower":0.20619843852039718,"ci_50_upper":0.2579806089074826},"262144":{"avg_score":0.20349386684490856,"min_score":0.002144772093743086,"max_score":1,"n_tests":141,"stddev":0.3618747597232187,"median_score":0.03302253410220146,"q1_score":0.023731587454676628,"q3_score":0.07020623236894608,"avg_token_efficiency":1.000003817936524,"is_incomplete":false,"total_prompt_cost_bin":0.8165530000000006,"total_cache_read_cost_bin":0.8416218249999976,"total_completion_cost_bin":0.0799695,"total_reasoning_cost_bin":1.5248130000000002,"total_cost_bin":3.2629573249999986,"total_on_demand_input_cost_bin":9.232771249999997,"ci_99_lower":0.13071799545705728,"ci_99_upper":0.27958054811376687,"ci_98_lower":0.13489813397387504,"ci_98_upper":0.273128888900689,"ci_95_lower":0.1472021234829428,"ci_95_upper":0.26312906072806924,"ci_90_lower":0.15421807716521335,"ci_90_upper":0.2544967368639781,"ci_75_lower":0.1687404290737472,"ci_75_upper":0.23847404353119142,"ci_50_lower":0.18195271392679835,"ci_50_upper":0.22269442498416422},"524288":{"avg_score":0.17074677422129692,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.3230713769541004,"median_score":0.03399723209440708,"q1_score":0.02087622368708253,"q3_score":0.06668775156140327,"avg_token_efficiency":1.0000019078540325,"is_incomplete":false,"total_prompt_cost_bin":1.341254499999999,"total_cache_read_cost_bin":2.958360224999988,"total_completion_cost_bin":0.12815099999999996,"total_reasoning_cost_bin":2.5558635000000005,"total_cost_bin":6.983629224999987,"total_on_demand_input_cost_bin":30.924856750000014,"ci_99_lower":0.11804981314558419,"ci_99_upper":0.22058298636630366,"ci_98_lower":0.12382855683484241,"ci_98_upper":0.21834320122851614,"ci_95_lower":0.13037733494834894,"ci_95_upper":0.21247963800560682,"ci_90_lower":0.13674483770086288,"ci_90_upper":0.20575583018017224,"ci_75_lower":0.147075736754124,"ci_75_upper":0.19434029384224155,"ci_50_lower":0.15648684050775355,"ci_50_upper":0.1834365114647603},"1048576":{"avg_score":0.1197311213743242,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.2583639194814849,"median_score":0.03184313327074051,"q1_score":0.021022579167038202,"q3_score":0.05402471497654915,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":2.2895917499999983,"total_cache_read_cost_bin":7.896023999999985,"total_completion_cost_bin":0.17722649999999981,"total_reasoning_cost_bin":4.3608449999999985,"total_cost_bin":14.72368724999998,"total_on_demand_input_cost_bin":81.2498317499998,"ci_99_lower":0.0870680667172156,"ci_99_upper":0.1585372810137999,"ci_98_lower":0.08881717542229942,"ci_98_upper":0.15658135725149974,"ci_95_lower":0.0921417232122903,"ci_95_upper":0.15107620185964005,"ci_90_lower":0.09535019112521811,"ci_90_upper":0.1454729409607112,"ci_75_lower":0.10271852514668998,"ci_75_upper":0.1370784023373891,"ci_50_lower":0.10913221395439318,"ci_50_upper":0.12847865454217927}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.1 Flash Lite Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":2.5e-7,"completion":0.0000015,"reasoning":0.0000015,"has_tiered_pricing":true},"api_created_at":"2026-03-03T04:37:53.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T01:53:13.666Z"},{"model_slug":"moonshotai/kimi-k3","reasoning_mode":"low","overall_metrics":{"auc_1m":0.1926176453563499,"auc_128k":0.49376547861950437,"cum_avg_1m":0.42683411852139164,"cum_avg_128k":0.6004401356712761,"avg_score_overall":0.41620409756947013,"avg_response_time_ms":33509.2092718995,"total_prompt_cost":11.668824000000003,"total_cache_read_cost":53.97803519999997,"total_completion_cost":16.385520000000003,"total_reasoning_cost":11.37093,"total_prompt_cost_128k":5.220623999999999,"total_completion_cost_128k":7.327215000000001,"total_on_demand_input_cost":551.449176000001,"total_on_demand_input_cost_128k":73.260816,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.996050877570598},"bin_metrics":{"8192":{"avg_score":0.9157320746453479,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.2737746871749377,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.002393554094993,"is_incomplete":false,"total_prompt_cost_bin":0.2859360000000001,"total_cache_read_cost_bin":0.13132799999999997,"total_completion_cost_bin":0.84714,"total_reasoning_cost_bin":0.41258999999999996,"total_cost_bin":1.676994,"total_on_demand_input_cost_bin":1.5992159999999997,"ci_99_lower":0.821873554250109,"ci_99_upper":0.9874004242246738,"ci_98_lower":0.8300500638859812,"ci_98_upper":0.9765176880946383,"ci_95_lower":0.8451710582722444,"ci_95_upper":0.9748734449659241,"ci_90_lower":0.8566627417667768,"ci_90_upper":0.9636140773014631,"ci_75_lower":0.8790401538397418,"ci_75_upper":0.9513221403874923,"ci_50_lower":0.8925770082627423,"ci_50_upper":0.9389948895084672},"16384":{"avg_score":0.7193396620627027,"min_score":0.0044469148851931095,"max_score":1,"n_tests":80,"stddev":0.4445657708017541,"median_score":1,"q1_score":0.03700730670243502,"q3_score":1,"avg_token_efficiency":0.9863410585921482,"is_incomplete":false,"total_prompt_cost_bin":0.18907199999999985,"total_cache_read_cost_bin":0.3071231999999996,"total_completion_cost_bin":0.9447450000000003,"total_reasoning_cost_bin":0.5237400000000001,"total_cost_bin":1.9646801999999999,"total_on_demand_input_cost_bin":3.260304,"ci_99_lower":0.5844495563320815,"ci_99_upper":0.830119489919074,"ci_98_lower":0.5973734584916384,"ci_98_upper":0.8289255989035009,"ci_95_lower":0.6213657113876251,"ci_95_upper":0.8053171416746044,"ci_90_lower":0.6336674185897573,"ci_90_upper":0.7929460193400154,"ci_75_lower":0.6586439410224557,"ci_75_upper":0.7688785051213927,"ci_50_lower":0.6830842888783081,"ci_50_upper":0.7550054294682924},"32768":{"avg_score":0.5705796123733949,"min_score":0.0016420361353084445,"max_score":1,"n_tests":136,"stddev":0.4780008591420381,"median_score":1,"q1_score":0.0348462862893939,"q3_score":1,"avg_token_efficiency":1.013008403395958,"is_incomplete":false,"total_prompt_cost_bin":0.20246400000000025,"total_cache_read_cost_bin":1.1582207999999994,"total_completion_cost_bin":2.130990000000001,"total_reasoning_cost_bin":1.3084799999999996,"total_cost_bin":4.8001548000000005,"total_on_demand_input_cost_bin":11.784672000000013,"ci_99_lower":0.46179009114801267,"ci_99_upper":0.6746537565153139,"ci_98_lower":0.47784183715965756,"ci_98_upper":0.6675961083895012,"ci_95_lower":0.49291293843786715,"ci_95_upper":0.6501738479143515,"ci_90_lower":0.5059827012290685,"ci_90_upper":0.6382826551640925,"ci_75_lower":0.5254985376082272,"ci_75_upper":0.6203620311996072,"ci_50_lower":0.5459726006265555,"ci_50_upper":0.5978704290678519},"65536":{"avg_score":0.4858352473279571,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.48779567557265496,"median_score":0.04961089417338371,"q1_score":0.02565392293035984,"q3_score":1,"avg_token_efficiency":0.9919747770303086,"is_incomplete":false,"total_prompt_cost_bin":0.22451399999999974,"total_cache_read_cost_bin":1.6321536000000014,"total_completion_cost_bin":1.5001949999999993,"total_reasoning_cost_bin":0.966255,"total_cost_bin":4.323117600000001,"total_on_demand_input_cost_bin":16.546049999999994,"ci_99_lower":0.3460314828863861,"ci_99_upper":0.6227196639517103,"ci_98_lower":0.36870296900231825,"ci_98_upper":0.6113360519655159,"ci_95_lower":0.38182396088233767,"ci_95_upper":0.5902148544437745,"ci_90_lower":0.3950154124822139,"ci_90_upper":0.5768926357705255,"ci_75_lower":0.4275398322394775,"ci_75_upper":0.5451848770745154,"ci_50_lower":0.45118915373048585,"ci_50_upper":0.5203054881860119},"131072":{"avg_score":0.3107140819469774,"min_score":0.0047505940310657024,"max_score":1,"n_tests":103,"stddev":0.4438755453870587,"median_score":0.033364225178956985,"q1_score":0.01999750081449747,"q3_score":1,"avg_token_efficiency":0.9908788032174033,"is_incomplete":false,"total_prompt_cost_bin":4.318637999999999,"total_cache_read_cost_bin":3.575193600000004,"total_completion_cost_bin":1.904145,"total_reasoning_cost_bin":1.2443849999999999,"total_cost_bin":11.042361600000003,"total_on_demand_input_cost_bin":40.07057399999999,"ci_99_lower":0.20557022995333576,"ci_99_upper":0.4223729137186802,"ci_98_lower":0.21541640159339434,"ci_98_upper":0.4073903358009761,"ci_95_lower":0.23164908979619286,"ci_95_upper":0.39629053822051713,"ci_90_lower":0.24264478727370428,"ci_90_upper":0.38535556669337945,"ci_75_lower":0.2623705328905557,"ci_75_upper":0.3583650788443742,"ci_50_lower":0.282488627432737,"ci_50_upper":0.33883906962707167},"262144":{"avg_score":0.21785095107167007,"min_score":0.0019762846641242504,"max_score":1,"n_tests":141,"stddev":0.3818896351203297,"median_score":0.03361344709992409,"q1_score":0.024518389254808426,"q3_score":0.05730833858251572,"avg_token_efficiency":0.9958583241229255,"is_incomplete":false,"total_prompt_cost_bin":4.787724000000005,"total_cache_read_cost_bin":10.554623999999993,"total_completion_cost_bin":3.362505000000001,"total_reasoning_cost_bin":2.5391250000000003,"total_cost_bin":21.243978,"total_on_demand_input_cost_bin":110.33396400000017,"ci_99_lower":0.14256232039051805,"ci_99_upper":0.3015128433745988,"ci_98_lower":0.14864362310429918,"ci_98_upper":0.292466753074001,"ci_95_lower":0.15608794459543052,"ci_95_upper":0.27922264267294167,"ci_90_lower":0.16771584410560547,"ci_90_upper":0.2716288603036433,"ci_75_lower":0.18237824588254464,"ci_75_upper":0.2534144675170884,"ci_50_lower":0.19566051993462924,"ci_50_upper":0.23782493747318334},"524288":{"avg_score":0.19462131874308275,"min_score":0.0029607696924358606,"max_score":1,"n_tests":236,"stddev":0.3598833442890287,"median_score":0.03237365372478962,"q1_score":0.020511317532509565,"q3_score":0.06478680670261383,"avg_token_efficiency":0.9912605814879746,"is_incomplete":false,"total_prompt_cost_bin":1.6604759999999974,"total_cache_read_cost_bin":36.619391999999976,"total_completion_cost_bin":5.695800000000002,"total_reasoning_cost_bin":4.376355,"total_cost_bin":48.352022999999974,"total_on_demand_input_cost_bin":367.8543960000008,"ci_99_lower":0.1395753959100447,"ci_99_upper":0.25453045917342426,"ci_98_lower":0.14352062773576077,"ci_98_upper":0.2478745170509032,"ci_95_lower":0.15035946127784466,"ci_95_upper":0.24064500463975522,"ci_90_lower":0.1565533481772113,"ci_90_upper":0.23297743834249313,"ci_75_lower":0.1694263873657091,"ci_75_upper":0.2225745045522009,"ci_50_lower":0.17893394199964568,"ci_50_upper":0.20973674557171762}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"max","sponsors":[],"openrouter_name":"MoonshotAI: Kimi K3","provider_name":"Moonshotai","run_provider_names":["Moonshot AI"],"pricing_metadata":{"prompt":0.000003,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-16T15:30:58.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-21T00:50:39.443Z"},{"model_slug":"google/gemini-3.5-flash-lite","reasoning_mode":"minimal","overall_metrics":{"auc_1m":0.18208857560991035,"auc_128k":0.24178353129834942,"cum_avg_1m":0.23528397165015205,"cum_avg_128k":0.2809166247762246,"avg_score_overall":0.20273508735372503,"avg_response_time_ms":8360.67659485228,"total_prompt_cost":7.001603700000004,"total_cache_read_cost":14.605138799999919,"total_completion_cost":1.2798824999999998,"total_reasoning_cost":0,"total_prompt_cost_128k":1.9645239,"total_completion_cost_128k":0.46434999999999993,"total_on_demand_input_cost":153.05299170000006,"total_on_demand_input_cost_128k":7.36404,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154894},"bin_metrics":{"8192":{"avg_score":0.4473486207076348,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.47859732266223426,"median_score":0.05953447334468365,"q1_score":0.02972931507974863,"q3_score":1,"avg_token_efficiency":1.0001553404747234,"is_incomplete":false,"total_prompt_cost_bin":0.15946320000000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06987499999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.22933820000000005,"total_on_demand_input_cost_bin":0.15946320000000005,"ci_99_lower":0.30325979024887784,"ci_99_upper":0.5777333818103071,"ci_98_lower":0.31461082110274585,"ci_98_upper":0.5668135248245671,"ci_95_lower":0.33829057276627283,"ci_95_upper":0.5551832312130136,"ci_90_lower":0.35205903375986963,"ci_90_upper":0.5350844616873653,"ci_75_lower":0.38492917572439184,"ci_75_upper":0.5081044897742687,"ci_50_lower":0.41087566410715226,"ci_50_upper":0.48329688676458316},"16384":{"avg_score":0.28447742578573526,"min_score":0.007738607004284859,"max_score":1,"n_tests":80,"stddev":0.42975689863414673,"median_score":0.03327174857258797,"q1_score":0.024387188255786896,"q3_score":1,"avg_token_efficiency":1.0000745135024474,"is_incomplete":false,"total_prompt_cost_bin":0.33056160000000007,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.07041,"total_reasoning_cost_bin":0,"total_cost_bin":0.4009716000000001,"total_on_demand_input_cost_bin":0.33056160000000007,"ci_99_lower":0.176306334597175,"ci_99_upper":0.4064429854087648,"ci_98_lower":0.18581195345998278,"ci_98_upper":0.3958280278283637,"ci_95_lower":0.1977749994341866,"ci_95_upper":0.3815506227969308,"ci_90_lower":0.21037141389388125,"ci_90_upper":0.3587020269478671,"ci_75_lower":0.23481261664710473,"ci_75_upper":0.33468888658680956,"ci_50_lower":0.25025664621207394,"ci_50_upper":0.31011738224769947},"32768":{"avg_score":0.16521777343587019,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.32865199528725664,"median_score":0.029865184798836708,"q1_score":0.02158403117209673,"q3_score":0.05936024896800518,"avg_token_efficiency":1.0000359090949913,"is_incomplete":false,"total_prompt_cost_bin":0.5090745000000002,"total_cache_read_cost_bin":0.06529503000000005,"total_completion_cost_bin":0.13065749999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.7050270300000002,"total_on_demand_input_cost_bin":1.1620247999999989,"ci_99_lower":0.09806901594034999,"ci_99_upper":0.24462850144997528,"ci_98_lower":0.10203437507590826,"ci_98_upper":0.23655164781836313,"ci_95_lower":0.11168842426555142,"ci_95_upper":0.2233342154044829,"ci_90_lower":0.11947973702190212,"ci_90_upper":0.2107933013289071,"ci_75_lower":0.13185405688273022,"ci_75_upper":0.19603885396495838,"ci_50_lower":0.1446787745662732,"ci_50_upper":0.1806190450601314},"65536":{"avg_score":0.22527599216384045,"min_score":0.008358662948012352,"max_score":1,"n_tests":85,"stddev":0.38985413165114047,"median_score":0.03414282202720642,"q1_score":0.023394852876663208,"q3_score":0.059739597141742706,"avg_token_efficiency":1.000015287852281,"is_incomplete":false,"total_prompt_cost_bin":0.4300160999999998,"total_cache_read_cost_bin":0.12380004000000021,"total_completion_cost_bin":0.08488000000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.63869614,"total_on_demand_input_cost_bin":1.6680164999999987,"ci_99_lower":0.13445046151664986,"ci_99_upper":0.34026812940824996,"ci_98_lower":0.1368260491490364,"ci_98_upper":0.32955468683131034,"ci_95_lower":0.15372143250670942,"ci_95_upper":0.30639104523120775,"ci_90_lower":0.1589857981036253,"ci_90_upper":0.29431009462103247,"ci_75_lower":0.17763606999891207,"ci_75_upper":0.2705745951327331,"ci_50_lower":0.1924427248811459,"ci_50_upper":0.24884916162392237},"131072":{"avg_score":0.2822633117880419,"min_score":0.003092384897172451,"max_score":1,"n_tests":103,"stddev":0.4299926693020455,"median_score":0.03214080631732941,"q1_score":0.019528918899595737,"q3_score":0.9978422224521637,"avg_token_efficiency":1.000007641057256,"is_incomplete":false,"total_prompt_cost_bin":0.5354085000000001,"total_cache_read_cost_bin":0.35085654000000044,"total_completion_cost_bin":0.10852749999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.9947925400000005,"total_on_demand_input_cost_bin":4.043973900000003,"ci_99_lower":0.17839299730644606,"ci_99_upper":0.3863939153578006,"ci_98_lower":0.18700081121067977,"ci_98_upper":0.3848195117622958,"ci_95_lower":0.206143884364181,"ci_95_upper":0.366188669389281,"ci_90_lower":0.2160162136984935,"ci_90_upper":0.34930483325004796,"ci_75_lower":0.23491958607591745,"ci_75_upper":0.3295298665134661,"ci_50_lower":0.2538245988645799,"ci_50_upper":0.31029499818466927},"262144":{"avg_score":0.1498985115210823,"min_score":0.0032573288772255182,"max_score":1,"n_tests":141,"stddev":0.3145946091111916,"median_score":0.032786883413791656,"q1_score":0.023717595264315605,"q3_score":0.04735375940799713,"avg_token_efficiency":1.0000038179365238,"is_incomplete":false,"total_prompt_cost_bin":0.8324262,"total_cache_read_cost_bin":1.0246899299999968,"total_completion_cost_bin":0.1358225000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.992938629999997,"total_on_demand_input_cost_bin":11.079325499999985,"ci_99_lower":0.0882572493156012,"ci_99_upper":0.21894746134351564,"ci_98_lower":0.09479540945559158,"ci_98_upper":0.2096687930888754,"ci_95_lower":0.09908871008289634,"ci_95_upper":0.20318511994621644,"ci_90_lower":0.10874517112970351,"ci_90_upper":0.19409354173789684,"ci_75_lower":0.11783370020318816,"ci_75_upper":0.18166983019497,"ci_50_lower":0.13080822073867715,"ci_50_upper":0.16399171202838195},"524288":{"avg_score":0.1970963095377941,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.35732611169510475,"median_score":0.033602263778448105,"q1_score":0.02189568802714348,"q3_score":0.06756501272320747,"avg_token_efficiency":1.0000019078540328,"is_incomplete":false,"total_prompt_cost_bin":1.4571435000000013,"total_cache_read_cost_bin":3.5652684599999964,"total_completion_cost_bin":0.2143150000000001,"total_reasoning_cost_bin":0,"total_cost_bin":5.236726959999998,"total_on_demand_input_cost_bin":37.10982809999999,"ci_99_lower":0.14233957328604754,"ci_99_upper":0.25608089854499777,"ci_98_lower":0.14513371705604414,"ci_98_upper":0.25113874754396864,"ci_95_lower":0.15298675533159148,"ci_95_upper":0.24331412524230564,"ci_90_lower":0.15890757783505496,"ci_90_upper":0.23546039567830168,"ci_75_lower":0.17225503109709234,"ci_75_upper":0.22465131899008478,"ci_50_lower":0.18130498599128847,"ci_50_upper":0.21174056426389143},"1048576":{"avg_score":0.13069382826121703,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.29147088986526815,"median_score":0.029743928462266922,"q1_score":0.018785081803798676,"q3_score":0.04604030679911375,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":2.7475101000000026,"total_cache_read_cost_bin":9.475228799999925,"total_completion_cost_bin":0.46539499999999984,"total_reasoning_cost_bin":0,"total_cost_bin":12.688133899999928,"total_on_demand_input_cost_bin":97.49979810000009,"ci_99_lower":0.09265838857288773,"ci_99_upper":0.1750661604318302,"ci_98_lower":0.09576367236789496,"ci_98_upper":0.17258084220021602,"ci_95_lower":0.0994418081618853,"ci_95_upper":0.1648302283013923,"ci_90_lower":0.1035588295418664,"ci_90_upper":0.15978085092629393,"ci_75_lower":0.11135497334781254,"ci_75_upper":0.1500883936202304,"ci_50_lower":0.11842028757936561,"ci_50_upper":0.1412926149451279}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"minimal","sponsors":[],"openrouter_name":"Google: Gemini 3.5 Flash Lite","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":3e-7,"completion":0.0000025,"reasoning":0.0000025,"has_tiered_pricing":true},"api_created_at":"2026-07-21T15:12:06.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":null},{"model_slug":"x-ai/grok-4.6","reasoning_mode":"low","overall_metrics":{"auc_1m":0.18071509243162023,"auc_128k":0.6364407786343187,"cum_avg_1m":0.5211892457815489,"cum_avg_128k":0.7504012258716664,"avg_score_overall":0.668511159998551,"avg_response_time_ms":20060.0280421875,"total_prompt_cost":59.73009400000001,"total_cache_read_cost":32.48761599999999,"total_completion_cost":5.119704,"total_reasoning_cost":3.742692000000001,"total_prompt_cost_128k":19.290847999999997,"total_completion_cost_128k":3.627528,"total_on_demand_input_cost":189.77130000000008,"total_on_demand_input_cost_128k":47.43472000000002,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9707781091017185},"bin_metrics":{"8192":{"avg_score":0.9758815949084237,"min_score":0.02591792680323124,"max_score":1,"n_tests":80,"stddev":0.15157697542073562,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9942756576635189,"is_incomplete":false,"total_prompt_cost_bin":0.9288799999999999,"total_cache_read_cost_bin":0.03200000000000007,"total_completion_cost_bin":0.413958,"total_reasoning_cost_bin":0.25288799999999995,"total_cost_bin":1.627726,"total_on_demand_input_cost_bin":1.0568799999999996,"ci_99_lower":0.9274099697532365,"ci_99_upper":1,"ci_98_lower":0.9278737585288473,"ci_98_upper":1,"ci_95_lower":0.9395871639018878,"ci_95_upper":1,"ci_90_lower":0.9400544573785737,"ci_90_upper":1,"ci_75_lower":0.9519968365551904,"ci_75_upper":1,"ci_50_lower":0.9639392157318071,"ci_50_upper":0.9880576208233833},"16384":{"avg_score":0.9507914919639007,"min_score":0.01199815422296524,"max_score":1,"n_tests":80,"stddev":0.21404855534292827,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9656677655727354,"is_incomplete":false,"total_prompt_cost_bin":1.5732640000000007,"total_cache_read_cost_bin":0.13843199999999994,"total_completion_cost_bin":0.4755599999999998,"total_reasoning_cost_bin":0.31673400000000007,"total_cost_bin":2.503990000000001,"total_on_demand_input_cost_bin":2.126991999999999,"ci_99_lower":0.8777289013075643,"ci_99_upper":0.9996045879274607,"ci_98_lower":0.8885568931167945,"ci_98_upper":0.9996026009321213,"ci_95_lower":0.8901887497288408,"ci_95_upper":0.9878368034260347,"ci_90_lower":0.9022123437584378,"ci_90_upper":0.9874794853618368,"ci_75_lower":0.9258449552813545,"ci_75_upper":0.975491790479282,"ci_50_lower":0.9382709773257375,"ci_50_upper":0.9633636647835374},"32768":{"avg_score":0.8245034557801452,"min_score":0.005333333276212215,"max_score":1,"n_tests":136,"stddev":0.37274164551290323,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9792383118514036,"is_incomplete":false,"total_prompt_cost_bin":3.006463999999995,"total_cache_read_cost_bin":1.1476480000000013,"total_completion_cost_bin":0.9785520000000002,"total_reasoning_cost_bin":0.6713520000000004,"total_cost_bin":5.804015999999997,"total_on_demand_input_cost_bin":7.597056000000006,"ci_99_lower":0.7389663131644791,"ci_99_upper":0.8962577032820006,"ci_98_lower":0.749418618121246,"ci_98_upper":0.8930124885218409,"ci_95_lower":0.7577249844723745,"ci_95_upper":0.8854766492793501,"ci_90_lower":0.7666340272164877,"ci_90_upper":0.8769606963835779,"ci_75_lower":0.7850628944117274,"ci_75_upper":0.8604676652543918,"ci_50_lower":0.8028086541787214,"ci_50_upper":0.8473235837060629},"65536":{"avg_score":0.5778274649754167,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4840346717328759,"median_score":1,"q1_score":0.023241335526108742,"q3_score":1,"avg_token_efficiency":0.9612760503125508,"is_incomplete":false,"total_prompt_cost_bin":4.319028000000001,"total_cache_read_cost_bin":1.5925760000000009,"total_completion_cost_bin":0.7445219999999999,"total_reasoning_cost_bin":0.5514,"total_cost_bin":7.207526000000002,"total_on_demand_input_cost_bin":10.689332000000002,"ci_99_lower":0.4386955776297871,"ci_99_upper":0.7048841911100727,"ci_98_lower":0.44989557803257857,"ci_98_upper":0.7004487549626434,"ci_95_lower":0.47524870232952865,"ci_95_upper":0.6811067024843,"ci_90_lower":0.4957865454826285,"ci_90_upper":0.6683461051894461,"ci_75_lower":0.5187933187014149,"ci_75_upper":0.6430397708735922,"ci_50_lower":0.5434093805303907,"ci_50_upper":0.6125378122991498},"131072":{"avg_score":0.4230021217304454,"min_score":0.0047505940310657024,"max_score":1,"n_tests":103,"stddev":0.48115916386819363,"median_score":0.03803780674934387,"q1_score":0.020738478749990463,"q3_score":1,"avg_token_efficiency":0.9630870168439467,"is_incomplete":false,"total_prompt_cost_bin":9.463212,"total_cache_read_cost_bin":4.125311999999998,"total_completion_cost_bin":1.0149359999999998,"total_reasoning_cost_bin":0.7686839999999998,"total_cost_bin":15.372144,"total_on_demand_input_cost_bin":25.964460000000013,"ci_99_lower":0.29839023967599043,"ci_99_upper":0.5359316339126328,"ci_98_lower":0.3108785214954625,"ci_98_upper":0.526491343558142,"ci_95_lower":0.32802122285684515,"ci_95_upper":0.5159732913822207,"ci_90_lower":0.34528760201460457,"ci_90_upper":0.5005623753334043,"ci_75_lower":0.36669587366253553,"ci_75_upper":0.47974910150571426,"ci_50_lower":0.3877682742206109,"ci_50_upper":0.45929161535386104},"262144":{"avg_score":0.4175078368940595,"min_score":0.002326934365555644,"max_score":1,"n_tests":141,"stddev":0.4747675181229302,"median_score":0.04730617627501488,"q1_score":0.027201145887374878,"q3_score":1,"avg_token_efficiency":0.9635319638751026,"is_incomplete":false,"total_prompt_cost_bin":40.43924600000001,"total_cache_read_cost_bin":25.451647999999985,"total_completion_cost_bin":1.4921759999999997,"total_reasoning_cost_bin":1.1816340000000005,"total_cost_bin":68.564704,"total_on_demand_input_cost_bin":142.33658000000005,"ci_99_lower":0.30825154719230263,"ci_99_upper":0.5255683838321038,"ci_98_lower":0.3207354402140688,"ci_98_upper":0.5149956122688382,"ci_95_lower":0.33509535127648,"ci_95_upper":0.5006875783198015,"ci_90_lower":0.3496045600630323,"ci_90_upper":0.486031054301478,"ci_75_lower":0.3696748421736745,"ci_75_upper":0.46500290031767794,"ci_50_lower":0.38979090201992095,"ci_50_upper":0.44496311034044544}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":500000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low","enabled"],"reasoning_default":"high","sponsors":[],"openrouter_name":"SpaceXAI: Grok 4.6","provider_name":"X-ai","run_provider_names":["SpaceXAI"],"pricing_metadata":{"prompt":0.000002,"completion":0.000006,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:35:57.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-14T01:35:43.986Z"},{"model_slug":"deepseek/deepseek-v4-pro-0813","reasoning_mode":null,"overall_metrics":{"auc_1m":0.17888276714900384,"auc_128k":0.31928643695002623,"cum_avg_1m":0.29272478984416617,"cum_avg_128k":0.3752652358685844,"avg_score_overall":0.30133151691882903,"avg_response_time_ms":5819.197237553081,"total_prompt_cost":0.016894964999999994,"total_cache_read_cost":0.668492687999999,"total_completion_cost":0.2811492000000001,"total_reasoning_cost":0,"total_prompt_cost_128k":0.010873260000000001,"total_completion_cost_128k":0.15905688,"total_on_demand_input_cost":80.236017525,"total_on_demand_input_cost_128k":10.6406307,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971342554822207},"bin_metrics":{"8192":{"avg_score":0.6255812055082061,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4732377032128615,"median_score":1,"q1_score":0.03443133644759655,"q3_score":1,"avg_token_efficiency":0.9915838193643983,"is_incomplete":false,"total_prompt_cost_bin":0.0022167600000000008,"total_cache_read_cost_bin":0.001893120000000001,"total_completion_cost_bin":0.024186870000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.028296750000000002,"total_on_demand_input_cost_bin":0.22939116000000012,"ci_99_lower":0.4790525513607426,"ci_99_upper":0.7605452870105509,"ci_98_lower":0.5008156280058902,"ci_98_upper":0.738272703037248,"ci_95_lower":0.5181717480125372,"ci_95_upper":0.7227020447090035,"ci_90_lower":0.5401518138311803,"ci_90_upper":0.7090563075360842,"ci_75_lower":0.5650863362578094,"ci_75_upper":0.6864124815168907,"ci_50_lower":0.5892199339548825,"ci_50_upper":0.6603414223209256},"16384":{"avg_score":0.3681833181995898,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.46409940477896544,"median_score":0.04244361259043217,"q1_score":0.02356732077896595,"q3_score":0.9993137419223785,"avg_token_efficiency":0.9890247337221417,"is_incomplete":false,"total_prompt_cost_bin":0.00202188,"total_cache_read_cost_bin":0.003931008000000001,"total_completion_cost_bin":0.024197310000000007,"total_reasoning_cost_bin":0,"total_cost_bin":0.030150198000000007,"total_on_demand_input_cost_bin":0.4737428399999999,"ci_99_lower":0.2348968461864861,"ci_99_upper":0.5012094349188847,"ci_98_lower":0.2462579951508669,"ci_98_upper":0.4889660961611662,"ci_95_lower":0.26059729257976866,"ci_95_upper":0.468447167240083,"ci_90_lower":0.2825569817912765,"ci_90_upper":0.4531909659702796,"ci_75_lower":0.30832571087521504,"ci_75_upper":0.4284372937341686,"ci_50_lower":0.33213496637472417,"ci_50_upper":0.40354225663468235},"32768":{"avg_score":0.30653201460646573,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.4338797626286314,"median_score":0.039359742775559425,"q1_score":0.023482299875468016,"q3_score":1,"avg_token_efficiency":1.010289265167121,"is_incomplete":false,"total_prompt_cost_bin":0.0028083599999999976,"total_cache_read_cost_bin":0.014164992000000006,"total_completion_cost_bin":0.0457794,"total_reasoning_cost_bin":0,"total_cost_bin":0.062752752,"total_on_demand_input_cost_bin":1.7026074000000035,"ci_99_lower":0.21691762587586283,"ci_99_upper":0.397439040086619,"ci_98_lower":0.22199999775775187,"ci_98_upper":0.38649023108612335,"ci_95_lower":0.2356902902603478,"ci_95_upper":0.37547804435872584,"ci_90_lower":0.24489023778895738,"ci_90_upper":0.3654176116503282,"ci_75_lower":0.2632995905013933,"ci_75_upper":0.350848579497645,"ci_50_lower":0.2804758620152236,"ci_50_upper":0.3310516004840953},"65536":{"avg_score":0.3502581831314327,"min_score":0.0029607696924358606,"max_score":1,"n_tests":85,"stddev":0.4577581751899575,"median_score":0.035944003611803055,"q1_score":0.025265689939260483,"q3_score":1,"avg_token_efficiency":0.9947451157769149,"is_incomplete":false,"total_prompt_cost_bin":0.0016151549999999983,"total_cache_read_cost_bin":0.020035520000000005,"total_completion_cost_bin":0.02822019000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.049870865000000014,"total_on_demand_input_cost_bin":2.4058775549999973,"ci_99_lower":0.24338908473358434,"ci_99_upper":0.4767089494098635,"ci_98_lower":0.24611855081945438,"ci_98_upper":0.46580085323959153,"ci_95_lower":0.25717300224234346,"ci_95_upper":0.45224338926539265,"ci_90_lower":0.269317227356985,"ci_90_upper":0.4393343246233759,"ci_75_lower":0.2922715426956796,"ci_75_upper":0.407407238461447,"ci_50_lower":0.31611064608921025,"ci_50_upper":0.3837779224007938},"131072":{"avg_score":0.22577145789722794,"min_score":0.0039748260751366615,"max_score":1,"n_tests":103,"stddev":0.3939121918836241,"median_score":0.029202409088611603,"q1_score":0.019659665413200855,"q3_score":0.05286543443799019,"avg_token_efficiency":0.994081296447494,"is_incomplete":false,"total_prompt_cost_bin":0.0022111050000000027,"total_cache_read_cost_bin":0.048556671999999974,"total_completion_cost_bin":0.036673109999999995,"total_reasoning_cost_bin":0,"total_cost_bin":0.08744088699999997,"total_on_demand_input_cost_bin":5.829011745,"ci_99_lower":0.1312040331139166,"ci_99_upper":0.3286658379572751,"ci_98_lower":0.1402467139912498,"ci_98_upper":0.31136097412723684,"ci_95_lower":0.1582483950996708,"ci_95_upper":0.2928744061318796,"ci_90_lower":0.167607501629532,"ci_90_upper":0.2829897268982695,"ci_75_lower":0.17959160396085544,"ci_75_upper":0.2651505076842952,"ci_50_lower":0.1986718219727531,"ci_50_upper":0.25249420556361846},"262144":{"avg_score":0.2591674783835122,"min_score":0.0023255813866853714,"max_score":1,"n_tests":141,"stddev":0.41140270243131993,"median_score":0.034246575087308884,"q1_score":0.02143104001879692,"q3_score":0.08377896249294281,"avg_token_efficiency":0.9994797316927638,"is_incomplete":false,"total_prompt_cost_bin":0.0015529500000000028,"total_cache_read_cost_bin":0.13379208000000004,"total_completion_cost_bin":0.04794048000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.1832855100000001,"total_on_demand_input_cost_bin":16.056602549999994,"ci_99_lower":0.17616785912896696,"ci_99_upper":0.3418966057024737,"ci_98_lower":0.17874850191344369,"ci_98_upper":0.33569675345152145,"ci_95_lower":0.19292684058206327,"ci_95_upper":0.32447834037341744,"ci_90_lower":0.20367808125893325,"ci_90_upper":0.31419947483983046,"ci_75_lower":0.21857681565014778,"ci_75_upper":0.2953330878071745,"ci_50_lower":0.23719384342825028,"ci_50_upper":0.2804496155057684},"524288":{"avg_score":0.20630466102689538,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.365752050865615,"median_score":0.035561906173825264,"q1_score":0.0212287618778646,"q3_score":0.07802350632846355,"avg_token_efficiency":0.9949755047319111,"is_incomplete":false,"total_prompt_cost_bin":0.004468754999999991,"total_cache_read_cost_bin":0.44611929599999894,"total_completion_cost_bin":0.07415184000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.524739890999999,"total_on_demand_input_cost_bin":53.53878427499999,"ci_99_lower":0.14578312978671834,"ci_99_upper":0.2677364583410593,"ci_98_lower":0.15130901251888984,"ci_98_upper":0.26102795328999184,"ci_95_lower":0.16143654179973904,"ci_95_upper":0.2547538746616722,"ci_90_lower":0.1682768369414666,"ci_90_upper":0.24593870984489036,"ci_75_lower":0.18088010213583494,"ci_75_upper":0.23404397275565209,"ci_50_lower":0.19127075263383894,"ci_50_upper":0.22107339680778904}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Pro 0813","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":0.00000132,"completion":0.00000396,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T15:42:44.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T17:36:10.932Z"},{"model_slug":"qwen/qwen3.8-27b","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.1593107504974293,"auc_128k":0.9278557991654294,"cum_avg_1m":0.5900466738848084,"cum_avg_128k":0.9440746782156934,"avg_score_overall":0.9397714435457721,"avg_response_time_ms":28194.987509483148,"total_prompt_cost":11.257057800000002,"total_cache_read_cost":0,"total_completion_cost":1.4939872,"total_reasoning_cost":0.8801216000000002,"total_prompt_cost_128k":11.257057800000002,"total_completion_cost_128k":1.4939872,"total_on_demand_input_cost":11.257057800000002,"total_on_demand_input_cost_128k":11.257057800000002,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207081000672176},"bin_metrics":{"8192":{"avg_score":0.9930317617952824,"min_score":0.9468473196029663,"max_score":1,"n_tests":80,"stddev":0.011503289814064335,"median_score":0.9974396526813507,"q1_score":0.99247607588768,"q3_score":1,"avg_token_efficiency":1.022497270322229,"is_incomplete":false,"total_prompt_cost_bin":0.24452640000000006,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.20763520000000008,"total_reasoning_cost_bin":0.1134272,"total_cost_bin":0.5655888000000002,"total_on_demand_input_cost_bin":0.24452640000000006,"ci_99_lower":0.9888791979774834,"ci_99_upper":0.9960807051174343,"ci_98_lower":0.989218166694045,"ci_98_upper":0.9958184349909425,"ci_95_lower":0.9903490461781621,"ci_95_upper":0.9953484713099896,"ci_90_lower":0.9907836832478643,"ci_90_upper":0.9950132907554506,"ci_75_lower":0.9914189880713821,"ci_75_upper":0.9944017673842608,"ci_50_lower":0.9921715682372452,"ci_50_upper":0.993868718110025},"16384":{"avg_score":0.9915868893265725,"min_score":0.9276835918426514,"max_score":1,"n_tests":80,"stddev":0.012721257185127281,"median_score":0.9965114593505859,"q1_score":0.9912072569131851,"q3_score":0.9990409761667252,"avg_token_efficiency":1.015970448833568,"is_incomplete":false,"total_prompt_cost_bin":0.5043168000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.20753279999999996,"total_reasoning_cost_bin":0.11327360000000002,"total_cost_bin":0.8251232000000001,"total_on_demand_input_cost_bin":0.5043168000000001,"ci_99_lower":0.9872226235382259,"ci_99_upper":0.9950238970294595,"ci_98_lower":0.9876124000549317,"ci_98_upper":0.9947846348658204,"ci_95_lower":0.9886308489367366,"ci_95_upper":0.9941014300286769,"ci_90_lower":0.9891791759803892,"ci_90_upper":0.9937613673508167,"ci_75_lower":0.9898775950074197,"ci_75_upper":0.9931436454877258,"ci_50_lower":0.9906441437080502,"ci_50_upper":0.9925465499982239},"32768":{"avg_score":0.9673101952409043,"min_score":0.02304147556424141,"max_score":1,"n_tests":136,"stddev":0.13982797629193758,"median_score":0.9958765506744385,"q1_score":0.9784428179264069,"q3_score":1,"avg_token_efficiency":1.0256830410679836,"is_incomplete":false,"total_prompt_cost_bin":1.7894736,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.38086400000000004,"total_reasoning_cost_bin":0.20087040000000006,"total_cost_bin":2.3712079999999998,"total_on_demand_input_cost_bin":1.7894736,"ci_99_lower":0.9253162052241319,"ci_99_upper":0.989739330623518,"ci_98_lower":0.9334764617686981,"ci_98_upper":0.9888439872466466,"ci_95_lower":0.9398231651099838,"ci_95_upper":0.9874114455326515,"ci_90_lower":0.9455489911248579,"ci_90_upper":0.9834378610365093,"ci_75_lower":0.9531434169586966,"ci_75_upper":0.9811807043254649,"ci_50_lower":0.959916739984799,"ci_50_upper":0.9752082411227199},"65536":{"avg_score":0.979116004004198,"min_score":0.03998491168022156,"max_score":1,"n_tests":85,"stddev":0.10420727621569065,"median_score":0.9974874258041382,"q1_score":0.9856215119361877,"q3_score":0.9992284178733826,"avg_token_efficiency":1.0209598255175438,"is_incomplete":false,"total_prompt_cost_bin":2.5544277000000006,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.2523776,"total_reasoning_cost_bin":0.14287680000000003,"total_cost_bin":2.9496821000000004,"total_on_demand_input_cost_bin":2.5544277000000006,"ci_99_lower":0.9436806037320811,"ci_99_upper":0.9935220703447567,"ci_98_lower":0.9453713064123602,"ci_98_upper":0.9933798230395599,"ci_95_lower":0.9528338276814012,"ci_95_upper":0.9925362348907134,"ci_90_lower":0.9567271508890038,"ci_90_upper":0.9921066068901735,"ci_75_lower":0.966986374031095,"ci_75_upper":0.9909386485815047,"ci_50_lower":0.9705032725544536,"ci_50_upper":0.9895454859032351},"131072":{"avg_score":0.7893285407115099,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.3879483907671641,"median_score":0.9894397258758545,"q1_score":0.9446562230587006,"q3_score":0.9995309114456177,"avg_token_efficiency":1.0162215936488597,"is_incomplete":false,"total_prompt_cost_bin":6.164313300000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.4455776,"total_reasoning_cost_bin":0.3096736000000001,"total_cost_bin":6.919564500000001,"total_on_demand_input_cost_bin":6.164313300000001,"ci_99_lower":0.6773201731465377,"ci_99_upper":0.8827214777334006,"ci_98_lower":0.6796981524582952,"ci_98_upper":0.8750827582530424,"ci_95_lower":0.7065171748754632,"ci_95_upper":0.864124246153221,"ci_90_lower":0.7229199929727983,"ci_90_upper":0.8521140527292109,"ci_75_lower":0.7430853395908167,"ci_75_upper":0.8319651424755948,"ci_50_lower":0.7624395599914262,"ci_50_upper":0.8164605430612177}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","medium","low"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"Qwen: Qwen3.8 27B","provider_name":"Qwen","run_provider_names":["AkashML"],"pricing_metadata":{"prompt":4.25e-7,"completion":0.00000255,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-14T15:55:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-16T14:56:17.473Z"},{"model_slug":"qwen/qwen3.8-27b","reasoning_mode":"low","overall_metrics":{"auc_1m":0.15909665724620037,"auc_128k":0.910691253320314,"cum_avg_1m":0.581644647650076,"cum_avg_128k":0.9306314362401216,"avg_score_overall":0.9278997638876635,"avg_response_time_ms":34186.5016121352,"total_prompt_cost":3.7570662000000006,"total_cache_read_cost":7.5065255999999945,"total_completion_cost":1.2704128,"total_reasoning_cost":0.6558495999999999,"total_prompt_cost_128k":3.7570662000000006,"total_completion_cost_128k":1.2704128,"total_on_demand_input_cost":11.263591799999997,"total_on_demand_input_cost_128k":11.263591799999997,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0222799049081523},"bin_metrics":{"8192":{"avg_score":0.9930888742208481,"min_score":0.9468473196029663,"max_score":1,"n_tests":80,"stddev":0.011536160745347518,"median_score":0.9976620674133301,"q1_score":0.99247607588768,"q3_score":1,"avg_token_efficiency":1.0271574845639369,"is_incomplete":false,"total_prompt_cost_bin":0.181044,"total_cache_read_cost_bin":0.06456240000000002,"total_completion_cost_bin":0.18123200000000003,"total_reasoning_cost_bin":0.08626239999999995,"total_cost_bin":0.5131008,"total_on_demand_input_cost_bin":0.24560640000000003,"ci_99_lower":0.988918978329748,"ci_99_upper":0.9961741709262132,"ci_98_lower":0.989248468823731,"ci_98_upper":0.9959149134010076,"ci_95_lower":0.9903846728429199,"ci_95_upper":0.99542262673378,"ci_90_lower":0.9908463894948364,"ci_90_upper":0.9950734737887978,"ci_75_lower":0.9914611033163965,"ci_75_upper":0.9944640257395804,"ci_50_lower":0.9922302674502135,"ci_50_upper":0.9939360657706857},"16384":{"avg_score":0.9558474860677961,"min_score":0.006335797253996134,"max_score":1,"n_tests":80,"stddev":0.18701037634999212,"median_score":0.9966711103916168,"q1_score":0.9912072569131851,"q3_score":0.9990913718938828,"avg_token_efficiency":1.0182058539069945,"is_incomplete":false,"total_prompt_cost_bin":0.29406960000000004,"total_cache_read_cost_bin":0.21132720000000008,"total_completion_cost_bin":0.19013119999999994,"total_reasoning_cost_bin":0.095856,"total_cost_bin":0.7913840000000001,"total_on_demand_input_cost_bin":0.5053968000000001,"ci_99_lower":0.8930216589620686,"ci_99_upper":0.9944884718880058,"ci_98_lower":0.8955899018459021,"ci_98_upper":0.99397723197937,"ci_95_lower":0.906728054031264,"ci_95_upper":0.9927213902771473,"ci_90_lower":0.9187047654303023,"ci_90_upper":0.9910803840681911,"ci_75_lower":0.9308679452311481,"ci_75_upper":0.9801748821490037,"ci_50_lower":0.9433658083624322,"ci_50_upper":0.9685521958104801},"32768":{"avg_score":0.9530150414211676,"min_score":0.009646302089095116,"max_score":1,"n_tests":136,"stddev":0.18190988520527931,"median_score":0.9960004985332489,"q1_score":0.976398304104805,"q3_score":1,"avg_token_efficiency":1.0267603139177033,"is_incomplete":false,"total_prompt_cost_bin":0.6761087999999998,"total_cache_read_cost_bin":1.1152008,"total_completion_cost_bin":0.34801280000000007,"total_reasoning_cost_bin":0.16861119999999996,"total_cost_bin":2.3079336,"total_on_demand_input_cost_bin":1.7913095999999995,"ci_99_lower":0.903195534869241,"ci_99_upper":0.987345560061581,"ci_98_lower":0.9092657024093547,"ci_98_upper":0.9839729022387387,"ci_95_lower":0.9169417667075963,"ci_95_upper":0.9815930524527657,"ci_90_lower":0.924117965676808,"ci_90_upper":0.9761953932811807,"ci_75_lower":0.9333726338835036,"ci_75_upper":0.9713234618812909,"ci_50_lower":0.9433067662586623,"ci_50_upper":0.9640387424425332},"65536":{"avg_score":0.9330924456829534,"min_score":0.009118541143834591,"max_score":1,"n_tests":85,"stddev":0.22960516326633523,"median_score":0.997481107711792,"q1_score":0.981246829032898,"q3_score":0.9993489384651184,"avg_token_efficiency":1.0214184610859864,"is_incomplete":false,"total_prompt_cost_bin":0.7862831999999998,"total_cache_read_cost_bin":1.7692919999999979,"total_completion_cost_bin":0.26217279999999993,"total_reasoning_cost_bin":0.15178239999999998,"total_cost_bin":2.969530399999998,"total_on_demand_input_cost_bin":2.5555752000000003,"ci_99_lower":0.8532808221391238,"ci_99_upper":0.9815746203104363,"ci_98_lower":0.8637545007282101,"ci_98_upper":0.9800210855123312,"ci_95_lower":0.8757472844019203,"ci_95_upper":0.9769158618554801,"ci_90_lower":0.8863089922475903,"ci_90_upper":0.9690831511232125,"ci_75_lower":0.906981314646135,"ci_75_upper":0.9594204390846083,"ci_50_lower":0.919241033724564,"ci_50_upper":0.9533606024322046},"131072":{"avg_score":0.8181133338078422,"min_score":0.01206181664019823,"max_score":1,"n_tests":103,"stddev":0.3659444369405978,"median_score":0.9917452931404114,"q1_score":0.9577004611492157,"q3_score":0.9994306564331055,"avg_token_efficiency":1.0164508253665498,"is_incomplete":false,"total_prompt_cost_bin":1.8195606000000006,"total_cache_read_cost_bin":4.346143199999997,"total_completion_cost_bin":0.28886400000000007,"total_reasoning_cost_bin":0.15333760000000007,"total_cost_bin":6.607905399999998,"total_on_demand_input_cost_bin":6.165703799999998,"ci_99_lower":0.7144253817569574,"ci_99_upper":0.9022827127033808,"ci_98_lower":0.7333842372271295,"ci_98_upper":0.893525227270297,"ci_95_lower":0.7429843936488484,"ci_95_upper":0.8847916580994334,"ci_90_lower":0.7527269948716954,"ci_90_upper":0.8749117230756878,"ci_75_lower":0.7723611928047883,"ci_75_upper":0.8569554259799403,"ci_50_lower":0.7912897919847187,"ci_50_upper":0.8445945895792357}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","medium","low"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"Qwen: Qwen3.8 27B","provider_name":"Qwen","run_provider_names":["AkashML"],"pricing_metadata":{"prompt":4.25e-7,"completion":0.00000255,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-14T15:55:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-16T15:43:29.450Z"},{"model_slug":"deepseek/deepseek-v4-pro","reasoning_mode":null,"overall_metrics":{"auc_1m":0.15843087928315783,"auc_128k":0.2656341028291076,"cum_avg_1m":0.25532264017840334,"cum_avg_128k":0.3271679478457396,"avg_score_overall":0.26310180312026366,"avg_response_time_ms":20202.533718722683,"total_prompt_cost":14.129058824999989,"total_cache_read_cost":0.5508167199999989,"total_completion_cost":0.28213143000000007,"total_reasoning_cost":0,"total_prompt_cost_128k":1.0302748799999997,"total_completion_cost_128k":0.16005042000000003,"total_on_demand_input_cost":80.22706522500002,"total_on_demand_input_cost_128k":10.631678400000002,"total_runs":858,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971324430132092},"bin_metrics":{"8192":{"avg_score":0.4944938808232062,"min_score":0.002900652587413788,"max_score":1,"n_tests":77,"stddev":0.4856747058716103,"median_score":0.06611569970846176,"q1_score":0.025923775508999825,"q3_score":1,"avg_token_efficiency":0.9913473725232907,"is_incomplete":false,"total_prompt_cost_bin":0.10429038,"total_cache_read_cost_bin":0.000967904,"total_completion_cost_bin":0.02291232,"total_reasoning_cost_bin":0,"total_cost_bin":0.12817060400000002,"total_on_demand_input_cost_bin":0.22043886000000013,"ci_99_lower":0.35990708549044326,"ci_99_upper":0.6322156020725908,"ci_98_lower":0.3681060791000506,"ci_98_upper":0.6207900290751176,"ci_95_lower":0.3809533678370153,"ci_95_upper":0.5977200459785661,"ci_90_lower":0.3952605288034281,"ci_90_upper":0.5817838258061606,"ci_75_lower":0.4318271577857169,"ci_75_upper":0.5571816490468714,"ci_50_lower":0.45754210413554564,"ci_50_upper":0.5324157755186951},"16384":{"avg_score":0.4018853992020013,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.47552518462398974,"median_score":0.03734876401722431,"q1_score":0.019047625828534365,"q3_score":0.9976304322481155,"avg_token_efficiency":0.9890247337221423,"is_incomplete":false,"total_prompt_cost_bin":0.11204556000000007,"total_cache_read_cost_bin":0.0030141440000000016,"total_completion_cost_bin":0.023741429999999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.13880113400000008,"total_on_demand_input_cost_bin":0.4737428399999999,"ci_99_lower":0.26741358799808945,"ci_99_upper":0.5348665792709072,"ci_98_lower":0.27913766483942165,"ci_98_upper":0.5335797514920123,"ci_95_lower":0.29172622240832424,"ci_95_upper":0.5090148770363885,"ci_90_lower":0.30512555385605084,"ci_90_upper":0.4889231821055,"ci_75_lower":0.34137279473507076,"ci_75_upper":0.4627651411192346,"ci_50_lower":0.36554673068385457,"ci_50_upper":0.4378588084873627},"32768":{"avg_score":0.2654122332220568,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.40465006112831065,"median_score":0.040646856650710106,"q1_score":0.0249644061550498,"q3_score":0.20882808044552803,"avg_token_efficiency":1.0102892651671205,"is_incomplete":false,"total_prompt_cost_bin":0.35754564,"total_cache_read_cost_bin":0.011208848000000009,"total_completion_cost_bin":0.046560660000000025,"total_reasoning_cost_bin":0,"total_cost_bin":0.415315148,"total_on_demand_input_cost_bin":1.7026074000000035,"ci_99_lower":0.18046825092691612,"ci_99_upper":0.3484937759402365,"ci_98_lower":0.18554219605014252,"ci_98_upper":0.34336626748862564,"ci_95_lower":0.1996238035713117,"ci_95_upper":0.3308066832215819,"ci_90_lower":0.2075943656347911,"ci_90_upper":0.3203729547050042,"ci_75_lower":0.2241906169164234,"ci_75_upper":0.3036128791414633,"ci_50_lower":0.24146775098223078,"ci_50_upper":0.28798179673578805},"65536":{"avg_score":0.22100344981505152,"min_score":0.0038729666266590357,"max_score":1,"n_tests":85,"stddev":0.3861443066123659,"median_score":0.028325745835900307,"q1_score":0.02151898667216301,"q3_score":0.05371247977018356,"avg_token_efficiency":0.994745115776915,"is_incomplete":false,"total_prompt_cost_bin":0.17132779499999995,"total_cache_read_cost_bin":0.01862124800000001,"total_completion_cost_bin":0.029315519999999994,"total_reasoning_cost_bin":0,"total_cost_bin":0.21926456299999994,"total_on_demand_input_cost_bin":2.4058775549999973,"ci_99_lower":0.13017662433690994,"ci_99_upper":0.3347788677637958,"ci_98_lower":0.13345527685052883,"ci_98_upper":0.32648616768101996,"ci_95_lower":0.14877132469034918,"ci_95_upper":0.3019649173196076,"ci_90_lower":0.15558068249374626,"ci_90_upper":0.2904233771343442,"ci_75_lower":0.173684570694219,"ci_75_upper":0.2661431687312228,"ci_50_lower":0.18847324140164454,"ci_50_upper":0.24409669246319132},"131072":{"avg_score":0.2530447761663823,"min_score":0.004092769231647253,"max_score":1,"n_tests":103,"stddev":0.4131998793330098,"median_score":0.030693676322698593,"q1_score":0.017930220812559128,"q3_score":0.05669598840177059,"avg_token_efficiency":0.9940812964474944,"is_incomplete":false,"total_prompt_cost_bin":0.28506550499999966,"total_cache_read_cost_bin":0.046199551999999984,"total_completion_cost_bin":0.03752049000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.36878554699999966,"total_on_demand_input_cost_bin":5.829011745000001,"ci_99_lower":0.1578431062354227,"ci_99_upper":0.35671222605903014,"ci_98_lower":0.1651256132913862,"ci_98_upper":0.3483204480791826,"ci_95_lower":0.1770053609183908,"ci_95_upper":0.328679941315882,"ci_90_lower":0.18740131145035757,"ci_90_upper":0.31918203887050445,"ci_75_lower":0.2061958659826823,"ci_75_upper":0.2989583067704462,"ci_50_lower":0.2252183326687302,"ci_50_upper":0.28044739038597577},"262144":{"avg_score":0.22548893451400048,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.38638914609357167,"median_score":0.033032454550266266,"q1_score":0.02277039922773838,"q3_score":0.07758919149637222,"avg_token_efficiency":0.999479731692764,"is_incomplete":false,"total_prompt_cost_bin":12.640634549999989,"total_cache_read_cost_bin":0.028466399999999982,"total_completion_cost_bin":0.04707657,"total_reasoning_cost_bin":0,"total_cost_bin":12.716177519999988,"total_on_demand_input_cost_bin":16.056602549999997,"ci_99_lower":0.15088482821802776,"ci_99_upper":0.3081694181081476,"ci_98_lower":0.15545329072506084,"ci_98_upper":0.3016500135467566,"ci_95_lower":0.1641292217189235,"ci_95_upper":0.2870814559904442,"ci_90_lower":0.17387150279923294,"ci_90_upper":0.2745547937118639,"ci_75_lower":0.18839013701757246,"ci_75_upper":0.2618912925600266,"ci_50_lower":0.20326560390171933,"ci_50_upper":0.24526682307907374},"524288":{"avg_score":0.1812524476845283,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.3430488341272407,"median_score":0.0338260680437088,"q1_score":0.022682021837681532,"q3_score":0.06603412516415119,"avg_token_efficiency":0.9949755047319108,"is_incomplete":false,"total_prompt_cost_bin":0.45814939499999946,"total_cache_read_cost_bin":0.44233862399999896,"total_completion_cost_bin":0.07500444000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.9754924589999985,"total_on_demand_input_cost_bin":53.538784275000026,"ci_99_lower":0.1271112183464905,"ci_99_upper":0.23520363230671354,"ci_98_lower":0.13157175666674326,"ci_98_upper":0.23215486689821926,"ci_95_lower":0.13823928960131124,"ci_95_upper":0.22693974068241587,"ci_90_lower":0.14440226901837594,"ci_90_upper":0.21804647286194653,"ci_75_lower":0.15717949020398594,"ci_75_upper":0.2065494128377611,"ci_50_lower":0.16645154163453005,"ci_50_upper":0.19571655067446014}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Pro 0423","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":8.7e-7,"completion":0.00000174,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T03:17:59.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-24T13:43:50.662Z"},{"model_slug":"openai/gpt-5.4","reasoning_mode":null,"overall_metrics":{"auc_1m":0.15828524688639095,"auc_128k":0.28580581466738714,"cum_avg_1m":0.2636534744666363,"cum_avg_128k":0.344755461084137,"avg_score_overall":0.2669465710570155,"avg_response_time_ms":32580.750290218268,"total_prompt_cost":24.674882500000006,"total_cache_read_cost":73.04079999999973,"total_completion_cost":4.833944999999999,"total_reasoning_cost":0,"total_prompt_cost_128k":3.0352625000000004,"total_completion_cost_128k":2.76042,"total_on_demand_input_cost":755.4055250000009,"total_on_demand_input_cost_128k":60.1347825,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9799666612778017},"bin_metrics":{"8192":{"avg_score":0.5782387542014475,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4816351907502593,"median_score":1,"q1_score":0.027285502292215824,"q3_score":1,"avg_token_efficiency":0.975904444959839,"is_incomplete":false,"total_prompt_cost_bin":0.20902000000000012,"total_cache_read_cost_bin":0.10889600000000005,"total_completion_cost_bin":0.40921500000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.7271310000000002,"total_on_demand_input_cost_bin":1.2979800000000004,"ci_99_lower":0.43507911621712264,"ci_99_upper":0.7141770684822986,"ci_98_lower":0.44764039981918174,"ci_98_upper":0.6999316311670118,"ci_95_lower":0.4711725878219295,"ci_95_upper":0.679400894512655,"ci_90_lower":0.4910637396093807,"ci_90_upper":0.6637698745081434,"ci_75_lower":0.5176158634341845,"ci_75_upper":0.6403812776501582,"ci_50_lower":0.5418634359564748,"ci_50_upper":0.6139932913807571},"16384":{"avg_score":0.357789891975699,"min_score":0.0044469148851931095,"max_score":1,"n_tests":80,"stddev":0.46143050255286605,"median_score":0.04245644435286522,"q1_score":0.023982561193406582,"q3_score":1,"avg_token_efficiency":0.9670687393188713,"is_incomplete":false,"total_prompt_cost_bin":0.2632600000000002,"total_cache_read_cost_bin":0.24009599999999995,"total_completion_cost_bin":0.40219499999999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.9055510000000001,"total_on_demand_input_cost_bin":2.6642199999999976,"ci_99_lower":0.22431477533455474,"ci_99_upper":0.4912520343188953,"ci_98_lower":0.23506535765761508,"ci_98_upper":0.4786619297604775,"ci_95_lower":0.25951590660610235,"ci_95_upper":0.45797747746531947,"ci_90_lower":0.2710279670756427,"ci_90_upper":0.44209376261133004,"ci_75_lower":0.2980097327163093,"ci_75_upper":0.4178623765939847,"ci_50_lower":0.32171100289269816,"ci_50_upper":0.3935105555923656},"32768":{"avg_score":0.22815873403348685,"min_score":0.005335111636668444,"max_score":1,"n_tests":136,"stddev":0.38639379207985664,"median_score":0.03455723822116852,"q1_score":0.024061167147010565,"q3_score":0.07961432822048664,"avg_token_efficiency":0.995210076854963,"is_incomplete":false,"total_prompt_cost_bin":0.6987599999999999,"total_cache_read_cost_bin":0.8951039999999996,"total_completion_cost_bin":0.81168,"total_reasoning_cost_bin":0,"total_cost_bin":2.4055439999999995,"total_on_demand_input_cost_bin":9.649800000000004,"ci_99_lower":0.1486996072631754,"ci_99_upper":0.31033053537104827,"ci_98_lower":0.15535685437091162,"ci_98_upper":0.3051029207952225,"ci_95_lower":0.16525037840508638,"ci_95_upper":0.2901607397089452,"ci_90_lower":0.17447877692466138,"ci_90_upper":0.28320347930986073,"ci_75_lower":0.19076463499796398,"ci_75_upper":0.2648355560955037,"ci_50_lower":0.20478537787001935,"ci_50_upper":0.24852279283154263},"65536":{"avg_score":0.2692235510029337,"min_score":0.004799040034413338,"max_score":1,"n_tests":85,"stddev":0.42136639252454494,"median_score":0.030576243996620178,"q1_score":0.021377671509981155,"q3_score":0.09436997026205063,"avg_token_efficiency":0.9776435846759913,"is_incomplete":false,"total_prompt_cost_bin":0.5472525000000001,"total_cache_read_cost_bin":1.3041919999999985,"total_completion_cost_bin":0.48866999999999994,"total_reasoning_cost_bin":0,"total_cost_bin":2.3401144999999985,"total_on_demand_input_cost_bin":13.589172500000004,"ci_99_lower":0.16567770560419953,"ci_99_upper":0.38671185895877286,"ci_98_lower":0.1709889587869539,"ci_98_upper":0.38141394655209254,"ci_95_lower":0.1887474870298277,"ci_95_upper":0.36077911050387607,"ci_90_lower":0.19984526885563836,"ci_90_upper":0.34056539988199985,"ci_75_lower":0.2127558565013768,"ci_75_upper":0.31576802706866364,"ci_50_lower":0.23519426171722657,"ci_50_upper":0.29495066056192365},"131072":{"avg_score":0.290366374207118,"min_score":0.003610108280554414,"max_score":1,"n_tests":103,"stddev":0.43531293386622133,"median_score":0.028699936345219612,"q1_score":0.018370932899415493,"q3_score":0.9942339360713959,"avg_token_efficiency":0.9772722316188369,"is_incomplete":false,"total_prompt_cost_bin":1.31697,"total_cache_read_cost_bin":3.1616639999999974,"total_completion_cost_bin":0.6486599999999998,"total_reasoning_cost_bin":0,"total_cost_bin":5.127293999999997,"total_on_demand_input_cost_bin":32.933609999999994,"ci_99_lower":0.18569135399329595,"ci_99_upper":0.3947421203750226,"ci_98_lower":0.18966799352860567,"ci_98_upper":0.38594124057725776,"ci_95_lower":0.21201627161918948,"ci_95_upper":0.3745072692507218,"ci_90_lower":0.22335342455415724,"ci_90_upper":0.3598319243842022,"ci_75_lower":0.24228610747546925,"ci_75_upper":0.3369744330072703,"ci_50_lower":0.2615812462517196,"ci_50_upper":0.31835520638885356},"262144":{"avg_score":0.20872564142048095,"min_score":0.002959674457088113,"max_score":1,"n_tests":141,"stddev":0.37429050203876285,"median_score":0.032882291823625565,"q1_score":0.022090058773756027,"q3_score":0.05013129487633705,"avg_token_efficiency":0.982419810338046,"is_incomplete":false,"total_prompt_cost_bin":3.384867500000001,"total_cache_read_cost_bin":8.731936000000013,"total_completion_cost_bin":0.796545,"total_reasoning_cost_bin":0,"total_cost_bin":12.913348500000014,"total_on_demand_input_cost_bin":90.70422749999993,"ci_99_lower":0.13231369880068528,"ci_99_upper":0.2860831034237582,"ci_98_lower":0.13986735155792737,"ci_98_upper":0.2817542303040961,"ci_95_lower":0.14820804022411907,"ci_95_upper":0.26967631029666905,"ci_90_lower":0.15919032255179374,"ci_90_upper":0.2602919058328501,"ci_75_lower":0.1733760120122568,"ci_75_upper":0.24472946544191865,"ci_50_lower":0.18659041287560768,"ci_50_upper":0.22849066503732368},"524288":{"avg_score":0.1767248488919241,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.34176975912346835,"median_score":0.029741683043539524,"q1_score":0.020493687130510807,"q3_score":0.052998924627900124,"avg_token_efficiency":0.9774785269226763,"is_incomplete":false,"total_prompt_cost_bin":18.254752500000006,"total_cache_read_cost_bin":58.59891199999972,"total_completion_cost_bin":1.276979999999999,"total_reasoning_cost_bin":0,"total_cost_bin":78.13064449999972,"total_on_demand_input_cost_bin":604.566515000001,"ci_99_lower":0.12355155451179846,"ci_99_upper":0.23120295490580217,"ci_98_lower":0.12749364480198663,"ci_98_upper":0.22710411691613955,"ci_95_lower":0.1341319138827927,"ci_95_upper":0.22256064220179678,"ci_90_lower":0.14068068059179065,"ci_90_upper":0.2132559449205735,"ci_75_lower":0.15289444936280555,"ci_75_upper":0.2020921868625318,"ci_50_lower":0.1618652688028583,"ci_50_upper":0.19039332111991578}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":0.0000025,"completion":0.000015,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-05T18:12:32.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T01:38:08.812Z"},{"model_slug":"xiaomi/mimo-v2.5","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.15771075500583687,"auc_128k":0.3190294951374932,"cum_avg_1m":0.3234462088128356,"cum_avg_128k":0.4350356177810955,"avg_score_overall":0.3154863292112315,"avg_response_time_ms":12635.940140745228,"total_prompt_cost":1.1174582999999998,"total_cache_read_cost":0.4938388224000003,"total_completion_cost":0.25088420000000006,"total_reasoning_cost":0.15915984000000002,"total_prompt_cost_128k":0.4263340200000002,"total_completion_cost_128k":0.16538312000000002,"total_on_demand_input_cost":25.809399419999995,"total_on_demand_input_cost_128k":3.431544900000002,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0023249715678455},"bin_metrics":{"8192":{"avg_score":0.7939900837722235,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.3950396643453153,"median_score":1,"q1_score":0.9897989481687546,"q3_score":1,"avg_token_efficiency":1.0282904085866937,"is_incomplete":false,"total_prompt_cost_bin":0.0228424,"total_cache_read_cost_bin":0.0010725119999999994,"total_completion_cost_bin":0.02438828,"total_reasoning_cost_bin":0.016548560000000007,"total_cost_bin":0.064851752,"total_on_demand_input_cost_bin":0.07646800000000005,"ci_99_lower":0.6731882847609232,"ci_99_upper":0.8939001122718327,"ci_98_lower":0.6839035814229865,"ci_98_upper":0.890709110795171,"ci_95_lower":0.6974354764181772,"ci_95_upper":0.8791042645013658,"ci_90_lower":0.7209250578976935,"ci_90_upper":0.8647096887981752,"ci_75_lower":0.7452906048216391,"ci_75_upper":0.8416173352306942,"ci_50_lower":0.7651429090445162,"ci_50_upper":0.822632150122081},"16384":{"avg_score":0.5140393505716929,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.4874603710124272,"median_score":0.5305703692138195,"q1_score":0.028501428198069334,"q3_score":1,"avg_token_efficiency":0.9993044103844311,"is_incomplete":false,"total_prompt_cost_bin":0.02527840000000001,"total_cache_read_cost_bin":0.0025754624000000003,"total_completion_cost_bin":0.031596880000000015,"total_reasoning_cost_bin":0.023855160000000004,"total_cost_bin":0.08330590240000002,"total_on_demand_input_cost_bin":0.15405151999999986,"ci_99_lower":0.3702681997094915,"ci_99_upper":0.648272409342113,"ci_98_lower":0.3805949891891796,"ci_98_upper":0.6347728412986616,"ci_95_lower":0.40112751470347574,"ci_95_upper":0.6220758859522901,"ci_90_lower":0.41775270319441915,"ci_90_upper":0.5999938392962212,"ci_75_lower":0.45335763830262293,"ci_75_upper":0.575885242782897,"ci_50_lower":0.4778749215300195,"ci_50_upper":0.5507145147137635},"32768":{"avg_score":0.3469631632153109,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.4497023975212501,"median_score":0.04179011657834053,"q1_score":0.025128189474344254,"q3_score":0.9997002184391022,"avg_token_efficiency":1.0185033069442804,"is_incomplete":false,"total_prompt_cost_bin":0.03193231999999998,"total_cache_read_cost_bin":0.01041815039999999,"total_completion_cost_bin":0.0489216,"total_reasoning_cost_bin":0.03360896000000001,"total_cost_bin":0.12488103039999998,"total_on_demand_input_cost_bin":0.5528398400000001,"ci_99_lower":0.25141927425015154,"ci_99_upper":0.4483274384370923,"ci_98_lower":0.25563104394265207,"ci_98_upper":0.4380283175807049,"ci_95_lower":0.2712805052125238,"ci_95_upper":0.4225468446425272,"ci_90_lower":0.2818663205049234,"ci_90_upper":0.41196539084348305,"ci_75_lower":0.3022360862348304,"ci_75_upper":0.3936356691800243,"ci_50_lower":0.31944226991874847,"ci_50_upper":0.3726854068355512},"65536":{"avg_score":0.2478784521439058,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.4060352096230572,"median_score":0.029862793162465096,"q1_score":0.02068965509533882,"q3_score":0.06970509141683578,"avg_token_efficiency":0.9957656248159551,"is_incomplete":false,"total_prompt_cost_bin":0.10073434000000006,"total_cache_read_cost_bin":0.013487308800000012,"total_completion_cost_bin":0.030152359999999996,"total_reasoning_cost_bin":0.020750520000000005,"total_cost_bin":0.16512452880000006,"total_on_demand_input_cost_bin":0.7750997800000001,"ci_99_lower":0.14599480095659106,"ci_99_upper":0.3597069722698892,"ci_98_lower":0.15594301966090193,"ci_98_upper":0.35227247486964747,"ci_95_lower":0.1696894809755716,"ci_95_upper":0.3355010756961179,"ci_90_lower":0.1797836809931323,"ci_90_upper":0.3238929365506834,"ci_75_lower":0.19522156724363895,"ci_75_upper":0.2937709624228506,"ci_50_lower":0.21374631761299337,"ci_50_upper":0.27330442812732036},"131072":{"avg_score":0.27230703920234467,"min_score":0.0047505940310657024,"max_score":1,"n_tests":103,"stddev":0.42291345239841,"median_score":0.03163580223917961,"q1_score":0.02335475478321314,"q3_score":0.507409606128931,"avg_token_efficiency":0.9925350580133276,"is_incomplete":false,"total_prompt_cost_bin":0.24554656000000014,"total_cache_read_cost_bin":0.03255078400000001,"total_completion_cost_bin":0.030324000000000007,"total_reasoning_cost_bin":0.018078200000000003,"total_cost_bin":0.32649954400000014,"total_on_demand_input_cost_bin":1.8730857600000017,"ci_99_lower":0.1689303082293035,"ci_99_upper":0.3761713581554393,"ci_98_lower":0.18530333673500102,"ci_98_upper":0.3670376245111896,"ci_95_lower":0.19700612840796505,"ci_95_upper":0.34812505838486085,"ci_90_lower":0.20619828777604746,"ci_90_upper":0.3389349270401776,"ci_75_lower":0.22538635342662197,"ci_75_upper":0.3184232970041865,"ci_50_lower":0.24374483787744006,"ci_50_upper":0.30006288452914837},"262144":{"avg_score":0.2803928654835764,"min_score":0.0024676125030964613,"max_score":1,"n_tests":141,"stddev":0.42234385836988586,"median_score":0.0393594428896904,"q1_score":0.022179903462529182,"q3_score":0.9580070972442627,"avg_token_efficiency":0.9987322284547321,"is_incomplete":false,"total_prompt_cost_bin":0.3978816800000001,"total_cache_read_cost_bin":0.09531791359999994,"total_completion_cost_bin":0.031110240000000015,"total_reasoning_cost_bin":0.015875159999999992,"total_cost_bin":0.5401849936000002,"total_on_demand_input_cost_bin":5.163777360000002,"ci_99_lower":0.19089387437104122,"ci_99_upper":0.37267294203372936,"ci_98_lower":0.1963291600779366,"ci_98_upper":0.3577398153800345,"ci_95_lower":0.21049140735714306,"ci_95_upper":0.34951837762859184,"ci_90_lower":0.2220072708986799,"ci_90_upper":0.3365609314882177,"ci_75_lower":0.23977709330654712,"ci_75_upper":0.3192066365177165,"ci_50_lower":0.2551267662685008,"ci_50_upper":0.30353111877760036},"524288":{"avg_score":0.13199871611363112,"min_score":0.002326934365555644,"max_score":1,"n_tests":236,"stddev":0.2918782693530533,"median_score":0.028555099852383137,"q1_score":0.018144196830689907,"q3_score":0.04807598702609539,"avg_token_efficiency":0.9940056439023583,"is_incomplete":false,"total_prompt_cost_bin":0.2932425999999995,"total_cache_read_cost_bin":0.33841669120000034,"total_completion_cost_bin":0.05439084,"total_reasoning_cost_bin":0.030443279999999996,"total_cost_bin":0.7164934111999999,"total_on_demand_input_cost_bin":17.21407715999999,"ci_99_lower":0.08800103928995914,"ci_99_upper":0.18035920689651003,"ci_98_lower":0.09114459105278956,"ci_98_upper":0.17597925677821552,"ci_95_lower":0.0974296731317921,"ci_95_upper":0.17135692113199985,"ci_90_lower":0.102471312036209,"ci_90_upper":0.1653731628329497,"ci_75_lower":0.11047342486871442,"ci_75_upper":0.1539823138173541,"ci_50_lower":0.11853466934713898,"ci_50_upper":0.14334603787845757}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2.5","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":1.4e-7,"completion":2.8e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-22T16:11:09.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-22T20:22:27.210Z"},{"model_slug":"x-ai/grok-4.20","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.15706353776626814,"auc_128k":0.422975753020026,"cum_avg_1m":0.3888970784920172,"cum_avg_128k":0.5506972142156454,"avg_score_overall":0.28751693650681276,"avg_response_time_ms":28721.183939633258,"total_prompt_cost":35.59819124999998,"total_cache_read_cost":183.35836380000055,"total_completion_cost":9.645575000000001,"total_reasoning_cost":8.603005000000003,"total_prompt_cost_128k":7.841710000000006,"total_completion_cost_128k":3.3281425000000002,"total_on_demand_input_cost":1181.9817362499998,"total_on_demand_input_cost_128k":29.19348375,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":0.952101774809972},"bin_metrics":{"8192":{"avg_score":0.9518300764029846,"min_score":0.028362305834889412,"max_score":1,"n_tests":80,"stddev":0.21099987478707985,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9652478859483746,"is_incomplete":false,"total_prompt_cost_bin":0.45398000000000005,"total_cache_read_cost_bin":0.030041600000000012,"total_completion_cost_bin":0.43206750000000005,"total_reasoning_cost_bin":0.3655,"total_cost_bin":1.2815891000000001,"total_on_demand_input_cost_bin":0.6417399999999999,"ci_99_lower":0.8796927671895828,"ci_99_upper":0.99994982752949,"ci_98_lower":0.8911881771446206,"ci_98_upper":0.9999289077296853,"ci_95_lower":0.8918646065430949,"ci_95_upper":0.9880072700511664,"ci_90_lower":0.9038273650826886,"ci_90_upper":0.9879479896370321,"ci_75_lower":0.9276005764724686,"ci_75_upper":0.9759567465633154,"ci_50_lower":0.9396928618196398,"ci_50_upper":0.9639984580222518},"16384":{"avg_score":0.5993212802568451,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.48110685800437747,"median_score":1,"q1_score":0.030626488383859396,"q3_score":1,"avg_token_efficiency":0.9481162912488784,"is_incomplete":false,"total_prompt_cost_bin":0.676465,"total_cache_read_cost_bin":0.10052719999999998,"total_completion_cost_bin":0.5982875,"total_reasoning_cost_bin":0.53382,"total_cost_bin":1.9090997,"total_on_demand_input_cost_bin":1.3047599999999995,"ci_99_lower":0.45393169045780085,"ci_99_upper":0.7336751072278129,"ci_98_lower":0.4654219654147746,"ci_98_upper":0.7211215032287873,"ci_95_lower":0.49128487692185446,"ci_95_upper":0.6976056136199623,"ci_90_lower":0.5129559766582679,"ci_90_upper":0.68467293734313,"ci_75_lower":0.5387462860657252,"ci_75_upper":0.6604468272722442,"ci_50_lower":0.5631006818352035,"ci_50_upper":0.6350379870913456},"32768":{"avg_score":0.6186537031471209,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.47023270388808536,"median_score":1,"q1_score":0.03912060521543026,"q3_score":1,"avg_token_efficiency":0.9647547101462162,"is_incomplete":false,"total_prompt_cost_bin":1.1758312500000014,"total_cache_read_cost_bin":0.5597758000000004,"total_completion_cost_bin":0.88087,"total_reasoning_cost_bin":0.7531825000000003,"total_cost_bin":3.3696595500000024,"total_on_demand_input_cost_bin":4.674429999999994,"ci_99_lower":0.5172313041572445,"ci_99_upper":0.7234363593180194,"ci_98_lower":0.525163301479068,"ci_98_upper":0.7115580474154558,"ci_95_lower":0.5398446074131403,"ci_95_upper":0.6974173751749143,"ci_90_lower":0.5541202422370106,"ci_90_upper":0.6878650270871516,"ci_75_lower":0.5710446772988208,"ci_75_upper":0.6666931906780946,"ci_50_lower":0.592152004082249,"ci_50_upper":0.6451834120076347},"65536":{"avg_score":0.3895270910935805,"min_score":0.003518648911267519,"max_score":1,"n_tests":85,"stddev":0.47053624699151825,"median_score":0.03554007038474083,"q1_score":0.02414928562939167,"q3_score":1,"avg_token_efficiency":0.9498488302416127,"is_incomplete":false,"total_prompt_cost_bin":1.9426937500000003,"total_cache_read_cost_bin":0.7453951999999994,"total_completion_cost_bin":0.60775,"total_reasoning_cost_bin":0.5251800000000001,"total_cost_bin":3.8210189499999996,"total_on_demand_input_cost_bin":6.601413749999998,"ci_99_lower":0.25887072048481446,"ci_99_upper":0.5305867729853729,"ci_98_lower":0.2738010162040043,"ci_98_upper":0.5173291781235705,"ci_95_lower":0.28920209367033645,"ci_95_upper":0.4892822955730919,"ci_90_lower":0.30509410511099677,"ci_90_upper":0.47593659843948183,"ci_75_lower":0.3300771573428338,"ci_75_upper":0.4491507945322048,"ci_50_lower":0.3547775008428075,"ci_50_upper":0.4250765083987704},"131072":{"avg_score":0.1941539201776958,"min_score":0.004462862387299538,"max_score":1,"n_tests":103,"stddev":0.35979968869690293,"median_score":0.030916843563318253,"q1_score":0.019950130954384804,"q3_score":0.04907715134322643,"avg_token_efficiency":0.9478554965136295,"is_incomplete":false,"total_prompt_cost_bin":3.592740000000005,"total_cache_read_cost_bin":1.9805439999999985,"total_completion_cost_bin":0.8091674999999998,"total_reasoning_cost_bin":0.7061500000000006,"total_cost_bin":7.088601500000005,"total_on_demand_input_cost_bin":15.971140000000007,"ci_99_lower":0.10933996110788356,"ci_99_upper":0.2850444771488224,"ci_98_lower":0.11678967385525699,"ci_98_upper":0.27955022594990947,"ci_95_lower":0.13088076788981817,"ci_95_upper":0.2623356470022152,"ci_90_lower":0.1403964088727874,"ci_90_upper":0.25141002108903543,"ci_75_lower":0.15466892469484467,"ci_75_upper":0.23177818433512726,"ci_50_lower":0.16959569649092707,"ci_50_upper":0.21448818761041585},"262144":{"avg_score":0.15897598781735234,"min_score":0.0043907794170081615,"max_score":1,"n_tests":141,"stddev":0.3169860684318592,"median_score":0.030598584562540054,"q1_score":0.020868584513664246,"q3_score":0.05067567527294159,"avg_token_efficiency":0.9522898196178772,"is_incomplete":false,"total_prompt_cost_bin":3.9033412500000084,"total_cache_read_cost_bin":13.416908799999966,"total_completion_cost_bin":1.0559949999999998,"total_reasoning_cost_bin":0.9362075000000001,"total_cost_bin":19.312452549999975,"total_on_demand_input_cost_bin":87.92240499999997,"ci_99_lower":0.09687359217450299,"ci_99_upper":0.22791131090483438,"ci_98_lower":0.10095128498020324,"ci_98_upper":0.22075911223333564,"ci_95_lower":0.10858035577025187,"ci_95_upper":0.21206133485233397,"ci_90_lower":0.11681715348596744,"ci_90_upper":0.2041225130725916,"ci_75_lower":0.12725586052290463,"ci_75_upper":0.1908375826248972,"ci_50_lower":0.13991879867736567,"ci_50_upper":0.17357089126264638},"524288":{"avg_score":0.11718198266748528,"min_score":0.002188183832913637,"max_score":1,"n_tests":236,"stddev":0.2664362747008346,"median_score":0.026690801605582237,"q1_score":0.01795600214973092,"q3_score":0.04477928392589092,"avg_token_efficiency":0.9465986079094495,"is_incomplete":false,"total_prompt_cost_bin":8.783781250000004,"total_cache_read_cost_bin":45.41552640000004,"total_completion_cost_bin":2.8313199999999994,"total_reasoning_cost_bin":2.6257050000000004,"total_cost_bin":59.656332650000046,"total_on_demand_input_cost_bin":292.733705,"ci_99_lower":0.07730368376668086,"ci_99_upper":0.1631907259959775,"ci_98_lower":0.08039246609529196,"ci_98_upper":0.15623965288096076,"ci_95_lower":0.08617618826338312,"ci_95_upper":0.15228735529161572,"ci_90_lower":0.08975779815040275,"ci_90_upper":0.14808984444510206,"ci_75_lower":0.09683156152003271,"ci_75_upper":0.13745563474564687,"ci_50_lower":0.10504813821862011,"ci_50_upper":0.1273182147607736},"1048576":{"avg_score":0.08153258637307331,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.20889993862086467,"median_score":0.025606284849345684,"q1_score":0.01702858041971922,"q3_score":0.038562336936593056,"avg_token_efficiency":0.9503193574278062,"is_incomplete":false,"total_prompt_cost_bin":15.069358749999958,"total_cache_read_cost_bin":121.10964480000054,"total_completion_cost_bin":2.4301175000000015,"total_reasoning_cost_bin":2.1572600000000017,"total_cost_bin":140.7663810500005,"total_on_demand_input_cost_bin":772.1321425,"ci_99_lower":0.053564453327350485,"ci_99_upper":0.11478947438779773,"ci_98_lower":0.056682806907668554,"ci_98_upper":0.10984309354199155,"ci_95_lower":0.058734055840215016,"ci_95_upper":0.10618962723607829,"ci_90_lower":0.062635178251207,"ci_90_upper":0.10236030646261848,"ci_75_lower":0.0678219129521993,"ci_75_upper":0.09453349933108023,"ci_50_lower":0.0726347399389206,"ci_50_upper":0.08969344635740027}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":2000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"SpaceXAI: Grok 4.20","provider_name":"X-ai","run_provider_names":["xAI"],"pricing_metadata":{"prompt":0.00000125,"completion":0.0000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-31T17:43:39.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T04:30:45.733Z"},{"model_slug":"qwen/qwen3.5-plus","reasoning_mode":null,"overall_metrics":{"auc_1m":0.1553251670361097,"auc_128k":0.30351506408868933,"cum_avg_1m":0.2802415202588012,"cum_avg_128k":0.3661642329125693,"avg_score_overall":0.2793622048969684,"avg_response_time_ms":15324.907890000453,"total_prompt_cost":59.64865945999999,"total_cache_read_cost":0,"total_completion_cost":0.517452,"total_reasoning_cost":0,"total_prompt_cost_128k":6.4697505600000005,"total_completion_cost_128k":0.2956231200000001,"total_on_demand_input_cost":59.64865945999999,"total_on_demand_input_cost_128k":6.4697505600000005,"total_runs":860,"total_candidates_generated":null,"avg_token_efficiency_overall":1.019826168626188},"bin_metrics":{"8192":{"avg_score":0.6108673275390174,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.47134716738262017,"median_score":0.9852707087993622,"q1_score":0.033514413982629776,"q3_score":0.9977800399065018,"avg_token_efficiency":1.0228079512716757,"is_incomplete":false,"total_prompt_cost_bin":0.1413235200000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.044038800000000024,"total_reasoning_cost_bin":0,"total_cost_bin":0.1853623200000001,"total_on_demand_input_cost_bin":0.1413235200000001,"ci_99_lower":0.4651510071305092,"ci_99_upper":0.751753063740849,"ci_98_lower":0.48042082191701047,"ci_98_upper":0.7291028020644444,"ci_95_lower":0.5048133531450003,"ci_95_upper":0.7078209227228945,"ci_90_lower":0.5253820806150907,"ci_90_upper":0.6945684445567895,"ci_75_lower":0.5509342220815596,"ci_75_upper":0.6709951688397269,"ci_50_lower":0.5745292469466221,"ci_50_upper":0.6455888191558188},"16384":{"avg_score":0.40059462790377437,"min_score":0.0050536952912807465,"max_score":1,"n_tests":80,"stddev":0.47287177972054734,"median_score":0.03981166519224644,"q1_score":0.021767777856439352,"q3_score":0.9942021518945694,"avg_token_efficiency":1.016119475838464,"is_incomplete":false,"total_prompt_cost_bin":0.2914246400000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.04238363999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.3338082800000002,"total_on_demand_input_cost_bin":0.2914246400000002,"ci_99_lower":0.26673785639589187,"ci_99_upper":0.5329164806301705,"ci_98_lower":0.27810012427344916,"ci_98_upper":0.5306562275000615,"ci_95_lower":0.2908433721610345,"ci_95_upper":0.5058773895047489,"ci_90_lower":0.3043912368890596,"ci_90_upper":0.4881986361177405,"ci_75_lower":0.3406831156098633,"ci_75_upper":0.46121177055611046,"ci_50_lower":0.3646490792889381,"ci_50_upper":0.4361331229913049},"32768":{"avg_score":0.27434778255273534,"min_score":0.005387931130826473,"max_score":1,"n_tests":136,"stddev":0.4140982116489578,"median_score":0.036918504163622856,"q1_score":0.023486973717808723,"q3_score":0.38843531534075737,"avg_token_efficiency":1.025754859257966,"is_incomplete":false,"total_prompt_cost_bin":1.0339888,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08513232000000004,"total_reasoning_cost_bin":0,"total_cost_bin":1.11912112,"total_on_demand_input_cost_bin":1.0339888,"ci_99_lower":0.1882670290791772,"ci_99_upper":0.3597069573795478,"ci_98_lower":0.1935270067005117,"ci_98_upper":0.3530244185282465,"ci_95_lower":0.2055678030804676,"ci_95_upper":0.3428467440899968,"ci_90_lower":0.21600633853231557,"ci_90_upper":0.33035973855602446,"ci_75_lower":0.2321596341609941,"ci_75_upper":0.31514859672033174,"ci_50_lower":0.25107376892735966,"ci_50_upper":0.29641045007498606},"65536":{"avg_score":0.32165565088884357,"min_score":0.0029607696924358606,"max_score":1,"n_tests":85,"stddev":0.44594275838588066,"median_score":0.033847685903310776,"q1_score":0.021377671509981155,"q3_score":0.9758948087692261,"avg_token_efficiency":1.020990401222107,"is_incomplete":false,"total_prompt_cost_bin":1.4759357600000007,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.05464524,"total_reasoning_cost_bin":0,"total_cost_bin":1.5305810000000006,"total_on_demand_input_cost_bin":1.4759357600000007,"ci_99_lower":0.20772747562333996,"ci_99_upper":0.4567928965357269,"ci_98_lower":0.21824103143090823,"ci_98_upper":0.44550946267069697,"ci_95_lower":0.23018892271670185,"ci_95_upper":0.4152655106704161,"ci_90_lower":0.24261689796098782,"ci_90_upper":0.40108472533702083,"ci_75_lower":0.26454494043308147,"ci_75_upper":0.3773990096366855,"ci_50_lower":0.28727345979115104,"ci_50_upper":0.3546036652741296},"131072":{"avg_score":0.22335577567847556,"min_score":0.003610108280554414,"max_score":1,"n_tests":102,"stddev":0.3891265072149195,"median_score":0.02898664027452469,"q1_score":0.019182818476110697,"q3_score":0.05170213617384434,"avg_token_efficiency":1.0162368709815355,"is_incomplete":false,"total_prompt_cost_bin":3.5270778399999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06942311999999999,"total_reasoning_cost_bin":0,"total_cost_bin":3.5965009599999997,"total_on_demand_input_cost_bin":3.5270778399999996,"ci_99_lower":0.12939103557911985,"ci_99_upper":0.3353796870737136,"ci_98_lower":0.13819592906356168,"ci_98_upper":0.31851370229723625,"ci_95_lower":0.15222825411714008,"ci_95_upper":0.303350295210668,"ci_90_lower":0.1651091367111304,"ci_90_upper":0.2889372953106466,"ci_75_lower":0.17742158463521515,"ci_75_upper":0.26301559040895844,"ci_50_lower":0.19568188285327281,"ci_50_upper":0.2481663939771334},"262144":{"avg_score":0.26444182639730207,"min_score":0.005767250433564186,"max_score":1,"n_tests":141,"stddev":0.4134783956512447,"median_score":0.036653731018304825,"q1_score":0.02277039922773838,"q3_score":0.08280254900455475,"avg_token_efficiency":1.018073678270462,"is_incomplete":false,"total_prompt_cost_bin":12.219487150000008,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08477819999999998,"total_reasoning_cost_bin":0,"total_cost_bin":12.304265350000009,"total_on_demand_input_cost_bin":12.219487150000008,"ci_99_lower":0.17685206671149623,"ci_99_upper":0.35046630562892417,"ci_98_lower":0.18780742747474674,"ci_98_upper":0.34459701009474514,"ci_95_lower":0.19759902522564002,"ci_95_upper":0.3327068592417582,"ci_90_lower":0.20896151911662164,"ci_90_upper":0.3186591763118718,"ci_75_lower":0.22410190938674707,"ci_75_upper":0.30418709417473166,"ci_50_lower":0.24029470104823247,"ci_50_upper":0.2857155965547018},"524288":{"avg_score":0.14666917111026123,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.30759711027495135,"median_score":0.028940952382981777,"q1_score":0.018578750547021627,"q3_score":0.05108683370053768,"avg_token_efficiency":1.0188343915699005,"is_incomplete":false,"total_prompt_cost_bin":40.95942174999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.13705067999999998,"total_reasoning_cost_bin":0,"total_cost_bin":41.096472429999984,"total_on_demand_input_cost_bin":40.95942174999998,"ci_99_lower":0.09896822179541864,"ci_99_upper":0.19723020414625475,"ci_98_lower":0.10327039367462099,"ci_98_upper":0.19308862916657588,"ci_95_lower":0.10859153002820676,"ci_95_upper":0.1860232932158643,"ci_90_lower":0.114759707027184,"ci_90_upper":0.1801415127369583,"ci_75_lower":0.12451850765809454,"ci_75_upper":0.16860233064110972,"ci_50_lower":0.13335296839216165,"ci_50_upper":0.15973778178060025}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5 Plus 2026-02-15","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":2.6e-7,"completion":0.00000156,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-16T08:10:16.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T05:58:51.721Z"},{"model_slug":"xiaomi/mimo-v2.5-pro","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.15345761724915058,"auc_128k":0.44515926055518257,"cum_avg_1m":0.3772672355311876,"cum_avg_128k":0.5438504749724101,"avg_score_overall":0.3621598789477196,"avg_response_time_ms":17129.486240790142,"total_prompt_cost":0.017849355000000018,"total_cache_read_cost":0.6635349503999997,"total_completion_cost":0.6864334799999999,"total_reasoning_cost":0.34736316,"total_prompt_cost_128k":0.011029425,"total_completion_cost_128k":0.38130186,"total_on_demand_input_cost":80.194989195,"total_on_demand_input_cost_128k":10.663142384999997,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.002447373584267},"bin_metrics":{"8192":{"avg_score":0.828909166820813,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.37237836803669666,"median_score":1,"q1_score":0.9972997605800629,"q3_score":1,"avg_token_efficiency":1.0289117704855877,"is_incomplete":false,"total_prompt_cost_bin":0.0016808400000000003,"total_cache_read_cost_bin":0.0019535615999999997,"total_completion_cost_bin":0.05616024000000001,"total_reasoning_cost_bin":0.03147486,"total_cost_bin":0.0912695016,"total_on_demand_input_cost_bin":0.2377361999999998,"ci_99_lower":0.7073226230980946,"ci_99_upper":0.9268120736341807,"ci_98_lower":0.7190796870979248,"ci_98_upper":0.9149822614102158,"ci_95_lower":0.7434930240080576,"ci_95_upper":0.9024943714332767,"ci_90_lower":0.7562358330766438,"ci_90_upper":0.8900057230424137,"ci_75_lower":0.7800062760637957,"ci_75_upper":0.8772751994052669,"ci_50_lower":0.8041921803553123,"ci_50_upper":0.8535047720826697},"16384":{"avg_score":0.6242030186811462,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.4747632324209848,"median_score":1,"q1_score":0.03429048415273428,"q3_score":1,"avg_token_efficiency":0.999602464394221,"is_incomplete":false,"total_prompt_cost_bin":0.0031807200000000006,"total_cache_read_cost_bin":0.003936153599999997,"total_completion_cost_bin":0.06396849000000002,"total_reasoning_cost_bin":0.040594200000000025,"total_cost_bin":0.11167956360000005,"total_on_demand_input_cost_bin":0.4787992800000001,"ci_99_lower":0.4786420398442424,"ci_99_upper":0.7587827774947508,"ci_98_lower":0.5004852921962738,"ci_98_upper":0.7349739395586075,"ci_95_lower":0.5150888354628114,"ci_95_upper":0.7215553053002804,"ci_90_lower":0.5389758369716583,"ci_90_upper":0.7083952166535892,"ci_75_lower":0.5635703011852456,"ci_75_upper":0.6851351547156809,"ci_50_lower":0.5880446027906145,"ci_50_upper":0.65992985223711},"32768":{"avg_score":0.545455249168513,"min_score":0.002536139916628599,"max_score":1,"n_tests":136,"stddev":0.48277690562306763,"median_score":0.9942873418331146,"q1_score":0.03098193509504199,"q3_score":1,"avg_token_efficiency":1.0186469433242435,"is_incomplete":false,"total_prompt_cost_bin":0.002376839999999999,"total_cache_read_cost_bin":0.014198169600000027,"total_completion_cost_bin":0.11913345000000004,"total_reasoning_cost_bin":0.07211951999999999,"total_cost_bin":0.20782797960000007,"total_on_demand_input_cost_bin":1.717988999999997,"ci_99_lower":0.4390059593123014,"ci_99_upper":0.6499306959004367,"ci_98_lower":0.44563249002126903,"ci_98_upper":0.639111797358149,"ci_95_lower":0.4672480751606653,"ci_95_upper":0.6239636797709596,"ci_90_lower":0.4806387108408243,"ci_90_upper":0.6117274737628415,"ci_75_lower":0.5010534648850701,"ci_75_upper":0.5947913842666073,"ci_50_lower":0.5180155350441483,"ci_50_upper":0.5737576768155764},"65536":{"avg_score":0.40376214431187907,"min_score":0.0014635931001976132,"max_score":1,"n_tests":85,"stddev":0.4746542531663648,"median_score":0.04197901114821434,"q1_score":0.02151898667216301,"q3_score":1,"avg_token_efficiency":0.9958267762250803,"is_incomplete":false,"total_prompt_cost_bin":0.001864845000000003,"total_cache_read_cost_bin":0.019916927999999987,"total_completion_cost_bin":0.07047869999999998,"total_reasoning_cost_bin":0.04101701999999998,"total_cost_bin":0.13327749299999994,"total_on_demand_input_cost_bin":2.4084936450000023,"ci_99_lower":0.276001934977577,"ci_99_upper":0.5408842695128131,"ci_98_lower":0.28841342414219806,"ci_98_upper":0.5285959735550672,"ci_95_lower":0.30082006714765586,"ci_95_upper":0.5044257471877535,"ci_90_lower":0.3139904413398985,"ci_90_upper":0.4861903816536891,"ci_75_lower":0.34531992137130785,"ci_75_upper":0.4614785031797296,"ci_50_lower":0.3694860677423147,"ci_50_upper":0.43798688554533705},"131072":{"avg_score":0.3169227958796998,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.44814007190786453,"median_score":0.030736619606614113,"q1_score":0.018167730420827866,"q3_score":0.9976342916488647,"avg_token_efficiency":0.9925656222423529,"is_incomplete":false,"total_prompt_cost_bin":0.001926179999999998,"total_cache_read_cost_bin":0.04815060479999999,"total_completion_cost_bin":0.07156097999999998,"total_reasoning_cost_bin":0.03441546,"total_cost_bin":0.15605322479999997,"total_on_demand_input_cost_bin":5.820124259999998,"ci_99_lower":0.21072953104265305,"ci_99_upper":0.43100390196207106,"ci_98_lower":0.2225525513915707,"ci_98_upper":0.4201320856166732,"ci_95_lower":0.23156044298869977,"ci_95_upper":0.40186151182742386,"ci_90_lower":0.24293471320511917,"ci_90_upper":0.3921788215954806,"ci_75_lower":0.26828102940225573,"ci_75_upper":0.36531114292181843,"ci_50_lower":0.2886981624760665,"ci_50_upper":0.3449516812745183},"262144":{"avg_score":0.16958993063836708,"min_score":0.0021436226088553667,"max_score":1,"n_tests":141,"stddev":0.3390138561115628,"median_score":0.03209827467799187,"q1_score":0.021808268502354622,"q3_score":0.05319149047136307,"avg_token_efficiency":0.9987475002008313,"is_incomplete":false,"total_prompt_cost_bin":0.0015660000000000027,"total_cache_read_cost_bin":0.1327719168000002,"total_completion_cost_bin":0.08800833000000002,"total_reasoning_cost_bin":0.04107791999999999,"total_cost_bin":0.26342416680000025,"total_on_demand_input_cost_bin":16.044839279999977,"ci_99_lower":0.09943799020000073,"ci_99_upper":0.23949400324030146,"ci_98_lower":0.10616160729460762,"ci_98_upper":0.23667303099584308,"ci_95_lower":0.11513093378289986,"ci_95_upper":0.22707025108137333,"ci_90_lower":0.12620749916387589,"ci_90_upper":0.2180216226052222,"ci_75_lower":0.13647386417616858,"ci_75_upper":0.20384115472531725,"ci_50_lower":0.14885685823945302,"ci_50_upper":0.18514288180996497},"524288":{"avg_score":0.12929557874908285,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.2869953921475238,"median_score":0.028187308460474014,"q1_score":0.019201085902750492,"q3_score":0.046999914571642876,"avg_token_efficiency":0.9940132753184773,"is_incomplete":false,"total_prompt_cost_bin":0.005253930000000015,"total_cache_read_cost_bin":0.4426076159999995,"total_completion_cost_bin":0.21712328999999994,"total_reasoning_cost_bin":0.08666417999999998,"total_cost_bin":0.7516490159999994,"total_on_demand_input_cost_bin":53.48700753000003,"ci_99_lower":0.08687684043178999,"ci_99_upper":0.17584973569841134,"ci_98_lower":0.0891047852288836,"ci_98_upper":0.17261147048906916,"ci_95_lower":0.09548658521731124,"ci_95_upper":0.16812405195674823,"ci_90_lower":0.10032954982667372,"ci_90_upper":0.16115701611314592,"ci_75_lower":0.10839608703592941,"ci_75_upper":0.15090522237273865,"ci_50_lower":0.11607531320753307,"ci_50_upper":0.14066148980326312}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2.5-Pro","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":4.35e-7,"completion":8.7e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-22T16:11:13.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-22T20:19:33.152Z"},{"model_slug":"google/gemini-3.1-flash-lite-preview","reasoning_mode":null,"overall_metrics":{"auc_1m":0.15206758157066394,"auc_128k":0.2208932422190824,"cum_avg_1m":0.22211834591594984,"cum_avg_128k":0.2688251841969108,"avg_score_overall":0.18225329555551195,"avg_response_time_ms":15784.534305562736,"total_prompt_cost":7.343337000000009,"total_cache_read_cost":12.020082274999968,"total_completion_cost":0.6754755,"total_reasoning_cost":0,"total_prompt_cost_128k":1.3602845000000001,"total_completion_cost_128k":0.27896699999999996,"total_on_demand_input_cost":127.54415974999985,"total_on_demand_input_cost_128k":6.136700000000002,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0000227521154894},"bin_metrics":{"8192":{"avg_score":0.4117675843925099,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4717380669910093,"median_score":0.058794207870960236,"q1_score":0.024268783628940582,"q3_score":1,"avg_token_efficiency":1.0001553404747232,"is_incomplete":false,"total_prompt_cost_bin":0.1267695,"total_cache_read_cost_bin":0.0006116499999999999,"total_completion_cost_bin":0.04159649999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.16897764999999998,"total_on_demand_input_cost_bin":0.13288599999999998,"ci_99_lower":0.27547440920662486,"ci_99_upper":0.5434669849096025,"ci_98_lower":0.2886058193633508,"ci_98_upper":0.5403599767769337,"ci_95_lower":0.2991229535658567,"ci_95_upper":0.5168280944769503,"ci_90_lower":0.31773376169898254,"ci_90_upper":0.4999009544568253,"ci_75_lower":0.3519817088521449,"ci_75_upper":0.47252069909682176,"ci_50_lower":0.3756492539458123,"ci_50_upper":0.4475081703349133},"16384":{"avg_score":0.344783001136966,"min_score":0.007738607004284859,"max_score":1,"n_tests":80,"stddev":0.45765491663504126,"median_score":0.03786655329167843,"q1_score":0.02262815786525607,"q3_score":1,"avg_token_efficiency":1.000074513502447,"is_incomplete":false,"total_prompt_cost_bin":0.13855675,"total_cache_read_cost_bin":0.013691125000000009,"total_completion_cost_bin":0.040716,"total_reasoning_cost_bin":0,"total_cost_bin":0.192963875,"total_on_demand_input_cost_bin":0.2754679999999998,"ci_99_lower":0.22090363637887642,"ci_99_upper":0.4805777404140681,"ci_98_lower":0.2233626069239108,"ci_98_upper":0.4670190161748322,"ci_95_lower":0.24770458109502216,"ci_95_upper":0.44265909311681756,"ci_90_lower":0.25852204044756943,"ci_90_upper":0.4288064073369605,"ci_75_lower":0.2846572640381055,"ci_75_upper":0.40485021397471427,"ci_50_lower":0.3093682170874672,"ci_50_upper":0.380245198099874},"32768":{"avg_score":0.18646605528192595,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.349929512766599,"median_score":0.035093534737825394,"q1_score":0.023661854676902294,"q3_score":0.06576568074524403,"avg_token_efficiency":1.0000359090949913,"is_incomplete":false,"total_prompt_cost_bin":0.2799527500000002,"total_cache_read_cost_bin":0.06884012499999996,"total_completion_cost_bin":0.08060549999999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.4293983750000001,"total_on_demand_input_cost_bin":0.9683539999999996,"ci_99_lower":0.11614796133912238,"ci_99_upper":0.2696913491485654,"ci_98_lower":0.11990480112439186,"ci_98_upper":0.2611912355626084,"ci_95_lower":0.12848314166681923,"ci_95_upper":0.248057347404435,"ci_90_lower":0.13799872291787968,"ci_90_upper":0.23560544206736853,"ci_75_lower":0.1508888439142645,"ci_75_upper":0.22031987510982778,"ci_50_lower":0.1651154370603549,"ci_50_upper":0.20395242987775847},"65536":{"avg_score":0.2132525264230721,"min_score":0.002102312631905079,"max_score":1,"n_tests":85,"stddev":0.38141164334121447,"median_score":0.03176620230078697,"q1_score":0.02310509979724884,"q3_score":0.052820052951574326,"avg_token_efficiency":1.000015287852281,"is_incomplete":false,"total_prompt_cost_bin":0.26953550000000004,"total_cache_read_cost_bin":0.11204782500000011,"total_completion_cost_bin":0.04966800000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.43125132500000013,"total_on_demand_input_cost_bin":1.3900137499999998,"ci_99_lower":0.11915518164421049,"ci_99_upper":0.3282641388663803,"ci_98_lower":0.12352162220526268,"ci_98_upper":0.3152146642256309,"ci_95_lower":0.1423984296214493,"ci_95_upper":0.29551648106836864,"ci_90_lower":0.15345486919998247,"ci_90_upper":0.28149090250406195,"ci_75_lower":0.1665911116897512,"ci_75_upper":0.2592032382524956,"ci_50_lower":0.18089114860631525,"ci_50_upper":0.23670297504183563},"131072":{"avg_score":0.18785675375008035,"min_score":0.005788711830973625,"max_score":1,"n_tests":103,"stddev":0.3608489507225131,"median_score":0.03209459409117699,"q1_score":0.01999750081449747,"q3_score":0.045163607224822044,"avg_token_efficiency":1.0000076410572558,"is_incomplete":false,"total_prompt_cost_bin":0.54547,"total_cache_read_cost_bin":0.2824508249999997,"total_completion_cost_bin":0.066381,"total_reasoning_cost_bin":0,"total_cost_bin":0.8943018249999997,"total_on_demand_input_cost_bin":3.3699782500000026,"ci_99_lower":0.1016979885394052,"ci_99_upper":0.28101633465952847,"ci_98_lower":0.1120664829394138,"ci_98_upper":0.27362068779893145,"ci_95_lower":0.12380694631448702,"ci_95_upper":0.26005460810791065,"ci_90_lower":0.13237934621408112,"ci_90_upper":0.24525681060526372,"ci_75_lower":0.14918752758112708,"ci_75_upper":0.22558892480588238,"ci_50_lower":0.16088174443547298,"ci_50_upper":0.20712455996036422},"262144":{"avg_score":0.19964539861729277,"min_score":0.002959674457088113,"max_score":1,"n_tests":141,"stddev":0.35741563606175786,"median_score":0.03273952752351761,"q1_score":0.023174971342086792,"q3_score":0.07530982047319412,"avg_token_efficiency":1.0000038179365243,"is_incomplete":false,"total_prompt_cost_bin":0.9393500000000005,"total_cache_read_cost_bin":0.8293421250000004,"total_completion_cost_bin":0.08121750000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.8499096250000007,"total_on_demand_input_cost_bin":9.232771249999999,"ci_99_lower":0.12793822682833345,"ci_99_upper":0.27334236885778485,"ci_98_lower":0.13118779814858253,"ci_98_upper":0.26844801821991976,"ci_95_lower":0.14187707379856643,"ci_95_upper":0.25965292408252433,"ci_90_lower":0.1511034501718146,"ci_90_upper":0.24993626915218956,"ci_75_lower":0.16551647596915767,"ci_75_upper":0.23349987569612662,"ci_50_lower":0.17877871112308485,"ci_50_upper":0.21858293354859681},"524288":{"avg_score":0.14020164388427275,"min_score":0.0013020833721384406,"max_score":1,"n_tests":236,"stddev":0.29320762603733064,"median_score":0.030255639925599098,"q1_score":0.021622472442686558,"q3_score":0.05926894303411245,"avg_token_efficiency":1.0000019078540328,"is_incomplete":false,"total_prompt_cost_bin":1.722101750000003,"total_cache_read_cost_bin":2.9202755000000122,"total_completion_cost_bin":0.12602850000000004,"total_reasoning_cost_bin":0,"total_cost_bin":4.768405750000015,"total_on_demand_input_cost_bin":30.924856750000032,"ci_99_lower":0.09569487526227514,"ci_99_upper":0.18711674455375218,"ci_98_lower":0.09927343688533302,"ci_98_upper":0.18420958625353476,"ci_95_lower":0.10491137861062563,"ci_95_upper":0.1790375255348439,"ci_90_lower":0.11029809404909907,"ci_90_upper":0.17268289775499812,"ci_75_lower":0.11821773896600322,"ci_75_upper":0.16189438041217696,"ci_50_lower":0.12728074283728286,"ci_50_upper":0.15263311812502206},"1048576":{"avg_score":0.09297380384147888,"min_score":0.0021834061481058598,"max_score":1,"n_tests":310,"stddev":0.2195995372445294,"median_score":0.030834699049592018,"q1_score":0.02162739308550954,"q3_score":0.04891287907958031,"avg_token_efficiency":1.000000953849041,"is_incomplete":false,"total_prompt_cost_bin":3.3216007500000058,"total_cache_read_cost_bin":7.792823099999955,"total_completion_cost_bin":0.1892625,"total_reasoning_cost_bin":0,"total_cost_bin":11.30368634999996,"total_on_demand_input_cost_bin":81.24983174999981,"ci_99_lower":0.06447459681396701,"ci_99_upper":0.12962107109819207,"ci_98_lower":0.06564461495079882,"ci_98_upper":0.12403850045732613,"ci_95_lower":0.06879777192675147,"ci_95_upper":0.11941045242898736,"ci_90_lower":0.07339216832785056,"ci_90_upper":0.11526795210669777,"ci_75_lower":0.07792899186625295,"ci_75_upper":0.10743044324035216,"ci_50_lower":0.08417024959494629,"ci_50_upper":0.10125353658616904}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low","minimal"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Google: Gemini 3.1 Flash Lite Preview","provider_name":"Google","run_provider_names":["Google AI Studio"],"pricing_metadata":{"prompt":2.5e-7,"completion":0.0000015,"reasoning":0.0000015,"has_tiered_pricing":true},"api_created_at":"2026-03-03T04:37:53.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-19T22:47:13.163Z"},{"model_slug":"deepseek/deepseek-v4-flash","reasoning_mode":null,"overall_metrics":{"auc_1m":0.1492666530543729,"auc_128k":0.2368974782521543,"cum_avg_1m":0.2267415868535516,"cum_avg_128k":0.282101458081823,"avg_score_overall":0.2346935280442939,"avg_response_time_ms":10482.247722630163,"total_prompt_cost":5.1193216200000045,"total_cache_read_cost":4.140752895999993,"total_completion_cost":0.09197608,"total_reasoning_cost":0,"total_prompt_cost_128k":0.3905714400000001,"total_completion_cost_128k":0.0525756,"total_on_demand_input_cost":25.823086100000005,"total_on_demand_input_cost_128k":3.4245708000000024,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971342554822208},"bin_metrics":{"8192":{"avg_score":0.472025318640226,"min_score":0.0011554014636203647,"max_score":1,"n_tests":80,"stddev":0.4810210342595763,"median_score":0.09171938896179199,"q1_score":0.028125859331339598,"q3_score":1,"avg_token_efficiency":0.9915838193643983,"is_incomplete":false,"total_prompt_cost_bin":0.04185776000000002,"total_cache_read_cost_bin":0.006393856000000002,"total_completion_cost_bin":0.007855119999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.056106736000000025,"total_on_demand_input_cost_bin":0.07382703999999998,"ci_99_lower":0.3274070715288471,"ci_99_upper":0.6032425881579911,"ci_98_lower":0.3422132020779972,"ci_98_upper":0.5907539829151938,"ci_95_lower":0.3611609287572355,"ci_95_upper":0.5778704260099765,"ci_90_lower":0.3766388612827723,"ci_90_upper":0.5586665125237779,"ci_75_lower":0.41154966860594866,"ci_75_upper":0.5337404777937992,"ci_50_lower":0.4358880350497202,"ci_50_upper":0.5081513402794372},"16384":{"avg_score":0.31775314623082523,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4479626539387769,"median_score":0.03231554664671421,"q1_score":0.022682589944452047,"q3_score":1,"avg_token_efficiency":0.9890247337221414,"is_incomplete":false,"total_prompt_cost_bin":0.04272688000000001,"total_cache_read_cost_bin":0.021948416,"total_completion_cost_bin":0.007803320000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.07247861600000001,"total_on_demand_input_cost_bin":0.15246896,"ci_99_lower":0.19643779174257361,"ci_99_upper":0.4536929274186405,"ci_98_lower":0.21016590349865147,"ci_98_upper":0.4404269431903085,"ci_95_lower":0.2213567925110692,"ci_95_upper":0.41489023761263527,"ci_90_lower":0.23454469756514298,"ci_90_upper":0.3924335483799223,"ci_75_lower":0.2575287921874405,"ci_75_upper":0.3692132461612346,"ci_50_lower":0.28305331631054287,"ci_50_upper":0.3524923460114224},"32768":{"avg_score":0.17671527722439564,"min_score":0.009250693954527378,"max_score":1,"n_tests":136,"stddev":0.3343788765339243,"median_score":0.03390283137559891,"q1_score":0.023859385401010513,"q3_score":0.06747523322701454,"avg_token_efficiency":1.0102892651671216,"is_incomplete":false,"total_prompt_cost_bin":0.1591016000000001,"total_cache_read_cost_bin":0.07777279999999993,"total_completion_cost_bin":0.015053360000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.25192776000000006,"total_on_demand_input_cost_bin":0.5479655999999999,"ci_99_lower":0.11013003228636295,"ci_99_upper":0.25707046970564135,"ci_98_lower":0.11363878426546067,"ci_98_upper":0.25135756584260993,"ci_95_lower":0.1219958609322716,"ci_95_upper":0.23616889239124517,"ci_90_lower":0.1308859706075643,"ci_90_upper":0.2252047777559389,"ci_75_lower":0.14270044843279345,"ci_75_upper":0.2083522212017081,"ci_50_lower":0.15720231207566993,"ci_50_upper":0.19238067384439464},"65536":{"avg_score":0.26730988509405185,"min_score":0.005247813183814287,"max_score":1,"n_tests":85,"stddev":0.4181771589957737,"median_score":0.03145589679479599,"q1_score":0.02028311789035797,"q3_score":0.09436997026205063,"avg_token_efficiency":0.9947451157769154,"is_incomplete":false,"total_prompt_cost_bin":0.10976013999999994,"total_cache_read_cost_bin":0.13290905599999986,"total_completion_cost_bin":0.009676800000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.2523459959999998,"total_on_demand_input_cost_bin":0.7743054199999999,"ci_99_lower":0.1644863714301575,"ci_99_upper":0.38353082940305633,"ci_98_lower":0.1691042467093862,"ci_98_upper":0.3783925893014735,"ci_95_lower":0.18707614902653458,"ci_95_upper":0.3580431117648807,"ci_90_lower":0.198427914962556,"ci_90_upper":0.3394676812527261,"ci_75_lower":0.21123900290044462,"ci_75_upper":0.31362569788539346,"ci_50_lower":0.23380901332232443,"ci_50_upper":0.29309481292661715},"131072":{"avg_score":0.1767036632196164,"min_score":0.003610108280554414,"max_score":1,"n_tests":103,"stddev":0.3536614300424977,"median_score":0.0282574575394392,"q1_score":0.017927194945514202,"q3_score":0.04126151278614998,"avg_token_efficiency":0.9940812964474941,"is_incomplete":false,"total_prompt_cost_bin":0.037125060000000036,"total_cache_read_cost_bin":0.36777574399999946,"total_completion_cost_bin":0.012186999999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.4170878039999995,"total_on_demand_input_cost_bin":1.8760037800000025,"ci_99_lower":0.09065355334770007,"ci_99_upper":0.2694237713487163,"ci_98_lower":0.1018804746437411,"ci_98_upper":0.26157165470700344,"ci_95_lower":0.11841818314753202,"ci_95_upper":0.24383378345909237,"ci_90_lower":0.1208187164806589,"ci_90_upper":0.23372747452513687,"ci_75_lower":0.13865776387443404,"ci_75_upper":0.2148455235868629,"ci_50_lower":0.15016033667539816,"ci_50_upper":0.19595506476991115},"262144":{"avg_score":0.2239435814066744,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.38543139671588644,"median_score":0.03267853707075119,"q1_score":0.025355596095323563,"q3_score":0.06407689303159714,"avg_token_efficiency":0.9994797316927643,"is_incomplete":false,"total_prompt_cost_bin":4.581299800000004,"total_cache_read_cost_bin":0.11726847999999995,"total_completion_cost_bin":0.015286320000000003,"total_reasoning_cost_bin":0,"total_cost_bin":4.713854600000004,"total_on_demand_input_cost_bin":5.167642200000001,"ci_99_lower":0.14933385657545356,"ci_99_upper":0.306574153127446,"ci_98_lower":0.15414205090930447,"ci_98_upper":0.2994486473224799,"ci_95_lower":0.16255363388051416,"ci_95_upper":0.28581093408824565,"ci_90_lower":0.1733516250604909,"ci_90_upper":0.2732231870467002,"ci_75_lower":0.1872150636792467,"ci_75_upper":0.2599470615126886,"ci_50_lower":0.2020283913432211,"ci_50_upper":0.24373798596372181},"524288":{"avg_score":0.17948182301262347,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.3407104978931716,"median_score":0.03451455570757389,"q1_score":0.022518334444612265,"q3_score":0.061437555588781834,"avg_token_efficiency":0.994975504731911,"is_incomplete":false,"total_prompt_cost_bin":0.14745038000000008,"total_cache_read_cost_bin":3.4166845439999944,"total_completion_cost_bin":0.024114159999999996,"total_reasoning_cost_bin":0,"total_cost_bin":3.5882490839999943,"total_on_demand_input_cost_bin":17.2308731,"ci_99_lower":0.12569003732293157,"ci_99_upper":0.23340874038697604,"ci_98_lower":0.12985657538361334,"ci_98_upper":0.23020250178927307,"ci_95_lower":0.13692856397616904,"ci_95_upper":0.22516592559810397,"ci_90_lower":0.14351861340161984,"ci_90_upper":0.2158301456326765,"ci_75_lower":0.15555073246052753,"ci_75_upper":0.20437576204913982,"ci_50_lower":0.16475374171284907,"ci_50_upper":0.193331086450213}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Flash 0423","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":8.106e-8,"completion":1.6212e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-24T03:17:46.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-24T13:26:08.597Z"},{"model_slug":"qwen/qwen3.8-27b","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.14909019680199967,"auc_128k":0.8659847686141403,"cum_avg_1m":0.567063504798213,"cum_avg_128k":0.9073016076771407,"avg_score_overall":0.9017002449013978,"avg_response_time_ms":177186.42397743414,"total_prompt_cost":11.2662054,"total_cache_read_cost":0,"total_completion_cost":10.887503999999996,"total_reasoning_cost":10.8712128,"total_prompt_cost_128k":11.2662054,"total_completion_cost_128k":10.887503999999996,"total_on_demand_input_cost":11.2662054,"total_on_demand_input_cost_128k":11.2662054,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0229086268445258},"bin_metrics":{"8192":{"avg_score":0.9999582476913929,"min_score":0.998678982257843,"max_score":1,"n_tests":80,"stddev":0.00017538462726477796,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.029021570260619,"is_incomplete":false,"total_prompt_cost_bin":0.24603839999999993,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.5949760000000001,"total_reasoning_cost_bin":0.5590975999999999,"total_cost_bin":1.4001119999999998,"total_on_demand_input_cost_bin":0.24603839999999993,"ci_99_lower":0.9998967360034584,"ci_99_upper":0.9999951340258122,"ci_98_lower":0.9999045692682267,"ci_98_upper":0.999993946403265,"ci_95_lower":0.9999131443724036,"ci_95_upper":0.999990229550749,"ci_90_lower":0.9999214550107717,"ci_90_upper":0.9999857529997825,"ci_75_lower":0.9999346675351263,"ci_75_upper":0.9999803505837918,"ci_50_lower":0.999946198426187,"ci_50_upper":0.9999731779098511},"16384":{"avg_score":0.9900946542620659,"min_score":0.21197521686553955,"max_score":1,"n_tests":80,"stddev":0.0880978038639632,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0191000159363646,"is_incomplete":false,"total_prompt_cost_bin":0.5058287999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.9983487999999996,"total_reasoning_cost_bin":0.9585215999999999,"total_cost_bin":2.4626991999999994,"total_on_demand_input_cost_bin":0.5058287999999996,"ci_99_lower":0.9605208795964717,"ci_99_upper":0.9999804026260972,"ci_98_lower":0.9605410908833147,"ci_98_upper":0.9999784312695265,"ci_95_lower":0.970372627209872,"ci_95_upper":0.9999722604267299,"ci_90_lower":0.9703984466567637,"ci_90_upper":0.9999651810899377,"ci_75_lower":0.9802301077172161,"ci_75_upper":0.999953626189381,"ci_50_lower":0.9802680399268866,"ci_50_upper":0.9999352723360062},"32768":{"avg_score":0.9293031761532321,"min_score":0.016029592603445053,"max_score":1,"n_tests":136,"stddev":0.251870556339642,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0271912230575917,"is_incomplete":false,"total_prompt_cost_bin":1.7920440000000009,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.5927551999999996,"total_reasoning_cost_bin":2.5709952,"total_cost_bin":6.9557944,"total_on_demand_input_cost_bin":1.7920440000000009,"ci_99_lower":0.866288002017502,"ci_99_upper":0.9788330413417561,"ci_98_lower":0.8729740491073908,"ci_98_upper":0.9720372626530553,"ci_95_lower":0.8799452564487343,"ci_95_upper":0.9715159628684085,"ci_90_lower":0.8926520522163415,"ci_90_upper":0.9645544680027182,"ci_75_lower":0.9013844281332294,"ci_75_upper":0.9572013478575494,"ci_50_lower":0.9150185653980931,"ci_50_upper":0.9436212889507742},"65536":{"avg_score":0.874066527198781,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.3210295724624408,"median_score":1,"q1_score":0.9995771646499634,"q3_score":1,"avg_token_efficiency":1.0216019153133646,"is_incomplete":false,"total_prompt_cost_bin":2.5560341999999983,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.7609983999999987,"total_reasoning_cost_bin":2.7928447999999992,"total_cost_bin":8.109877399999997,"total_on_demand_input_cost_bin":2.5560341999999983,"ci_99_lower":0.7677904054056852,"ci_99_upper":0.9541859040312359,"ci_98_lower":0.7779123675925109,"ci_98_upper":0.9499436273411573,"ci_95_lower":0.7902645878197955,"ci_95_upper":0.935468409893596,"ci_90_lower":0.8085643558352089,"ci_90_upper":0.9274284800815888,"ci_75_lower":0.8352077170220368,"ci_75_upper":0.9120142746141509,"ci_50_lower":0.850482371011201,"ci_50_upper":0.8969938055831281},"131072":{"avg_score":0.7430854330802318,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4288893754099733,"median_score":1,"q1_score":0.07005829364061356,"q3_score":1,"avg_token_efficiency":1.0165425180536243,"is_incomplete":false,"total_prompt_cost_bin":6.166260000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":3.9404256,"total_reasoning_cost_bin":3.989753600000001,"total_cost_bin":14.096439200000003,"total_on_demand_input_cost_bin":6.166260000000001,"ci_99_lower":0.6286833906288539,"ci_99_upper":0.8487738512352887,"ci_98_lower":0.6317757019790757,"ci_98_upper":0.8384711992522493,"ci_95_lower":0.651247099169487,"ci_95_upper":0.819984068148046,"ci_90_lower":0.6690292341911542,"ci_90_upper":0.8110179520813405,"ci_75_lower":0.6936259474435784,"ci_75_upper":0.7904913107886034,"ci_50_lower":0.7136289291112069,"ci_50_upper":0.7715258763778674}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","medium","low"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"Qwen: Qwen3.8 27B","provider_name":"Qwen","run_provider_names":["AkashML"],"pricing_metadata":{"prompt":4.25e-7,"completion":0.00000255,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-14T15:55:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-16T04:28:10.533Z"},{"model_slug":"qwen/qwen3.6-max-preview","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.14563607553581104,"auc_128k":0.851075146019708,"cum_avg_1m":0.5502544658392233,"cum_avg_128k":0.8804071453427573,"avg_score_overall":0.8814487984073964,"avg_response_time_ms":60834.970384329805,"total_prompt_cost":33.24124316999998,"total_cache_read_cost":0.02512042,"total_completion_cost":11.399780105999998,"total_reasoning_cost":10.217772941999996,"total_prompt_cost_128k":33.24124316999998,"total_completion_cost_128k":11.399780105999998,"total_on_demand_input_cost":33.26636358999998,"total_on_demand_input_cost_128k":33.26636358999998,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207081000672176},"bin_metrics":{"8192":{"avg_score":1,"min_score":1,"max_score":1,"n_tests":80,"stddev":0,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.022497270322228,"is_incomplete":false,"total_prompt_cost_bin":0.558063584,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.7769790739999995,"total_reasoning_cost_bin":1.598447448,"total_cost_bin":3.9334901059999994,"total_on_demand_input_cost_bin":0.558063584,"ci_99_lower":1,"ci_99_upper":1,"ci_98_lower":1,"ci_98_upper":1,"ci_95_lower":1,"ci_95_upper":1,"ci_90_lower":1,"ci_90_upper":1,"ci_75_lower":1,"ci_75_upper":1,"ci_50_lower":1,"ci_50_upper":1},"16384":{"avg_score":0.8536931463284418,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.35049638606549294,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.015970448833568,"is_incomplete":false,"total_prompt_cost_bin":1.150963008,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.902203238,"total_reasoning_cost_bin":1.7260501439999998,"total_cost_bin":4.77921639,"total_on_demand_input_cost_bin":1.150963008,"ci_99_lower":0.7433143802434206,"ci_99_upper":0.9513117682832526,"ci_98_lower":0.7565363641090226,"ci_98_upper":0.9389825310923624,"ci_95_lower":0.7689094774919796,"ci_95_upper":0.926572769418708,"ci_90_lower":0.7810572841484099,"ci_90_upper":0.9145320255844853,"ci_75_lower":0.804913749563275,"ci_75_upper":0.9020160505926469,"ci_50_lower":0.8291921662457753,"ci_50_upper":0.8780457974644378},"32768":{"avg_score":0.9573514702736728,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.1993003870927236,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0256830410679845,"is_incomplete":false,"total_prompt_cost_bin":4.058855996,"total_cache_read_cost_bin":0.02512042,"total_completion_cost_bin":3.4484893560000005,"total_reasoning_cost_bin":3.110423549999998,"total_cost_bin":10.642889321999998,"total_on_demand_input_cost_bin":4.0839764160000005,"ci_99_lower":0.9071178112382425,"ci_99_upper":0.9931302343988243,"ci_98_lower":0.9135399888282958,"ci_98_upper":0.9929367459707839,"ci_95_lower":0.9214852262183044,"ci_95_upper":0.9860690196031047,"ci_90_lower":0.9283011390195872,"ci_90_upper":0.9856286037177436,"ci_75_lower":0.9361631137983162,"ci_75_upper":0.9785985106821445,"ci_50_lower":0.9436996549646959,"ci_50_upper":0.9712011589110583},"65536":{"avg_score":0.8747843097676249,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.3267504608562336,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0209598255175443,"is_incomplete":false,"total_prompt_cost_bin":5.829771661999997,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.92750441,"total_reasoning_cost_bin":1.7074840379999998,"total_cost_bin":9.464760109999997,"total_on_demand_input_cost_bin":5.829771661999997,"ci_99_lower":0.7717171059473253,"ci_99_upper":0.9543647376076482,"ci_98_lower":0.7823873149088638,"ci_98_upper":0.9539118770365329,"ci_95_lower":0.7942490873945987,"ci_95_upper":0.9426765746256226,"ci_90_lower":0.8067263392172754,"ci_90_upper":0.931647313889116,"ci_75_lower":0.8302700993550175,"ci_75_upper":0.9097398912588902,"ci_50_lower":0.8515198564156891,"ci_50_upper":0.8977988371625543},"131072":{"avg_score":0.7162068003440475,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.44483399279517605,"median_score":1,"q1_score":0.037455614656209946,"q3_score":1,"avg_token_efficiency":1.0162215936488594,"is_incomplete":false,"total_prompt_cost_bin":21.643588919999985,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.3446040279999982,"total_reasoning_cost_bin":2.075367762,"total_cost_bin":26.063560709999983,"total_on_demand_input_cost_bin":21.643588919999985,"ci_99_lower":0.5933233677869825,"ci_99_upper":0.8301013222957613,"ci_98_lower":0.6026027349428075,"ci_98_upper":0.811297895301843,"ci_95_lower":0.6214827523284222,"ci_95_upper":0.8006705010470875,"ci_90_lower":0.6401020940248539,"ci_90_upper":0.7908129124971133,"ci_75_lower":0.6603410472300523,"ci_75_upper":0.7636529218614015,"ci_50_lower":0.6795849060024886,"ci_50_upper":0.7447811376605211}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 Max Preview","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":0.000001027,"completion":0.000006162,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-27T03:24:02.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T14:51:29.972Z"},{"model_slug":"deepseek/deepseek-v4-flash-0731","reasoning_mode":null,"overall_metrics":{"auc_1m":0.1452997537309188,"auc_128k":0.23959134457744,"cum_avg_1m":0.22808517218077567,"cum_avg_128k":0.28298108274045286,"avg_score_overall":0.23574789634686175,"avg_response_time_ms":5035.526018698733,"total_prompt_cost":0.0034955100000000003,"total_cache_read_cost":3.319411968000003,"total_completion_cost":0.057550859999999995,"total_reasoning_cost":0,"total_prompt_cost_128k":0.0022496399999999994,"total_completion_cost_128k":0.03285414,"total_on_demand_input_cost":16.60055535,"total_on_demand_input_cost_128k":2.2015098,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9971342554822207},"bin_metrics":{"8192":{"avg_score":0.46270134349906583,"min_score":0.0011554014636203647,"max_score":1,"n_tests":80,"stddev":0.4774783465114941,"median_score":0.08228861913084984,"q1_score":0.028125859331339598,"q3_score":1,"avg_token_efficiency":0.991583819364398,"is_incomplete":false,"total_prompt_cost_bin":0.00045863999999999985,"total_cache_read_cost_bin":0.00940032,"total_completion_cost_bin":0.004921020000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.014779980000000003,"total_on_demand_input_cost_bin":0.04746024,"ci_99_lower":0.32692802565947926,"ci_99_upper":0.5918747060675014,"ci_98_lower":0.33668197266060446,"ci_98_upper":0.5788592928659783,"ci_95_lower":0.3488349343906884,"ci_95_upper":0.5675489075098086,"ci_90_lower":0.36747073619837467,"ci_90_upper":0.5532491505714279,"ci_75_lower":0.40137551400603116,"ci_75_upper":0.5256992389678998,"ci_50_lower":0.42613636629212126,"ci_50_upper":0.49936235714194477},"16384":{"avg_score":0.2951481828262331,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.43679058750970856,"median_score":0.03272862918674946,"q1_score":0.02305248472839594,"q3_score":1,"avg_token_efficiency":0.989024733722142,"is_incomplete":false,"total_prompt_cost_bin":0.00041831999999999997,"total_cache_read_cost_bin":0.019519488,"total_completion_cost_bin":0.004835519999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.024773328,"total_on_demand_input_cost_bin":0.09801576,"ci_99_lower":0.18386357052881797,"ci_99_upper":0.4285358434970404,"ci_98_lower":0.18670741524797632,"ci_98_upper":0.4061842742256994,"ci_95_lower":0.20831400933930128,"ci_95_upper":0.3920073768075235,"ci_90_lower":0.22056433679972542,"ci_90_upper":0.36955663323635235,"ci_75_lower":0.2441245700501895,"ci_75_upper":0.34740506233647467,"ci_50_lower":0.2613454427460965,"ci_50_upper":0.32110093792143746},"32768":{"avg_score":0.22127572917516397,"min_score":0.005194805096834898,"max_score":1,"n_tests":136,"stddev":0.3806791558710798,"median_score":0.034428661689162254,"q1_score":0.022002163343131542,"q3_score":0.0881910901516676,"avg_token_efficiency":1.0102892651671211,"is_incomplete":false,"total_prompt_cost_bin":0.00058104,"total_cache_read_cost_bin":0.07033651200000011,"total_completion_cost_bin":0.009479879999999994,"total_reasoning_cost_bin":0,"total_cost_bin":0.08039743200000012,"total_on_demand_input_cost_bin":0.3522636000000004,"ci_99_lower":0.1478870924625046,"ci_99_upper":0.30264900349375023,"ci_98_lower":0.15110768618565465,"ci_98_upper":0.29864267134241185,"ci_95_lower":0.1594057050986414,"ci_95_upper":0.2855308416019611,"ci_90_lower":0.16883006348593077,"ci_90_upper":0.2759844921205727,"ci_75_lower":0.18357350563575678,"ci_75_upper":0.25762959211501496,"ci_50_lower":0.1988157269969473,"ci_50_upper":0.2415605024271987},"65536":{"avg_score":0.2564247011700097,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.41531881747340316,"median_score":0.025779375806450844,"q1_score":0.017600269988179207,"q3_score":0.10603290796279907,"avg_token_efficiency":0.9947451157769149,"is_incomplete":false,"total_prompt_cost_bin":0.00033417000000000017,"total_cache_read_cost_bin":0.09948671999999993,"total_completion_cost_bin":0.005879160000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.10570004999999993,"total_on_demand_input_cost_bin":0.49776777000000044,"ci_99_lower":0.15388639786127298,"ci_99_upper":0.37426490967287956,"ci_98_lower":0.16452180112449125,"ci_98_upper":0.36393406094566855,"ci_95_lower":0.17594933011021247,"ci_95_upper":0.3470206357341478,"ci_90_lower":0.18631193793543122,"ci_90_upper":0.33372242508894384,"ci_75_lower":0.20157522537389438,"ci_75_upper":0.30239027787328643,"ci_50_lower":0.2228618261678254,"ci_50_upper":0.28244153404991856},"131072":{"avg_score":0.17935545703179193,"min_score":0.004308487754315138,"max_score":1,"n_tests":103,"stddev":0.35396550618773004,"median_score":0.02967946231365204,"q1_score":0.017776082269847393,"q3_score":0.04681859351694584,"avg_token_efficiency":0.9940812964474942,"is_incomplete":false,"total_prompt_cost_bin":0.0004574699999999997,"total_cache_read_cost_bin":0.24110899199999983,"total_completion_cost_bin":0.0077385599999999985,"total_reasoning_cost_bin":0,"total_cost_bin":0.24930502199999982,"total_on_demand_input_cost_bin":1.2060024299999996,"ci_99_lower":0.09318523959340347,"ci_99_upper":0.27263119060727,"ci_98_lower":0.10479733759012905,"ci_98_upper":0.2640336942717487,"ci_95_lower":0.12099214327223237,"ci_95_upper":0.2465564252864072,"ci_90_lower":0.12385019570480561,"ci_90_upper":0.2364197042729995,"ci_75_lower":0.14085252163253578,"ci_75_upper":0.21737075746466308,"ci_50_lower":0.15337642323162443,"ci_50_upper":0.1984045205556435},"262144":{"avg_score":0.2602038066999339,"min_score":0.006493506487458944,"max_score":1,"n_tests":141,"stddev":0.4104775646513237,"median_score":0.037351444363594055,"q1_score":0.02379349060356617,"q3_score":0.08876560628414154,"avg_token_efficiency":0.999479731692764,"is_incomplete":false,"total_prompt_cost_bin":0.00032129999999999995,"total_cache_read_cost_bin":0.6643468799999995,"total_completion_cost_bin":0.009580320000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.6742484999999995,"total_on_demand_input_cost_bin":3.322055700000004,"ci_99_lower":0.1772752642294174,"ci_99_upper":0.34346857132854147,"ci_98_lower":0.18026934729245034,"ci_98_upper":0.3369380968295249,"ci_95_lower":0.19338206924004347,"ci_95_upper":0.3260319310734181,"ci_90_lower":0.20446608001993746,"ci_90_upper":0.3148709658318668,"ci_75_lower":0.21951524739224687,"ci_75_upper":0.29614561827562697,"ci_50_lower":0.23825081543260457,"ci_50_upper":0.2817124139256643},"524288":{"avg_score":0.1495721570440071,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.31356332755865557,"median_score":0.029993184842169285,"q1_score":0.02079364052042365,"q3_score":0.05469909589737654,"avg_token_efficiency":0.9949755047319108,"is_incomplete":false,"total_prompt_cost_bin":0.0009245700000000008,"total_cache_read_cost_bin":2.2152130560000036,"total_completion_cost_bin":0.015116399999999999,"total_reasoning_cost_bin":0,"total_cost_bin":2.2312540260000038,"total_on_demand_input_cost_bin":11.076989849999999,"ci_99_lower":0.10015225273273122,"ci_99_upper":0.20320726059001554,"ci_98_lower":0.10535444249660941,"ci_98_upper":0.1979631738215823,"ci_95_lower":0.11068583135892661,"ci_95_upper":0.18930905143284166,"ci_90_lower":0.11808018268853568,"ci_90_upper":0.18338512054652417,"ci_75_lower":0.12722879411487187,"ci_75_upper":0.1711569257373813,"ci_50_lower":0.1358185464183808,"ci_50_upper":0.16220984631834623}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1310720,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"DeepSeek: DeepSeek V4 Flash 0731","provider_name":"Deepseek","run_provider_names":["DeepSeek"],"pricing_metadata":{"prompt":6e-8,"completion":1.2e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-31T06:21:48.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-02T01:56:58.833Z"},{"model_slug":"qwen/qwen3.5-flash","reasoning_mode":null,"overall_metrics":{"auc_1m":0.1445604021204355,"auc_128k":0.32019458998219374,"cum_avg_1m":0.258207051756026,"cum_avg_128k":0.34587387251541324,"avg_score_overall":0.27236646571341555,"avg_response_time_ms":13714.213077530008,"total_prompt_cost":8.877897859999997,"total_cache_read_cost":0,"total_completion_cost":0.07737834,"total_reasoning_cost":0,"total_prompt_cost_128k":1.6260823799999997,"total_completion_cost_128k":0.04969328,"total_on_demand_input_cost":8.877897859999997,"total_on_demand_input_cost_128k":1.6260823799999997,"total_runs":755,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0199778247045406},"bin_metrics":{"8192":{"avg_score":0.4792525725031737,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4820745631005219,"median_score":0.09359140321612358,"q1_score":0.027709668036550283,"q3_score":0.9979728311300278,"avg_token_efficiency":1.0228079512716761,"is_incomplete":false,"total_prompt_cost_bin":0.035330880000000016,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.00751816,"total_reasoning_cost_bin":0,"total_cost_bin":0.04284904000000002,"total_on_demand_input_cost_bin":0.035330880000000016,"ci_99_lower":0.3365163295440434,"ci_99_upper":0.6092668687015539,"ci_98_lower":0.34706006132857875,"ci_98_upper":0.5989678662256337,"ci_95_lower":0.3685766489866364,"ci_95_upper":0.5853015849733492,"ci_90_lower":0.383103159085731,"ci_90_upper":0.565303441461292,"ci_75_lower":0.4196213234536117,"ci_75_upper":0.5405496390878397,"ci_50_lower":0.44299177664943273,"ci_50_upper":0.5155958920848207},"16384":{"avg_score":0.3569421205698745,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.45806479978428927,"median_score":0.03363822586834431,"q1_score":0.023134059738367796,"q3_score":0.9896649271249771,"avg_token_efficiency":1.0161194758384633,"is_incomplete":false,"total_prompt_cost_bin":0.07285616000000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.007673119999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.08052928000000004,"total_on_demand_input_cost_bin":0.07285616000000004,"ci_99_lower":0.224768928023419,"ci_99_upper":0.49020182854213634,"ci_98_lower":0.23386950123880523,"ci_98_upper":0.4777122987876937,"ci_95_lower":0.2549573998834967,"ci_95_upper":0.4587690661079978,"ci_90_lower":0.27057532403327056,"ci_90_upper":0.4408532785746502,"ci_75_lower":0.3014917008680641,"ci_75_upper":0.4158093767648097,"ci_50_lower":0.3215849159074423,"ci_50_upper":0.39096510622621283},"32768":{"avg_score":0.25315926966825775,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4086184979063417,"median_score":0.030609832145273685,"q1_score":0.015106625156477094,"q3_score":0.10489795356988907,"avg_token_efficiency":1.0257548592579662,"is_incomplete":false,"total_prompt_cost_bin":0.2584972,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.013855140000000004,"total_reasoning_cost_bin":0,"total_cost_bin":0.27235233999999997,"total_on_demand_input_cost_bin":0.2584972,"ci_99_lower":0.16795082295637898,"ci_99_upper":0.3370144303107586,"ci_98_lower":0.17263803764282357,"ci_98_upper":0.3338249737926526,"ci_95_lower":0.1869827971507381,"ci_95_upper":0.32051016466160515,"ci_90_lower":0.19588858450862223,"ci_90_upper":0.30858349008990715,"ci_75_lower":0.2118318389969799,"ci_75_upper":0.29189086533498243,"ci_50_lower":0.22899159359718288,"ci_50_upper":0.27577913839623386},"65536":{"avg_score":0.35416998713934683,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.46020884894952163,"median_score":0.03306504338979721,"q1_score":0.022657541558146477,"q3_score":0.9856981635093689,"avg_token_efficiency":1.0209904012221072,"is_incomplete":false,"total_prompt_cost_bin":0.3689839400000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0093223,"total_reasoning_cost_bin":0,"total_cost_bin":0.3783062400000001,"total_on_demand_input_cost_bin":0.3689839400000001,"ci_99_lower":0.23900594161212554,"ci_99_upper":0.48111802544494103,"ci_98_lower":0.2415492972629145,"ci_98_upper":0.4694502263601376,"ci_95_lower":0.26138626913807134,"ci_95_upper":0.4564059504805653,"ci_90_lower":0.273899036378271,"ci_90_upper":0.4378341624494093,"ci_75_lower":0.29475030472317215,"ci_75_upper":0.4115392516165123,"ci_50_lower":0.3199549738173921,"ci_50_upper":0.3875945648418137},"131072":{"avg_score":0.28584541269641334,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.42882055021795856,"median_score":0.029684601351618767,"q1_score":0.018787242472171783,"q3_score":0.9421403408050537,"avg_token_efficiency":1.0162368757633713,"is_incomplete":false,"total_prompt_cost_bin":0.8904141999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.01132456,"total_reasoning_cost_bin":0,"total_cost_bin":0.9017387599999996,"total_on_demand_input_cost_bin":0.8904141999999996,"ci_99_lower":0.1824756943122529,"ci_99_upper":0.3882765039649467,"ci_98_lower":0.186134242281381,"ci_98_upper":0.37953117440056816,"ci_95_lower":0.2085258240002271,"ci_95_upper":0.36857128413043716,"ci_90_lower":0.21996902307355323,"ci_90_upper":0.3539847788728719,"ci_75_lower":0.23841903426569155,"ci_75_upper":0.33178456982618054,"ci_50_lower":0.2572542575371081,"ci_50_upper":0.3136058655088243},"262144":{"avg_score":0.20301206469265443,"min_score":0.002918642945587635,"max_score":1,"n_tests":124,"stddev":0.3699834842784221,"median_score":0.03208021819591522,"q1_score":0.021159084979444742,"q3_score":0.056166814640164375,"avg_token_efficiency":1.0180738075035256,"is_incomplete":false,"total_prompt_cost_bin":2.1492429400000006,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.01267734,"total_reasoning_cost_bin":0,"total_cost_bin":2.161920280000001,"total_on_demand_input_cost_bin":2.1492429400000006,"ci_99_lower":0.1215328182498624,"ci_99_upper":0.2822642589375309,"ci_98_lower":0.12554314312757925,"ci_98_upper":0.2740729402369939,"ci_95_lower":0.13873013832846717,"ci_95_upper":0.2645905568599588,"ci_90_lower":0.14929001956817603,"ci_90_upper":0.2565134645999229,"ci_75_lower":0.16513411337209324,"ci_75_upper":0.24144473467904684,"ci_50_lower":0.18011633756442838,"ci_50_upper":0.22183538300188016},"524288":{"avg_score":0.13327498677848731,"min_score":0,"max_score":1,"n_tests":147,"stddev":0.2855832541132323,"median_score":0.03298429399728775,"q1_score":0.016130640171468258,"q3_score":0.0671435296535492,"avg_token_efficiency":1.0188344774563953,"is_incomplete":false,"total_prompt_cost_bin":5.102572539999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.015007719999999999,"total_reasoning_cost_bin":0,"total_cost_bin":5.117580259999996,"total_on_demand_input_cost_bin":5.102572539999996,"ci_99_lower":0.07687345435659933,"ci_99_upper":0.1958254000931688,"ci_98_lower":0.0823593625635048,"ci_98_upper":0.19047441304731025,"ci_95_lower":0.09233524706186949,"ci_95_upper":0.18396693117515467,"ci_90_lower":0.09723129355749369,"ci_90_upper":0.17293373210273277,"ci_75_lower":0.10657709052166141,"ci_75_upper":0.1588207847806213,"ci_50_lower":0.1157563948167526,"ci_50_upper":0.1479583897991787}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-Flash","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":6.5e-8,"completion":2.6e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-25T21:09:36.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T04:01:03.081Z"},{"model_slug":"xiaomi/mimo-v2.5","reasoning_mode":null,"overall_metrics":{"auc_1m":0.14287830323750858,"auc_128k":0.27270530463350634,"cum_avg_1m":0.2328719309818349,"cum_avg_128k":0.29848649812427597,"avg_score_overall":0.23985871900782707,"avg_response_time_ms":10577.953203402883,"total_prompt_cost":3.0300590600000024,"total_cache_read_cost":0.4555916288000005,"total_completion_cost":0.09083759999999999,"total_reasoning_cost":0,"total_prompt_cost_128k":0.7073207400000001,"total_completion_cost_128k":0.05171096,"total_on_demand_input_cost":25.809640500000015,"total_on_demand_input_cost_128k":3.43168042,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0023861725760572},"bin_metrics":{"8192":{"avg_score":0.4097105948603712,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4724752373395544,"median_score":0.056538550183176994,"q1_score":0.025831264443695545,"q3_score":1,"avg_token_efficiency":1.0286010895361408,"is_incomplete":false,"total_prompt_cost_bin":0.03294480000000001,"total_cache_read_cost_bin":0.0008709119999999994,"total_completion_cost_bin":0.00799204,"total_reasoning_cost_bin":0,"total_cost_bin":0.04180775200000001,"total_on_demand_input_cost_bin":0.07649040000000004,"ci_99_lower":0.27412746091984447,"ci_99_upper":0.541843640592182,"ci_98_lower":0.2866223195749335,"ci_98_upper":0.5395849688726012,"ci_95_lower":0.29779064797388854,"ci_95_upper":0.5151759991893777,"ci_90_lower":0.3151416730787605,"ci_90_upper":0.4973515127546852,"ci_75_lower":0.3497752889539697,"ci_75_upper":0.4702495305304183,"ci_50_lower":0.37344120922789437,"ci_50_upper":0.445374045780045},"16384":{"avg_score":0.294974143296713,"min_score":0.0044469148851931095,"max_score":1,"n_tests":80,"stddev":0.43707035868860955,"median_score":0.03163466043770313,"q1_score":0.023614450357854366,"q3_score":1,"avg_token_efficiency":0.9994534373893262,"is_incomplete":false,"total_prompt_cost_bin":0.054008640000000004,"total_cache_read_cost_bin":0.002001305599999999,"total_completion_cost_bin":0.0077448,"total_reasoning_cost_bin":0,"total_cost_bin":0.0637547456,"total_on_demand_input_cost_bin":0.1540739200000001,"ci_99_lower":0.18346378538411226,"ci_99_upper":0.42875056410461554,"ci_98_lower":0.18664919333864238,"ci_98_upper":0.40620826897089124,"ci_95_lower":0.208073399267887,"ci_95_upper":0.39207161436570453,"ci_90_lower":0.22041631979751397,"ci_90_upper":0.3692831659319927,"ci_75_lower":0.2439600116762449,"ci_75_upper":0.3474208288389491,"ci_50_lower":0.26117170327779604,"ci_50_upper":0.32103107176080814},"32768":{"avg_score":0.26957343246013965,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4154564241329519,"median_score":0.035368675366044044,"q1_score":0.02256704680621624,"q3_score":0.13413005508482456,"avg_token_efficiency":1.0185751251342623,"is_incomplete":false,"total_prompt_cost_bin":0.12589807999999997,"total_cache_read_cost_bin":0.008539596799999997,"total_completion_cost_bin":0.014834680000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.14927235679999998,"total_on_demand_input_cost_bin":0.5528779200000005,"ci_99_lower":0.18302925564608066,"ci_99_upper":0.3525357868637794,"ci_98_lower":0.1900157787037931,"ci_98_upper":0.34846475950461087,"ci_95_lower":0.20028103192132274,"ci_95_upper":0.33671008877097525,"ci_90_lower":0.21228603857076345,"ci_90_upper":0.32636451083843204,"ci_75_lower":0.22776881186419973,"ci_75_upper":0.3083950841567404,"ci_50_lower":0.2460691479162265,"ci_50_upper":0.29219861455888296},"65536":{"avg_score":0.28092273986514876,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4264233345729941,"median_score":0.033847685903310776,"q1_score":0.021377671509981155,"q3_score":0.9537546038627625,"avg_token_efficiency":0.9957962005205171,"is_incomplete":false,"total_prompt_cost_bin":0.06436262,"total_cache_read_cost_bin":0.014215219200000014,"total_completion_cost_bin":0.009179239999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.0877570792,"total_on_demand_input_cost_bin":0.7751235800000001,"ci_99_lower":0.17066440927001703,"ci_99_upper":0.40494287610158103,"ci_98_lower":0.1869798311548965,"ci_98_upper":0.3947063700114103,"ci_95_lower":0.1929583052740268,"ci_95_upper":0.3739285788854913,"ci_90_lower":0.21072598623741856,"ci_90_upper":0.35939907267771876,"ci_75_lower":0.2243668618865366,"ci_75_upper":0.3287933552281602,"ci_50_lower":0.24697718842995958,"ci_50_upper":0.31069806494667906},"131072":{"avg_score":0.23725158013900724,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4000067559584928,"median_score":0.029702970758080482,"q1_score":0.018427281640470028,"q3_score":0.06527271494269371,"avg_token_efficiency":0.9925503401278395,"is_incomplete":false,"total_prompt_cost_bin":0.4301066000000001,"total_cache_read_cost_bin":0.028860159999999992,"total_completion_cost_bin":0.011960199999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.47092696000000006,"total_on_demand_input_cost_bin":1.8731145999999996,"ci_99_lower":0.14281255030858744,"ci_99_upper":0.33898024601322707,"ci_98_lower":0.1513138431077371,"ci_98_upper":0.32880044089173627,"ci_95_lower":0.16698543720622094,"ci_95_upper":0.3131343521863828,"ci_90_lower":0.17658043685572639,"ci_90_upper":0.29993746763590307,"ci_75_lower":0.1911108411664315,"ci_75_upper":0.2802606023086836,"ci_50_lower":0.21068105007718468,"ci_50_upper":0.2612635543274928},"262144":{"avg_score":0.23454623527705987,"min_score":0.005409582518041134,"max_score":1,"n_tests":141,"stddev":0.39122066626095287,"median_score":0.034739453345537186,"q1_score":0.023276634514331818,"q3_score":0.08153077960014343,"avg_token_efficiency":0.9987398643277834,"is_incomplete":false,"total_prompt_cost_bin":0.8624254800000004,"total_cache_read_cost_bin":0.08602782719999998,"total_completion_cost_bin":0.015306480000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.9637597872000004,"total_on_demand_input_cost_bin":5.163816840000004,"ci_99_lower":0.15899937807509662,"ci_99_upper":0.31728785785207364,"ci_98_lower":0.1650567372387328,"ci_98_upper":0.3151425094468559,"ci_95_lower":0.17159316364332292,"ci_95_upper":0.2977595543923147,"ci_90_lower":0.18149834667045145,"ci_90_upper":0.28455865614657805,"ci_75_lower":0.19648404315804255,"ci_75_upper":0.2699259774584376,"ci_50_lower":0.21248441441843274,"ci_50_upper":0.254318821362834},"524288":{"avg_score":0.1359967219562396,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.2913146221565299,"median_score":0.0314888171851635,"q1_score":0.02153055462986231,"q3_score":0.05400692392140627,"avg_token_efficiency":0.9940094596104206,"is_incomplete":false,"total_prompt_cost_bin":1.4603128400000018,"total_cache_read_cost_bin":0.31507660800000054,"total_completion_cost_bin":0.023820159999999996,"total_reasoning_cost_bin":0,"total_cost_bin":1.7992096080000024,"total_on_demand_input_cost_bin":17.21414324000001,"ci_99_lower":0.09189809387639042,"ci_99_upper":0.18400001423433915,"ci_98_lower":0.0953625630434421,"ci_98_upper":0.18003804467071904,"ci_95_lower":0.10197376923330258,"ci_95_upper":0.17519801137961497,"ci_90_lower":0.106384093778936,"ci_90_upper":0.16885970390966099,"ci_75_lower":0.11442161327653655,"ci_75_upper":0.1579886370757052,"ci_50_lower":0.12253740015049994,"ci_50_upper":0.14761130334646638}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2.5","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":1.4e-7,"completion":2.8e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-22T16:11:09.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-22T18:52:58.588Z"},{"model_slug":"xiaomi/mimo-v2.5-pro","reasoning_mode":null,"overall_metrics":{"auc_1m":0.1388415520507679,"auc_128k":0.3083176958611393,"cum_avg_1m":0.24185805866346582,"cum_avg_128k":0.32196289065795813,"avg_score_overall":0.24252460707418738,"avg_response_time_ms":23267.50122410578,"total_prompt_cost":1.486239704999999,"total_cache_read_cost":0.6513889535999997,"total_completion_cost":0.2820252900000001,"total_reasoning_cost":0,"total_prompt_cost_128k":0.23879194500000023,"total_completion_cost_128k":0.16322331,"total_on_demand_input_cost":80.19573826499996,"total_on_demand_input_cost_128k":10.663563464999996,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0025085745924778},"bin_metrics":{"8192":{"avg_score":0.47328946725319837,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4799724740980604,"median_score":0.11820477992296219,"q1_score":0.028036646079272032,"q3_score":1,"avg_token_efficiency":1.0292224514350352,"is_incomplete":false,"total_prompt_cost_bin":0.05250276000000003,"total_cache_read_cost_bin":0.0015335423999999994,"total_completion_cost_bin":0.024376530000000004,"total_reasoning_cost_bin":0,"total_cost_bin":0.07841283240000003,"total_on_demand_input_cost_bin":0.23780579999999996,"ci_99_lower":0.329049405889833,"ci_99_upper":0.6036628016861433,"ci_98_lower":0.34572896883483917,"ci_98_upper":0.5908118677696329,"ci_95_lower":0.3612289498781684,"ci_95_upper":0.5780844218085621,"ci_90_lower":0.378785119978711,"ci_90_upper":0.5613099105970469,"ci_75_lower":0.4124526499332205,"ci_75_upper":0.5354336693082586,"ci_50_lower":0.43705832686828217,"ci_50_upper":0.5095988199434941},"16384":{"avg_score":0.27288926820037884,"min_score":0.007502679713070393,"max_score":1,"n_tests":80,"stddev":0.4225961933152381,"median_score":0.03494700975716114,"q1_score":0.02406835462898016,"q3_score":0.3162619788199663,"avg_token_efficiency":0.9997514913991175,"is_incomplete":false,"total_prompt_cost_bin":0.04141896,"total_cache_read_cost_bin":0.0036202752000000014,"total_completion_cost_bin":0.024226019999999994,"total_reasoning_cost_bin":0,"total_cost_bin":0.06926525519999999,"total_on_demand_input_cost_bin":0.4788688800000002,"ci_99_lower":0.16399564941978317,"ci_99_upper":0.39354908309422904,"ci_98_lower":0.16532508715544825,"ci_98_upper":0.3826483784621814,"ci_95_lower":0.18579790935182247,"ci_95_upper":0.3689444467824069,"ci_90_lower":0.19937197363877204,"ci_90_upper":0.3473208423581672,"ci_75_lower":0.22329691737541,"ci_75_upper":0.3217357538480428,"ci_50_lower":0.24447609991766514,"ci_50_upper":0.2982317514281021},"32768":{"avg_score":0.24459156665184043,"min_score":0.004460551775991917,"max_score":1,"n_tests":136,"stddev":0.3955192140886158,"median_score":0.03920864872634411,"q1_score":0.024189640302211046,"q3_score":0.10764018259942532,"avg_token_efficiency":1.018718761514224,"is_incomplete":false,"total_prompt_cost_bin":0.05611500000000007,"total_cache_read_cost_bin":0.013754419200000024,"total_completion_cost_bin":0.04710093000000004,"total_reasoning_cost_bin":0,"total_cost_bin":0.11697034920000013,"total_on_demand_input_cost_bin":1.718107319999998,"ci_99_lower":0.15869831185305133,"ci_99_upper":0.3226709792991032,"ci_98_lower":0.16867531169870154,"ci_98_upper":0.3189572746353224,"ci_95_lower":0.18229574456002892,"ci_95_upper":0.3087945167125915,"ci_90_lower":0.1901368864079225,"ci_90_upper":0.3010279851359562,"ci_75_lower":0.20382142211105125,"ci_75_upper":0.2824095358870074,"ci_50_lower":0.22179370667093762,"ci_50_upper":0.2653607178705416},"65536":{"avg_score":0.3844177486484542,"min_score":0.003880983218550682,"max_score":1,"n_tests":85,"stddev":0.46939186801988186,"median_score":0.04285714402794838,"q1_score":0.02166847325861454,"q3_score":1,"avg_token_efficiency":0.9958573519296433,"is_incomplete":false,"total_prompt_cost_bin":0.030252075000000052,"total_cache_read_cost_bin":0.019682611199999988,"total_completion_cost_bin":0.030463050000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.08039773620000004,"total_on_demand_input_cost_bin":2.4085675950000014,"ci_99_lower":0.2480953979812124,"ci_99_upper":0.531168154655901,"ci_98_lower":0.2710590784422615,"ci_98_upper":0.5103063233542967,"ci_95_lower":0.2835557115907945,"ci_95_upper":0.4865608907477273,"ci_90_lower":0.3032531870885149,"ci_90_upper":0.47382895984412987,"ci_75_lower":0.32647793517338436,"ci_75_upper":0.4422123900642071,"ci_50_lower":0.3496675621170332,"ci_50_upper":0.41849163031095965},"131072":{"avg_score":0.23462640253591885,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4010303541986522,"median_score":0.0304151251912117,"q1_score":0.019651595503091812,"q3_score":0.0497902724891901,"avg_token_efficiency":0.9925809043568657,"is_incomplete":false,"total_prompt_cost_bin":0.05850315000000009,"total_cache_read_cost_bin":0.04768312319999999,"total_completion_cost_bin":0.03705677999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.14324305320000005,"total_on_demand_input_cost_bin":5.8202138699999955,"ci_99_lower":0.13964120509220487,"ci_99_upper":0.3371978993580261,"ci_98_lower":0.1485976022615287,"ci_98_upper":0.3281385644976936,"ci_95_lower":0.1658330254150482,"ci_95_upper":0.3097599961707884,"ci_90_lower":0.17570928603668418,"ci_90_upper":0.29993484441747037,"ci_75_lower":0.1877591139670841,"ci_75_upper":0.27959264346253737,"ci_50_lower":0.20693283122331263,"ci_50_upper":0.25730534227343405},"262144":{"avg_score":0.18630417985907682,"min_score":0.002102312631905079,"max_score":1,"n_tests":141,"stddev":0.35159557547324377,"median_score":0.0353446900844574,"q1_score":0.02375478856265545,"q3_score":0.06064140051603317,"avg_token_efficiency":0.9987551360738827,"is_incomplete":false,"total_prompt_cost_bin":1.015370909999999,"total_cache_read_cost_bin":0.12438282240000026,"total_completion_cost_bin":0.045148650000000005,"total_reasoning_cost_bin":0,"total_cost_bin":1.1849023823999993,"total_on_demand_input_cost_bin":16.044961949999994,"ci_99_lower":0.11195185205620112,"ci_99_upper":0.25618851721806923,"ci_98_lower":0.1196257504379271,"ci_98_upper":0.2519771497085334,"ci_95_lower":0.13001887973840515,"ci_95_upper":0.24745734905459132,"ci_90_lower":0.13973431121161645,"ci_90_upper":0.23472347081975733,"ci_75_lower":0.15227273410741002,"ci_75_upper":0.22024147081765155,"ci_50_lower":0.16566170528134766,"ci_50_upper":0.204940497757147},"524288":{"avg_score":0.13874583615885921,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.29652461015761694,"median_score":0.03050669003278017,"q1_score":0.0195956421084702,"q3_score":0.049826331436634064,"avg_token_efficiency":0.9940170910265357,"is_incomplete":false,"total_prompt_cost_bin":0.23207684999999975,"total_cache_read_cost_bin":0.4407321599999995,"total_completion_cost_bin":0.07365333000000009,"total_reasoning_cost_bin":0,"total_cost_bin":0.7464623399999993,"total_on_demand_input_cost_bin":53.48721284999997,"ci_99_lower":0.09412498748980135,"ci_99_upper":0.18615456172688843,"ci_98_lower":0.09774026946941099,"ci_98_upper":0.1831077834479984,"ci_95_lower":0.10322213765360529,"ci_95_upper":0.17722718916143784,"ci_90_lower":0.10940990030417273,"ci_90_upper":0.17170128561048253,"ci_75_lower":0.11657205280986954,"ci_75_upper":0.1606088083186108,"ci_50_lower":0.12514129426785137,"ci_50_upper":0.1513224337449775}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1050000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2.5-Pro","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":4.35e-7,"completion":8.7e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-22T16:11:13.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-22T19:05:52.057Z"},{"model_slug":"xiaomi/mimo-v2-pro","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.13579397152145883,"auc_128k":0.34989528735389513,"cum_avg_1m":0.3158008181104485,"cum_avg_128k":0.44256147423977676,"avg_score_overall":0.2999104891214671,"avg_response_time_ms":32285.11042698806,"total_prompt_cost":3.8504030000000053,"total_cache_read_cost":68.06461440000012,"total_completion_cost":2.3945399999999997,"total_reasoning_cost":1.414779,"total_prompt_cost_128k":0.6963230000000002,"total_completion_cost_128k":1.3362749999999999,"total_on_demand_input_cost":344.1971470000003,"total_on_demand_input_cost_128k":24.512002999999993,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0023861725760568},"bin_metrics":{"8192":{"avg_score":0.72240921704215,"min_score":0.006839945446699858,"max_score":1,"n_tests":80,"stddev":0.4399722998451922,"median_score":1,"q1_score":0.0654174406081438,"q3_score":1,"avg_token_efficiency":1.0286010895361408,"is_incomplete":false,"total_prompt_cost_bin":0.0843439999999999,"total_cache_read_cost_bin":0.09240319999999992,"total_completion_cost_bin":0.2058329999999999,"total_reasoning_cost_bin":0.12142799999999997,"total_cost_bin":0.5040081999999997,"total_on_demand_input_cost_bin":0.54636,"ci_99_lower":0.5885155538069085,"ci_99_upper":0.8320820802252565,"ci_98_lower":0.6003056080112583,"ci_98_upper":0.8304274449828662,"ci_95_lower":0.6239188967816881,"ci_95_upper":0.808643653330073,"ci_90_lower":0.6383119104497018,"ci_90_upper":0.7958542644820409,"ci_75_lower":0.6629741212447698,"ci_75_upper":0.7731832367673632,"ci_50_lower":0.6870736725133612,"ci_50_upper":0.7569088629214092},"16384":{"avg_score":0.5464779036032269,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4873595725256282,"median_score":0.9956421256065369,"q1_score":0.025198862422257662,"q3_score":1,"avg_token_efficiency":0.9994534373893261,"is_incomplete":false,"total_prompt_cost_bin":0.07748799999999995,"total_cache_read_cost_bin":0.20460799999999996,"total_completion_cost_bin":0.23066699999999996,"total_reasoning_cost_bin":0.14806499999999997,"total_cost_bin":0.6608279999999999,"total_on_demand_input_cost_bin":1.1005280000000002,"ci_99_lower":0.401347092737371,"ci_99_upper":0.6899894354498392,"ci_98_lower":0.42096783576247976,"ci_98_upper":0.6707032194456551,"ci_95_lower":0.43595243368530645,"ci_95_upper":0.6472419457256909,"ci_90_lower":0.4589418578521872,"ci_90_upper":0.6342017712697271,"ci_75_lower":0.48586695305275496,"ci_75_upper":0.6094966962537002,"ci_50_lower":0.5108671419722668,"ci_50_upper":0.5836849919789528},"32768":{"avg_score":0.3562173860726248,"min_score":0.0027758500073105097,"max_score":1,"n_tests":136,"stddev":0.452315437014901,"median_score":0.04779568128287792,"q1_score":0.02614011289551854,"q3_score":1,"avg_token_efficiency":1.0185751251342625,"is_incomplete":false,"total_prompt_cost_bin":0.3314000000000003,"total_cache_read_cost_bin":0.7235455999999996,"total_completion_cost_bin":0.39105599999999996,"total_reasoning_cost_bin":0.22676400000000005,"total_cost_bin":1.6727655999999997,"total_on_demand_input_cost_bin":3.949127999999999,"ci_99_lower":0.2604631126965037,"ci_99_upper":0.4610283376510779,"ci_98_lower":0.2645423583148102,"ci_98_upper":0.45375751701930045,"ci_95_lower":0.2798294825784123,"ci_95_upper":0.4316528211955515,"ci_90_lower":0.2908838057939326,"ci_90_upper":0.42144523527086303,"ci_75_lower":0.31181873326311366,"ci_75_upper":0.4038482377616353,"ci_50_lower":0.3289264836536912,"ci_50_upper":0.38219406514861687},"65536":{"avg_score":0.3240221151205547,"min_score":0.00469667324796319,"max_score":1,"n_tests":85,"stddev":0.45110062434130016,"median_score":0.033041466027498245,"q1_score":0.02079395018517971,"q3_score":1,"avg_token_efficiency":0.9957962005205167,"is_incomplete":false,"total_prompt_cost_bin":0.06901300000000005,"total_cache_read_cost_bin":1.0935167999999986,"total_completion_cost_bin":0.244551,"total_reasoning_cost_bin":0.14296499999999993,"total_cost_bin":1.5500457999999986,"total_on_demand_input_cost_bin":5.5365969999999995,"ci_99_lower":0.20765353299907466,"ci_99_upper":0.4605229191072559,"ci_98_lower":0.2187571222257307,"ci_98_upper":0.449707940354505,"ci_95_lower":0.23154603562605403,"ci_95_upper":0.4177981387191067,"ci_90_lower":0.24454517315196642,"ci_90_upper":0.4045555971395772,"ci_75_lower":0.2662953198825776,"ci_75_upper":0.38035379324163143,"ci_50_lower":0.28931905498509025,"ci_50_upper":0.35736495216157943},"131072":{"avg_score":0.2636807493603273,"min_score":0.0051701851189136505,"max_score":1,"n_tests":103,"stddev":0.41771690483108526,"median_score":0.03178483992815018,"q1_score":0.020531407557427883,"q3_score":0.08674989268183708,"avg_token_efficiency":0.9925503401278392,"is_incomplete":false,"total_prompt_cost_bin":0.13407800000000009,"total_cache_read_cost_bin":2.649062399999999,"total_completion_cost_bin":0.264168,"total_reasoning_cost_bin":0.13391099999999997,"total_cost_bin":3.1812193999999994,"total_on_demand_input_cost_bin":13.379389999999992,"ci_99_lower":0.16708554931683467,"ci_99_upper":0.36548460979751557,"ci_98_lower":0.1771932231568397,"ci_98_upper":0.35891386687665167,"ci_95_lower":0.18875594541274499,"ci_95_upper":0.3397769507615221,"ci_90_lower":0.19784717485059994,"ci_90_upper":0.3298415381793674,"ci_75_lower":0.21651316786329577,"ci_75_upper":0.31001504787946915,"ci_50_lower":0.23593685624881386,"ci_50_upper":0.2910131950336011},"262144":{"avg_score":0.21520005245131593,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.3825685571473654,"median_score":0.031240960583090782,"q1_score":0.020455602556467056,"q3_score":0.051304347813129425,"avg_token_efficiency":0.9987398643277824,"is_incomplete":false,"total_prompt_cost_bin":1.0585319999999983,"total_cache_read_cost_bin":14.53962240000002,"total_completion_cost_bin":0.37554,"total_reasoning_cost_bin":0.213954,"total_cost_bin":16.18764840000002,"total_on_demand_input_cost_bin":73.768812,"ci_99_lower":0.1398202591712078,"ci_99_upper":0.2992267983558039,"ci_98_lower":0.14632267022713463,"ci_98_upper":0.28991724965182386,"ci_95_lower":0.1535423577256197,"ci_95_upper":0.2775143219980391,"ci_90_lower":0.16442100710750734,"ci_90_upper":0.26863301850340154,"ci_75_lower":0.17959898318443118,"ci_75_upper":0.2510961064675011,"ci_50_lower":0.19235065227608916,"ci_50_upper":0.23482134138276894},"524288":{"avg_score":0.0983991212333884,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.23840727831294486,"median_score":0.028858046978712082,"q1_score":0.01863128039985895,"q3_score":0.04424487706273794,"avg_token_efficiency":0.9940094596104204,"is_incomplete":false,"total_prompt_cost_bin":2.0955480000000066,"total_cache_read_cost_bin":48.761856000000115,"total_completion_cost_bin":0.6827249999999999,"total_reasoning_cost_bin":0.4276920000000001,"total_cost_bin":51.96782100000012,"total_on_demand_input_cost_bin":245.9163320000003,"ci_99_lower":0.064036699964214,"ci_99_upper":0.14320998895309583,"ci_98_lower":0.06599449314168157,"ci_98_upper":0.13737756508428176,"ci_95_lower":0.07043389446606221,"ci_95_upper":0.1305974858294052,"ci_90_lower":0.07327294397647335,"ci_90_upper":0.12531367096748058,"ci_75_lower":0.08084825810116038,"ci_75_upper":0.11629219589125014,"ci_50_lower":0.08750218208889098,"ci_50_upper":0.10814913607082109}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2-Pro","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":0.000001,"completion":0.000003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-18T19:54:03.000Z","api_updated_at":"2026-04-25T12:33:03.020Z","latest_run_timestamp":"2026-04-20T06:44:42.695Z"},{"model_slug":"openai/gpt-5.4-nano","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.1317005202324501,"auc_128k":0.4250411562338859,"cum_avg_1m":0.37937974923875295,"cum_avg_128k":0.5410265815607851,"avg_score_overall":0.4877523261876516,"avg_response_time_ms":97915.8221304829,"total_prompt_cost":0.3069561999999998,"total_cache_read_cost":1.0374246399999998,"total_completion_cost":9.799121250000002,"total_reasoning_cost":9.517352500000003,"total_prompt_cost_128k":0.19173079999999992,"total_completion_cost_128k":8.370293750000002,"total_on_demand_input_cost":10.681202599999995,"total_on_demand_input_cost_128k":4.814373999999997,"total_runs":607,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9810874232763337},"bin_metrics":{"8192":{"avg_score":0.8572966862935573,"min_score":0.01620626077055931,"max_score":1,"n_tests":80,"stddev":0.3407233999120131,"median_score":1,"q1_score":0.9997944980859756,"q3_score":1,"avg_token_efficiency":0.9759044449598389,"is_incomplete":false,"total_prompt_cost_bin":0.0209456,"total_cache_read_cost_bin":0.008289280000000005,"total_completion_cost_bin":0.34623499999999996,"total_reasoning_cost_bin":0.31170125000000004,"total_cost_bin":0.6871711300000001,"total_on_demand_input_cost_bin":0.10383839999999991,"ci_99_lower":0.7489213195465272,"ci_99_upper":0.9523857695827028,"ci_98_lower":0.7648772518592887,"ci_98_upper":0.9389231349816546,"ci_95_lower":0.7765127198904519,"ci_95_upper":0.9265014787047403,"ci_90_lower":0.7915518880984747,"ci_90_upper":0.9158574714441785,"ci_75_lower":0.8103215481998632,"ci_75_upper":0.9019942690210883,"ci_50_lower":0.8324964822386391,"ci_50_upper":0.8813881520356517},"16384":{"avg_score":0.6463563876808621,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.4711784800774626,"median_score":0.9996964037418365,"q1_score":0.027342600282281637,"q3_score":1,"avg_token_efficiency":0.9670687393188716,"is_incomplete":false,"total_prompt_cost_bin":0.01996,"total_cache_read_cost_bin":0.019317760000000007,"total_completion_cost_bin":1.0469050000000002,"total_reasoning_cost_bin":1.01330125,"total_cost_bin":2.0994840100000003,"total_on_demand_input_cost_bin":0.21313760000000004,"ci_99_lower":0.5107059933024576,"ci_99_upper":0.7692692606477648,"ci_98_lower":0.5126782879416133,"ci_98_upper":0.7561853023162112,"ci_95_lower":0.5475458502105903,"ci_95_upper":0.7435046374666854,"ci_90_lower":0.5609330753254471,"ci_90_upper":0.7313506267621415,"ci_75_lower":0.5855067833996145,"ci_75_upper":0.706954873437644,"ci_50_lower":0.6096120654576226,"ci_50_upper":0.6825803449115484},"32768":{"avg_score":0.5410959730900632,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.4816345438936098,"median_score":0.9989043176174164,"q1_score":0.03221877012401819,"q3_score":1,"avg_token_efficiency":0.9952100768549628,"is_incomplete":false,"total_prompt_cost_bin":0.03132479999999995,"total_cache_read_cost_bin":0.07406592000000005,"total_completion_cost_bin":2.6477225000000004,"total_reasoning_cost_bin":2.5808100000000005,"total_cost_bin":5.333923220000001,"total_on_demand_input_cost_bin":0.7719840000000008,"ci_99_lower":0.4350751070969786,"ci_99_upper":0.6472660024540572,"ci_98_lower":0.4402359742866761,"ci_98_upper":0.6348261818110683,"ci_95_lower":0.4602497325373834,"ci_95_upper":0.6223696579760786,"ci_90_lower":0.4762526373749578,"ci_90_upper":0.6058763826645993,"ci_75_lower":0.49652974194802535,"ci_75_upper":0.5907348902467621,"ci_50_lower":0.5156076514643192,"ci_50_upper":0.5691279171609066},"65536":{"avg_score":0.3563055283061283,"min_score":0.0029607696924358606,"max_score":1,"n_tests":85,"stddev":0.4620539053151481,"median_score":0.034682080149650574,"q1_score":0.021835196763277054,"q3_score":0.998400866985321,"avg_token_efficiency":0.9776435846759911,"is_incomplete":false,"total_prompt_cost_bin":0.06164900000000005,"total_cache_read_cost_bin":0.10254847999999987,"total_completion_cost_bin":2.0563825,"total_reasoning_cost_bin":2.01456625,"total_cost_bin":4.23514623,"total_on_demand_input_cost_bin":1.087133799999999,"ci_99_lower":0.2409263133406776,"ci_99_upper":0.4830188507903778,"ci_98_lower":0.24338230734124014,"ci_98_upper":0.47178498291835075,"ci_95_lower":0.2633524133132168,"ci_95_upper":0.45908707106335306,"ci_90_lower":0.2755609427552725,"ci_90_upper":0.4393996059695969,"ci_75_lower":0.29750014356275856,"ci_75_upper":0.4136745506147717,"ci_50_lower":0.3221004641929861,"ci_50_upper":0.3901843946940704},"131072":{"avg_score":0.3040783324333143,"min_score":0.0054580895230174065,"max_score":1,"n_tests":103,"stddev":0.43706275781633885,"median_score":0.03832833096385002,"q1_score":0.02298072073608637,"q3_score":0.9910716116428375,"avg_token_efficiency":0.9786043714640693,"is_incomplete":false,"total_prompt_cost_bin":0.057851399999999956,"total_cache_read_cost_bin":0.2580428800000003,"total_completion_cost_bin":2.273048750000001,"total_reasoning_cost_bin":2.2201187500000006,"total_cost_bin":4.809061780000002,"total_on_demand_input_cost_bin":2.6382801999999974,"ci_99_lower":0.1991854936180374,"ci_99_upper":0.4115839283624935,"ci_98_lower":0.21555910103307874,"ci_98_upper":0.40002337331051746,"ci_95_lower":0.225470449327767,"ci_95_upper":0.3876124884857507,"ci_90_lower":0.2354034517170608,"ci_90_upper":0.37436057994847444,"ci_75_lower":0.25545338033223797,"ci_75_upper":0.3510386060247451,"ci_50_lower":0.27510376051814267,"ci_50_upper":0.3313364364665671},"262144":{"avg_score":0.32990508610609826,"min_score":0.005610098131000996,"max_score":1,"n_tests":123,"stddev":0.4383126888468222,"median_score":0.04518134891986847,"q1_score":0.028377664275467396,"q3_score":0.9865490794181824,"avg_token_efficiency":0.9824202095601985,"is_incomplete":false,"total_prompt_cost_bin":0.11522539999999987,"total_cache_read_cost_bin":0.5751603199999996,"total_completion_cost_bin":1.4288274999999997,"total_reasoning_cost_bin":1.3768550000000004,"total_cost_bin":3.4960682199999993,"total_on_demand_input_cost_bin":5.866828599999997,"ci_99_lower":0.23250385199325174,"ci_99_upper":0.42944405696366156,"ci_98_lower":0.24530874891789645,"ci_98_upper":0.4261991579552007,"ci_95_lower":0.2552976027733033,"ci_95_upper":0.4096207243076154,"ci_90_lower":0.26533257160297924,"ci_90_upper":0.3947753309677317,"ci_75_lower":0.28421482038911344,"ci_75_upper":0.37390166902876787,"ci_50_lower":0.3020404963735188,"ci_50_upper":0.3554068370476153}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Nano","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":2e-7,"completion":0.00000125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:47.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T18:18:55.157Z"},{"model_slug":"qwen/qwen3.6-35b-a3b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.12670241015582892,"auc_128k":0.7563386874416677,"cum_avg_1m":0.5223973886581285,"cum_avg_128k":0.8358358218530055,"avg_score_overall":0.8353234381041651,"avg_response_time_ms":78406.35671971849,"total_prompt_cost":0.17830903999999997,"total_cache_read_cost":3.3238867200000026,"total_completion_cost":2.0848829999999996,"total_reasoning_cost":1.9831720000000002,"total_prompt_cost_128k":0.17830903999999997,"total_completion_cost_128k":2.0848829999999996,"total_on_demand_input_cost":3.5021957599999984,"total_on_demand_input_cost_128k":3.5021957599999984,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207081000672178},"bin_metrics":{"8192":{"avg_score":0.987980803148821,"min_score":0.03866809979081154,"max_score":1,"n_tests":80,"stddev":0.10747988781409927,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222288,"is_incomplete":false,"total_prompt_cost_bin":0.01767808,"total_cache_read_cost_bin":0.05839680000000008,"total_completion_cost_bin":0.30501300000000003,"total_reasoning_cost_bin":0.29492399999999996,"total_cost_bin":0.6760118800000001,"total_on_demand_input_cost_bin":0.07607488000000004,"ci_99_lower":0.9519283087924123,"ci_99_upper":1,"ci_98_lower":0.9519308568909765,"ci_98_upper":1,"ci_95_lower":0.9639449575450272,"ci_95_upper":1,"ci_90_lower":0.9639475056435913,"ci_90_upper":1,"ci_75_lower":0.9759638358838856,"ci_75_upper":1,"ci_50_lower":0.9759667024947702,"ci_50_upper":0.9999974519014359},"16384":{"avg_score":0.9513524346752092,"min_score":0.025401761755347252,"max_score":1,"n_tests":80,"stddev":0.2133276844680747,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0159704488335684,"is_incomplete":false,"total_prompt_cost_bin":0.016154880000000003,"total_cache_read_cost_bin":0.14074368000000004,"total_completion_cost_bin":0.3481179999999999,"total_reasoning_cost_bin":0.3249269999999999,"total_cost_bin":0.8299435599999998,"total_on_demand_input_cost_bin":0.1568985600000001,"ci_99_lower":0.8783870404527988,"ci_99_upper":0.9999869080632925,"ci_98_lower":0.8902931368669961,"ci_98_upper":0.9999868422746658,"ci_95_lower":0.8906607555219671,"ci_95_upper":0.9878797410055995,"ci_90_lower":0.9027504238951951,"ci_90_upper":0.9878243035869674,"ci_75_lower":0.9269696781644597,"ci_75_upper":0.9756981092272327,"ci_50_lower":0.9391392099671065,"ci_50_upper":0.963553347857669},"32768":{"avg_score":0.9428275086889591,"min_score":0.008643042296171188,"max_score":1,"n_tests":136,"stddev":0.22952339848620226,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0256830410679845,"is_incomplete":false,"total_prompt_cost_bin":0.025092479999999993,"total_cache_read_cost_bin":0.5316326400000001,"total_completion_cost_bin":0.620362,"total_reasoning_cost_bin":0.5888960000000002,"total_cost_bin":1.76598312,"total_on_demand_input_cost_bin":0.55672512,"ci_99_lower":0.8854136371296858,"ci_99_upper":0.9858235342393165,"ci_98_lower":0.8926736418509802,"ci_98_upper":0.9855369450414883,"ci_95_lower":0.8996702193850925,"ci_95_upper":0.9785434215083061,"ci_90_lower":0.9070864595301613,"ci_90_upper":0.9716940619558206,"ci_75_lower":0.920865124105202,"ci_75_upper":0.9644695208527777,"ci_50_lower":0.9286324483178117,"ci_50_upper":0.9569820057744544},"65536":{"avg_score":0.7037526404813809,"min_score":0.008361839689314365,"max_score":1,"n_tests":85,"stddev":0.4490723799240563,"median_score":1,"q1_score":0.03862803801894188,"q3_score":1,"avg_token_efficiency":1.020959825517544,"is_incomplete":false,"total_prompt_cost_bin":0.049597240000000015,"total_cache_read_cost_bin":0.7451135999999988,"total_completion_cost_bin":0.3690309999999999,"total_reasoning_cost_bin":0.353108,"total_cost_bin":1.5168498399999988,"total_on_demand_input_cost_bin":0.7947108399999997,"ci_99_lower":0.5769672514041337,"ci_99_upper":0.8171529620819012,"ci_98_lower":0.588186967127573,"ci_98_upper":0.8085452639249099,"ci_95_lower":0.6103298720657169,"ci_95_upper":0.7946359723776251,"ci_90_lower":0.6221744343912339,"ci_90_upper":0.7824638507504235,"ci_75_lower":0.645438823062817,"ci_75_upper":0.760542207026361,"ci_50_lower":0.6693012374562813,"ci_50_upper":0.736761643803295},"131072":{"avg_score":0.5932657222706571,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.48247057567443097,"median_score":1,"q1_score":0.02950091753154993,"q3_score":1,"avg_token_efficiency":1.0162215936488594,"is_incomplete":false,"total_prompt_cost_bin":0.06978635999999995,"total_cache_read_cost_bin":1.8480000000000036,"total_completion_cost_bin":0.442359,"total_reasoning_cost_bin":0.42131700000000005,"total_cost_bin":2.781462360000004,"total_on_demand_input_cost_bin":1.917786359999999,"ci_99_lower":0.4702927629959327,"ci_99_upper":0.7152184628472364,"ci_98_lower":0.4875617972885268,"ci_98_upper":0.7056024106345998,"ci_95_lower":0.5070158894983058,"ci_95_upper":0.6874101246985183,"ci_90_lower":0.5171068633477051,"ci_90_upper":0.6768946038455097,"ci_75_lower":0.5365314579385703,"ci_75_upper":0.6502844919021088,"ci_50_lower":0.5638608758844146,"ci_50_upper":0.6297863462659558}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 35B A3B","provider_name":"Qwen","run_provider_names":["Parasail"],"pricing_metadata":{"prompt":1.4e-7,"completion":0.000001,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-27T03:24:15.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T05:40:18.386Z"},{"model_slug":"qwen/qwen3.6-27b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.12366932741052461,"auc_128k":0.7452016744149647,"cum_avg_1m":0.5143572658590312,"cum_avg_128k":0.8229716253744499,"avg_score_overall":0.821720921774087,"avg_response_time_ms":71915.08209581612,"total_prompt_cost":7.504705200000005,"total_cache_read_cost":0,"total_completion_cost":4.144137999999999,"total_reasoning_cost":3.757814,"total_prompt_cost_128k":7.504705200000005,"total_completion_cost_128k":4.144137999999999,"total_on_demand_input_cost":7.504705200000005,"total_on_demand_input_cost_128k":7.504705200000005,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207081000672176},"bin_metrics":{"8192":{"avg_score":0.9756600440014154,"min_score":0.01769087463617325,"max_score":1,"n_tests":80,"stddev":0.15293543870739512,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222277,"is_incomplete":false,"total_prompt_cost_bin":0.16301760000000007,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.6406680000000001,"total_reasoning_cost_bin":0.5829539999999999,"total_cost_bin":1.3866396,"total_on_demand_input_cost_bin":0.16301760000000007,"ci_99_lower":0.9267590643621516,"ci_99_upper":1,"ci_98_lower":0.9271989755807445,"ci_98_upper":1,"ci_95_lower":0.9390409382432698,"ci_95_upper":0.9999975472688675,"ci_90_lower":0.9394948820583522,"ci_90_upper":0.9999973110854625,"ci_75_lower":0.9515483796130866,"ci_75_upper":0.9999873048625887,"ci_50_lower":0.9635989520698786,"ci_50_upper":0.9879337664227933},"16384":{"avg_score":0.9396397956646979,"min_score":0.02601809985935688,"max_score":1,"n_tests":80,"stddev":0.23511621506156816,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0159704488335684,"is_incomplete":false,"total_prompt_cost_bin":0.33621120000000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.6519739999999997,"total_reasoning_cost_bin":0.5946699999999999,"total_cost_bin":1.5828551999999996,"total_on_demand_input_cost_bin":0.33621120000000004,"ci_99_lower":0.8556200948036277,"ci_99_upper":0.9881284777501131,"ci_98_lower":0.867176915461896,"ci_98_upper":0.9880394016969949,"ci_95_lower":0.8790309443708975,"ci_95_upper":0.9878482331847772,"ci_90_lower":0.8912649267818779,"ci_90_upper":0.9760282587446272,"ci_75_lower":0.9036158154718578,"ci_75_upper":0.9640576265053824,"ci_50_lower":0.927260967931943,"ci_50_upper":0.9520906535210089},"32768":{"avg_score":0.9282064854273545,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.2538147929934466,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0256830410679847,"is_incomplete":false,"total_prompt_cost_bin":1.1929824000000007,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.2273559999999997,"total_reasoning_cost_bin":1.1163280000000002,"total_cost_bin":3.5366664000000005,"total_on_demand_input_cost_bin":1.1929824000000007,"ci_99_lower":0.8650639848391052,"ci_99_upper":0.9776719127168968,"ci_98_lower":0.8711964538255672,"ci_98_upper":0.9715581740647116,"ci_95_lower":0.8789221945488809,"ci_95_upper":0.9705227628812495,"ci_90_lower":0.8912100075929844,"ci_90_upper":0.9636914625258857,"ci_75_lower":0.900217794253609,"ci_75_upper":0.9557426281019562,"ci_50_lower":0.9137725318621104,"ci_50_upper":0.9427071006537197},"65536":{"avg_score":0.7053543686647625,"min_score":0.010593811050057411,"max_score":1,"n_tests":85,"stddev":0.44678321780539854,"median_score":1,"q1_score":0.04198895022273064,"q3_score":1,"avg_token_efficiency":1.0209598255175434,"is_incomplete":false,"total_prompt_cost_bin":1.7029518,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.7585159999999999,"total_reasoning_cost_bin":0.68705,"total_cost_bin":3.1485178,"total_on_demand_input_cost_bin":1.7029518,"ci_99_lower":0.5791942905314705,"ci_99_upper":0.8180203456750469,"ci_98_lower":0.5902567353099584,"ci_98_upper":0.809526103015989,"ci_95_lower":0.6120530618787469,"ci_95_upper":0.7958902403531486,"ci_90_lower":0.6242688103184543,"ci_90_upper":0.783646477528793,"ci_75_lower":0.6473507842049002,"ci_75_upper":0.7616058663547257,"ci_50_lower":0.6711684071732794,"ci_50_upper":0.738070561928565},"131072":{"avg_score":0.5659974331140193,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4855072836544222,"median_score":1,"q1_score":0.026360411196947098,"q3_score":1,"avg_token_efficiency":1.0162215936488599,"is_incomplete":false,"total_prompt_cost_bin":4.109542200000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.8656240000000001,"total_reasoning_cost_bin":0.7768120000000001,"total_cost_bin":5.751978200000005,"total_on_demand_input_cost_bin":4.109542200000004,"ci_99_lower":0.44327319339627524,"ci_99_upper":0.6976280687042418,"ci_98_lower":0.453875554764476,"ci_98_upper":0.6702244650081107,"ci_95_lower":0.47221579201670943,"ci_95_upper":0.6605275997837771,"ci_90_lower":0.4904396495527533,"ci_90_upper":0.6437732978508064,"ci_75_lower":0.5102056610031795,"ci_75_upper":0.6229644058180692,"ci_50_lower":0.5319259347754657,"ci_50_upper":0.6032603479999804}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 27B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":3.2e-7,"completion":0.0000032,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-27T01:57:44.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T06:09:14.436Z"},{"model_slug":"xiaomi/mimo-v2-pro","reasoning_mode":null,"overall_metrics":{"auc_1m":0.12281029454614495,"auc_128k":0.2703792723007373,"cum_avg_1m":0.2296124547469243,"cum_avg_128k":0.3023491194282035,"avg_score_overall":0.2278693515405683,"avg_response_time_ms":21033.29565940086,"total_prompt_cost":3.4468590000000026,"total_cache_read_cost":68.14565120000013,"total_completion_cost":0.9537599999999999,"total_reasoning_cost":0,"total_prompt_cost_128k":0.8151149999999993,"total_completion_cost_128k":0.5551470000000001,"total_on_demand_input_cost":344.19962300000014,"total_on_demand_input_cost_128k":24.512971,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0024473735842667},"bin_metrics":{"8192":{"avg_score":0.4616809645202011,"min_score":0.009021200239658356,"max_score":1,"n_tests":80,"stddev":0.47877772587493367,"median_score":0.07080858200788498,"q1_score":0.026283251587301493,"q3_score":1,"avg_token_efficiency":1.0289117704855872,"is_incomplete":false,"total_prompt_cost_bin":0.10063199999999996,"total_cache_read_cost_bin":0.08917759999999997,"total_completion_cost_bin":0.08520900000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.2750185999999999,"total_on_demand_input_cost_bin":0.5465199999999999,"ci_99_lower":0.3253591479611351,"ci_99_upper":0.591747161482228,"ci_98_lower":0.3362901356194634,"ci_98_upper":0.5791236453154125,"ci_95_lower":0.3484523854823783,"ci_95_upper":0.5663243812718429,"ci_90_lower":0.36625335907447154,"ci_90_upper":0.5527822012384422,"ci_75_lower":0.4002823173388606,"ci_75_upper":0.5243244974728442,"ci_50_lower":0.4250491078128107,"ci_50_upper":0.4983376792399213},"16384":{"avg_score":0.2867460228386335,"min_score":0.008738393895328045,"max_score":1,"n_tests":80,"stddev":0.4279619369741107,"median_score":0.03734876401722431,"q1_score":0.025638720951974392,"q3_score":0.9871983528137207,"avg_token_efficiency":0.999602464394221,"is_incomplete":false,"total_prompt_cost_bin":0.08583999999999999,"total_cache_read_cost_bin":0.2029695999999999,"total_completion_cost_bin":0.08274000000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.3715495999999999,"total_on_demand_input_cost_bin":1.1006879999999994,"ci_99_lower":0.17936940888443495,"ci_99_upper":0.40838738908723465,"ci_98_lower":0.1876987883509137,"ci_98_upper":0.39821392977004866,"ci_95_lower":0.19987317104678368,"ci_95_upper":0.3834239853679901,"ci_90_lower":0.21315967903181446,"ci_90_upper":0.36084805215417876,"ci_75_lower":0.2368937179679051,"ci_75_upper":0.33681832198926714,"ci_50_lower":0.2531739300728077,"ci_50_upper":0.31270665711490436},"32768":{"avg_score":0.2642928681076121,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.40714329299163915,"median_score":0.03598370403051376,"q1_score":0.02522915694862604,"q3_score":0.16479674354195595,"avg_token_efficiency":1.0186469433242438,"is_incomplete":false,"total_prompt_cost_bin":0.4249840000000003,"total_cache_read_cost_bin":0.7048831999999992,"total_completion_cost_bin":0.161604,"total_reasoning_cost_bin":0,"total_cost_bin":1.2914711999999995,"total_on_demand_input_cost_bin":3.9493999999999976,"ci_99_lower":0.1790242431020734,"ci_99_upper":0.3473033214148874,"ci_98_lower":0.1847200786570936,"ci_98_upper":0.3433599593360936,"ci_95_lower":0.19872998606845027,"ci_95_upper":0.33184068867429983,"ci_90_lower":0.20638674542287308,"ci_90_upper":0.32014202127568697,"ci_75_lower":0.22281256428173304,"ci_75_upper":0.3033531192734646,"ci_50_lower":0.2400784203542999,"ci_50_upper":0.2868930065374383},"65536":{"avg_score":0.3028739984859439,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4397358048123305,"median_score":0.03145589679479599,"q1_score":0.020283976569771767,"q3_score":1,"avg_token_efficiency":0.9958267762250804,"is_incomplete":false,"total_prompt_cost_bin":0.0693749999999998,"total_cache_read_cost_bin":1.0934783999999986,"total_completion_cost_bin":0.09858000000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.2614333999999985,"total_on_demand_input_cost_bin":5.536767000000005,"ci_99_lower":0.1884548596910146,"ci_99_upper":0.43885144565199663,"ci_98_lower":0.1995581109846997,"ci_98_upper":0.41808560716919596,"ci_95_lower":0.21162767202867305,"ci_95_upper":0.40303835829336415,"ci_90_lower":0.22452580642174272,"ci_90_upper":0.38168180937550084,"ci_75_lower":0.2464251956669614,"ci_75_upper":0.358246174528647,"ci_50_lower":0.26812466832480447,"ci_50_upper":0.3358406277976053},"131072":{"avg_score":0.196151743188627,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.3719192666081988,"median_score":0.026934010908007622,"q1_score":0.019082140177488327,"q3_score":0.04420059360563755,"avg_token_efficiency":0.9925656222423526,"is_incomplete":false,"total_prompt_cost_bin":0.1342839999999993,"total_cache_read_cost_bin":2.649062399999999,"total_completion_cost_bin":0.12701400000000002,"total_reasoning_cost_bin":0,"total_cost_bin":2.9103603999999983,"total_on_demand_input_cost_bin":13.379596000000001,"ci_99_lower":0.11066453134856928,"ci_99_upper":0.28966185007224005,"ci_98_lower":0.11851836181971363,"ci_98_upper":0.28231843919769034,"ci_95_lower":0.13089215293412496,"ci_95_upper":0.263538209298734,"ci_90_lower":0.13929628079205533,"ci_90_upper":0.253619510151369,"ci_75_lower":0.15740312155996206,"ci_75_upper":0.23437831451516208,"ci_50_lower":0.169748535690585,"ci_50_upper":0.21590523368190723},"262144":{"avg_score":0.23480789927957277,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.3937973821843285,"median_score":0.034300126135349274,"q1_score":0.019999999552965164,"q3_score":0.06778087466955185,"avg_token_efficiency":0.9987475002008315,"is_incomplete":false,"total_prompt_cost_bin":0.5357209999999998,"total_cache_read_cost_bin":14.64422400000002,"total_completion_cost_bin":0.160962,"total_reasoning_cost_bin":0,"total_cost_bin":15.340907000000021,"total_on_demand_input_cost_bin":73.76937599999998,"ci_99_lower":0.1610837091917605,"ci_99_upper":0.3175406710058863,"ci_98_lower":0.16456010511001332,"ci_98_upper":0.3131954334962763,"ci_95_lower":0.1715126220843629,"ci_95_upper":0.29813651847937933,"ci_90_lower":0.18263947744052925,"ci_90_upper":0.28586506785838106,"ci_75_lower":0.19661933999960243,"ci_75_upper":0.2721098832400509,"ci_50_lower":0.21191207375311366,"ci_50_upper":0.25530625031088605},"524288":{"avg_score":0.09034614155480393,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.2273510069771997,"median_score":0.026642514392733574,"q1_score":0.016706922091543674,"q3_score":0.04393212869763374,"avg_token_efficiency":0.9940132753184772,"is_incomplete":false,"total_prompt_cost_bin":2.0960230000000037,"total_cache_read_cost_bin":48.761856000000115,"total_completion_cost_bin":0.23765099999999983,"total_reasoning_cost_bin":0,"total_cost_bin":51.09553000000012,"total_on_demand_input_cost_bin":245.91727600000016,"ci_99_lower":0.05689116131739742,"ci_99_upper":0.13499617140458736,"ci_98_lower":0.05956635000738632,"ci_98_upper":0.128469248067191,"ci_95_lower":0.06328614056049338,"ci_95_upper":0.12037426273127494,"ci_90_lower":0.06628289578331564,"ci_90_upper":0.11635613398531736,"ci_75_lower":0.07383970533774735,"ci_75_upper":0.10760472988578716,"ci_50_lower":0.07960605551026242,"ci_50_upper":0.09969709330748705}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2-Pro","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":0.000001,"completion":0.000003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-18T19:54:03.000Z","api_updated_at":"2026-04-25T12:33:03.020Z","latest_run_timestamp":"2026-04-20T05:56:32.583Z"},{"model_slug":"openai/gpt-5.4-mini","reasoning_mode":"high","overall_metrics":{"auc_1m":0.12008397734214986,"auc_128k":0.40448458040283,"cum_avg_1m":0.3518538478524049,"cum_avg_128k":0.5064312122830111,"avg_score_overall":0.4478941197756678,"avg_response_time_ms":32820.478887304685,"total_prompt_cost":1.8859537499999999,"total_cache_read_cost":4.297977599999996,"total_completion_cost":12.7786005,"total_reasoning_cost":11.708221499999999,"total_prompt_cost_128k":0.66789075,"total_completion_cost_128k":9.030051,"total_on_demand_input_cost":44.86572974999997,"total_on_demand_input_cost_128k":18.040434749999985,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061856180517},"bin_metrics":{"8192":{"avg_score":0.6879830994410441,"min_score":0.014189189299941063,"max_score":1,"n_tests":80,"stddev":0.4521473114490695,"median_score":1,"q1_score":0.05346966441720724,"q3_score":1,"avg_token_efficiency":0.9759044449598393,"is_incomplete":false,"total_prompt_cost_bin":0.05109,"total_cache_read_cost_bin":0.0338304,"total_completion_cost_bin":0.5160644999999999,"total_reasoning_cost_bin":0.3915404999999998,"total_cost_bin":0.9925253999999997,"total_on_demand_input_cost_bin":0.38939399999999996,"ci_99_lower":0.5438672491440083,"ci_99_upper":0.8085253233521944,"ci_98_lower":0.5666201382018625,"ci_98_upper":0.7965061235392932,"ci_95_lower":0.5911620642041089,"ci_95_upper":0.7831369584740605,"ci_90_lower":0.6041329727414996,"ci_90_upper":0.7607943914923817,"ci_75_lower":0.6288015821308364,"ci_75_upper":0.7461466859153006,"ci_50_lower":0.6524119384004735,"ci_50_upper":0.723461920482805},"16384":{"avg_score":0.7083212863770314,"min_score":0.007738607004284859,"max_score":1,"n_tests":80,"stddev":0.44247059785538073,"median_score":1,"q1_score":0.047395993024110794,"q3_score":1,"avg_token_efficiency":0.9670687393188718,"is_incomplete":false,"total_prompt_cost_bin":0.07734600000000007,"total_cache_read_cost_bin":0.07219200000000009,"total_completion_cost_bin":1.0227914999999996,"total_reasoning_cost_bin":0.9024480000000002,"total_cost_bin":2.0747775,"total_on_demand_input_cost_bin":0.7992660000000001,"ci_99_lower":0.5730263153092819,"ci_99_upper":0.826339188461774,"ci_98_lower":0.5863432629852322,"ci_98_upper":0.8106093108705245,"ci_95_lower":0.6106980129162548,"ci_95_upper":0.7969914230896392,"ci_90_lower":0.6227362279564841,"ci_90_upper":0.787271819912712,"ci_75_lower":0.6500249145203271,"ci_75_upper":0.7589541026900406,"ci_50_lower":0.6735572844772832,"ci_50_upper":0.7399870139284759},"32768":{"avg_score":0.5181713503826519,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.4837810109494695,"median_score":0.5554050020873547,"q1_score":0.03183934139087796,"q3_score":1,"avg_token_efficiency":0.9952100768549632,"is_incomplete":false,"total_prompt_cost_bin":0.13167599999999996,"total_cache_read_cost_bin":0.2763264000000007,"total_completion_cost_bin":2.398554,"total_reasoning_cost_bin":2.158677,"total_cost_bin":4.965233400000001,"total_on_demand_input_cost_bin":2.8949399999999983,"ci_99_lower":0.41224379026387503,"ci_99_upper":0.6247198214688209,"ci_98_lower":0.4209180204001913,"ci_98_upper":0.6172056504728629,"ci_95_lower":0.43879534028100725,"ci_95_upper":0.5954228350789087,"ci_90_lower":0.44942133076992297,"ci_90_upper":0.5830342386739658,"ci_75_lower":0.4690370130620602,"ci_75_upper":0.5676795875170842,"ci_50_lower":0.49032005816201596,"ci_50_upper":0.5467809778561145},"65536":{"avg_score":0.31777993737555604,"min_score":0.004449388012290001,"max_score":1,"n_tests":85,"stddev":0.44360728085695017,"median_score":0.03248446062207222,"q1_score":0.022587930783629417,"q3_score":0.999069333076477,"avg_token_efficiency":0.977643584675991,"is_incomplete":false,"total_prompt_cost_bin":0.20430374999999998,"total_cache_read_cost_bin":0.38724479999999933,"total_completion_cost_bin":2.1132629999999994,"total_reasoning_cost_bin":1.9586655,"total_cost_bin":4.663477049999999,"total_on_demand_input_cost_bin":4.076751749999996,"ci_99_lower":0.20426413596613224,"ci_99_upper":0.45161657760264895,"ci_98_lower":0.21039460303776844,"ci_98_upper":0.4418206583426937,"ci_95_lower":0.22657799146138133,"ci_95_upper":0.41674943550270704,"ci_90_lower":0.24311056853436372,"ci_90_upper":0.3970862047105808,"ci_75_lower":0.2598024921683485,"ci_75_upper":0.3712256701937055,"ci_50_lower":0.28239017593236093,"ci_50_upper":0.3501239552874776},"131072":{"avg_score":0.2999003878387722,"min_score":0.0057299453765153885,"max_score":1,"n_tests":103,"stddev":0.4314641962773002,"median_score":0.03775133192539215,"q1_score":0.02133385557681322,"q3_score":0.9973851442337036,"avg_token_efficiency":0.9772722316188368,"is_incomplete":false,"total_prompt_cost_bin":0.20347500000000002,"total_cache_read_cost_bin":0.9676607999999973,"total_completion_cost_bin":2.979378000000001,"total_reasoning_cost_bin":2.7808559999999996,"total_cost_bin":6.931369799999999,"total_on_demand_input_cost_bin":9.88008299999999,"ci_99_lower":0.19598234906573372,"ci_99_upper":0.4008680238686637,"ci_98_lower":0.2068108044196716,"ci_98_upper":0.3957678817588276,"ci_95_lower":0.22071327440343452,"ci_95_upper":0.3812154452207959,"ci_90_lower":0.23318987153606147,"ci_90_upper":0.37080262533289565,"ci_75_lower":0.25188775946503705,"ci_75_upper":0.3451265052070579,"ci_50_lower":0.2717523906974447,"ci_50_upper":0.3274962792310918},"262144":{"avg_score":0.28267472140418354,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.4186384629287644,"median_score":0.03981677070260048,"q1_score":0.024769196286797523,"q3_score":0.9144049882888794,"avg_token_efficiency":0.9824198316490135,"is_incomplete":false,"total_prompt_cost_bin":1.218063,"total_cache_read_cost_bin":2.560723199999998,"total_completion_cost_bin":3.7485494999999998,"total_reasoning_cost_bin":3.5160344999999986,"total_cost_bin":11.043370199999996,"total_on_demand_input_cost_bin":26.82529499999999,"ci_99_lower":0.1944487091405515,"ci_99_upper":0.37385129881305257,"ci_98_lower":0.19911906705483964,"ci_98_upper":0.3596123342050061,"ci_95_lower":0.21278359644339548,"ci_95_upper":0.351601502237716,"ci_90_lower":0.22503380533621536,"ci_90_upper":0.338559206457584,"ci_75_lower":0.24281238747371778,"ci_75_upper":0.32083820930582224,"ci_50_lower":0.25864844827447087,"ci_50_upper":0.3059092282436471}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Mini","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.0000045,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T03:17:26.965Z"},{"model_slug":"openai/gpt-5.4-mini","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.1187834999768704,"auc_128k":0.4268665033422491,"cum_avg_1m":0.35728215568591914,"cum_avg_128k":0.5212707355364916,"avg_score_overall":0.4567339139903197,"avg_response_time_ms":108537.54276034706,"total_prompt_cost":1.0746652499999998,"total_cache_read_cost":3.9159359999999963,"total_completion_cost":42.35569200000001,"total_reasoning_cost":41.327298,"total_prompt_cost_128k":0.6421627499999998,"total_completion_cost_128k":37.45952550000001,"total_on_demand_input_cost":40.234025249999995,"total_on_demand_input_cost_128k":18.040434749999985,"total_runs":608,"total_candidates_generated":null,"avg_token_efficiency_overall":0.980863889919113},"bin_metrics":{"8192":{"avg_score":0.7705279791844077,"min_score":0.012341446243226528,"max_score":1,"n_tests":80,"stddev":0.4137321539905359,"median_score":1,"q1_score":0.9986235350370407,"q3_score":1,"avg_token_efficiency":0.9759044449598395,"is_incomplete":false,"total_prompt_cost_bin":0.060690000000000015,"total_cache_read_cost_bin":0.03287040000000001,"total_completion_cost_bin":2.2025609999999993,"total_reasoning_cost_bin":2.0784914999999993,"total_cost_bin":4.374612899999999,"total_on_demand_input_cost_bin":0.3893939999999998,"ci_99_lower":0.6487373859388172,"ci_99_upper":0.8799415125459782,"ci_98_lower":0.6602859934752342,"ci_98_upper":0.8678453048374504,"ci_95_lower":0.6754473191098078,"ci_95_upper":0.8558786975426482,"ci_90_lower":0.6967147679277697,"ci_90_upper":0.8429821508191526,"ci_75_lower":0.7209248592465883,"ci_75_upper":0.8193174933054251,"ci_50_lower":0.735834010122926,"ci_50_upper":0.8052942965383408},"16384":{"avg_score":0.6246423740754835,"min_score":0.00514579750597477,"max_score":1,"n_tests":80,"stddev":0.46941899474254123,"median_score":0.9996319115161896,"q1_score":0.040001073852181435,"q3_score":1,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.060834,"total_cache_read_cost_bin":0.07384320000000011,"total_completion_cost_bin":4.756599000000001,"total_reasoning_cost_bin":4.6328084999999986,"total_cost_bin":9.5240847,"total_on_demand_input_cost_bin":0.7992659999999999,"ci_99_lower":0.4820273109322297,"ci_99_upper":0.7601429263030295,"ci_98_lower":0.49421382878371517,"ci_98_upper":0.7400235598837026,"ci_95_lower":0.5199102788988967,"ci_95_upper":0.7215629776314016,"ci_90_lower":0.539861797943595,"ci_90_upper":0.7071631991810864,"ci_75_lower":0.5649680393005837,"ci_75_upper":0.6851770608947846,"ci_50_lower":0.5889720872888574,"ci_50_upper":0.6590170170005877},"32768":{"avg_score":0.5090415390680872,"min_score":0.0067437379620969296,"max_score":1,"n_tests":136,"stddev":0.4785301293234512,"median_score":0.1750735491514206,"q1_score":0.03335666377097368,"q3_score":1,"avg_token_efficiency":0.995210076854963,"is_incomplete":false,"total_prompt_cost_bin":0.1556759999999999,"total_cache_read_cost_bin":0.2739264000000006,"total_completion_cost_bin":10.775241000000003,"total_reasoning_cost_bin":10.5372495,"total_cost_bin":21.742092900000003,"total_on_demand_input_cost_bin":2.894940000000001,"ci_99_lower":0.40795041831492734,"ci_99_upper":0.6174421539104901,"ci_98_lower":0.4149137216680752,"ci_98_upper":0.6055450507433167,"ci_95_lower":0.42966954886112085,"ci_95_upper":0.5855711319956325,"ci_90_lower":0.4425195961605128,"ci_90_upper":0.5759301545550389,"ci_75_lower":0.46488429257485275,"ci_75_upper":0.5575143536839545,"ci_50_lower":0.48245088404099296,"ci_50_upper":0.5372463458402575},"65536":{"avg_score":0.37253912040306364,"min_score":0.003488372080028057,"max_score":1,"n_tests":85,"stddev":0.4591532132568795,"median_score":0.03819805011153221,"q1_score":0.021728144958615303,"q3_score":0.9996564984321594,"avg_token_efficiency":0.9776435846759911,"is_incomplete":false,"total_prompt_cost_bin":0.1614877499999999,"total_cache_read_cost_bin":0.39152639999999933,"total_completion_cost_bin":9.206041500000003,"total_reasoning_cost_bin":9.051979500000003,"total_cost_bin":18.811035150000006,"total_on_demand_input_cost_bin":4.076751749999998,"ci_99_lower":0.24740056053025866,"ci_99_upper":0.5045533754468128,"ci_98_lower":0.2647885046847274,"ci_98_upper":0.4941246904058053,"ci_95_lower":0.27566305051064666,"ci_95_upper":0.4725081203564289,"ci_90_lower":0.29171596857335635,"ci_90_upper":0.45713881251566546,"ci_75_lower":0.3108915133251096,"ci_75_upper":0.4296325534452027,"ci_50_lower":0.33824436644174377,"ci_50_upper":0.40654801643754845},"131072":{"avg_score":0.3296026649514159,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.45069301746453494,"median_score":0.03346613422036171,"q1_score":0.02141695562750101,"q3_score":0.9994550049304962,"avg_token_efficiency":0.9772722316188368,"is_incomplete":false,"total_prompt_cost_bin":0.203475,"total_cache_read_cost_bin":0.9676607999999973,"total_completion_cost_bin":10.519083000000006,"total_reasoning_cost_bin":10.324894499999997,"total_cost_bin":22.015113300000003,"total_on_demand_input_cost_bin":9.880082999999987,"ci_99_lower":0.2234469584381497,"ci_99_upper":0.4431831734841328,"ci_98_lower":0.23246704372713,"ci_98_upper":0.4308620307986054,"ci_95_lower":0.2427950134232993,"ci_95_upper":0.41691975547950166,"ci_90_lower":0.25562263674179575,"ci_90_upper":0.40534498300674615,"ci_75_lower":0.2781064054336471,"ci_75_upper":0.38075690245305555,"ci_50_lower":0.30151338069541467,"ci_50_upper":0.3579067116766011},"262144":{"avg_score":0.25190356780489487,"min_score":0,"max_score":1,"n_tests":124,"stddev":0.40144650789414416,"median_score":0.03869294747710228,"q1_score":0.0255599282681942,"q3_score":0.10990717820823193,"avg_token_efficiency":0.9824199622745924,"is_incomplete":false,"total_prompt_cost_bin":0.4325025,"total_cache_read_cost_bin":2.176108799999999,"total_completion_cost_bin":4.896166500000002,"total_reasoning_cost_bin":4.701874500000001,"total_cost_bin":12.206652300000002,"total_on_demand_input_cost_bin":22.19359050000001,"ci_99_lower":0.1610531391737412,"ci_99_upper":0.3521180345168555,"ci_98_lower":0.1708412418106661,"ci_98_upper":0.33580315759413004,"ci_95_lower":0.18044272231824335,"ci_95_upper":0.31852818598356775,"ci_90_lower":0.1908343099065185,"ci_90_upper":0.308485832927756,"ci_75_lower":0.20937343497785005,"ci_75_upper":0.2914968260409942,"ci_50_lower":0.22673553005405403,"ci_50_upper":0.27521739269967277}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Mini","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.0000045,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T20:19:49.095Z"},{"model_slug":"qwen/qwen3.6-max-preview","reasoning_mode":null,"overall_metrics":{"auc_1m":0.11601171746567346,"auc_128k":0.6665639888224054,"cum_avg_1m":0.44657589777271034,"cum_avg_128k":0.7145214364363366,"avg_score_overall":0.7061721029347925,"avg_response_time_ms":9807.285024595654,"total_prompt_cost":33.26747164399999,"total_cache_read_cost":0,"total_completion_cost":1.161463056,"total_reasoning_cost":0,"total_prompt_cost_128k":33.26747164399999,"total_completion_cost_128k":1.161463056,"total_on_demand_input_cost":33.26747164399999,"total_on_demand_input_cost_128k":33.26747164399999,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0208128870566133},"bin_metrics":{"8192":{"avg_score":0.8442783953156322,"min_score":0.019757520407438278,"max_score":1,"n_tests":80,"stddev":0.3559110363708825,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0228079512716755,"is_incomplete":false,"total_prompt_cost_bin":0.5582279040000003,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.17511787800000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.7333457820000003,"total_on_demand_input_cost_bin":0.5582279040000003,"ci_99_lower":0.7336420094314963,"ci_99_upper":0.9411736865125131,"ci_98_lower":0.7379768331074157,"ci_98_upper":0.9279228901525493,"ci_95_lower":0.7600925020588328,"ci_95_upper":0.9162032798223663,"ci_90_lower":0.7740459144639317,"ci_90_upper":0.904224915796658,"ci_75_lower":0.7964186935307225,"ci_75_upper":0.8914083330251742,"ci_50_lower":0.8194169963651803,"ci_50_upper":0.868143178866012},"16384":{"avg_score":0.7814104243880138,"min_score":0.01270289346575737,"max_score":1,"n_tests":80,"stddev":0.40810104642903516,"median_score":1,"q1_score":0.9999001175165176,"q3_score":1,"avg_token_efficiency":1.0161194758384646,"is_incomplete":false,"total_prompt_cost_bin":1.1511273279999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.17255448599999998,"total_reasoning_cost_bin":0,"total_cost_bin":1.3236818139999995,"total_on_demand_input_cost_bin":1.1511273279999996,"ci_99_lower":0.6601712660415796,"ci_99_upper":0.8905422636492877,"ci_98_lower":0.6733418629905209,"ci_98_upper":0.8791109487824142,"ci_95_lower":0.6848232530447421,"ci_95_upper":0.8669327584962594,"ci_90_lower":0.7079370823537465,"ci_90_upper":0.8540940248931294,"ci_75_lower":0.7320423913712147,"ci_75_upper":0.8297371733759064,"ci_50_lower":0.746285379503388,"ci_50_upper":0.8069165782828349},"32768":{"avg_score":0.6978493684402887,"min_score":0.003944773226976395,"max_score":1,"n_tests":136,"stddev":0.44653873094129776,"median_score":1,"q1_score":0.052756499499082565,"q3_score":1,"avg_token_efficiency":1.0257548592579655,"is_incomplete":false,"total_prompt_cost_bin":4.08425576,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.33797953800000013,"total_reasoning_cost_bin":0,"total_cost_bin":4.4222352979999995,"total_on_demand_input_cost_bin":4.08425576,"ci_99_lower":0.5951241564441315,"ci_99_upper":0.7905390736274764,"ci_98_lower":0.5995312986611594,"ci_98_upper":0.7820199124496909,"ci_95_lower":0.6183080298048854,"ci_95_upper":0.7702171504183378,"ci_90_lower":0.6343468642252607,"ci_90_upper":0.7599126097174389,"ci_75_lower":0.6534711626051884,"ci_75_upper":0.7392956413951812,"ci_50_lower":0.6721685724718126,"ci_50_upper":0.7218888696207775},"65536":{"avg_score":0.6571904583121924,"min_score":0.00757862813770771,"max_score":1,"n_tests":85,"stddev":0.46699033843777443,"median_score":1,"q1_score":0.03301799297332764,"q3_score":1,"avg_token_efficiency":1.0209904012221072,"is_incomplete":false,"total_prompt_cost_bin":5.829946251999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.21541735799999992,"total_reasoning_cost_bin":0,"total_cost_bin":6.0453636099999954,"total_on_demand_input_cost_bin":5.829946251999996,"ci_99_lower":0.5202933223394348,"ci_99_upper":0.7717907745828304,"ci_98_lower":0.5320337724462152,"ci_98_upper":0.7703951811355484,"ci_95_lower":0.554661525775788,"ci_95_upper":0.7492430461455574,"ci_90_lower":0.5766286100425264,"ci_90_upper":0.7374839081396075,"ci_75_lower":0.5989531987250837,"ci_75_upper":0.714407148146454,"ci_50_lower":0.6226128452320528,"ci_50_upper":0.6912274285767447},"131072":{"avg_score":0.591878535725556,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.484485147842355,"median_score":1,"q1_score":0.026250883005559444,"q3_score":1,"avg_token_efficiency":1.0162368757633713,"is_incomplete":false,"total_prompt_cost_bin":21.643914399999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.260393796,"total_reasoning_cost_bin":0,"total_cost_bin":21.904308195999995,"total_on_demand_input_cost_bin":21.643914399999996,"ci_99_lower":0.46868252032893054,"ci_99_upper":0.7145409914651814,"ci_98_lower":0.4860778825268945,"ci_98_upper":0.7048524334060606,"ci_95_lower":0.5054393036460539,"ci_95_upper":0.6863988245159243,"ci_90_lower":0.5152353480200563,"ci_90_upper":0.6760334434822988,"ci_75_lower":0.5348161879765294,"ci_75_upper":0.6489667063624174,"ci_50_lower":0.5627209197004208,"ci_50_upper":0.6287024998257515}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 Max Preview","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":0.000001027,"completion":0.000006162,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-27T03:24:02.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T12:54:19.322Z"},{"model_slug":"x-ai/grok-4.20","reasoning_mode":null,"overall_metrics":{"auc_1m":0.11477366047172041,"auc_128k":0.15917482772819455,"cum_avg_1m":0.17093647933003436,"cum_avg_128k":0.2104817893850953,"avg_score_overall":0.1448785635908345,"avg_response_time_ms":20732.46743333981,"total_prompt_cost":37.99529249999998,"total_cache_read_cost":182.96642080000055,"total_completion_cost":1.0635024999999998,"total_reasoning_cost":0,"total_prompt_cost_128k":9.06630375,"total_completion_cost_128k":0.44431750000000003,"total_on_demand_input_cost":1181.977091249998,"total_on_demand_input_cost_128k":29.192273750000012,"total_runs":1171,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9520562705789948},"bin_metrics":{"8192":{"avg_score":0.36592104263254444,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4563483822646095,"median_score":0.043215805664658546,"q1_score":0.028903400991111994,"q3_score":1,"avg_token_efficiency":0.9649372049989274,"is_incomplete":false,"total_prompt_cost_bin":0.4723399999999999,"total_cache_read_cost_bin":0.02707200000000001,"total_completion_cost_bin":0.0666,"total_reasoning_cost_bin":0,"total_cost_bin":0.566012,"total_on_demand_input_cost_bin":0.6415400000000003,"ci_99_lower":0.2337998448428407,"ci_99_upper":0.4974364108810259,"ci_98_lower":0.2437327476341743,"ci_98_upper":0.4860570846346673,"ci_95_lower":0.2671707311476348,"ci_95_upper":0.4676258994625822,"ci_90_lower":0.2773472814177512,"ci_90_upper":0.4483078782257508,"ci_75_lower":0.3087540162407095,"ci_75_upper":0.42514928959280956,"ci_50_lower":0.3309960072321701,"ci_50_upper":0.4001291047650738},"16384":{"avg_score":0.24217250008368865,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.40517885533676185,"median_score":0.0303514851257205,"q1_score":0.018197179306298494,"q3_score":0.048165462911129,"avg_token_efficiency":0.947967264243983,"is_incomplete":false,"total_prompt_cost_bin":0.525665,"total_cache_read_cost_bin":0.12462320000000006,"total_completion_cost_bin":0.0646175,"total_reasoning_cost_bin":0,"total_cost_bin":0.7149057000000001,"total_on_demand_input_cost_bin":1.3045600000000002,"ci_99_lower":0.13548766906460513,"ci_99_upper":0.35399735394684834,"ci_98_lower":0.14683363990776707,"ci_98_upper":0.3454726973074721,"ci_95_lower":0.15813579716108508,"ci_95_upper":0.32790923045977255,"ci_90_lower":0.17301355286035686,"ci_90_upper":0.31240527120884504,"ci_75_lower":0.1926789624558296,"ci_75_upper":0.2908178111625602,"ci_50_lower":0.20804137266532052,"ci_50_upper":0.2689364776975708},"32768":{"avg_score":0.1874061705174801,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.34936573425647577,"median_score":0.03603421710431576,"q1_score":0.023258299101144075,"q3_score":0.07183669693768024,"avg_token_efficiency":0.964682891956235,"is_incomplete":false,"total_prompt_cost_bin":1.6746950000000005,"total_cache_read_cost_bin":0.47990320000000014,"total_completion_cost_bin":0.12849250000000004,"total_reasoning_cost_bin":0,"total_cost_bin":2.2830907000000007,"total_on_demand_input_cost_bin":4.674090000000003,"ci_99_lower":0.11656781567585901,"ci_99_upper":0.2700774209950332,"ci_98_lower":0.12159240819952068,"ci_98_upper":0.26186931635294214,"ci_95_lower":0.12924437567870722,"ci_95_upper":0.24899276056950087,"ci_90_lower":0.13889992632047163,"ci_90_upper":0.2370548596758026,"ci_75_lower":0.15206766032112204,"ci_75_upper":0.22142121423316627,"ci_50_lower":0.16583556305404806,"ci_50_upper":0.20493531450612115},"65536":{"avg_score":0.12577384908080977,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.28937479811220973,"median_score":0.025073746219277382,"q1_score":0.015963032841682434,"q3_score":0.04344677925109863,"avg_token_efficiency":0.949818254537049,"is_incomplete":false,"total_prompt_cost_bin":1.8645612499999993,"total_cache_read_cost_bin":0.7578623999999993,"total_completion_cost_bin":0.08154000000000002,"total_reasoning_cost_bin":0,"total_cost_bin":2.7039636499999986,"total_on_demand_input_cost_bin":6.601201250000003,"ci_99_lower":0.05827230694722932,"ci_99_upper":0.21335484033075217,"ci_98_lower":0.062005647084894866,"ci_98_upper":0.20431818880902272,"ci_95_lower":0.07070736355750876,"ci_95_upper":0.19483417833237632,"ci_90_lower":0.07777284631313866,"ci_90_upper":0.1820421507267062,"ci_75_lower":0.09133340600289076,"ci_75_upper":0.1606700913946363,"ci_50_lower":0.10323180466041187,"ci_50_upper":0.1443752433228142},"131072":{"avg_score":0.13113538461095356,"min_score":0.004299226216971874,"max_score":1,"n_tests":103,"stddev":0.2964172664095212,"median_score":0.026483051478862762,"q1_score":0.019448080100119114,"q3_score":0.04015681520104408,"avg_token_efficiency":0.9478402143991185,"is_incomplete":false,"total_prompt_cost_bin":4.529042499999998,"total_cache_read_cost_bin":1.8306943999999994,"total_completion_cost_bin":0.10306750000000002,"total_reasoning_cost_bin":0,"total_cost_bin":6.462804399999998,"total_on_demand_input_cost_bin":15.970882500000005,"ci_99_lower":0.06296779816606289,"ci_99_upper":0.2101611369070416,"ci_98_lower":0.0710037279789544,"ci_98_upper":0.20745974760629166,"ci_95_lower":0.08211606880186524,"ci_95_upper":0.18846321899310495,"ci_90_lower":0.08752297640109019,"ci_90_upper":0.1763337524146637,"ci_75_lower":0.10003308791529332,"ci_75_upper":0.16154626048977028,"ci_50_lower":0.11147735249115498,"ci_50_upper":0.14864298489591268},"262144":{"avg_score":0.10420967518279316,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.23068006986089162,"median_score":0.03348729759454727,"q1_score":0.022689076140522957,"q3_score":0.0531800203025341,"avg_token_efficiency":0.9522821837448268,"is_incomplete":false,"total_prompt_cost_bin":5.093006250000001,"total_cache_read_cost_bin":13.215974399999965,"total_completion_cost_bin":0.12334000000000003,"total_reasoning_cost_bin":0,"total_cost_bin":18.432320649999966,"total_on_demand_input_cost_bin":87.92169999999996,"ci_99_lower":0.06211068641146685,"ci_99_upper":0.15934715422766998,"ci_98_lower":0.06369968584844354,"ci_98_upper":0.15465960727317643,"ci_95_lower":0.06935169708383342,"ci_95_upper":0.14518482197102503,"ci_90_lower":0.07421456761063731,"ci_90_upper":0.13852335310091943,"ci_75_lower":0.08231880421401827,"ci_75_upper":0.12671597614712643,"ci_50_lower":0.08946442364711737,"ci_50_upper":0.11643117735020367},"524288":{"avg_score":0.11816089885542169,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.2576892336446508,"median_score":0.03130391053855419,"q1_score":0.019079791847616434,"q3_score":0.05529290344566107,"avg_token_efficiency":0.9465947922013894,"is_incomplete":false,"total_prompt_cost_bin":8.767410000000002,"total_cache_read_cost_bin":45.42064640000005,"total_completion_cost_bin":0.19996999999999987,"total_reasoning_cost_bin":0,"total_cost_bin":54.38802640000005,"total_on_demand_input_cost_bin":292.7325249999999,"ci_99_lower":0.07949885570333177,"ci_99_upper":0.16351683776255924,"ci_98_lower":0.08215215677691433,"ci_98_upper":0.15603217078678672,"ci_95_lower":0.08819229049826144,"ci_95_upper":0.15245996652268684,"ci_90_lower":0.09114872234270678,"ci_90_upper":0.14782405638188256,"ci_75_lower":0.09865055839043757,"ci_75_upper":0.13775471253313176,"ci_50_lower":0.10602919812589466,"ci_50_upper":0.12788443534311414},"1048576":{"avg_score":0.09271231367658343,"min_score":0,"max_score":1,"n_tests":310,"stddev":0.22373245611138304,"median_score":0.03011072427034378,"q1_score":0.01904699532315135,"q3_score":0.04654296673834324,"avg_token_efficiency":0.9503174497297268,"is_incomplete":false,"total_prompt_cost_bin":15.068572499999977,"total_cache_read_cost_bin":121.10964480000052,"total_completion_cost_bin":0.2958749999999999,"total_reasoning_cost_bin":0,"total_cost_bin":136.4740923000005,"total_on_demand_input_cost_bin":772.1305924999982,"ci_99_lower":0.06369667537973052,"ci_99_upper":0.1298223621281119,"ci_98_lower":0.0649252957843259,"ci_98_upper":0.12387436369143334,"ci_95_lower":0.06818644535855481,"ci_95_upper":0.12001407637655162,"ci_90_lower":0.07283419497517658,"ci_90_upper":0.11596633369790299,"ci_75_lower":0.07743717227365836,"ci_75_upper":0.10739449189258589,"ci_50_lower":0.08383124638724351,"ci_50_upper":0.10102746916818432}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":2000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"SpaceXAI: Grok 4.20","provider_name":"X-ai","run_provider_names":["xAI"],"pricing_metadata":{"prompt":0.00000125,"completion":0.0000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-31T17:43:39.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T02:33:25.199Z"},{"model_slug":"openai/gpt-5.4-mini","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.11358601122113703,"auc_128k":0.40784091758716,"cum_avg_1m":0.33050685136941144,"cum_avg_128k":0.48280294071873253,"avg_score_overall":0.420806363992393,"avg_response_time_ms":19548.08936953125,"total_prompt_cost":1.5667110000000006,"total_cache_read_cost":4.368499199999996,"total_completion_cost":7.292312999999998,"total_reasoning_cost":6.217335,"total_prompt_cost_128k":0.7383547499999996,"total_completion_cost_128k":4.614781499999999,"total_on_demand_input_cost":45.25170299999997,"total_on_demand_input_cost_128k":18.04043474999998,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061808102971},"bin_metrics":{"8192":{"avg_score":0.6768206703942269,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.45539275916910693,"median_score":1,"q1_score":0.0449463315308094,"q3_score":1,"avg_token_efficiency":0.9759044449598389,"is_incomplete":false,"total_prompt_cost_bin":0.05953800000000002,"total_cache_read_cost_bin":0.0329856,"total_completion_cost_bin":0.3260429999999999,"total_reasoning_cost_bin":0.19932750000000002,"total_cost_bin":0.6178940999999999,"total_on_demand_input_cost_bin":0.38939399999999974,"ci_99_lower":0.530516538761789,"ci_99_upper":0.7971438382195773,"ci_98_lower":0.5463840326478239,"ci_98_upper":0.788511585253058,"ci_95_lower":0.5764510802394943,"ci_95_upper":0.770936665541958,"ci_90_lower":0.5958915273309684,"ci_90_upper":0.7527308141998946,"ci_75_lower":0.6182035865233048,"ci_75_upper":0.7342172212403967,"ci_50_lower":0.6404371519689448,"ci_50_upper":0.7119728324469179},"16384":{"avg_score":0.5473921843804419,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.48267846546624504,"median_score":0.9987680613994598,"q1_score":0.030932892113924026,"q3_score":1,"avg_token_efficiency":0.9670687393188718,"is_incomplete":false,"total_prompt_cost_bin":0.07388999999999998,"total_cache_read_cost_bin":0.0725376000000001,"total_completion_cost_bin":0.5105879999999999,"total_reasoning_cost_bin":0.3884264999999999,"total_cost_bin":1.0454420999999998,"total_on_demand_input_cost_bin":0.7992659999999999,"ci_99_lower":0.40686516941635636,"ci_99_upper":0.6856187641474535,"ci_98_lower":0.4219019815453794,"ci_98_upper":0.6730225726831705,"ci_95_lower":0.4373587430993211,"ci_95_upper":0.6481199335245764,"ci_90_lower":0.45794874073704706,"ci_90_upper":0.6365399872610578,"ci_75_lower":0.48794529519655044,"ci_75_upper":0.6113166640468989,"ci_50_lower":0.5126589851977769,"ci_50_upper":0.5841331811796409},"32768":{"avg_score":0.4946613899082877,"min_score":0.005958292167633772,"max_score":1,"n_tests":136,"stddev":0.4780048859831273,"median_score":0.10546177253127098,"q1_score":0.0335443913936615,"q3_score":1,"avg_token_efficiency":0.9952100768549625,"is_incomplete":false,"total_prompt_cost_bin":0.1971479999999999,"total_cache_read_cost_bin":0.26977920000000066,"total_completion_cost_bin":1.2835845,"total_reasoning_cost_bin":1.0441575,"total_cost_bin":2.7946692000000004,"total_on_demand_input_cost_bin":2.8949400000000005,"ci_99_lower":0.3935393087388365,"ci_99_upper":0.5997284113904051,"ci_98_lower":0.40064859825124804,"ci_98_upper":0.5886836823364309,"ci_95_lower":0.41589884542644384,"ci_95_upper":0.575233563130742,"ci_90_lower":0.4264699971400927,"ci_90_upper":0.5609273884222871,"ci_75_lower":0.4502896420719266,"ci_75_upper":0.5431305375669206,"ci_50_lower":0.4679588695298127,"ci_50_upper":0.5228580336654833},"65536":{"avg_score":0.3467845733359675,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.46052185452682776,"median_score":0.03207184001803398,"q1_score":0.0216621533036232,"q3_score":1,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":0.20430374999999995,"total_cache_read_cost_bin":0.38724479999999933,"total_completion_cost_bin":1.0161854999999995,"total_reasoning_cost_bin":0.8608725000000004,"total_cost_bin":2.468606549999999,"total_on_demand_input_cost_bin":4.076751749999996,"ci_99_lower":0.24028288255533314,"ci_99_upper":0.4733313632943421,"ci_98_lower":0.2420638922485797,"ci_98_upper":0.4625418339429478,"ci_95_lower":0.2537094036032966,"ci_95_upper":0.44957506592686775,"ci_90_lower":0.2657214260894312,"ci_90_upper":0.4366063838356229,"ci_75_lower":0.2888383269275757,"ci_75_upper":0.4042515727811191,"ci_50_lower":0.3123388448401409,"ci_50_upper":0.3807238726547974},"131072":{"avg_score":0.3483558855747388,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4556435890027751,"median_score":0.03472931683063507,"q1_score":0.02197199407964945,"q3_score":0.9995213747024536,"avg_token_efficiency":0.9772722316188369,"is_incomplete":false,"total_prompt_cost_bin":0.2034749999999999,"total_cache_read_cost_bin":0.9676607999999973,"total_completion_cost_bin":1.4783805,"total_reasoning_cost_bin":1.2846600000000006,"total_cost_bin":3.934176299999998,"total_on_demand_input_cost_bin":9.880082999999983,"ci_99_lower":0.24085434204966588,"ci_99_upper":0.46083685115900896,"ci_98_lower":0.24580243950445224,"ci_98_upper":0.4528450803088874,"ci_95_lower":0.2606706104104552,"ci_95_upper":0.4358120676889016,"ci_90_lower":0.27376059830851074,"ci_90_upper":0.4233517588016767,"ci_75_lower":0.294458603294842,"ci_75_upper":0.3988496481186091,"ci_50_lower":0.3199930118093972,"ci_50_upper":0.37750290758179916},"262144":{"avg_score":0.23004010736162886,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.3842078671599505,"median_score":0.03728251904249191,"q1_score":0.023779723793268204,"q3_score":0.06420546025037766,"avg_token_efficiency":0.982419810338046,"is_incomplete":false,"total_prompt_cost_bin":0.8283562500000011,"total_cache_read_cost_bin":2.6382911999999985,"total_completion_cost_bin":2.6775314999999993,"total_reasoning_cost_bin":2.439891,"total_cost_bin":8.584069949999998,"total_on_demand_input_cost_bin":27.21126824999999,"ci_99_lower":0.15618676701042114,"ci_99_upper":0.31247356973446455,"ci_98_lower":0.15930492037801558,"ci_98_upper":0.3069822232075802,"ci_95_lower":0.16726560752214667,"ci_95_upper":0.2918135207126591,"ci_90_lower":0.17793097921914638,"ci_90_upper":0.28076732094145335,"ci_75_lower":0.19329935499897938,"ci_75_upper":0.2659842615921532,"ci_50_lower":0.20747511903755367,"ci_50_upper":0.2493360640038571}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Mini","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.0000045,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T02:32:05.024Z"},{"model_slug":"inclusionai/ring-2.6-1t","reasoning_mode":"high","overall_metrics":{"auc_1m":0.10845681065146133,"auc_128k":0.6405976263564015,"cum_avg_1m":0.4424461198980885,"cum_avg_128k":0.7079137918369416,"avg_score_overall":0.6970377571311729,"avg_response_time_ms":16343.230427891755,"total_prompt_cost":0.24384705000000012,"total_cache_read_cost":0.33593356500000027,"total_completion_cost":0.5009999999999999,"total_reasoning_cost":0.39565312500000005,"total_prompt_cost_128k":0.24384705000000012,"total_completion_cost_128k":0.5009999999999999,"total_on_demand_input_cost":1.9235148749999995,"total_on_demand_input_cost_128k":1.9235148749999995,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0476121803738963},"bin_metrics":{"8192":{"avg_score":0.9097927315393463,"min_score":0.008075370453298092,"max_score":1,"n_tests":80,"stddev":0.27543598291749744,"median_score":0.9977972209453583,"q1_score":0.9933072030544281,"q3_score":1,"avg_token_efficiency":1.0531622272267291,"is_incomplete":false,"total_prompt_cost_bin":0.02631660000000001,"total_cache_read_cost_bin":0.0031249200000000006,"total_completion_cost_bin":0.05768,"total_reasoning_cost_bin":0.042331875000000026,"total_cost_bin":0.12945339500000003,"total_on_demand_input_cost_bin":0.04194120000000002,"ci_99_lower":0.8146305672819143,"ci_99_upper":0.9804527336748433,"ci_98_lower":0.8244046405073022,"ci_98_upper":0.9716854293567594,"ci_95_lower":0.8385961976466932,"ci_95_upper":0.9676649768862989,"ci_90_lower":0.8502450656384463,"ci_90_upper":0.9581944026437123,"ci_75_lower":0.8727644159429474,"ci_75_upper":0.9454257243924076,"ci_50_lower":0.8864316096762195,"ci_50_upper":0.9328109467896867},"16384":{"avg_score":0.7759023438498843,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.405697216937491,"median_score":0.996351420879364,"q1_score":0.9492583572864532,"q3_score":0.9992415457963943,"avg_token_efficiency":1.0492618050264615,"is_incomplete":false,"total_prompt_cost_bin":0.023934375,"total_cache_read_cost_bin":0.012583365000000001,"total_completion_cost_bin":0.08450374999999999,"total_reasoning_cost_bin":0.06637562499999998,"total_cost_bin":0.18739711499999995,"total_on_demand_input_cost_bin":0.08685120000000002,"ci_99_lower":0.6566737606450043,"ci_99_upper":0.8834999047877209,"ci_98_lower":0.6690203799598385,"ci_98_upper":0.8730149573965463,"ci_95_lower":0.6805107559834142,"ci_95_upper":0.8620479122735559,"ci_90_lower":0.7026399001554818,"ci_90_upper":0.8480825279082638,"ci_75_lower":0.7263650736575074,"ci_75_upper":0.8243739430552524,"ci_50_lower":0.7416544333449564,"ci_50_upper":0.802732273179572},"32768":{"avg_score":0.6972553832079832,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.44018056348952017,"median_score":0.9880377352237701,"q1_score":0.05300329998135567,"q3_score":1,"avg_token_efficiency":1.0489053037512415,"is_incomplete":false,"total_prompt_cost_bin":0.02600879999999999,"total_cache_read_cost_bin":0.05578752000000009,"total_completion_cost_bin":0.15310875,"total_reasoning_cost_bin":0.12327687500000005,"total_cost_bin":0.3581819450000001,"total_on_demand_input_cost_bin":0.30494639999999995,"ci_99_lower":0.5980527887791772,"ci_99_upper":0.7894686528374544,"ci_98_lower":0.6029387766199515,"ci_98_upper":0.7810255619689055,"ci_95_lower":0.6220820281308775,"ci_95_upper":0.7689234463263454,"ci_90_lower":0.6348953861738076,"ci_90_upper":0.7591708581783356,"ci_75_lower":0.6527530691913426,"ci_75_upper":0.7384188469244263,"ci_50_lower":0.671053254224449,"ci_50_upper":0.720563413003487},"65536":{"avg_score":0.6359871809137985,"min_score":0.001463057822547853,"max_score":1,"n_tests":85,"stddev":0.4667241770233913,"median_score":0.9943060278892517,"q1_score":0.034448493272066116,"q3_score":0.9989053010940552,"avg_token_efficiency":1.044777939647338,"is_incomplete":false,"total_prompt_cost_bin":0.05719964999999997,"total_cache_read_cost_bin":0.07569408,"total_completion_cost_bin":0.06830187499999998,"total_reasoning_cost_bin":0.04859062499999999,"total_cost_bin":0.24978622999999994,"total_on_demand_input_cost_bin":0.43567004999999986,"ci_99_lower":0.49454281052019833,"ci_99_upper":0.7601215183205821,"ci_98_lower":0.5142821515228133,"ci_98_upper":0.745041429245009,"ci_95_lower":0.5339870092666515,"ci_95_upper":0.7309996993490152,"ci_90_lower":0.5541644758420915,"ci_90_upper":0.71974388643209,"ci_75_lower":0.5788484649160904,"ci_75_upper":0.6932873642323744,"ci_50_lower":0.6005066179201308,"ci_50_upper":0.6697611516081345},"131072":{"avg_score":0.5206313196736958,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.47975200678999874,"median_score":0.916304349899292,"q1_score":0.028386819176375866,"q3_score":0.9944174885749817,"avg_token_efficiency":1.0426517139856113,"is_incomplete":false,"total_prompt_cost_bin":0.11038762500000017,"total_cache_read_cost_bin":0.18874368000000016,"total_completion_cost_bin":0.137405625,"total_reasoning_cost_bin":0.11507812499999996,"total_cost_bin":0.5516150550000003,"total_on_demand_input_cost_bin":1.0541060249999996,"ci_99_lower":0.3904750109180691,"ci_99_upper":0.6403557045205759,"ci_98_lower":0.41502474978690496,"ci_98_upper":0.6293437960780975,"ci_95_lower":0.43061001837446844,"ci_95_upper":0.6119080306604409,"ci_90_lower":0.4453945706098892,"ci_90_upper":0.6025155617661087,"ci_75_lower":0.46552479543691516,"ci_75_upper":0.5772853872656271,"ci_50_lower":0.48745491877718133,"ci_50_upper":0.5560214134629419}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"inclusionAI: Ring-2.6-1T","provider_name":"Inclusionai","run_provider_names":["NovitaAI"],"pricing_metadata":{"prompt":7.5e-8,"completion":6.25e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-08T13:37:20.000Z","api_updated_at":"2026-08-23T14:20:02.494Z","latest_run_timestamp":"2026-06-23T04:38:36.701Z"},{"model_slug":"inclusionai/ring-2.6-1t","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.10775152872539673,"auc_128k":0.6469791795284006,"cum_avg_1m":0.44014793933288443,"cum_avg_128k":0.7042367029326151,"avg_score_overall":0.6984821387065366,"avg_response_time_ms":14212.799121194634,"total_prompt_cost":0.3002285999999997,"total_cache_read_cost":0.3246645150000002,"total_completion_cost":0.43658062499999994,"total_reasoning_cost":0.3338968750000001,"total_prompt_cost_128k":0.3002285999999997,"total_completion_cost_128k":0.43658062499999994,"total_on_demand_input_cost":1.9235511749999987,"total_on_demand_input_cost_128k":1.9235511749999987,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0476645738685941},"bin_metrics":{"8192":{"avg_score":0.8155038638040424,"min_score":0.010457987897098064,"max_score":1,"n_tests":80,"stddev":0.37884884596830637,"median_score":0.9989337921142578,"q1_score":0.9877743571996689,"q3_score":1,"avg_token_efficiency":1.0533175677014532,"is_incomplete":false,"total_prompt_cost_bin":0.02680657499999999,"total_cache_read_cost_bin":0.0030281250000000004,"total_completion_cost_bin":0.04999812500000001,"total_reasoning_cost_bin":0.03461000000000001,"total_cost_bin":0.114442825,"total_on_demand_input_cost_bin":0.04194719999999997,"ci_99_lower":0.6949351019423922,"ci_99_upper":0.9241363362557021,"ci_98_lower":0.7085986949586077,"ci_98_upper":0.9119565419005229,"ci_95_lower":0.7272097503967233,"ci_95_upper":0.8896444012966821,"ci_90_lower":0.7420845952397213,"ci_90_upper":0.8797839265788204,"ci_75_lower":0.7662472058684215,"ci_75_upper":0.862819565285463,"ci_50_lower":0.7902511995198438,"ci_50_upper":0.8409010892326478},"16384":{"avg_score":0.8236764151311945,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.3702071056500116,"median_score":0.9961158335208893,"q1_score":0.9726253896951675,"q3_score":0.9990530014038086,"avg_token_efficiency":1.04933631852891,"is_incomplete":false,"total_prompt_cost_bin":0.01959045,"total_cache_read_cost_bin":0.013453349999999992,"total_completion_cost_bin":0.05772000000000001,"total_reasoning_cost_bin":0.04086250000000001,"total_cost_bin":0.13162630000000003,"total_on_demand_input_cost_bin":0.08685720000000004,"ci_99_lower":0.702835504873452,"ci_99_upper":0.9223460741941817,"ci_98_lower":0.7140347732664085,"ci_98_upper":0.9110415765815996,"ci_95_lower":0.7387536449772597,"ci_95_upper":0.8979848173170467,"ci_90_lower":0.7517203520194744,"ci_90_upper":0.885406037196808,"ci_75_lower":0.7749616402477841,"ci_75_upper":0.8705413752446475,"ci_50_lower":0.7983732958236942,"ci_50_upper":0.8488620343981894},"32768":{"avg_score":0.7452197207466644,"min_score":0.008937438018620014,"max_score":1,"n_tests":136,"stddev":0.4154757036580034,"median_score":0.988149493932724,"q1_score":0.18775242194533348,"q3_score":0.9995775818824768,"avg_token_efficiency":1.0489412128462319,"is_incomplete":false,"total_prompt_cost_bin":0.03830699999999998,"total_cache_read_cost_bin":0.053329920000000114,"total_completion_cost_bin":0.11922124999999999,"total_reasoning_cost_bin":0.08905625000000002,"total_cost_bin":0.29991442000000007,"total_on_demand_input_cost_bin":0.30495659999999986,"ci_99_lower":0.653509349037674,"ci_99_upper":0.8350622384657275,"ci_98_lower":0.6620829818862053,"ci_98_upper":0.8301906874750549,"ci_95_lower":0.674691610995454,"ci_95_upper":0.8127106730641304,"ci_90_lower":0.6836875810784371,"ci_90_upper":0.8011468367856543,"ci_75_lower":0.7036642599046258,"ci_75_upper":0.7852564347755534,"ci_50_lower":0.7198019602579092,"ci_50_upper":0.7681794445499476},"65536":{"avg_score":0.6393139580812525,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.46787727690058545,"median_score":0.9874186515808105,"q1_score":0.031302403658628464,"q3_score":0.9987834692001343,"avg_token_efficiency":1.044793227499619,"is_incomplete":false,"total_prompt_cost_bin":0.07195162499999998,"total_cache_read_cost_bin":0.07274495999999998,"total_completion_cost_bin":0.12253437499999999,"total_reasoning_cost_bin":0.10651812500000006,"total_cost_bin":0.37374908500000004,"total_on_demand_input_cost_bin":0.435676425,"ci_99_lower":0.5013142654086079,"ci_99_upper":0.7643983290344477,"ci_98_lower":0.5122075792327523,"ci_98_upper":0.7517888118082329,"ci_95_lower":0.5371224312424002,"ci_95_upper":0.7325686851146576,"ci_90_lower":0.5590347563086406,"ci_90_upper":0.7205134336129927,"ci_75_lower":0.581712472637403,"ci_75_upper":0.696556163859992,"ci_50_lower":0.60465423726915,"ci_50_upper":0.6734854598967907},"131072":{"avg_score":0.4974695568999221,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.478701353454801,"median_score":0.09195402264595032,"q1_score":0.025882162153720856,"q3_score":0.9949365556240082,"avg_token_efficiency":1.0426593550428676,"is_incomplete":false,"total_prompt_cost_bin":0.14357294999999975,"total_cache_read_cost_bin":0.18210816000000013,"total_completion_cost_bin":0.08710687499999997,"total_reasoning_cost_bin":0.06284999999999999,"total_cost_bin":0.4756379849999999,"total_on_demand_input_cost_bin":1.0541137499999989,"ci_99_lower":0.37392602177869155,"ci_99_upper":0.6119719601684427,"ci_98_lower":0.3913938652831175,"ci_98_upper":0.6086933587263428,"ci_95_lower":0.4062121805088432,"ci_95_upper":0.5879484213690851,"ci_90_lower":0.41905029290660906,"ci_90_upper":0.5733283641811423,"ci_75_lower":0.44271351671206893,"ci_75_upper":0.5557187909016824,"ci_50_lower":0.4658273949165765,"ci_50_upper":0.5306303730778152}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"inclusionAI: Ring-2.6-1T","provider_name":"Inclusionai","run_provider_names":[],"pricing_metadata":{"prompt":7.5e-8,"completion":6.25e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-08T13:37:20.000Z","api_updated_at":"2026-08-23T14:20:02.494Z","latest_run_timestamp":"2026-06-23T03:12:39.605Z"},{"model_slug":"inclusionai/ling-3.0-flash","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.10761942762818466,"auc_128k":0.6688410548603361,"cum_avg_1m":0.46766866779392613,"cum_avg_128k":0.7482698684702818,"avg_score_overall":0.7428361171820924,"avg_response_time_ms":5326.229676554026,"total_prompt_cost":0.05383469699999999,"total_cache_read_cost":0.09694379520000002,"total_completion_cost":0.042034104,"total_reasoning_cost":0.030390444000000003,"total_prompt_cost_128k":0.05383469699999999,"total_completion_cost_128k":0.042034104,"total_on_demand_input_cost":0.5385536730000002,"total_on_demand_input_cost_128k":0.5385536730000002,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0474549998898026},"bin_metrics":{"8192":{"avg_score":0.9974538922309876,"min_score":0.9468473196029663,"max_score":1,"n_tests":80,"stddev":0.008038497635263826,"median_score":1,"q1_score":0.9991320818662643,"q3_score":1,"avg_token_efficiency":1.0526962058025586,"is_incomplete":false,"total_prompt_cost_bin":0.0038559360000000003,"total_cache_read_cost_bin":0.001576512,"total_completion_cost_bin":0.005250294000000003,"total_reasoning_cost_bin":0.003610781999999998,"total_cost_bin":0.014293524,"total_on_demand_input_cost_bin":0.011738496000000001,"ci_99_lower":0.9946464775353671,"ci_99_upper":0.9992786736525595,"ci_98_lower":0.9950344901904463,"ci_98_upper":0.9991878211274742,"ci_95_lower":0.9953658194281161,"ci_95_upper":0.998967444486916,"ci_90_lower":0.9957650939747691,"ci_90_upper":0.9987611801922321,"ci_75_lower":0.9963871829211711,"ci_75_upper":0.9984668977558613,"ci_50_lower":0.9968992615118623,"ci_50_upper":0.9980628302320838},"16384":{"avg_score":0.7855934473336674,"min_score":0.010664081200957298,"max_score":1,"n_tests":80,"stddev":0.3969760643829619,"median_score":0.9962050914764404,"q1_score":0.9606459438800812,"q3_score":0.9996891021728516,"avg_token_efficiency":1.0490382645191199,"is_incomplete":false,"total_prompt_cost_bin":0.0051760800000000004,"total_cache_read_cost_bin":0.0038274432000000034,"total_completion_cost_bin":0.006415038000000001,"total_reasoning_cost_bin":0.004537575,"total_cost_bin":0.019956136200000005,"total_on_demand_input_cost_bin":0.024313295999999998,"ci_99_lower":0.6656947905926499,"ci_99_upper":0.8877916017787647,"ci_98_lower":0.6743488122210837,"ci_98_upper":0.8839263375297887,"ci_95_lower":0.6905769409850473,"ci_95_upper":0.8722927123249974,"ci_90_lower":0.7109762734593823,"ci_90_upper":0.8561719779478153,"ci_75_lower":0.7367365328973392,"ci_75_upper":0.8332591375758057,"ci_50_lower":0.7544756208226316,"ci_50_upper":0.8139499481767416},"32768":{"avg_score":0.8285168131542913,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.3602190681974227,"median_score":0.9954627454280853,"q1_score":0.9656716585159302,"q3_score":1,"avg_token_efficiency":1.0487975764662696,"is_incomplete":false,"total_prompt_cost_bin":0.006316967999999997,"total_cache_read_cost_bin":0.01581189119999996,"total_completion_cost_bin":0.010928862,"total_reasoning_cost_bin":0.007526673000000001,"total_cost_bin":0.04058439419999996,"total_on_demand_input_cost_bin":0.08537642400000009,"ci_99_lower":0.7474650523443074,"ci_99_upper":0.8999456009350765,"ci_98_lower":0.7544744770197533,"ci_98_upper":0.8942231893239488,"ci_95_lower":0.7647723967105455,"ci_95_upper":0.8868533094993942,"ci_90_lower":0.7722435543085927,"ci_90_upper":0.8784790415074574,"ci_75_lower":0.7882914178966638,"ci_75_upper":0.8637264906415916,"ci_50_lower":0.8073279760244666,"ci_50_upper":0.8501272228934207},"65536":{"avg_score":0.6754037538981613,"min_score":0.006762987934052944,"max_score":1,"n_tests":85,"stddev":0.4577513978045887,"median_score":0.9955320954322815,"q1_score":0.03782821446657181,"q3_score":0.9989561438560486,"avg_token_efficiency":1.0447320760904928,"is_incomplete":false,"total_prompt_cost_bin":0.011560562999999998,"total_cache_read_cost_bin":0.022084339200000004,"total_completion_cost_bin":0.009029097000000002,"total_reasoning_cost_bin":0.006879222,"total_cost_bin":0.0495532212,"total_on_demand_input_cost_bin":0.12198225900000005,"ci_99_lower":0.5388726256268428,"ci_99_upper":0.7795532350953779,"ci_98_lower":0.5585790045764297,"ci_98_upper":0.7781486392579973,"ci_95_lower":0.5815248296648154,"ci_95_upper":0.7672014484729837,"ci_90_lower":0.5947942405018737,"ci_90_upper":0.7547463427330641,"ci_75_lower":0.6181164386463078,"ci_75_upper":0.732562863950015,"ci_50_lower":0.6409756774301915,"ci_50_upper":0.7088837769402959},"131072":{"avg_score":0.45438143573430123,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.476027950845742,"median_score":0.04246962070465088,"q1_score":0.025172675028443336,"q3_score":0.9962901175022125,"avg_token_efficiency":1.0426287908138419,"is_incomplete":false,"total_prompt_cost_bin":0.02692515,"total_cache_read_cost_bin":0.05364360960000006,"total_completion_cost_bin":0.010410812999999998,"total_reasoning_cost_bin":0.007836192000000002,"total_cost_bin":0.09881576460000006,"total_on_demand_input_cost_bin":0.29514319800000005,"ci_99_lower":0.3291777538752988,"ci_99_upper":0.5638023482110956,"ci_98_lower":0.3451136945565548,"ci_98_upper":0.5551421711191499,"ci_95_lower":0.3571299109615847,"ci_95_upper":0.5443023692345184,"ci_90_lower":0.37877015407802983,"ci_90_upper":0.5323901502955796,"ci_75_lower":0.39878413763837617,"ci_75_upper":0.5117717170404062,"ci_50_lower":0.4223811784940003,"ci_50_upper":0.48852937734568297}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Ling-3.0-flash","provider_name":"Inclusionai","run_provider_names":["NovitaAI"],"pricing_metadata":{"prompt":2.1e-8,"completion":6.3e-8,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-23T14:56:20.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-12T19:34:25.691Z"},{"model_slug":"qwen/qwen3.5-397b-a17b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.10594795451851333,"auc_128k":0.6732530474347783,"cum_avg_1m":0.47850320345182046,"cum_avg_128k":0.7656051255229127,"avg_score_overall":0.7554309912379985,"avg_response_time_ms":115466.05471978306,"total_prompt_cost":9.37150812,"total_cache_read_cost":0.38460864000000017,"total_completion_cost":7.9993438200000035,"total_reasoning_cost":7.5459922200000005,"total_prompt_cost_128k":9.37150812,"total_completion_cost_128k":7.9993438200000035,"total_on_demand_input_cost":9.75611676,"total_on_demand_input_cost_128k":9.75611676,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207081000672173},"bin_metrics":{"8192":{"avg_score":0.9441011326853186,"min_score":0.01399631705135107,"max_score":1,"n_tests":80,"stddev":0.21413571424341626,"median_score":0.9971585273742676,"q1_score":0.9887349009513855,"q3_score":1,"avg_token_efficiency":1.0224972703222286,"is_incomplete":false,"total_prompt_cost_bin":0.10085087999999999,"total_cache_read_cost_bin":0.11107200000000003,"total_completion_cost_bin":0.94109184,"total_reasoning_cost_bin":0.8739993599999998,"total_cost_bin":2.02701408,"total_on_demand_input_cost_bin":0.2119228800000001,"ci_99_lower":0.8712244795713341,"ci_99_upper":0.9936579795517028,"ci_98_lower":0.8806747490761336,"ci_98_upper":0.9924100356325507,"ci_95_lower":0.8839749466627835,"ci_95_upper":0.9825337682920507,"ci_90_lower":0.8959567043592688,"ci_90_upper":0.9804572666389868,"ci_75_lower":0.9184938128455543,"ci_75_upper":0.9691709269347484,"ci_50_lower":0.9307443813071586,"ci_50_upper":0.9575774809141876},"16384":{"avg_score":0.9671842320822179,"min_score":0.0172955971211195,"max_score":1,"n_tests":80,"stddev":0.15281352237574264,"median_score":0.9964780211448669,"q1_score":0.9902336895465851,"q3_score":0.9988732784986496,"avg_token_efficiency":1.0159704488335681,"is_incomplete":false,"total_prompt_cost_bin":0.16353791999999998,"total_cache_read_cost_bin":0.2735366400000001,"total_completion_cost_bin":1.0138120200000003,"total_reasoning_cost_bin":0.9470026799999997,"total_cost_bin":2.39788926,"total_on_demand_input_cost_bin":0.4370745599999997,"ci_99_lower":0.9160287763122469,"ci_99_upper":0.9940355029925704,"ci_98_lower":0.918082707490772,"ci_98_upper":0.9937277705371379,"ci_95_lower":0.9295338391675615,"ci_95_upper":0.9927263621427118,"ci_90_lower":0.9321322678867727,"ci_90_upper":0.9918487052246927,"ci_75_lower":0.9438310006109532,"ci_75_upper":0.9886718011461199,"ci_50_lower":0.9556091808131896,"ci_50_upper":0.9794602839043365},"32768":{"avg_score":0.828441383981365,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.3539588901267824,"median_score":0.9926130473613739,"q1_score":0.962560623884201,"q3_score":0.9994260519742966,"avg_token_efficiency":1.0256830410679836,"is_incomplete":false,"total_prompt_cost_bin":1.5508771199999989,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.0959122600000013,"total_reasoning_cost_bin":1.9668168000000004,"total_cost_bin":5.613606180000001,"total_on_demand_input_cost_bin":1.5508771199999989,"ci_99_lower":0.7506575587882853,"ci_99_upper":0.8985168825170394,"ci_98_lower":0.7547114180169506,"ci_98_upper":0.8951824781599947,"ci_95_lower":0.7656872586444745,"ci_95_upper":0.8850248665096896,"ci_90_lower":0.7727895063604913,"ci_90_upper":0.8783948215173201,"ci_75_lower":0.7896540704815005,"ci_75_upper":0.863093650312187,"ci_50_lower":0.8074298852554742,"ci_50_upper":0.8494499896170216},"65536":{"avg_score":0.6687245648244724,"min_score":0.007713073864579201,"max_score":1,"n_tests":85,"stddev":0.45351943809931694,"median_score":0.9855595827102661,"q1_score":0.040453072637319565,"q3_score":0.9985548853874207,"avg_token_efficiency":1.0209598255175443,"is_incomplete":false,"total_prompt_cost_bin":2.21383734,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.5828906600000001,"total_reasoning_cost_bin":1.49973876,"total_cost_bin":5.2964667599999995,"total_on_demand_input_cost_bin":2.21383734,"ci_99_lower":0.5335514706306926,"ci_99_upper":0.7761963868252384,"ci_98_lower":0.5499866008962341,"ci_98_upper":0.7720895477785346,"ci_95_lower":0.5756056744375212,"ci_95_upper":0.7602098780990961,"ci_90_lower":0.5877709936991553,"ci_90_upper":0.7458466999555993,"ci_75_lower":0.6123185768490658,"ci_75_upper":0.7266511085867773,"ci_50_lower":0.6341860372052692,"ci_50_upper":0.7003622969694654},"131072":{"avg_score":0.4195743140411898,"min_score":0.0010498687624931335,"max_score":1,"n_tests":103,"stddev":0.47264904231164745,"median_score":0.04701627418398857,"q1_score":0.02402450703084469,"q3_score":0.9917313158512115,"avg_token_efficiency":1.0162215936488594,"is_incomplete":false,"total_prompt_cost_bin":5.34240486,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.3656370400000015,"total_reasoning_cost_bin":2.2584346200000005,"total_cost_bin":9.966476520000002,"total_on_demand_input_cost_bin":5.34240486,"ci_99_lower":0.29720393441870663,"ci_99_upper":0.5315832410983342,"ci_98_lower":0.31043232532037257,"ci_98_upper":0.5204285145557296,"ci_95_lower":0.32604651448013966,"ci_95_upper":0.5077728197105156,"ci_90_lower":0.3426744501133567,"ci_90_upper":0.4968411608685025,"ci_75_lower":0.36432855884607845,"ci_75_upper":0.47542931006665257,"ci_50_lower":0.38613044852345674,"ci_50_upper":0.45450834603980184}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5 397B A17B","provider_name":"Qwen","run_provider_names":["AtlasCloud"],"pricing_metadata":{"prompt":3.9e-7,"completion":0.00000234,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-16T06:23:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T06:56:35.230Z"},{"model_slug":"upstage/solar-pro4","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.10033997979241076,"auc_128k":0.4798677467427955,"cum_avg_1m":0.40046206240798427,"cum_avg_128k":0.6119329417756407,"avg_score_overall":0.4973369454715001,"avg_response_time_ms":188103.25548142655,"total_prompt_cost":0.7458019499999998,"total_cache_read_cost":0.21294988799999998,"total_completion_cost":0.6471499199999999,"total_reasoning_cost":0.6116973600000001,"total_prompt_cost_128k":0.30355617000000007,"total_completion_cost_128k":0.47095175999999994,"total_on_demand_input_cost":1.8105513899999996,"total_on_demand_input_cost_128k":0.7212502499999995,"total_runs":624,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9839016531109587},"bin_metrics":{"8192":{"avg_score":0.874143854691647,"min_score":0.017316017299890518,"max_score":1,"n_tests":80,"stddev":0.314195175899193,"median_score":0.9976620674133301,"q1_score":0.9876617044210434,"q3_score":1,"avg_token_efficiency":0.9793386024398524,"is_incomplete":false,"total_prompt_cost_bin":0.01268136,"total_cache_read_cost_bin":0.0005875200000000001,"total_completion_cost_bin":0.05153628,"total_reasoning_cost_bin":0.04093044,"total_cost_bin":0.1057356,"total_on_demand_input_cost_bin":0.015618960000000001,"ci_99_lower":0.7770153320075478,"ci_99_upper":0.9589710139494855,"ci_98_lower":0.7882842969133053,"ci_98_upper":0.9462167699658312,"ci_95_lower":0.7980929954926251,"ci_95_upper":0.936633497857838,"ci_90_lower":0.8098305930546486,"ci_90_upper":0.9247010555467569,"ci_75_lower":0.8342090478370665,"ci_75_upper":0.910714720323449,"ci_50_lower":0.8510439298872371,"ci_50_upper":0.8977698542177677},"16384":{"avg_score":0.8475631311535835,"min_score":0.005645730532705784,"max_score":1,"n_tests":80,"stddev":0.34822675535098435,"median_score":0.9964700639247894,"q1_score":0.9773350358009338,"q3_score":0.9991892874240875,"avg_token_efficiency":0.9804485654841614,"is_incomplete":false,"total_prompt_cost_bin":0.022685999999999994,"total_cache_read_cost_bin":0.0019453440000000005,"total_completion_cost_bin":0.05824343999999998,"total_reasoning_cost_bin":0.0549702,"total_cost_bin":0.13784498399999998,"total_on_demand_input_cost_bin":0.032412719999999985,"ci_99_lower":0.7374663684396073,"ci_99_upper":0.9449334609386278,"ci_98_lower":0.7507688746231143,"ci_98_upper":0.9322756742959609,"ci_95_lower":0.7649190805174294,"ci_95_upper":0.9199319885653676,"ci_90_lower":0.7768323981616413,"ci_90_upper":0.9078608779987553,"ci_75_lower":0.799340712511912,"ci_75_upper":0.8938565456759534,"ci_50_lower":0.8230101328081219,"ci_50_upper":0.871862686710665},"32768":{"avg_score":0.6720157317741884,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4533720530471658,"median_score":0.9869400858879089,"q1_score":0.034922017715871334,"q3_score":1,"avg_token_efficiency":0.9957145677635983,"is_incomplete":false,"total_prompt_cost_bin":0.05935608000000001,"total_cache_read_cost_bin":0.011286912000000003,"total_completion_cost_bin":0.1308492,"total_reasoning_cost_bin":0.12452232000000006,"total_cost_bin":0.3260145120000001,"total_on_demand_input_cost_bin":0.11579063999999982,"ci_99_lower":0.5659060144205476,"ci_99_upper":0.7671347823956816,"ci_98_lower":0.57317618243927,"ci_98_upper":0.7556808323870576,"ci_95_lower":0.5891494331337847,"ci_95_upper":0.7448089536133065,"ci_90_lower":0.6084735688467524,"ci_90_upper":0.7358595680818647,"ci_75_lower":0.6284487381413141,"ci_75_upper":0.7153707854017471,"ci_50_lower":0.644736592668136,"ci_50_upper":0.6954392532600597},"65536":{"avg_score":0.40489220835401546,"min_score":0,"max_score":1,"n_tests":84,"stddev":0.4726101753414653,"median_score":0.03614087402820587,"q1_score":0.02323983609676361,"q3_score":0.9945959746837616,"avg_token_efficiency":0.9793249938245555,"is_incomplete":false,"total_prompt_cost_bin":0.06487958999999997,"total_cache_read_cost_bin":0.019309823999999996,"total_completion_cost_bin":0.10679951999999998,"total_reasoning_cost_bin":0.10279967999999998,"total_cost_bin":0.2937886139999999,"total_on_demand_input_cost_bin":0.16142871,"ci_99_lower":0.28781814198852296,"ci_99_upper":0.5205031847172704,"ci_98_lower":0.29122246051395667,"ci_98_upper":0.5104792530225988,"ci_95_lower":0.3028516519920058,"ci_95_upper":0.4978274178020434,"ci_90_lower":0.3225600342909318,"ci_90_upper":0.48553529227363124,"ci_75_lower":0.34656560225855737,"ci_75_upper":0.463301081193744,"ci_50_lower":0.36989130518521135,"ci_50_upper":0.4392858153635947},"131072":{"avg_score":0.26104978290476916,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.41439144840241265,"median_score":0.029728343710303307,"q1_score":0.019665293395519257,"q3_score":0.12456942349672318,"avg_token_efficiency":0.9792403602520147,"is_incomplete":false,"total_prompt_cost_bin":0.1439531400000001,"total_cache_read_cost_bin":0.050409216000000034,"total_completion_cost_bin":0.12352331999999999,"total_reasoning_cost_bin":0.11849963999999996,"total_cost_bin":0.4363853160000001,"total_on_demand_input_cost_bin":0.3959992199999997,"ci_99_lower":0.16389897956958185,"ci_99_upper":0.36158343759215056,"ci_98_lower":0.17461304870565217,"ci_98_upper":0.35553433956894837,"ci_95_lower":0.18730839501249205,"ci_95_upper":0.33651782415832115,"ci_90_lower":0.1961150860543887,"ci_90_upper":0.32649119433255197,"ci_75_lower":0.21432614271829667,"ci_75_upper":0.30653434522869294,"ci_50_lower":0.23318832198799888,"ci_50_upper":0.2876759806678414},"262144":{"avg_score":0.14403179038567007,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.30451768642268484,"median_score":0.02983539178967476,"q1_score":0.02012808807194233,"q3_score":0.04398447647690773,"avg_token_efficiency":0.9831873780443089,"is_incomplete":false,"total_prompt_cost_bin":0.44224577999999976,"total_cache_read_cost_bin":0.12941107199999993,"total_completion_cost_bin":0.17619815999999994,"total_reasoning_cost_bin":0.1699750800000001,"total_cost_bin":0.9178300919999998,"total_on_demand_input_cost_bin":1.0893011400000001,"ci_99_lower":0.08399960664227267,"ci_99_upper":0.21188011795663808,"ci_98_lower":0.08907960186354116,"ci_98_upper":0.20170491428941764,"ci_95_lower":0.09483032578784899,"ci_95_upper":0.19518209190726288,"ci_90_lower":0.10398914637671558,"ci_90_upper":0.18730045189108394,"ci_75_lower":0.11390458911319647,"ci_75_upper":0.17467857868198455,"ci_50_lower":0.12565494655856058,"ci_50_upper":0.15901090992872224}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":524288,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Upstage: Solar Pro 4","provider_name":"Upstage","run_provider_names":["Upstage"],"pricing_metadata":{"prompt":3e-8,"completion":1.2e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-10T14:20:36.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T00:48:47.881Z"},{"model_slug":"openai/gpt-5.4-nano","reasoning_mode":"high","overall_metrics":{"auc_1m":0.09837879788652347,"auc_128k":0.29332386552248585,"cum_avg_1m":0.28722094240335827,"cum_avg_128k":0.40687212079266705,"avg_score_overall":0.3616750489667058,"avg_response_time_ms":18768.681732226563,"total_prompt_cost":0.983344800000001,"total_cache_read_cost":1.1083776000000005,"total_completion_cost":1.6644762500000003,"total_reasoning_cost":1.3663675,"total_prompt_cost_128k":0.24957940000000003,"total_completion_cost_128k":1.2565500000000003,"total_on_demand_input_cost":12.067120799999998,"total_on_demand_input_cost_128k":4.810782599999998,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061808102971},"bin_metrics":{"8192":{"avg_score":0.744781776657328,"min_score":0.01769087463617325,"max_score":1,"n_tests":80,"stddev":0.4253208001643442,"median_score":1,"q1_score":0.12096920609474182,"q3_score":1,"avg_token_efficiency":0.9759044449598386,"is_incomplete":false,"total_prompt_cost_bin":0.019767999999999987,"total_cache_read_cost_bin":0.008407040000000006,"total_completion_cost_bin":0.11934875000000002,"total_reasoning_cost_bin":0.08485625000000001,"total_cost_bin":0.23238004,"total_on_demand_input_cost_bin":0.10383840000000004,"ci_99_lower":0.6223671707698376,"ci_99_upper":0.8488266373348888,"ci_98_lower":0.6318576141258236,"ci_98_upper":0.8420757961480412,"ci_95_lower":0.6485367209860123,"ci_95_upper":0.8311056136665865,"ci_90_lower":0.6627570831403137,"ci_90_upper":0.824191332575865,"ci_75_lower":0.6873285412264521,"ci_75_upper":0.7956916160648688,"ci_50_lower":0.71110760680167,"ci_50_upper":0.7779778525757137},"16384":{"avg_score":0.48853188583161683,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.48873604185076136,"median_score":0.04824812524020672,"q1_score":0.025880069471895695,"q3_score":1,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.014737599999999995,"total_cache_read_cost_bin":0.019840000000000007,"total_completion_cost_bin":0.15922250000000002,"total_reasoning_cost_bin":0.12519749999999996,"total_cost_bin":0.3189976,"total_on_demand_input_cost_bin":0.21313760000000004,"ci_99_lower":0.3435370454490767,"ci_99_upper":0.6220545488553471,"ci_98_lower":0.35768018061609475,"ci_98_upper":0.6096599024927709,"ci_95_lower":0.3779021029805881,"ci_95_upper":0.5886150668517801,"ci_90_lower":0.394901992114028,"ci_90_upper":0.5750309633585853,"ci_75_lower":0.42802229366934624,"ci_75_upper":0.5497237143543316,"ci_50_lower":0.45200395413558003,"ci_50_upper":0.5250349523004844},"32768":{"avg_score":0.3561638858261765,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4522693793460123,"median_score":0.0456022173166275,"q1_score":0.024995672050863504,"q3_score":1,"avg_token_efficiency":0.9952100768549628,"is_incomplete":false,"total_prompt_cost_bin":0.04694080000000003,"total_cache_read_cost_bin":0.07250432000000018,"total_completion_cost_bin":0.37429875000000007,"total_reasoning_cost_bin":0.3082849999999999,"total_cost_bin":0.8020288700000002,"total_on_demand_input_cost_bin":0.7719840000000008,"ci_99_lower":0.26045911266882765,"ci_99_upper":0.4595820484563164,"ci_98_lower":0.26416548554598945,"ci_98_upper":0.4534666163495721,"ci_95_lower":0.2799455522928517,"ci_95_upper":0.43147715492043914,"ci_90_lower":0.2912474412468488,"ci_90_upper":0.42102157237223925,"ci_75_lower":0.3120750046670283,"ci_75_upper":0.4037560836188598,"ci_50_lower":0.328825021695058,"ci_50_upper":0.3819048286677571},"65536":{"avg_score":0.22332269284490716,"min_score":0.002958579920232296,"max_score":1,"n_tests":85,"stddev":0.3899981357972883,"median_score":0.02871437929570675,"q1_score":0.022916197776794434,"q3_score":0.052147239446640015,"avg_token_efficiency":0.9776435846759911,"is_incomplete":false,"total_prompt_cost_bin":0.06277540000000004,"total_cache_read_cost_bin":0.10243583999999988,"total_completion_cost_bin":0.27645000000000014,"total_reasoning_cost_bin":0.23413874999999995,"total_cost_bin":0.67579999,"total_on_demand_input_cost_bin":1.0871337999999988,"ci_99_lower":0.1318244350867565,"ci_99_upper":0.33728732126617034,"ci_98_lower":0.13571079146075882,"ci_98_upper":0.3283125047845053,"ci_95_lower":0.1512489348538357,"ci_95_upper":0.30521181152695237,"ci_90_lower":0.15703102015465606,"ci_90_upper":0.29337479175999764,"ci_75_lower":0.17494539029533318,"ci_75_upper":0.2684939315864433,"ci_50_lower":0.1904333946014316,"ci_50_upper":0.2468285947716247},"131072":{"avg_score":0.22156036280330668,"min_score":0.0047505940310657024,"max_score":1,"n_tests":103,"stddev":0.3874102875753568,"median_score":0.029565032571554184,"q1_score":0.021262059919536114,"q3_score":0.049221303313970566,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":0.10535759999999997,"total_cache_read_cost_bin":0.25293312000000034,"total_completion_cost_bin":0.32723,"total_reasoning_cost_bin":0.2731062500000001,"total_cost_bin":0.9586269700000004,"total_on_demand_input_cost_bin":2.6346887999999984,"ci_99_lower":0.1296086040813277,"ci_99_upper":0.32326720695906475,"ci_98_lower":0.13808738351419783,"ci_98_upper":0.309524913718291,"ci_95_lower":0.15343574513918584,"ci_95_upper":0.2913056310861896,"ci_90_lower":0.1632519262096872,"ci_90_upper":0.2790920752496232,"ci_75_lower":0.17842353174001774,"ci_75_upper":0.2624342171350964,"ci_50_lower":0.19688942060846543,"ci_50_upper":0.24496995673618882},"262144":{"avg_score":0.2634069352635308,"min_score":0.005073995795100927,"max_score":1,"n_tests":141,"stddev":0.4036790304883234,"median_score":0.03874603658914566,"q1_score":0.025517573580145836,"q3_score":0.1748807579278946,"avg_token_efficiency":0.9824198103380463,"is_incomplete":false,"total_prompt_cost_bin":0.733765400000001,"total_cache_read_cost_bin":0.65225728,"total_completion_cost_bin":0.40792624999999993,"total_reasoning_cost_bin":0.34078375,"total_cost_bin":2.134732680000001,"total_on_demand_input_cost_bin":7.2563382,"ci_99_lower":0.17965114096729481,"ci_99_upper":0.3462581139377728,"ci_98_lower":0.18794247520516183,"ci_98_upper":0.34026974882256794,"ci_95_lower":0.19971312717220857,"ci_95_upper":0.3284714346805237,"ci_90_lower":0.20882140833055537,"ci_90_upper":0.3168889401535048,"ci_75_lower":0.224266415436303,"ci_75_upper":0.3015236708765737,"ci_50_lower":0.24109328969824292,"ci_50_upper":0.28448945060884245}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Nano","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":2e-7,"completion":0.00000125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:47.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T03:25:14.780Z"},{"model_slug":"qwen/qwen3.5-27b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.09800115932473165,"auc_128k":0.6044322768781759,"cum_avg_1m":0.46102305374620073,"cum_avg_128k":0.7376368859939212,"avg_score_overall":0.733358321043003,"avg_response_time_ms":249841.17119954427,"total_prompt_cost":4.7743218899999995,"total_cache_read_cost":0,"total_completion_cost":4.72638816,"total_reasoning_cost":4.425919680000001,"total_prompt_cost_128k":4.7743218899999995,"total_completion_cost_128k":4.72638816,"total_on_demand_input_cost":4.7743218899999995,"total_on_demand_input_cost_128k":4.7743218899999995,"total_runs":480,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207454527517659},"bin_metrics":{"8192":{"avg_score":0.9999975472688675,"min_score":0.9998037815093994,"max_score":1,"n_tests":80,"stddev":0.00002193789417126568,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222286,"is_incomplete":false,"total_prompt_cost_bin":0.10596144000000006,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.5072433599999999,"total_reasoning_cost_bin":0.46204548,"total_cost_bin":1.0752502800000001,"total_on_demand_input_cost_bin":0.10596144000000006,"ci_99_lower":0.99999018907547,"ci_99_upper":1,"ci_98_lower":0.99999018907547,"ci_98_upper":1,"ci_95_lower":0.9999926418066025,"ci_95_upper":1,"ci_90_lower":0.9999926418066025,"ci_90_upper":1,"ci_75_lower":0.9999950945377349,"ci_75_upper":1,"ci_50_lower":0.9999950945377349,"ci_50_upper":1},"16384":{"avg_score":0.9398692955961451,"min_score":0.02601809985935688,"max_score":1,"n_tests":80,"stddev":0.23417706919257403,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0159704488335684,"is_incomplete":false,"total_prompt_cost_bin":0.21853727999999972,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.66532596,"total_reasoning_cost_bin":0.6205633200000001,"total_cost_bin":1.5044265599999997,"total_on_demand_input_cost_bin":0.21853727999999972,"ci_99_lower":0.8562308589628664,"ci_99_upper":0.9882144349520094,"ci_98_lower":0.8676826083834748,"ci_98_upper":0.9881131817884744,"ci_95_lower":0.8794072298490209,"ci_95_upper":0.9878711223788559,"ci_90_lower":0.8916028563876172,"ci_90_upper":0.9761749144690112,"ci_75_lower":0.9040865859715268,"ci_75_upper":0.9642947332118638,"ci_50_lower":0.9273374027281535,"ci_50_upper":0.9524160655681043},"32768":{"avg_score":0.829436105405501,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.36923948634696074,"median_score":1,"q1_score":0.9999550729990005,"q3_score":1,"avg_token_efficiency":1.0256830410679845,"is_incomplete":false,"total_prompt_cost_bin":0.7754385599999994,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.2511309199999996,"total_reasoning_cost_bin":1.1642794800000003,"total_cost_bin":3.1908489599999994,"total_on_demand_input_cost_bin":0.7754385599999994,"ci_99_lower":0.7444519504222177,"ci_99_upper":0.9009769318311011,"ci_98_lower":0.7521750342360739,"ci_98_upper":0.8946654492212966,"ci_95_lower":0.7650703127173234,"ci_95_upper":0.887174752701043,"ci_90_lower":0.772333428449244,"ci_90_upper":0.8797715856837491,"ci_75_lower":0.7875384068649831,"ci_75_upper":0.8651379438250681,"ci_50_lower":0.8080034612563287,"ci_50_upper":0.8509742940788034},"65536":{"avg_score":0.4964235965655569,"min_score":0.005082591902464628,"max_score":1,"n_tests":85,"stddev":0.48871709167125066,"median_score":0.06811477243900299,"q1_score":0.024004174396395683,"q3_score":1,"avg_token_efficiency":1.0209598255175445,"is_incomplete":false,"total_prompt_cost_bin":1.1069186699999995,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.9309830400000001,"total_reasoning_cost_bin":0.8754860400000004,"total_cost_bin":2.91338775,"total_on_demand_input_cost_bin":1.1069186699999995,"ci_99_lower":0.35853671522509745,"ci_99_upper":0.6337736892018686,"ci_98_lower":0.3693002544553293,"ci_98_upper":0.6215715329244403,"ci_95_lower":0.3927985529667315,"ci_95_upper":0.6020291801138012,"ci_90_lower":0.4059192963122555,"ci_90_upper":0.587959961529054,"ci_75_lower":0.43817084635032666,"ci_75_upper":0.5555111031006912,"ci_50_lower":0.46228744384887466,"ci_50_upper":0.5310295845217564},"131072":{"avg_score":0.42245788513353527,"min_score":0.007272727321833372,"max_score":1,"n_tests":99,"stddev":0.4778260171224326,"median_score":0.0400702990591526,"q1_score":0.02680982928723097,"q3_score":1,"avg_token_efficiency":1.01622142458734,"is_incomplete":false,"total_prompt_cost_bin":2.5674659400000013,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.3717048800000005,"total_reasoning_cost_bin":1.3035453600000002,"total_cost_bin":5.242716180000002,"total_on_demand_input_cost_bin":2.5674659400000013,"ci_99_lower":0.2866624665330635,"ci_99_upper":0.5410910327491031,"ci_98_lower":0.3030875948975991,"ci_98_upper":0.5329392026616447,"ci_95_lower":0.3236458332855207,"ci_95_upper":0.5118091937742487,"ci_90_lower":0.34438583439855713,"ci_90_upper":0.5007216397621151,"ci_75_lower":0.36560637818712904,"ci_75_upper":0.48017407284821434,"ci_50_lower":0.3915180592889418,"ci_50_upper":0.4523942101984332}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-27B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":1.95e-7,"completion":0.00000156,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-25T21:10:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T07:56:16.069Z"},{"model_slug":"qwen/qwen3.6-27b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.09423294178944702,"auc_128k":0.5176587510764523,"cum_avg_1m":0.34300971663519964,"cum_avg_128k":0.5488155466163194,"avg_score_overall":0.5377715676445668,"avg_response_time_ms":16571.14185049514,"total_prompt_cost":7.504995599999998,"total_cache_read_cost":0,"total_completion_cost":0.37943000000000005,"total_reasoning_cost":0,"total_prompt_cost_128k":7.504995599999998,"total_completion_cost_128k":0.37943000000000005,"total_on_demand_input_cost":7.504995599999998,"total_on_demand_input_cost_128k":7.504995599999998,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0208128870566135},"bin_metrics":{"8192":{"avg_score":0.686197330133291,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4553815160558284,"median_score":1,"q1_score":0.0456464234739542,"q3_score":1,"avg_token_efficiency":1.0228079512716766,"is_incomplete":false,"total_prompt_cost_bin":0.16306559999999992,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.05743199999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.2204975999999999,"total_on_demand_input_cost_bin":0.16306559999999992,"ci_99_lower":0.5405433820823964,"ci_99_upper":0.8077086515104457,"ci_98_lower":0.562847796689195,"ci_98_upper":0.795479442992655,"ci_95_lower":0.5881974432524293,"ci_95_upper":0.7819539166726462,"ci_90_lower":0.6016737097012811,"ci_90_upper":0.7598633945494658,"ci_75_lower":0.6270488885740633,"ci_75_upper":0.7441623497630644,"ci_50_lower":0.6504992963251426,"ci_50_upper":0.7219389997291727},"16384":{"avg_score":0.550457351183286,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4878387381294089,"median_score":1,"q1_score":0.030190876219421625,"q3_score":1,"avg_token_efficiency":1.016119475838464,"is_incomplete":false,"total_prompt_cost_bin":0.33625919999999976,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.055692000000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.3919511999999998,"total_on_demand_input_cost_bin":0.33625919999999976,"ci_99_lower":0.40404930067871464,"ci_99_upper":0.6972043233184959,"ci_98_lower":0.4258992618433549,"ci_98_upper":0.6734387143407947,"ci_95_lower":0.43995771843066905,"ci_95_upper":0.6501545791109673,"ci_90_lower":0.46432415695657253,"ci_90_upper":0.6365101023137686,"ci_75_lower":0.4892486429154814,"ci_75_upper":0.6128860568373057,"ci_50_lower":0.5139606240816648,"ci_50_upper":0.5865729666838888},"32768":{"avg_score":0.46592205033555406,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.48402824009276185,"median_score":0.05999325215816498,"q1_score":0.025060547050088644,"q3_score":1,"avg_token_efficiency":1.025754859257966,"is_incomplete":false,"total_prompt_cost_bin":1.1930639999999995,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.10905600000000004,"total_reasoning_cost_bin":0,"total_cost_bin":1.3021199999999995,"total_on_demand_input_cost_bin":1.1930639999999995,"ci_99_lower":0.35741532302374257,"ci_99_upper":0.5691562418872319,"ci_98_lower":0.3735893093827454,"ci_98_upper":0.558234577603426,"ci_95_lower":0.38610922438914286,"ci_95_upper":0.5460577013111427,"ci_90_lower":0.396626655998989,"ci_90_upper":0.5356347147977966,"ci_75_lower":0.41995680068366176,"ci_75_upper":0.5160867442297304,"ci_50_lower":0.4373491995836102,"ci_50_upper":0.49513963435482067},"65536":{"avg_score":0.5161632087903426,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.49254122868886313,"median_score":0.9968010187149048,"q1_score":0.02028311789035797,"q3_score":1,"avg_token_efficiency":1.0209904012221067,"is_incomplete":false,"total_prompt_cost_bin":1.7030028000000006,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.070186,"total_reasoning_cost_bin":0,"total_cost_bin":1.7731888000000007,"total_on_demand_input_cost_bin":1.7030028000000006,"ci_99_lower":0.376430107628954,"ci_99_upper":0.6522582622907289,"ci_98_lower":0.38870878353884775,"ci_98_upper":0.6430909479181008,"ci_95_lower":0.4104102586402887,"ci_95_upper":0.6204451647561098,"ci_90_lower":0.42529932002071297,"ci_90_upper":0.6083937411404707,"ci_75_lower":0.45684045856503547,"ci_75_upper":0.5840619283070897,"ci_50_lower":0.48123000567121543,"ci_50_upper":0.5509189245934762},"131072":{"avg_score":0.5253377926391233,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.49102767100520855,"median_score":0.9996252059936523,"q1_score":0.02194868866354227,"q3_score":1,"avg_token_efficiency":1.0162368757633715,"is_incomplete":false,"total_prompt_cost_bin":4.109603999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08706400000000002,"total_reasoning_cost_bin":0,"total_cost_bin":4.196667999999998,"total_on_demand_input_cost_bin":4.109603999999998,"ci_99_lower":0.3931535895259653,"ci_99_upper":0.6489547729772714,"ci_98_lower":0.4193934288727644,"ci_98_upper":0.6385551973628513,"ci_95_lower":0.4311232154371751,"ci_95_upper":0.6199409570179888,"ci_90_lower":0.4487450629748513,"ci_90_upper":0.6099844573560687,"ci_75_lower":0.4684490061170666,"ci_75_upper":0.5823831959645652,"ci_50_lower":0.49353609553615124,"ci_50_upper":0.5626065321854379}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 27B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":3.2e-7,"completion":0.0000032,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-27T01:57:44.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T03:56:24.091Z"},{"model_slug":"nex-agi/nex-n2-mini","reasoning_mode":null,"overall_metrics":{"auc_1m":0.09246867684867574,"auc_128k":0.6025962134587793,"cum_avg_1m":0.4292271932975296,"cum_avg_128k":0.6867635092760473,"avg_score_overall":0.6788788674496927,"avg_response_time_ms":57760.94034847971,"total_prompt_cost":0.5854818500000002,"total_cache_read_cost":0.05103616000000002,"total_completion_cost":0.1766923,"total_reasoning_cost":0.15842749999999997,"total_prompt_cost_128k":0.5854818500000002,"total_completion_cost_128k":0.1766923,"total_on_demand_input_cost":1.0958434499999998,"total_on_demand_input_cost_128k":1.0958434499999998,"total_runs":472,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6251803382741377},"bin_metrics":{"8192":{"avg_score":0.9033882021671161,"min_score":0.013846153393387794,"max_score":1,"n_tests":80,"stddev":0.29134441701493036,"median_score":1,"q1_score":0.9996924996376038,"q3_score":1,"avg_token_efficiency":1.0223419298475052,"is_incomplete":false,"total_prompt_cost_bin":0.013582800000000011,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.015382200000000004,"total_reasoning_cost_bin":0.012496500000000002,"total_cost_bin":0.04146150000000002,"total_on_demand_input_cost_bin":0.013582800000000011,"ci_99_lower":0.8177400256206747,"ci_99_upper":0.9755611188897164,"ci_98_lower":0.8182515349427704,"ci_98_upper":0.9643259272542781,"ci_95_lower":0.8303064284950962,"ci_95_upper":0.9629162058135262,"ci_90_lower":0.8428999885485973,"ci_90_upper":0.9517248977674172,"ci_75_lower":0.8659399807278533,"ci_75_upper":0.9394571004755563,"ci_50_lower":0.8795552648080047,"ci_50_upper":0.9271318423561753},"16384":{"avg_score":0.8537577636656352,"min_score":0.01270289346575737,"max_score":1,"n_tests":80,"stddev":0.3496432775601145,"median_score":1,"q1_score":0.9993381202220917,"q3_score":1,"avg_token_efficiency":1.01589593533112,"is_incomplete":false,"total_prompt_cost_bin":0.028015600000000015,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.021442600000000003,"total_reasoning_cost_bin":0.018658299999999996,"total_cost_bin":0.06811650000000002,"total_on_demand_input_cost_bin":0.028015600000000015,"ci_99_lower":0.7434994127533282,"ci_99_upper":0.9511366892050137,"ci_98_lower":0.756810485261958,"ci_98_upper":0.9387494068886153,"ci_95_lower":0.7693264302937314,"ci_95_upper":0.9264361183420988,"ci_90_lower":0.7813681283348705,"ci_90_upper":0.9144519357581157,"ci_75_lower":0.8050853887267294,"ci_75_upper":0.9018977059618919,"ci_50_lower":0.8293012836249546,"ci_50_upper":0.8780844755383441},"32768":{"avg_score":0.6924241128367344,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.447592768306933,"median_score":0.9996320903301239,"q1_score":0.060373575426638126,"q3_score":1,"avg_token_efficiency":1.8564429198051478,"is_incomplete":false,"total_prompt_cost_bin":0.09941180000000012,"total_cache_read_cost_bin":0.008273919999999985,"total_completion_cost_bin":0.046552100000000006,"total_reasoning_cost_bin":0.04113829999999999,"total_cost_bin":0.1953761200000001,"total_on_demand_input_cost_bin":0.18215100000000006,"ci_99_lower":0.5857777479311506,"ci_99_upper":0.784060995100667,"ci_98_lower":0.5950379971854614,"ci_98_upper":0.7771123740901341,"ci_95_lower":0.6094730889309122,"ci_95_upper":0.7675241838411345,"ci_90_lower":0.6322746328385513,"ci_90_upper":0.7554397738136365,"ci_75_lower":0.6471216259119769,"ci_75_upper":0.7339487444974913,"ci_50_lower":0.6672034978529315,"ci_50_upper":0.7172476559708765},"65536":{"avg_score":0.6461751229732352,"min_score":0.00840978603810072,"max_score":1,"n_tests":85,"stddev":0.46939240965677564,"median_score":0.9994645714759827,"q1_score":0.03412401303648949,"q3_score":1,"avg_token_efficiency":1.9992744880529731,"is_incomplete":false,"total_prompt_cost_bin":0.14191052500000015,"total_cache_read_cost_bin":0.01359872000000001,"total_completion_cost_bin":0.0501739,"total_reasoning_cost_bin":0.0467115,"total_cost_bin":0.25239464500000014,"total_on_demand_input_cost_bin":0.2778977249999998,"ci_99_lower":0.5077820703039931,"ci_99_upper":0.7709079903120942,"ci_98_lower":0.5186879182627754,"ci_98_upper":0.7587556716708795,"ci_95_lower":0.5436378000128795,"ci_95_upper":0.7391611911430407,"ci_90_lower":0.5652252152994933,"ci_90_upper":0.7263755988970617,"ci_75_lower":0.5881102356991238,"ci_75_upper":0.7032283269060665,"ci_50_lower":0.6114361743634457,"ci_50_upper":0.6803543582120362},"131072":{"avg_score":0.33807234473751635,"min_score":0,"max_score":1,"n_tests":91,"stddev":0.455306441308965,"median_score":0.03692571818828583,"q1_score":0.02176347840577364,"q3_score":0.9994844794273376,"avg_token_efficiency":1.9957310095945073,"is_incomplete":false,"total_prompt_cost_bin":0.3025611249999999,"total_cache_read_cost_bin":0.029163520000000023,"total_completion_cost_bin":0.0431415,"total_reasoning_cost_bin":0.03942289999999999,"total_cost_bin":0.41428904499999986,"total_on_demand_input_cost_bin":0.5941963250000001,"ci_99_lower":0.21832288213440085,"ci_99_upper":0.4551492652788266,"ci_98_lower":0.22951052106599643,"ci_98_upper":0.44487325618524554,"ci_95_lower":0.24977831134219539,"ci_95_upper":0.4329669657229845,"ci_90_lower":0.2619316937534937,"ci_90_upper":0.4121330211092564,"ci_75_lower":0.2843329111474904,"ci_75_upper":0.39142862283675883,"ci_50_lower":0.30572820921264754,"ci_50_upper":0.3701925705184464}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Nex AGI: Nex-N2-Mini","provider_name":"Nex-agi","run_provider_names":["Nex AGI"],"pricing_metadata":{"prompt":2.5e-8,"completion":1e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-06-24T14:56:04.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T01:05:47.312Z"},{"model_slug":"minimax/minimax-m3","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.0924097562117389,"auc_128k":0.3924427649594915,"cum_avg_1m":0.35531134299144673,"cum_avg_128k":0.5313775845585363,"avg_score_overall":0.4379853424459696,"avg_response_time_ms":52077.647469140626,"total_prompt_cost":0.8865299999999998,"total_cache_read_cost":3.367690740000004,"total_completion_cost":2.237172,"total_reasoning_cost":2.2087116,"total_prompt_cost_128k":0.4275959999999999,"total_completion_cost_128k":1.5544872000000003,"total_on_demand_input_cost":17.72498369999998,"total_on_demand_input_cost_128k":7.074586199999995,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9662257896676263},"bin_metrics":{"8192":{"avg_score":0.912677965639159,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.2761694208511948,"median_score":1,"q1_score":0.9976594597101212,"q3_score":1,"avg_token_efficiency":0.9813365228541354,"is_incomplete":false,"total_prompt_cost_bin":0.035375099999999965,"total_cache_read_cost_bin":0.023872199999999993,"total_completion_cost_bin":0.08442360000000002,"total_reasoning_cost_bin":0.059988,"total_cost_bin":0.20365889999999998,"total_on_demand_input_cost_bin":0.15473609999999985,"ci_99_lower":0.8177894427160499,"ci_99_upper":0.983528434605687,"ci_98_lower":0.8266552899607923,"ci_98_upper":0.9744091182015836,"ci_95_lower":0.8412951085664099,"ci_95_upper":0.9712723089358769,"ci_90_lower":0.8526346893631853,"ci_90_upper":0.9611878808471374,"ci_75_lower":0.8755570535751758,"ci_75_upper":0.9482685242837761,"ci_50_lower":0.8891957785235718,"ci_50_upper":0.9359476430807262},"16384":{"avg_score":0.6933209315291606,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.45158671298298375,"median_score":0.9978914558887482,"q1_score":0.04365826118737459,"q3_score":1,"avg_token_efficiency":0.9597271169467988,"is_incomplete":false,"total_prompt_cost_bin":0.015160500000000004,"total_cache_read_cost_bin":0.05943828000000005,"total_completion_cost_bin":0.18643560000000006,"total_reasoning_cost_bin":0.17062200000000002,"total_cost_bin":0.4316563800000002,"total_on_demand_input_cost_bin":0.3123518999999998,"ci_99_lower":0.5474815697813173,"ci_99_upper":0.8051085822632885,"ci_98_lower":0.571094836143544,"ci_98_upper":0.8008369602016173,"ci_95_lower":0.595644346898771,"ci_95_upper":0.7891309627983719,"ci_90_lower":0.608821134826867,"ci_90_upper":0.7746735453431028,"ci_75_lower":0.6335416018730029,"ci_75_upper":0.7430667041073321,"ci_50_lower":0.6574567787611159,"ci_50_upper":0.7287298934184947},"32768":{"avg_score":0.5220935317252845,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4798063898497532,"median_score":0.9268854856491089,"q1_score":0.032901338301599026,"q3_score":1,"avg_token_efficiency":0.977133847209817,"is_incomplete":false,"total_prompt_cost_bin":0.09117120000000002,"total_cache_read_cost_bin":0.20902752000000013,"total_completion_cost_bin":0.4851828000000002,"total_reasoning_cost_bin":0.47700839999999994,"total_cost_bin":1.2623899200000002,"total_on_demand_input_cost_bin":1.1363088000000003,"ci_99_lower":0.41551485787916187,"ci_99_upper":0.6289558185569117,"ci_98_lower":0.42544119920814405,"ci_98_upper":0.6154656064349076,"ci_95_lower":0.4415261344165843,"ci_95_upper":0.5988558222935192,"ci_90_lower":0.4541277720908778,"ci_90_upper":0.5873870066996246,"ci_75_lower":0.4767941888426054,"ci_75_upper":0.5717868691424561,"ci_50_lower":0.4944964958421598,"ci_50_upper":0.5504239181969109},"65536":{"avg_score":0.35442926175344514,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.45782611969634696,"median_score":0.03412401303648949,"q1_score":0.0217162873595953,"q3_score":0.9939780235290527,"avg_token_efficiency":0.9601521233041697,"is_incomplete":false,"total_prompt_cost_bin":0.1339394999999999,"total_cache_read_cost_bin":0.2935171200000001,"total_completion_cost_bin":0.32852639999999994,"total_reasoning_cost_bin":0.3275256,"total_cost_bin":1.08350862,"total_on_demand_input_cost_bin":1.601525099999999,"ci_99_lower":0.24046706145437544,"ci_99_upper":0.48117603942179393,"ci_98_lower":0.24289212297971413,"ci_98_upper":0.4700685086056371,"ci_95_lower":0.26169735027025176,"ci_95_upper":0.4558817953387659,"ci_90_lower":0.274065905334428,"ci_90_upper":0.4379677215185672,"ci_75_lower":0.2953395006349584,"ci_75_upper":0.41131453105412863,"ci_50_lower":0.32054187065428674,"ci_50_upper":0.38797098707873373},"131072":{"avg_score":0.17436623214563193,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.35230117319587184,"median_score":0.02673669159412384,"q1_score":0.018412547186017036,"q3_score":0.03505023010075092,"avg_token_efficiency":0.9569037694470507,"is_incomplete":false,"total_prompt_cost_bin":0.15194970000000002,"total_cache_read_cost_bin":0.7435429200000013,"total_completion_cost_bin":0.4699188,"total_reasoning_cost_bin":0.47089559999999997,"total_cost_bin":1.8363070200000013,"total_on_demand_input_cost_bin":3.869664299999996,"ci_99_lower":0.08868415625440408,"ci_99_upper":0.2666373376433344,"ci_98_lower":0.09969293681974747,"ci_98_upper":0.25798887463313647,"ci_95_lower":0.11632385307366877,"ci_95_upper":0.24138077557097115,"ci_90_lower":0.11892663727568699,"ci_90_upper":0.23129747826730362,"ci_75_lower":0.1361796184003837,"ci_75_upper":0.21220497114191655,"ci_50_lower":0.1477853873427008,"ci_50_upper":0.19337643372365138},"262144":{"avg_score":0.1856028211388922,"min_score":0.002959674457088113,"max_score":1,"n_tests":141,"stddev":0.3501629272933846,"median_score":0.03001464158296585,"q1_score":0.021112464368343353,"q3_score":0.062209054827690125,"avg_token_efficiency":0.9612894000266375,"is_incomplete":false,"total_prompt_cost_bin":0.4589339999999999,"total_cache_read_cost_bin":2.038292700000002,"total_completion_cost_bin":0.6826847999999999,"total_reasoning_cost_bin":0.702672,"total_cost_bin":3.8825835000000017,"total_on_demand_input_cost_bin":10.650397499999984,"ci_99_lower":0.11235355655379521,"ci_99_upper":0.2555717027825237,"ci_98_lower":0.11940362781613192,"ci_98_upper":0.25106095937827055,"ci_95_lower":0.13016380491990148,"ci_95_upper":0.2462235224555438,"ci_90_lower":0.13914177236370145,"ci_90_upper":0.23398086725823342,"ci_75_lower":0.15151373674066784,"ci_75_upper":0.21918112904548395,"ci_50_lower":0.16498125754775653,"ci_50_upper":0.20343702244663164}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"MiniMax: MiniMax M3","provider_name":"Minimax","run_provider_names":["MiniMax"],"pricing_metadata":{"prompt":3e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-31T16:36:14.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T04:49:25.570Z"},{"model_slug":"nex-agi/nex-n2-mini","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.09040053434345477,"auc_128k":0.5920612466597194,"cum_avg_1m":0.4208251134042775,"cum_avg_128k":0.6733201814468439,"avg_score_overall":0.6624174152826794,"avg_response_time_ms":68753.00271991864,"total_prompt_cost":0.6220551500000004,"total_cache_read_cost":0.054640640000000004,"total_completion_cost":0.18623260000000005,"total_reasoning_cost":0.1672342,"total_prompt_cost_128k":0.6220551500000004,"total_completion_cost_128k":0.18623260000000005,"total_on_demand_input_cost":1.1684615500000004,"total_on_demand_input_cost_128k":1.1684615500000004,"total_runs":483,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6320455675552938},"bin_metrics":{"8192":{"avg_score":0.8682720754179172,"min_score":0.008579599671065807,"max_score":1,"n_tests":80,"stddev":0.3320043932747088,"median_score":1,"q1_score":0.9996395856142044,"q3_score":1,"avg_token_efficiency":1.022341929847504,"is_incomplete":false,"total_prompt_cost_bin":0.01358280000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0156284,"total_reasoning_cost_bin":0.012746700000000003,"total_cost_bin":0.04195790000000001,"total_on_demand_input_cost_bin":0.01358280000000001,"ci_99_lower":0.7601175634629327,"ci_99_upper":0.9533602625650819,"ci_98_lower":0.7817965996017446,"ci_98_upper":0.9409176725891885,"ci_95_lower":0.78631338711828,"ci_95_upper":0.9289718087675283,"ci_90_lower":0.797838792426628,"ci_90_upper":0.9271823944506469,"ci_75_lower":0.822147049056366,"ci_75_upper":0.9050402810331434,"ci_50_lower":0.843843854239094,"ci_50_upper":0.8923223836289254},"16384":{"avg_score":0.8292202303768136,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.3723111347792176,"median_score":0.9997917413711548,"q1_score":0.9991132915019989,"q3_score":1,"avg_token_efficiency":1.01589593533112,"is_incomplete":false,"total_prompt_cost_bin":0.028015600000000022,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.020128700000000006,"total_reasoning_cost_bin":0.0173123,"total_cost_bin":0.06545660000000003,"total_on_demand_input_cost_bin":0.028015600000000022,"ci_99_lower":0.7076429925476551,"ci_99_upper":0.927208531732089,"ci_98_lower":0.7191090090209619,"ci_98_upper":0.9153795528197659,"ci_95_lower":0.7439683095170768,"ci_95_upper":0.9028347692935496,"ci_90_lower":0.7566751821123763,"ci_90_upper":0.8903864122013329,"ci_75_lower":0.7804185334200157,"ci_75_upper":0.8774129194527632,"ci_50_lower":0.8042875624581938,"ci_50_upper":0.8539361856732284},"32768":{"avg_score":0.72057066719486,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4345045169933657,"median_score":0.9997232258319855,"q1_score":0.0870493371039629,"q3_score":1,"avg_token_efficiency":1.8564429198051478,"is_incomplete":false,"total_prompt_cost_bin":0.09941180000000015,"total_cache_read_cost_bin":0.008273919999999985,"total_completion_cost_bin":0.04426360000000001,"total_reasoning_cost_bin":0.0388099,"total_cost_bin":0.19075922000000015,"total_on_demand_input_cost_bin":0.18215100000000012,"ci_99_lower":0.6161589297991903,"ci_99_upper":0.8122728206363836,"ci_98_lower":0.6236695323532503,"ci_98_upper":0.8032521582503036,"ci_95_lower":0.6452276373032124,"ci_95_upper":0.7901000736719545,"ci_90_lower":0.6572070695624194,"ci_90_upper":0.7805613388314241,"ci_75_lower":0.6772089791090419,"ci_75_upper":0.76270191078177,"ci_50_lower":0.6939176608815187,"ci_50_upper":0.7438822554883219},"65536":{"avg_score":0.6235442890292582,"min_score":0.004698512144386768,"max_score":1,"n_tests":85,"stddev":0.47504805933181493,"median_score":0.9995090961456299,"q1_score":0.03230403736233711,"q3_score":1,"avg_token_efficiency":1.975700111733825,"is_incomplete":false,"total_prompt_cost_bin":0.1419105250000002,"total_cache_read_cost_bin":0.01327104000000001,"total_completion_cost_bin":0.04688640000000002,"total_reasoning_cost_bin":0.043399400000000005,"total_cost_bin":0.24546736500000024,"total_on_demand_input_cost_bin":0.27462092499999996,"ci_99_lower":0.48481885653071327,"ci_99_upper":0.7385949326968916,"ci_98_lower":0.5063289583144599,"ci_98_upper":0.7375653655273731,"ci_95_lower":0.5211812666203718,"ci_95_upper":0.7252619749611682,"ci_90_lower":0.5413811993905727,"ci_90_upper":0.713349572236595,"ci_75_lower":0.565896059152232,"ci_75_upper":0.6813016532002674,"ci_50_lower":0.5887063593904981,"ci_50_upper":0.6576086566372611},"131072":{"avg_score":0.3249936452153705,"min_score":0.0072053526528179646,"max_score":1,"n_tests":102,"stddev":0.4482276350258271,"median_score":0.03608946315944195,"q1_score":0.025573521852493286,"q3_score":0.9995276629924774,"avg_token_efficiency":2.0079238561180555,"is_incomplete":false,"total_prompt_cost_bin":0.33913442499999996,"total_cache_read_cost_bin":0.03309568000000001,"total_completion_cost_bin":0.0593255,"total_reasoning_cost_bin":0.05496589999999999,"total_cost_bin":0.486521505,"total_on_demand_input_cost_bin":0.6700912250000003,"ci_99_lower":0.2112833617608903,"ci_99_upper":0.4486128491029947,"ci_98_lower":0.22968124616000912,"ci_98_upper":0.43959999096347013,"ci_95_lower":0.24132046672314697,"ci_95_upper":0.4198298780397083,"ci_90_lower":0.2569849067363048,"ci_90_upper":0.4018001344073199,"ci_75_lower":0.27647648097527233,"ci_75_upper":0.3738147674953592,"ci_50_lower":0.29425094166300353,"ci_50_upper":0.3536120366451202}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Nex AGI: Nex-N2-Mini","provider_name":"Nex-agi","run_provider_names":["Nex AGI"],"pricing_metadata":{"prompt":2.5e-8,"completion":1e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-06-24T14:56:04.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-12T22:21:29.146Z"},{"model_slug":"qwen/qwen3.5-122b-a10b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.08962565248266292,"auc_128k":0.5736055783074528,"cum_avg_1m":0.4367340461430639,"cum_avg_128k":0.6987744738289022,"avg_score_overall":0.6849754825758763,"avg_response_time_ms":136849.69559901214,"total_prompt_cost":6.504077840000001,"total_cache_read_cost":0,"total_completion_cost":7.3134672,"total_reasoning_cost":6.91173184,"total_prompt_cost_128k":6.504077840000001,"total_completion_cost_128k":7.3134672,"total_on_demand_input_cost":6.504077840000001,"total_on_demand_input_cost_128k":6.504077840000001,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207081000672178},"bin_metrics":{"8192":{"avg_score":0.9507619165349752,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.21566932271714645,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222288,"is_incomplete":false,"total_prompt_cost_bin":0.14128191999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.77681344,"total_reasoning_cost_bin":0.7161980799999998,"total_cost_bin":1.6342934399999998,"total_on_demand_input_cost_bin":0.14128191999999998,"ci_99_lower":0.8769643257443095,"ci_99_upper":0.9999417028501629,"ci_98_lower":0.8883948811469599,"ci_98_upper":0.9999224933460354,"ci_95_lower":0.8897411998960889,"ci_95_upper":0.9879210532177239,"ci_90_lower":0.901811728423927,"ci_90_upper":0.9876505998033099,"ci_75_lower":0.9257614377595018,"ci_75_upper":0.975479134041234,"ci_50_lower":0.9381725220009685,"ci_50_upper":0.9633212959859521},"16384":{"avg_score":0.9368071145378053,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.23814304401459896,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0159704488335684,"is_incomplete":false,"total_prompt_cost_bin":0.2913830400000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.96690256,"total_reasoning_cost_bin":0.9072585599999998,"total_cost_bin":2.1655441599999996,"total_on_demand_input_cost_bin":0.2913830400000001,"ci_99_lower":0.8515705003701151,"ci_99_upper":0.9873909517785069,"ci_98_lower":0.8628651968464255,"ci_98_upper":0.9865227548796683,"ci_95_lower":0.8750506732764188,"ci_95_upper":0.9853473285026848,"ci_90_lower":0.8876989543414675,"ci_90_upper":0.9743121672724373,"ci_75_lower":0.9010490776563529,"ci_75_upper":0.9628955442400183,"ci_50_lower":0.9228516888921148,"ci_50_upper":0.9508581627276725},"32768":{"avg_score":0.739911478740396,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.42694508638428635,"median_score":1,"q1_score":0.13632392324507236,"q3_score":1,"avg_token_efficiency":1.0256830410679845,"is_incomplete":false,"total_prompt_cost_bin":1.0339180799999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.93588304,"total_reasoning_cost_bin":1.8219739200000007,"total_cost_bin":4.79177504,"total_on_demand_input_cost_bin":1.0339180799999996,"ci_99_lower":0.6434519729144985,"ci_99_upper":0.8359246699947535,"ci_98_lower":0.6547446544321684,"ci_98_upper":0.8243978588174864,"ci_95_lower":0.667585481937396,"ci_95_upper":0.8079361466289008,"ci_90_lower":0.6770469942778137,"ci_90_upper":0.7960858443197245,"ci_75_lower":0.6965912085817014,"ci_75_upper":0.7816469834172441,"ci_50_lower":0.7134094999196617,"ci_50_upper":0.7651538483088884},"65536":{"avg_score":0.5190950854955351,"min_score":0.002923976629972458,"max_score":1,"n_tests":85,"stddev":0.48855124469925515,"median_score":0.9629040956497192,"q1_score":0.026308264583349228,"q3_score":1,"avg_token_efficiency":1.0209598255175438,"is_incomplete":false,"total_prompt_cost_bin":1.4758915600000018,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.5625188799999996,"total_reasoning_cost_bin":1.4890449599999995,"total_cost_bin":4.527455400000001,"total_on_demand_input_cost_bin":1.4758915600000018,"ci_99_lower":0.3805091241490754,"ci_99_upper":0.6548351942403242,"ci_98_lower":0.3928635340732906,"ci_98_upper":0.6450709593369461,"ci_95_lower":0.4140541821266251,"ci_95_upper":0.6226457416315508,"ci_90_lower":0.42874077568214164,"ci_90_upper":0.6105323208545279,"ci_75_lower":0.46057244944386183,"ci_75_upper":0.5865646777606077,"ci_50_lower":0.484599070368773,"ci_50_upper":0.5535840333713328},"131072":{"avg_score":0.34729677383579965,"min_score":0.006864007096737623,"max_score":1,"n_tests":103,"stddev":0.45871536620565423,"median_score":0.036894384771585464,"q1_score":0.020005999132990837,"q3_score":0.9997437298297882,"avg_token_efficiency":1.016221593648859,"is_incomplete":false,"total_prompt_cost_bin":3.5616032399999993,"total_cache_read_cost_bin":0,"total_completion_cost_bin":2.0713492799999997,"total_reasoning_cost_bin":1.9772563200000002,"total_cost_bin":7.610208839999999,"total_on_demand_input_cost_bin":3.5616032399999993,"ci_99_lower":0.2414005075991425,"ci_99_upper":0.4598799326075964,"ci_98_lower":0.2434469817023616,"ci_98_upper":0.4517722500414018,"ci_95_lower":0.26069512184103494,"ci_95_upper":0.43311942301484874,"ci_90_lower":0.2719795335129409,"ci_90_upper":0.42242590298659155,"ci_75_lower":0.2915251084950355,"ci_75_upper":0.4015271606989594,"ci_50_lower":0.31900940713070297,"ci_50_upper":0.37610445319656827}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-122B-A10B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":2.6e-7,"completion":0.00000208,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-25T21:09:49.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T04:01:02.405Z"},{"model_slug":"moonshotai/kimi-k2.6","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.08804755476228349,"auc_128k":0.537694163923986,"cum_avg_1m":0.4114667632668474,"cum_avg_128k":0.6583468212269559,"avg_score_overall":0.6463493582251875,"avg_response_time_ms":117527.58064307852,"total_prompt_cost":2.5799070960000017,"total_cache_read_cost":2.967874848,"total_completion_cost":7.217338860000001,"total_reasoning_cost":7.438783859999999,"total_prompt_cost_128k":2.5799070960000017,"total_completion_cost_128k":7.217338860000001,"total_on_demand_input_cost":16.677312624000002,"total_on_demand_input_cost_128k":16.677312624000002,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9943036508110018},"bin_metrics":{"8192":{"avg_score":0.9513883754378185,"min_score":0.0233728289604187,"max_score":1,"n_tests":80,"stddev":0.21313279778125382,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9901216565918285,"is_incomplete":false,"total_prompt_cost_bin":0.019545984000000013,"total_cache_read_cost_bin":0.071737344,"total_completion_cost_bin":0.7896643199999999,"total_reasoning_cost_bin":0.81647712,"total_cost_bin":1.6974247679999999,"total_on_demand_input_cost_bin":0.36029836800000015,"ci_99_lower":0.8785055405915483,"ci_99_upper":0.9999837271869183,"ci_98_lower":0.8902964709016961,"ci_98_upper":0.9999755803719163,"ci_95_lower":0.8908462770906045,"ci_95_upper":0.9879319566767663,"ci_90_lower":0.9028641161834821,"ci_90_upper":0.9878287215018645,"ci_75_lower":0.9269564233487472,"ci_75_upper":0.9757312048692256,"ci_50_lower":0.9391404695343227,"ci_50_upper":0.9636370146647096},"16384":{"avg_score":0.8056953091174364,"min_score":0.01270289346575737,"max_score":1,"n_tests":80,"stddev":0.39101760952079473,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9804544918987934,"is_incomplete":false,"total_prompt_cost_bin":0.0060410879999999965,"total_cache_read_cost_bin":0.15431270399999997,"total_completion_cost_bin":1.3641661800000005,"total_reasoning_cost_bin":1.40398182,"total_cost_bin":2.9285017920000005,"total_on_demand_input_cost_bin":0.7390264319999996,"ci_99_lower":0.6841904929539887,"ci_99_upper":0.9152580502568745,"ci_98_lower":0.695648618072737,"ci_98_upper":0.9030150705275591,"ci_95_lower":0.7087537951732521,"ci_95_upper":0.8906005382596051,"ci_90_lower":0.7327328437310643,"ci_90_upper":0.878404558019247,"ci_75_lower":0.7566614110022782,"ci_75_upper":0.8542025042930618,"ci_50_lower":0.7807398864300922,"ci_50_upper":0.830546619510278},"32768":{"avg_score":0.6825234910082894,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.45304378871892376,"median_score":1,"q1_score":0.04860869608819485,"q3_score":1,"avg_token_efficiency":1.0101715848916966,"is_incomplete":false,"total_prompt_cost_bin":0.15543215999999996,"total_cache_read_cost_bin":0.5313945600000012,"total_completion_cost_bin":2.1892514400000005,"total_reasoning_cost_bin":2.2542759,"total_cost_bin":5.130354060000002,"total_on_demand_input_cost_bin":2.6795563200000023,"ci_99_lower":0.5769272377627374,"ci_99_upper":0.7761690219870069,"ci_98_lower":0.5831268709222102,"ci_98_upper":0.7660441867763396,"ci_95_lower":0.599295841673259,"ci_95_upper":0.7550385098121135,"ci_90_lower":0.6189519185737214,"ci_90_upper":0.74648968953629,"ci_75_lower":0.6385328353045384,"ci_75_upper":0.7253547546737108,"ci_50_lower":0.6554667848009985,"ci_50_upper":0.7057328115111244},"65536":{"avg_score":0.46254855607745843,"min_score":0.0016868147067725658,"max_score":1,"n_tests":85,"stddev":0.48556479657959345,"median_score":0.04961089417338371,"q1_score":0.023241335526108742,"q3_score":1,"avg_token_efficiency":0.9907670367000743,"is_incomplete":false,"total_prompt_cost_bin":0.6225131879999996,"total_cache_read_cost_bin":0.662188032,"total_completion_cost_bin":1.4728264199999996,"total_reasoning_cost_bin":1.5336374400000001,"total_cost_bin":4.291165079999999,"total_on_demand_input_cost_bin":3.767906339999998,"ci_99_lower":0.32447416350474195,"ci_99_upper":0.5992381565212135,"ci_98_lower":0.3387563488484799,"ci_98_upper":0.5875227291264626,"ci_95_lower":0.3573674259562155,"ci_95_upper":0.5661349391017576,"ci_90_lower":0.3726488204070312,"ci_90_upper":0.5511533378924736,"ci_75_lower":0.40240032424403843,"ci_75_upper":0.5224503720894812,"ci_50_lower":0.4277904374257881,"ci_50_upper":0.4971057692707023},"131072":{"avg_score":0.38957837449377675,"min_score":0.003610108280554414,"max_score":1,"n_tests":103,"stddev":0.4696479378517534,"median_score":0.036173783242702484,"q1_score":0.021306640468537807,"q3_score":1,"avg_token_efficiency":0.990275159694156,"is_incomplete":false,"total_prompt_cost_bin":1.776374676000002,"total_cache_read_cost_bin":1.5482422079999991,"total_completion_cost_bin":1.4014304999999998,"total_reasoning_cost_bin":1.4304115799999997,"total_cost_bin":6.1564589640000005,"total_on_demand_input_cost_bin":9.130525164000003,"ci_99_lower":0.2725166105444416,"ci_99_upper":0.5110973645297096,"ci_98_lower":0.2834356619240671,"ci_98_upper":0.492454211566873,"ci_95_lower":0.29806018234756143,"ci_95_upper":0.4809341488352375,"ci_90_lower":0.3126105374050806,"ci_90_upper":0.466361107321872,"ci_75_lower":0.3375175730808461,"ci_75_upper":0.44575132639062576,"ci_50_lower":0.35857509017483447,"ci_50_upper":0.4216960402204456}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"MoonshotAI: Kimi K2.6","provider_name":"Moonshotai","run_provider_names":["Moonshot AI"],"pricing_metadata":{"prompt":9.5e-7,"completion":0.000004,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-20T15:36:42.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T08:02:30.787Z"},{"model_slug":"qwen/qwen3.6-35b-a3b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.08716931974449584,"auc_128k":0.4772819205718007,"cum_avg_1m":0.33403547947417767,"cum_avg_128k":0.5344567671586843,"avg_score_overall":0.5320357034643842,"avg_response_time_ms":13650.46165453698,"total_prompt_cost":0.13305767999999998,"total_cache_read_cost":3.3692736000000023,"total_completion_cost":0.190119,"total_reasoning_cost":0,"total_prompt_cost_128k":0.13305767999999998,"total_completion_cost_128k":0.190119,"total_on_demand_input_cost":3.502331279999998,"total_on_demand_input_cost_128k":3.502331279999998,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0208128870566133},"bin_metrics":{"8192":{"avg_score":0.7104774467588868,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.44519860922839927,"median_score":1,"q1_score":0.05565536953508854,"q3_score":1,"avg_token_efficiency":1.0228079512716763,"is_incomplete":false,"total_prompt_cost_bin":0.018735360000000006,"total_cache_read_cost_bin":0.057361920000000045,"total_completion_cost_bin":0.028399999999999995,"total_reasoning_cost_bin":0,"total_cost_bin":0.10449728000000004,"total_on_demand_input_cost_bin":0.07609727999999999,"ci_99_lower":0.5767220428827567,"ci_99_upper":0.8299146120920485,"ci_98_lower":0.587914735682658,"ci_98_upper":0.8170774951337371,"ci_95_lower":0.6118125049814989,"ci_95_upper":0.7975994561839617,"ci_90_lower":0.6252580558374756,"ci_90_upper":0.7925956802201108,"ci_75_lower":0.6507589361797727,"ci_75_upper":0.7603514788315807,"ci_50_lower":0.6751267847022974,"ci_50_upper":0.7446455007797339},"16384":{"avg_score":0.5239304167742376,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.49130803686817826,"median_score":1,"q1_score":0.024398380424827337,"q3_score":1,"avg_token_efficiency":1.0161194758384642,"is_incomplete":false,"total_prompt_cost_bin":0.016177280000000006,"total_cache_read_cost_bin":0.14074367999999998,"total_completion_cost_bin":0.02872899999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.18564996,"total_on_demand_input_cost_bin":0.15692096,"ci_99_lower":0.3884763677844894,"ci_99_upper":0.6580964065265725,"ci_98_lower":0.39123306417424464,"ci_98_upper":0.6459371618015576,"ci_95_lower":0.41123694053385407,"ci_95_upper":0.632685634180234,"ci_90_lower":0.42696272986970146,"ci_90_upper":0.6204265547424439,"ci_75_lower":0.46223307127002045,"ci_75_upper":0.5853957843086391,"ci_50_lower":0.4875333471252816,"ci_50_upper":0.5609952259546844},"32768":{"avg_score":0.5437808912606937,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4778045329480705,"median_score":1,"q1_score":0.03858308959752321,"q3_score":1,"avg_token_efficiency":1.0257548592579655,"is_incomplete":false,"total_prompt_cost_bin":0.025130559999999993,"total_cache_read_cost_bin":0.53163264,"total_completion_cost_bin":0.055571999999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.6123352,"total_on_demand_input_cost_bin":0.5567631999999999,"ci_99_lower":0.4379213165923241,"ci_99_upper":0.6491575574306155,"ci_98_lower":0.44399508909077506,"ci_98_upper":0.6366776950626039,"ci_95_lower":0.4628331312878478,"ci_95_upper":0.6237792300805536,"ci_90_lower":0.47947826724275194,"ci_90_upper":0.6080737120875924,"ci_75_lower":0.4990454006215527,"ci_75_upper":0.5934326584088815,"ci_50_lower":0.5181480157702668,"ci_50_upper":0.5717162140623687},"65536":{"avg_score":0.40518573112785816,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4768468318866574,"median_score":0.04197901114821434,"q1_score":0.021984733641147614,"q3_score":1,"avg_token_efficiency":1.020990401222107,"is_incomplete":false,"total_prompt_cost_bin":0.02167928,"total_cache_read_cost_bin":0.7730553599999987,"total_completion_cost_bin":0.03435500000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.8290896399999987,"total_on_demand_input_cost_bin":0.7947346399999993,"ci_99_lower":0.27698872016441517,"ci_99_upper":0.5426069595038014,"ci_98_lower":0.2896143882335986,"ci_98_upper":0.5309781586293351,"ci_95_lower":0.30136975949191874,"ci_95_upper":0.5065741961484994,"ci_90_lower":0.31486408854177333,"ci_90_upper":0.4877606703319094,"ci_75_lower":0.3467339145772926,"ci_75_upper":0.4632514758651857,"ci_50_lower":0.3707554233863073,"ci_50_upper":0.4399409898995038},"131072":{"avg_score":0.4889093498717451,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.48926049353646023,"median_score":0.04927007481455803,"q1_score":0.022313310764729977,"q3_score":1,"avg_token_efficiency":1.0162368757633717,"is_incomplete":false,"total_prompt_cost_bin":0.05133519999999999,"total_cache_read_cost_bin":1.8664800000000037,"total_completion_cost_bin":0.04306300000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.9608782000000038,"total_on_demand_input_cost_bin":1.917815199999999,"ci_99_lower":0.365067684842676,"ci_99_upper":0.6123510017124166,"ci_98_lower":0.3753540385331398,"ci_98_upper":0.6020431509188114,"ci_95_lower":0.3950793588930714,"ci_95_upper":0.5749572151151793,"ci_90_lower":0.41237251750098597,"ci_90_upper":0.5724970446673483,"ci_75_lower":0.4325063325277105,"ci_75_upper":0.5461222389379506,"ci_50_lower":0.4594594899913097,"ci_50_upper":0.5254146208041754}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.6 35B A3B","provider_name":"Qwen","run_provider_names":["Parasail"],"pricing_metadata":{"prompt":1.4e-7,"completion":0.000001,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-27T03:24:15.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T04:01:15.377Z"},{"model_slug":"qwen/qwen3.5-35b-a3b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.08708614762562046,"auc_128k":0.567191091741294,"cum_avg_1m":0.43063008939795794,"cum_avg_128k":0.6890081430367327,"avg_score_overall":0.6747574308407993,"avg_response_time_ms":86346.80117268207,"total_prompt_cost":3.5021957599999984,"total_cache_read_cost":0,"total_completion_cost":3.9953640000000004,"total_reasoning_cost":3.8030739999999996,"total_prompt_cost_128k":3.5021957599999984,"total_completion_cost_128k":3.9953640000000004,"total_on_demand_input_cost":3.5021957599999984,"total_on_demand_input_cost_128k":3.5021957599999984,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0207081000672176},"bin_metrics":{"8192":{"avg_score":0.937680358835496,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.2378974988671113,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0224972703222286,"is_incomplete":false,"total_prompt_cost_bin":0.07607488000000004,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.42855999999999994,"total_reasoning_cost_bin":0.3998369999999998,"total_cost_bin":0.9044718799999998,"total_on_demand_input_cost_bin":0.07607488000000004,"ci_99_lower":0.8538744394754758,"ci_99_upper":0.9879728910699486,"ci_98_lower":0.8642020418348256,"ci_98_upper":0.987695854483638,"ci_95_lower":0.8758935956022469,"ci_95_upper":0.9863551525637739,"ci_90_lower":0.8885677609441336,"ci_90_upper":0.9754809877974913,"ci_75_lower":0.9019373883085791,"ci_75_upper":0.9635786097933305,"ci_50_lower":0.9236745717527811,"ci_50_upper":0.9516383334819692},"16384":{"avg_score":0.9268788310233503,"min_score":0.016410255804657936,"max_score":1,"n_tests":80,"stddev":0.2580276964048934,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.015970448833568,"is_incomplete":false,"total_prompt_cost_bin":0.15689855999999994,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.4750209999999998,"total_reasoning_cost_bin":0.446193,"total_cost_bin":1.0781125599999997,"total_on_demand_input_cost_bin":0.15689855999999994,"ci_99_lower":0.8413618928954238,"ci_99_upper":0.9877413634191035,"ci_98_lower":0.8529453738315498,"ci_98_upper":0.9876150420182385,"ci_95_lower":0.8657020362903131,"ci_95_upper":0.9757061810465528,"ci_90_lower":0.8771626113483217,"ci_90_upper":0.9753444077563472,"ci_75_lower":0.8904001412272919,"ci_75_upper":0.9632157627580454,"ci_50_lower":0.9028419811686035,"ci_50_upper":0.9509619593387469},"32768":{"avg_score":0.7320542028995559,"min_score":0.004637680947780609,"max_score":1,"n_tests":136,"stddev":0.4301831275071574,"median_score":1,"q1_score":0.07938993535935879,"q3_score":1,"avg_token_efficiency":1.025683041067984,"is_incomplete":false,"total_prompt_cost_bin":0.5567251200000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.1419940000000008,"total_reasoning_cost_bin":1.087368999999999,"total_cost_bin":2.7860881199999996,"total_on_demand_input_cost_bin":0.5567251200000001,"ci_99_lower":0.6324612508539919,"ci_99_upper":0.8291533502968206,"ci_98_lower":0.6404696364341962,"ci_98_upper":0.8150645296333824,"ci_95_lower":0.6568654654485939,"ci_95_upper":0.802019879571847,"ci_90_lower":0.6703444400216755,"ci_90_upper":0.7889087130681967,"ci_75_lower":0.6882421963622192,"ci_75_upper":0.7745472109767005,"ci_50_lower":0.7048600008112023,"ci_50_upper":0.7575496713346664},"65536":{"avg_score":0.5294180258834625,"min_score":0.005082591902464628,"max_score":1,"n_tests":85,"stddev":0.4869893325475098,"median_score":0.9587835073471069,"q1_score":0.025459688156843185,"q3_score":1,"avg_token_efficiency":1.020959825517544,"is_incomplete":false,"total_prompt_cost_bin":0.7947108399999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.791607,"total_reasoning_cost_bin":0.7565460000000004,"total_cost_bin":2.3428638399999997,"total_on_demand_input_cost_bin":0.7947108399999996,"ci_99_lower":0.39090263174622153,"ci_99_upper":0.6650826492686756,"ci_98_lower":0.4035840470225496,"ci_98_upper":0.6542824637190274,"ci_95_lower":0.4253378983940381,"ci_95_upper":0.6324181609799373,"ci_90_lower":0.4396433410809978,"ci_90_upper":0.6206890774479903,"ci_75_lower":0.47115060905309614,"ci_75_upper":0.5966923127731527,"ci_50_lower":0.4951677250100629,"ci_50_upper":0.5645061714026858},"131072":{"avg_score":0.31900929654179844,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.44597657800728246,"median_score":0.03178483992815018,"q1_score":0.018701749853789806,"q3_score":0.9955264627933502,"avg_token_efficiency":1.0162215936488594,"is_incomplete":false,"total_prompt_cost_bin":1.9177863599999987,"total_cache_read_cost_bin":0,"total_completion_cost_bin":1.1581820000000003,"total_reasoning_cost_bin":1.1131290000000007,"total_cost_bin":4.18909736,"total_on_demand_input_cost_bin":1.9177863599999987,"ci_99_lower":0.21257100298110104,"ci_99_upper":0.43144917451039644,"ci_98_lower":0.22501877194315703,"ci_98_upper":0.4215452723003454,"ci_95_lower":0.23425512281110855,"ci_95_upper":0.4037796748420043,"ci_90_lower":0.24603999175755217,"ci_90_upper":0.3936166377439733,"ci_75_lower":0.27002429768094993,"ci_75_upper":0.3678917314109022,"ci_50_lower":0.29070748160005483,"ci_50_upper":0.34714430699450444}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-35B-A3B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":2.5e-7,"completion":0.00000125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-25T21:10:22.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T06:20:13.753Z"},{"model_slug":"openai/gpt-5.4-mini","reasoning_mode":"low","overall_metrics":{"auc_1m":0.08579966969768063,"auc_128k":0.31404684517639525,"cum_avg_1m":0.2778261593600807,"cum_avg_128k":0.4084113528876735,"avg_score_overall":0.3447465222261846,"avg_response_time_ms":11113.33439609375,"total_prompt_cost":1.9996710000000002,"total_cache_read_cost":4.325203199999996,"total_completion_cost":2.234475,"total_reasoning_cost":1.1745809999999994,"total_prompt_cost_128k":0.7861627500000001,"total_completion_cost_128k":1.5329385,"total_on_demand_input_cost":45.25170299999997,"total_on_demand_input_cost_128k":18.04043474999998,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061808102971},"bin_metrics":{"8192":{"avg_score":0.6362649622082245,"min_score":0.006942034233361483,"max_score":1,"n_tests":80,"stddev":0.4724918391181238,"median_score":1,"q1_score":0.03247983939945698,"q3_score":1,"avg_token_efficiency":0.9759044449598386,"is_incomplete":false,"total_prompt_cost_bin":0.056658,"total_cache_read_cost_bin":0.0332736,"total_completion_cost_bin":0.1820025,"total_reasoning_cost_bin":0.05875649999999998,"total_cost_bin":0.3306906,"total_on_demand_input_cost_bin":0.3893939999999997,"ci_99_lower":0.48976337237446566,"ci_99_upper":0.7692010011050151,"ci_98_lower":0.5028167407494037,"ci_98_upper":0.7459683213813696,"ci_95_lower":0.527918983228301,"ci_95_upper":0.7331214322515007,"ci_90_lower":0.5505777956018573,"ci_90_upper":0.7208115606885985,"ci_75_lower":0.5752422222503811,"ci_75_upper":0.6968531115671794,"ci_50_lower":0.5998600050355889,"ci_50_upper":0.6719561333055026},"16384":{"avg_score":0.5570148219238036,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.48417690263047947,"median_score":1,"q1_score":0.028307717759162188,"q3_score":1,"avg_token_efficiency":0.9670687393188718,"is_incomplete":false,"total_prompt_cost_bin":0.06563400000000005,"total_cache_read_cost_bin":0.07336320000000009,"total_completion_cost_bin":0.20190600000000003,"total_reasoning_cost_bin":0.078453,"total_cost_bin":0.4193562000000002,"total_on_demand_input_cost_bin":0.7992659999999998,"ci_99_lower":0.42169361741811734,"ci_99_upper":0.6974756834146683,"ci_98_lower":0.42801134528708645,"ci_98_upper":0.6844647468188777,"ci_95_lower":0.44797935497044816,"ci_95_upper":0.6603938876310712,"ci_90_lower":0.4708797839377075,"ci_90_upper":0.6470109483780107,"ci_75_lower":0.49678092575340993,"ci_75_upper":0.6220965578744654,"ci_50_lower":0.5210889054025756,"ci_50_upper":0.5939516518817982},"32768":{"avg_score":0.3574883791152388,"min_score":0.003955174703150988,"max_score":1,"n_tests":136,"stddev":0.453993722588367,"median_score":0.050333721563220024,"q1_score":0.029237812850624323,"q3_score":1,"avg_token_efficiency":0.9952100768549628,"is_incomplete":false,"total_prompt_cost_bin":0.208092,"total_cache_read_cost_bin":0.26868480000000067,"total_completion_cost_bin":0.4452255,"total_reasoning_cost_bin":0.20534399999999997,"total_cost_bin":1.1273463000000006,"total_on_demand_input_cost_bin":2.8949399999999983,"ci_99_lower":0.2602621764764327,"ci_99_upper":0.4625119961221422,"ci_98_lower":0.26506443723587675,"ci_98_upper":0.45678869797348265,"ci_95_lower":0.28062274392860304,"ci_95_upper":0.43304673210882083,"ci_90_lower":0.29123195411863406,"ci_90_upper":0.42241494556758824,"ci_75_lower":0.3129529466844842,"ci_75_upper":0.40544986559132884,"ci_50_lower":0.3298995701291406,"ci_50_upper":0.3837945584630635},"65536":{"avg_score":0.25971421077299645,"min_score":0.004449388012290001,"max_score":1,"n_tests":85,"stddev":0.4140318785092827,"median_score":0.03145589679479599,"q1_score":0.019632678478956223,"q3_score":0.08787943422794342,"avg_token_efficiency":0.9776435846759911,"is_incomplete":false,"total_prompt_cost_bin":0.15649575000000002,"total_cache_read_cost_bin":0.3920255999999993,"total_completion_cost_bin":0.269289,"total_reasoning_cost_bin":0.11819700000000001,"total_cost_bin":0.9360073499999993,"total_on_demand_input_cost_bin":4.076751749999995,"ci_99_lower":0.1574178373135407,"ci_99_upper":0.3797716854642803,"ci_98_lower":0.166198229090475,"ci_98_upper":0.3693682429755983,"ci_95_lower":0.1793905611733413,"ci_95_upper":0.34975002874099814,"ci_90_lower":0.189728491071009,"ci_90_upper":0.33491926613154216,"ci_75_lower":0.20709275412236286,"ci_75_upper":0.3056817851159448,"ci_50_lower":0.22606472972205238,"ci_50_upper":0.28777374399908107},"131072":{"avg_score":0.23157439041810418,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.3928142770387445,"median_score":0.03346613422036171,"q1_score":0.022860242053866386,"q3_score":0.06919866800308228,"avg_token_efficiency":0.9772722316188368,"is_incomplete":false,"total_prompt_cost_bin":0.2992830000000001,"total_cache_read_cost_bin":0.9580799999999974,"total_completion_cost_bin":0.4345155000000001,"total_reasoning_cost_bin":0.248004,"total_cost_bin":1.9398824999999977,"total_on_demand_input_cost_bin":9.880082999999988,"ci_99_lower":0.13793370448256329,"ci_99_upper":0.3323708086165609,"ci_98_lower":0.14685324125642915,"ci_98_upper":0.31858766455809107,"ci_95_lower":0.1620321418660539,"ci_95_upper":0.3011841987053981,"ci_90_lower":0.17226776312901051,"ci_90_upper":0.29304063805125796,"ci_75_lower":0.1881286275123237,"ci_75_upper":0.2735920169863181,"ci_50_lower":0.20637640663088092,"ci_50_upper":0.25505481041217387},"262144":{"avg_score":0.18055251044227827,"min_score":0.003857280593365431,"max_score":1,"n_tests":141,"stddev":0.343541414267422,"median_score":0.03570416569709778,"q1_score":0.023225806653499603,"q3_score":0.05714285746216774,"avg_token_efficiency":0.9824198103380465,"is_incomplete":false,"total_prompt_cost_bin":1.21350825,"total_cache_read_cost_bin":2.5997759999999985,"total_completion_cost_bin":0.7015365000000001,"total_reasoning_cost_bin":0.4658264999999995,"total_cost_bin":4.980647249999999,"total_on_demand_input_cost_bin":27.21126824999999,"ci_99_lower":0.11042434578541839,"ci_99_upper":0.2507880000001256,"ci_98_lower":0.11562576727816135,"ci_98_upper":0.2477884766340573,"ci_95_lower":0.12636819566874138,"ci_95_upper":0.24068036211170718,"ci_90_lower":0.13541146488761172,"ci_90_upper":0.23023436191435637,"ci_75_lower":0.14694490615346184,"ci_75_upper":0.21447758837223238,"ci_50_lower":0.15984919010281748,"ci_50_upper":0.19829092267322757}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Mini","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.0000045,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T01:59:31.622Z"},{"model_slug":"openai/gpt-5.4-nano","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.08282890288590533,"auc_128k":0.2634640804531362,"cum_avg_1m":0.23944806645967992,"cum_avg_128k":0.3427650519115682,"avg_score_overall":0.29902489431239665,"avg_response_time_ms":14847.586815625,"total_prompt_cost":0.7602152000000004,"total_cache_read_cost":1.1306905600000006,"total_completion_cost":1.14893125,"total_reasoning_cost":0.8506575000000001,"total_prompt_cost_128k":0.23119860000000012,"total_completion_cost_128k":0.8816712499999999,"total_on_demand_input_cost":12.067120799999998,"total_on_demand_input_cost_128k":4.8107826,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061808102976},"bin_metrics":{"8192":{"avg_score":0.6135396052093711,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.47429520190940583,"median_score":1,"q1_score":0.032082052901387215,"q3_score":1,"avg_token_efficiency":0.9759044449598393,"is_incomplete":false,"total_prompt_cost_bin":0.023710399999999993,"total_cache_read_cost_bin":0.008012800000000006,"total_completion_cost_bin":0.09590124999999998,"total_reasoning_cost_bin":0.06138124999999998,"total_cost_bin":0.18900569999999994,"total_on_demand_input_cost_bin":0.10383839999999997,"ci_99_lower":0.4665512541270582,"ci_99_upper":0.7540968459364086,"ci_98_lower":0.48276766118360687,"ci_98_upper":0.7316477456342546,"ci_95_lower":0.50654317831737,"ci_95_upper":0.7115440312988357,"ci_90_lower":0.5262415818186128,"ci_90_upper":0.6980021750993911,"ci_75_lower":0.5533544852187333,"ci_75_upper":0.67408048966754,"ci_50_lower":0.5772958415691392,"ci_50_upper":0.6481226327014156},"16384":{"avg_score":0.3800694045959972,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.4692253898134843,"median_score":0.0366845466196537,"q1_score":0.02370831836014986,"q3_score":1,"avg_token_efficiency":0.9670687393188716,"is_incomplete":false,"total_prompt_cost_bin":0.021598400000000018,"total_cache_read_cost_bin":0.019153920000000005,"total_completion_cost_bin":0.12081625,"total_reasoning_cost_bin":0.08717000000000003,"total_cost_bin":0.24873857000000005,"total_on_demand_input_cost_bin":0.21313760000000004,"ci_99_lower":0.24624777601181996,"ci_99_upper":0.5130936338163447,"ci_98_lower":0.25668351499328856,"ci_98_upper":0.5010327641104814,"ci_95_lower":0.2717428039843799,"ci_95_upper":0.48883868101023836,"ci_90_lower":0.29310606240294873,"ci_90_upper":0.4646076848765369,"ci_75_lower":0.3201385060703615,"ci_75_upper":0.44098743784416006,"ci_50_lower":0.34370668075280264,"ci_50_upper":0.4158640252862824},"32768":{"avg_score":0.27164615633527694,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4143249478785408,"median_score":0.040547195822000504,"q1_score":0.02297296468168497,"q3_score":0.18434650637209415,"avg_token_efficiency":0.9952100768549637,"is_incomplete":false,"total_prompt_cost_bin":0.04878400000000009,"total_cache_read_cost_bin":0.07232000000000002,"total_completion_cost_bin":0.28175500000000003,"total_reasoning_cost_bin":0.21496000000000007,"total_cost_bin":0.6178190000000002,"total_on_demand_input_cost_bin":0.7719840000000004,"ci_99_lower":0.18545535109560554,"ci_99_upper":0.35459088037240744,"ci_98_lower":0.1912339210835532,"ci_98_upper":0.3506563863221912,"ci_95_lower":0.2029852187675287,"ci_95_upper":0.33877375208154686,"ci_90_lower":0.2133057832485065,"ci_90_upper":0.3272284977055867,"ci_75_lower":0.22960583175753366,"ci_75_upper":0.31078914562556964,"ci_50_lower":0.2477503774108995,"ci_50_upper":0.2942612935901921},"65536":{"avg_score":0.23293422731187413,"min_score":0.002923976629972458,"max_score":1,"n_tests":85,"stddev":0.3975310306754993,"median_score":0.030543318018317223,"q1_score":0.020700637251138687,"q3_score":0.05271565541625023,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":0.05012900000000004,"total_cache_read_cost_bin":0.10370047999999987,"total_completion_cost_bin":0.16683249999999997,"total_reasoning_cost_bin":0.12468875,"total_cost_bin":0.4453507299999999,"total_on_demand_input_cost_bin":1.0871337999999988,"ci_99_lower":0.1401647139620036,"ci_99_upper":0.34797262063901874,"ci_98_lower":0.14173790002219816,"ci_98_upper":0.335940384314012,"ci_95_lower":0.15430931195835856,"ci_95_upper":0.316465730271302,"ci_90_lower":0.16537357091958468,"ci_90_upper":0.306376613434404,"ci_75_lower":0.18451850862954472,"ci_75_upper":0.27861486647332856,"ci_50_lower":0.20017710635438563,"ci_50_upper":0.2565960134708268},"131072":{"avg_score":0.2156358661053214,"min_score":0.0033821871038526297,"max_score":1,"n_tests":103,"stddev":0.3851762769432379,"median_score":0.0317460335791111,"q1_score":0.019283168949186802,"q3_score":0.05300911329686642,"avg_token_efficiency":0.9772722316188368,"is_incomplete":false,"total_prompt_cost_bin":0.08697679999999997,"total_cache_read_cost_bin":0.25477120000000036,"total_completion_cost_bin":0.21636625,"total_reasoning_cost_bin":0.16220375000000004,"total_cost_bin":0.7203180000000005,"total_on_demand_input_cost_bin":2.6346887999999997,"ci_99_lower":0.12145415877731228,"ci_99_upper":0.3165080308716278,"ci_98_lower":0.1317687420852853,"ci_98_upper":0.3016724012676598,"ci_95_lower":0.1492778444412906,"ci_95_upper":0.2830706047325946,"ci_90_lower":0.15865416691154785,"ci_90_upper":0.27282315048296457,"ci_75_lower":0.1753351207270436,"ci_75_upper":0.25439591262855754,"ci_50_lower":0.1884824324244389,"ci_50_upper":0.23862657969415152},"262144":{"avg_score":0.2017592721195984,"min_score":0.0029484028927981853,"max_score":1,"n_tests":141,"stddev":0.3609966340823126,"median_score":0.03666841238737106,"q1_score":0.024758219718933105,"q3_score":0.05681818351149559,"avg_token_efficiency":0.9824198103380458,"is_incomplete":false,"total_prompt_cost_bin":0.5290166000000003,"total_cache_read_cost_bin":0.6727321600000002,"total_completion_cost_bin":0.2672600000000001,"total_reasoning_cost_bin":0.20025374999999995,"total_cost_bin":1.6692625100000005,"total_on_demand_input_cost_bin":7.256338199999998,"ci_99_lower":0.12897756267411647,"ci_99_upper":0.27672714915947255,"ci_98_lower":0.13476004508971268,"ci_98_upper":0.27276203858134407,"ci_95_lower":0.14465711433580805,"ci_95_upper":0.2613040346515213,"ci_90_lower":0.15212447406552362,"ci_90_upper":0.25291564425697927,"ci_75_lower":0.1672678037066878,"ci_75_upper":0.2357863888721616,"ci_50_lower":0.1803240742235517,"ci_50_upper":0.22105144515653397}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Nano","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":2e-7,"completion":0.00000125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:47.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T02:31:53.846Z"},{"model_slug":"nvidia/nemotron-3-super-120b-a12b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.08160461351409015,"auc_128k":0.48779539034669994,"cum_avg_1m":0.30991976865546195,"cum_avg_128k":0.4958716298487391,"avg_score_overall":0.5029073893430513,"avg_response_time_ms":65194.79033036665,"total_prompt_cost":2.1321233399999993,"total_cache_read_cost":0,"total_completion_cost":0.44977599999999995,"total_reasoning_cost":0.38760399999999995,"total_prompt_cost_128k":2.1321233399999993,"total_completion_cost_128k":0.44977599999999995,"total_on_demand_input_cost":2.1321233399999993,"total_on_demand_input_cost_128k":2.1321233399999993,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.024499276521396},"bin_metrics":{"8192":{"avg_score":0.5409587980131618,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.47700128399116204,"median_score":0.9320750832557678,"q1_score":0.002175489440560341,"q3_score":1,"avg_token_efficiency":1.0250824169116066,"is_incomplete":false,"total_prompt_cost_bin":0.04630391999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0522592,"total_reasoning_cost_bin":0.049539200000000005,"total_cost_bin":0.14810231999999998,"total_on_demand_input_cost_bin":0.04630391999999998,"ci_99_lower":0.40434777312539516,"ci_99_upper":0.670438910535071,"ci_98_lower":0.4124297313992865,"ci_98_upper":0.6588941579026869,"ci_95_lower":0.43015662291552875,"ci_95_upper":0.6405963646562304,"ci_90_lower":0.4525558688817546,"ci_90_upper":0.6307790984481108,"ci_75_lower":0.48256791134044763,"ci_75_upper":0.602450617327122,"ci_50_lower":0.5053852695535169,"ci_50_upper":0.5774307672720169},"16384":{"avg_score":0.4474456589159672,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.49097396277670313,"median_score":0.04201420210301876,"q1_score":0.0015156193112488836,"q3_score":1,"avg_token_efficiency":1.0154579765577258,"is_incomplete":false,"total_prompt_cost_bin":0.09509255999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08069240000000001,"total_reasoning_cost_bin":0.05685199999999999,"total_cost_bin":0.23263695999999998,"total_on_demand_input_cost_bin":0.09509255999999998,"ci_99_lower":0.30819615468280975,"ci_99_upper":0.5796456564694527,"ci_98_lower":0.31946962247426564,"ci_98_upper":0.5682951543744129,"ci_95_lower":0.3327227174865766,"ci_95_upper":0.555432662278472,"ci_90_lower":0.3497570898932463,"ci_90_upper":0.5407902298193948,"ci_75_lower":0.3836171477623793,"ci_75_upper":0.5108096044235936,"ci_50_lower":0.41000312571195535,"ci_50_upper":0.4851849551912891},"32768":{"avg_score":0.6029084735983976,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4750882905201003,"median_score":0.9996490776538849,"q1_score":0.026732579804956913,"q3_score":1,"avg_token_efficiency":1.0369335178031158,"is_incomplete":false,"total_prompt_cost_bin":0.34172040000000015,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.122692,"total_reasoning_cost_bin":0.0973416,"total_cost_bin":0.5617540000000002,"total_on_demand_input_cost_bin":0.34172040000000015,"ci_99_lower":0.49155908310144963,"ci_99_upper":0.7038663215778284,"ci_98_lower":0.5115724242054506,"ci_98_upper":0.6969841467868837,"ci_95_lower":0.5251796182313175,"ci_95_upper":0.683593242963648,"ci_90_lower":0.5356400814563719,"ci_90_upper":0.6717378155834226,"ci_75_lower":0.5555737440309338,"ci_75_upper":0.6513095549711151,"ci_50_lower":0.577285669523703,"ci_50_upper":0.6291400549104283},"65536":{"avg_score":0.5071883360800498,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4895629760706097,"median_score":0.054421767592430115,"q1_score":0.023104693740606308,"q3_score":1,"avg_token_efficiency":1.0197200706747458,"is_incomplete":false,"total_prompt_cost_bin":0.4819171049999997,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.09515880000000002,"total_reasoning_cost_bin":0.09206040000000001,"total_cost_bin":0.6691363049999998,"total_on_demand_input_cost_bin":0.4819171049999997,"ci_99_lower":0.35936499637655694,"ci_99_upper":0.6344067336868713,"ci_98_lower":0.3794625011982725,"ci_98_upper":0.63310022210275,"ci_95_lower":0.40329334060880634,"ci_95_upper":0.620269843041075,"ci_90_lower":0.41578757150263035,"ci_90_upper":0.5986909734720693,"ci_75_lower":0.4492353792220135,"ci_75_upper":0.566308666620513,"ci_50_lower":0.47266340602244084,"ci_50_upper":0.5413457248716013},"131072":{"avg_score":0.3808568826361189,"min_score":0.005772005766630173,"max_score":1,"n_tests":103,"stddev":0.46263984092142785,"median_score":0.038770053535699844,"q1_score":0.023716501891613007,"q3_score":0.9999460875988007,"avg_token_efficiency":1.0185946983517673,"is_incomplete":false,"total_prompt_cost_bin":1.1670893549999994,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.09897360000000001,"total_reasoning_cost_bin":0.09181079999999994,"total_cost_bin":1.3578737549999995,"total_on_demand_input_cost_bin":1.1670893549999994,"ci_99_lower":0.27160556528735363,"ci_99_upper":0.4948723219814755,"ci_98_lower":0.2781558180227876,"ci_98_upper":0.4846428664288149,"ci_95_lower":0.29073372281821064,"ci_95_upper":0.46921889684909,"ci_90_lower":0.30445854867157496,"ci_90_upper":0.4549190510417,"ci_75_lower":0.32861148042703786,"ci_75_upper":0.43469168038279254,"ci_50_lower":0.35084573558341825,"ci_50_upper":0.4100063245752555}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"NVIDIA: Nemotron 3 Super","provider_name":"Nvidia","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":8.5e-8,"completion":4e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-11T16:07:19.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:59:39.719Z"},{"model_slug":"tencent/hy3","reasoning_mode":"high","overall_metrics":{"auc_1m":0.08133146128542246,"auc_128k":0.5280155413529456,"cum_avg_1m":0.4271802913137274,"cum_avg_128k":0.6834884661019639,"avg_score_overall":0.667877532272361,"avg_response_time_ms":64453.84682294357,"total_prompt_cost":2.610039564000001,"total_cache_read_cost":0.16167782399999997,"total_completion_cost":1.7289861599999998,"total_reasoning_cost":1.6278366720000002,"total_prompt_cost_128k":2.610039564000001,"total_completion_cost_128k":1.7289861599999998,"total_on_demand_input_cost":3.2567508600000012,"total_on_demand_input_cost_128k":3.2567508600000012,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.007042553226076},"bin_metrics":{"8192":{"avg_score":0.9640002839965746,"min_score":0.027107903733849525,"max_score":1,"n_tests":80,"stddev":0.18352369321552653,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.000662777245702,"is_incomplete":false,"total_prompt_cost_bin":0.06016665600000005,"total_cache_read_cost_bin":0.002515392,"total_completion_cost_bin":0.14682623999999997,"total_reasoning_cost_bin":0.13157812800000004,"total_cost_bin":0.34108641600000006,"total_on_demand_input_cost_bin":0.07022822400000006,"ci_99_lower":0.9036868940852583,"ci_99_upper":1,"ci_98_lower":0.9040255620626267,"ci_98_upper":1,"ci_95_lower":0.9159925693646074,"ci_95_upper":0.9999956749379635,"ci_90_lower":0.9278901202755514,"ci_90_upper":0.9999872410669923,"ci_75_lower":0.9399713115533814,"ci_75_upper":0.9879752417560667,"ci_50_lower":0.951927998661995,"ci_50_upper":0.976157110137865},"16384":{"avg_score":0.9986904814839364,"min_score":0.9413084387779236,"max_score":1,"n_tests":80,"stddev":0.007436710252761421,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0025338616228132,"is_incomplete":false,"total_prompt_cost_bin":0.11904076800000005,"total_cache_read_cost_bin":0.006663359999999999,"total_completion_cost_bin":0.210357312,"total_reasoning_cost_bin":0.19535260800000007,"total_cost_bin":0.5314140480000001,"total_on_demand_input_cost_bin":0.14569420799999994,"ci_99_lower":0.9959136739373207,"ci_99_upper":1,"ci_98_lower":0.996362935334444,"ci_98_upper":1,"ci_95_lower":0.9969267414137721,"ci_95_upper":1,"ci_90_lower":0.9972231924533844,"ci_90_upper":1,"ci_75_lower":0.9976740315556526,"ci_75_upper":0.99970693141222,"ci_50_lower":0.9982396423816681,"ci_50_upper":0.9992663554847241},"32768":{"avg_score":0.7286988394557471,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.43608564620135254,"median_score":1,"q1_score":0.05400631297379732,"q3_score":1,"avg_token_efficiency":1.0192594719247559,"is_incomplete":false,"total_prompt_cost_bin":0.41204064000000007,"total_cache_read_cost_bin":0.027335615999999997,"total_completion_cost_bin":0.5525995199999999,"total_reasoning_cost_bin":0.523629744,"total_cost_bin":1.5156055199999998,"total_on_demand_input_cost_bin":0.5213831040000001,"ci_99_lower":0.628402165325593,"ci_99_upper":0.8271750873600575,"ci_98_lower":0.6355762723467471,"ci_98_upper":0.8140767200487521,"ci_95_lower":0.6508639233379011,"ci_95_upper":0.7997057791002375,"ci_90_lower":0.6657178001133348,"ci_90_upper":0.7861074483083725,"ci_75_lower":0.6850617065463345,"ci_75_upper":0.7716081195976585,"ci_50_lower":0.7009658203447435,"ci_50_upper":0.7548304757900255},"65536":{"avg_score":0.42494491770425263,"min_score":0.008689839392900467,"max_score":1,"n_tests":85,"stddev":0.4780366825585382,"median_score":0.04197901114821434,"q1_score":0.021421976387500763,"q3_score":1,"avg_token_efficiency":1.0041162094815022,"is_incomplete":false,"total_prompt_cost_bin":0.6338460479999997,"total_cache_read_cost_bin":0.025772735999999997,"total_completion_cost_bin":0.34941350400000004,"total_reasoning_cost_bin":0.33099528,"total_cost_bin":1.3400275679999998,"total_on_demand_input_cost_bin":0.7369369919999997,"ci_99_lower":0.2849391493555706,"ci_99_upper":0.5599227098047405,"ci_98_lower":0.3024712455655064,"ci_98_upper":0.5525888304161456,"ci_95_lower":0.32169179631506695,"ci_95_upper":0.5268952879460309,"ci_90_lower":0.33429539229446914,"ci_90_upper":0.5078801675754435,"ci_75_lower":0.36624725422520626,"ci_75_upper":0.4845482056125012,"ci_50_lower":0.39015637424719685,"ci_50_upper":0.46133315050152734},"131072":{"avg_score":0.30110780786930863,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.4384801226949689,"median_score":0.033453311771154404,"q1_score":0.022792640142142773,"q3_score":1,"avg_token_efficiency":1.001783482177137,"is_incomplete":false,"total_prompt_cost_bin":1.384945452000001,"total_cache_read_cost_bin":0.09939071999999997,"total_completion_cost_bin":0.4697895839999999,"total_reasoning_cost_bin":0.4462809119999999,"total_cost_bin":2.400406668000001,"total_on_demand_input_cost_bin":1.7825083320000012,"ci_99_lower":0.19670475033285928,"ci_99_upper":0.40402196423239883,"ci_98_lower":0.211978660642296,"ci_98_upper":0.3969864959605897,"ci_95_lower":0.2232688395665886,"ci_95_upper":0.3849781336733569,"ci_90_lower":0.23434256803068035,"ci_90_upper":0.3692800489279161,"ci_75_lower":0.2527365225849863,"ci_75_upper":0.34826158591262424,"ci_50_lower":0.27262337914005697,"ci_50_upper":0.32928252836024774}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","low","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Tencent: Hy3","provider_name":"Tencent","run_provider_names":["Tencent Cloud"],"pricing_metadata":{"prompt":1.32e-7,"completion":5.28e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-06T13:20:48.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-12T23:23:07.833Z"},{"model_slug":"moonshotai/kimi-k2.6","reasoning_mode":null,"overall_metrics":{"auc_1m":0.07970608044375775,"auc_128k":0.46434364363851965,"cum_avg_1m":0.32972981227908127,"cum_avg_128k":0.52756769964653,"avg_score_overall":0.5187588656198018,"avg_response_time_ms":12206.140921096172,"total_prompt_cost":2.478479472000001,"total_cache_read_cost":2.9892280319999998,"total_completion_cost":0.6585654599999999,"total_reasoning_cost":0,"total_prompt_cost_128k":2.478479472000001,"total_completion_cost_128k":0.6585654599999999,"total_on_demand_input_cost":16.677312624000002,"total_on_demand_input_cost_128k":16.677312624000002,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9943036508110019},"bin_metrics":{"8192":{"avg_score":0.70671193113958,"min_score":0.0038135594222694635,"max_score":1,"n_tests":80,"stddev":0.4477420273094692,"median_score":1,"q1_score":0.03854077961295843,"q3_score":1,"avg_token_efficiency":0.9901216565918295,"is_incomplete":false,"total_prompt_cost_bin":0.16085491199999985,"total_cache_read_cost_bin":0.04198809600000003,"total_completion_cost_bin":0.09417996000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.2970229679999999,"total_on_demand_input_cost_bin":0.3602983680000002,"ci_99_lower":0.5731343620913977,"ci_99_upper":0.8269651959627662,"ci_98_lower":0.5841829462874739,"ci_98_upper":0.8136821180681,"ci_95_lower":0.6084608401238802,"ci_95_upper":0.7940038353024282,"ci_90_lower":0.6215507449912548,"ci_90_upper":0.7892832602724957,"ci_75_lower":0.6468089274047087,"ci_75_upper":0.7567734851079876,"ci_50_lower":0.6713017200207105,"ci_50_upper":0.7410635494343296},"16384":{"avg_score":0.5872726225876249,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.48253243571432314,"median_score":1,"q1_score":0.03303896449506283,"q3_score":1,"avg_token_efficiency":0.9804544918987936,"is_incomplete":false,"total_prompt_cost_bin":0.22246963200000003,"total_cache_read_cost_bin":0.10874880000000008,"total_completion_cost_bin":0.09276066000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.4239790920000001,"total_on_demand_input_cost_bin":0.7390264319999997,"ci_99_lower":0.4411884860068094,"ci_99_upper":0.7330016487695393,"ci_98_lower":0.4543780394885689,"ci_98_upper":0.7088587171983672,"ci_95_lower":0.4881796975224279,"ci_95_upper":0.6955987203077529,"ci_90_lower":0.502177641511662,"ci_90_upper":0.6722726440447151,"ci_75_lower":0.5269559213891625,"ci_75_upper":0.6484275926282863,"ci_50_lower":0.5509091055922908,"ci_50_upper":0.6233468954451382},"32768":{"avg_score":0.5142848404870807,"min_score":0.003955174703150988,"max_score":1,"n_tests":136,"stddev":0.4811114528941379,"median_score":0.20264124125242233,"q1_score":0.0318937529809773,"q3_score":1,"avg_token_efficiency":1.0101715848916961,"is_incomplete":false,"total_prompt_cost_bin":0.44925667200000025,"total_cache_read_cost_bin":0.469536768000001,"total_completion_cost_bin":0.18632160000000003,"total_reasoning_cost_bin":0,"total_cost_bin":1.1051150400000014,"total_on_demand_input_cost_bin":2.679556319999998,"ci_99_lower":0.41180481880704656,"ci_99_upper":0.6189074979088721,"ci_98_lower":0.42035053165054276,"ci_98_upper":0.6084375132828855,"ci_95_lower":0.4341699913029408,"ci_95_upper":0.590653984091135,"ci_90_lower":0.4450131726374178,"ci_90_upper":0.5806098394782404,"ci_75_lower":0.46919211321801413,"ci_75_upper":0.5623241509554859,"ci_50_lower":0.4874121673304163,"ci_50_upper":0.5440924943806534},"65536":{"avg_score":0.4348794087959344,"min_score":0.0038729666266590357,"max_score":1,"n_tests":85,"stddev":0.48089313134158707,"median_score":0.042375244200229645,"q1_score":0.022327791899442673,"q3_score":1,"avg_token_efficiency":0.9907670367000744,"is_incomplete":false,"total_prompt_cost_bin":0.3124040039999999,"total_cache_read_cost_bin":0.7274741759999998,"total_completion_cost_bin":0.11653992000000003,"total_reasoning_cost_bin":0,"total_cost_bin":1.1564180999999998,"total_on_demand_input_cost_bin":3.7679063399999975,"ci_99_lower":0.2922683580661713,"ci_99_upper":0.5716969694176633,"ci_98_lower":0.31071232630027573,"ci_98_upper":0.5639885513764149,"ci_95_lower":0.3267164838027811,"ci_95_upper":0.5374897575510583,"ci_90_lower":0.3441559583982727,"ci_90_upper":0.5192280367601608,"ci_75_lower":0.3741597939108033,"ci_75_upper":0.49505379893782775,"ci_50_lower":0.3999865049503141,"ci_50_upper":0.4723411120378467},"131072":{"avg_score":0.39468969522242986,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.47341483472400275,"median_score":0.03458010032773018,"q1_score":0.021358217112720013,"q3_score":1,"avg_token_efficiency":0.9902751596941556,"is_incomplete":false,"total_prompt_cost_bin":1.333494252000001,"total_cache_read_cost_bin":1.6414801919999988,"total_completion_cost_bin":0.16876331999999994,"total_reasoning_cost_bin":0,"total_cost_bin":3.143737764,"total_on_demand_input_cost_bin":9.130525164000005,"ci_99_lower":0.274863136378569,"ci_99_upper":0.5093241978504071,"ci_98_lower":0.28692850984425,"ci_98_upper":0.4990317946673344,"ci_95_lower":0.3018997035775949,"ci_95_upper":0.48771195392219296,"ci_90_lower":0.31761940782166365,"ci_90_upper":0.47049133424393974,"ci_75_lower":0.3425526333717376,"ci_75_upper":0.45080638822231406,"ci_50_lower":0.3638846158022874,"ci_50_upper":0.4271854227838261}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"MoonshotAI: Kimi K2.6","provider_name":"Moonshotai","run_provider_names":["Moonshot AI"],"pricing_metadata":{"prompt":9.5e-7,"completion":0.000004,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-20T15:36:42.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T05:27:17.132Z"},{"model_slug":"tencent/hy3","reasoning_mode":"low","overall_metrics":{"auc_1m":0.07690689739875879,"auc_128k":0.5167136871905115,"cum_avg_1m":0.42279990226817193,"cum_avg_128k":0.6764798436290751,"avg_score_overall":0.6609603748772263,"avg_response_time_ms":54273.4415656476,"total_prompt_cost":2.604607500000001,"total_cache_read_cost":0.16303583999999996,"total_completion_cost":1.3053971039999996,"total_reasoning_cost":1.2040923840000002,"total_prompt_cost_128k":2.604607500000001,"total_completion_cost_128k":1.3053971039999996,"total_on_demand_input_cost":3.2567508600000017,"total_on_demand_input_cost_128k":3.2567508600000017,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.007042553226076},"bin_metrics":{"8192":{"avg_score":0.9877759197261184,"min_score":0.02207357808947563,"max_score":1,"n_tests":80,"stddev":0.10933549781925361,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.0006627772457017,"is_incomplete":false,"total_prompt_cost_bin":0.06510873600000007,"total_cache_read_cost_bin":0.001279872,"total_completion_cost_bin":0.10713489599999997,"total_reasoning_cost_bin":0.09206841600000003,"total_cost_bin":0.2655919200000001,"total_on_demand_input_cost_bin":0.07022822400000008,"ci_99_lower":0.9511036789044738,"ci_99_upper":1,"ci_98_lower":0.9511036789044738,"ci_98_upper":1,"ci_95_lower":0.9633277591783553,"ci_95_upper":1,"ci_90_lower":0.9633277591783553,"ci_90_upper":1,"ci_75_lower":0.9755518394522369,"ci_75_upper":1,"ci_50_lower":0.9755518394522369,"ci_50_upper":1},"16384":{"avg_score":0.9756192969623954,"min_score":0.0314318984746933,"max_score":1,"n_tests":80,"stddev":0.1502561760297489,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.002533861622813,"is_incomplete":false,"total_prompt_cost_bin":0.12981196799999997,"total_cache_read_cost_bin":0.00397056,"total_completion_cost_bin":0.16492819199999997,"total_reasoning_cost_bin":0.14982475199999995,"total_cost_bin":0.44853547199999994,"total_on_demand_input_cost_bin":0.14569420799999996,"ci_99_lower":0.9270720794319641,"ci_99_upper":0.9999881227612496,"ci_98_lower":0.9279284084597603,"ci_98_upper":0.9999826714396477,"ci_95_lower":0.9393646976456511,"ci_95_upper":0.9999349266290665,"ci_90_lower":0.9405878459219821,"ci_90_upper":0.9998419472575187,"ci_75_lower":0.9521208143793046,"ci_75_upper":0.9986915070563555,"ci_50_lower":0.9638564081978984,"ci_50_upper":0.9877083275932819},"32768":{"avg_score":0.7388393135813257,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4290274597029178,"median_score":1,"q1_score":0.06467493437230587,"q3_score":1,"avg_token_efficiency":1.0192594719247559,"is_incomplete":false,"total_prompt_cost_bin":0.4088979840000002,"total_cache_read_cost_bin":0.028121279999999995,"total_completion_cost_bin":0.4169510399999999,"total_reasoning_cost_bin":0.38811009599999985,"total_cost_bin":1.2420803999999999,"total_on_demand_input_cost_bin":0.5213831040000008,"ci_99_lower":0.640045487472851,"ci_99_upper":0.8360191873319941,"ci_98_lower":0.6532284383241067,"ci_98_upper":0.823747828172739,"ci_95_lower":0.6669063116859059,"ci_95_upper":0.8074259926396787,"ci_90_lower":0.6750939035847242,"ci_90_upper":0.7956632344511009,"ci_75_lower":0.6949863261489715,"ci_75_upper":0.7808039265803339,"ci_50_lower":0.7120213172533755,"ci_50_upper":0.7643812755426658},"65536":{"avg_score":0.4281058551524492,"min_score":0.008645533584058285,"max_score":1,"n_tests":85,"stddev":0.4809543509735071,"median_score":0.03999999910593033,"q1_score":0.021377671509981155,"q3_score":1,"avg_token_efficiency":1.0041162094815024,"is_incomplete":false,"total_prompt_cost_bin":0.6331279679999995,"total_cache_read_cost_bin":0.025952255999999996,"total_completion_cost_bin":0.266546016,"total_reasoning_cost_bin":0.2480364480000001,"total_cost_bin":1.1736626879999996,"total_on_demand_input_cost_bin":0.7369369919999997,"ci_99_lower":0.28827386331229526,"ci_99_upper":0.5651566920974017,"ci_98_lower":0.30284817883109344,"ci_98_upper":0.555306693073362,"ci_95_lower":0.3238520710792064,"ci_95_upper":0.5307133087913012,"ci_90_lower":0.3370382471201832,"ci_90_upper":0.5112629799676293,"ci_75_lower":0.3697652261074194,"ci_75_upper":0.48636933407636684,"ci_50_lower":0.39313466940623,"ci_50_upper":0.4634832353605067},"131072":{"avg_score":0.25205883272308677,"min_score":0.0034722222480922937,"max_score":1,"n_tests":103,"stddev":0.40747847129303905,"median_score":0.03199343755841255,"q1_score":0.020283479243516922,"q3_score":0.07623224705457687,"avg_token_efficiency":1.0017834821771372,"is_incomplete":false,"total_prompt_cost_bin":1.367660844000001,"total_cache_read_cost_bin":0.10371187199999997,"total_completion_cost_bin":0.3498369599999997,"total_reasoning_cost_bin":0.32605267200000015,"total_cost_bin":2.147262348000001,"total_on_demand_input_cost_bin":1.7825083320000012,"ci_99_lower":0.1590177897150963,"ci_99_upper":0.352800704260307,"ci_98_lower":0.1626325158173562,"ci_98_upper":0.34570912289512296,"ci_95_lower":0.17711625791322078,"ci_95_upper":0.3270790762232404,"ci_90_lower":0.18843832620164128,"ci_90_upper":0.31768091944118654,"ci_75_lower":0.20641900276119857,"ci_75_upper":0.2960635905034808,"ci_50_lower":0.2248352535557255,"ci_50_upper":0.27804902365071776}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","low","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Tencent: Hy3","provider_name":"Tencent","run_provider_names":["Tencent Cloud"],"pricing_metadata":{"prompt":1.32e-7,"completion":5.28e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-06T13:20:48.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T02:32:53.146Z"},{"model_slug":"z-ai/glm-5.1","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.07653707162375568,"auc_128k":0.5020599591945613,"cum_avg_1m":0.3956603190909225,"cum_avg_128k":0.633056510545476,"avg_score_overall":0.6205318696890219,"avg_response_time_ms":87782.60533961777,"total_prompt_cost":7.388121593999995,"total_cache_read_cost":2.9939879424000004,"total_completion_cost":5.844518592,"total_reasoning_cost":5.278996799999999,"total_prompt_cost_128k":7.388121593999995,"total_completion_cost_128k":5.844518592,"total_on_demand_input_cost":23.509595129999973,"total_on_demand_input_cost_128k":23.509595129999973,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9924993579410007},"bin_metrics":{"8192":{"avg_score":0.9601108406437561,"min_score":0.02207357808947563,"max_score":1,"n_tests":80,"stddev":0.1823591720023088,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9896789088399329,"is_incomplete":false,"total_prompt_cost_bin":0.4069767659999999,"total_cache_read_cost_bin":0.018783897600000002,"total_completion_cost_bin":0.6235215359999999,"total_reasoning_cost_bin":0.5392452119999999,"total_cost_bin":1.5885274115999999,"total_on_demand_input_cost_bin":0.5081208299999999,"ci_99_lower":0.8988916105458048,"ci_99_upper":0.9991149311624468,"ci_98_lower":0.9017137346155941,"ci_98_upper":0.9988237520903349,"ci_95_lower":0.9116490445815726,"ci_95_upper":0.995547140557319,"ci_90_lower":0.9230186421144754,"ci_90_upper":0.9895287738740444,"ci_75_lower":0.935682932054624,"ci_75_upper":0.9836889946338487,"ci_50_lower":0.9477175033069215,"ci_50_upper":0.9738406087213661},"16384":{"avg_score":0.7925838971277699,"min_score":0.01194353960454464,"max_score":1,"n_tests":80,"stddev":0.3990017587258396,"median_score":1,"q1_score":0.9912072569131851,"q3_score":1,"avg_token_efficiency":0.9784005286431926,"is_incomplete":false,"total_prompt_cost_bin":0.39861796799999993,"total_cache_read_cost_bin":0.1193627136,"total_completion_cost_bin":0.7614621959999999,"total_reasoning_cost_bin":0.677935764,"total_cost_bin":1.9573786415999996,"total_on_demand_input_cost_bin":1.041340272000001,"ci_99_lower":0.6713666418652283,"ci_99_upper":0.8910555646258871,"ci_98_lower":0.68230209574732,"ci_98_upper":0.8900232222047635,"ci_95_lower":0.695733113172464,"ci_95_upper":0.8780667366401758,"ci_90_lower":0.7193153680709657,"ci_90_upper":0.8648683950863778,"ci_75_lower":0.743784584646346,"ci_75_upper":0.8409828410571207,"ci_50_lower":0.7644770212064033,"ci_50_upper":0.8186636506288778},"32768":{"avg_score":0.6887438635851996,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.45104171315063285,"median_score":1,"q1_score":0.049879797734320164,"q3_score":1,"avg_token_efficiency":1.007393097590867,"is_incomplete":false,"total_prompt_cost_bin":0.6380159519999993,"total_cache_read_cost_bin":0.5822089728000003,"total_completion_cost_bin":1.8045042839999998,"total_reasoning_cost_bin":1.6392305159999998,"total_cost_bin":4.663959724799999,"total_on_demand_input_cost_bin":3.7729873439999912,"ci_99_lower":0.5814072155391263,"ci_99_upper":0.7812864895731892,"ci_98_lower":0.593027267474938,"ci_98_upper":0.7740228261573493,"ci_95_lower":0.6046711813908855,"ci_95_upper":0.7606325979257235,"ci_90_lower":0.6283168366553796,"ci_90_upper":0.7520182779817226,"ci_75_lower":0.6449960197400394,"ci_75_upper":0.7309957012490914,"ci_50_lower":0.6634214238986127,"ci_50_upper":0.714528437005784},"65536":{"avg_score":0.4501803628333351,"min_score":0.007713073864579201,"max_score":1,"n_tests":85,"stddev":0.4831576983941503,"median_score":0.043887145817279816,"q1_score":0.025072885677218437,"q3_score":1,"avg_token_efficiency":0.9892191866740655,"is_incomplete":false,"total_prompt_cost_bin":1.066955694000001,"total_cache_read_cost_bin":0.7885562880000013,"total_completion_cost_bin":1.1613519720000003,"total_reasoning_cost_bin":1.0577454359999998,"total_cost_bin":4.074609390000003,"total_on_demand_input_cost_bin":5.313028013999995,"ci_99_lower":0.3133366425875026,"ci_99_upper":0.5866164815728826,"ci_98_lower":0.32512944050198966,"ci_98_upper":0.5676923122137566,"ci_95_lower":0.3456534989106962,"ci_95_upper":0.553136596664622,"ci_90_lower":0.3587887780751814,"ci_90_upper":0.5396904789229087,"ci_75_lower":0.3914694511509665,"ci_75_upper":0.5083132258808131,"ci_50_lower":0.4160821131457958,"ci_50_upper":0.4851439951179439},"131072":{"avg_score":0.2736635885373192,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4231804081096282,"median_score":0.03232533857226372,"q1_score":0.02255538571625948,"q3_score":0.5269882902503014,"avg_token_efficiency":0.9886819621858336,"is_incomplete":false,"total_prompt_cost_bin":4.877555213999995,"total_cache_read_cost_bin":1.485076070399999,"total_completion_cost_bin":1.4936786040000005,"total_reasoning_cost_bin":1.364839872,"total_cost_bin":9.221149760399994,"total_on_demand_input_cost_bin":12.874118669999985,"ci_99_lower":0.17014500899036933,"ci_99_upper":0.37712386711869045,"ci_98_lower":0.18611021527572924,"ci_98_upper":0.36874334602978404,"ci_95_lower":0.19789325232613036,"ci_95_upper":0.34992320355733186,"ci_90_lower":0.20748337433781944,"ci_90_upper":0.34036491136111824,"ci_75_lower":0.22680541068525734,"ci_75_upper":0.3197932310421608,"ci_50_lower":0.2448617525724242,"ci_50_upper":0.3012786852512328}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":204800,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Z.ai: GLM 5.1","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":0.00000126,"completion":0.00000396,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-07T16:07:05.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:40:09.845Z"},{"model_slug":"anthropic/claude-opus-4.7","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.07617222719606996,"auc_128k":0.3110151892521478,"cum_avg_1m":0.31791112341766914,"cum_avg_128k":0.48551179557492075,"avg_score_overall":0.28814825418606155,"avg_response_time_ms":26172.795760488123,"total_prompt_cost":9.295505000000004,"total_cache_read_cost":124.13051949999964,"total_completion_cost":15.321100000000007,"total_reasoning_cost":1.4482500000000003,"total_prompt_cost_128k":2.8056200000000002,"total_completion_cost_128k":9.021300000000002,"total_on_demand_input_cost":1250.6006999999997,"total_on_demand_input_cost_128k":124.86667500000004,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6295672248304611},"bin_metrics":{"8192":{"avg_score":0.8199288152041845,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.3729178577877484,"median_score":1,"q1_score":0.9819401502609253,"q3_score":1,"avg_token_efficiency":1.66187971598256,"is_incomplete":false,"total_prompt_cost_bin":0.3488849999999999,"total_cache_read_cost_bin":0.4065294999999996,"total_completion_cost_bin":1.50985,"total_reasoning_cost_bin":0.07655,"total_cost_bin":2.3418144999999995,"total_on_demand_input_cost_bin":4.414180000000002,"ci_99_lower":0.699911098988785,"ci_99_upper":0.9275634843280423,"ci_98_lower":0.7140715128920274,"ci_98_upper":0.9152190491729416,"ci_95_lower":0.7292706768887002,"ci_95_upper":0.8967028896298143,"ci_90_lower":0.7467061989754438,"ci_90_upper":0.8870826592802769,"ci_75_lower":0.7692114005389158,"ci_75_upper":0.8648858009983086,"ci_50_lower":0.7935506927373354,"ci_50_upper":0.8470843431918182},"16384":{"avg_score":0.8622538493480534,"min_score":0.009156310930848122,"max_score":1,"n_tests":80,"stddev":0.3352830547069026,"median_score":0.9992893934249878,"q1_score":0.9901812672615051,"q3_score":1,"avg_token_efficiency":1.5738574220527404,"is_incomplete":false,"total_prompt_cost_bin":0.3291350000000001,"total_cache_read_cost_bin":0.8330924999999997,"total_completion_cost_bin":1.788275,"total_reasoning_cost_bin":0.13572499999999996,"total_cost_bin":3.0862274999999997,"total_on_demand_input_cost_bin":8.660059999999998,"ci_99_lower":0.7533839368148475,"ci_99_upper":0.9490117011816473,"ci_98_lower":0.7759500939541031,"ci_98_upper":0.9363737467247993,"ci_95_lower":0.7788538235542365,"ci_95_upper":0.9239866387832444,"ci_90_lower":0.7916282173083165,"ci_90_upper":0.9214656335266772,"ci_75_lower":0.8153766224655555,"ci_75_upper":0.8993023197137517,"ci_50_lower":0.8377780827519019,"ci_50_upper":0.8867377826361916},"32768":{"avg_score":0.4942024819525507,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.44190233361175857,"median_score":0.4325220137834549,"q1_score":0.03550065495073795,"q3_score":0.9988379329442978,"avg_token_efficiency":1.6528166626033114,"is_incomplete":false,"total_prompt_cost_bin":0.505995,"total_cache_read_cost_bin":3.1611774999999964,"total_completion_cost_bin":3.552200000000002,"total_reasoning_cost_bin":0.3330000000000001,"total_cost_bin":7.552372499999999,"total_on_demand_input_cost_bin":32.11777000000005,"ci_99_lower":0.39154493505573446,"ci_99_upper":0.589226835791179,"ci_98_lower":0.4035906107790138,"ci_98_upper":0.5797244480804211,"ci_95_lower":0.41861222531297243,"ci_95_upper":0.5673173295534284,"ci_90_lower":0.4319624818964204,"ci_90_upper":0.5566291681518231,"ci_75_lower":0.4533858600719695,"ci_75_upper":0.539529279765488,"ci_50_lower":0.46803222095374675,"ci_50_upper":0.5201153361652842},"65536":{"avg_score":0.23728994291056604,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.37170408400545046,"median_score":0.02775941789150238,"q1_score":0,"q3_score":0.3701188564300537,"avg_token_efficiency":1.633311858261692,"is_incomplete":false,"total_prompt_cost_bin":0.55795,"total_cache_read_cost_bin":4.431369999999998,"total_completion_cost_bin":1.9756750000000007,"total_reasoning_cost_bin":0.18595000000000003,"total_cost_bin":7.150944999999998,"total_on_demand_input_cost_bin":44.871649999999995,"ci_99_lower":0.14955448292771026,"ci_99_upper":0.3506674868408591,"ci_98_lower":0.15694809781201186,"ci_98_upper":0.3387749510051354,"ci_95_lower":0.1641852910556447,"ci_95_upper":0.3174305813734913,"ci_90_lower":0.17143221204278664,"ci_90_upper":0.30338233109289664,"ci_75_lower":0.19021959277417733,"ci_75_upper":0.28311272271607507,"ci_50_lower":0.20684063116581564,"ci_50_upper":0.2616412558236762},"131072":{"avg_score":0.013883888459249028,"min_score":0,"max_score":0.7286432385444641,"n_tests":103,"stddev":0.07551960070062347,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6117058522678807,"is_incomplete":false,"total_prompt_cost_bin":1.0636550000000002,"total_cache_read_cost_bin":3.373936,"total_completion_cost_bin":0.19530000000000003,"total_reasoning_cost_bin":0.032375,"total_cost_bin":4.665266000000001,"total_on_demand_input_cost_bin":34.803014999999995,"ci_99_lower":0.002285296262187648,"ci_99_upper":0.040759555943111335,"ci_98_lower":0.0027829213419621724,"ci_98_upper":0.03820877549672516,"ci_95_lower":0.003428453405487161,"ci_95_upper":0.0313259798291962,"ci_90_lower":0.004061727185543735,"ci_90_upper":0.027716609272310167,"ci_75_lower":0.005723272741871696,"ci_75_upper":0.022399731526305013,"ci_50_lower":0.008088537335884079,"ci_50_upper":0.01851930415926917},"262144":{"avg_score":0.02739443945663763,"min_score":0,"max_score":0.9994357824325562,"n_tests":141,"stddev":0.12054598868927825,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6299638830267156,"is_incomplete":false,"total_prompt_cost_bin":2.1514800000000007,"total_cache_read_cost_bin":12.806010000000011,"total_completion_cost_bin":0.6755749999999997,"total_reasoning_cost_bin":0.09567500000000001,"total_cost_bin":15.728740000000013,"total_on_demand_input_cost_bin":130.21157999999994,"ci_99_lower":0.007254317341659693,"ci_99_upper":0.05996422170537955,"ci_98_lower":0.008502184998698975,"ci_98_upper":0.0560079916563638,"ci_95_lower":0.010574603438014512,"ci_95_upper":0.049966340265986064,"ci_90_lower":0.013187632798497953,"ci_90_upper":0.045367100591096755,"ci_75_lower":0.01641567161277635,"ci_75_upper":0.0392064639868843,"ci_50_lower":0.020299681161195106,"ci_50_upper":0.03419515359636928},"524288":{"avg_score":0.08833557001011194,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.20342280044022923,"median_score":0.016504526138305664,"q1_score":0,"q3_score":0.05943936016410589,"avg_token_efficiency":1.630310276184535,"is_incomplete":false,"total_prompt_cost_bin":4.338405000000003,"total_cache_read_cost_bin":99.11840399999963,"total_completion_cost_bin":5.6242250000000045,"total_reasoning_cost_bin":0.588975,"total_cost_bin":109.67000899999965,"total_on_demand_input_cost_bin":995.5224449999998,"ci_99_lower":0.059137950920385415,"ci_99_upper":0.12524914202964035,"ci_98_lower":0.06011468250491909,"ci_98_upper":0.12080494836598271,"ci_95_lower":0.06413247423284485,"ci_95_upper":0.11589971146093961,"ci_90_lower":0.0682017303863639,"ci_90_upper":0.11142482384560842,"ci_75_lower":0.07307596656533799,"ci_75_upper":0.10421664168430746,"ci_50_lower":0.0791136524149205,"ci_50_upper":0.09598966697956066}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.7","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-16T14:51:40.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T06:36:50.864Z"},{"model_slug":"anthropic/claude-opus-4.7","reasoning_mode":"high","overall_metrics":{"auc_1m":0.07596730162565886,"auc_128k":0.3237305665066977,"cum_avg_1m":0.31264632778235574,"cum_avg_128k":0.47921239029705454,"avg_score_overall":0.27961529648290034,"avg_response_time_ms":18945.277031130907,"total_prompt_cost":9.295505000000006,"total_cache_read_cost":124.13051949999964,"total_completion_cost":14.093250000000005,"total_reasoning_cost":1.3324250000000006,"total_prompt_cost_128k":2.8056200000000002,"total_completion_cost_128k":8.2677,"total_on_demand_input_cost":1250.6006999999995,"total_on_demand_input_cost_128k":124.86667500000004,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6295672248304613},"bin_metrics":{"8192":{"avg_score":0.7882684772586799,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.39738744368714063,"median_score":1,"q1_score":0.9748023003339767,"q3_score":1,"avg_token_efficiency":1.6618797159825593,"is_incomplete":false,"total_prompt_cost_bin":0.348885,"total_cache_read_cost_bin":0.40652949999999966,"total_completion_cost_bin":1.4411999999999998,"total_reasoning_cost_bin":0.0699,"total_cost_bin":2.2665144999999995,"total_on_demand_input_cost_bin":4.414180000000001,"ci_99_lower":0.6702205828655423,"ci_99_upper":0.8973528491847172,"ci_98_lower":0.6814146398720913,"ci_98_upper":0.8901633922334004,"ci_95_lower":0.6921775580985923,"ci_95_upper":0.8741745914629063,"ci_90_lower":0.7129495866879734,"ci_90_upper":0.8573580383322406,"ci_75_lower":0.7380685167971024,"ci_75_upper":0.8345534232985301,"ci_50_lower":0.7583591330101626,"ci_50_upper":0.8178792117865668},"16384":{"avg_score":0.863599805207923,"min_score":0.012921089306473732,"max_score":1,"n_tests":80,"stddev":0.33198435390926573,"median_score":0.9998892843723297,"q1_score":0.9905573129653931,"q3_score":1,"avg_token_efficiency":1.5738574220527406,"is_incomplete":false,"total_prompt_cost_bin":0.329135,"total_cache_read_cost_bin":0.8330924999999995,"total_completion_cost_bin":1.7183750000000004,"total_reasoning_cost_bin":0.1198,"total_cost_bin":3.0004025000000003,"total_on_demand_input_cost_bin":8.66006,"ci_99_lower":0.755495289436658,"ci_99_upper":0.9493179802523227,"ci_98_lower":0.7781578150866554,"ci_98_upper":0.9369399724297691,"ci_95_lower":0.7809151054988615,"ci_95_upper":0.9246245604497381,"ci_90_lower":0.7935025930416304,"ci_90_upper":0.9223152025090531,"ci_75_lower":0.8169704275875119,"ci_75_upper":0.9003450328862528,"ci_50_lower":0.8393988786498086,"ci_50_upper":0.8878006552695297},"32768":{"avg_score":0.42537985660687005,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.42769924558144634,"median_score":0.2887425124645233,"q1_score":0.019082377199083567,"q3_score":0.9927425980567932,"avg_token_efficiency":1.6528166626033123,"is_incomplete":false,"total_prompt_cost_bin":0.505995,"total_cache_read_cost_bin":3.1611774999999973,"total_completion_cost_bin":2.287499999999999,"total_reasoning_cost_bin":0.2140250000000001,"total_cost_bin":6.168697499999997,"total_on_demand_input_cost_bin":32.11777000000004,"ci_99_lower":0.3277737815080121,"ci_99_upper":0.514467294523899,"ci_98_lower":0.3386282076557879,"ci_98_upper":0.5109045697238745,"ci_95_lower":0.352572124633469,"ci_95_upper":0.497112470494133,"ci_90_lower":0.36561557199602246,"ci_90_upper":0.4859948192065263,"ci_75_lower":0.38429874824588994,"ci_75_upper":0.4686411155700412,"ci_50_lower":0.40042982876184396,"ci_50_upper":0.4515748987937781},"65536":{"avg_score":0.30751486584571575,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4122550882529937,"median_score":0.030245747417211533,"q1_score":0.00841423962265253,"q3_score":0.6995760202407837,"avg_token_efficiency":1.6333118582616915,"is_incomplete":false,"total_prompt_cost_bin":0.55795,"total_cache_read_cost_bin":4.431369999999998,"total_completion_cost_bin":2.65835,"total_reasoning_cost_bin":0.24044999999999997,"total_cost_bin":7.888119999999998,"total_on_demand_input_cost_bin":44.87165000000001,"ci_99_lower":0.2039755182884058,"ci_99_upper":0.42951364145595716,"ci_98_lower":0.209485177503077,"ci_98_upper":0.415825153577881,"ci_95_lower":0.22526647406435052,"ci_95_upper":0.4001665931832908,"ci_90_lower":0.23550640747040125,"ci_90_upper":0.37872227197622554,"ci_75_lower":0.2541804181499174,"ci_75_upper":0.35772535945596995,"ci_50_lower":0.27423007419976575,"ci_50_upper":0.33709387525690115},"131072":{"avg_score":0.011298946566084057,"min_score":0,"max_score":0.7286432385444641,"n_tests":103,"stddev":0.07281343737888507,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6117058522678809,"is_incomplete":false,"total_prompt_cost_bin":1.0636550000000002,"total_cache_read_cost_bin":3.373936,"total_completion_cost_bin":0.16227500000000006,"total_reasoning_cost_bin":0.030375000000000003,"total_cost_bin":4.630241000000001,"total_on_demand_input_cost_bin":34.803014999999995,"ci_99_lower":0.0016129499282100176,"ci_99_upper":0.03865275658479467,"ci_98_lower":0.0019643363461477083,"ci_98_upper":0.03294227135910208,"ci_95_lower":0.0022328904721415736,"ci_95_upper":0.027268554497785976,"ci_90_lower":0.0026629871715909068,"ci_90_upper":0.025068417414163862,"ci_75_lower":0.0036224501130614176,"ci_75_upper":0.019241321895235374,"ci_50_lower":0.004892248502091585,"ci_50_upper":0.01690167411867963},"262144":{"avg_score":0.014321623366748821,"min_score":0,"max_score":0.2926829159259796,"n_tests":141,"stddev":0.047487015507654315,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6299638830267154,"is_incomplete":false,"total_prompt_cost_bin":2.1514800000000003,"total_cache_read_cost_bin":12.806010000000011,"total_completion_cost_bin":0.6219,"total_reasoning_cost_bin":0.0872,"total_cost_bin":15.66659000000001,"total_on_demand_input_cost_bin":130.21157999999994,"ci_99_lower":0.005636139958997832,"ci_99_upper":0.025560948361943187,"ci_98_lower":0.006324621568374495,"ci_98_upper":0.024765938241695255,"ci_95_lower":0.007477083512423362,"ci_95_upper":0.023024315226918862,"ci_90_lower":0.008313357661121545,"ci_90_upper":0.021221312436486544,"ci_75_lower":0.009922473091575984,"ci_75_upper":0.019045410108511462,"ci_50_lower":0.011452801611459425,"ci_50_upper":0.016778963926770057},"524288":{"avg_score":0.09078704740682428,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.20706122036734181,"median_score":0.019324839115142822,"q1_score":0,"q3_score":0.06353592313826084,"avg_token_efficiency":1.630310276184536,"is_incomplete":false,"total_prompt_cost_bin":4.338405000000004,"total_cache_read_cost_bin":99.11840399999963,"total_completion_cost_bin":5.203650000000004,"total_reasoning_cost_bin":0.5706750000000004,"total_cost_bin":109.23113399999964,"total_on_demand_input_cost_bin":995.5224449999996,"ci_99_lower":0.06013231443704927,"ci_99_upper":0.12792391494593297,"ci_98_lower":0.06205526067616844,"ci_98_upper":0.12396077089961735,"ci_95_lower":0.06590047136234747,"ci_95_upper":0.11810677056449911,"ci_90_lower":0.069988498803459,"ci_90_upper":0.11411234969214015,"ci_75_lower":0.07556283505489433,"ci_75_upper":0.10655433472807783,"ci_50_lower":0.08143010308450063,"ci_50_upper":0.09897797477556891}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.7","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-16T14:51:40.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T12:33:25.043Z"},{"model_slug":"moonshotai/kimi-k2.5","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.07532864652112284,"auc_128k":0.46168099405928786,"cum_avg_1m":0.3783446410578857,"cum_avg_128k":0.6053514256926171,"avg_score_overall":0.5921624722615874,"avg_response_time_ms":73735.88470711195,"total_prompt_cost":0.5962160100000001,"total_cache_read_cost":2.2169240849999987,"total_completion_cost":3.4657368,"total_reasoning_cost":3.3001917000000005,"total_prompt_cost_128k":0.5962160100000001,"total_completion_cost_128k":3.4657368,"total_on_demand_input_cost":13.897760519999995,"total_on_demand_input_cost_128k":13.897760519999995,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.994303650811002},"bin_metrics":{"8192":{"avg_score":0.9511613255832344,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.21313403420948665,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":0.9901216565918297,"is_incomplete":false,"total_prompt_cost_bin":0.08997792000000003,"total_cache_read_cost_bin":0.03504512,"total_completion_cost_bin":0.4484646,"total_reasoning_cost_bin":0.4314672,"total_cost_bin":1.00495484,"total_on_demand_input_cost_bin":0.3002486400000002,"ci_99_lower":0.8781986132513266,"ci_99_upper":0.9998517274856568,"ci_98_lower":0.888988127856981,"ci_98_upper":0.9997539624571801,"ci_95_lower":0.8909904185379856,"ci_95_upper":0.9879706538747997,"ci_90_lower":0.9028885462367907,"ci_90_upper":0.9876942828297615,"ci_75_lower":0.9263768778764643,"ci_75_upper":0.9757027519575786,"ci_50_lower":0.9386742392089219,"ci_50_upper":0.9637354593258352},"16384":{"avg_score":0.769274306728039,"min_score":0.008561643771827221,"max_score":1,"n_tests":80,"stddev":0.41602323252931594,"median_score":1,"q1_score":0.9996761679649353,"q3_score":1,"avg_token_efficiency":0.9804544918987934,"is_incomplete":false,"total_prompt_cost_bin":0.09419135999999993,"total_cache_read_cost_bin":0.08694399999999995,"total_completion_cost_bin":0.57313215,"total_reasoning_cost_bin":0.5485879499999998,"total_cost_bin":1.3028554599999995,"total_on_demand_input_cost_bin":0.6158553600000009,"ci_99_lower":0.6470988327580272,"ci_99_upper":0.8788714581417735,"ci_98_lower":0.6591844495694386,"ci_98_upper":0.866865270313341,"ci_95_lower":0.6728398749165354,"ci_95_upper":0.85465254429786,"ci_90_lower":0.695651993319625,"ci_90_upper":0.8421016044577119,"ci_75_lower":0.7200971180674969,"ci_75_upper":0.8182569337834138,"ci_50_lower":0.7336968007672112,"ci_50_upper":0.8051261997083202},"32768":{"avg_score":0.6272751298606368,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.46787704251288126,"median_score":1,"q1_score":0.03731883969157934,"q3_score":1,"avg_token_efficiency":1.0101715848916963,"is_incomplete":false,"total_prompt_cost_bin":0.11113632000000002,"total_cache_read_cost_bin":0.35363788,"total_completion_cost_bin":0.9352303499999999,"total_reasoning_cost_bin":0.8694039,"total_cost_bin":2.26940845,"total_on_demand_input_cost_bin":2.232963599999998,"ci_99_lower":0.5216437006135718,"ci_99_upper":0.7243830233097319,"ci_98_lower":0.5360404015889201,"ci_98_upper":0.7193162419472697,"ci_95_lower":0.5485989161244051,"ci_95_upper":0.703899077590091,"ci_90_lower":0.562192471757846,"ci_90_upper":0.6962130069102654,"ci_75_lower":0.579122092034702,"ci_75_upper":0.6723778828120831,"ci_50_lower":0.6012127881288453,"ci_50_upper":0.6522038361911067},"65536":{"avg_score":0.34885596662951524,"min_score":0.001463057822547853,"max_score":1,"n_tests":85,"stddev":0.4589372518995409,"median_score":0.03531459718942642,"q1_score":0.02375861257314682,"q3_score":1,"avg_token_efficiency":0.9907670367000744,"is_incomplete":false,"total_prompt_cost_bin":0.1495617300000001,"total_cache_read_cost_bin":0.49839336999999967,"total_completion_cost_bin":0.6830794499999999,"total_reasoning_cost_bin":0.6594957000000002,"total_cost_bin":1.99053025,"total_on_demand_input_cost_bin":3.1399219499999997,"ci_99_lower":0.24227684500287563,"ci_99_upper":0.47508797988262685,"ci_98_lower":0.2436720485652331,"ci_98_upper":0.46442425964142686,"ci_95_lower":0.25527716026768116,"ci_95_upper":0.45136185707743554,"ci_90_lower":0.2674506675025366,"ci_90_upper":0.4378024041546536,"ci_75_lower":0.2905404267325888,"ci_75_upper":0.4058254549740334,"ci_50_lower":0.31436217056231247,"ci_50_upper":0.3825043407689972},"131072":{"avg_score":0.3301903996616602,"min_score":0.00544959120452404,"max_score":1,"n_tests":103,"stddev":0.4519612668060211,"median_score":0.030490955337882042,"q1_score":0.022889805026352406,"q3_score":1,"avg_token_efficiency":0.9902751596941558,"is_incomplete":false,"total_prompt_cost_bin":0.15134867999999999,"total_cache_read_cost_bin":1.2429037149999989,"total_completion_cost_bin":0.8258302500000001,"total_reasoning_cost_bin":0.7912369500000004,"total_cost_bin":3.0113195949999993,"total_on_demand_input_cost_bin":7.608770969999997,"ci_99_lower":0.22336558020280367,"ci_99_upper":0.4430733389718439,"ci_98_lower":0.23264906101796815,"ci_98_upper":0.4333132784230887,"ci_95_lower":0.24276768985098154,"ci_95_upper":0.4172030384000164,"ci_90_lower":0.2551540303236163,"ci_90_upper":0.4060686878059782,"ci_75_lower":0.27813253630669243,"ci_75_upper":0.3814856943943384,"ci_50_lower":0.30210060973932007,"ci_50_upper":0.35866590568601403}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"MoonshotAI: Kimi K2.5","provider_name":"Moonshotai","run_provider_names":["Moonshot AI"],"pricing_metadata":{"prompt":6e-7,"completion":0.000003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-01-27T04:11:16.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T13:04:21.293Z"},{"model_slug":"inclusionai/ling-3.0-flash","reasoning_mode":null,"overall_metrics":{"auc_1m":0.07510588615486884,"auc_128k":0.44336827754232083,"cum_avg_1m":0.320055453745984,"cum_avg_128k":0.5120887259935744,"avg_score_overall":0.501899481359446,"avg_response_time_ms":3052.9334642394515,"total_prompt_cost":0.056355453000000014,"total_cache_read_cost":0.09644167680000003,"total_completion_cost":0.012037725,"total_reasoning_cost":0,"total_prompt_cost_128k":0.056355453000000014,"total_completion_cost_128k":0.012037725,"total_on_demand_input_cost":0.5385638369999999,"total_on_demand_input_cost_128k":0.5385638369999999,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.047507393384501},"bin_metrics":{"8192":{"avg_score":0.6614487698534504,"min_score":0.013631090521812439,"max_score":1,"n_tests":80,"stddev":0.46436085920048764,"median_score":1,"q1_score":0.036061935126781464,"q3_score":1,"avg_token_efficiency":1.0528515462772823,"is_incomplete":false,"total_prompt_cost_bin":0.00412104,"total_cache_read_cost_bin":0.0015238272000000002,"total_completion_cost_bin":0.0018730529999999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.0075179202,"total_on_demand_input_cost_bin":0.011740175999999986,"ci_99_lower":0.5162194938069443,"ci_99_upper":0.7835719031357439,"ci_98_lower":0.53965714773559,"ci_98_upper":0.7710489643448962,"ci_95_lower":0.5636074006487615,"ci_95_upper":0.7577175974589773,"ci_90_lower":0.5765116701635997,"ci_90_upper":0.744880041956203,"ci_75_lower":0.6010140919679542,"ci_75_upper":0.721283251576824,"ci_50_lower":0.6247796313778963,"ci_50_upper":0.6974488694686442},"16384":{"avg_score":0.6367625578539446,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.4719419127992339,"median_score":1,"q1_score":0.03658138029277325,"q3_score":1,"avg_token_efficiency":1.0491127780215674,"is_incomplete":false,"total_prompt_cost_bin":0.006540575999999999,"total_cache_read_cost_bin":0.003554880000000003,"total_completion_cost_bin":0.0018219600000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.011917416000000004,"total_on_demand_input_cost_bin":0.024314975999999978,"ci_99_lower":0.49045145127957224,"ci_99_upper":0.7693585773402591,"ci_98_lower":0.5037077879523859,"ci_98_upper":0.7465071808653884,"ci_95_lower":0.5285195022739936,"ci_95_upper":0.7333677802921739,"ci_90_lower":0.5510437746962998,"ci_90_upper":0.7214232421678025,"ci_75_lower":0.5757139135355829,"ci_75_upper":0.6972452870657435,"ci_50_lower":0.6004547483520581,"ci_50_upper":0.6723051763081458},"32768":{"avg_score":0.4955856758341029,"min_score":0.005855290684849024,"max_score":1,"n_tests":136,"stddev":0.4782680402939276,"median_score":0.13107780367136002,"q1_score":0.03469214867800474,"q3_score":1,"avg_token_efficiency":1.0488334855612607,"is_incomplete":false,"total_prompt_cost_bin":0.007092624000000002,"total_cache_read_cost_bin":0.01565733119999996,"total_completion_cost_bin":0.0034891920000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.026239147199999965,"total_on_demand_input_cost_bin":0.08537927999999993,"ci_99_lower":0.39431996593287777,"ci_99_upper":0.6006834857242167,"ci_98_lower":0.40206459594274163,"ci_98_upper":0.5892995281795532,"ci_95_lower":0.41661228724868343,"ci_95_upper":0.5764236495379865,"ci_90_lower":0.4277353383463752,"ci_90_upper":0.5619487688529154,"ci_75_lower":0.4510675635285545,"ci_75_upper":0.5439610812232679,"ci_50_lower":0.46880186176841726,"ci_50_upper":0.5236722711329896},"65536":{"avg_score":0.4056562041096828,"min_score":0.00757862813770771,"max_score":1,"n_tests":85,"stddev":0.4762686351633062,"median_score":0.03781979903578758,"q1_score":0.025072885677218437,"q3_score":1,"avg_token_efficiency":1.044747363942775,"is_incomplete":false,"total_prompt_cost_bin":0.011620139999999998,"total_cache_read_cost_bin":0.022072780800000004,"total_completion_cost_bin":0.002222514,"total_reasoning_cost_bin":0,"total_cost_bin":0.0359154348,"total_on_demand_input_cost_bin":0.12198404400000004,"ci_99_lower":0.2784981440483209,"ci_99_upper":0.5429364395154201,"ci_98_lower":0.29045223962416983,"ci_98_upper":0.5312278913822682,"ci_95_lower":0.3024091619822909,"ci_95_upper":0.5072995621680886,"ci_90_lower":0.3149879357163959,"ci_90_upper":0.4871765716045218,"ci_75_lower":0.3476384521598982,"ci_75_upper":0.4634506199495209,"ci_50_lower":0.37132882718270754,"ci_50_upper":0.44017868419451744},"131072":{"avg_score":0.36099042231669126,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.46122820683715565,"median_score":0.037472594529390335,"q1_score":0.02301869262009859,"q3_score":1,"avg_token_efficiency":1.0426364318710986,"is_incomplete":false,"total_prompt_cost_bin":0.02698107300000001,"total_cache_read_cost_bin":0.05363285760000006,"total_completion_cost_bin":0.002631006,"total_reasoning_cost_bin":0,"total_cost_bin":0.08324493660000008,"total_on_demand_input_cost_bin":0.29514536099999994,"ci_99_lower":0.24904051559710494,"ci_99_upper":0.4731913765295164,"ci_98_lower":0.2613382859769485,"ci_98_upper":0.46398142265409087,"ci_95_lower":0.2686185431445144,"ci_95_upper":0.4524410418682343,"ci_90_lower":0.2854359556739987,"ci_90_upper":0.4354604162755015,"ci_75_lower":0.30662038657267227,"ci_75_upper":0.4156686846783963,"ci_50_lower":0.33098472931729717,"ci_50_upper":0.38997641560458807}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Ling-3.0-flash","provider_name":"Inclusionai","run_provider_names":["NovitaAI"],"pricing_metadata":{"prompt":2.1e-8,"completion":6.3e-8,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-23T14:56:20.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-12T19:44:19.309Z"},{"model_slug":"bytedance-seed/seed-2-1-turbo","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.07446371696507348,"auc_128k":0.5040750739683735,"cum_avg_1m":0.39564057060603863,"cum_avg_128k":0.6330249129696618,"avg_score_overall":0.6207496765136324,"avg_response_time_ms":249616.98818095986,"total_prompt_cost":1.0423064999999996,"total_cache_read_cost":11.380543999999986,"total_completion_cost":21.3381725,"total_reasoning_cost":20.861475,"total_prompt_cost_128k":1.0423064999999996,"total_completion_cost_128k":21.3381725,"total_on_demand_input_cost":12.4228505,"total_on_demand_input_cost_128k":12.4228505,"total_runs":482,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0262579292172704},"bin_metrics":{"8192":{"avg_score":0.9756180462893098,"min_score":0.02108139730989933,"max_score":1,"n_tests":80,"stddev":0.15322702054055892,"median_score":1,"q1_score":1,"q3_score":1,"avg_token_efficiency":1.035886368838225,"is_incomplete":false,"total_prompt_cost_bin":0.0696079999999999,"total_cache_read_cost_bin":0.20556799999999986,"total_completion_cost_bin":2.1576100000000005,"total_reasoning_cost_bin":2.0860425000000014,"total_cost_bin":4.5188285000000015,"total_on_demand_input_cost_bin":0.27517600000000003,"ci_99_lower":0.9267626724545844,"ci_99_upper":1,"ci_98_lower":0.9269433299973607,"ci_98_upper":1,"ci_95_lower":0.9389996100449934,"ci_95_upper":1,"ci_90_lower":0.9391816327581182,"ci_90_upper":1,"ci_75_lower":0.951327103935182,"ci_75_upper":1,"ci_50_lower":0.9634725751122459,"ci_50_upper":0.9878545288229361},"16384":{"avg_score":0.7825025122612714,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4059866408973592,"median_score":1,"q1_score":0.9980420768260956,"q3_score":1,"avg_token_efficiency":1.0234062021243353,"is_incomplete":false,"total_prompt_cost_bin":0.09959199999999999,"total_cache_read_cost_bin":0.4647519999999998,"total_completion_cost_bin":3.4021600000000007,"total_reasoning_cost_bin":3.3304774999999998,"total_cost_bin":7.2969815,"total_on_demand_input_cost_bin":0.5643440000000001,"ci_99_lower":0.6622903399413917,"ci_99_upper":0.8910302629367215,"ci_98_lower":0.6760880077555775,"ci_98_upper":0.8804701826286037,"ci_95_lower":0.6870054978062399,"ci_95_upper":0.8678298190393252,"ci_90_lower":0.7092995938414243,"ci_90_upper":0.854553419178119,"ci_75_lower":0.7330382923100842,"ci_75_upper":0.8305467458936618,"ci_50_lower":0.7481474733387585,"ci_50_upper":0.8084015593922231},"32768":{"avg_score":0.6895064111792154,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4510979458761337,"median_score":1,"q1_score":0.049879797734320164,"q3_score":1,"avg_token_efficiency":1.0282972468514218,"is_incomplete":false,"total_prompt_cost_bin":0.19816400000000023,"total_cache_read_cost_bin":1.7952720000000013,"total_completion_cost_bin":7.604480000000001,"total_reasoning_cost_bin":7.4687399999999995,"total_cost_bin":17.066656000000002,"total_on_demand_input_cost_bin":1.993436000000001,"ci_99_lower":0.5821640611164484,"ci_99_upper":0.7818732575672124,"ci_98_lower":0.5925828140585423,"ci_98_upper":0.7748144118426887,"ci_95_lower":0.6062751938502462,"ci_95_upper":0.7619335110779502,"ci_90_lower":0.6295692230835009,"ci_90_upper":0.7525619971978491,"ci_75_lower":0.6453970513051472,"ci_75_upper":0.7314724008525338,"ci_50_lower":0.6642304891446495,"ci_50_upper":0.7152449647110188},"65536":{"avg_score":0.48052685198867146,"min_score":0.00469667324796319,"max_score":1,"n_tests":85,"stddev":0.4849629605692856,"median_score":0.06062961369752884,"q1_score":0.0216621533036232,"q3_score":1,"avg_token_efficiency":1.023394371105109,"is_incomplete":false,"total_prompt_cost_bin":0.31714100000000006,"total_cache_read_cost_bin":2.527880000000001,"total_completion_cost_bin":4.170769999999998,"total_reasoning_cost_bin":4.0834325,"total_cost_bin":11.099223499999999,"total_on_demand_input_cost_bin":2.845021000000001,"ci_99_lower":0.34455912754829865,"ci_99_upper":0.6107447485856382,"ci_98_lower":0.3589490164629019,"ci_98_upper":0.6091202905554982,"ci_95_lower":0.3741320678565706,"ci_95_upper":0.589618478691178,"ci_90_lower":0.39086149038609996,"ci_90_upper":0.5717203892290811,"ci_75_lower":0.41857884963974357,"ci_75_upper":0.5427192192873918,"ci_50_lower":0.4471562962773639,"ci_50_upper":0.5146445399136557},"131072":{"avg_score":0.23697074312984132,"min_score":0.0034722222480922937,"max_score":1,"n_tests":101,"stddev":0.4036042873722065,"median_score":0.02890770137310028,"q1_score":0.018018018454313278,"q3_score":0.04430379718542099,"avg_token_efficiency":1.0205541494058608,"is_incomplete":false,"total_prompt_cost_bin":0.35780149999999955,"total_cache_read_cost_bin":6.387071999999984,"total_completion_cost_bin":4.003152500000001,"total_reasoning_cost_bin":3.892782500000001,"total_cost_bin":14.640808499999986,"total_on_demand_input_cost_bin":6.744873499999998,"ci_99_lower":0.14068450067857383,"ci_99_upper":0.3349496767738843,"ci_98_lower":0.14983337558891238,"ci_98_upper":0.33207642903124984,"ci_95_lower":0.1666986092453071,"ci_95_upper":0.3132409043794517,"ci_90_lower":0.17753790248102405,"ci_90_upper":0.30397404014434903,"ci_75_lower":0.189563323880804,"ci_75_upper":0.28404929884911123,"ci_50_lower":0.20842746715119,"ci_50_upper":0.2645929350764352}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"ByteDance Seed: Seed 2.1 Turbo","provider_name":"Bytedance-seed","run_provider_names":["Seed"],"pricing_metadata":{"prompt":5e-7,"completion":0.0000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T16:29:36.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T14:26:05.478Z"},{"model_slug":"anthropic/claude-opus-4.7","reasoning_mode":"xhigh","overall_metrics":{"auc_1m":0.07293888797782873,"auc_128k":0.3039086467313325,"cum_avg_1m":0.3183586987174312,"cum_avg_128k":0.4878909634999421,"avg_score_overall":0.28542999397629587,"avg_response_time_ms":18547.495815015696,"total_prompt_cost":9.295505000000007,"total_cache_read_cost":124.13051949999964,"total_completion_cost":14.494975000000002,"total_reasoning_cost":1.3819999999999997,"total_prompt_cost_128k":2.8056199999999993,"total_completion_cost_128k":8.577449999999999,"total_on_demand_input_cost":1250.6007000000002,"total_on_demand_input_cost_128k":124.86667500000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6295672248304613},"bin_metrics":{"8192":{"avg_score":0.8896256760213873,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.3028524498002176,"median_score":1,"q1_score":0.9935799390077591,"q3_score":1,"avg_token_efficiency":1.6618797159825593,"is_incomplete":false,"total_prompt_cost_bin":0.34888499999999983,"total_cache_read_cost_bin":0.4065294999999997,"total_completion_cost_bin":1.56525,"total_reasoning_cost_bin":0.08369999999999997,"total_cost_bin":2.4043644999999993,"total_on_demand_input_cost_bin":4.414180000000003,"ci_99_lower":0.7950719365589975,"ci_99_upper":0.9658736312449909,"ci_98_lower":0.8018493914068531,"ci_98_upper":0.9594829828116345,"ci_95_lower":0.8143386290226045,"ci_95_upper":0.947292036302424,"ci_90_lower":0.826329076701877,"ci_90_upper":0.939192689739284,"ci_75_lower":0.8476308578885438,"ci_75_upper":0.9251717665027173,"ci_50_lower":0.8654599620629596,"ci_50_upper":0.9109976958065089},"16384":{"avg_score":0.8399807866895571,"min_score":0.016675930470228195,"max_score":1,"n_tests":80,"stddev":0.35729045572798984,"median_score":0.9998892843723297,"q1_score":0.9902487397193909,"q3_score":1,"avg_token_efficiency":1.5738574220527404,"is_incomplete":false,"total_prompt_cost_bin":0.32913499999999996,"total_cache_read_cost_bin":0.8330924999999996,"total_completion_cost_bin":1.7499499999999997,"total_reasoning_cost_bin":0.12692499999999998,"total_cost_bin":3.0391024999999994,"total_on_demand_input_cost_bin":8.66006,"ci_99_lower":0.7298734593575354,"ci_99_upper":0.9367539643127238,"ci_98_lower":0.731560306308791,"ci_98_upper":0.9248432470052503,"ci_95_lower":0.7556857399235014,"ci_95_upper":0.9131027551501756,"ci_90_lower":0.7685927674360573,"ci_90_upper":0.9003809583676048,"ci_75_lower":0.791844017218682,"ci_75_upper":0.8871708209539064,"ci_50_lower":0.8153752338606866,"ci_50_upper":0.8642545736394823},"32768":{"avg_score":0.4627777750882909,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4296008763200507,"median_score":0.3304823338985443,"q1_score":0.03471910208463669,"q3_score":0.9870645552873611,"avg_token_efficiency":1.652816662603312,"is_incomplete":false,"total_prompt_cost_bin":0.505995,"total_cache_read_cost_bin":3.1611774999999978,"total_completion_cost_bin":2.5609250000000006,"total_reasoning_cost_bin":0.24164999999999998,"total_cost_bin":6.469747499999999,"total_on_demand_input_cost_bin":32.117770000000036,"ci_99_lower":0.36354960647329465,"ci_99_upper":0.5529015045727155,"ci_98_lower":0.3752574178783963,"ci_98_upper":0.5454920315341801,"ci_95_lower":0.39024669928681655,"ci_95_upper":0.5345425537873618,"ci_90_lower":0.40182811832064014,"ci_90_upper":0.5245989591932277,"ci_75_lower":0.42230182059136173,"ci_75_upper":0.5060123897973801,"ci_50_lower":0.4370634230917323,"ci_50_upper":0.4879691937648578},"65536":{"avg_score":0.23861501942909158,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.3611633984901891,"median_score":0.03097345121204853,"q1_score":0.009871668182313442,"q3_score":0.3464977741241455,"avg_token_efficiency":1.6333118582616915,"is_incomplete":false,"total_prompt_cost_bin":0.5579499999999998,"total_cache_read_cost_bin":4.431369999999998,"total_completion_cost_bin":2.496725,"total_reasoning_cost_bin":0.24509999999999993,"total_cost_bin":7.731144999999997,"total_on_demand_input_cost_bin":44.871649999999995,"ci_99_lower":0.1499833440643475,"ci_99_upper":0.3522252611494486,"ci_98_lower":0.15713465181299868,"ci_98_upper":0.33616274878677144,"ci_95_lower":0.16749201020643606,"ci_95_upper":0.31608066562768605,"ci_90_lower":0.17473649063825553,"ci_90_upper":0.3022584693328313,"ci_75_lower":0.1932793973131886,"ci_75_upper":0.2837138645501588,"ci_50_lower":0.20977399347808814,"ci_50_upper":0.2622203687202487},"131072":{"avg_score":0.008455560271384067,"min_score":0,"max_score":0.3571428656578064,"n_tests":103,"stddev":0.037395858078385504,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6117058522678813,"is_incomplete":false,"total_prompt_cost_bin":1.063655,"total_cache_read_cost_bin":3.373936,"total_completion_cost_bin":0.2046,"total_reasoning_cost_bin":0.032025,"total_cost_bin":4.674216,"total_on_demand_input_cost_bin":34.803014999999995,"ci_99_lower":0.002292814385371952,"ci_99_upper":0.022366102328117105,"ci_98_lower":0.0027013303614357143,"ci_98_upper":0.01961948021703842,"ci_95_lower":0.0031023514282989125,"ci_95_upper":0.01672161167026362,"ci_90_lower":0.0035133840480041736,"ci_90_upper":0.015035196617516929,"ci_75_lower":0.004355256944144308,"ci_75_upper":0.01278479910767592,"ci_50_lower":0.005579279866792913,"ci_50_upper":0.010823928592817007},"262144":{"avg_score":0.0216226865610121,"min_score":0,"max_score":0.7931727170944214,"n_tests":141,"stddev":0.09751265288566735,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6299638830267156,"is_incomplete":false,"total_prompt_cost_bin":2.1514800000000007,"total_cache_read_cost_bin":12.806010000000011,"total_completion_cost_bin":0.6758999999999998,"total_reasoning_cost_bin":0.099675,"total_cost_bin":15.733065000000012,"total_on_demand_input_cost_bin":130.21157999999997,"ci_99_lower":0.005406591431634065,"ci_99_upper":0.04827905160341914,"ci_98_lower":0.006511010368823582,"ci_98_upper":0.044743589253845154,"ci_95_lower":0.008225394680382723,"ci_95_upper":0.0394756902275105,"ci_90_lower":0.010160794346890552,"ci_90_upper":0.036332902783884645,"ci_75_lower":0.0127525701862925,"ci_75_upper":0.03090537683988743,"ci_50_lower":0.015811778636358624,"ci_50_upper":0.027141985211006185},"524288":{"avg_score":0.08579208567872706,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.19568475920481476,"median_score":0.018269157968461514,"q1_score":0,"q3_score":0.05959927011281252,"avg_token_efficiency":1.6303102761845352,"is_incomplete":false,"total_prompt_cost_bin":4.338405000000007,"total_cache_read_cost_bin":99.11840399999963,"total_completion_cost_bin":5.241625000000003,"total_reasoning_cost_bin":0.5529249999999999,"total_cost_bin":109.25135899999964,"total_on_demand_input_cost_bin":995.5224450000003,"ci_99_lower":0.056911723011635024,"ci_99_upper":0.12050513523242089,"ci_98_lower":0.059068071414160116,"ci_98_upper":0.11752106867884522,"ci_95_lower":0.0625214313804093,"ci_95_upper":0.11181486730781477,"ci_90_lower":0.06634599752625514,"ci_90_upper":0.10790254931001118,"ci_75_lower":0.07130480356148436,"ci_75_upper":0.10095114276376434,"ci_50_lower":0.07710973123212787,"ci_50_upper":0.09331926221118733}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.7","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-16T14:51:40.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T13:13:46.944Z"},{"model_slug":"qwen/qwen3.8-27b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.07272626767568705,"auc_128k":0.4309454009830886,"cum_avg_1m":0.29170872418819604,"cum_avg_128k":0.46673395870111367,"avg_score_overall":0.466173862534275,"avg_response_time_ms":23529.26385901585,"total_prompt_cost":4.507723799999999,"total_cache_read_cost":6.749769599999995,"total_completion_cost":0.6072384,"total_reasoning_cost":0,"total_prompt_cost_128k":4.507723799999999,"total_completion_cost_128k":0.6072384,"total_on_demand_input_cost":11.257493400000008,"total_on_demand_input_cost_128k":11.257493400000008,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.020812887056613},"bin_metrics":{"8192":{"avg_score":0.6029650793178007,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.47713245045963953,"median_score":1,"q1_score":0.03311022836714983,"q3_score":1,"avg_token_efficiency":1.0228079512716752,"is_incomplete":false,"total_prompt_cost_bin":0.17933040000000014,"total_cache_read_cost_bin":0.06526800000000002,"total_completion_cost_bin":0.0898144,"total_reasoning_cost_bin":0,"total_cost_bin":0.3344128000000002,"total_on_demand_input_cost_bin":0.24459840000000016,"ci_99_lower":0.45859186805051283,"ci_99_upper":0.7365382589633808,"ci_98_lower":0.4697879225353244,"ci_98_upper":0.7234186859217007,"ci_95_lower":0.4969395521914703,"ci_95_upper":0.701165456364397,"ci_90_lower":0.5162545861193212,"ci_90_upper":0.6879290028428658,"ci_75_lower":0.5431831723777577,"ci_75_upper":0.6638255119833049,"ci_50_lower":0.5670452910388122,"ci_50_upper":0.63765570858086},"16384":{"avg_score":0.44099207847029903,"min_score":0.0044469148851931095,"max_score":1,"n_tests":80,"stddev":0.4837087892148955,"median_score":0.043738072738051414,"q1_score":0.024936684872955084,"q3_score":1,"avg_token_efficiency":1.0161194758384635,"is_incomplete":false,"total_prompt_cost_bin":0.31987440000000017,"total_cache_read_cost_bin":0.18451440000000002,"total_completion_cost_bin":0.0906464,"total_reasoning_cost_bin":0,"total_cost_bin":0.5950352000000002,"total_on_demand_input_cost_bin":0.5043888000000002,"ci_99_lower":0.2959386302584317,"ci_99_upper":0.5736300754444091,"ci_98_lower":0.3066004171304521,"ci_98_upper":0.5622293418223854,"ci_95_lower":0.3309095224621706,"ci_95_upper":0.5503105900865921,"ci_90_lower":0.3441566722712014,"ci_90_upper":0.5278018174367025,"ci_75_lower":0.37941257410639084,"ci_75_upper":0.50220424384097,"ci_50_lower":0.4042819254100323,"ci_50_upper":0.4774020181444939},"32768":{"avg_score":0.5146839039687834,"min_score":0.0021052630618214607,"max_score":1,"n_tests":136,"stddev":0.48066886676815823,"median_score":0.18946504592895508,"q1_score":0.034202023409307,"q3_score":1,"avg_token_efficiency":1.0257548592579657,"is_incomplete":false,"total_prompt_cost_bin":0.7548335999999999,"total_cache_read_cost_bin":1.0347623999999997,"total_completion_cost_bin":0.17460799999999987,"total_reasoning_cost_bin":0,"total_cost_bin":1.9642039999999994,"total_on_demand_input_cost_bin":1.7895959999999997,"ci_99_lower":0.41208502635489874,"ci_99_upper":0.6193401619039564,"ci_98_lower":0.4209658778553335,"ci_98_upper":0.6090603934572529,"ci_95_lower":0.43509513862167404,"ci_95_upper":0.5910974069976981,"ci_90_lower":0.44479365301530754,"ci_90_upper":0.5812315011664073,"ci_75_lower":0.46930992182238057,"ci_75_upper":0.5626249260856317,"ci_50_lower":0.48756182460569747,"ci_50_upper":0.5440869040253079},"65536":{"avg_score":0.4285218592404443,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4811569875124563,"median_score":0.043887145817279816,"q1_score":0.02454473450779915,"q3_score":1,"avg_token_efficiency":1.0209904012221072,"is_incomplete":false,"total_prompt_cost_bin":0.9051641999999996,"total_cache_read_cost_bin":1.6493399999999983,"total_completion_cost_bin":0.1119008,"total_reasoning_cost_bin":0,"total_cost_bin":2.666404999999998,"total_on_demand_input_cost_bin":2.5545041999999993,"ci_99_lower":0.28934554128431,"ci_99_upper":0.5650068241185783,"ci_98_lower":0.3033276414508548,"ci_98_upper":0.5548128127585001,"ci_95_lower":0.32395898902898324,"ci_95_upper":0.5309381455478862,"ci_90_lower":0.3374940842663979,"ci_90_upper":0.5108023644759155,"ci_75_lower":0.37020227464381605,"ci_75_upper":0.4864670391701271,"ci_50_lower":0.3937719155185143,"ci_50_upper":0.46354902952306853},"131072":{"avg_score":0.34650687250824086,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4611637795231345,"median_score":0.03319108486175537,"q1_score":0.0199356097728014,"q3_score":1,"avg_token_efficiency":1.0162368757633713,"is_incomplete":false,"total_prompt_cost_bin":2.3485211999999995,"total_cache_read_cost_bin":3.815884799999998,"total_completion_cost_bin":0.14026880000000003,"total_reasoning_cost_bin":0,"total_cost_bin":6.304674799999998,"total_on_demand_input_cost_bin":6.1644060000000085,"ci_99_lower":0.23945142883488693,"ci_99_upper":0.4601579987262918,"ci_98_lower":0.24210180009823762,"ci_98_upper":0.45214986659816997,"ci_95_lower":0.2591778914750445,"ci_95_upper":0.4333410792090044,"ci_90_lower":0.27063344901007175,"ci_90_upper":0.42185787962392446,"ci_75_lower":0.2905387233104105,"ci_75_upper":0.40150523834943197,"ci_50_lower":0.31806200248125976,"ci_50_upper":0.37556812176435683}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","medium","low"],"reasoning_default":"xhigh","sponsors":[],"openrouter_name":"Qwen: Qwen3.8 27B","provider_name":"Qwen","run_provider_names":["AkashML"],"pricing_metadata":{"prompt":4.25e-7,"completion":0.00000255,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-14T15:55:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-16T19:53:01.778Z"},{"model_slug":"anthropic/claude-opus-4.7","reasoning_mode":"low","overall_metrics":{"auc_1m":0.07195200332931476,"auc_128k":0.3311701260064203,"cum_avg_1m":0.32260934654638185,"cum_avg_128k":0.4972146158089796,"avg_score_overall":0.2863479883680591,"avg_response_time_ms":24167.079851848324,"total_prompt_cost":9.295505000000006,"total_cache_read_cost":124.13051949999964,"total_completion_cost":15.091325000000001,"total_reasoning_cost":1.4718000000000004,"total_prompt_cost_128k":2.8056200000000002,"total_completion_cost_128k":9.261149999999999,"total_on_demand_input_cost":1250.6006999999995,"total_on_demand_input_cost_128k":124.86667500000004,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6295672248304613},"bin_metrics":{"8192":{"avg_score":0.865542647158145,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.3336782913288088,"median_score":1,"q1_score":0.9942052066326141,"q3_score":1,"avg_token_efficiency":1.6618797159825598,"is_incomplete":false,"total_prompt_cost_bin":0.348885,"total_cache_read_cost_bin":0.4065294999999996,"total_completion_cost_bin":1.5385749999999994,"total_reasoning_cost_bin":0.08787500000000002,"total_cost_bin":2.381864499999999,"total_on_demand_input_cost_bin":4.414180000000002,"ci_99_lower":0.7631369310997034,"ci_99_upper":0.9532900963345382,"ci_98_lower":0.777859187796712,"ci_98_upper":0.9409640444953548,"ci_95_lower":0.7866024073094013,"ci_95_upper":0.9278796942448388,"ci_90_lower":0.7970329047954874,"ci_90_upper":0.9221456313240923,"ci_75_lower":0.8227120012921659,"ci_75_upper":0.9035413192992564,"ci_50_lower":0.8409753664789604,"ci_50_upper":0.8904459794823196},"16384":{"avg_score":0.846133433363866,"min_score":0.009156310930848122,"max_score":1,"n_tests":80,"stddev":0.34738974417539975,"median_score":0.9989176392555237,"q1_score":0.9893791675567627,"q3_score":1,"avg_token_efficiency":1.5738574220527408,"is_incomplete":false,"total_prompt_cost_bin":0.32913499999999996,"total_cache_read_cost_bin":0.8330924999999996,"total_completion_cost_bin":1.9032000000000004,"total_reasoning_cost_bin":0.13907500000000003,"total_cost_bin":3.2045025000000003,"total_on_demand_input_cost_bin":8.660060000000001,"ci_99_lower":0.7368366869080928,"ci_99_upper":0.9427542269591941,"ci_98_lower":0.7458108954969793,"ci_98_upper":0.9255095159255433,"ci_95_lower":0.7616584966314258,"ci_95_upper":0.9178488797383033,"ci_90_lower":0.7798024097637972,"ci_90_upper":0.9055560730892467,"ci_75_lower":0.7993444368985365,"ci_75_upper":0.8884360490279505,"ci_50_lower":0.8216997511626686,"ci_50_upper":0.8702971026446904},"32768":{"avg_score":0.4525138567906277,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.43383724811445484,"median_score":0.3131067603826523,"q1_score":0.027200979180634022,"q3_score":0.9927856624126434,"avg_token_efficiency":1.6528166626033116,"is_incomplete":false,"total_prompt_cost_bin":0.505995,"total_cache_read_cost_bin":3.1611774999999964,"total_completion_cost_bin":3.2320999999999995,"total_reasoning_cost_bin":0.3254250000000002,"total_cost_bin":7.224697499999996,"total_on_demand_input_cost_bin":32.11777000000004,"ci_99_lower":0.35207858944390275,"ci_99_upper":0.5426649445109819,"ci_98_lower":0.3652042092276978,"ci_98_upper":0.5372422405382703,"ci_95_lower":0.37911774238891666,"ci_95_upper":0.5254373056979602,"ci_90_lower":0.3909742910927461,"ci_90_upper":0.5144393667313766,"ci_75_lower":0.4112381090842117,"ci_75_upper":0.4966953306148913,"ci_50_lower":0.4264088375838422,"ci_50_upper":0.4784859628434788},"65536":{"avg_score":0.3102531395852566,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.41742247695024187,"median_score":0.03001745231449604,"q1_score":0,"q3_score":0.7403072714805603,"avg_token_efficiency":1.633311858261692,"is_incomplete":false,"total_prompt_cost_bin":0.5579499999999998,"total_cache_read_cost_bin":4.431369999999998,"total_completion_cost_bin":2.3228249999999995,"total_reasoning_cost_bin":0.20215000000000005,"total_cost_bin":7.514294999999997,"total_on_demand_input_cost_bin":44.87165000000002,"ci_99_lower":0.20292705596977953,"ci_99_upper":0.4347148219141698,"ci_98_lower":0.21284194274986273,"ci_98_upper":0.41938645473897784,"ci_95_lower":0.2272059807403917,"ci_95_upper":0.40494069061550203,"ci_90_lower":0.23772032602306675,"ci_90_upper":0.3834941030266311,"ci_75_lower":0.2566968557986376,"ci_75_upper":0.3612039165324805,"ci_50_lower":0.2772055051077212,"ci_50_upper":0.3410927868817988},"131072":{"avg_score":0.011630002147002706,"min_score":0,"max_score":0.2731451094150543,"n_tests":103,"stddev":0.04434881077717111,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6117058522678815,"is_incomplete":false,"total_prompt_cost_bin":1.063655,"total_cache_read_cost_bin":3.373936,"total_completion_cost_bin":0.2644499999999999,"total_reasoning_cost_bin":0.039699999999999985,"total_cost_bin":4.741741,"total_on_demand_input_cost_bin":34.803014999999995,"ci_99_lower":0.002368018099658408,"ci_99_upper":0.0260205562409122,"ci_98_lower":0.0030780469800698236,"ci_98_upper":0.02378576791320828,"ci_95_lower":0.004196896204878621,"ci_95_upper":0.020825381892579563,"ci_90_lower":0.005131751554792078,"ci_90_upper":0.018882425022718394,"ci_75_lower":0.006800582632422447,"ci_75_upper":0.016440074465879535,"ci_50_lower":0.008441592437820961,"ci_50_upper":0.014378717350004946},"262144":{"avg_score":0.019715365171036187,"min_score":0,"max_score":0.4763107895851135,"n_tests":141,"stddev":0.07346932843685985,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6299638830267162,"is_incomplete":false,"total_prompt_cost_bin":2.1514799999999994,"total_cache_read_cost_bin":12.806010000000011,"total_completion_cost_bin":0.7355500000000003,"total_reasoning_cost_bin":0.109075,"total_cost_bin":15.802115000000011,"total_on_demand_input_cost_bin":130.2115799999999,"ci_99_lower":0.006652342620758169,"ci_99_upper":0.03835757137703914,"ci_98_lower":0.008151413129893599,"ci_98_upper":0.0363858345757298,"ci_95_lower":0.009310838129166646,"ci_95_upper":0.03303710417285336,"ci_90_lower":0.010572442570404012,"ci_90_upper":0.030354175030861498,"ci_75_lower":0.01292867544734626,"ci_75_upper":0.02684021768397949,"ci_50_lower":0.0152047517204142,"ci_50_upper":0.02332487421651893},"524288":{"avg_score":0.0750863281551204,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.17180862445920064,"median_score":0.0171325895935297,"q1_score":0,"q3_score":0.06109481863677502,"avg_token_efficiency":1.6303102761845347,"is_incomplete":false,"total_prompt_cost_bin":4.338405000000007,"total_cache_read_cost_bin":99.11840399999963,"total_completion_cost_bin":5.094625000000002,"total_reasoning_cost_bin":0.5685000000000001,"total_cost_bin":109.11993399999965,"total_on_demand_input_cost_bin":995.5224449999995,"ci_99_lower":0.0501047289889144,"ci_99_upper":0.10573382066462142,"ci_98_lower":0.05118644505450147,"ci_98_upper":0.10364752647185292,"ci_95_lower":0.0547140707282455,"ci_95_upper":0.09749448135201927,"ci_90_lower":0.058106850609267166,"ci_90_upper":0.09429324170178931,"ci_75_lower":0.0626522222171504,"ci_75_upper":0.08801388265351692,"ci_50_lower":0.06726942584522846,"ci_50_upper":0.08183018157297599}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.7","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-16T14:51:40.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:59:17.669Z"},{"model_slug":"moonshotai/kimi-k2.5","reasoning_mode":null,"overall_metrics":{"auc_1m":0.07180237786675354,"auc_128k":0.43961859577563905,"cum_avg_1m":0.33487893644498284,"cum_avg_128k":0.5358062983119726,"avg_score_overall":0.5332767741537214,"avg_response_time_ms":12772.901221409316,"total_prompt_cost":0.9302878800000001,"total_cache_read_cost":2.1612454399999983,"total_completion_cost":0.52577655,"total_reasoning_cost":0,"total_prompt_cost_128k":0.9302878800000001,"total_completion_cost_128k":0.52577655,"total_on_demand_input_cost":13.897760519999995,"total_on_demand_input_cost_128k":13.897760519999995,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.994303650811002},"bin_metrics":{"8192":{"avg_score":0.7593451489927248,"min_score":0.00991735514253378,"max_score":1,"n_tests":80,"stddev":0.4169783105852801,"median_score":1,"q1_score":0.7876463159918785,"q3_score":1,"avg_token_efficiency":0.9901216565918286,"is_incomplete":false,"total_prompt_cost_bin":0.11784864000000002,"total_cache_read_cost_bin":0.030400000000000017,"total_completion_cost_bin":0.07889084999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.22713949000000003,"total_on_demand_input_cost_bin":0.30024864000000007,"ci_99_lower":0.6391437766197487,"ci_99_upper":0.8671040756924195,"ci_98_lower":0.6463213710855926,"ci_98_upper":0.8558670991621912,"ci_95_lower":0.6701935932668857,"ci_95_upper":0.8451825241037295,"ci_90_lower":0.683053483806434,"ci_90_upper":0.8310045231453842,"ci_75_lower":0.7084609045414254,"ci_75_upper":0.8095201451651519,"ci_50_lower":0.7258814406755846,"ci_50_upper":0.7903530535812024},"16384":{"avg_score":0.6248520862311124,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.4745266722881286,"median_score":1,"q1_score":0.0349748395383358,"q3_score":1,"avg_token_efficiency":0.9804544918987936,"is_incomplete":false,"total_prompt_cost_bin":0.24755328000000004,"total_cache_read_cost_bin":0.061383680000000024,"total_completion_cost_bin":0.07769100000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.38662796000000005,"total_on_demand_input_cost_bin":0.6158553600000007,"ci_99_lower":0.4788513098198455,"ci_99_upper":0.7591591664590525,"ci_98_lower":0.501848781903158,"ci_98_upper":0.7351739876435603,"ci_95_lower":0.5158871811503195,"ci_95_upper":0.7220238534771488,"ci_90_lower":0.5397495339502348,"ci_90_upper":0.7090053560858359,"ci_75_lower":0.5642276018028496,"ci_75_upper":0.6856074185052422,"ci_50_lower":0.5886778016138123,"ci_50_upper":0.6606792011880316},"32768":{"avg_score":0.620173847371274,"min_score":0.003955174703150988,"max_score":1,"n_tests":136,"stddev":0.46982294172490136,"median_score":1,"q1_score":0.0404161661863327,"q3_score":1,"avg_token_efficiency":1.0101715848916968,"is_incomplete":false,"total_prompt_cost_bin":0.22700135999999999,"total_cache_read_cost_bin":0.33432704000000013,"total_completion_cost_bin":0.15405389999999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.7153823,"total_on_demand_input_cost_bin":2.232963599999998,"ci_99_lower":0.5193957688837819,"ci_99_upper":0.7239015340056913,"ci_98_lower":0.5268519110836581,"ci_98_upper":0.712129576168338,"ci_95_lower":0.5401066172328738,"ci_95_upper":0.6993162229523638,"ci_90_lower":0.5555456054140129,"ci_90_upper":0.689374928936869,"ci_75_lower":0.5728217678834505,"ci_75_upper":0.6674488161361831,"ci_50_lower":0.5936692177698097,"ci_50_upper":0.6461476177851985},"65536":{"avg_score":0.3590825274093624,"min_score":0.0026773761492222548,"max_score":1,"n_tests":85,"stddev":0.46426629596861,"median_score":0.03652210161089897,"q1_score":0.019221041351556778,"q3_score":1,"avg_token_efficiency":0.9907670367000743,"is_incomplete":false,"total_prompt_cost_bin":0.33417219,"total_cache_read_cost_bin":0.4676249599999997,"total_completion_cost_bin":0.09546075000000004,"total_reasoning_cost_bin":0,"total_cost_bin":0.8972578999999996,"total_on_demand_input_cost_bin":3.1399219500000006,"ci_99_lower":0.24091509106189615,"ci_99_upper":0.4865337860815857,"ci_98_lower":0.2459176260950044,"ci_98_upper":0.4763293350093231,"ci_95_lower":0.265721242993399,"ci_95_upper":0.4616361337309868,"ci_90_lower":0.2782010291245602,"ci_90_upper":0.44426017609616625,"ci_75_lower":0.29961550916401286,"ci_75_upper":0.41681877261418027,"ci_50_lower":0.32454717848312986,"ci_50_upper":0.3928200776449076},"131072":{"avg_score":0.3155778815553894,"min_score":0.0047505940310657024,"max_score":1,"n_tests":103,"stddev":0.44370528280477384,"median_score":0.03501199185848236,"q1_score":0.021484007127583027,"q3_score":1,"avg_token_efficiency":0.990275159694156,"is_incomplete":false,"total_prompt_cost_bin":0.0037124100000000045,"total_cache_read_cost_bin":1.2675097599999987,"total_completion_cost_bin":0.11968004999999997,"total_reasoning_cost_bin":0,"total_cost_bin":1.3909022199999987,"total_on_demand_input_cost_bin":7.608770969999995,"ci_99_lower":0.2118472852100148,"ci_99_upper":0.42723228640822714,"ci_98_lower":0.2228127636883444,"ci_98_upper":0.41803519708276227,"ci_95_lower":0.23455050091713542,"ci_95_upper":0.40072146880517273,"ci_90_lower":0.24417746698899756,"ci_90_upper":0.3900687945986311,"ci_75_lower":0.2654270168187015,"ci_75_upper":0.3646847816699555,"ci_50_lower":0.2881009940259177,"ci_50_upper":0.3431617926932799}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"MoonshotAI: Kimi K2.5","provider_name":"Moonshotai","run_provider_names":["Moonshot AI"],"pricing_metadata":{"prompt":6e-7,"completion":0.000003,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-01-27T04:11:16.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T13:21:10.574Z"},{"model_slug":"upstage/solar-pro4","reasoning_mode":null,"overall_metrics":{"auc_1m":0.06504266404279103,"auc_128k":0.2433407031920267,"cum_avg_1m":0.19693523075067118,"cum_avg_128k":0.28868201878876343,"avg_score_overall":0.24723423484191298,"avg_response_time_ms":13301.523294726563,"total_prompt_cost":0.5907441,"total_cache_read_cost":0.24434956799999996,"total_completion_cost":0.029755199999999996,"total_reasoning_cost":0,"total_prompt_cost_128k":0.29485185000000014,"total_completion_cost_128k":0.022615679999999996,"total_on_demand_input_cost":1.812491939999999,"total_on_demand_input_cost_128k":0.7231865699999993,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9839357509757091},"bin_metrics":{"8192":{"avg_score":0.4479191750229802,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4767756663091645,"median_score":0.08098097890615463,"q1_score":0.030593177769333124,"q3_score":1,"avg_token_efficiency":0.9794939429145753,"is_incomplete":false,"total_prompt_cost_bin":0.012781680000000002,"total_cache_read_cost_bin":0.0005679360000000001,"total_completion_cost_bin":0.00330408,"total_reasoning_cost_bin":0,"total_cost_bin":0.016653696000000003,"total_on_demand_input_cost_bin":0.01562136,"ci_99_lower":0.3052075043695222,"ci_99_upper":0.5769658534209885,"ci_98_lower":0.3151262417830876,"ci_98_upper":0.5670637298976071,"ci_95_lower":0.33888806552218737,"ci_95_upper":0.5553557837889821,"ci_90_lower":0.3531111340798089,"ci_90_upper":0.535650695178483,"ci_75_lower":0.3861136716834153,"ci_75_upper":0.5092455806021463,"ci_50_lower":0.41164488597132726,"ci_50_upper":0.4837216189713217},"16384":{"avg_score":0.30632292400114236,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4420794312718093,"median_score":0.035387612879276276,"q1_score":0.02322081197053194,"q3_score":0.9981440156698227,"avg_token_efficiency":0.9805230789866091,"is_incomplete":false,"total_prompt_cost_bin":0.021611280000000004,"total_cache_read_cost_bin":0.0021607680000000004,"total_completion_cost_bin":0.003326399999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.027098448000000004,"total_on_demand_input_cost_bin":0.03241512,"ci_99_lower":0.19518271022761471,"ci_99_upper":0.4388686840358423,"ci_98_lower":0.19704358143126594,"ci_98_upper":0.42877222482219796,"ci_95_lower":0.21950600303302054,"ci_95_upper":0.40262399306520824,"ci_90_lower":0.2242223674093839,"ci_90_upper":0.3805454266263405,"ci_75_lower":0.2468335456796922,"ci_75_upper":0.35793491242802705,"ci_50_lower":0.27157023462059443,"ci_50_upper":0.33549362323537935},"32768":{"avg_score":0.26939832037725653,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.41446240780908383,"median_score":0.03708367608487606,"q1_score":0.02182007348164916,"q3_score":0.2035675197839737,"avg_token_efficiency":0.9957504768585885,"is_incomplete":false,"total_prompt_cost_bin":0.0535176,"total_cache_read_cost_bin":0.012455424000000007,"total_completion_cost_bin":0.006275159999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.07224818400000001,"total_on_demand_input_cost_bin":0.11579471999999974,"ci_99_lower":0.1824849902202293,"ci_99_upper":0.35150012814638426,"ci_98_lower":0.1901999988382244,"ci_98_upper":0.3477994518566822,"ci_95_lower":0.20075849301065318,"ci_95_upper":0.3358970572658376,"ci_90_lower":0.212075943560408,"ci_90_upper":0.32608648983912325,"ci_75_lower":0.2276136189671781,"ci_75_upper":0.3080113496047595,"ci_50_lower":0.24588181673179327,"ci_50_upper":0.2917271687388968},"65536":{"avg_score":0.2396964575328371,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.39897345947963875,"median_score":0.029336562380194664,"q1_score":0.02004251442849636,"q3_score":0.06604000926017761,"avg_token_efficiency":0.979340176315329,"is_incomplete":false,"total_prompt_cost_bin":0.06680394,"total_cache_read_cost_bin":0.019309823999999996,"total_completion_cost_bin":0.004209599999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.090323364,"total_on_demand_input_cost_bin":0.16335306000000005,"ci_99_lower":0.1416889141596963,"ci_99_upper":0.35351840003914037,"ci_98_lower":0.15049179557623232,"ci_98_upper":0.3389054727354909,"ci_95_lower":0.16421656261828232,"ci_95_upper":0.3248682061306146,"ci_90_lower":0.17359200709554204,"ci_90_upper":0.31142093732898285,"ci_75_lower":0.18854166996281813,"ci_75_upper":0.2856945427422247,"ci_50_lower":0.2071258213754524,"ci_50_upper":0.265195733947022},"131072":{"avg_score":0.18007321700960108,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.3541213616236008,"median_score":0.029099488630890846,"q1_score":0.01795289758592844,"q3_score":0.044478461146354675,"avg_token_efficiency":0.9792480013092701,"is_incomplete":false,"total_prompt_cost_bin":0.1401373500000001,"total_cache_read_cost_bin":0.051172992000000035,"total_completion_cost_bin":0.005500439999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.19681078200000013,"total_on_demand_input_cost_bin":0.39600230999999947,"ci_99_lower":0.09419164611117065,"ci_99_upper":0.27330572083601457,"ci_98_lower":0.10534159269292379,"ci_98_upper":0.2647386664574405,"ci_95_lower":0.12034081601553588,"ci_95_upper":0.24808419557193348,"ci_90_lower":0.1255065668259015,"ci_90_upper":0.2379910924885585,"ci_75_lower":0.1413048551688016,"ci_75_upper":0.2183002980173909,"ci_50_lower":0.15569484860238447,"ci_50_upper":0.19903121414273361},"262144":{"avg_score":0.1320717520615522,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.29984758154190083,"median_score":0.029411764815449715,"q1_score":0.017125383019447327,"q3_score":0.043513957411050797,"avg_token_efficiency":0.983191195980834,"is_incomplete":false,"total_prompt_cost_bin":0.2958922499999999,"total_cache_read_cost_bin":0.1586826239999999,"total_completion_cost_bin":0.007139520000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.46171439399999986,"total_on_demand_input_cost_bin":1.0893053699999997,"ci_99_lower":0.0754674254087335,"ci_99_upper":0.20038736313607888,"ci_98_lower":0.07858858001794904,"ci_98_upper":0.19377543210243503,"ci_95_lower":0.08423268443967566,"ci_95_upper":0.18246898437711467,"ci_90_lower":0.09175272459042068,"ci_90_upper":0.1749725048976838,"ci_75_lower":0.1041306066207079,"ci_75_upper":0.16047849818231893,"ci_50_lower":0.11504641519562202,"ci_50_upper":0.14721089831457623}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":524288,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Upstage: Solar Pro 4","provider_name":"Upstage","run_provider_names":["Upstage"],"pricing_metadata":{"prompt":3e-8,"completion":1.2e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-10T14:20:36.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T01:50:53.389Z"},{"model_slug":"x-ai/grok-4.3","reasoning_mode":null,"overall_metrics":{"auc_1m":0.06462018312336516,"auc_128k":0.14295434238864288,"cum_avg_1m":0.12795572313328174,"cum_avg_128k":0.17689186107002466,"avg_score_overall":0.1230888646735603,"avg_response_time_ms":9254.615629575677,"total_prompt_cost":107.16856874999996,"total_cache_read_cost":47.850777600000015,"total_completion_cost":0.5706175000000001,"total_reasoning_cost":0,"total_prompt_cost_128k":10.343951249999998,"total_completion_cost_128k":0.331805,"total_on_demand_input_cost":406.9451362499998,"total_on_demand_input_cost_128k":26.290911250000004,"total_runs":779,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9513658864838969},"bin_metrics":{"8192":{"avg_score":0.25323753754575073,"min_score":0,"max_score":1,"n_tests":70,"stddev":0.39650318076863006,"median_score":0.03684973530471325,"q1_score":0.026762468740344048,"q3_score":0.15151870623230934,"avg_token_efficiency":0.9691037108256302,"is_incomplete":false,"total_prompt_cost_bin":0.42503749999999996,"total_cache_read_cost_bin":0.023270400000000007,"total_completion_cost_bin":0.05522250000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.5035304,"total_on_demand_input_cost_bin":0.5704775000000001,"ci_99_lower":0.13738763013043873,"ci_99_upper":0.36989970719695386,"ci_98_lower":0.14572885692858006,"ci_98_upper":0.3617717783771056,"ci_95_lower":0.16752304142041666,"ci_95_upper":0.3469623436103159,"ci_90_lower":0.17964698004561278,"ci_90_upper":0.32506811299818633,"ci_75_lower":0.20071038865093474,"ci_75_upper":0.30333855385459696,"ci_50_lower":0.22196400405913924,"ci_50_upper":0.2833118233393179},"16384":{"avg_score":0.24350271688890643,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.39952646170843464,"median_score":0.029740067198872566,"q1_score":0.020684629678726196,"q3_score":0.11730187758803368,"avg_token_efficiency":0.9479672642439831,"is_incomplete":false,"total_prompt_cost_bin":0.7316800000000001,"total_cache_read_cost_bin":0.09166080000000004,"total_completion_cost_bin":0.06216249999999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.8855033000000002,"total_on_demand_input_cost_bin":1.3045599999999993,"ci_99_lower":0.1389127904693014,"ci_99_upper":0.3565094953238149,"ci_98_lower":0.14920340463705362,"ci_98_upper":0.3435732640612987,"ci_95_lower":0.16149727133641137,"ci_95_upper":0.3267500952030241,"ci_90_lower":0.1766803311550757,"ci_90_upper":0.31280879448866467,"ci_75_lower":0.19269675585237564,"ci_75_upper":0.2906779238423042,"ci_50_lower":0.21077548401663082,"ci_50_upper":0.2704557316625142},"32768":{"avg_score":0.1434303551868652,"min_score":0,"max_score":1,"n_tests":64,"stddev":0.304663983566123,"median_score":0.03355016000568867,"q1_score":0.020035077817738056,"q3_score":0.05314568243920803,"avg_token_efficiency":0.9595176888646235,"is_incomplete":true,"total_prompt_cost_bin":0.6965900000000002,"total_cache_read_cost_bin":0.18355199999999994,"total_completion_cost_bin":0.04877000000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.9289120000000001,"total_on_demand_input_cost_bin":1.843790000000001,"ci_99_lower":0.06139560990017344,"ci_99_upper":0.2619868593084539,"ci_98_lower":0.06855030960286967,"ci_98_upper":0.24547896941250655,"ci_95_lower":0.07848877961205289,"ci_95_upper":0.2244960859487036,"ci_90_lower":0.08428244072238157,"ci_90_upper":0.2048097894705279,"ci_75_lower":0.1001496733933891,"ci_75_upper":0.18679515019448445,"ci_50_lower":0.11609088002114731,"ci_50_upper":0.16484718596439052},"65536":{"avg_score":0.12249822162420435,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.27825939825486057,"median_score":0.027306579053401947,"q1_score":0.018171703442931175,"q3_score":0.04599696397781372,"avg_token_efficiency":0.9498182545370497,"is_incomplete":false,"total_prompt_cost_bin":2.485761249999999,"total_cache_read_cost_bin":0.6584703999999995,"total_completion_cost_bin":0.073575,"total_reasoning_cost_bin":0,"total_cost_bin":3.217806649999998,"total_on_demand_input_cost_bin":6.60120125,"ci_99_lower":0.05555949190075454,"ci_99_upper":0.2070129164622494,"ci_98_lower":0.06019061703815618,"ci_98_upper":0.19895875750591652,"ci_95_lower":0.06898536095491557,"ci_95_upper":0.19049390147316872,"ci_90_lower":0.07569232359528541,"ci_90_upper":0.17526058913756376,"ci_75_lower":0.08859108136213549,"ci_75_upper":0.15601161009604658,"ci_50_lower":0.10072666427586227,"ci_50_upper":0.14060861216858028},"131072":{"avg_score":0.12179047410439664,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.2878353052251353,"median_score":0.025848960503935814,"q1_score":0.016968092881143093,"q3_score":0.03547879680991173,"avg_token_efficiency":0.9478402143991189,"is_incomplete":false,"total_prompt_cost_bin":6.004882499999999,"total_cache_read_cost_bin":1.5945599999999989,"total_completion_cost_bin":0.09207500000000002,"total_reasoning_cost_bin":0,"total_cost_bin":7.691517499999998,"total_on_demand_input_cost_bin":15.970882500000004,"ci_99_lower":0.056432847550401045,"ci_99_upper":0.20470087229814468,"ci_98_lower":0.06419420100215062,"ci_98_upper":0.19699169331365854,"ci_95_lower":0.07298369876479784,"ci_95_upper":0.17855295313926847,"ci_90_lower":0.08119870266058817,"ci_90_upper":0.1670891836335322,"ci_75_lower":0.09207481013902563,"ci_75_upper":0.1516815772533272,"ci_50_lower":0.10209368356137347,"ci_50_upper":0.13859515824441085},"262144":{"avg_score":0.06636777909882102,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.17773333664663926,"median_score":0.02320185676217079,"q1_score":0.009315657429397106,"q3_score":0.0430164635181427,"avg_token_efficiency":0.9522821837448265,"is_incomplete":false,"total_prompt_cost_bin":23.808397499999987,"total_cache_read_cost_bin":10.238412799999994,"total_completion_cost_bin":0.08522250000000003,"total_reasoning_cost_bin":0,"total_cost_bin":34.13203279999998,"total_on_demand_input_cost_bin":87.92170000000002,"ci_99_lower":0.03464464764850519,"ci_99_upper":0.10949838383557566,"ci_98_lower":0.03679853984851602,"ci_98_upper":0.10646929154899103,"ci_95_lower":0.040741467533355696,"ci_95_upper":0.09906491935187092,"ci_90_lower":0.04409974628484788,"ci_90_upper":0.09247801629017298,"ci_75_lower":0.04989711404323961,"ci_75_upper":0.08363084034566562,"ci_50_lower":0.05560868758054331,"ci_50_upper":0.07592209027370056},"524288":{"avg_score":0.0728187006173095,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.19090239424308744,"median_score":0.022712324745953083,"q1_score":0.010418855119496584,"q3_score":0.037672633305191994,"avg_token_efficiency":0.9465947922013899,"is_incomplete":false,"total_prompt_cost_bin":73.01621999999998,"total_cache_read_cost_bin":35.06085120000002,"total_completion_cost_bin":0.15359000000000003,"total_reasoning_cost_bin":0,"total_cost_bin":108.2306612,"total_on_demand_input_cost_bin":292.7325249999998,"ci_99_lower":0.0458279227768219,"ci_99_upper":0.10952287342982915,"ci_98_lower":0.047351201112831996,"ci_98_upper":0.10499719841202021,"ci_95_lower":0.05051645440298825,"ci_95_upper":0.09758954925962794,"ci_90_lower":0.05395565007120703,"ci_90_upper":0.0936974497223411,"ci_75_lower":0.058977581288194934,"ci_75_upper":0.08732454683799375,"ci_50_lower":0.06403000432057247,"ci_50_upper":0.08030266334279046}},"has_incomplete_bins":true,"has_incomplete_bins_128k":true,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low"],"reasoning_default":"low","sponsors":[],"openrouter_name":"SpaceXAI: Grok 4.3","provider_name":"X-ai","run_provider_names":["xAI"],"pricing_metadata":{"prompt":0.00000125,"completion":0.0000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-30T23:30:21.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-05-15T23:00:34.023Z"},{"model_slug":"openai/gpt-5.4-nano","reasoning_mode":"low","overall_metrics":{"auc_1m":0.06443726242684963,"auc_128k":0.18986002326919943,"cum_avg_1m":0.1755355007749639,"cum_avg_128k":0.24704347580401512,"avg_score_overall":0.21867516929805278,"avg_response_time_ms":9152.438450585938,"total_prompt_cost":0.5804007999999996,"total_cache_read_cost":1.1486720000000006,"total_completion_cost":0.48501249999999996,"total_reasoning_cost":0.18129000000000003,"total_prompt_cost_128k":0.25679860000000004,"total_completion_cost_128k":0.38050125,"total_on_demand_input_cost":12.067120799999998,"total_on_demand_input_cost_128k":4.810782599999998,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061808102971},"bin_metrics":{"8192":{"avg_score":0.45803550349664873,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.481169826243013,"median_score":0.08320523053407669,"q1_score":0.027089818380773067,"q3_score":1,"avg_token_efficiency":0.9759044449598389,"is_incomplete":false,"total_prompt_cost_bin":0.02094559999999999,"total_cache_read_cost_bin":0.008289280000000007,"total_completion_cost_bin":0.05625625000000001,"total_reasoning_cost_bin":0.0210175,"total_cost_bin":0.10650862999999999,"total_on_demand_input_cost_bin":0.10383840000000004,"ci_99_lower":0.3227606734526053,"ci_99_upper":0.5883784311614,"ci_98_lower":0.3340529821588425,"ci_98_upper":0.5760675957867643,"ci_95_lower":0.3462361029132444,"ci_95_upper":0.5643020955534302,"ci_90_lower":0.3620366763984202,"ci_90_upper":0.5481644378972121,"ci_75_lower":0.3967133233534696,"ci_75_upper":0.5204984630712715,"ci_50_lower":0.4212744858421502,"ci_50_upper":0.4950266193263815},"16384":{"avg_score":0.2707385842455551,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4213746542667978,"median_score":0.03329555504024029,"q1_score":0.02325875498354435,"q3_score":0.28523835353553295,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.020779200000000008,"total_cache_read_cost_bin":0.019235840000000004,"total_completion_cost_bin":0.056847499999999995,"total_reasoning_cost_bin":0.021825,"total_cost_bin":0.11868754000000001,"total_on_demand_input_cost_bin":0.21313760000000004,"ci_99_lower":0.16175727865367662,"ci_99_upper":0.38989377975813116,"ci_98_lower":0.16360537442879286,"ci_98_upper":0.3798965898309834,"ci_95_lower":0.1837241614621598,"ci_95_upper":0.36537981897010463,"ci_90_lower":0.19723705972370226,"ci_90_upper":0.3449317292461637,"ci_75_lower":0.22097180888813456,"ci_75_upper":0.319674629655492,"ci_50_lower":0.2417367821588414,"ci_50_upper":0.29596857807773636},"32768":{"avg_score":0.16879226501024439,"min_score":0.003516998840495944,"max_score":1,"n_tests":136,"stddev":0.3337510735439245,"median_score":0.030774922110140324,"q1_score":0.02257610810920596,"q3_score":0.05241545382887125,"avg_token_efficiency":0.9952100768549632,"is_incomplete":false,"total_prompt_cost_bin":0.04694080000000008,"total_cache_read_cost_bin":0.07250432000000001,"total_completion_cost_bin":0.11425374999999999,"total_reasoning_cost_bin":0.047635000000000025,"total_cost_bin":0.2813338700000001,"total_on_demand_input_cost_bin":0.7719840000000007,"ci_99_lower":0.10252956045998657,"ci_99_upper":0.24826206845995757,"ci_98_lower":0.10452388695487409,"ci_98_upper":0.24241516360049203,"ci_95_lower":0.1151210071031956,"ci_95_upper":0.22786447390437173,"ci_90_lower":0.12313877813028419,"ci_90_upper":0.21701223137865516,"ci_75_lower":0.1346218289284426,"ci_75_upper":0.20129721946196855,"ci_50_lower":0.148484839590845,"ci_50_upper":0.18411815609279608},"65536":{"avg_score":0.1778969104100457,"min_score":0.002610966097563505,"max_score":1,"n_tests":85,"stddev":0.34925280381390494,"median_score":0.031481482088565826,"q1_score":0.02259279228746891,"q3_score":0.04444444552063942,"avg_token_efficiency":0.9776435846759913,"is_incomplete":false,"total_prompt_cost_bin":0.06277540000000004,"total_cache_read_cost_bin":0.10243583999999988,"total_completion_cost_bin":0.06953000000000004,"total_reasoning_cost_bin":0.026026250000000008,"total_cost_bin":0.26076749,"total_on_demand_input_cost_bin":1.0871337999999988,"ci_99_lower":0.08740514780487865,"ci_99_upper":0.28259160094143937,"ci_98_lower":0.09817607349297032,"ci_98_upper":0.2695010123215342,"ci_95_lower":0.10922620104002657,"ci_95_upper":0.257804980071823,"ci_90_lower":0.12002004949444467,"ci_90_upper":0.24495648186037536,"ci_75_lower":0.1331518873315551,"ci_75_upper":0.22293069408384755,"ci_50_lower":0.15266826921895912,"ci_50_upper":0.20018857853172126},"131072":{"avg_score":0.15975411585758179,"min_score":0.004308487754315138,"max_score":1,"n_tests":103,"stddev":0.3335950560366394,"median_score":0.028650647029280663,"q1_score":0.019202586263418198,"q3_score":0.04551844485104084,"avg_token_efficiency":0.9772722316188367,"is_incomplete":false,"total_prompt_cost_bin":0.10535759999999994,"total_cache_read_cost_bin":0.25293312000000034,"total_completion_cost_bin":0.08361374999999997,"total_reasoning_cost_bin":0.028979999999999992,"total_cost_bin":0.4708844700000003,"total_on_demand_input_cost_bin":2.6346887999999984,"ci_99_lower":0.07510161277302314,"ci_99_upper":0.24539204612896479,"ci_98_lower":0.08639301927394044,"ci_98_upper":0.24454610575623448,"ci_95_lower":0.10266146160791023,"ci_95_upper":0.2251316023724904,"ci_90_lower":0.11131929552399562,"ci_90_upper":0.21528573800511294,"ci_75_lower":0.12227246330913101,"ci_75_upper":0.1956432772573147,"ci_50_lower":0.13647200857307218,"ci_50_upper":0.1785143228864876},"262144":{"avg_score":0.16906662717963575,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.32122999234922656,"median_score":0.03288121521472931,"q1_score":0.022659113630652428,"q3_score":0.06220095604658127,"avg_token_efficiency":0.982419810338046,"is_incomplete":false,"total_prompt_cost_bin":0.3236021999999996,"total_cache_read_cost_bin":0.6932736000000004,"total_completion_cost_bin":0.10451124999999994,"total_reasoning_cost_bin":0.035806250000000005,"total_cost_bin":1.1571933,"total_on_demand_input_cost_bin":7.256338199999999,"ci_99_lower":0.10561105756614199,"ci_99_upper":0.23469842564833107,"ci_98_lower":0.10843429852561061,"ci_98_upper":0.23223525439461704,"ci_95_lower":0.11752156268009349,"ci_95_upper":0.22198586989531013,"ci_90_lower":0.1266095083206892,"ci_90_upper":0.21426346886586004,"ci_75_lower":0.13703922757462778,"ci_75_upper":0.2009852891003528,"ci_50_lower":0.15014879081186885,"ci_50_upper":0.18446520824771348}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Nano","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":2e-7,"completion":0.00000125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:47.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T01:59:22.774Z"},{"model_slug":"openai/gpt-5.4-mini","reasoning_mode":null,"overall_metrics":{"auc_1m":0.05792321508229907,"auc_128k":0.19434414389029725,"cum_avg_1m":0.162204619335327,"cum_avg_128k":0.2351112597458318,"avg_score_overall":0.2082961377121508,"avg_response_time_ms":8939.531087597656,"total_prompt_cost":2.8767270000000003,"total_cache_read_cost":4.237497599999996,"total_completion_cost":1.0492695000000003,"total_reasoning_cost":0,"total_prompt_cost_128k":1.0739707499999995,"total_completion_cost_128k":0.8105580000000001,"total_on_demand_input_cost":45.251702999999964,"total_on_demand_input_cost_128k":18.040434749999978,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061808102971},"bin_metrics":{"8192":{"avg_score":0.33021262553520503,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.44200495667779094,"median_score":0.04379182308912277,"q1_score":0.02525523165240884,"q3_score":1,"avg_token_efficiency":0.9759044449598392,"is_incomplete":false,"total_prompt_cost_bin":0.07643400000000003,"total_cache_read_cost_bin":0.031296,"total_completion_cost_bin":0.11824650000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.22597650000000008,"total_on_demand_input_cost_bin":0.3893939999999997,"ci_99_lower":0.21101825450680917,"ci_99_upper":0.46570344540924996,"ci_98_lower":0.22336592275183648,"ci_98_upper":0.4535306839208351,"ci_95_lower":0.23510583456634773,"ci_95_upper":0.4257779287765151,"ci_90_lower":0.24992806012509391,"ci_90_upper":0.4068576318235136,"ci_75_lower":0.2709518431540346,"ci_75_upper":0.38575106453354235,"ci_50_lower":0.29856722031254324,"ci_50_upper":0.3619354155234759},"16384":{"avg_score":0.26688727786531674,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4191806636741307,"median_score":0.03313360922038555,"q1_score":0.019878135062754154,"q3_score":0.26637212559580803,"avg_token_efficiency":0.9670687393188716,"is_incomplete":false,"total_prompt_cost_bin":0.11229000000000006,"total_cache_read_cost_bin":0.06869760000000008,"total_completion_cost_bin":0.11851650000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.29950410000000016,"total_on_demand_input_cost_bin":0.7992660000000001,"ci_99_lower":0.15862242209154645,"ci_99_upper":0.3812436653443146,"ci_98_lower":0.1612908372146776,"ci_98_upper":0.37528569054661787,"ci_95_lower":0.17971325885504486,"ci_95_upper":0.3583382798885577,"ci_90_lower":0.19425743737258017,"ci_90_upper":0.34139534002868455,"ci_75_lower":0.2159235365543282,"ci_75_upper":0.3159092971473001,"ci_50_lower":0.23646999160118864,"ci_50_upper":0.29335692098829896},"32768":{"avg_score":0.24774194654310122,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4024664003707946,"median_score":0.03569009155035019,"q1_score":0.019689257722347975,"q3_score":0.0906244870275259,"avg_token_efficiency":0.9952100768549632,"is_incomplete":false,"total_prompt_cost_bin":0.23516399999999996,"total_cache_read_cost_bin":0.26597760000000054,"total_completion_cost_bin":0.2271374999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.7282791000000004,"total_on_demand_input_cost_bin":2.8949399999999965,"ci_99_lower":0.16306749234818305,"ci_99_upper":0.3269055597393376,"ci_98_lower":0.16709447132563904,"ci_98_upper":0.32425604276115977,"ci_95_lower":0.18007703143709533,"ci_95_upper":0.3160846259031506,"ci_90_lower":0.18978235674970648,"ci_90_upper":0.3034163960579248,"ci_75_lower":0.20633252232544372,"ci_75_upper":0.2879768140072188,"ci_50_lower":0.2243219170743353,"ci_50_upper":0.2692841551247779},"65536":{"avg_score":0.14182064700871705,"min_score":0.004449388012290001,"max_score":1,"n_tests":85,"stddev":0.3153130224220719,"median_score":0.025073746219277382,"q1_score":0.018142549321055412,"q3_score":0.04226870834827423,"avg_token_efficiency":0.9776435846759911,"is_incomplete":false,"total_prompt_cost_bin":0.15918374999999996,"total_cache_read_cost_bin":0.39175679999999935,"total_completion_cost_bin":0.1589490000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.7098895499999994,"total_on_demand_input_cost_bin":4.076751749999995,"ci_99_lower":0.06268584874954403,"ci_99_upper":0.24542168998562128,"ci_98_lower":0.07204031070239622,"ci_98_upper":0.23344068035494314,"ci_95_lower":0.08212204201503054,"ci_95_upper":0.21115774116629515,"ci_90_lower":0.08693322657689673,"ci_90_upper":0.20004571899525164,"ci_75_lower":0.10478900583145921,"ci_75_upper":0.17713247403666812,"ci_50_lower":0.11803341297816267,"ci_50_upper":0.16363171708277044},"131072":{"avg_score":0.18889380177681886,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.36207562416278827,"median_score":0.030736619606614113,"q1_score":0.020572214387357235,"q3_score":0.04333352483808994,"avg_token_efficiency":0.9772722316188371,"is_incomplete":false,"total_prompt_cost_bin":0.49089899999999964,"total_cache_read_cost_bin":0.9389183999999975,"total_completion_cost_bin":0.18770850000000006,"total_reasoning_cost_bin":0,"total_cost_bin":1.617525899999997,"total_on_demand_input_cost_bin":9.880082999999987,"ci_99_lower":0.10291731722589618,"ci_99_upper":0.2829132529526262,"ci_98_lower":0.11265105098006722,"ci_98_upper":0.2745174251536576,"ci_95_lower":0.12395160387686083,"ci_95_upper":0.2615428739281606,"ci_90_lower":0.13344576914604864,"ci_90_upper":0.2464842569720434,"ci_75_lower":0.1502999515618532,"ci_75_upper":0.22672149564424368,"ci_50_lower":0.1615808850100029,"ci_50_upper":0.2083017114048046},"262144":{"avg_score":0.12208065595345717,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.2807949522380539,"median_score":0.030712980777025223,"q1_score":0.02148168534040451,"q3_score":0.04755944758653641,"avg_token_efficiency":0.9824198103380458,"is_incomplete":false,"total_prompt_cost_bin":1.8027562500000007,"total_cache_read_cost_bin":2.5408511999999988,"total_completion_cost_bin":0.23871150000000008,"total_reasoning_cost_bin":0,"total_cost_bin":4.5823189499999994,"total_on_demand_input_cost_bin":27.211268249999986,"ci_99_lower":0.06768357980913807,"ci_99_upper":0.18733209181354044,"ci_98_lower":0.07446566830316545,"ci_98_upper":0.17921798042635964,"ci_95_lower":0.0792394857139472,"ci_95_upper":0.17024986112361498,"ci_90_lower":0.08554011260948627,"ci_90_upper":0.16209838939450863,"ci_75_lower":0.09507460300794122,"ci_75_upper":0.14944141323542792,"ci_50_lower":0.1064253746868765,"ci_50_upper":0.13647355876387435}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Mini","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":7.5e-7,"completion":0.0000045,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-19T22:52:52.064Z"},{"model_slug":"qwen/qwen3.5-27b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.05494130340370229,"auc_128k":0.32979187350375444,"cum_avg_1m":0.23718928675821435,"cum_avg_128k":0.37950285881314294,"avg_score_overall":0.3681642567036024,"avg_response_time_ms":38067.22980105852,"total_prompt_cost":4.748577405000001,"total_cache_read_cost":0,"total_completion_cost":0.294918,"total_reasoning_cost":0,"total_prompt_cost_128k":4.748577405000001,"total_completion_cost_128k":0.294918,"total_on_demand_input_cost":4.748577405000001,"total_on_demand_input_cost_128k":4.748577405000001,"total_runs":479,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0208607275060022},"bin_metrics":{"8192":{"avg_score":0.5902802430035081,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4789990716548674,"median_score":0.9976193010807037,"q1_score":0.034163713455200195,"q3_score":1,"avg_token_efficiency":1.0228079512716752,"is_incomplete":false,"total_prompt_cost_bin":0.1059926399999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.04533048000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.15132311999999992,"total_on_demand_input_cost_bin":0.1059926399999999,"ci_99_lower":0.4442614232923079,"ci_99_upper":0.7345948074927728,"ci_98_lower":0.45978450060600884,"ci_98_upper":0.7112586979996995,"ci_95_lower":0.4900589158049843,"ci_95_upper":0.6972972686339927,"ci_90_lower":0.5054521522248979,"ci_90_upper":0.6744503448606701,"ci_75_lower":0.530816292900272,"ci_75_upper":0.6518709158364799,"ci_50_lower":0.5541830344925984,"ci_50_upper":0.6252716414877796},"16384":{"avg_score":0.37931853186455555,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.46923697130513264,"median_score":0.037299370393157005,"q1_score":0.024587559048086405,"q3_score":0.9976304322481155,"avg_token_efficiency":1.0161194758384637,"is_incomplete":false,"total_prompt_cost_bin":0.21856848000000012,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.044091839999999986,"total_reasoning_cost_bin":0,"total_cost_bin":0.2626603200000001,"total_on_demand_input_cost_bin":0.21856848000000012,"ci_99_lower":0.2459699432056805,"ci_99_upper":0.5125422553906391,"ci_98_lower":0.25635986974881964,"ci_98_upper":0.5004205511115143,"ci_95_lower":0.27119513928730155,"ci_95_upper":0.4879983834241284,"ci_90_lower":0.2926080432115123,"ci_90_upper":0.4645172417155118,"ci_75_lower":0.3193238550746173,"ci_75_upper":0.4402719200428692,"ci_50_lower":0.3429104413022287,"ci_50_upper":0.4151804522611201},"32768":{"avg_score":0.32568869149990326,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.44078613080943685,"median_score":0.04124149680137634,"q1_score":0.024819147307425737,"q3_score":0.9864146560430527,"avg_token_efficiency":1.025754859257966,"is_incomplete":false,"total_prompt_cost_bin":0.7754916000000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08590140000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.8613930000000002,"total_on_demand_input_cost_bin":0.7754916000000002,"ci_99_lower":0.23667427130469793,"ci_99_upper":0.4183565283051233,"ci_98_lower":0.23858366602437692,"ci_98_upper":0.41189881996900485,"ci_95_lower":0.25276647406473574,"ci_95_upper":0.39999638042846686,"ci_90_lower":0.26080538603273457,"ci_90_upper":0.38911676322989963,"ci_75_lower":0.2821145350370738,"ci_75_upper":0.36874952814469,"ci_50_lower":0.29892931169460474,"ci_50_upper":0.35098343945719535},"65536":{"avg_score":0.3483933989835136,"min_score":0.009118541143834591,"max_score":1,"n_tests":85,"stddev":0.4557699975157975,"median_score":0.03642524406313896,"q1_score":0.022934583947062492,"q3_score":0.994310736656189,"avg_token_efficiency":1.0209904012221067,"is_incomplete":false,"total_prompt_cost_bin":1.1069518200000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.054047760000000014,"total_reasoning_cost_bin":0,"total_cost_bin":1.1609995800000001,"total_on_demand_input_cost_bin":1.1069518200000001,"ci_99_lower":0.24263089812448357,"ci_99_upper":0.4749515990311937,"ci_98_lower":0.24435220327337875,"ci_98_upper":0.46353168812635187,"ci_95_lower":0.25576735863407307,"ci_95_upper":0.4505128187100019,"ci_90_lower":0.2678654527439572,"ci_90_upper":0.4365483362375594,"ci_75_lower":0.2908538032336818,"ci_75_upper":0.4050560675561428,"ci_50_lower":0.31438352908007805,"ci_50_upper":0.3816884136484826},"131072":{"avg_score":0.2538334287142343,"min_score":0.005767250433564186,"max_score":1,"n_tests":98,"stddev":0.41160027129528054,"median_score":0.029819397255778313,"q1_score":0.020163854584097862,"q3_score":0.0658502895385027,"avg_token_efficiency":1.016237238200015,"is_incomplete":false,"total_prompt_cost_bin":2.5415728649999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06554652,"total_reasoning_cost_bin":0,"total_cost_bin":2.6071193849999994,"total_on_demand_input_cost_bin":2.5415728649999996,"ci_99_lower":0.15612238664966438,"ci_99_upper":0.37129693243671585,"ci_98_lower":0.165586861818031,"ci_98_upper":0.3618960646133661,"ci_95_lower":0.18197672862787637,"ci_95_upper":0.3334846856026216,"ci_90_lower":0.1881147492170923,"ci_90_upper":0.32224803785912276,"ci_75_lower":0.20486630292191188,"ci_75_upper":0.301414034955978,"ci_50_lower":0.2243568812513591,"ci_50_upper":0.28205200491653637}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-27B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":1.95e-7,"completion":0.00000156,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-25T21:10:10.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T05:39:18.376Z"},{"model_slug":"anthropic/claude-opus-4.7","reasoning_mode":null,"overall_metrics":{"auc_1m":0.053612081873778465,"auc_128k":0.14131668684197135,"cum_avg_1m":0.1637572509790753,"cum_avg_128k":0.24140014895222528,"avg_score_overall":0.1512058934153022,"avg_response_time_ms":21170.000568669017,"total_prompt_cost":9.271430000000006,"total_cache_read_cost":106.14423949999968,"total_completion_cost":4.214125,"total_reasoning_cost":0,"total_prompt_cost_128k":2.7965049999999994,"total_completion_cost_128k":3.4302749999999995,"total_on_demand_input_cost":1070.7138249999996,"total_on_demand_input_cost_128k":100.87839000000002,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.6293985094092707},"bin_metrics":{"8192":{"avg_score":0.5272968030185439,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4890965371061529,"median_score":1,"q1_score":0.02259412407875061,"q3_score":1,"avg_token_efficiency":1.6601281011370588,"is_incomplete":false,"total_prompt_cost_bin":0.34745499999999996,"total_cache_read_cost_bin":0.3856169999999998,"total_completion_cost_bin":1.0487,"total_reasoning_cost_bin":0,"total_cost_bin":1.7817719999999997,"total_on_demand_input_cost_bin":4.2036250000000015,"ci_99_lower":0.3917736571744026,"ci_99_upper":0.662432133678638,"ci_98_lower":0.39742952571221396,"ci_98_upper":0.6499990684943041,"ci_95_lower":0.41177406988761506,"ci_95_upper":0.6325115751320844,"ci_90_lower":0.4361910372026614,"ci_90_upper":0.6199318895654868,"ci_75_lower":0.46497303879514224,"ci_75_upper":0.5898123477694753,"ci_50_lower":0.49070160001720065,"ci_50_upper":0.5644867206312483},"16384":{"avg_score":0.37550730920629577,"min_score":0.0035834265872836113,"max_score":1,"n_tests":80,"stddev":0.46351243647205953,"median_score":0.0378870964050293,"q1_score":0.027383452747017145,"q3_score":1,"avg_token_efficiency":1.57385742205274,"is_incomplete":false,"total_prompt_cost_bin":0.32913499999999973,"total_cache_read_cost_bin":0.8330924999999999,"total_completion_cost_bin":1.1226249999999995,"total_reasoning_cost_bin":0,"total_cost_bin":2.2848524999999995,"total_on_demand_input_cost_bin":8.660059999999998,"ci_99_lower":0.2421127141697216,"ci_99_upper":0.5082022721273243,"ci_98_lower":0.2502564822173444,"ci_98_upper":0.4911193910026923,"ci_95_lower":0.270886242337292,"ci_95_upper":0.4792175312372273,"ci_90_lower":0.2890350102633238,"ci_90_upper":0.46203613516991027,"ci_75_lower":0.31690761636418757,"ci_75_upper":0.43636551877425517,"ci_50_lower":0.3402427558612544,"ci_50_upper":0.40761603806167834},"32768":{"avg_score":0.17403754131502117,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.29145269346357433,"median_score":0.03928740136325359,"q1_score":0.017925535328686237,"q3_score":0.15640167519450188,"avg_token_efficiency":1.6528166626033132,"is_incomplete":false,"total_prompt_cost_bin":0.5059950000000001,"total_cache_read_cost_bin":3.161177499999999,"total_completion_cost_bin":0.7290249999999999,"total_reasoning_cost_bin":0,"total_cost_bin":4.396197499999999,"total_on_demand_input_cost_bin":32.11777000000002,"ci_99_lower":0.11618784645202088,"ci_99_upper":0.24477170701520448,"ci_98_lower":0.1202011122595885,"ci_98_upper":0.23745106921648532,"ci_95_lower":0.12677718583040318,"ci_95_upper":0.22686950309282966,"ci_90_lower":0.1332535951207854,"ci_90_upper":0.21588163325581902,"ci_75_lower":0.1447857361531265,"ci_75_upper":0.20196385484889426,"ci_50_lower":0.1566635099124516,"ci_50_upper":0.18886178192919675},"65536":{"avg_score":0.1250459742403644,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.21570855200194533,"median_score":0.02821197174489498,"q1_score":0.014098690822720528,"q3_score":0.0833333358168602,"avg_token_efficiency":1.6333090125114886,"is_incomplete":false,"total_prompt_cost_bin":0.5559149999999996,"total_cache_read_cost_bin":4.057639999999996,"total_completion_cost_bin":0.5154,"total_reasoning_cost_bin":0,"total_cost_bin":5.128954999999995,"total_on_demand_input_cost_bin":41.132315,"ci_99_lower":0.07314502976495116,"ci_99_upper":0.1828249592380324,"ci_98_lower":0.07573160168555948,"ci_98_upper":0.18112044369314306,"ci_95_lower":0.08253480358925812,"ci_95_upper":0.17416372166426083,"ci_90_lower":0.08909479583147914,"ci_90_upper":0.16575016568825746,"ci_75_lower":0.09810520646381465,"ci_75_upper":0.15284683387384146,"ci_50_lower":0.10752978146980133,"ci_50_upper":0.13878438790493153},"131072":{"avg_score":0.005113116980901042,"min_score":0,"max_score":0.09598740935325623,"n_tests":103,"stddev":0.01598762876638101,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6116698163062615,"is_incomplete":false,"total_prompt_cost_bin":1.058005,"total_cache_read_cost_bin":1.3706615,"total_completion_cost_bin":0.014524999999999998,"total_reasoning_cost_bin":0,"total_cost_bin":2.4431915,"total_on_demand_input_cost_bin":14.76462,"ci_99_lower":0.0017864224676368307,"ci_99_upper":0.009525682697416217,"ci_98_lower":0.0019164548118065282,"ci_98_upper":0.00913242398987262,"ci_95_lower":0.0023101328505944567,"ci_95_upper":0.008185451100263925,"ci_90_lower":0.0027169500540905785,"ci_90_upper":0.007617950084103831,"ci_75_lower":0.0033987616912181514,"ci_75_upper":0.006788378730597138,"ci_50_lower":0.004004493726471675,"ci_50_upper":0.006072255705926314},"262144":{"avg_score":0.01244456116407317,"min_score":0,"max_score":0.248094841837883,"n_tests":141,"stddev":0.03960347207247479,"median_score":0,"q1_score":0,"q3_score":0,"avg_token_efficiency":1.6299556904390495,"is_incomplete":false,"total_prompt_cost_bin":2.1424649999999996,"total_cache_read_cost_bin":6.189571499999997,"total_completion_cost_bin":0.08930000000000003,"total_reasoning_cost_bin":0,"total_cost_bin":8.421336499999997,"total_on_demand_input_cost_bin":64.03818000000001,"ci_99_lower":0.0053281759401017145,"ci_99_upper":0.021994540685980172,"ci_98_lower":0.005618141733460023,"ci_98_upper":0.021340944802938418,"ci_95_lower":0.00685182092397586,"ci_95_upper":0.019738958407720178,"ci_90_lower":0.007494249200144559,"ci_90_upper":0.01810900331631712,"ci_75_lower":0.008708607027280733,"ci_75_upper":0.01629666005253399,"ci_50_lower":0.010068705931086264,"ci_50_upper":0.014527956520641514},"524288":{"avg_score":0.09061270190740273,"min_score":0,"max_score":1,"n_tests":236,"stddev":0.16013364060590798,"median_score":0.03333286941051483,"q1_score":0.013529487885534763,"q3_score":0.09905861504375935,"avg_token_efficiency":1.6303101658356784,"is_incomplete":false,"total_prompt_cost_bin":4.332460000000006,"total_cache_read_cost_bin":90.14647949999969,"total_completion_cost_bin":0.6945500000000002,"total_reasoning_cost_bin":0,"total_cost_bin":95.1734894999997,"total_on_demand_input_cost_bin":905.7972549999996,"ci_99_lower":0.06664661866158417,"ci_99_upper":0.12045099421241272,"ci_98_lower":0.06836573466136203,"ci_98_upper":0.1162607587853563,"ci_95_lower":0.07170183740417381,"ci_95_upper":0.11207503523413434,"ci_90_lower":0.07414039926212754,"ci_90_upper":0.1086410997085218,"ci_75_lower":0.07913888329660995,"ci_75_upper":0.10205725947340952,"ci_50_lower":0.08380101265023676,"ci_50_upper":0.09670843063916329}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","xhigh","high","medium","low"],"reasoning_default":"high","sponsors":["openrouter"],"openrouter_name":"Anthropic: Claude Opus 4.7","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000005,"completion":0.000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-16T14:51:40.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:23:18.067Z"},{"model_slug":"qwen/qwen3.5-122b-a10b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.050779135445919485,"auc_128k":0.2991637584811195,"cum_avg_1m":0.22270513514555879,"cum_avg_128k":0.35632821623289407,"avg_score_overall":0.3447102831427348,"avg_response_time_ms":18471.413887245526,"total_prompt_cost":6.435172639999999,"total_cache_read_cost":0,"total_completion_cost":0.39392704000000006,"total_reasoning_cost":0,"total_prompt_cost_128k":6.435172639999999,"total_completion_cost_128k":0.39392704000000006,"total_on_demand_input_cost":6.435172639999999,"total_on_demand_input_cost_128k":6.435172639999999,"total_runs":482,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0208319038227718},"bin_metrics":{"8192":{"avg_score":0.5534389727923553,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4790032643575704,"median_score":0.9745472371578217,"q1_score":0.02841759007424116,"q3_score":1,"avg_token_efficiency":1.0228079512716757,"is_incomplete":false,"total_prompt_cost_bin":0.14132352000000015,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.05922799999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.20055152000000015,"total_on_demand_input_cost_bin":0.14132352000000015,"ci_99_lower":0.4103378289810207,"ci_99_upper":0.6970261169283185,"ci_98_lower":0.4294869820037275,"ci_98_upper":0.6741371825556852,"ci_95_lower":0.44357950372039345,"ci_95_upper":0.6544418425214825,"ci_90_lower":0.4683849157107761,"ci_90_upper":0.6393600373415392,"ci_75_lower":0.4934323833360395,"ci_75_upper":0.6163412601068557,"ci_50_lower":0.5175059081535437,"ci_50_upper":0.5886866866727359},"16384":{"avg_score":0.39021677748824,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.4705441299434612,"median_score":0.036586178466677666,"q1_score":0.02532999776303768,"q3_score":0.9954689294099808,"avg_token_efficiency":1.016119475838464,"is_incomplete":false,"total_prompt_cost_bin":0.2914246400000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0586352,"total_reasoning_cost_bin":0,"total_cost_bin":0.3500598400000002,"total_on_demand_input_cost_bin":0.2914246400000002,"ci_99_lower":0.2572970878888882,"ci_99_upper":0.5224358330697433,"ci_98_lower":0.26822992790798894,"ci_98_upper":0.5106130425619777,"ci_95_lower":0.28202248165471244,"ci_95_upper":0.4893828062358077,"ci_90_lower":0.29463740262479404,"ci_90_upper":0.4763030832866207,"ci_75_lower":0.3297368575062137,"ci_75_upper":0.45104884640604725,"ci_50_lower":0.3541852754264255,"ci_50_upper":0.4257353021159361},"32768":{"avg_score":0.30404226820374947,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.43042958883938626,"median_score":0.03672958165407181,"q1_score":0.025663322303444147,"q3_score":0.9721610397100449,"avg_token_efficiency":1.0257548592579664,"is_incomplete":false,"total_prompt_cost_bin":1.0339887999999997,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.11421488000000009,"total_reasoning_cost_bin":0,"total_cost_bin":1.1482036799999997,"total_on_demand_input_cost_bin":1.0339887999999997,"ci_99_lower":0.21533820730206035,"ci_99_upper":0.3940973686717831,"ci_98_lower":0.22043694330199606,"ci_98_upper":0.38324090214964923,"ci_95_lower":0.23314654501758494,"ci_95_upper":0.37198725932064597,"ci_90_lower":0.24310758594331086,"ci_90_upper":0.3617509448166375,"ci_75_lower":0.2611320256595825,"ci_75_upper":0.34719195217407745,"ci_50_lower":0.2786062331108363,"ci_50_upper":0.32815728701654523},"65536":{"avg_score":0.2887563346282524,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.43190652030411725,"median_score":0.030576243996620178,"q1_score":0.020283976569771767,"q3_score":0.9629040956497192,"avg_token_efficiency":1.0209904012221067,"is_incomplete":false,"total_prompt_cost_bin":1.4759357600000005,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.06855888,"total_reasoning_cost_bin":0,"total_cost_bin":1.5444946400000006,"total_on_demand_input_cost_bin":1.4759357600000005,"ci_99_lower":0.17500964159551352,"ci_99_upper":0.42380955705747886,"ci_98_lower":0.1871351356350762,"ci_98_upper":0.4028655163613969,"ci_95_lower":0.19905352974173557,"ci_95_upper":0.3896776045831468,"ci_90_lower":0.21120126189444874,"ci_90_upper":0.3672116615973851,"ci_75_lower":0.2324855379335692,"ci_75_upper":0.336480043614831,"ci_50_lower":0.2544957812902901,"ci_50_upper":0.32052804559250087},"131072":{"avg_score":0.24518672805187283,"min_score":0.004621605854481459,"max_score":1,"n_tests":101,"stddev":0.403422201442963,"median_score":0.02982601523399353,"q1_score":0.02224094420671463,"q3_score":0.04639101028442383,"avg_token_efficiency":1.0162370149584388,"is_incomplete":false,"total_prompt_cost_bin":3.4924999199999984,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.09329007999999998,"total_reasoning_cost_bin":0,"total_cost_bin":3.5857899999999985,"total_on_demand_input_cost_bin":3.4924999199999984,"ci_99_lower":0.14980398412690138,"ci_99_upper":0.3523396402168881,"ci_98_lower":0.1591372868774625,"ci_98_upper":0.3384729821689538,"ci_95_lower":0.17044541186221993,"ci_95_upper":0.32119258055970584,"ci_90_lower":0.1799611540769439,"ci_90_upper":0.3110808831999208,"ci_75_lower":0.1982048783153941,"ci_75_upper":0.29172743644535376,"ci_50_lower":0.21713413354839284,"ci_50_upper":0.27182487414638046}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-122B-A10B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":2.6e-7,"completion":0.00000208,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-25T21:09:49.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T05:48:03.262Z"},{"model_slug":"qwen/qwen3.5-397b-a17b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.05037131433291217,"auc_128k":0.3087739448437033,"cum_avg_1m":0.22906983614947535,"cum_avg_128k":0.3665117378391606,"avg_score_overall":0.3501965358029774,"avg_response_time_ms":10024.530884797909,"total_prompt_cost":9.499481160000002,"total_cache_read_cost":0.2570131200000001,"total_completion_cost":0.4486131000000001,"total_reasoning_cost":0,"total_prompt_cost_128k":9.499481160000002,"total_completion_cost_128k":0.4486131000000001,"total_on_demand_input_cost":9.756494280000002,"total_on_demand_input_cost_128k":9.756494280000002,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0208128870566133},"bin_metrics":{"8192":{"avg_score":0.5506921908294317,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.48060456938202784,"median_score":0.9785349369049072,"q1_score":0.02781451726332307,"q3_score":0.9976888746023178,"avg_token_efficiency":1.0228079512716761,"is_incomplete":false,"total_prompt_cost_bin":0.15283007999999992,"total_cache_read_cost_bin":0.05915520000000001,"total_completion_cost_bin":0.06720948000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.27919476,"total_on_demand_input_cost_bin":0.21198527999999994,"ci_99_lower":0.4067868583917443,"ci_99_upper":0.6947979426416568,"ci_98_lower":0.42744901320064677,"ci_98_upper":0.6724506502490839,"ci_95_lower":0.4410102226794697,"ci_95_upper":0.6503718871096498,"ci_90_lower":0.46555846779607235,"ci_90_upper":0.6360435977968153,"ci_75_lower":0.4907515848244657,"ci_75_upper":0.6131297533363977,"ci_50_lower":0.5146593217577902,"ci_50_upper":0.586308132513659},"16384":{"avg_score":0.45008244569180533,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.48034227200858953,"median_score":0.05175222270190716,"q1_score":0.02509907726198435,"q3_score":0.995401069521904,"avg_token_efficiency":1.016119475838464,"is_incomplete":false,"total_prompt_cost_bin":0.23927904000000014,"total_cache_read_cost_bin":0.19785792000000008,"total_completion_cost_bin":0.0645957,"total_reasoning_cost_bin":0,"total_cost_bin":0.5017326600000003,"total_on_demand_input_cost_bin":0.4371369600000003,"ci_99_lower":0.3153720460811164,"ci_99_upper":0.5807077314367053,"ci_98_lower":0.3258986710248282,"ci_98_upper":0.5690859454553574,"ci_95_lower":0.338468247941928,"ci_95_upper":0.5579160201267223,"ci_90_lower":0.3536815647856565,"ci_90_upper":0.5386667112598659,"ci_75_lower":0.38877966168074635,"ci_75_upper":0.5112931911586202,"ci_50_lower":0.41373345649917614,"ci_50_upper":0.48616401110775764},"32768":{"avg_score":0.28939265681273196,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4221032808281253,"median_score":0.03931949846446514,"q1_score":0.024524828419089317,"q3_score":0.9665705114603043,"avg_token_efficiency":1.025754859257966,"is_incomplete":false,"total_prompt_cost_bin":1.5509832000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.12821562000000003,"total_reasoning_cost_bin":0,"total_cost_bin":1.67919882,"total_on_demand_input_cost_bin":1.5509832000000001,"ci_99_lower":0.20192158095096066,"ci_99_upper":0.38470031681062267,"ci_98_lower":0.20639424130755393,"ci_98_upper":0.3698660042327452,"ci_95_lower":0.21977397316171418,"ci_95_upper":0.3581040896182136,"ci_90_lower":0.22868509679623564,"ci_90_upper":0.3483140179837766,"ci_75_lower":0.24749230686960372,"ci_75_upper":0.33166087978893566,"ci_50_lower":0.2653791722285164,"ci_50_upper":0.31239529238904223},"65536":{"avg_score":0.3216979274087969,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.4465081739495105,"median_score":0.0301204826682806,"q1_score":0.020368363708257675,"q3_score":0.9758948087692261,"avg_token_efficiency":1.020990401222107,"is_incomplete":false,"total_prompt_cost_bin":2.2139036399999994,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.08048664000000003,"total_reasoning_cost_bin":0,"total_cost_bin":2.2943902799999996,"total_on_demand_input_cost_bin":2.2139036399999994,"ci_99_lower":0.2074564256106558,"ci_99_upper":0.4565899729289881,"ci_98_lower":0.21831264431895142,"ci_98_upper":0.4452775991068167,"ci_95_lower":0.22976758652366697,"ci_95_upper":0.41606935319435934,"ci_90_lower":0.2424786061963395,"ci_90_upper":0.4012968178329003,"ci_75_lower":0.2641519988435047,"ci_75_upper":0.3776480820824338,"ci_50_lower":0.28741688961437084,"ci_50_upper":0.35442204638658203},"131072":{"avg_score":0.22069346845303708,"min_score":0.003610108280554414,"max_score":1,"n_tests":103,"stddev":0.3887007765213767,"median_score":0.028722601011395454,"q1_score":0.018006102181971073,"q3_score":0.041022079065442085,"avg_token_efficiency":1.0162368757633713,"is_incomplete":false,"total_prompt_cost_bin":5.342485200000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.10810566,"total_reasoning_cost_bin":0,"total_cost_bin":5.450590860000002,"total_on_demand_input_cost_bin":5.342485200000002,"ci_99_lower":0.12705807353468224,"ci_99_upper":0.32331834830312073,"ci_98_lower":0.13601180454892406,"ci_98_upper":0.3046422400401489,"ci_95_lower":0.15400982404108501,"ci_95_upper":0.2882396598664131,"ci_90_lower":0.16322367478826635,"ci_90_upper":0.2772371201734149,"ci_75_lower":0.17603154085040798,"ci_75_upper":0.2601659048523288,"ci_50_lower":0.19402796532777905,"ci_50_upper":0.24560934963163658}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5 397B A17B","provider_name":"Qwen","run_provider_names":["AtlasCloud"],"pricing_metadata":{"prompt":3.9e-7,"completion":0.00000234,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-16T06:23:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T05:43:09.371Z"},{"model_slug":"stepfun/step-3.7-flash","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.050003990130619076,"auc_128k":0.3069044510757704,"cum_avg_1m":0.24176447588491168,"cum_avg_128k":0.3868231614158587,"avg_score_overall":0.3765231643395483,"avg_response_time_ms":33348.17531213682,"total_prompt_cost":0.15637040000000002,"total_cache_read_cost":0.9475232000000013,"total_completion_cost":2.301397250000001,"total_reasoning_cost":2.0175554,"total_prompt_cost_128k":0.15637040000000002,"total_completion_cost_128k":2.301397250000001,"total_on_demand_input_cost":4.893986399999996,"total_on_demand_input_cost_128k":4.893986399999996,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9984466628317946},"bin_metrics":{"8192":{"avg_score":0.6535845728532876,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4632911352989876,"median_score":1,"q1_score":0.042804510332643986,"q3_score":1,"avg_token_efficiency":0.9943799479094226,"is_incomplete":false,"total_prompt_cost_bin":0.028609599999999995,"total_cache_read_cost_bin":0.015429119999999998,"total_completion_cost_bin":0.24758464999999996,"total_reasoning_cost_bin":0.11110035,"total_cost_bin":0.40272371999999995,"total_on_demand_input_cost_bin":0.10575520000000002,"ci_99_lower":0.5158125976280427,"ci_99_upper":0.778702572014794,"ci_98_lower":0.5240144669678993,"ci_98_upper":0.7619471459368942,"ci_95_lower":0.5540305645644549,"ci_95_upper":0.7487723222799831,"ci_90_lower":0.5690442763085594,"ci_90_upper":0.7350437287738896,"ci_75_lower":0.5933918224429362,"ci_75_upper":0.7111408015829511,"ci_50_lower":0.6169309223201708,"ci_50_upper":0.6878396603860891},"16384":{"avg_score":0.40870980334002527,"min_score":0.008149010129272938,"max_score":1,"n_tests":80,"stddev":0.4736229846116112,"median_score":0.04484527185559273,"q1_score":0.023501567542552948,"q3_score":1,"avg_token_efficiency":0.9903659767661971,"is_incomplete":false,"total_prompt_cost_bin":0.017748800000000002,"total_cache_read_cost_bin":0.040070399999999985,"total_completion_cost_bin":0.35355715,"total_reasoning_cost_bin":0.3362082500000001,"total_cost_bin":0.7475846,"total_on_demand_input_cost_bin":0.21810079999999984,"ci_99_lower":0.27316080032574247,"ci_99_upper":0.5412362119730679,"ci_98_lower":0.28553764104319274,"ci_98_upper":0.5391902256489266,"ci_95_lower":0.29710400153999217,"ci_95_upper":0.5140294192094006,"ci_90_lower":0.3146562560729217,"ci_90_upper":0.4969137779780431,"ci_75_lower":0.3484048371916288,"ci_75_upper":0.4689713736574049,"ci_50_lower":0.372362023126334,"ci_50_upper":0.4442700631829211},"32768":{"avg_score":0.3768279385009972,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.46116633906268933,"median_score":0.048089878633618355,"q1_score":0.02433614945039153,"q3_score":1,"avg_token_efficiency":1.0109356288769533,"is_incomplete":false,"total_prompt_cost_bin":0.029867200000000035,"total_cache_read_cost_bin":0.15068608000000008,"total_completion_cost_bin":0.5947179000000001,"total_reasoning_cost_bin":0.42170155,"total_cost_bin":1.1969727300000002,"total_on_demand_input_cost_bin":0.7832975999999996,"ci_99_lower":0.276279915493925,"ci_99_upper":0.48175496265694484,"ci_98_lower":0.28903907893078323,"ci_98_upper":0.4689641701027422,"ci_95_lower":0.29916867576522393,"ci_95_upper":0.45258284115133235,"ci_90_lower":0.31042943609251833,"ci_90_upper":0.4421062423946911,"ci_75_lower":0.3293566662402608,"ci_75_upper":0.4224398923275551,"ci_50_lower":0.34815134312991763,"ci_50_upper":0.4041616768128519},"65536":{"avg_score":0.2766259948284749,"min_score":0.006511123385280371,"max_score":1,"n_tests":85,"stddev":0.4213613261214444,"median_score":0.031481482088565826,"q1_score":0.021835196763277054,"q3_score":0.6918873190879822,"avg_token_efficiency":0.9950202971179802,"is_incomplete":false,"total_prompt_cost_bin":0.026997399999999994,"total_cache_read_cost_bin":0.2158918400000004,"total_completion_cost_bin":0.3865782500000001,"total_reasoning_cost_bin":0.3926652,"total_cost_bin":1.0221326900000005,"total_on_demand_input_cost_bin":1.1064565999999987,"ci_99_lower":0.16831669280623252,"ci_99_upper":0.3984007929795531,"ci_98_lower":0.1818481122346266,"ci_98_upper":0.3882227881463683,"ci_95_lower":0.19094143533914845,"ci_95_upper":0.3687184803532984,"ci_90_lower":0.2074194790729705,"ci_90_upper":0.35230863741653795,"ci_75_lower":0.22133548085180604,"ci_75_upper":0.3246461688102606,"ci_50_lower":0.2439235951477552,"ci_50_upper":0.3051856305614552},"131072":{"avg_score":0.2183674975565083,"min_score":0.0036513006780296564,"max_score":1,"n_tests":103,"stddev":0.38337571792463626,"median_score":0.03100775182247162,"q1_score":0.020419476553797722,"q3_score":0.05060974322259426,"avg_token_efficiency":0.9942188354781073,"is_incomplete":false,"total_prompt_cost_bin":0.05314740000000001,"total_cache_read_cost_bin":0.5254457600000009,"total_completion_cost_bin":0.7189593000000005,"total_reasoning_cost_bin":0.75588005,"total_cost_bin":2.0534325100000013,"total_on_demand_input_cost_bin":2.6803761999999978,"ci_99_lower":0.1270661882591633,"ci_99_upper":0.31886345188437454,"ci_98_lower":0.1358159451636721,"ci_98_upper":0.30562713621229437,"ci_95_lower":0.15039277425729805,"ci_95_upper":0.2881698813895781,"ci_90_lower":0.16021240265786793,"ci_90_upper":0.27484683922505526,"ci_75_lower":0.17657702911822623,"ci_75_upper":0.2585594130731391,"ci_50_lower":0.19434499455100487,"ci_50_upper":0.24213141348841377}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"StepFun: Step 3.7 Flash","provider_name":"Stepfun","run_provider_names":[],"pricing_metadata":{"prompt":2e-7,"completion":0.00000115,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-28T16:17:49.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-23T01:12:49.180Z"},{"model_slug":"thinkingmachines/inkling-small","reasoning_mode":"max","overall_metrics":{"auc_1m":0.04964765168299448,"auc_128k":0.24938312573796112,"cum_avg_1m":0.2457185022572078,"cum_avg_128k":0.3781001559595932,"avg_score_overall":0.29200587636027486,"avg_response_time_ms":67561.49532421875,"total_prompt_cost":0.34876619999999975,"total_cache_read_cost":5.956556799999997,"total_completion_cost":2.9555160000000003,"total_reasoning_cost":2.9057580000000005,"total_prompt_cost_128k":0.2301376499999998,"total_completion_cost_128k":2.4902856,"total_on_demand_input_cost":27.15327180000004,"total_on_demand_input_cost_128k":10.82600325,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9812376596004894},"bin_metrics":{"8192":{"avg_score":0.7220178408897482,"min_score":0.01323472149670124,"max_score":1,"n_tests":80,"stddev":0.4404603484469436,"median_score":1,"q1_score":0.047062129713594913,"q3_score":1,"avg_token_efficiency":0.9771471687576284,"is_incomplete":false,"total_prompt_cost_bin":0.0228204,"total_cache_read_cost_bin":0.04691199999999999,"total_completion_cost_bin":0.38595599999999997,"total_reasoning_cost_bin":0.3868379999999999,"total_cost_bin":0.8425263999999999,"total_on_demand_input_cost_bin":0.23392440000000006,"ci_99_lower":0.5885140564714675,"ci_99_upper":0.8314360900577157,"ci_98_lower":0.6004625766043318,"ci_98_upper":0.8301109191720607,"ci_95_lower":0.624591260564921,"ci_95_upper":0.8076123378134799,"ci_90_lower":0.6372593237424735,"ci_90_upper":0.7951593277376378,"ci_75_lower":0.6621236615668749,"ci_75_upper":0.7715900538605638,"ci_50_lower":0.6861612412816612,"ci_50_upper":0.7571105975657701},"16384":{"avg_score":0.5570703924167901,"min_score":0.008628128096461296,"max_score":1,"n_tests":80,"stddev":0.48332031081582466,"median_score":0.999593585729599,"q1_score":0.028455635532736778,"q3_score":1,"avg_token_efficiency":0.9676648473384523,"is_incomplete":false,"total_prompt_cost_bin":0.029703600000000004,"total_cache_read_cost_bin":0.10003200000000002,"total_completion_cost_bin":0.5298779999999998,"total_reasoning_cost_bin":0.5204964000000001,"total_cost_bin":1.18011,"total_on_demand_input_cost_bin":0.4798475999999996,"ci_99_lower":0.42162227619643095,"ci_99_upper":0.6980066034725169,"ci_98_lower":0.42826758354250344,"ci_98_upper":0.6845173556704539,"ci_95_lower":0.448169849505939,"ci_95_upper":0.6609645707037999,"ci_90_lower":0.47080420098616743,"ci_90_upper":0.6466835125465877,"ci_75_lower":0.4964620485552586,"ci_75_upper":0.622076510588522,"ci_50_lower":0.5207407901616534,"ci_50_upper":0.5934776171605335},"32768":{"avg_score":0.31156670402988845,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.43257811371866955,"median_score":0.039838362485170364,"q1_score":0.02615354536101222,"q3_score":0.9994466453790665,"avg_token_efficiency":0.9954973496148867,"is_incomplete":false,"total_prompt_cost_bin":0.04988880000000002,"total_cache_read_cost_bin":0.37501440000000047,"total_completion_cost_bin":0.7618776,"total_reasoning_cost_bin":0.7576488000000005,"total_cost_bin":1.9444296000000012,"total_on_demand_input_cost_bin":1.7374536,"ci_99_lower":0.22281922624471276,"ci_99_upper":0.4005996719932248,"ci_98_lower":0.2270551691820112,"ci_98_upper":0.39372737231380167,"ci_95_lower":0.23993613828339114,"ci_95_upper":0.38004493675419804,"ci_90_lower":0.24986400356000935,"ci_90_upper":0.36987584494679193,"ci_75_lower":0.2691583641880082,"ci_75_upper":0.355369621205575,"ci_50_lower":0.28554968610671283,"ci_50_upper":0.3360115486023406},"65536":{"avg_score":0.20502444753176807,"min_score":0.0016872890992090106,"max_score":1,"n_tests":85,"stddev":0.3620862426139534,"median_score":0.029297716915607452,"q1_score":0.0202312134206295,"q3_score":0.05736137554049492,"avg_token_efficiency":0.977765887494243,"is_incomplete":false,"total_prompt_cost_bin":0.060737849999999934,"total_cache_read_cost_bin":0.5301375999999997,"total_completion_cost_bin":0.41117640000000005,"total_reasoning_cost_bin":0.40347240000000006,"total_cost_bin":1.4055242499999998,"total_on_demand_input_cost_bin":2.4463570499999987,"ci_99_lower":0.11339833491791368,"ci_99_upper":0.31393241221283097,"ci_98_lower":0.1215406821946997,"ci_98_upper":0.306620347895276,"ci_95_lower":0.13259038689121713,"ci_95_upper":0.28538938223329535,"ci_90_lower":0.14559547032393955,"ci_90_upper":0.2712050242809003,"ci_75_lower":0.15862095764485487,"ci_75_upper":0.24754617355596137,"ci_50_lower":0.17691830260549013,"ci_50_upper":0.22693255069145166},"131072":{"avg_score":0.09482139492977085,"min_score":0.0011587485205382109,"max_score":0.9988207817077637,"n_tests":103,"stddev":0.2397407107002489,"median_score":0.029255319386720657,"q1_score":0.01901637203991413,"q3_score":0.0417045671492815,"avg_token_efficiency":0.977333360076887,"is_incomplete":false,"total_prompt_cost_bin":0.06698699999999985,"total_cache_read_cost_bin":1.3025408,"total_completion_cost_bin":0.40139759999999997,"total_reasoning_cost_bin":0.3907619999999999,"total_cost_bin":2.1616874,"total_on_demand_input_cost_bin":5.928420600000002,"ci_99_lower":0.04142693767805574,"ci_99_upper":0.1687758360295647,"ci_98_lower":0.048356962832464004,"ci_98_upper":0.15855250416065847,"ci_95_lower":0.05516929972337025,"ci_95_upper":0.1422151937382861,"ci_90_lower":0.05827896212865102,"ci_90_upper":0.13251859464341303,"ci_75_lower":0.06743132618557689,"ci_75_upper":0.12121496535865905,"ci_50_lower":0.07697676781404456,"ci_50_upper":0.11017553899346769},"262144":{"avg_score":0.07524723825969648,"min_score":0.003864734200760722,"max_score":1,"n_tests":141,"stddev":0.16951845579637126,"median_score":0.03061782382428646,"q1_score":0.020773638039827347,"q3_score":0.050357937812805176,"avg_token_efficiency":0.9824503538302468,"is_incomplete":false,"total_prompt_cost_bin":0.11862854999999993,"total_cache_read_cost_bin":3.6019199999999967,"total_completion_cost_bin":0.4652304000000001,"total_reasoning_cost_bin":0.4465404,"total_cost_bin":4.632319349999997,"total_on_demand_input_cost_bin":16.327268550000042,"ci_99_lower":0.044600091023446586,"ci_99_upper":0.11600669597699934,"ci_98_lower":0.04673397058370054,"ci_98_upper":0.11411643459762162,"ci_95_lower":0.050712324505017954,"ci_95_upper":0.10618672365486186,"ci_90_lower":0.054698137635718506,"ci_90_upper":0.10035280771148952,"ci_75_lower":0.05937186675455837,"ci_75_upper":0.09098458251831656,"ci_50_lower":0.06500092196791823,"ci_50_upper":0.08397688054279853}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low","minimal","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Thinking Machines: Inkling Small","provider_name":"Thinkingmachines","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":4.5e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-30T20:25:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T13:14:26.334Z"},{"model_slug":"thinkingmachines/inkling-small","reasoning_mode":"minimal","overall_metrics":{"auc_1m":0.0488017158314515,"auc_128k":0.20046752829254075,"cum_avg_1m":0.19818009100176215,"cum_avg_128k":0.297031084650858,"avg_score_overall":0.24352157099992036,"avg_response_time_ms":25160.916616796876,"total_prompt_cost":0.3019949999999998,"total_cache_read_cost":5.966950399999996,"total_completion_cost":1.6621020000000002,"total_reasoning_cost":1.5036395999999999,"total_prompt_cost_128k":0.18336644999999985,"total_completion_cost_128k":1.2407712000000002,"total_on_demand_input_cost":27.15327180000005,"total_on_demand_input_cost_128k":10.826003250000003,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9812376596004891},"bin_metrics":{"8192":{"avg_score":0.5376371601654683,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4883096787947355,"median_score":1,"q1_score":0.02525781188160181,"q3_score":1,"avg_token_efficiency":0.9771471687576279,"is_incomplete":false,"total_prompt_cost_bin":0.030942000000000008,"total_cache_read_cost_bin":0.04510719999999997,"total_completion_cost_bin":0.08995320000000001,"total_reasoning_cost_bin":0.061472399999999976,"total_cost_bin":0.22747479999999998,"total_on_demand_input_cost_bin":0.23392439999999998,"ci_99_lower":0.4023389816952695,"ci_99_upper":0.6717592473709374,"ci_98_lower":0.40515755356568844,"ci_98_upper":0.6595126708196476,"ci_95_lower":0.42536254829741665,"ci_95_upper":0.645201642598695,"ci_90_lower":0.45050998358783545,"ci_90_upper":0.6250273788679623,"ci_75_lower":0.4764810275337367,"ci_75_upper":0.5991158882672607,"ci_50_lower":0.5013982334028697,"ci_50_upper":0.5740073655586456},"16384":{"avg_score":0.4174099988944363,"min_score":0.006237006280571222,"max_score":1,"n_tests":80,"stddev":0.47192282280653014,"median_score":0.04596088454127312,"q1_score":0.024936684872955084,"q3_score":1,"avg_token_efficiency":0.9676648473384528,"is_incomplete":false,"total_prompt_cost_bin":0.019335600000000005,"total_cache_read_cost_bin":0.10233600000000002,"total_completion_cost_bin":0.11313600000000003,"total_reasoning_cost_bin":0.08467919999999998,"total_cost_bin":0.31948680000000007,"total_on_demand_input_cost_bin":0.4798475999999999,"ci_99_lower":0.27863652398574057,"ci_99_upper":0.5498451133538037,"ci_98_lower":0.2908106433718931,"ci_98_upper":0.5428233121733064,"ci_95_lower":0.30577849024281023,"ci_95_upper":0.5220911658156546,"ci_90_lower":0.3257346393581247,"ci_90_upper":0.5098790500379982,"ci_75_lower":0.3552444782297243,"ci_75_upper":0.4759233189906809,"ci_50_lower":0.38224184966966274,"ci_50_upper":0.4527758039243054},"32768":{"avg_score":0.29086027373662493,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.42634097143632965,"median_score":0.03626870736479759,"q1_score":0.023187905550003052,"q3_score":0.991087332367897,"avg_token_efficiency":0.9954973496148869,"is_incomplete":false,"total_prompt_cost_bin":0.03422160000000001,"total_cache_read_cost_bin":0.37849600000000044,"total_completion_cost_bin":0.3085896,"total_reasoning_cost_bin":0.266868,"total_cost_bin":0.9881752000000005,"total_on_demand_input_cost_bin":1.7374535999999994,"ci_99_lower":0.2024578426681247,"ci_99_upper":0.38755855338028217,"ci_98_lower":0.20682398306452396,"ci_98_upper":0.3709450624617816,"ci_95_lower":0.2199542699588432,"ci_95_upper":0.36083081354098634,"ci_90_lower":0.22919852885294376,"ci_90_upper":0.3499261601726175,"ci_75_lower":0.24863398670512493,"ci_75_upper":0.3334529189716491,"ci_50_lower":0.26622780635379434,"ci_50_upper":0.31369282256834113},"65536":{"avg_score":0.14125328146140365,"min_score":0.005048911552876234,"max_score":1,"n_tests":85,"stddev":0.3144706840056737,"median_score":0.025344762951135635,"q1_score":0.017706576734781265,"q3_score":0.04197698086500168,"avg_token_efficiency":0.9777658874942426,"is_incomplete":false,"total_prompt_cost_bin":0.03199544999999998,"total_cache_read_cost_bin":0.5365247999999997,"total_completion_cost_bin":0.3387780000000001,"total_reasoning_cost_bin":0.3258408,"total_cost_bin":1.2331390499999997,"total_on_demand_input_cost_bin":2.4463570499999987,"ci_99_lower":0.06248158010393928,"ci_99_upper":0.24467726758505928,"ci_98_lower":0.0713588564605726,"ci_98_upper":0.23170964616027606,"ci_95_lower":0.08131885154926055,"ci_95_upper":0.21140117800219313,"ci_90_lower":0.08708377903272561,"ci_90_upper":0.19917485671337035,"ci_75_lower":0.10432623032932443,"ci_75_upper":0.17618776922429197,"ci_50_lower":0.11706507846795242,"ci_50_upper":0.16287411175503888},"131072":{"avg_score":0.09799470899635655,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.2320550648726577,"median_score":0.025577042251825333,"q1_score":0.01588775310665369,"q3_score":0.03972952999174595,"avg_token_efficiency":0.9773333600768865,"is_incomplete":false,"total_prompt_cost_bin":0.06687179999999986,"total_cache_read_cost_bin":1.3025664,"total_completion_cost_bin":0.39031440000000006,"total_reasoning_cost_bin":0.3837863999999999,"total_cost_bin":2.1435389999999996,"total_on_demand_input_cost_bin":5.928420600000006,"ci_99_lower":0.047894318782005604,"ci_99_upper":0.1637161313636382,"ci_98_lower":0.0517462984145194,"ci_98_upper":0.1559199669844588,"ci_95_lower":0.05907786781662279,"ci_95_upper":0.14312084754058654,"ci_90_lower":0.06320316678385512,"ci_90_upper":0.13619943463686598,"ci_75_lower":0.07232706103608401,"ci_75_upper":0.12227652236837351,"ci_50_lower":0.08131442961420918,"ci_50_upper":0.11219924456186736},"262144":{"avg_score":0.1002853047598074,"min_score":0.002560819499194622,"max_score":1,"n_tests":141,"stddev":0.21003840265214221,"median_score":0.03336604684591293,"q1_score":0.021805495023727417,"q3_score":0.05082135647535324,"avg_token_efficiency":0.9824503538302466,"is_incomplete":false,"total_prompt_cost_bin":0.11862854999999996,"total_cache_read_cost_bin":3.6019199999999967,"total_completion_cost_bin":0.42133079999999995,"total_reasoning_cost_bin":0.38099279999999996,"total_cost_bin":4.522872149999996,"total_on_demand_input_cost_bin":16.327268550000046,"ci_99_lower":0.06120011903096241,"ci_99_upper":0.15074009074119465,"ci_98_lower":0.06358376875866521,"ci_98_upper":0.14708236493087695,"ci_95_lower":0.06874493771572827,"ci_95_upper":0.1356590869065361,"ci_90_lower":0.07277646924415274,"ci_90_upper":0.13133211067253525,"ci_75_lower":0.0797444470495098,"ci_75_upper":0.12100161281282228,"ci_50_lower":0.08748994485709605,"ci_50_upper":0.11088507132792964}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low","minimal","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Thinking Machines: Inkling Small","provider_name":"Thinkingmachines","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":4.5e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-30T20:25:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T21:07:01.662Z"},{"model_slug":"thinkingmachines/inkling-small","reasoning_mode":"high","overall_metrics":{"auc_1m":0.04861049588859343,"auc_128k":0.21902648410746187,"cum_avg_1m":0.22164763690600764,"cum_avg_128k":0.3395323552876868,"avg_score_overall":0.26837980577126147,"avg_response_time_ms":50323.36869140625,"total_prompt_cost":1.6417709999999999,"total_cache_read_cost":5.669222399999996,"total_completion_cost":2.233314,"total_reasoning_cost":2.0995596,"total_prompt_cost_128k":1.39607685,"total_completion_cost_128k":1.7750207999999998,"total_on_demand_input_cost":27.15327180000005,"total_on_demand_input_cost_128k":10.826003250000001,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9812376596004895},"bin_metrics":{"8192":{"avg_score":0.6835763483832125,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.45878899162793496,"median_score":1,"q1_score":0.03581077978014946,"q3_score":1,"avg_token_efficiency":0.9771471687576276,"is_incomplete":false,"total_prompt_cost_bin":0.049143599999999954,"total_cache_read_cost_bin":0.041062400000000006,"total_completion_cost_bin":0.2771963999999999,"total_reasoning_cost_bin":0.2687868,"total_cost_bin":0.6361891999999998,"total_on_demand_input_cost_bin":0.2339244000000001,"ci_99_lower":0.5379239929805335,"ci_99_upper":0.8057401237777085,"ci_98_lower":0.5611611337828216,"ci_98_upper":0.7932710297736338,"ci_95_lower":0.5859193106919701,"ci_95_upper":0.7806358378566802,"ci_90_lower":0.5984302833775291,"ci_90_upper":0.7571485508806653,"ci_75_lower":0.6231848177936627,"ci_75_upper":0.7435150772544147,"ci_50_lower":0.6473605563398451,"ci_50_upper":0.7199560160530383},"16384":{"avg_score":0.42991692817595323,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.4806566848068119,"median_score":0.04315559193491936,"q1_score":0.025893906131386757,"q3_score":1,"avg_token_efficiency":0.9676648473384528,"is_incomplete":false,"total_prompt_cost_bin":0.036212400000000006,"total_cache_read_cost_bin":0.09858560000000005,"total_completion_cost_bin":0.4010856,"total_reasoning_cost_bin":0.386418,"total_cost_bin":0.9223015999999999,"total_on_demand_input_cost_bin":0.47984759999999965,"ci_99_lower":0.28405827145905643,"ci_99_upper":0.5629636250263721,"ci_98_lower":0.2964041935536661,"ci_98_upper":0.5515758936242201,"ci_95_lower":0.3182995741645573,"ci_95_upper":0.5387863584585284,"ci_90_lower":0.3340764140443935,"ci_90_upper":0.516149771795026,"ci_75_lower":0.3685446423591202,"ci_75_upper":0.490619591500581,"ci_50_lower":0.3933021485245262,"ci_50_upper":0.46652062746361483},"32768":{"avg_score":0.31245309597237364,"min_score":0.008937438018620014,"max_score":1,"n_tests":136,"stddev":0.4378645730934435,"median_score":0.039273496717214584,"q1_score":0.023883139714598656,"q3_score":1,"avg_token_efficiency":0.9954973496148877,"is_incomplete":false,"total_prompt_cost_bin":0.08041679999999998,"total_cache_read_cost_bin":0.3682304000000004,"total_completion_cost_bin":0.4612392000000001,"total_reasoning_cost_bin":0.4315872,"total_cost_bin":1.3414736000000005,"total_on_demand_input_cost_bin":1.7374535999999994,"ci_99_lower":0.22321877784475766,"ci_99_upper":0.40362826569156496,"ci_98_lower":0.22619848303825538,"ci_98_upper":0.39684511758695246,"ci_95_lower":0.23983403143236715,"ci_95_upper":0.382312002618667,"ci_90_lower":0.2505417812639601,"ci_90_upper":0.37152218141435533,"ci_75_lower":0.26895821602289177,"ci_75_upper":0.355618548409171,"ci_50_lower":0.28547911287408645,"ci_50_upper":0.3374271890035529},"65536":{"avg_score":0.13725639580584625,"min_score":0.0047846888191998005,"max_score":1,"n_tests":85,"stddev":0.30023003927417596,"median_score":0.024930043146014214,"q1_score":0.0190400630235672,"q3_score":0.04565303772687912,"avg_token_efficiency":0.9777658874942426,"is_incomplete":false,"total_prompt_cost_bin":0.15952185,"total_cache_read_cost_bin":0.5081855999999996,"total_completion_cost_bin":0.2796816,"total_reasoning_cost_bin":0.2616851999999999,"total_cost_bin":1.2090742499999994,"total_on_demand_input_cost_bin":2.44635705,"ci_99_lower":0.06519663637030103,"ci_99_upper":0.23249878796483117,"ci_98_lower":0.07378441305781769,"ci_98_upper":0.22220398287880505,"ci_95_lower":0.07862321220160297,"ci_95_upper":0.20718843985088717,"ci_90_lower":0.08768205010496517,"ci_90_upper":0.1957204818046268,"ci_75_lower":0.10021356973002721,"ci_75_upper":0.17277537687777483,"ci_50_lower":0.1126948349218925,"ci_50_upper":0.15726518222295185},"131072":{"avg_score":0.1344590081010484,"min_score":0.007801072672009468,"max_score":1,"n_tests":103,"stddev":0.28569445676641503,"median_score":0.029036687687039375,"q1_score":0.01991682592779398,"q3_score":0.04976000264286995,"avg_token_efficiency":0.9773333600768869,"is_incomplete":false,"total_prompt_cost_bin":1.0707822000000002,"total_cache_read_cost_bin":1.0794751999999994,"total_completion_cost_bin":0.355818,"total_reasoning_cost_bin":0.3406332,"total_cost_bin":2.8467086,"total_on_demand_input_cost_bin":5.928420600000003,"ci_99_lower":0.06813338661557333,"ci_99_upper":0.2135828948655681,"ci_98_lower":0.07652055555846073,"ci_98_upper":0.2063629905698658,"ci_95_lower":0.08533518663856192,"ci_95_upper":0.19006057606978358,"ci_90_lower":0.09295170045937005,"ci_90_upper":0.17750114587461124,"ci_75_lower":0.10395611048160681,"ci_75_upper":0.1652373514427505,"ci_50_lower":0.11490026027307783,"ci_50_upper":0.15008717479446487},"262144":{"avg_score":0.07551931880962712,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.15787631825003326,"median_score":0.035024695098400116,"q1_score":0.021088629961013794,"q3_score":0.05238828808069229,"avg_token_efficiency":0.9824503538302469,"is_incomplete":false,"total_prompt_cost_bin":0.24569414999999992,"total_cache_read_cost_bin":3.573683199999997,"total_completion_cost_bin":0.4582932,"total_reasoning_cost_bin":0.41044920000000007,"total_cost_bin":4.688119749999998,"total_on_demand_input_cost_bin":16.327268550000046,"ci_99_lower":0.047534248967413914,"ci_99_upper":0.11406246280723667,"ci_98_lower":0.048693795288460504,"ci_98_upper":0.1100424166526216,"ci_95_lower":0.05276789907560063,"ci_95_upper":0.10474846067612145,"ci_90_lower":0.05573126872339293,"ci_90_upper":0.09893013377086068,"ci_75_lower":0.06068322284672251,"ci_75_upper":0.09030421804211511,"ci_50_lower":0.066158014927175,"ci_50_upper":0.0835704435024688}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low","minimal","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Thinking Machines: Inkling Small","provider_name":"Thinkingmachines","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":4.5e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-30T20:25:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T14:24:19.492Z"},{"model_slug":"qwen/qwen3.5-9b","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.04800519065687109,"auc_128k":0.3244625007505014,"cum_avg_1m":0.2868187279528993,"cum_avg_128k":0.4589099647246389,"avg_score_overall":0.4327769967749492,"avg_response_time_ms":294653.7204105598,"total_prompt_cost":2.5066174000000006,"total_cache_read_cost":0,"total_completion_cost":0.832791,"total_reasoning_cost":0.8282172,"total_prompt_cost_128k":2.5066174000000006,"total_completion_cost_128k":0.832791,"total_on_demand_input_cost":2.5066174000000006,"total_on_demand_input_cost_128k":2.5066174000000006,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0215051978672283},"bin_metrics":{"8192":{"avg_score":0.9023171714274213,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.29131697346281016,"median_score":1,"q1_score":0.9995790868997574,"q3_score":1,"avg_token_efficiency":1.0224972703222286,"is_incomplete":false,"total_prompt_cost_bin":0.05433919999999998,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.07129125000000001,"total_reasoning_cost_bin":0.06838875000000001,"total_cost_bin":0.1940192,"total_on_demand_input_cost_bin":0.05433919999999998,"ci_99_lower":0.8150714404310566,"ci_99_upper":0.9749650001206901,"ci_98_lower":0.8167526630365756,"ci_98_upper":0.9643997414407786,"ci_95_lower":0.8293011336651397,"ci_95_upper":0.9613877750199753,"ci_90_lower":0.8421309065481182,"ci_90_upper":0.950663524827687,"ci_75_lower":0.8639430380862905,"ci_75_upper":0.9382182249682955,"ci_50_lower":0.878703261591727,"ci_50_upper":0.9254338925122284},"16384":{"avg_score":0.558786847817828,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.48716764317018485,"median_score":0.9880603849887848,"q1_score":0.024336533155292273,"q3_score":1,"avg_token_efficiency":1.0159704488335684,"is_incomplete":false,"total_prompt_cost_bin":0.11207039999999995,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.1263675,"total_reasoning_cost_bin":0.12174869999999997,"total_cost_bin":0.3601865999999999,"total_on_demand_input_cost_bin":0.11207039999999995,"ci_99_lower":0.4224308533218427,"ci_99_upper":0.7031872245604608,"ci_98_lower":0.42554213873657865,"ci_98_upper":0.690792792549415,"ci_95_lower":0.45077336735936113,"ci_95_upper":0.6672592214420728,"ci_90_lower":0.47233658780940463,"ci_90_upper":0.6449950745498063,"ci_75_lower":0.4981325663782627,"ci_75_upper":0.6210797771305807,"ci_50_lower":0.522153368411091,"ci_50_upper":0.5958176649000961},"32768":{"avg_score":0.3718819156364428,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.45825864502012964,"median_score":0.04567169398069382,"q1_score":0.025824857410043478,"q3_score":0.9998592287302017,"avg_token_efficiency":1.0256830410679838,"is_incomplete":false,"total_prompt_cost_bin":0.39766080000000015,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.25305104999999994,"total_reasoning_cost_bin":0.2524140000000001,"total_cost_bin":0.9031258500000001,"total_on_demand_input_cost_bin":0.39766080000000015,"ci_99_lower":0.27076765820962684,"ci_99_upper":0.47304362572193737,"ci_98_lower":0.28255563202879036,"ci_98_upper":0.4658432247637949,"ci_95_lower":0.2960462979599149,"ci_95_upper":0.44733174798448627,"ci_90_lower":0.30711684553808644,"ci_90_upper":0.4385587568228777,"ci_75_lower":0.3256515449962428,"ci_75_upper":0.417855064598638,"ci_50_lower":0.34369459694854043,"ci_50_upper":0.39778508825368},"65536":{"avg_score":0.3078486759708646,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4411282385778763,"median_score":0.027194561436772346,"q1_score":0.02056148648262024,"q3_score":0.9917770624160767,"avg_token_efficiency":1.0209598255175443,"is_incomplete":false,"total_prompt_cost_bin":0.5676505999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.16629285,"total_reasoning_cost_bin":0.16684139999999995,"total_cost_bin":0.9007848499999999,"total_on_demand_input_cost_bin":0.5676505999999999,"ci_99_lower":0.195126730110688,"ci_99_upper":0.43727391683841665,"ci_98_lower":0.19976467950926982,"ci_98_upper":0.4260218113317866,"ci_95_lower":0.21671950577347376,"ci_95_upper":0.4033205075430519,"ci_90_lower":0.23356576247688599,"ci_90_upper":0.3835765385252478,"ci_75_lower":0.2515447688275291,"ci_75_upper":0.35990799746736335,"ci_50_lower":0.2725717574467554,"ci_50_upper":0.34000153128222066},"131072":{"avg_score":0.15371521277063824,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.3274375465421726,"median_score":0.027335872873663902,"q1_score":0.017466280609369278,"q3_score":0.03800334967672825,"avg_token_efficiency":1.0199671794275498,"is_incomplete":false,"total_prompt_cost_bin":1.3748964000000008,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.21578835000000005,"total_reasoning_cost_bin":0.21882435,"total_cost_bin":1.8095091000000008,"total_on_demand_input_cost_bin":1.3748964000000008,"ci_99_lower":0.07106796143569723,"ci_99_upper":0.24081399345567034,"ci_98_lower":0.08158109325739257,"ci_98_upper":0.2344567311944856,"ci_95_lower":0.0965873490750012,"ci_95_upper":0.2169892540156335,"ci_90_lower":0.10689943166341162,"ci_90_upper":0.20669419023985927,"ci_75_lower":0.11721500577359885,"ci_75_upper":0.18858511917444526,"ci_50_lower":0.13023629370688017,"ci_50_upper":0.17224692747231973}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-9B","provider_name":"Qwen","run_provider_names":["Together"],"pricing_metadata":{"prompt":1e-7,"completion":1.5e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-10T14:19:56.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T09:22:00.859Z"},{"model_slug":"qwen/qwen3.5-35b-a3b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.04758685365296632,"auc_128k":0.2944941469785024,"cum_avg_1m":0.2129150202898319,"cum_avg_128k":0.340664032463731,"avg_score_overall":0.3310187449500788,"avg_response_time_ms":9247.09637652941,"total_prompt_cost":3.502331279999998,"total_cache_read_cost":0,"total_completion_cost":0.193964,"total_reasoning_cost":0,"total_prompt_cost_128k":3.502331279999998,"total_completion_cost_128k":0.193964,"total_on_demand_input_cost":3.502331279999998,"total_on_demand_input_cost_128k":3.502331279999998,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0208128870566133},"bin_metrics":{"8192":{"avg_score":0.46904209938948044,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.478300604883371,"median_score":0.10837116464972496,"q1_score":0.028159708715975285,"q3_score":0.9980257451534271,"avg_token_efficiency":1.0228079512716763,"is_incomplete":false,"total_prompt_cost_bin":0.07609727999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.02820599999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.10430327999999996,"total_on_demand_input_cost_bin":0.07609727999999996,"ci_99_lower":0.3253739646324539,"ci_99_upper":0.598985981055972,"ci_98_lower":0.34184651374892566,"ci_98_upper":0.5864352538146195,"ci_95_lower":0.35807319048501085,"ci_95_upper":0.573730833982263,"ci_90_lower":0.3742639513942413,"ci_90_upper":0.5560988354272558,"ci_75_lower":0.4085541264234053,"ci_75_upper":0.5308017254581501,"ci_50_lower":0.4326515806140378,"ci_50_upper":0.5050623203293072},"16384":{"avg_score":0.41421971913950983,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.4750361989269464,"median_score":0.03895406424999237,"q1_score":0.025606202892959118,"q3_score":0.9949467331171036,"avg_token_efficiency":1.0161194758384635,"is_incomplete":false,"total_prompt_cost_bin":0.15692095999999997,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.03297499999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.18989595999999997,"total_on_demand_input_cost_bin":0.15692095999999997,"ci_99_lower":0.2797983958723635,"ci_99_upper":0.5465375183444703,"ci_98_lower":0.2827173704308807,"ci_98_upper":0.5349070334789285,"ci_95_lower":0.30389434940625504,"ci_95_upper":0.5222157248142321,"ci_90_lower":0.3183731301577063,"ci_90_upper":0.5093024626192346,"ci_75_lower":0.3537977156258421,"ci_75_upper":0.47430433086155976,"ci_50_lower":0.3782150033162907,"ci_50_upper":0.44999829306179884},"32768":{"avg_score":0.3174196447173873,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.43576717476370525,"median_score":0.039671579375863075,"q1_score":0.023628925438970327,"q3_score":0.9740709364414215,"avg_token_efficiency":1.0257548592579657,"is_incomplete":false,"total_prompt_cost_bin":0.5567631999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.05441500000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.6111781999999999,"total_on_demand_input_cost_bin":0.5567631999999999,"ci_99_lower":0.2280156166728942,"ci_99_upper":0.4094704338933875,"ci_98_lower":0.23484446327147238,"ci_98_upper":0.4018411718725453,"ci_95_lower":0.2447853208050005,"ci_95_upper":0.388263757797583,"ci_90_lower":0.25386350237719907,"ci_90_upper":0.37755819318348693,"ci_75_lower":0.2741716093598156,"ci_75_upper":0.36075523965187395,"ci_50_lower":0.2917576850150534,"ci_50_upper":0.3431234037292169},"65536":{"avg_score":0.29937392291612924,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.4362711640471456,"median_score":0.03155491128563881,"q1_score":0.021377671509981155,"q3_score":0.9758948087692261,"avg_token_efficiency":1.020990401222107,"is_incomplete":false,"total_prompt_cost_bin":0.7947346399999993,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.034351,"total_reasoning_cost_bin":0,"total_cost_bin":0.8290856399999993,"total_on_demand_input_cost_bin":0.7947346399999993,"ci_99_lower":0.18535586111405042,"ci_99_upper":0.4332085038910839,"ci_98_lower":0.19792027806241394,"ci_98_upper":0.4145659248306799,"ci_95_lower":0.20947620218085564,"ci_95_upper":0.3971376173261909,"ci_90_lower":0.22172852985410238,"ci_90_upper":0.37684620396836715,"ci_75_lower":0.24319739829344839,"ci_75_upper":0.3532276090772083,"ci_50_lower":0.2647792669204886,"ci_50_upper":0.33127367258455387},"131072":{"avg_score":0.2032647761561483,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.37313838055875276,"median_score":0.02963796630501747,"q1_score":0.018700744956731796,"q3_score":0.04356520250439644,"avg_token_efficiency":1.0162368757633713,"is_incomplete":false,"total_prompt_cost_bin":1.9178151999999993,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.044016999999999994,"total_reasoning_cost_bin":0,"total_cost_bin":1.9618321999999992,"total_on_demand_input_cost_bin":1.9178151999999993,"ci_99_lower":0.11704016186446134,"ci_99_upper":0.29723159479354827,"ci_98_lower":0.12676059591000702,"ci_98_upper":0.2881143329270835,"ci_95_lower":0.1375404680124032,"ci_95_upper":0.2751589591415979,"ci_90_lower":0.1474007043230248,"ci_90_upper":0.26094186005557807,"ci_75_lower":0.16360486571188762,"ci_75_upper":0.24111458603903035,"ci_50_lower":0.17672733892234566,"ci_50_upper":0.2235343255423624}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-35B-A3B","provider_name":"Qwen","run_provider_names":["Alibaba Cloud Int."],"pricing_metadata":{"prompt":2.5e-7,"completion":0.00000125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-02-25T21:10:22.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T05:26:43.964Z"},{"model_slug":"nvidia/nemotron-3-super-120b-a12b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.045132716191829875,"auc_128k":0.24948332072464333,"cum_avg_1m":0.18285283528694177,"cum_avg_128k":0.29256453645910685,"avg_score_overall":0.2929865854428138,"avg_response_time_ms":16456.76164434764,"total_prompt_cost":2.132123339999999,"total_cache_read_cost":0,"total_completion_cost":0.1353956,"total_reasoning_cost":0,"total_prompt_cost_128k":2.132123339999999,"total_completion_cost_128k":0.1353956,"total_on_demand_input_cost":2.132123339999999,"total_on_demand_input_cost_128k":2.132123339999999,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0244992765213963},"bin_metrics":{"8192":{"avg_score":0.4208709577273112,"min_score":0.006942034233361483,"max_score":1,"n_tests":80,"stddev":0.4714482016693867,"median_score":0.044361695647239685,"q1_score":0.025039538275450468,"q3_score":1,"avg_token_efficiency":1.025082416911607,"is_incomplete":false,"total_prompt_cost_bin":0.04630391999999999,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0209216,"total_reasoning_cost_bin":0,"total_cost_bin":0.06722551999999998,"total_on_demand_input_cost_bin":0.04630391999999999,"ci_99_lower":0.2841195150420826,"ci_99_upper":0.5511684269660618,"ci_98_lower":0.2914633620507666,"ci_98_upper":0.5432741595292463,"ci_95_lower":0.3101066342050035,"ci_95_upper":0.5260486157739069,"ci_90_lower":0.32751931488892294,"ci_90_upper":0.5158317853556945,"ci_75_lower":0.35901686793586124,"ci_75_upper":0.48133294245490105,"ci_50_lower":0.38570711982029027,"ci_50_upper":0.4564358776508015},"16384":{"avg_score":0.2783937156200409,"min_score":0.003485838882625103,"max_score":1,"n_tests":80,"stddev":0.41358585159671374,"median_score":0.03982326574623585,"q1_score":0.020906470715999603,"q3_score":0.4840626046061516,"avg_token_efficiency":1.015457976557726,"is_incomplete":false,"total_prompt_cost_bin":0.09509256000000003,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.014507999999999997,"total_reasoning_cost_bin":0,"total_cost_bin":0.10960056000000003,"total_on_demand_input_cost_bin":0.09509256000000003,"ci_99_lower":0.17520145512279123,"ci_99_upper":0.3991768216821365,"ci_98_lower":0.17941651719168292,"ci_98_upper":0.38574510962539355,"ci_95_lower":0.19009262640407543,"ci_95_upper":0.36825437282386697,"ci_90_lower":0.20708224591566252,"ci_90_upper":0.350539234834141,"ci_75_lower":0.2261740816727979,"ci_75_upper":0.32818716097681316,"ci_50_lower":0.24798004969779866,"ci_50_upper":0.30637160791957285},"32768":{"avg_score":0.32324100291332475,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4366914729187624,"median_score":0.0431128665804863,"q1_score":0.02176587749272585,"q3_score":0.9955146163702011,"avg_token_efficiency":1.036933517803116,"is_incomplete":false,"total_prompt_cost_bin":0.3417204000000001,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.050575599999999984,"total_reasoning_cost_bin":0,"total_cost_bin":0.3922960000000001,"total_on_demand_input_cost_bin":0.3417204000000001,"ci_99_lower":0.23418271510417124,"ci_99_upper":0.4123524533167074,"ci_98_lower":0.2389383311630723,"ci_98_upper":0.40934962232446515,"ci_95_lower":0.2488268896646118,"ci_95_upper":0.39544949086457026,"ci_90_lower":0.2600756876070912,"ci_90_upper":0.3830960865234802,"ci_75_lower":0.27929792135236153,"ci_75_upper":0.3671481388835887,"ci_50_lower":0.29737481672975047,"ci_50_upper":0.34879260188287897},"65536":{"avg_score":0.19161636284826433,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.36237300988233206,"median_score":0.025072885677218437,"q1_score":0.01617562025785446,"q3_score":0.050428904592990875,"avg_token_efficiency":1.0197200706747458,"is_incomplete":false,"total_prompt_cost_bin":0.4819171049999997,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.028716800000000008,"total_reasoning_cost_bin":0,"total_cost_bin":0.5106339049999997,"total_on_demand_input_cost_bin":0.4819171049999997,"ci_99_lower":0.09988171769930598,"ci_99_upper":0.3050196490063168,"ci_98_lower":0.1086536467252847,"ci_98_upper":0.2894043808712021,"ci_95_lower":0.12116933844764444,"ci_95_upper":0.27159850703963234,"ci_90_lower":0.13154905794243166,"ci_90_upper":0.2576882650423795,"ci_75_lower":0.1462706984815133,"ci_75_upper":0.23611355309703333,"ci_50_lower":0.16440699219922808,"ci_50_upper":0.21312620322055675},"131072":{"avg_score":0.24870064318659307,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.4003698694598392,"median_score":0.03458010032773018,"q1_score":0.02184957917779684,"q3_score":0.08980288729071617,"avg_token_efficiency":1.018594698351767,"is_incomplete":false,"total_prompt_cost_bin":1.1670893549999992,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.020673600000000007,"total_reasoning_cost_bin":0,"total_cost_bin":1.1877629549999993,"total_on_demand_input_cost_bin":1.1670893549999992,"ci_99_lower":0.15761394439323906,"ci_99_upper":0.34784145142151796,"ci_98_lower":0.1639233603837529,"ci_98_upper":0.3406453255026656,"ci_95_lower":0.17697882072884788,"ci_95_upper":0.32279664014084086,"ci_90_lower":0.18729655368590403,"ci_90_upper":0.31207556785332957,"ci_75_lower":0.2030518143480187,"ci_75_upper":0.29083002630075283,"ci_50_lower":0.22244940293236246,"ci_50_upper":0.2734817407140282}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1000000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"NVIDIA: Nemotron 3 Super","provider_name":"Nvidia","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":8.5e-8,"completion":4e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-11T16:07:19.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T06:12:51.796Z"},{"model_slug":"thinkingmachines/inkling-small","reasoning_mode":"medium","overall_metrics":{"auc_1m":0.04496095674151622,"auc_128k":0.2080134870416825,"cum_avg_1m":0.20309781551270684,"cum_avg_128k":0.3085192772857648,"avg_score_overall":0.24610551590826363,"avg_response_time_ms":23227.32086640625,"total_prompt_cost":0.29035979999999995,"total_cache_read_cost":5.969535999999998,"total_completion_cost":1.7441004000000004,"total_reasoning_cost":1.6035960000000005,"total_prompt_cost_128k":0.17173125,"total_completion_cost_128k":1.2590412000000004,"total_on_demand_input_cost":27.15327180000005,"total_on_demand_input_cost_128k":10.826003250000003,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9812376596004895},"bin_metrics":{"8192":{"avg_score":0.5654343100031838,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4837327008096122,"median_score":1,"q1_score":0.026766886468976736,"q3_score":1,"avg_token_efficiency":0.9771471687576275,"is_incomplete":false,"total_prompt_cost_bin":0.0193068,"total_cache_read_cost_bin":0.04769279999999999,"total_completion_cost_bin":0.14731799999999995,"total_reasoning_cost_bin":0.12826079999999995,"total_cost_bin":0.3425783999999999,"total_on_demand_input_cost_bin":0.23392439999999995,"ci_99_lower":0.4293190862117917,"ci_99_upper":0.7078516912895138,"ci_98_lower":0.4354747577126836,"ci_98_upper":0.6960993690220639,"ci_95_lower":0.46352862727857425,"ci_95_upper":0.671769268422504,"ci_90_lower":0.47847838373621926,"ci_90_upper":0.6515420452586841,"ci_75_lower":0.5050347076947219,"ci_75_upper":0.6286133244691883,"ci_50_lower":0.528363841807004,"ci_50_upper":0.6014082618494285},"16384":{"avg_score":0.444961468805559,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.48114847801946786,"median_score":0.04288073442876339,"q1_score":0.025906750466674566,"q3_score":1,"avg_token_efficiency":0.9676648473384521,"is_incomplete":false,"total_prompt_cost_bin":0.019335600000000015,"total_cache_read_cost_bin":0.10233600000000007,"total_completion_cost_bin":0.1557648000000001,"total_reasoning_cost_bin":0.13192200000000004,"total_cost_bin":0.4093584000000002,"total_on_demand_input_cost_bin":0.4798475999999997,"ci_99_lower":0.2996162320697913,"ci_99_upper":0.575083313295443,"ci_98_lower":0.31488856172154195,"ci_98_upper":0.5652177502493141,"ci_95_lower":0.3323314044563449,"ci_95_upper":0.5514859404313028,"ci_90_lower":0.3485100224270718,"ci_90_upper":0.5362862414505798,"ci_75_lower":0.3812050943291979,"ci_75_upper":0.5057666355947731,"ci_50_lower":0.4083458978187991,"ci_50_upper":0.4804945037700236},"32768":{"avg_score":0.2896059080684448,"min_score":0.004637680947780609,"max_score":1,"n_tests":136,"stddev":0.4219081685213428,"median_score":0.035354310646653175,"q1_score":0.02316714683547616,"q3_score":0.9996526092290878,"avg_token_efficiency":0.9954973496148877,"is_incomplete":false,"total_prompt_cost_bin":0.034221600000000005,"total_cache_read_cost_bin":0.37849600000000044,"total_completion_cost_bin":0.3162000000000002,"total_reasoning_cost_bin":0.27444840000000004,"total_cost_bin":1.0033660000000006,"total_on_demand_input_cost_bin":1.7374535999999996,"ci_99_lower":0.20120417551805184,"ci_99_upper":0.37719561896458104,"ci_98_lower":0.20659961627159432,"ci_98_upper":0.3712352267237079,"ci_95_lower":0.2205269455533091,"ci_95_upper":0.3607464816339095,"ci_90_lower":0.23008588492212928,"ci_90_upper":0.34750010948525467,"ci_75_lower":0.24731225457542358,"ci_75_upper":0.3317806881155668,"ci_50_lower":0.2657904701721717,"ci_50_upper":0.3133685479149469},"65536":{"avg_score":0.16542321250171346,"min_score":0.00469667324796319,"max_score":1,"n_tests":85,"stddev":0.33359048560670845,"median_score":0.025562994182109833,"q1_score":0.017785683274269104,"q3_score":0.04391328617930412,"avg_token_efficiency":0.977765887494243,"is_incomplete":false,"total_prompt_cost_bin":0.031995449999999974,"total_cache_read_cost_bin":0.5365247999999997,"total_completion_cost_bin":0.27135479999999995,"total_reasoning_cost_bin":0.24996960000000001,"total_cost_bin":1.0898446499999996,"total_on_demand_input_cost_bin":2.446357049999999,"ci_99_lower":0.08384894574354129,"ci_99_upper":0.2687246788874926,"ci_98_lower":0.08647519694520708,"ci_98_upper":0.2570750750293407,"ci_95_lower":0.10087102912763571,"ci_95_upper":0.24116998632520653,"ci_90_lower":0.11098051037118933,"ci_90_upper":0.22852143166614147,"ci_75_lower":0.1229436401359956,"ci_75_upper":0.20505322168704454,"ci_50_lower":0.13871128128786736,"ci_50_upper":0.18669655243318312},"131072":{"avg_score":0.077171487049923,"min_score":0.0015278839273378253,"max_score":1,"n_tests":103,"stddev":0.18479547269064295,"median_score":0.029818955808877945,"q1_score":0.020676342770457268,"q3_score":0.043398117646574974,"avg_token_efficiency":0.977333360076887,"is_incomplete":false,"total_prompt_cost_bin":0.06687180000000001,"total_cache_read_cost_bin":1.3025664000000006,"total_completion_cost_bin":0.36840360000000005,"total_reasoning_cost_bin":0.3600912000000001,"total_cost_bin":2.0979330000000007,"total_on_demand_input_cost_bin":5.928420600000004,"ci_99_lower":0.03946525229316304,"ci_99_upper":0.13141026246224108,"ci_98_lower":0.04107756596605646,"ci_98_upper":0.12351298063628384,"ci_95_lower":0.04577455673625611,"ci_95_upper":0.11341459263275486,"ci_90_lower":0.049844700078597495,"ci_90_upper":0.10810662719592887,"ci_75_lower":0.05654240610361669,"ci_75_upper":0.09761826030190049,"ci_50_lower":0.06364637808489965,"ci_50_upper":0.08790612314574564},"262144":{"avg_score":0.08218613767283077,"min_score":0.001715108286589384,"max_score":1,"n_tests":141,"stddev":0.18022091630856749,"median_score":0.03119015134871006,"q1_score":0.020706890150904655,"q3_score":0.0517241396009922,"avg_token_efficiency":0.9824503538302469,"is_incomplete":false,"total_prompt_cost_bin":0.11862854999999996,"total_cache_read_cost_bin":3.6019199999999967,"total_completion_cost_bin":0.4850592,"total_reasoning_cost_bin":0.45890400000000037,"total_cost_bin":4.664511749999997,"total_on_demand_input_cost_bin":16.327268550000046,"ci_99_lower":0.04945608483777705,"ci_99_upper":0.12347736907216343,"ci_98_lower":0.05129848506470072,"ci_98_upper":0.1217445638509639,"ci_95_lower":0.056366106867707655,"ci_95_upper":0.11562145776140127,"ci_90_lower":0.05924950384779285,"ci_90_upper":0.10838430389804857,"ci_75_lower":0.06499222852289677,"ci_75_upper":0.09940945158724124,"ci_50_lower":0.07148300366416732,"ci_50_upper":0.0916015807166035}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low","minimal","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Thinking Machines: Inkling Small","provider_name":"Thinkingmachines","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":4.5e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-30T20:25:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T18:05:33.997Z"},{"model_slug":"thinkingmachines/inkling-small","reasoning_mode":"low","overall_metrics":{"auc_1m":0.04472693705422853,"auc_128k":0.20453462451645532,"cum_avg_1m":0.19106070041239917,"cum_avg_128k":0.28935374189542656,"avg_score_overall":0.2289110279917717,"avg_response_time_ms":28142.737895703125,"total_prompt_cost":0.2960045999999998,"total_cache_read_cost":5.968281599999997,"total_completion_cost":1.7248884,"total_reasoning_cost":1.5096744,"total_prompt_cost_128k":0.17737604999999984,"total_completion_cost_128k":1.2314268,"total_on_demand_input_cost":27.153271800000045,"total_on_demand_input_cost_128k":10.826003250000003,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9812376596004894},"bin_metrics":{"8192":{"avg_score":0.5426528741954826,"min_score":0.009532888419926167,"max_score":1,"n_tests":80,"stddev":0.48466899714858247,"median_score":1,"q1_score":0.02747252769768238,"q3_score":1,"avg_token_efficiency":0.9771471687576276,"is_incomplete":false,"total_prompt_cost_bin":0.0249516,"total_cache_read_cost_bin":0.04643839999999998,"total_completion_cost_bin":0.0878028,"total_reasoning_cost_bin":0.05869080000000001,"total_cost_bin":0.21788359999999998,"total_on_demand_input_cost_bin":0.23392439999999998,"ci_99_lower":0.40734640719147863,"ci_99_upper":0.6765060822265222,"ci_98_lower":0.41363411574903874,"ci_98_upper":0.6651278968353291,"ci_95_lower":0.42949915991164744,"ci_95_upper":0.6479185459404834,"ci_90_lower":0.45427921381895436,"ci_90_upper":0.6301540594734251,"ci_75_lower":0.4818870002723997,"ci_75_upper":0.6045505471643992,"ci_50_lower":0.506186076803715,"ci_50_upper":0.5788730245927581},"16384":{"avg_score":0.39372525734361263,"min_score":0.002871500328183174,"max_score":1,"n_tests":80,"stddev":0.4649746237422235,"median_score":0.04824812524020672,"q1_score":0.02565621118992567,"q3_score":1,"avg_token_efficiency":0.9676648473384526,"is_incomplete":false,"total_prompt_cost_bin":0.0193356,"total_cache_read_cost_bin":0.10233600000000002,"total_completion_cost_bin":0.11559720000000004,"total_reasoning_cost_bin":0.0875568,"total_cost_bin":0.32482560000000005,"total_on_demand_input_cost_bin":0.47984759999999976,"ci_99_lower":0.2619302746246685,"ci_99_upper":0.5250315075991675,"ci_98_lower":0.2706775998211233,"ci_98_upper":0.5148704559860053,"ci_95_lower":0.287355556631228,"ci_95_upper":0.49539065302727975,"ci_90_lower":0.30146804069052446,"ci_90_upper":0.48214517008862456,"ci_75_lower":0.3349395811092109,"ci_75_upper":0.45585100129246714,"ci_50_lower":0.35816742267343216,"ci_50_upper":0.42772273734444755},"32768":{"avg_score":0.23115479213062345,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.38464701673962803,"median_score":0.03537116199731827,"q1_score":0.023213544860482216,"q3_score":0.09764880128204823,"avg_token_efficiency":0.9954973496148869,"is_incomplete":false,"total_prompt_cost_bin":0.03422160000000001,"total_cache_read_cost_bin":0.37849600000000044,"total_completion_cost_bin":0.33212280000000005,"total_reasoning_cost_bin":0.292908,"total_cost_bin":1.0377484000000006,"total_on_demand_input_cost_bin":1.7374535999999996,"ci_99_lower":0.15203370205448558,"ci_99_upper":0.3136650619838175,"ci_98_lower":0.15837794134434421,"ci_98_upper":0.3073575500501936,"ci_95_lower":0.16819541325813214,"ci_95_upper":0.2931970770287615,"ci_90_lower":0.17755338654165184,"ci_90_upper":0.28515794491737745,"ci_75_lower":0.1933589154128533,"ci_75_upper":0.26788323023399296,"ci_50_lower":0.2074592228146845,"ci_50_upper":0.2510960790695047},"65536":{"avg_score":0.19784876250607125,"min_score":0.0023023791145533323,"max_score":1,"n_tests":85,"stddev":0.3677614290917889,"median_score":0.02599858120083809,"q1_score":0.017658187076449394,"q3_score":0.05198776721954346,"avg_token_efficiency":0.977765887494243,"is_incomplete":false,"total_prompt_cost_bin":0.031995449999999974,"total_cache_read_cost_bin":0.5365247999999997,"total_completion_cost_bin":0.3556439999999999,"total_reasoning_cost_bin":0.3423912,"total_cost_bin":1.2665554499999996,"total_on_demand_input_cost_bin":2.4463570499999987,"ci_99_lower":0.10698228565813042,"ci_99_upper":0.3120021304391352,"ci_98_lower":0.11061303146986072,"ci_98_upper":0.2992068067475715,"ci_95_lower":0.1262162115329685,"ci_95_upper":0.27804186542614306,"ci_90_lower":0.1386223904302289,"ci_90_upper":0.26256609820144466,"ci_75_lower":0.15113052397398005,"ci_75_upper":0.24414895476862405,"ci_50_lower":0.17131814738215112,"ci_50_upper":0.22016296877306613},"131072":{"avg_score":0.08138702330134302,"min_score":0.004428978078067303,"max_score":1,"n_tests":103,"stddev":0.2044772178339728,"median_score":0.027062581852078438,"q1_score":0.0178461242467165,"q3_score":0.036906931549310684,"avg_token_efficiency":0.9773333600768869,"is_incomplete":false,"total_prompt_cost_bin":0.06687179999999984,"total_cache_read_cost_bin":1.3025664000000001,"total_completion_cost_bin":0.3402599999999999,"total_reasoning_cost_bin":0.3243384000000001,"total_cost_bin":2.0340366000000003,"total_on_demand_input_cost_bin":5.9284206000000035,"ci_99_lower":0.038381245581962126,"ci_99_upper":0.14273806949905615,"ci_98_lower":0.04014954635023492,"ci_98_upper":0.13106642425634438,"ci_95_lower":0.046354237949199864,"ci_95_upper":0.12056034135957727,"ci_90_lower":0.05086816864391084,"ci_90_upper":0.11428458308303747,"ci_75_lower":0.059185609285317325,"ci_75_upper":0.10325794987882332,"ci_50_lower":0.067386690514768,"ci_50_upper":0.0936460316789255},"262144":{"avg_score":0.0817168938220604,"min_score":0.005488474387675524,"max_score":1,"n_tests":141,"stddev":0.17423100771712816,"median_score":0.03489439934492111,"q1_score":0.02301255241036415,"q3_score":0.055216506123542786,"avg_token_efficiency":0.9824503538302468,"is_incomplete":false,"total_prompt_cost_bin":0.11862854999999996,"total_cache_read_cost_bin":3.6019199999999967,"total_completion_cost_bin":0.49346160000000006,"total_reasoning_cost_bin":0.40378919999999985,"total_cost_bin":4.617799349999997,"total_on_demand_input_cost_bin":16.327268550000042,"ci_99_lower":0.050357682717656914,"ci_99_upper":0.1232155274327164,"ci_98_lower":0.052010105922322795,"ci_98_upper":0.12029070238067589,"ci_95_lower":0.056463032060282976,"ci_95_upper":0.1138543138606639,"ci_90_lower":0.05981269286456683,"ci_90_upper":0.10730426600879944,"ci_75_lower":0.06539674973526495,"ci_75_upper":0.09834407026390711,"ci_50_lower":0.07138425099303458,"ci_50_upper":0.09100488820426986}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low","minimal","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Thinking Machines: Inkling Small","provider_name":"Thinkingmachines","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":4.5e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-30T20:25:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T19:04:29.274Z"},{"model_slug":"z-ai/glm-5.1","reasoning_mode":null,"overall_metrics":{"auc_1m":0.04341497601784093,"auc_128k":0.2631112674598829,"cum_avg_1m":0.1976088636283419,"cum_avg_128k":0.31617418180534707,"avg_score_overall":0.30294037801511814,"avg_response_time_ms":15112.066501081483,"total_prompt_cost":8.087856252000002,"total_cache_read_cost":2.8637751167999994,"total_completion_cost":0.57438084,"total_reasoning_cost":0,"total_prompt_cost_128k":8.087856252000002,"total_completion_cost_128k":0.57438084,"total_on_demand_input_cost":23.508183803999977,"total_on_demand_input_cost_128k":23.508183803999977,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9918943078224756},"bin_metrics":{"8192":{"avg_score":0.5060521877137945,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.48514163066072324,"median_score":0.11230244114995003,"q1_score":0.03047934966161847,"q3_score":1,"avg_token_efficiency":0.9860183556228543,"is_incomplete":false,"total_prompt_cost_bin":0.345039744,"total_cache_read_cost_bin":0.030024384,"total_completion_cost_bin":0.08342928000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.458493408,"total_on_demand_input_cost_bin":0.5067095040000001,"ci_99_lower":0.37139539796445753,"ci_99_upper":0.6390297750445897,"ci_98_lower":0.3817022185699316,"ci_98_upper":0.6258637334946543,"ci_95_lower":0.3952442949800752,"ci_95_upper":0.6053121748170814,"ci_90_lower":0.41029301277943886,"ci_90_upper":0.594149132888997,"ci_75_lower":0.44596986871911215,"ci_75_upper":0.5677654893544969,"ci_50_lower":0.4700780213897815,"ci_50_upper":0.5429021656280383},"16384":{"avg_score":0.35492193192476407,"min_score":0.008643042296171188,"max_score":1,"n_tests":80,"stddev":0.46343626359425366,"median_score":0.030895894393324852,"q1_score":0.02115301415324211,"q3_score":1,"avg_token_efficiency":0.9784005286431927,"is_incomplete":false,"total_prompt_cost_bin":0.5325905759999996,"total_cache_read_cost_bin":0.09448208639999998,"total_completion_cost_bin":0.08241829199999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.7094909543999995,"total_on_demand_input_cost_bin":1.0413402720000007,"ci_99_lower":0.2205806970818085,"ci_99_upper":0.4888378891487955,"ci_98_lower":0.23194251119054388,"ci_98_upper":0.4762488830001093,"ci_95_lower":0.25649628661427415,"ci_95_upper":0.4551451452405307,"ci_90_lower":0.2679779041028814,"ci_90_upper":0.43973812585521954,"ci_75_lower":0.2950098419532878,"ci_75_upper":0.41544720922684064,"ci_50_lower":0.3186152927810326,"ci_50_upper":0.39088336300628723},"32768":{"avg_score":0.25545416019432354,"min_score":0.003944773226976395,"max_score":1,"n_tests":136,"stddev":0.4038118884094812,"median_score":0.03888236917555332,"q1_score":0.023560710716992617,"q3_score":0.0907918568700552,"avg_token_efficiency":1.007393097590867,"is_incomplete":false,"total_prompt_cost_bin":1.203520080000001,"total_cache_read_cost_bin":0.4771867775999997,"total_completion_cost_bin":0.16242296399999995,"total_reasoning_cost_bin":0,"total_cost_bin":1.8431298216000005,"total_on_demand_input_cost_bin":3.7729873439999944,"ci_99_lower":0.1703888698466814,"ci_99_upper":0.3372053119812421,"ci_98_lower":0.1756770845702233,"ci_98_upper":0.33328085268700414,"ci_95_lower":0.19075399356515846,"ci_95_upper":0.32295838859795095,"ci_90_lower":0.19833696100446532,"ci_90_upper":0.3114309779145782,"ci_75_lower":0.21418265895344496,"ci_75_upper":0.2935933578795384,"ci_50_lower":0.23182586818310322,"ci_50_upper":0.27711456759938946},"65536":{"avg_score":0.26856351139790874,"min_score":0.00441988930106163,"max_score":1,"n_tests":85,"stddev":0.42166006777907616,"median_score":0.03145589679479599,"q1_score":0.02166847325861454,"q3_score":0.08787943422794342,"avg_token_efficiency":0.9892191866740662,"is_incomplete":false,"total_prompt_cost_bin":1.6289358540000014,"total_cache_read_cost_bin":0.6841885440000008,"total_completion_cost_bin":0.107189016,"total_reasoning_cost_bin":0,"total_cost_bin":2.420313414000002,"total_on_demand_input_cost_bin":5.313028013999994,"ci_99_lower":0.1649213218866464,"ci_99_upper":0.3861931936004035,"ci_98_lower":0.1693638083739754,"ci_98_upper":0.38111664413069096,"ci_95_lower":0.18789691693396035,"ci_95_upper":0.3604486028420027,"ci_90_lower":0.19945390972308813,"ci_90_upper":0.3392381412481119,"ci_75_lower":0.21193704643177197,"ci_75_upper":0.3150000996148104,"ci_50_lower":0.23459264160517385,"ci_50_upper":0.29389878676053793},"131072":{"avg_score":0.19587911779594436,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.37102964942046585,"median_score":0.02926829271018505,"q1_score":0.019432597793638706,"q3_score":0.040258072316646576,"avg_token_efficiency":0.988681962185834,"is_incomplete":false,"total_prompt_cost_bin":4.377769998000001,"total_cache_read_cost_bin":1.5778933247999989,"total_completion_cost_bin":0.13892128800000003,"total_reasoning_cost_bin":0,"total_cost_bin":6.094584610799999,"total_on_demand_input_cost_bin":12.874118669999985,"ci_99_lower":0.1107532032654571,"ci_99_upper":0.28892275694818736,"ci_98_lower":0.11811567918033547,"ci_98_upper":0.28212010700757467,"ci_95_lower":0.13044919831468046,"ci_95_upper":0.2631519951574499,"ci_90_lower":0.1391711606243987,"ci_90_upper":0.2530490035051028,"ci_75_lower":0.15729307872356252,"ci_75_upper":0.23395280624786166,"ci_50_lower":0.16941284613844315,"ci_50_upper":0.21566633166105587}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":204800,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Z.ai: GLM 5.1","provider_name":"Z-ai","run_provider_names":["Z.ai"],"pricing_metadata":{"prompt":0.00000126,"completion":0.00000396,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-07T16:07:05.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:57:17.454Z"},{"model_slug":"stepfun/step-3.7-flash","reasoning_mode":"high","overall_metrics":{"auc_1m":0.04297841593090876,"auc_128k":0.27812928514329477,"cum_avg_1m":0.22528957645144163,"cum_avg_128k":0.3604633223223066,"avg_score_overall":0.3532673549936499,"avg_response_time_ms":28680.945338729984,"total_prompt_cost":0.1934208,"total_cache_read_cost":0.9400467199999996,"total_completion_cost":2.3101280499999994,"total_reasoning_cost":1.2745910000000003,"total_prompt_cost_128k":0.1934208,"total_completion_cost_128k":2.3101280499999994,"total_on_demand_input_cost":4.893654400000001,"total_on_demand_input_cost_128k":4.893654400000001,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9982482351012699},"bin_metrics":{"8192":{"avg_score":0.6126137607905549,"min_score":0.00764818349853158,"max_score":1,"n_tests":80,"stddev":0.4773742076574266,"median_score":1,"q1_score":0.03541491273790598,"q3_score":1,"avg_token_efficiency":0.9933785930411834,"is_incomplete":false,"total_prompt_cost_bin":0.030884399999999996,"total_cache_read_cost_bin":0.01495296,"total_completion_cost_bin":0.26867105,"total_reasoning_cost_bin":0.05553465000000001,"total_cost_bin":0.37004306000000003,"total_on_demand_input_cost_bin":0.1056492,"ci_99_lower":0.4667158540447999,"ci_99_upper":0.7571717138636741,"ci_98_lower":0.480504634289071,"ci_98_upper":0.7327176440634648,"ci_95_lower":0.5040255704433366,"ci_95_upper":0.7100233313863283,"ci_90_lower":0.5271789681122755,"ci_90_upper":0.6972548879103851,"ci_75_lower":0.5519448484272289,"ci_75_upper":0.673263517156738,"ci_50_lower":0.5761617010197368,"ci_50_upper":0.6482901176423184},"16384":{"avg_score":0.395260666415561,"min_score":0.007741935551166534,"max_score":1,"n_tests":80,"stddev":0.4718325382631902,"median_score":0.03978479094803333,"q1_score":0.023252153303474188,"q3_score":1,"avg_token_efficiency":0.9902756304702761,"is_incomplete":false,"total_prompt_cost_bin":0.0156336,"total_cache_read_cost_bin":0.04049024,"total_completion_cost_bin":0.35356404999999985,"total_reasoning_cost_bin":0.32548335000000006,"total_cost_bin":0.7351712399999999,"total_on_demand_input_cost_bin":0.21808479999999997,"ci_99_lower":0.26165346216631585,"ci_99_upper":0.526821380853944,"ci_98_lower":0.27238343343790616,"ci_98_upper":0.5187690902817994,"ci_95_lower":0.28624459868879054,"ci_95_upper":0.4967121473251608,"ci_90_lower":0.3010275719343918,"ci_90_upper":0.4840560321364319,"ci_75_lower":0.33368682043364967,"ci_75_upper":0.45593138388794613,"ci_50_lower":0.3593601129425224,"ci_50_upper":0.43059861857036597},"32768":{"avg_score":0.39077727426090003,"min_score":0.005387931130826473,"max_score":1,"n_tests":136,"stddev":0.4634113329199247,"median_score":0.044119205325841904,"q1_score":0.02492425125092268,"q3_score":1,"avg_token_efficiency":1.0109295065279296,"is_incomplete":false,"total_prompt_cost_bin":0.02262800000000002,"total_cache_read_cost_bin":0.15213312,"total_completion_cost_bin":0.5142673499999998,"total_reasoning_cost_bin":0.5074628000000001,"total_cost_bin":1.1964912699999999,"total_on_demand_input_cost_bin":0.7832936,"ci_99_lower":0.2900260287382202,"ci_99_upper":0.496017813398537,"ci_98_lower":0.29913064395878736,"ci_98_upper":0.4850661019502235,"ci_95_lower":0.31239135962352843,"ci_95_upper":0.4664483904396869,"ci_90_lower":0.32423641339101933,"ci_90_upper":0.4547111014048021,"ci_75_lower":0.3446917070447371,"ci_75_upper":0.4367301607074777,"ci_50_lower":0.36202480104368395,"ci_50_upper":0.4184694681795972},"65536":{"avg_score":0.242874966885018,"min_score":0.004658385179936886,"max_score":1,"n_tests":85,"stddev":0.39678439598905946,"median_score":0.03412401303648949,"q1_score":0.021377671509981155,"q3_score":0.08787943422794342,"avg_token_efficiency":0.9950202971179805,"is_incomplete":false,"total_prompt_cost_bin":0.01398619999999999,"total_cache_read_cost_bin":0.21849408000000045,"total_completion_cost_bin":0.3758222999999998,"total_reasoning_cost_bin":0.3861102,"total_cost_bin":0.9944127800000002,"total_on_demand_input_cost_bin":1.106456599999999,"ci_99_lower":0.1482584250094643,"ci_99_upper":0.3587772888553,"ci_98_lower":0.1530282758608022,"ci_98_upper":0.3464980656619129,"ci_95_lower":0.16707813818199452,"ci_95_upper":0.32954045008484495,"ci_90_lower":0.1772005494623719,"ci_90_upper":0.31620568317443354,"ci_75_lower":0.19214302208548523,"ci_75_upper":0.2878284913323381,"ci_50_lower":0.21072236372787942,"ci_50_upper":0.2674677702255876},"131072":{"avg_score":0.16078994325949875,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.3245436923789685,"median_score":0.02888800948858261,"q1_score":0.017353895120322704,"q3_score":0.0446302704513073,"avg_token_efficiency":0.9941424249055446,"is_incomplete":false,"total_prompt_cost_bin":0.11028859999999999,"total_cache_read_cost_bin":0.513976319999999,"total_completion_cost_bin":0.7978033000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.422068219999999,"total_on_demand_input_cost_bin":2.6801702000000014,"ci_99_lower":0.08105355599962935,"ci_99_upper":0.24720631281804584,"ci_98_lower":0.09105162150918243,"ci_98_upper":0.2384375708730602,"ci_95_lower":0.103894445413918,"ci_95_upper":0.22287283324754065,"ci_90_lower":0.11396545243056184,"ci_90_upper":0.21170935108782996,"ci_75_lower":0.12503976026466296,"ci_75_upper":0.19424935933699816,"ci_50_lower":0.13786773609553546,"ci_50_upper":0.17836398483385124}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"StepFun: Step 3.7 Flash","provider_name":"Stepfun","run_provider_names":[],"pricing_metadata":{"prompt":2e-7,"completion":0.00000115,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-28T16:17:49.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-23T00:11:28.185Z"},{"model_slug":"bytedance-seed/seed-2-1-turbo","reasoning_mode":null,"overall_metrics":{"auc_1m":0.04287538718309595,"auc_128k":0.2752983425343833,"cum_avg_1m":0.22648775860429626,"cum_avg_128k":0.362380413766874,"avg_score_overall":0.34860589524120933,"avg_response_time_ms":10268.176626474376,"total_prompt_cost":0.7048274999999996,"total_cache_read_cost":10.448631999999995,"total_completion_cost":0.4898525,"total_reasoning_cost":0,"total_prompt_cost_128k":0.7048274999999996,"total_completion_cost_128k":0.4898525,"total_on_demand_input_cost":11.1534595,"total_on_demand_input_cost_128k":11.1534595,"total_runs":461,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0264383903943504},"bin_metrics":{"8192":{"avg_score":0.6394376818323508,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.46871996089700985,"median_score":1,"q1_score":0.038527097553014755,"q3_score":1,"avg_token_efficiency":1.035731028363502,"is_incomplete":false,"total_prompt_cost_bin":0.11127599999999997,"total_cache_read_cost_bin":0.16386000000000006,"total_completion_cost_bin":0.11238500000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.38752100000000006,"total_on_demand_input_cost_bin":0.275136,"ci_99_lower":0.49294139754929345,"ci_99_upper":0.7705593970348127,"ci_98_lower":0.5069931778286119,"ci_98_upper":0.7483620370789431,"ci_95_lower":0.532265510368743,"ci_95_upper":0.7353862638448482,"ci_90_lower":0.5533993412344717,"ci_90_upper":0.7231905211531557,"ci_75_lower":0.5785053596177022,"ci_75_upper":0.6991432418377371,"ci_50_lower":0.6034253866964718,"ci_50_upper":0.6741411777125904},"16384":{"avg_score":0.4671372759505175,"min_score":0.009212344884872437,"max_score":1,"n_tests":80,"stddev":0.485115098452181,"median_score":0.04705960489809513,"q1_score":0.027858505491167307,"q3_score":1,"avg_token_efficiency":1.0233316886218882,"is_incomplete":false,"total_prompt_cost_bin":0.11215199999999992,"total_cache_read_cost_bin":0.4521519999999995,"total_completion_cost_bin":0.06963999999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.6339439999999994,"total_on_demand_input_cost_bin":0.5643039999999998,"ci_99_lower":0.32152689285721864,"ci_99_upper":0.6000694910864696,"ci_98_lower":0.33581021942559164,"ci_98_upper":0.5878822877572383,"ci_95_lower":0.3569017030994291,"ci_95_upper":0.5749489097794868,"ci_90_lower":0.3703278464480536,"ci_90_upper":0.5529205554450163,"ci_75_lower":0.40625770433980507,"ci_75_upper":0.5284570284173241,"ci_50_lower":0.4309667217981769,"ci_50_upper":0.5035572090448113},"32768":{"avg_score":0.2646625081919517,"min_score":0.003944773226976395,"max_score":1,"n_tests":136,"stddev":0.41021103713350593,"median_score":0.03574427030980587,"q1_score":0.0236886627972126,"q3_score":0.10196165554225445,"avg_token_efficiency":1.0282613377564316,"is_incomplete":false,"total_prompt_cost_bin":0.200044,"total_cache_read_cost_bin":1.7933240000000024,"total_completion_cost_bin":0.13387000000000004,"total_reasoning_cost_bin":0,"total_cost_bin":2.1272380000000024,"total_on_demand_input_cost_bin":1.993367999999999,"ci_99_lower":0.1790235487190041,"ci_99_upper":0.3494454848698361,"ci_98_lower":0.18447925402279533,"ci_98_upper":0.34523624797262636,"ci_95_lower":0.19827556854194398,"ci_95_upper":0.3329687902727849,"ci_90_lower":0.2064360062336645,"ci_90_upper":0.3210979194923044,"ci_75_lower":0.22299821901063244,"ci_75_upper":0.30347327447987693,"ci_50_lower":0.2408865539875799,"ci_50_upper":0.28708813038161574},"65536":{"avg_score":0.2762022235798707,"min_score":0.00469667324796319,"max_score":1,"n_tests":81,"stddev":0.4226473414993267,"median_score":0.0301204826682806,"q1_score":0.02166847325861454,"q3_score":0.6397862434387207,"avg_token_efficiency":1.0233783365167566,"is_incomplete":false,"total_prompt_cost_bin":0.10308700000000004,"total_cache_read_cost_bin":2.608008000000001,"total_completion_cost_bin":0.08041500000000001,"total_reasoning_cost_bin":0,"total_cost_bin":2.791510000000001,"total_on_demand_input_cost_bin":2.711095,"ci_99_lower":0.16177172052239952,"ci_99_upper":0.4043196809250735,"ci_98_lower":0.17477261138361314,"ci_98_upper":0.376204792662972,"ci_95_lower":0.19019676368557104,"ci_95_upper":0.36498938188730035,"ci_90_lower":0.20607091875079972,"ci_90_upper":0.35123472608263706,"ci_75_lower":0.22673289302339844,"ci_75_upper":0.3282177769909555,"ci_50_lower":0.24304105430686224,"ci_50_upper":0.30588561213966226},"131072":{"avg_score":0.1644623792796795,"min_score":0.00445831473916769,"max_score":1,"n_tests":84,"stddev":0.34342108352862893,"median_score":0.028107796795666218,"q1_score":0.016149123199284077,"q3_score":0.038123237900435925,"avg_token_efficiency":1.0205463502408607,"is_incomplete":false,"total_prompt_cost_bin":0.17826849999999964,"total_cache_read_cost_bin":5.4312879999999915,"total_completion_cost_bin":0.0935425,"total_reasoning_cost_bin":0,"total_cost_bin":5.703098999999991,"total_on_demand_input_cost_bin":5.609556500000003,"ci_99_lower":0.0812817054907126,"ci_99_upper":0.25774391089632576,"ci_98_lower":0.08650971184074435,"ci_98_upper":0.2555994644219477,"ci_95_lower":0.09515684746771252,"ci_95_upper":0.24440903395727565,"ci_90_lower":0.1066306778551301,"ci_90_upper":0.2237347306071093,"ci_75_lower":0.11955786455794061,"ci_75_upper":0.20922215797084692,"ci_50_lower":0.1392119900265243,"ci_50_upper":0.18747176042441943}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"ByteDance Seed: Seed 2.1 Turbo","provider_name":"Bytedance-seed","run_provider_names":["Seed"],"pricing_metadata":{"prompt":5e-7,"completion":0.0000025,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-08-12T16:29:36.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T15:20:45.117Z"},{"model_slug":"poolside/laguna-s-2.1","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.04275892897614048,"auc_128k":0.11580521027792266,"cum_avg_1m":0.10553150481194455,"cum_avg_128k":0.15109212356602117,"avg_score_overall":0.10148476662858455,"avg_response_time_ms":24697.9380668968,"total_prompt_cost":0.27275920000000053,"total_cache_read_cost":1.8880003200000006,"total_completion_cost":0.07780300000000001,"total_reasoning_cost":0.00103,"total_prompt_cost_128k":0.026858399999999998,"total_completion_cost_128k":0.04627420000000001,"total_on_demand_input_cost":19.152762400000004,"total_on_demand_input_cost_128k":2.535552800000001,"total_runs":861,"total_candidates_generated":null,"avg_token_efficiency_overall":1.038063037974025},"bin_metrics":{"8192":{"avg_score":0.23818371354136617,"min_score":0.011167512275278568,"max_score":1,"n_tests":80,"stddev":0.38485386054569254,"median_score":0.04135696776211262,"q1_score":0.02590903313830495,"q3_score":0.12033736333251,"avg_token_efficiency":1.0356967144463256,"is_incomplete":false,"total_prompt_cost_bin":0.005313599999999998,"total_cache_read_cost_bin":0.004972480000000001,"total_completion_cost_bin":0.006301800000000001,"total_reasoning_cost_bin":0.000368,"total_cost_bin":0.01695588,"total_on_demand_input_cost_bin":0.05503840000000004,"ci_99_lower":0.1419293815420824,"ci_99_upper":0.34973308775399337,"ci_98_lower":0.14871263875172008,"ci_98_upper":0.3361232375781983,"ci_95_lower":0.16180418624542656,"ci_95_upper":0.3198899205564521,"ci_90_lower":0.17403152855811638,"ci_90_upper":0.3035431109747151,"ci_75_lower":0.19051471704151482,"ci_75_upper":0.28361285657010743,"ci_50_lower":0.2071900184673723,"ci_50_upper":0.26308784704306165},"16384":{"avg_score":0.20332509638974444,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.370440141910392,"median_score":0.03351544961333275,"q1_score":0.018763286992907524,"q3_score":0.049286942929029465,"avg_token_efficiency":1.041538797797136,"is_incomplete":false,"total_prompt_cost_bin":0.013607199999999995,"total_cache_read_cost_bin":0.01014112,"total_completion_cost_bin":0.005482200000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.029230519999999996,"total_on_demand_input_cost_bin":0.1150184000000001,"ci_99_lower":0.11183443145127968,"ci_99_upper":0.314206092358625,"ci_98_lower":0.11736937541200314,"ci_98_upper":0.2897205082356231,"ci_95_lower":0.12959061416215265,"ci_95_upper":0.2806709018046967,"ci_90_lower":0.13992963997356128,"ci_90_upper":0.27230973990750496,"ci_75_lower":0.15598652232292806,"ci_75_upper":0.2479441520117689,"ci_50_lower":0.17595229574362745,"ci_50_upper":0.2276225763896946},"32768":{"avg_score":0.14780281612868695,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.302144124717786,"median_score":0.034022511914372444,"q1_score":0.020305476151406765,"q3_score":0.05772643722593784,"avg_token_efficiency":1.0489906253597152,"is_incomplete":false,"total_prompt_cost_bin":0.007346400000000003,"total_cache_read_cost_bin":0.03992639999999992,"total_completion_cost_bin":0.011004400000000004,"total_reasoning_cost_bin":0,"total_cost_bin":0.058277199999999925,"total_on_demand_input_cost_bin":0.4066104000000002,"ci_99_lower":0.09015744807083942,"ci_99_upper":0.22378410340916802,"ci_98_lower":0.09314212317496318,"ci_98_upper":0.21373164918126455,"ci_95_lower":0.09924279856675318,"ci_95_upper":0.20044340302137081,"ci_90_lower":0.10657713219595334,"ci_90_upper":0.19112543863936,"ci_75_lower":0.1176835864145473,"ci_75_upper":0.17634734701106722,"ci_50_lower":0.13015740304185516,"ci_50_upper":0.16352229463775397},"65536":{"avg_score":0.10249711867312299,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.24245301928222304,"median_score":0.02770562842488289,"q1_score":0.018635105341672897,"q3_score":0.043931663036346436,"avg_token_efficiency":1.0388369004117979,"is_incomplete":false,"total_prompt_cost_bin":0.00013419999999999996,"total_cache_read_cost_bin":0.05774559999999989,"total_completion_cost_bin":0.006661800000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.0645415999999999,"total_on_demand_input_cost_bin":0.5775902000000002,"ci_99_lower":0.04441262074965326,"ci_99_upper":0.17790931278850663,"ci_98_lower":0.04572483840245096,"ci_98_upper":0.17349979418368242,"ci_95_lower":0.053803067975403636,"ci_95_upper":0.1609775751956519,"ci_90_lower":0.06047177866128657,"ci_90_upper":0.1476572526396964,"ci_75_lower":0.07193052292226212,"ci_75_upper":0.13284297334646586,"ci_50_lower":0.08325381581808496,"ci_50_upper":0.11701367182711907},"131072":{"avg_score":0.06365187309718537,"min_score":0,"max_score":0.9989429116249084,"n_tests":103,"stddev":0.15659754191351155,"median_score":0.029728343710303307,"q1_score":0.020974709652364254,"q3_score":0.04636893421411514,"avg_token_efficiency":1.0347606731041992,"is_incomplete":false,"total_prompt_cost_bin":0.00045699999999999994,"total_cache_read_cost_bin":0.1380838399999999,"total_completion_cost_bin":0.016824,"total_reasoning_cost_bin":0,"total_cost_bin":0.15536483999999992,"total_on_demand_input_cost_bin":1.3812954000000004,"ci_99_lower":0.033315792371659055,"ci_99_upper":0.11477430715496895,"ci_98_lower":0.034698411265876396,"ci_98_upper":0.10635823966740873,"ci_95_lower":0.03750717805244155,"ci_95_upper":0.09687008773275081,"ci_90_lower":0.041757547290480326,"ci_90_upper":0.0908976330763199,"ci_75_lower":0.046423122916343505,"ci_75_upper":0.080627214210447,"ci_50_lower":0.05238915525051286,"ci_50_upper":0.07339013134269497},"262144":{"avg_score":0.04491238962158772,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.08673439550364039,"median_score":0.030987493693828583,"q1_score":0.019607843831181526,"q3_score":0.043130990117788315,"avg_token_efficiency":1.0350996714344212,"is_incomplete":false,"total_prompt_cost_bin":0.02768399999999998,"total_cache_read_cost_bin":0.37950368,"total_completion_cost_bin":0.012402200000000002,"total_reasoning_cost_bin":0.00042259999999999997,"total_cost_bin":0.42001247999999997,"total_on_demand_input_cost_bin":3.8227207999999977,"ci_99_lower":0.03232191663045529,"ci_99_upper":0.06963952488399686,"ci_98_lower":0.03347498339191343,"ci_98_upper":0.06804593352108851,"ci_95_lower":0.034464243753955216,"ci_95_upper":0.061986638608154114,"ci_90_lower":0.035288182930882143,"ci_90_upper":0.05847417181886151,"ci_75_lower":0.037020646061266094,"ci_75_upper":0.05301097011670853,"ci_50_lower":0.03967719380198879,"ci_50_upper":0.049786903596555826},"524288":{"avg_score":0.043879031043862766,"min_score":0,"max_score":0.7202316522598267,"n_tests":236,"stddev":0.05920857686735635,"median_score":0.02949132863432169,"q1_score":0.018652827013283968,"q3_score":0.04586413688957691,"avg_token_efficiency":1.0343227547891893,"is_incomplete":false,"total_prompt_cost_bin":0.21821680000000054,"total_cache_read_cost_bin":1.2576272000000008,"total_completion_cost_bin":0.019126600000000004,"total_reasoning_cost_bin":0.0002394,"total_cost_bin":1.4952100000000013,"total_on_demand_input_cost_bin":12.794488800000005,"ci_99_lower":0.03569034015339054,"ci_99_upper":0.05573750314924281,"ci_98_lower":0.03627679321011213,"ci_98_upper":0.05486738923559226,"ci_95_lower":0.03713913962303689,"ci_95_upper":0.05278997802416305,"ci_90_lower":0.03807161492433683,"ci_90_upper":0.05085752451120504,"ci_75_lower":0.039744159860844265,"ci_75_upper":0.048302792100866236,"ci_50_lower":0.04127902135729466,"ci_50_upper":0.0461426418774339}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Poolside: Laguna S 2.1","provider_name":"Poolside","run_provider_names":["Poolside"],"pricing_metadata":{"prompt":9e-8,"completion":1.8e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-21T16:51:23.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-22T16:00:45.925Z"},{"model_slug":"stepfun/step-3.7-flash","reasoning_mode":"low","overall_metrics":{"auc_1m":0.04273159847958676,"auc_128k":0.27212636301577225,"cum_avg_1m":0.22541039655156264,"cum_avg_128k":0.36065663448250024,"avg_score_overall":0.3397167102858994,"avg_response_time_ms":29388.08328574945,"total_prompt_cost":0.1788952,"total_cache_read_cost":0.9430182400000015,"total_completion_cost":1.9061847999999997,"total_reasoning_cost":1.60760455,"total_prompt_cost_128k":0.1788952,"total_completion_cost_128k":1.9061847999999997,"total_on_demand_input_cost":4.893986399999996,"total_on_demand_input_cost_128k":4.893986399999996,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9984466628317947},"bin_metrics":{"8192":{"avg_score":0.6927337456669193,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.45259445977176377,"median_score":1,"q1_score":0.03939663991332054,"q3_score":1,"avg_token_efficiency":0.9943799479094221,"is_incomplete":false,"total_prompt_cost_bin":0.02032159999999999,"total_cache_read_cost_bin":0.017086720000000003,"total_completion_cost_bin":0.1865461,"total_reasoning_cost_bin":0.1529569,"total_cost_bin":0.37691132,"total_on_demand_input_cost_bin":0.10575519999999998,"ci_99_lower":0.5461794655433042,"ci_99_upper":0.8057420764616982,"ci_98_lower":0.5689996034906363,"ci_98_upper":0.8014214987466695,"ci_95_lower":0.5953231165579929,"ci_95_upper":0.7871300108337891,"ci_90_lower":0.6084242632813402,"ci_90_upper":0.7699612410747795,"ci_75_lower":0.634471162869886,"ci_75_upper":0.7439566670058411,"ci_50_lower":0.6571136765778647,"ci_50_upper":0.7285672056605108},"16384":{"avg_score":0.3984605376957916,"min_score":0.009216589853167534,"max_score":1,"n_tests":80,"stddev":0.46991255115554187,"median_score":0.044444017112255096,"q1_score":0.023525837808847427,"q3_score":1,"avg_token_efficiency":0.9903659767661971,"is_incomplete":false,"total_prompt_cost_bin":0.024123200000000004,"total_cache_read_cost_bin":0.03879552,"total_completion_cost_bin":0.2720739,"total_reasoning_cost_bin":0.08574284999999997,"total_cost_bin":0.42073546999999994,"total_on_demand_input_cost_bin":0.21810079999999984,"ci_99_lower":0.2655142384726205,"ci_99_upper":0.5294322537304251,"ci_98_lower":0.27581692952767484,"ci_98_upper":0.5223581855510129,"ci_95_lower":0.28974806691694543,"ci_95_upper":0.5014644413153292,"ci_90_lower":0.3054207465430955,"ci_90_upper":0.48884910807479176,"ci_75_lower":0.33686662123218414,"ci_75_upper":0.45857928047335006,"ci_50_lower":0.36272591802407983,"ci_50_upper":0.4333730900078081},"32768":{"avg_score":0.266147059340523,"min_score":0.003944773226976395,"max_score":1,"n_tests":136,"stddev":0.40941536852330745,"median_score":0.03650434501469135,"q1_score":0.024401037953794003,"q3_score":0.16030378267169,"avg_token_efficiency":1.010935628876953,"is_incomplete":false,"total_prompt_cost_bin":0.028238400000000018,"total_cache_read_cost_bin":0.15101184000000006,"total_completion_cost_bin":0.5040127999999996,"total_reasoning_cost_bin":0.43025870000000005,"total_cost_bin":1.1135217399999997,"total_on_demand_input_cost_bin":0.7832975999999994,"ci_99_lower":0.18025987433461363,"ci_99_upper":0.3507642126135247,"ci_98_lower":0.1856700754218036,"ci_98_upper":0.3459067287233055,"ci_95_lower":0.1996296473984223,"ci_95_upper":0.3333800325816436,"ci_90_lower":0.20867688550394686,"ci_90_upper":0.3214920585956219,"ci_75_lower":0.22441776726833162,"ci_75_upper":0.30502097752683127,"ci_50_lower":0.2422067527522278,"ci_50_upper":0.2885947636001648},"65536":{"avg_score":0.2778146345004001,"min_score":0.007499999832361937,"max_score":1,"n_tests":85,"stddev":0.41816617804204853,"median_score":0.02929115481674671,"q1_score":0.02004251442849636,"q3_score":0.4727047085762024,"avg_token_efficiency":0.9950202971179809,"is_incomplete":false,"total_prompt_cost_bin":0.027010200000000005,"total_cache_read_cost_bin":0.2158892800000004,"total_completion_cost_bin":0.3468549500000002,"total_reasoning_cost_bin":0.3425240500000001,"total_cost_bin":0.9322784800000007,"total_on_demand_input_cost_bin":1.1064565999999991,"ci_99_lower":0.170583935739856,"ci_99_upper":0.3969159469900598,"ci_98_lower":0.18275004036505432,"ci_98_upper":0.38945229891797195,"ci_95_lower":0.19602529682136854,"ci_95_upper":0.36968630870420705,"ci_90_lower":0.2075400999278816,"ci_90_upper":0.35186275228173197,"ci_75_lower":0.22261389766422593,"ci_75_upper":0.32544694458380996,"ci_50_lower":0.24584790484514088,"ci_50_upper":0.305495369409704},"131072":{"avg_score":0.1681271952088669,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.34047192053847664,"median_score":0.030927835032343864,"q1_score":0.018447481095790863,"q3_score":0.04321908950805664,"avg_token_efficiency":0.9942188354781075,"is_incomplete":false,"total_prompt_cost_bin":0.07920179999999999,"total_cache_read_cost_bin":0.520234880000001,"total_completion_cost_bin":0.5966970499999998,"total_reasoning_cost_bin":0.59612205,"total_cost_bin":1.7922557800000007,"total_on_demand_input_cost_bin":2.6803761999999978,"ci_99_lower":0.08239636473337333,"ci_99_upper":0.25493584424038007,"ci_98_lower":0.09488366264420407,"ci_98_upper":0.24464606047847562,"ci_95_lower":0.11041223260198801,"ci_95_upper":0.2336504755494212,"ci_90_lower":0.11683909068792804,"ci_90_upper":0.22393867795161215,"ci_75_lower":0.1308227995381175,"ci_75_upper":0.20529338395849178,"ci_50_lower":0.14379738260046984,"ci_50_upper":0.1870468576558531}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","medium","low"],"reasoning_default":"high","sponsors":[],"openrouter_name":"StepFun: Step 3.7 Flash","provider_name":"Stepfun","run_provider_names":[],"pricing_metadata":{"prompt":2e-7,"completion":0.00000115,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-05-28T16:17:49.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-23T03:13:31.972Z"},{"model_slug":"xiaomi/mimo-v2-omni","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.0421292433427808,"auc_128k":0.2775492156423949,"cum_avg_1m":0.24586893402902507,"cum_avg_128k":0.39339029444644014,"avg_score_overall":0.3784904894385341,"avg_response_time_ms":16213.280826284865,"total_prompt_cost":0.10057240000000002,"total_cache_read_cost":1.9408844800000011,"total_completion_cost":1.746986,"total_reasoning_cost":1.3733339999999996,"total_prompt_cost_128k":0.10057240000000002,"total_completion_cost_128k":1.746986,"total_on_demand_input_cost":9.804994800000003,"total_on_demand_input_cost_128k":9.804994800000003,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0075853299610373},"bin_metrics":{"8192":{"avg_score":0.7062411883031018,"min_score":0.0011560693383216858,"max_score":1,"n_tests":80,"stddev":0.44512435382942894,"median_score":1,"q1_score":0.054326536133885384,"q3_score":1,"avg_token_efficiency":1.0287564300108643,"is_incomplete":false,"total_prompt_cost_bin":0.00962879999999999,"total_cache_read_cost_bin":0.04178944000000002,"total_completion_cost_bin":0.21471000000000004,"total_reasoning_cost_bin":0.15922999999999995,"total_cost_bin":0.42535824,"total_on_demand_input_cost_bin":0.21857599999999994,"ci_99_lower":0.5729987341550296,"ci_99_upper":0.820851361317269,"ci_98_lower":0.5830307291470235,"ci_98_upper":0.8071257180301472,"ci_95_lower":0.6078843346302164,"ci_95_upper":0.7956493066321126,"ci_90_lower":0.6220828351791716,"ci_90_upper":0.7869626606930978,"ci_75_lower":0.647821395500796,"ci_75_upper":0.7575528566521824,"ci_50_lower":0.6720333272824064,"ci_50_upper":0.7374988210998709},"16384":{"avg_score":0.5143112888676115,"min_score":0.008665511384606361,"max_score":1,"n_tests":80,"stddev":0.48577923154183184,"median_score":0.5215901993215084,"q1_score":0.03084167605265975,"q3_score":1,"avg_token_efficiency":0.9995279508917738,"is_incomplete":false,"total_prompt_cost_bin":0.026112000000000034,"total_cache_read_cost_bin":0.08282624,"total_completion_cost_bin":0.22389399999999998,"total_reasoning_cost_bin":0.16938199999999998,"total_cost_bin":0.50221424,"total_on_demand_input_cost_bin":0.44024319999999983,"ci_99_lower":0.3705375434928574,"ci_99_upper":0.6475826200476149,"ci_98_lower":0.381060146178701,"ci_98_upper":0.634735024401918,"ci_95_lower":0.4023430479509989,"ci_95_upper":0.6221152788968175,"ci_90_lower":0.4179807870072545,"ci_90_upper":0.6000610981363569,"ci_75_lower":0.4540002588910284,"ci_75_upper":0.5755550158384721,"ci_50_lower":0.47800899003341324,"ci_50_upper":0.5506861916393972},"32768":{"avg_score":0.3809831152372199,"min_score":0.002711864421144128,"max_score":1,"n_tests":136,"stddev":0.4646300633765605,"median_score":0.04349261894822121,"q1_score":0.024326454382389784,"q3_score":0.9997866898775101,"avg_token_efficiency":1.0186110342292523,"is_incomplete":false,"total_prompt_cost_bin":0.03546239999999998,"total_cache_read_cost_bin":0.30884864000000006,"total_completion_cost_bin":0.6263219999999998,"total_reasoning_cost_bin":0.5186439999999999,"total_cost_bin":1.4892770399999997,"total_on_demand_input_cost_bin":1.5797056000000003,"ci_99_lower":0.2781389918798745,"ci_99_upper":0.4866093325917695,"ci_98_lower":0.2887988564897815,"ci_98_upper":0.4741589475377045,"ci_95_lower":0.30218407172180695,"ci_95_upper":0.45835033977427353,"ci_90_lower":0.314724111465501,"ci_90_upper":0.444817050881701,"ci_75_lower":0.3336675792653456,"ci_75_upper":0.4266925641859416,"ci_50_lower":0.3522500173323137,"ci_50_upper":0.4095475852745414},"65536":{"avg_score":0.2170189210871125,"min_score":0.0030075188260525465,"max_score":1,"n_tests":85,"stddev":0.3859306183252852,"median_score":0.02762543223798275,"q1_score":0.01922614686191082,"q3_score":0.05107252299785614,"avg_token_efficiency":0.9958114883727984,"is_incomplete":false,"total_prompt_cost_bin":0.027639200000000027,"total_cache_read_cost_bin":0.43740671999999975,"total_completion_cost_bin":0.338038,"total_reasoning_cost_bin":0.270018,"total_cost_bin":1.0731019199999996,"total_on_demand_input_cost_bin":2.214672800000001,"ci_99_lower":0.12291047738303486,"ci_99_upper":0.3315396165335825,"ci_98_lower":0.13058638628298308,"ci_98_upper":0.32235404016104907,"ci_95_lower":0.14456764222300778,"ci_95_upper":0.29952090009074545,"ci_90_lower":0.15261079648345272,"ci_90_upper":0.2866593167917145,"ci_75_lower":0.16853198701009045,"ci_75_upper":0.2629774382676902,"ci_50_lower":0.1846582260886755,"ci_50_upper":0.24108435342649875},"131072":{"avg_score":0.1483969587371549,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.32220592900802586,"median_score":0.025968916714191437,"q1_score":0.018556777387857437,"q3_score":0.04009448550641537,"avg_token_efficiency":0.9925579811850956,"is_incomplete":false,"total_prompt_cost_bin":0.0017299999999999982,"total_cache_read_cost_bin":1.0700134400000012,"total_completion_cost_bin":0.344022,"total_reasoning_cost_bin":0.25605999999999995,"total_cost_bin":1.6718254400000012,"total_on_demand_input_cost_bin":5.351797200000001,"ci_99_lower":0.0716006340343478,"ci_99_upper":0.2318281115918619,"ci_98_lower":0.07430767569464894,"ci_98_upper":0.22396016081419853,"ci_95_lower":0.0918950949629398,"ci_95_upper":0.21307415584954476,"ci_90_lower":0.1015088890028159,"ci_90_upper":0.19594352330980838,"ci_75_lower":0.11177487354228291,"ci_75_upper":0.17958490767920826,"ci_50_lower":0.12754922328082824,"ci_50_upper":0.16708007912508674}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2-Omni","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":4e-7,"completion":0.000002,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-18T19:55:03.000Z","api_updated_at":"2026-04-25T12:33:03.020Z","latest_run_timestamp":"2026-04-20T05:56:18.611Z"},{"model_slug":"minimax/minimax-m2.7","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.04031925723989216,"auc_128k":0.251706932987734,"cum_avg_1m":0.21318329918649415,"cum_avg_128k":0.3410932786983906,"avg_score_overall":0.33290143719425186,"avg_response_time_ms":51830.48480211867,"total_prompt_cost":4.582715249999998,"total_cache_read_cost":0.2577721,"total_completion_cost":1.00372,"total_reasoning_cost":0.9563680000000001,"total_prompt_cost_128k":4.582715249999998,"total_completion_cost_128k":1.00372,"total_on_demand_input_cost":5.871575750000001,"total_on_demand_input_cost_128k":5.871575750000001,"total_runs":479,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9609707866412914},"bin_metrics":{"8192":{"avg_score":0.6356104114209302,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4690707442280861,"median_score":0.9969784915447235,"q1_score":0.037147567607462406,"q3_score":1,"avg_token_efficiency":0.9588265886974587,"is_incomplete":false,"total_prompt_cost_bin":0.08200550000000001,"total_cache_read_cost_bin":0.00910525,"total_completion_cost_bin":0.24525499999999997,"total_reasoning_cost_bin":0.24717400000000006,"total_cost_bin":0.58353975,"total_on_demand_input_cost_bin":0.12753174999999994,"ci_99_lower":0.4896670922067133,"ci_99_upper":0.7693617182464222,"ci_98_lower":0.5038217928980012,"ci_98_upper":0.7449663697459036,"ci_95_lower":0.5299684363402777,"ci_95_upper":0.7317649962569703,"ci_90_lower":0.5494501487939851,"ci_90_upper":0.718983842554735,"ci_75_lower":0.5747726497997063,"ci_75_upper":0.6962613701354713,"ci_50_lower":0.5993791451415746,"ci_50_upper":0.6703803071402945},"16384":{"avg_score":0.3464835295943837,"min_score":0,"max_score":1,"n_tests":76,"stddev":0.4582755002079948,"median_score":0.03605814836919308,"q1_score":0.02373050758615136,"q3_score":0.9966136962175369,"avg_token_efficiency":0.9493669590080466,"is_incomplete":false,"total_prompt_cost_bin":0.10520650000000004,"total_cache_read_cost_bin":0.028204150000000004,"total_completion_cost_bin":0.24608399999999991,"total_reasoning_cost_bin":0.23222500000000004,"total_cost_bin":0.61171965,"total_on_demand_input_cost_bin":0.24622724999999998,"ci_99_lower":0.2178940651887464,"ci_99_upper":0.4763926568307521,"ci_98_lower":0.22981291923598435,"ci_98_upper":0.47321304113898227,"ci_95_lower":0.24540448611124274,"ci_95_upper":0.45006437415110045,"ci_90_lower":0.25815297789491815,"ci_90_upper":0.4261239668771033,"ci_75_lower":0.2837233239046518,"ci_75_upper":0.4082661363538232,"ci_50_lower":0.30938592641062934,"ci_50_upper":0.3826041032505026},"32768":{"avg_score":0.35530860049443114,"min_score":0,"max_score":1,"n_tests":135,"stddev":0.4511596508523636,"median_score":0.046222835779190063,"q1_score":0.024211524985730648,"q3_score":0.9929787516593933,"avg_token_efficiency":0.9719064988856416,"is_incomplete":false,"total_prompt_cost_bin":0.7552807499999998,"total_cache_read_cost_bin":0.03616934999999999,"total_completion_cost_bin":0.23575800000000005,"total_reasoning_cost_bin":0.22778399999999996,"total_cost_bin":1.2549921,"total_on_demand_input_cost_bin":0.9361274999999992,"ci_99_lower":0.25743370095578333,"ci_99_upper":0.4537514940188439,"ci_98_lower":0.26963549222397026,"ci_98_upper":0.44762546025271766,"ci_95_lower":0.2808600297865147,"ci_95_upper":0.4336569623462856,"ci_90_lower":0.29433392743821496,"ci_90_upper":0.4188448311029761,"ci_75_lower":0.3108274385074361,"ci_75_upper":0.3997599350383367,"ci_50_lower":0.3272936707224559,"ci_50_upper":0.37994496087474683},"65536":{"avg_score":0.19994614265092156,"min_score":0.004648074507713318,"max_score":1,"n_tests":85,"stddev":0.3647732992475799,"median_score":0.025561438873410225,"q1_score":0.017801513895392418,"q3_score":0.04330927133560181,"avg_token_efficiency":0.9599949098925789,"is_incomplete":false,"total_prompt_cost_bin":0.9564597499999995,"total_cache_read_cost_bin":0.07558519999999999,"total_completion_cost_bin":0.12360199999999996,"total_reasoning_cost_bin":0.10920999999999999,"total_cost_bin":1.2648569499999995,"total_on_demand_input_cost_bin":1.3343857500000003,"ci_99_lower":0.1081642069898217,"ci_99_upper":0.3095931679391138,"ci_98_lower":0.11642813778049586,"ci_98_upper":0.30122407136731505,"ci_95_lower":0.12963529280893615,"ci_95_upper":0.28024660735158247,"ci_90_lower":0.13993866436512153,"ci_90_upper":0.26501766132683874,"ci_75_lower":0.15449520733399208,"ci_75_upper":0.2427095804390881,"ci_50_lower":0.17218010797959696,"ci_50_upper":0.22060136514003661},"131072":{"avg_score":0.16811770933128675,"min_score":0.005540166050195694,"max_score":1,"n_tests":103,"stddev":0.3397205283372422,"median_score":0.026483051478862762,"q1_score":0.016633475199341774,"q3_score":0.045730117708444595,"avg_token_efficiency":0.9576703507799953,"is_incomplete":false,"total_prompt_cost_bin":2.6837627499999988,"total_cache_read_cost_bin":0.10870814999999999,"total_completion_cost_bin":0.15302100000000002,"total_reasoning_cost_bin":0.13997499999999996,"total_cost_bin":3.0854668999999983,"total_on_demand_input_cost_bin":3.2273035000000014,"ci_99_lower":0.08289033129935615,"ci_99_upper":0.25449092281955177,"ci_98_lower":0.09535339835140565,"ci_98_upper":0.24692156434881962,"ci_95_lower":0.10891204039041784,"ci_95_upper":0.23477500491511094,"ci_90_lower":0.11732205956952772,"ci_90_upper":0.22355562515163485,"ci_75_lower":0.13063008880109336,"ci_75_upper":0.20526169823229456,"ci_50_lower":0.14539000944424313,"ci_50_upper":0.18654284335433244}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":204800,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"MiniMax: MiniMax M2.7","provider_name":"Minimax","run_provider_names":["MiniMax"],"pricing_metadata":{"prompt":3e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-18T12:24:57.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T06:45:45.981Z"},{"model_slug":"openai/gpt-5.4-nano","reasoning_mode":null,"overall_metrics":{"auc_1m":0.03744369085163809,"auc_128k":0.11694972871072232,"cum_avg_1m":0.0957117247182916,"cum_avg_128k":0.13352214580958124,"avg_score_overall":0.12314074615370482,"avg_response_time_ms":8987.990690771485,"total_prompt_cost":0.6022631999999998,"total_cache_read_cost":1.1464857600000007,"total_completion_cost":0.24970125000000007,"total_reasoning_cost":0,"total_prompt_cost_128k":0.27866100000000005,"total_completion_cost_128k":0.20453500000000008,"total_on_demand_input_cost":12.067120799999994,"total_on_demand_input_cost_128k":4.810782599999998,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9809061808102971},"bin_metrics":{"8192":{"avg_score":0.1981185810174793,"min_score":0.009231604635715485,"max_score":1,"n_tests":80,"stddev":0.35674808444324313,"median_score":0.03708426654338837,"q1_score":0.025801117531955242,"q3_score":0.08256016299128532,"avg_token_efficiency":0.9759044449598392,"is_incomplete":false,"total_prompt_cost_bin":0.02795999999999998,"total_cache_read_cost_bin":0.0075878400000000045,"total_completion_cost_bin":0.03423125000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.06977909,"total_on_demand_input_cost_bin":0.1038384,"ci_99_lower":0.11186241931037512,"ci_99_upper":0.29511301597423145,"ci_98_lower":0.11755124209180941,"ci_98_upper":0.28692750943603457,"ci_95_lower":0.12573684171424246,"ci_95_upper":0.2730495366588001,"ci_90_lower":0.13591810482146685,"ci_90_upper":0.2620949853648199,"ci_75_lower":0.15306410133925966,"ci_75_upper":0.24033683655725327,"ci_50_lower":0.17123208726698066,"ci_50_upper":0.2210698150534881},"16384":{"avg_score":0.13593709574779494,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.30922585484151327,"median_score":0.029041282832622528,"q1_score":0.018914510030299425,"q3_score":0.03908670041710138,"avg_token_efficiency":0.9670687393188715,"is_incomplete":false,"total_prompt_cost_bin":0.03040480000000002,"total_cache_read_cost_bin":0.018273280000000003,"total_completion_cost_bin":0.031347500000000014,"total_reasoning_cost_bin":0,"total_cost_bin":0.08002558000000004,"total_on_demand_input_cost_bin":0.21313760000000007,"ci_99_lower":0.06319297722936608,"ci_99_upper":0.24607848006987482,"ci_98_lower":0.06561442150385119,"ci_98_upper":0.22216474302415726,"ci_95_lower":0.07480547925195424,"ci_95_upper":0.208725818892417,"ci_90_lower":0.08531927053758409,"ci_90_upper":0.19586516236537135,"ci_75_lower":0.09829835175769404,"ci_75_upper":0.17307925460045226,"ci_50_lower":0.11127284118265378,"ci_50_upper":0.15897930240316782},"32768":{"avg_score":0.12942400916054955,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.2963658109361453,"median_score":0.029782415367662907,"q1_score":0.013856276869773865,"q3_score":0.04758573230355978,"avg_token_efficiency":0.995210076854963,"is_incomplete":false,"total_prompt_cost_bin":0.06480960000000006,"total_cache_read_cost_bin":0.07071744000000006,"total_completion_cost_bin":0.05287500000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.18840204000000016,"total_on_demand_input_cost_bin":0.7719840000000007,"ci_99_lower":0.0728872140303172,"ci_99_upper":0.19914271371348363,"ci_98_lower":0.07746838217649116,"ci_98_upper":0.19446218692926992,"ci_95_lower":0.08527931594512399,"ci_95_upper":0.18532777861580732,"ci_90_lower":0.09017866032852051,"ci_90_upper":0.17171668383594163,"ci_75_lower":0.0994690742394168,"ci_75_upper":0.15809301500506562,"ci_50_lower":0.11282741492323112,"ci_50_upper":0.14455885169873534},"65536":{"avg_score":0.1232423982802121,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.29238453312908874,"median_score":0.022916197776794434,"q1_score":0.012814645655453205,"q3_score":0.03854707255959511,"avg_token_efficiency":0.9776435846759914,"is_incomplete":false,"total_prompt_cost_bin":0.050129000000000035,"total_cache_read_cost_bin":0.10370047999999987,"total_completion_cost_bin":0.03681750000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.1906469799999999,"total_on_demand_input_cost_bin":1.0871337999999988,"ci_99_lower":0.05519766172933776,"ci_99_upper":0.21245702084068976,"ci_98_lower":0.06011325769959127,"ci_98_upper":0.2030864915478536,"ci_95_lower":0.06753258440616157,"ci_95_upper":0.1928554458036909,"ci_90_lower":0.074601515584954,"ci_90_upper":0.18040805991026843,"ci_75_lower":0.08856233257307287,"ci_75_upper":0.158190846658887,"ci_50_lower":0.10016467952963841,"ci_50_upper":0.14185052728691303},"131072":{"avg_score":0.08088864484187035,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.22110812140281638,"median_score":0.023320376873016357,"q1_score":0.014872403349727392,"q3_score":0.035131754353642464,"avg_token_efficiency":0.9772722316188366,"is_incomplete":false,"total_prompt_cost_bin":0.10535759999999997,"total_cache_read_cost_bin":0.25293312000000034,"total_completion_cost_bin":0.04926375000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.4075544700000003,"total_on_demand_input_cost_bin":2.634688799999999,"ci_99_lower":0.0331382377175117,"ci_99_upper":0.14794513336993137,"ci_98_lower":0.036478567517823676,"ci_98_upper":0.13596286835397586,"ci_95_lower":0.043135588636816234,"ci_95_upper":0.12311366110456905,"ci_90_lower":0.04714475382521785,"ci_90_upper":0.1160473532934031,"ci_75_lower":0.05491020991093959,"ci_75_upper":0.10397806536752133,"ci_50_lower":0.06394597981238141,"ci_50_upper":0.09311890658624108},"262144":{"avg_score":0.09808306869842667,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.2385127062354018,"median_score":0.028622539713978767,"q1_score":0.014621008187532425,"q3_score":0.04384615272283554,"avg_token_efficiency":0.9824198103380456,"is_incomplete":false,"total_prompt_cost_bin":0.32360219999999973,"total_cache_read_cost_bin":0.6932736000000004,"total_completion_cost_bin":0.04516625,"total_reasoning_cost_bin":0,"total_cost_bin":1.0620420500000003,"total_on_demand_input_cost_bin":7.256338199999997,"ci_99_lower":0.05516060849547251,"ci_99_upper":0.15650899944884752,"ci_98_lower":0.05730642873598163,"ci_98_upper":0.15162514628089485,"ci_95_lower":0.06126004914238765,"ci_95_upper":0.13950015908600066,"ci_90_lower":0.06682892315282075,"ci_90_upper":0.13227373520709104,"ci_75_lower":0.07512407551056388,"ci_75_upper":0.12122074597459848,"ci_50_lower":0.08243411444069765,"ci_50_upper":0.10970767422171362}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":400000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["xhigh","high","medium","low"],"reasoning_default":"medium","sponsors":[],"openrouter_name":"OpenAI: GPT-5.4 Nano","provider_name":"Openai","run_provider_names":["openai"],"pricing_metadata":{"prompt":2e-7,"completion":0.00000125,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-17T11:49:47.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-19T22:51:24.330Z"},{"model_slug":"anthropic/claude-haiku-4.5","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.037380962298541036,"auc_128k":0.24297291438808688,"cum_avg_1m":0.24047039189279115,"cum_avg_128k":0.38475262702846585,"avg_score_overall":0.36274810207498737,"avg_response_time_ms":23102.998057467878,"total_prompt_cost":1.6746270000000014,"total_cache_read_cost":2.6063493999999983,"total_completion_cost":5.474120000000001,"total_reasoning_cost":2.5144200000000003,"total_prompt_cost_128k":1.6746270000000014,"total_completion_cost_128k":5.474120000000001,"total_on_demand_input_cost":27.738121000000014,"total_on_demand_input_cost_128k":27.738121000000014,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1347219939845858},"bin_metrics":{"8192":{"avg_score":0.8249466324559762,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.36810115160067736,"median_score":0.996441125869751,"q1_score":0.9797597080469131,"q3_score":1,"avg_token_efficiency":1.1422554365030504,"is_incomplete":false,"total_prompt_cost_bin":0.05363000000000003,"total_cache_read_cost_bin":0.05527380000000002,"total_completion_cost_bin":0.5826350000000001,"total_reasoning_cost_bin":0.320945,"total_cost_bin":1.0124838000000003,"total_on_demand_input_cost_bin":0.6063679999999997,"ci_99_lower":0.7048691293021039,"ci_99_upper":0.9229310920492789,"ci_98_lower":0.7158670709775907,"ci_98_upper":0.9116672855296346,"ci_95_lower":0.7400103249986205,"ci_95_upper":0.8993828242539894,"ci_90_lower":0.7539280422551383,"ci_90_upper":0.8861516776165808,"ci_75_lower":0.7764741427236003,"ci_75_upper":0.8718555261524671,"ci_50_lower":0.7997722351537959,"ci_50_upper":0.8499633283237926},"16384":{"avg_score":0.4754933915246511,"min_score":0.002711864421144128,"max_score":1,"n_tests":80,"stddev":0.47927845421697796,"median_score":0.054164422675967216,"q1_score":0.027124859392642975,"q3_score":0.9926138073205948,"avg_token_efficiency":1.1249181718691132,"is_incomplete":false,"total_prompt_cost_bin":0.08770400000000003,"total_cache_read_cost_bin":0.11534320000000003,"total_completion_cost_bin":0.854465,"total_reasoning_cost_bin":0.40302,"total_cost_bin":1.4605322,"total_on_demand_input_cost_bin":1.2411359999999994,"ci_99_lower":0.3330830918354477,"ci_99_upper":0.6045294578006141,"ci_98_lower":0.3449434615485661,"ci_98_upper":0.594005391897983,"ci_95_lower":0.36692291676175953,"ci_95_upper":0.581515760789116,"ci_90_lower":0.3805754325202724,"ci_90_upper":0.5609094688479671,"ci_75_lower":0.41609850912500407,"ci_75_upper":0.5360370339330984,"ci_50_lower":0.43940737727025403,"ci_50_upper":0.5113374211832706},"32768":{"avg_score":0.3151204102135072,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.4346153204946655,"median_score":0.040040887892246246,"q1_score":0.024159247986972332,"q3_score":0.9679259657859802,"avg_token_efficiency":1.145945749290596,"is_incomplete":false,"total_prompt_cost_bin":0.12610399999999983,"total_cache_read_cost_bin":0.4320632000000002,"total_completion_cost_bin":1.5047350000000004,"total_reasoning_cost_bin":0.6806999999999999,"total_cost_bin":2.7436022,"total_on_demand_input_cost_bin":4.446736,"ci_99_lower":0.2249522317547577,"ci_99_upper":0.40738679455947613,"ci_98_lower":0.23213321931280115,"ci_98_upper":0.39855270598451814,"ci_95_lower":0.24329912100986215,"ci_95_upper":0.3856594989309943,"ci_90_lower":0.2519978465112645,"ci_90_upper":0.3749643443353467,"ci_75_lower":0.2719817228914026,"ci_75_upper":0.3580069082942346,"ci_50_lower":0.290026340250573,"ci_50_upper":0.3405389158193858},"65536":{"avg_score":0.17035413893651874,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.3436133959362016,"median_score":0.026758410036563873,"q1_score":0.017536168918013573,"q3_score":0.041650597006082535,"avg_token_efficiency":1.1320224755765207,"is_incomplete":false,"total_prompt_cost_bin":0.07851499999999981,"total_cache_read_cost_bin":0.621549599999999,"total_completion_cost_bin":1.3837350000000006,"total_reasoning_cost_bin":0.6045750000000001,"total_cost_bin":2.6883745999999995,"total_on_demand_input_cost_bin":6.294011000000002,"ci_99_lower":0.08166133407104816,"ci_99_upper":0.2741472405229613,"ci_98_lower":0.09225695568999713,"ci_98_upper":0.2613539879600079,"ci_95_lower":0.10235504099126796,"ci_95_upper":0.24888816336984745,"ci_90_lower":0.11330651270157163,"ci_90_upper":0.23637523597106339,"ci_75_lower":0.12667384650455515,"ci_75_upper":0.21413432720309014,"ci_50_lower":0.14471603814471407,"ci_50_upper":0.1919444865841583},"131072":{"avg_score":0.13784856201167608,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.2990829527016504,"median_score":0.02870991826057434,"q1_score":0.017504772171378136,"q3_score":0.03971799276769161,"avg_token_efficiency":1.1238934377790415,"is_incomplete":false,"total_prompt_cost_bin":1.3286740000000017,"total_cache_read_cost_bin":1.382119599999999,"total_completion_cost_bin":1.1485500000000004,"total_reasoning_cost_bin":0.50518,"total_cost_bin":4.364523600000001,"total_on_demand_input_cost_bin":15.149870000000016,"ci_99_lower":0.06658351644041757,"ci_99_upper":0.21658758827039826,"ci_98_lower":0.07363138380650322,"ci_98_upper":0.21125009865961172,"ci_95_lower":0.088184869821857,"ci_95_upper":0.19915028215036729,"ci_90_lower":0.09419541067253906,"ci_90_upper":0.18247236602383582,"ci_75_lower":0.10615549866845876,"ci_75_upper":0.16903695007204078,"ci_50_lower":0.11763366890191868,"ci_50_upper":0.1557331939337513}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":200000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Anthropic: Claude Haiku 4.5","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000001,"completion":0.000005,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2025-10-15T17:00:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T01:23:18.148Z"},{"model_slug":"mistralai/mistral-medium-3.5","reasoning_mode":"high","overall_metrics":{"auc_1m":0.03460441202871566,"auc_128k":0.22282466166866252,"cum_avg_1m":0.21488045078795345,"cum_avg_128k":0.34380872126072554,"avg_score_overall":0.32054177619500496,"avg_response_time_ms":39928.605257901276,"total_prompt_cost":28.095444000000015,"total_cache_read_cost":9.529535999999998,"total_completion_cost":22.076722500000002,"total_reasoning_cost":22.523445000000002,"total_prompt_cost_128k":28.095444000000015,"total_completion_cost_128k":22.076722500000002,"total_on_demand_input_cost":37.62498000000002,"total_on_demand_input_cost_128k":37.62498000000002,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0244468830266986},"bin_metrics":{"8192":{"avg_score":0.7166874364425894,"min_score":0.0052659292705357075,"max_score":1,"n_tests":80,"stddev":0.4405408029564445,"median_score":0.9998535811901093,"q1_score":0.044645827263593674,"q3_score":1,"avg_token_efficiency":1.024927076436883,"is_incomplete":false,"total_prompt_cost_bin":0.7898880000000001,"total_cache_read_cost_bin":0.02712,"total_completion_cost_bin":3.7696425000000002,"total_reasoning_cost_bin":4.0370025,"total_cost_bin":8.623653000000001,"total_on_demand_input_cost_bin":0.817008,"ci_99_lower":0.5844098140411953,"ci_99_upper":0.8299467844558531,"ci_98_lower":0.5944318677365663,"ci_98_upper":0.8185984042805502,"ci_95_lower":0.6196033087682735,"ci_95_upper":0.8056263704462617,"ci_90_lower":0.6319898979112621,"ci_90_upper":0.7927081518864725,"ci_75_lower":0.6577200095925946,"ci_75_upper":0.7679112605794217,"ci_50_lower":0.6822962342499523,"ci_50_upper":0.747786653872754},"16384":{"avg_score":0.45153561570623424,"min_score":0.002706359839066863,"max_score":1,"n_tests":80,"stddev":0.482856256564834,"median_score":0.042808106169104576,"q1_score":0.025894413702189922,"q3_score":1,"avg_token_efficiency":1.0153834630552776,"is_incomplete":false,"total_prompt_cost_bin":1.5332159999999997,"total_cache_read_cost_bin":0.1447679999999999,"total_completion_cost_bin":4.5203025,"total_reasoning_cost_bin":4.573417500000001,"total_cost_bin":10.771704,"total_on_demand_input_cost_bin":1.6779840000000004,"ci_99_lower":0.3165622329411562,"ci_99_upper":0.583615886468513,"ci_98_lower":0.3276286276107712,"ci_98_upper":0.5712536458603281,"ci_95_lower":0.3398031950995938,"ci_95_upper":0.5598178712133085,"ci_90_lower":0.3547732154879486,"ci_90_upper":0.5399649061390661,"ci_75_lower":0.3899761601460341,"ci_75_upper":0.5129858261349,"ci_50_lower":0.41498224150127505,"ci_50_upper":0.4878255709816585},"32768":{"avg_score":0.24503431367509834,"min_score":0.0027072757948189974,"max_score":1,"n_tests":136,"stddev":0.3913973304474882,"median_score":0.03922852873802185,"q1_score":0.022521347738802433,"q3_score":0.10330003313720226,"avg_token_efficiency":1.0368976087081263,"is_incomplete":false,"total_prompt_cost_bin":4.537716000000003,"total_cache_read_cost_bin":1.4924400000000004,"total_completion_cost_bin":6.932752500000001,"total_reasoning_cost_bin":7.041502500000003,"total_cost_bin":20.004411000000008,"total_on_demand_input_cost_bin":6.030156000000001,"ci_99_lower":0.1598664081885705,"ci_99_upper":0.322815247859561,"ci_98_lower":0.16770076825747418,"ci_98_upper":0.319019548613715,"ci_95_lower":0.1828796998305359,"ci_95_upper":0.3092839294593432,"ci_90_lower":0.1906370642940815,"ci_90_upper":0.3009096902938087,"ci_75_lower":0.20517976178405875,"ci_75_upper":0.282744055856978,"ci_50_lower":0.22158565117313306,"ci_50_upper":0.26561295656917056},"65536":{"avg_score":0.17423744015258683,"min_score":0.001463057822547853,"max_score":1,"n_tests":85,"stddev":0.3390284063622651,"median_score":0.02669352851808071,"q1_score":0.01740812323987484,"q3_score":0.04628949239850044,"avg_token_efficiency":1.0197047828224646,"is_incomplete":false,"total_prompt_cost_bin":6.504516000000004,"total_cache_read_cost_bin":1.9997760000000009,"total_completion_cost_bin":3.4055550000000006,"total_reasoning_cost_bin":3.447532499999999,"total_cost_bin":15.357379500000006,"total_on_demand_input_cost_bin":8.504292000000003,"ci_99_lower":0.08676866688981981,"ci_99_upper":0.27877477236505016,"ci_98_lower":0.09565132852445137,"ci_98_upper":0.2626140751954825,"ci_95_lower":0.1088343981747269,"ci_95_upper":0.2509550108909196,"ci_90_lower":0.11805554654997062,"ci_90_upper":0.23820155419929717,"ci_75_lower":0.13270920376012593,"ci_75_upper":0.21635367062408478,"ci_50_lower":0.1473151730017408,"ci_50_upper":0.19481425987426465},"131072":{"avg_score":0.13154880032711888,"min_score":0.003610108280554414,"max_score":1,"n_tests":103,"stddev":0.2648517838261379,"median_score":0.029739776626229286,"q1_score":0.01584541890770197,"q3_score":0.05663307197391987,"avg_token_efficiency":1.0185870572945108,"is_incomplete":false,"total_prompt_cost_bin":14.730108000000007,"total_cache_read_cost_bin":5.8654319999999975,"total_completion_cost_bin":3.448470000000001,"total_reasoning_cost_bin":3.4239900000000003,"total_cost_bin":27.468000000000004,"total_on_demand_input_cost_bin":20.595540000000014,"ci_99_lower":0.07239872026380413,"ci_99_upper":0.20159461201900997,"ci_98_lower":0.0760115657572734,"ci_98_upper":0.19758494959488618,"ci_95_lower":0.08645414005946911,"ci_95_upper":0.18243905186485862,"ci_90_lower":0.09293130021836175,"ci_90_upper":0.1716633935918968,"ci_75_lower":0.10296823988381537,"ci_75_upper":0.15984163965856954,"ci_50_lower":0.11253329574580313,"ci_50_upper":0.1470572457024918}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Mistral: Mistral Medium 3.5","provider_name":"Mistralai","run_provider_names":["Mistral"],"pricing_metadata":{"prompt":0.0000015,"completion":0.0000075,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-30T17:33:59.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T03:45:06.120Z"},{"model_slug":"thinkingmachines/inkling-small","reasoning_mode":null,"overall_metrics":{"auc_1m":0.034595590417707246,"auc_128k":0.14057228127506044,"cum_avg_1m":0.11221781434331464,"cum_avg_128k":0.16585698505466498,"avg_score_overall":0.13963918778710066,"avg_response_time_ms":6779.669561132812,"total_prompt_cost":0.3324212999999999,"total_cache_read_cost":5.960063999999997,"total_completion_cost":0.40444080000000004,"total_reasoning_cost":0.0002748,"total_prompt_cost_128k":0.2139196499999999,"total_completion_cost_128k":0.31137600000000004,"total_on_demand_input_cost":27.152709300000012,"total_on_demand_input_cost_128k":10.825567650000005,"total_runs":625,"total_candidates_generated":null,"avg_token_efficiency_overall":0.9811547899029414},"bin_metrics":{"8192":{"avg_score":0.28089613385964185,"min_score":0.0054054055362939835,"max_score":1,"n_tests":80,"stddev":0.41885286424280505,"median_score":0.03597739711403847,"q1_score":0.025994518771767616,"q3_score":0.39769814163446426,"avg_token_efficiency":0.9768364878081804,"is_incomplete":false,"total_prompt_cost_bin":0.052009199999999985,"total_cache_read_cost_bin":0.0404096,"total_completion_cost_bin":0.0314376,"total_reasoning_cost_bin":0,"total_cost_bin":0.12385639999999998,"total_on_demand_input_cost_bin":0.23385239999999993,"ci_99_lower":0.17085319833259563,"ci_99_upper":0.4006981414424954,"ci_98_lower":0.17709758201823572,"ci_98_upper":0.3900247656153515,"ci_95_lower":0.19199372564035005,"ci_95_upper":0.3733203518280061,"ci_90_lower":0.20671296451822857,"ci_90_upper":0.35699514138628724,"ci_75_lower":0.22997678539250047,"ci_75_upper":0.32971377052599565,"ci_50_lower":0.2507258523372002,"ci_50_upper":0.30782623032573614},"16384":{"avg_score":0.1552878605085425,"min_score":0.008665511384606361,"max_score":1,"n_tests":80,"stddev":0.31808590991066693,"median_score":0.0315085593611002,"q1_score":0.022704831324517727,"q3_score":0.04786762036383152,"avg_token_efficiency":0.9675158203335567,"is_incomplete":false,"total_prompt_cost_bin":0.0288252,"total_cache_read_cost_bin":0.10021119999999997,"total_completion_cost_bin":0.03160080000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.16063719999999998,"total_on_demand_input_cost_bin":0.4797755999999997,"ci_99_lower":0.08079894866544055,"ci_99_upper":0.2664232812289847,"ci_98_lower":0.08311821635125671,"ci_98_upper":0.24178618901758453,"ci_95_lower":0.09133322933950694,"ci_95_upper":0.22610660040867514,"ci_90_lower":0.09915211588551756,"ci_90_upper":0.21535149339353665,"ci_75_lower":0.11523350074712653,"ci_75_upper":0.19395786808308912,"ci_50_lower":0.13014588514051867,"ci_50_upper":0.1767725254583638},"32768":{"avg_score":0.15429427328533218,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.32167087354328106,"median_score":0.03055349551141262,"q1_score":0.02013654913753271,"q3_score":0.04741096869111061,"avg_token_efficiency":0.9954255314249072,"is_incomplete":false,"total_prompt_cost_bin":0.03409919999999996,"total_cache_read_cost_bin":0.37849600000000044,"total_completion_cost_bin":0.0596004,"total_reasoning_cost_bin":0,"total_cost_bin":0.4721956000000004,"total_on_demand_input_cost_bin":1.7373312000000012,"ci_99_lower":0.08864705157346393,"ci_99_upper":0.23361528697849923,"ci_98_lower":0.0940351724022698,"ci_98_upper":0.22479256586041574,"ci_95_lower":0.10258335793409702,"ci_95_upper":0.2117492922642508,"ci_90_lower":0.10964462487934405,"ci_90_upper":0.19837610525941912,"ci_75_lower":0.12109633773712525,"ci_75_upper":0.18423290544588405,"ci_50_lower":0.13442527826683703,"ci_50_upper":0.16960787810368105},"65536":{"avg_score":0.15854745554902097,"min_score":0.004765687044709921,"max_score":1,"n_tests":85,"stddev":0.3263097774696552,"median_score":0.02804107405245304,"q1_score":0.02164226584136486,"q3_score":0.044348035007715225,"avg_token_efficiency":0.977735311789681,"is_incomplete":false,"total_prompt_cost_bin":0.03191894999999997,"total_cache_read_cost_bin":0.5365247999999997,"total_completion_cost_bin":0.04143000000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.6098737499999997,"total_on_demand_input_cost_bin":2.446280550000001,"ci_99_lower":0.07748727832003223,"ci_99_upper":0.2613424650530934,"ci_98_lower":0.07943433671466568,"ci_98_upper":0.24713430914277318,"ci_95_lower":0.09206071262960049,"ci_95_upper":0.23568871346083195,"ci_90_lower":0.1036764271064278,"ci_90_upper":0.22222012264270546,"ci_75_lower":0.11640664810205208,"ci_75_upper":0.19635493432870135,"ci_50_lower":0.13244480818975718,"ci_50_upper":0.1796885142773938},"131072":{"avg_score":0.0802592020707874,"min_score":0,"max_score":0.9997298121452332,"n_tests":103,"stddev":0.21046487474095235,"median_score":0.025830257683992386,"q1_score":0.015768752433359623,"q3_score":0.0434101577848196,"avg_token_efficiency":0.9773180779623741,"is_incomplete":false,"total_prompt_cost_bin":0.06706709999999999,"total_cache_read_cost_bin":1.3025024000000003,"total_completion_cost_bin":0.14730720000000003,"total_reasoning_cost_bin":0,"total_cost_bin":1.5168767000000003,"total_on_demand_input_cost_bin":5.928327900000004,"ci_99_lower":0.0350477535571695,"ci_99_upper":0.14207414078056016,"ci_98_lower":0.03861600309316503,"ci_98_upper":0.13079450134965542,"ci_95_lower":0.04365640099587607,"ci_95_upper":0.12051042964584781,"ci_90_lower":0.04880127577920381,"ci_90_upper":0.11372897254949597,"ci_75_lower":0.05694465433841897,"ci_75_upper":0.10266337117679797,"ci_50_lower":0.06567053447332849,"ci_50_upper":0.09276257799395803},"262144":{"avg_score":0.06845758947319226,"min_score":0,"max_score":1,"n_tests":141,"stddev":0.14809134728512982,"median_score":0.03194103017449379,"q1_score":0.021280432119965553,"q3_score":0.0479360856115818,"avg_token_efficiency":0.9824427179571965,"is_incomplete":false,"total_prompt_cost_bin":0.11850165000000003,"total_cache_read_cost_bin":3.6019199999999967,"total_completion_cost_bin":0.0930648,"total_reasoning_cost_bin":0.0002748,"total_cost_bin":3.813761249999997,"total_on_demand_input_cost_bin":16.32714165000001,"ci_99_lower":0.042447759560317606,"ci_99_upper":0.10380737605679727,"ci_98_lower":0.04448286898378019,"ci_98_upper":0.10012119084390224,"ci_95_lower":0.04726650209627643,"ci_95_upper":0.09570684941598635,"ci_90_lower":0.05063906332101435,"ci_90_upper":0.09057388139726819,"ci_75_lower":0.054923449579794746,"ci_75_upper":0.08262684270830375,"ci_50_lower":0.05947670798644701,"ci_50_upper":0.07616323517001372}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":1048576,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["max","high","medium","low","minimal","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Thinking Machines: Inkling Small","provider_name":"Thinkingmachines","run_provider_names":["DeepInfra"],"pricing_metadata":{"prompt":4.5e-7,"completion":0.0000012,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-30T20:25:17.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T23:04:28.602Z"},{"model_slug":"tencent/hy3","reasoning_mode":null,"overall_metrics":{"auc_1m":0.03308359112750438,"auc_128k":0.2001996383627551,"cum_avg_1m":0.16837806457101218,"cum_avg_128k":0.2694049033136195,"avg_score_overall":0.25345874727176393,"avg_response_time_ms":9021.890086780895,"total_prompt_cost":2.588266428,"total_cache_read_cost":0.16716902399999994,"total_completion_cost":0.09369571199999999,"total_reasoning_cost":0,"total_prompt_cost_128k":2.588266428,"total_completion_cost_128k":0.09369571199999999,"total_on_demand_input_cost":3.256942523999999,"total_on_demand_input_cost_128k":3.256942523999999,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.007199733710169},"bin_metrics":{"8192":{"avg_score":0.484218694322044,"min_score":0.002900652587413788,"max_score":1,"n_tests":80,"stddev":0.4820165666361667,"median_score":0.12175419181585312,"q1_score":0.03166438080370426,"q3_score":1,"avg_token_efficiency":1.0011287986698734,"is_incomplete":false,"total_prompt_cost_bin":0.06617107200000005,"total_cache_read_cost_bin":0.0010222080000000001,"total_completion_cost_bin":0.015501552000000002,"total_reasoning_cost_bin":0,"total_cost_bin":0.08269483200000005,"total_on_demand_input_cost_bin":0.07025990400000005,"ci_99_lower":0.3417303938831319,"ci_99_upper":0.6144242380104261,"ci_98_lower":0.35224724647455147,"ci_98_upper":0.6038963849805877,"ci_95_lower":0.37353544719371706,"ci_95_upper":0.589411870046606,"ci_90_lower":0.38824893471377436,"ci_90_upper":0.5707110389042644,"ci_75_lower":0.42489881494693693,"ci_75_upper":0.5458295316275326,"ci_50_lower":0.4479869388625957,"ci_50_upper":0.5207274382730247},"16384":{"avg_score":0.3444890644168481,"min_score":0.007502679713070393,"max_score":1,"n_tests":80,"stddev":0.45642096849592584,"median_score":0.041440924629569054,"q1_score":0.024526177905499935,"q3_score":1,"avg_token_efficiency":1.002757402130155,"is_incomplete":false,"total_prompt_cost_bin":0.12982675200000002,"total_cache_read_cost_bin":0.003974784,"total_completion_cost_bin":0.014185247999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.14798678400000004,"total_on_demand_input_cost_bin":0.14572588800000003,"ci_99_lower":0.2215602466626442,"ci_99_upper":0.4799557718051946,"ci_98_lower":0.22393637149222195,"ci_98_upper":0.4659868808101163,"ci_95_lower":0.24767031894618413,"ci_95_upper":0.4424196535555529,"ci_90_lower":0.2583973350696033,"ci_90_upper":0.42857565327954944,"ci_75_lower":0.2842939833906712,"ci_75_upper":0.4042399999481858,"ci_50_lower":0.30909347528940995,"ci_50_upper":0.3797924408427207},"32768":{"avg_score":0.18869784696075573,"min_score":0.0031725887674838305,"max_score":1,"n_tests":136,"stddev":0.34910331580583936,"median_score":0.0339908953756094,"q1_score":0.02246238011866808,"q3_score":0.08195121213793755,"avg_token_efficiency":1.019367199209727,"is_incomplete":false,"total_prompt_cost_bin":0.4088842560000002,"total_cache_read_cost_bin":0.028138175999999997,"total_completion_cost_bin":0.027561072000000013,"total_reasoning_cost_bin":0,"total_cost_bin":0.46458350400000026,"total_on_demand_input_cost_bin":0.5214369600000001,"ci_99_lower":0.11807462155313046,"ci_99_upper":0.27080727772021096,"ci_98_lower":0.12188860265820575,"ci_98_upper":0.2637278355433612,"ci_95_lower":0.13048002096265382,"ci_95_upper":0.24993482775030046,"ci_90_lower":0.1404308407966335,"ci_90_upper":0.23799311345972132,"ci_75_lower":0.15341299167823094,"ci_75_upper":0.22199196317459985,"ci_50_lower":0.16707865389659726,"ci_50_upper":0.20647044230795206},"65536":{"avg_score":0.17979122364948338,"min_score":0.002102312631905079,"max_score":1,"n_tests":85,"stddev":0.35169133708697636,"median_score":0.02666666731238365,"q1_score":0.01861812174320221,"q3_score":0.04968944191932678,"avg_token_efficiency":1.0041620730383456,"is_incomplete":false,"total_prompt_cost_bin":0.6331616280000001,"total_cache_read_cost_bin":0.025952255999999996,"total_completion_cost_bin":0.014287679999999994,"total_reasoning_cost_bin":0,"total_cost_bin":0.6734015640000001,"total_on_demand_input_cost_bin":0.7369706520000003,"ci_99_lower":0.08888023867081468,"ci_99_upper":0.287949121195823,"ci_98_lower":0.09806907382382847,"ci_98_upper":0.27195922483172374,"ci_95_lower":0.11052866191815532,"ci_95_upper":0.2593641351744779,"ci_90_lower":0.12036602273871028,"ci_90_upper":0.2454993223270182,"ci_75_lower":0.13491259204079945,"ci_75_upper":0.22434969800764093,"ci_50_lower":0.15272208861938186,"ci_50_upper":0.20108586769099074},"131072":{"avg_score":0.14982768721896614,"min_score":0.005529954098165035,"max_score":1,"n_tests":103,"stddev":0.32324338299057687,"median_score":0.026422765105962753,"q1_score":0.01799336913973093,"q3_score":0.038904743269085884,"avg_token_efficiency":1.0018064053489053,"is_incomplete":false,"total_prompt_cost_bin":1.3502227199999994,"total_cache_read_cost_bin":0.10808159999999993,"total_completion_cost_bin":0.02216015999999999,"total_reasoning_cost_bin":0,"total_cost_bin":1.4804644799999995,"total_on_demand_input_cost_bin":1.7825491199999985,"ci_99_lower":0.07244786266307572,"ci_99_upper":0.23445072332711311,"ci_98_lower":0.07566705234208623,"ci_98_upper":0.22532774151632906,"ci_95_lower":0.09297687642183608,"ci_95_upper":0.21427882436959036,"ci_90_lower":0.10297550094580658,"ci_90_upper":0.19758316316433902,"ci_75_lower":0.11308208263300594,"ci_75_upper":0.18074702749817712,"ci_50_lower":0.1290588359409509,"ci_50_upper":0.1685533683997968}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high","low","none"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Tencent: Hy3","provider_name":"Tencent","run_provider_names":["Tencent Cloud"],"pricing_metadata":{"prompt":1.32e-7,"completion":5.28e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-06T13:20:48.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-08-13T02:54:29.254Z"},{"model_slug":"qwen/qwen3.5-9b","reasoning_mode":null,"overall_metrics":{"auc_1m":0.031856926443782345,"auc_128k":0.19590697879556299,"cum_avg_1m":0.14842128596872828,"cum_avg_128k":0.23747405754996526,"avg_score_overall":0.22599345909514318,"avg_response_time_ms":20995.406521631667,"total_prompt_cost":2.5307567999999994,"total_cache_read_cost":0,"total_completion_cost":0.027783600000000002,"total_reasoning_cost":0,"total_prompt_cost_128k":2.5307567999999994,"total_completion_cost_128k":0.027783600000000002,"total_on_demand_input_cost":2.5307567999999994,"total_on_demand_input_cost_128k":2.5307567999999994,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0263362815165993},"bin_metrics":{"8192":{"avg_score":0.3936141751386458,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4631638146240184,"median_score":0.05924108810722828,"q1_score":0.02300674607977271,"q3_score":0.9941604435443878,"avg_token_efficiency":1.0228079512716761,"is_incomplete":false,"total_prompt_cost_bin":0.05435520000000002,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.004306500000000003,"total_reasoning_cost_bin":0,"total_cost_bin":0.058661700000000025,"total_on_demand_input_cost_bin":0.05435520000000002,"ci_99_lower":0.262479301397354,"ci_99_upper":0.5221921127675888,"ci_98_lower":0.2707348149713362,"ci_98_upper":0.5109697180463408,"ci_95_lower":0.28769045917782926,"ci_95_upper":0.4944571871095832,"ci_90_lower":0.3018996356827848,"ci_90_upper":0.48188566985510983,"ci_75_lower":0.3347569882073003,"ci_75_upper":0.454760552029984,"ci_50_lower":0.35849740630219457,"ci_50_upper":0.427253278131684},"16384":{"avg_score":0.2664501081017079,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4209564316775107,"median_score":0.02890919428318739,"q1_score":0.017163286916911602,"q3_score":0.29409668780863285,"avg_token_efficiency":1.0161194758384637,"is_incomplete":false,"total_prompt_cost_bin":0.11208639999999996,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.0042956999999999995,"total_reasoning_cost_bin":0,"total_cost_bin":0.11638209999999996,"total_on_demand_input_cost_bin":0.11208639999999996,"ci_99_lower":0.15832200690601894,"ci_99_upper":0.3847479036928707,"ci_98_lower":0.15983911699458256,"ci_98_upper":0.37574024593364447,"ci_95_lower":0.17971753469399118,"ci_95_upper":0.3610534956852644,"ci_90_lower":0.19314877860699198,"ci_90_upper":0.3402334160708414,"ci_75_lower":0.21663078657293228,"ci_75_upper":0.31546682746957233,"ci_50_lower":0.23773939912352943,"ci_50_upper":0.2918308490909112},"32768":{"avg_score":0.18163578357951551,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.3472658439151369,"median_score":0.030454088933765888,"q1_score":0.02152482094243169,"q3_score":0.05806853715330362,"avg_token_efficiency":1.0298112216030706,"is_incomplete":false,"total_prompt_cost_bin":0.39901309999999995,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.007987049999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.40700014999999995,"total_on_demand_input_cost_bin":0.39901309999999995,"ci_99_lower":0.11114100994783632,"ci_99_upper":0.2641138444908275,"ci_98_lower":0.11539608285967808,"ci_98_upper":0.2559206296452013,"ci_95_lower":0.12374381854670526,"ci_95_upper":0.2419972501867211,"ci_90_lower":0.13402046043886323,"ci_90_upper":0.2310011189084274,"ci_75_lower":0.14661826294126085,"ci_75_upper":0.21536638316641019,"ci_50_lower":0.1604223383555483,"ci_50_upper":0.19907215329465755},"65536":{"avg_score":0.20726709887659286,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.37260980390003506,"median_score":0.03042743355035782,"q1_score":0.01959412172436714,"q3_score":0.05105948448181152,"avg_token_efficiency":1.020990401222107,"is_incomplete":false,"total_prompt_cost_bin":0.5676675999999997,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.005199449999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.5728670499999997,"total_on_demand_input_cost_bin":0.5676675999999997,"ci_99_lower":0.11317624521733481,"ci_99_upper":0.3228006090559066,"ci_98_lower":0.12138199316466447,"ci_98_upper":0.30764769463112357,"ci_95_lower":0.13451093116117752,"ci_95_upper":0.28932360269889396,"ci_90_lower":0.14612774778910748,"ci_90_upper":0.272425438191276,"ci_75_lower":0.16053678139989427,"ci_75_upper":0.2536726018961738,"ci_50_lower":0.17694077424444804,"ci_50_upper":0.23012149473720725},"131072":{"avg_score":0.13840312205336408,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.31115786302391135,"median_score":0.02570694126188755,"q1_score":0.014744995627552271,"q3_score":0.03784385696053505,"avg_token_efficiency":1.0368354936245263,"is_incomplete":false,"total_prompt_cost_bin":1.3976344999999997,"total_cache_read_cost_bin":0,"total_completion_cost_bin":0.005994900000000004,"total_reasoning_cost_bin":0,"total_cost_bin":1.4036293999999996,"total_on_demand_input_cost_bin":1.3976344999999997,"ci_99_lower":0.06504655665861561,"ci_99_upper":0.22046105564658944,"ci_98_lower":0.07057651754344352,"ci_98_upper":0.21387345576514172,"ci_95_lower":0.0867365976132665,"ci_95_upper":0.2021692115530367,"ci_90_lower":0.09223830081303197,"ci_90_upper":0.18473710189019646,"ci_75_lower":0.10403611767953444,"ci_75_upper":0.16926258833693494,"ci_50_lower":0.11745498989577613,"ci_50_upper":0.15705489262359812}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Qwen: Qwen3.5-9B","provider_name":"Qwen","run_provider_names":["Together"],"pricing_metadata":{"prompt":1e-7,"completion":1.5e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-10T14:19:56.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-21T05:27:42.203Z"},{"model_slug":"xiaomi/mimo-v2-omni","reasoning_mode":null,"overall_metrics":{"auc_1m":0.02768474252300973,"auc_128k":0.18467426483888444,"cum_avg_1m":0.1345008007394018,"cum_avg_128k":0.21520128118304288,"avg_score_overall":0.20563760871973952,"avg_response_time_ms":5593.787023875339,"total_prompt_cost":0.31118680000000004,"total_cache_read_cost":1.898839040000001,"total_completion_cost":0.3734,"total_reasoning_cost":0,"total_prompt_cost_128k":0.31118680000000004,"total_completion_cost_128k":0.3734,"total_on_demand_input_cost":9.805381999999994,"total_on_demand_input_cost_128k":9.805381999999994,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0076901169504333},"bin_metrics":{"8192":{"avg_score":0.3056474497599993,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4308628949056454,"median_score":0.04102826118469238,"q1_score":0.024191271979361773,"q3_score":0.9994779676198959,"avg_token_efficiency":1.029067110960312,"is_incomplete":false,"total_prompt_cost_bin":0.037007999999999985,"total_cache_read_cost_bin":0.0363264,"total_completion_cost_bin":0.05709999999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.13043439999999998,"total_on_demand_input_cost_bin":0.2186400000000001,"ci_99_lower":0.1928164309775166,"ci_99_upper":0.43613778123175145,"ci_98_lower":0.2009967790696537,"ci_98_upper":0.4189229860927443,"ci_95_lower":0.21744720489805333,"ci_95_upper":0.4003894847477204,"ci_90_lower":0.22820237158884993,"ci_90_upper":0.3822418632751214,"ci_75_lower":0.2512545518111438,"ci_75_upper":0.36051161795112424,"ci_50_lower":0.2757247462970554,"ci_50_upper":0.33526347151928343},"16384":{"avg_score":0.28214340239064767,"min_score":0.008738393895328045,"max_score":1,"n_tests":80,"stddev":0.43101942798058107,"median_score":0.030966666527092457,"q1_score":0.019256314728409052,"q3_score":0.9986956864595413,"avg_token_efficiency":0.9996769778966689,"is_incomplete":false,"total_prompt_cost_bin":0.05548800000000007,"total_cache_read_cost_bin":0.07696383999999996,"total_completion_cost_bin":0.05545800000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.18790984000000005,"total_on_demand_input_cost_bin":0.4403071999999997,"ci_99_lower":0.17346280768507857,"ci_99_upper":0.4048701554992003,"ci_98_lower":0.18358492184674832,"ci_98_upper":0.39322632076335134,"ci_95_lower":0.19539959237881702,"ci_95_upper":0.37956132891005834,"ci_90_lower":0.20821145292487928,"ci_90_upper":0.3563166497950442,"ci_75_lower":0.23250948205241,"ci_75_upper":0.3321834510832559,"ci_50_lower":0.24750066582928412,"ci_50_upper":0.3076717617252143},"32768":{"avg_score":0.18291869883678907,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.35142126478554037,"median_score":0.031038818880915642,"q1_score":0.021027777809649706,"q3_score":0.05530564393848181,"avg_token_efficiency":1.018682852419234,"is_incomplete":false,"total_prompt_cost_bin":0.13715200000000008,"total_cache_read_cost_bin":0.28853248000000015,"total_completion_cost_bin":0.10494600000000005,"total_reasoning_cost_bin":0,"total_cost_bin":0.5306304800000003,"total_on_demand_input_cost_bin":1.579814399999998,"ci_99_lower":0.11202729394856278,"ci_99_upper":0.2663949374026561,"ci_98_lower":0.11616070223073406,"ci_98_upper":0.2579592792661895,"ci_95_lower":0.12489738052686067,"ci_95_upper":0.2450796698281706,"ci_90_lower":0.13441099863429212,"ci_90_upper":0.23232622470446476,"ci_75_lower":0.14713927903336727,"ci_75_upper":0.21688995102714914,"ci_50_lower":0.1613110641894095,"ci_50_upper":0.2003706864484497},"65536":{"avg_score":0.21206581394790727,"min_score":0.006896551698446274,"max_score":1,"n_tests":85,"stddev":0.3817256156582216,"median_score":0.030886195600032806,"q1_score":0.02083333395421505,"q3_score":0.05164169520139694,"avg_token_efficiency":0.9958420640773624,"is_incomplete":false,"total_prompt_cost_bin":0.02778399999999999,"total_cache_read_cost_bin":0.4373913599999998,"total_completion_cost_bin":0.06995799999999999,"total_reasoning_cost_bin":0,"total_cost_bin":0.5351333599999998,"total_on_demand_input_cost_bin":2.214740800000001,"ci_99_lower":0.11885080079276882,"ci_99_upper":0.3273803881776026,"ci_98_lower":0.1218499881526017,"ci_98_upper":0.3145474698881564,"ci_95_lower":0.1416549875510528,"ci_95_upper":0.2938440879159058,"ci_90_lower":0.15274694406591793,"ci_90_upper":0.2801128855301067,"ci_75_lower":0.16573357893674473,"ci_75_upper":0.2581639997414587,"ci_50_lower":0.17922437272637207,"ci_50_upper":0.23541327547829816},"131072":{"avg_score":0.09323104097987119,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.24606416672650613,"median_score":0.026631157845258713,"q1_score":0.01768844574689865,"q3_score":0.03545357286930084,"avg_token_efficiency":0.9925732632996084,"is_incomplete":false,"total_prompt_cost_bin":0.05375479999999996,"total_cache_read_cost_bin":1.0596249600000012,"total_completion_cost_bin":0.08593800000000001,"total_reasoning_cost_bin":0,"total_cost_bin":1.1993177600000011,"total_on_demand_input_cost_bin":5.351879599999997,"ci_99_lower":0.03832570852894896,"ci_99_upper":0.16834715512656895,"ci_98_lower":0.04540648643377246,"ci_98_upper":0.15913827374819395,"ci_95_lower":0.052635551841355384,"ci_95_upper":0.14120786494312093,"ci_90_lower":0.0557588823826857,"ci_90_upper":0.1318073513282427,"ci_75_lower":0.06491318953198662,"ci_75_upper":0.12077825957314718,"ci_50_lower":0.07488029802871361,"ci_50_upper":0.1090059608551086}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Xiaomi: MiMo-V2-Omni","provider_name":"Xiaomi","run_provider_names":["Xiaomi"],"pricing_metadata":{"prompt":4e-7,"completion":0.000002,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-18T19:55:03.000Z","api_updated_at":"2026-04-25T12:33:03.020Z","latest_run_timestamp":"2026-04-20T05:39:35.540Z"},{"model_slug":"mistralai/mistral-medium-3.5","reasoning_mode":null,"overall_metrics":{"auc_1m":0.024435380672326437,"auc_128k":0.16089225693486367,"cum_avg_1m":0.11556424867274322,"cum_avg_128k":0.18490279787638914,"avg_score_overall":0.1763873216821158,"avg_response_time_ms":10376.386399828698,"total_prompt_cost":28.290780000000005,"total_cache_read_cost":9.3342,"total_completion_cost":1.3389749999999996,"total_reasoning_cost":0,"total_prompt_cost_128k":28.290780000000005,"total_completion_cost_128k":1.3389749999999996,"total_on_demand_input_cost":37.624980000000015,"total_on_demand_input_cost_128k":37.624980000000015,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.024446883026699},"bin_metrics":{"8192":{"avg_score":0.32046539498260246,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.4363124776580445,"median_score":0.04400465451180935,"q1_score":0.02361624175682664,"q3_score":0.9992467314004898,"avg_token_efficiency":1.0249270764368836,"is_incomplete":false,"total_prompt_cost_bin":0.796728,"total_cache_read_cost_bin":0.020279999999999996,"total_completion_cost_bin":0.19863000000000006,"total_reasoning_cost_bin":0,"total_cost_bin":1.015638,"total_on_demand_input_cost_bin":0.8170079999999998,"ci_99_lower":0.20714155274967197,"ci_99_upper":0.45036177339981076,"ci_98_lower":0.2110029337799642,"ci_98_upper":0.4462786637017271,"ci_95_lower":0.23103358451189707,"ci_95_upper":0.414246986998769,"ci_90_lower":0.2412651138956426,"ci_90_upper":0.3961064050294226,"ci_75_lower":0.26365790355630453,"ci_75_upper":0.3745361407389282,"ci_50_lower":0.28902064331632576,"ci_50_upper":0.3516102654568385},"16384":{"avg_score":0.16612842225003988,"min_score":0.008316008374094963,"max_score":1,"n_tests":80,"stddev":0.3370225375303565,"median_score":0.02652297355234623,"q1_score":0.01834366749972105,"q3_score":0.046110667288303375,"avg_token_efficiency":1.0153834630552774,"is_incomplete":false,"total_prompt_cost_bin":1.532256,"total_cache_read_cost_bin":0.14572799999999997,"total_completion_cost_bin":0.2034825,"total_reasoning_cost_bin":0,"total_cost_bin":1.8814665,"total_on_demand_input_cost_bin":1.6779840000000004,"ci_99_lower":0.07900922589906258,"ci_99_upper":0.2811328025521943,"ci_98_lower":0.09028483598737512,"ci_98_upper":0.2588050428262213,"ci_95_lower":0.09942656816245289,"ci_95_upper":0.23904377685685174,"ci_90_lower":0.10503130756318568,"ci_90_upper":0.2269467908988008,"ci_75_lower":0.1233503411305719,"ci_75_upper":0.20736117384658428,"ci_50_lower":0.14031383860565255,"ci_50_upper":0.18758319459448106},"32768":{"avg_score":0.15036171229953385,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.30439598342565816,"median_score":0.03570779412984848,"q1_score":0.024707610718905926,"q3_score":0.05748591851443052,"avg_token_efficiency":1.0368976087081272,"is_incomplete":false,"total_prompt_cost_bin":5.159843999999998,"total_cache_read_cost_bin":0.8703120000000018,"total_completion_cost_bin":0.4024049999999997,"total_reasoning_cost_bin":0,"total_cost_bin":6.432561,"total_on_demand_input_cost_bin":6.030156000000002,"ci_99_lower":0.09101622224403615,"ci_99_upper":0.22729101288575773,"ci_98_lower":0.09454736032540334,"ci_98_upper":0.21787504882479825,"ci_95_lower":0.10254642405026757,"ci_95_upper":0.20371502950523898,"ci_90_lower":0.10805701607725282,"ci_90_upper":0.1931192302220391,"ci_75_lower":0.11976841706328901,"ci_75_upper":0.17875084958444967,"ci_50_lower":0.13231658010473063,"ci_50_upper":0.16591024435525897},"65536":{"avg_score":0.20131977342945687,"min_score":0,"max_score":1,"n_tests":85,"stddev":0.3641946676304608,"median_score":0.025830257683992386,"q1_score":0.01719479262828827,"q3_score":0.08283518254756927,"avg_token_efficiency":1.0197047828224648,"is_incomplete":false,"total_prompt_cost_bin":6.603684000000001,"total_cache_read_cost_bin":1.9006080000000019,"total_completion_cost_bin":0.24189000000000002,"total_reasoning_cost_bin":0,"total_cost_bin":8.746182000000003,"total_on_demand_input_cost_bin":8.504292000000005,"ci_99_lower":0.10890628185638171,"ci_99_upper":0.31361108241405555,"ci_98_lower":0.11720299153742106,"ci_98_upper":0.29841818824914446,"ci_95_lower":0.12923955895677758,"ci_95_upper":0.28199682727880665,"ci_90_lower":0.14120581084643216,"ci_90_upper":0.2667672515879659,"ci_75_lower":0.15531351268222993,"ci_75_upper":0.24684057887985972,"ci_50_lower":0.1738253382322214,"ci_50_upper":0.22280502779929737},"131072":{"avg_score":0.08623868642031278,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.2313567159753957,"median_score":0.024523161351680756,"q1_score":0.014202881138771772,"q3_score":0.03802657686173916,"avg_token_efficiency":1.0185870572945108,"is_incomplete":false,"total_prompt_cost_bin":14.198268000000008,"total_cache_read_cost_bin":6.397271999999995,"total_completion_cost_bin":0.2925674999999999,"total_reasoning_cost_bin":0,"total_cost_bin":20.888107500000004,"total_on_demand_input_cost_bin":20.595540000000007,"ci_99_lower":0.03583712574750076,"ci_99_upper":0.15604292697266411,"ci_98_lower":0.0407060405221831,"ci_98_upper":0.14405250168238196,"ci_95_lower":0.04709759046316816,"ci_95_upper":0.13270837303513242,"ci_90_lower":0.05167289134001957,"ci_90_upper":0.12310182490946672,"ci_75_lower":0.059366422079761234,"ci_75_upper":0.11007131609864515,"ci_50_lower":0.06971333655686958,"ci_50_upper":0.09944445254095802}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["high"],"reasoning_default":"high","sponsors":[],"openrouter_name":"Mistral: Mistral Medium 3.5","provider_name":"Mistralai","run_provider_names":["Mistral"],"pricing_metadata":{"prompt":0.0000015,"completion":0.0000075,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-04-30T17:33:59.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-06-01T03:18:28.887Z"},{"model_slug":"anthropic/claude-haiku-4.5","reasoning_mode":null,"overall_metrics":{"auc_1m":0.024189412032017892,"auc_128k":0.15578591013588863,"cum_avg_1m":0.11573875195689746,"cum_avg_128k":0.18518200313103594,"avg_score_overall":0.17681182713189147,"avg_response_time_ms":8818.283804144741,"total_prompt_cost":0.4893059999999996,"total_cache_read_cost":2.723477899999998,"total_completion_cost":1.056945,"total_reasoning_cost":0,"total_prompt_cost_128k":0.4893059999999996,"total_completion_cost_128k":1.056945,"total_on_demand_input_cost":27.72408499999999,"total_on_demand_input_cost_128k":27.72408499999999,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.1332025826383496},"bin_metrics":{"8192":{"avg_score":0.2898932094612974,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.42139632132861365,"median_score":0.03836905211210251,"q1_score":0.02585004549473524,"q3_score":0.9539165198802948,"avg_token_efficiency":1.1377505627360662,"is_incomplete":false,"total_prompt_cost_bin":0.10287100000000012,"total_cache_read_cost_bin":0.05011770000000003,"total_completion_cost_bin":0.162185,"total_reasoning_cost_bin":0,"total_cost_bin":0.3151737000000001,"total_on_demand_input_cost_bin":0.6040479999999999,"ci_99_lower":0.18523920157048995,"ci_99_upper":0.41136065733627764,"ci_98_lower":0.18908261579717509,"ci_98_upper":0.40116107125888806,"ci_95_lower":0.20317319694459002,"ci_95_upper":0.38566105495745434,"ci_90_lower":0.2160235978921264,"ci_90_upper":0.3636927069042576,"ci_75_lower":0.2390490476936975,"ci_75_upper":0.34124018993388744,"ci_50_lower":0.2595885254489986,"ci_50_upper":0.3179106885429064},"16384":{"avg_score":0.21862364450353197,"min_score":0.003010752610862255,"max_score":1,"n_tests":80,"stddev":0.3861687006586811,"median_score":0.02875489741563797,"q1_score":0.017131523694843054,"q3_score":0.05426719877868891,"avg_token_efficiency":1.1227572802981338,"is_incomplete":false,"total_prompt_cost_bin":0.08598600000000005,"total_cache_read_cost_bin":0.11528299999999994,"total_completion_cost_bin":0.1554250000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.35669400000000007,"total_on_demand_input_cost_bin":1.2388159999999997,"ci_99_lower":0.12268567780099693,"ci_99_upper":0.32947467912509454,"ci_98_lower":0.1322848992358195,"ci_98_upper":0.3148532372440095,"ci_95_lower":0.14434684161868064,"ci_95_upper":0.3015983902971493,"ci_90_lower":0.1558856788734556,"ci_90_upper":0.28239134518516923,"ci_75_lower":0.1705304062044888,"ci_75_upper":0.2652448079119495,"ci_50_lower":0.18467186999187107,"ci_50_upper":0.24337716997251843},"32768":{"avg_score":0.15221190686091124,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.3167109931589567,"median_score":0.029898938722908497,"q1_score":0.019410750828683376,"q3_score":0.05364569369703531,"avg_token_efficiency":1.1449043855358674,"is_incomplete":false,"total_prompt_cost_bin":0.0708920000000001,"total_cache_read_cost_bin":0.43719000000000013,"total_completion_cost_bin":0.2959800000000001,"total_reasoning_cost_bin":0,"total_cost_bin":0.8040620000000003,"total_on_demand_input_cost_bin":4.442792,"ci_99_lower":0.08777098646510721,"ci_99_upper":0.2304402803066392,"ci_98_lower":0.09365660282160046,"ci_98_upper":0.22134675099319254,"ci_95_lower":0.1016013183434124,"ci_95_upper":0.2087248015921967,"ci_90_lower":0.10806413952942581,"ci_90_upper":0.19576759776066274,"ci_75_lower":0.11933848256526021,"ci_75_upper":0.18176455120727256,"ci_50_lower":0.13303242468100507,"ci_50_upper":0.1676461034512613},"65536":{"avg_score":0.1732729203259463,"min_score":0.0038729666266590357,"max_score":1,"n_tests":85,"stddev":0.34979230889628393,"median_score":0.025327511131763458,"q1_score":0.015789473429322243,"q3_score":0.040449436753988266,"avg_token_efficiency":1.131579127860358,"is_incomplete":false,"total_prompt_cost_bin":0.07789800000000008,"total_cache_read_cost_bin":0.6213647999999995,"total_completion_cost_bin":0.19649999999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.8957627999999996,"total_on_demand_input_cost_bin":6.291545999999997,"ci_99_lower":0.08214343388099224,"ci_99_upper":0.277988792205065,"ci_98_lower":0.09368344164404141,"ci_98_upper":0.26478973141501605,"ci_95_lower":0.10409637658925289,"ci_95_upper":0.25302850309670355,"ci_90_lower":0.1152725087139098,"ci_90_upper":0.240195776168257,"ci_75_lower":0.12849607510416877,"ci_75_upper":0.21834524171118735,"ci_50_lower":0.1481843337074251,"ci_50_upper":0.19557703644352253},"131072":{"avg_score":0.09190833450349283,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.23797873568507502,"median_score":0.026843026280403137,"q1_score":0.015636660624295473,"q3_score":0.038330405950546265,"avg_token_efficiency":1.1236718471186096,"is_incomplete":false,"total_prompt_cost_bin":0.1516589999999993,"total_cache_read_cost_bin":1.4995223999999985,"total_completion_cost_bin":0.24685500000000002,"total_reasoning_cost_bin":0,"total_cost_bin":1.8980363999999978,"total_on_demand_input_cost_bin":15.146882999999995,"ci_99_lower":0.03816129911586968,"ci_99_upper":0.16280424346231842,"ci_98_lower":0.04475380696044349,"ci_98_upper":0.15428705846988008,"ci_95_lower":0.051988467226681996,"ci_95_upper":0.13954485903428504,"ci_90_lower":0.055428771351495744,"ci_90_upper":0.1303230576956355,"ci_75_lower":0.06509199690382285,"ci_75_upper":0.11710551595660063,"ci_50_lower":0.07449830352505037,"ci_50_upper":0.10620997179507558}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":200000,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Anthropic: Claude Haiku 4.5","provider_name":"Anthropic","run_provider_names":["Amazon Bedrock"],"pricing_metadata":{"prompt":0.000001,"completion":0.000005,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2025-10-15T17:00:38.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T00:48:08.324Z"},{"model_slug":"mistralai/mistral-small-4","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.023447390898708537,"auc_128k":0.1615245892784219,"cum_avg_1m":0.1676103744727957,"cum_avg_128k":0.2681765991564731,"avg_score_overall":0.25353239414384127,"avg_response_time_ms":12165.12257120432,"total_prompt_cost":0.3894468000000001,"total_cache_read_cost":0.3373051200000003,"total_completion_cost":0.474324,"total_reasoning_cost":0.40998,"total_prompt_cost_128k":0.3894468000000001,"total_completion_cost_128k":0.474324,"total_on_demand_input_cost":3.762498,"total_on_demand_input_cost_128k":3.762498,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0244468830266988},"bin_metrics":{"8192":{"avg_score":0.5872367579606361,"min_score":0.010362694039940834,"max_score":1,"n_tests":80,"stddev":0.47714425678525174,"median_score":0.9975277185440063,"q1_score":0.033856010995805264,"q3_score":1,"avg_token_efficiency":1.0249270764368836,"is_incomplete":false,"total_prompt_cost_bin":0.03420480000000001,"total_cache_read_cost_bin":0.004749600000000002,"total_completion_cost_bin":0.08177819999999998,"total_reasoning_cost_bin":0.073998,"total_cost_bin":0.19473059999999998,"total_on_demand_input_cost_bin":0.0817008,"ci_99_lower":0.44314876058994557,"ci_99_upper":0.730300590867817,"ci_98_lower":0.45901128473551944,"ci_98_upper":0.7081058699459537,"ci_95_lower":0.4851225842602435,"ci_95_upper":0.689014271502965,"ci_90_lower":0.5028507871052715,"ci_90_upper":0.6728829973761459,"ci_75_lower":0.5285231998248492,"ci_75_upper":0.650019163408433,"ci_50_lower":0.5505695904488676,"ci_50_upper":0.6223088480561274},"16384":{"avg_score":0.34225706638826525,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.44695888735027955,"median_score":0.03733566962182522,"q1_score":0.021833826787769794,"q3_score":0.9988667070865631,"avg_token_efficiency":1.0153834630552774,"is_incomplete":false,"total_prompt_cost_bin":0.04271040000000004,"total_cache_read_cost_bin":0.012508799999999999,"total_completion_cost_bin":0.0900582,"total_reasoning_cost_bin":0.08135760000000004,"total_cost_bin":0.2266350000000001,"total_on_demand_input_cost_bin":0.16779840000000004,"ci_99_lower":0.21987492919637588,"ci_99_upper":0.4725764935413171,"ci_98_lower":0.22582919418509118,"ci_98_upper":0.46210835047430004,"ci_95_lower":0.24448061393741227,"ci_95_upper":0.4393809431363479,"ci_90_lower":0.2574232645244046,"ci_90_upper":0.4215363536226505,"ci_75_lower":0.2845579276723583,"ci_75_upper":0.3990612010045879,"ci_50_lower":0.3100032525289862,"ci_50_upper":0.37450105469179107},"32768":{"avg_score":0.23791090558241942,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.38706472704316963,"median_score":0.034008877351880074,"q1_score":0.020621465984731913,"q3_score":0.10706287063658237,"avg_token_efficiency":1.0368976087081265,"is_incomplete":false,"total_prompt_cost_bin":0.07117320000000002,"total_cache_read_cost_bin":0.053184240000000015,"total_completion_cost_bin":0.14862180000000005,"total_reasoning_cost_bin":0.1304514,"total_cost_bin":0.40343064000000006,"total_on_demand_input_cost_bin":0.6030156000000002,"ci_99_lower":0.15349399705687947,"ci_99_upper":0.31637994545549564,"ci_98_lower":0.16357490461398436,"ci_98_upper":0.31225514184539754,"ci_95_lower":0.17619857676977785,"ci_95_upper":0.301280502458394,"ci_90_lower":0.18495957818180186,"ci_90_upper":0.2931097786498653,"ci_75_lower":0.19865326372853492,"ci_75_upper":0.27457259774254295,"ci_50_lower":0.21514939581410383,"ci_50_upper":0.25732067294987193},"65536":{"avg_score":0.10410954023108762,"min_score":0,"max_score":0.9996388554573059,"n_tests":85,"stddev":0.22006087685551204,"median_score":0.029675252735614777,"q1_score":0.01919323392212391,"q3_score":0.06336311995983124,"avg_token_efficiency":1.0197047828224648,"is_incomplete":false,"total_prompt_cost_bin":0.08065560000000005,"total_cache_read_cost_bin":0.0769773600000001,"total_completion_cost_bin":0.0713232,"total_reasoning_cost_bin":0.058936800000000004,"total_cost_bin":0.28789296000000014,"total_on_demand_input_cost_bin":0.8504291999999997,"ci_99_lower":0.052172530251856454,"ci_99_upper":0.16999055579625177,"ci_98_lower":0.05607695632241666,"ci_98_upper":0.166696983317735,"ci_95_lower":0.06065117391634403,"ci_95_upper":0.1530835090320119,"ci_90_lower":0.06704065394067371,"ci_90_upper":0.14478129534680834,"ci_75_lower":0.07622243102009901,"ci_75_upper":0.1307713367880377,"ci_50_lower":0.08678894574918292,"ci_50_upper":0.11796436571954366},"131072":{"avg_score":0.069368725619957,"min_score":0,"max_score":1,"n_tests":103,"stddev":0.17185444263139044,"median_score":0.024551996961236,"q1_score":0.015789721626788378,"q3_score":0.03902499936521053,"avg_token_efficiency":1.0185870572945106,"is_incomplete":false,"total_prompt_cost_bin":0.16070279999999998,"total_cache_read_cost_bin":0.18988512000000018,"total_completion_cost_bin":0.08254259999999995,"total_reasoning_cost_bin":0.0652362,"total_cost_bin":0.49836672000000015,"total_on_demand_input_cost_bin":2.059554,"ci_99_lower":0.033291624727075725,"ci_99_upper":0.12096372457704893,"ci_98_lower":0.03639631675036389,"ci_98_upper":0.11200119222577624,"ci_95_lower":0.0397102016366756,"ci_95_upper":0.10437970785390206,"ci_90_lower":0.045088541609669124,"ci_90_upper":0.0978658298340183,"ci_75_lower":0.05060081069303441,"ci_75_upper":0.08781683451249671,"ci_50_lower":0.05697054668099086,"ci_50_upper":0.08009609792074099}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Mistral: Mistral Small 4","provider_name":"Mistralai","run_provider_names":["Mistral"],"pricing_metadata":{"prompt":1.5e-7,"completion":6e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-16T21:14:45.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T06:00:43.692Z"},{"model_slug":"mistralai/mistral-small-4","reasoning_mode":null,"overall_metrics":{"auc_1m":0.01731378866984677,"auc_128k":0.10777102345836846,"cum_avg_1m":0.09198726437519746,"cum_avg_128k":0.14717962300031592,"avg_score_overall":0.13961116128460666,"avg_response_time_ms":3716.355572787198,"total_prompt_cost":0.384714,"total_cache_read_cost":0.33777840000000026,"total_completion_cost":0.08028539999999998,"total_reasoning_cost":0,"total_prompt_cost_128k":0.384714,"total_completion_cost_128k":0.08028539999999998,"total_on_demand_input_cost":3.762497999999998,"total_on_demand_input_cost_128k":3.762497999999998,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0244468830266988},"bin_metrics":{"8192":{"avg_score":0.24511871760478243,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.39112904288718,"median_score":0.035754358395934105,"q1_score":0.021321858279407024,"q3_score":0.20490367338061333,"avg_token_efficiency":1.0249270764368832,"is_incomplete":false,"total_prompt_cost_bin":0.03482879999999999,"total_cache_read_cost_bin":0.004687199999999997,"total_completion_cost_bin":0.015351599999999995,"total_reasoning_cost_bin":0,"total_cost_bin":0.05486759999999998,"total_on_demand_input_cost_bin":0.08170079999999995,"ci_99_lower":0.14492104611266407,"ci_99_upper":0.358639800478908,"ci_98_lower":0.1541354490974918,"ci_98_upper":0.3440622347815078,"ci_95_lower":0.16694591042702087,"ci_95_upper":0.324260316081636,"ci_90_lower":0.17934031893120844,"ci_90_upper":0.31342112727055793,"ci_75_lower":0.19611505015127478,"ci_75_upper":0.2904435149590427,"ci_50_lower":0.21371213596430605,"ci_50_upper":0.2720230539620388},"16384":{"avg_score":0.21166587925399655,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.36582778574796215,"median_score":0.02511936891824007,"q1_score":0.013496800092980266,"q3_score":0.13774743676185608,"avg_token_efficiency":1.0153834630552776,"is_incomplete":false,"total_prompt_cost_bin":0.04105200000000003,"total_cache_read_cost_bin":0.012674639999999996,"total_completion_cost_bin":0.013513199999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.06723984000000002,"total_on_demand_input_cost_bin":0.16779840000000007,"ci_99_lower":0.12215208461447037,"ci_99_upper":0.3249398919426603,"ci_98_lower":0.1286709567566286,"ci_98_upper":0.3038487609025323,"ci_95_lower":0.13828802728821757,"ci_95_upper":0.2876617815217469,"ci_90_lower":0.14936538397654658,"ci_90_upper":0.27715594787325243,"ci_75_lower":0.16546971593343188,"ci_75_upper":0.25395515416239506,"ci_50_lower":0.1828395489399554,"ci_50_upper":0.23484970058780164},"32768":{"avg_score":0.11753416078863665,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.26307101403412153,"median_score":0.026493425481021404,"q1_score":0.013330599293112755,"q3_score":0.058030507527291775,"avg_token_efficiency":1.0368976087081265,"is_incomplete":false,"total_prompt_cost_bin":0.0674724,"total_cache_read_cost_bin":0.05355432000000004,"total_completion_cost_bin":0.022996799999999998,"total_reasoning_cost_bin":0,"total_cost_bin":0.14402352000000002,"total_on_demand_input_cost_bin":0.6030156000000003,"ci_99_lower":0.0678640886289256,"ci_99_upper":0.18528299216470365,"ci_98_lower":0.0724875070876442,"ci_98_upper":0.17756463014087567,"ci_95_lower":0.07540074265814517,"ci_95_upper":0.16429617702931007,"ci_90_lower":0.08228997816348806,"ci_90_upper":0.15435255319265276,"ci_75_lower":0.09102369378610199,"ci_75_upper":0.14220041465801736,"ci_50_lower":0.10069419396698803,"ci_50_upper":0.13120338154583483},"65536":{"avg_score":0.08879363120478742,"min_score":0,"max_score":0.9997035264968872,"n_tests":85,"stddev":0.1902174209931178,"median_score":0.02631578966975212,"q1_score":0.014461930841207504,"q3_score":0.06903991103172302,"avg_token_efficiency":1.0197047828224648,"is_incomplete":false,"total_prompt_cost_bin":0.0806604,"total_cache_read_cost_bin":0.07697688000000011,"total_completion_cost_bin":0.011484599999999996,"total_reasoning_cost_bin":0,"total_cost_bin":0.16912188000000009,"total_on_demand_input_cost_bin":0.8504291999999996,"ci_99_lower":0.04628936604541891,"ci_99_upper":0.14687194205489118,"ci_98_lower":0.04827259815895163,"ci_98_upper":0.13942260277967025,"ci_95_lower":0.052806442278464705,"ci_95_upper":0.13135409541428086,"ci_90_lower":0.057479275488020745,"ci_90_upper":0.12376606572994635,"ci_75_lower":0.06543618306202595,"ci_75_upper":0.11321559513590354,"ci_50_lower":0.07360815625117326,"ci_50_upper":0.10112661152311107},"131072":{"avg_score":0.07278572614937659,"min_score":0,"max_score":0.9998297095298767,"n_tests":103,"stddev":0.2030635904401268,"median_score":0.021978022530674934,"q1_score":0.010699544567614794,"q3_score":0.03163979575037956,"avg_token_efficiency":1.0185870572945106,"is_incomplete":false,"total_prompt_cost_bin":0.16070040000000002,"total_cache_read_cost_bin":0.18988536000000011,"total_completion_cost_bin":0.0169392,"total_reasoning_cost_bin":0,"total_cost_bin":0.3675249600000001,"total_on_demand_input_cost_bin":2.059553999999998,"ci_99_lower":0.029248277625516603,"ci_99_upper":0.13325161799689011,"ci_98_lower":0.0324125224388955,"ci_98_upper":0.12198795319401466,"ci_95_lower":0.038146095628142626,"ci_95_upper":0.11151135354947063,"ci_90_lower":0.04234599468978664,"ci_90_upper":0.10558663603693733,"ci_75_lower":0.05045537192892121,"ci_75_upper":0.09420988357849691,"ci_50_lower":0.058589378339622744,"ci_50_upper":0.08494310857257942}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Mistral: Mistral Small 4","provider_name":"Mistralai","run_provider_names":["Mistral"],"pricing_metadata":{"prompt":1.5e-7,"completion":6e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-03-16T21:14:45.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-04-20T05:43:19.849Z"},{"model_slug":"poolside/laguna-xs-2.1","reasoning_mode":"enabled","overall_metrics":{"auc_1m":0.013796296111225435,"auc_128k":0.09474927520679062,"cum_avg_1m":0.09158724561996376,"cum_avg_128k":0.14653959299194202,"avg_score_overall":0.1400229633467807,"avg_response_time_ms":15509.56145521432,"total_prompt_cost":0.025228919999999995,"total_cache_read_cost":0.7516929600000004,"total_completion_cost":0.0724764,"total_reasoning_cost":0.04124051999999999,"total_prompt_cost_128k":0.025228919999999995,"total_completion_cost_128k":0.0724764,"total_on_demand_input_cost":1.5286148400000004,"total_on_demand_input_cost_128k":1.5286148400000004,"total_runs":484,"total_candidates_generated":null,"avg_token_efficiency_overall":1.0392306758477505},"bin_metrics":{"8192":{"avg_score":0.2992974446096923,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.42102778909116123,"median_score":0.04607783816754818,"q1_score":0.027002280578017235,"q3_score":0.8788188844919205,"avg_token_efficiency":1.0311918406793412,"is_incomplete":false,"total_prompt_cost_bin":0.0060835199999999985,"total_cache_read_cost_bin":0.01340016000000001,"total_completion_cost_bin":0.008591159999999999,"total_reasoning_cost_bin":0.005152679999999999,"total_cost_bin":0.03322752000000001,"total_on_demand_input_cost_bin":0.03288384,"ci_99_lower":0.19084305927716194,"ci_99_upper":0.4254744373143185,"ci_98_lower":0.1976320144510246,"ci_98_upper":0.41008648444426943,"ci_95_lower":0.21235390948102575,"ci_95_upper":0.39212077637028414,"ci_90_lower":0.22464830436976627,"ci_90_upper":0.37479329853609666,"ci_75_lower":0.24591326278823544,"ci_75_upper":0.3517133485511295,"ci_50_lower":0.2692221858727862,"ci_50_upper":0.32781197998992867},"16384":{"avg_score":0.1833099479728844,"min_score":0,"max_score":1,"n_tests":80,"stddev":0.36030372775126207,"median_score":0.025297646410763264,"q1_score":0.016772761940956116,"q3_score":0.04309865552932024,"avg_token_efficiency":1.0393779062261577,"is_incomplete":false,"total_prompt_cost_bin":0.002607840000000001,"total_cache_read_cost_bin":0.03313199999999999,"total_completion_cost_bin":0.009261,"total_reasoning_cost_bin":0.0058212,"total_cost_bin":0.050822039999999985,"total_on_demand_input_cost_bin":0.06887183999999998,"ci_99_lower":0.09754826698149555,"ci_99_upper":0.2812859119535132,"ci_98_lower":0.09974594616680404,"ci_98_upper":0.2713919075601734,"ci_95_lower":0.11019356499076821,"ci_95_upper":0.25847329473093844,"ci_90_lower":0.12150289292272645,"ci_90_upper":0.24584130969218676,"ci_75_lower":0.13628968291086493,"ci_75_upper":0.22327486687354395,"ci_50_lower":0.15751640690286875,"ci_50_upper":0.20711975171871017},"32768":{"avg_score":0.14039844363972354,"min_score":0,"max_score":1,"n_tests":136,"stddev":0.29015389600187036,"median_score":0.030233035795390606,"q1_score":0.01510320883244276,"q3_score":0.057785763405263424,"avg_token_efficiency":1.0479492616049857,"is_incomplete":false,"total_prompt_cost_bin":0.003885119999999998,"total_cache_read_cost_bin":0.11992224000000003,"total_completion_cost_bin":0.029854919999999993,"total_reasoning_cost_bin":0.015089399999999992,"total_cost_bin":0.16875168000000002,"total_on_demand_input_cost_bin":0.2437295999999998,"ci_99_lower":0.0854398348929595,"ci_99_upper":0.21036352003417796,"ci_98_lower":0.08779683961607482,"ci_98_upper":0.2050702334642119,"ci_95_lower":0.09475142701389011,"ci_95_upper":0.19172198734490575,"ci_90_lower":0.10079834171094049,"ci_90_upper":0.18276026178941454,"ci_75_lower":0.11156027146117754,"ci_75_upper":0.16839873245868042,"ci_50_lower":0.12277505271905444,"ci_50_upper":0.15491773489766808},"65536":{"avg_score":0.0683308189844384,"min_score":0,"max_score":0.9022606611251831,"n_tests":85,"stddev":0.1558323459225656,"median_score":0.02631578966975212,"q1_score":0.016786079853773117,"q3_score":0.03881610929965973,"avg_token_efficiency":1.038393552695637,"is_incomplete":false,"total_prompt_cost_bin":0.004267020000000001,"total_cache_read_cost_bin":0.17106960000000027,"total_completion_cost_bin":0.013310160000000001,"total_reasoning_cost_bin":0.009109199999999998,"total_cost_bin":0.19775598000000028,"total_on_demand_input_cost_bin":0.3464062199999997,"ci_99_lower":0.03323568034585675,"ci_99_upper":0.11793679133780739,"ci_98_lower":0.03555177270670366,"ci_98_upper":0.11382635886139947,"ci_95_lower":0.038861712976769706,"ci_95_upper":0.10591835324803148,"ci_90_lower":0.042640094669426185,"ci_90_upper":0.09786220372462753,"ci_75_lower":0.049400096615392936,"ci_75_upper":0.08791349599042031,"ci_50_lower":0.056192662439528196,"ci_50_upper":0.0781548088786247},"131072":{"avg_score":0.041361309752971365,"min_score":0,"max_score":0.893878698348999,"n_tests":103,"stddev":0.1014526204323226,"median_score":0.02096519060432911,"q1_score":0.011633739806711674,"q3_score":0.03622769936919212,"avg_token_efficiency":1.0345389883540208,"is_incomplete":false,"total_prompt_cost_bin":0.00838542,"total_cache_read_cost_bin":0.41416896000000014,"total_completion_cost_bin":0.011459160000000003,"total_reasoning_cost_bin":0.00606804,"total_cost_bin":0.44008158000000014,"total_on_demand_input_cost_bin":0.836723340000001,"ci_99_lower":0.023127255871896495,"ci_99_upper":0.07590746054324113,"ci_98_lower":0.02399350146722338,"ci_98_upper":0.0690213042687742,"ci_95_lower":0.025649427322645954,"ci_95_upper":0.06357379921466717,"ci_90_lower":0.027302111805873264,"ci_90_upper":0.05929982855857249,"ci_75_lower":0.030335292900523662,"ci_75_upper":0.053449431832685385,"ci_50_lower":0.03396840482493591,"ci_50_upper":0.04742221110353463}},"has_incomplete_bins":false,"has_incomplete_bins_128k":false,"max_context_length":262144,"is_new":false,"is_deprecated":false,"has_sufficient_data":true,"unranked":null,"reasoning_levels":["enabled"],"reasoning_default":"enabled","sponsors":[],"openrouter_name":"Poolside: Laguna XS 2.1","provider_name":"Poolside","run_provider_names":["Poolside"],"pricing_metadata":{"prompt":6e-8,"completion":1.2e-7,"reasoning":null,"has_tiered_pricing":true},"api_created_at":"2026-07-02T14:27:09.000Z","api_updated_at":"2026-08-26T16:03:11.042Z","latest_run_timestamp":"2026-07-21T19:10:48.150Z"}]}