{ "schema_version": "2.0.0", "generated_at": "2026-03-27T05:25:50.551344700+00:00", "tool_version": "0.1.0", "machine": { "label": "vivobook-pinto", "os": "windows x86_64", "hostname": "Vivobook-Pinto" }, "hardware": { "cpu_brand": "Intel(R) Core(TM) Ultra 9 285H", "cpu_cores_physical": 16, "cpu_cores_logical": 16, "gpus": [ { "name": "NVIDIA GeForce RTX 4050 Laptop GPU", "vendor": "Nvidia", "vram_mb": 4095 }, { "name": "Intel(R) Arc(TM) 140T GPU (8GB)", "vendor": "Intel", "vram_mb": 2047 } ], "npu": "Intel Gaussian Neural Accelerator | 1.0 TOPS", "memory_total_gb": 15.365234375 }, "test_config": { "backends": [ "cpu", "directml", "openvino_npu" ], "models": [ "qwen2.5-1.5b-instruct", "qwen3-4b" ], "runs_per_prompt": 3, "warmup_runs": 1, "temperature": 0.0, "repetition_penalty": 1.0, "prompt_corpus_hash": "1672e3a7311c9f6f" }, "results": [ { "backend": "cpu", "model_id": "qwen2.5-1.5b-instruct", "cold_start_ttft_ms": 115, "idle_baseline": { "rss_mb": 1925.91796875, "cpu_percent": 0.0 }, "reliability": { "total_runs": 30, "successful_runs": 30, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.3, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 44, "tokens_per_second": 52.625831604003906, "tpot_ms": 19.007874015748033, "total_time_ms": 2458, "total_tokens": 128, "peak_memory_mb": 1938.6015625, "mean_cpu_percent": 584.5318, "peak_cpu_percent": 601.83, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 43, "tokens_per_second": 52.545658111572266, "tpot_ms": 19.03937007874016, "total_time_ms": 2461, "total_tokens": 128, "peak_memory_mb": 1938.69140625, "mean_cpu_percent": 561.30347, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 42, "tokens_per_second": 52.01903533935547, "tpot_ms": 19.236220472440944, "total_time_ms": 2485, "total_tokens": 128, "peak_memory_mb": 1938.6953125, "mean_cpu_percent": 555.89594, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 43.0, "median": 43.0, "p90": 43.8, "p95": 43.9, "std_dev": 0.816496580927726, "min": 42.0, "max": 44.0 }, "tokens_per_second": { "mean": 52.39684168497721, "median": 52.545658111572266, "p90": 52.60979690551758, "p95": 52.617814254760745, "std_dev": 0.26914701474128044, "min": 52.01903533935547, "max": 52.625831604003906 }, "tpot": { "mean": 19.09448818897638, "median": 19.03937007874016, "p90": 19.19685039370079, "p95": 19.216535433070867, "std_dev": 0.1010413467449867, "min": 19.007874015748033, "max": 19.236220472440944 }, "total_time": { "mean": 2468.0, "median": 2461.0, "p90": 2480.2, "p95": 2482.6000000000004, "std_dev": 12.083045973594572, "min": 2458.0, "max": 2485.0 }, "peak_memory_mb": { "mean": 1938.6627604166667, "median": 1938.69140625, "p90": 1938.6945312500002, "p95": 1938.694921875, "std_dev": 0.0433028363237393, "min": 1938.6015625, "max": 1938.6953125 }, "mean_cpu_percent": { "mean": 567.2437337239584, "median": 561.303466796875, "p90": 579.8861328125, "p95": 582.2089660644532, "std_dev": 12.422245068739008, "min": 555.8959350585938, "max": 584.5317993164062 }, "peak_cpu_percent": { "mean": 600.6100056966146, "median": 600.0, "p90": 601.464013671875, "p95": 601.6470153808593, "std_dev": 0.8626783292771913, "min": 600.0, "max": 601.8300170898438 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 118, "tokens_per_second": 51.94465637207031, "tpot_ms": 19.251968503937007, "total_time_ms": 2563, "total_tokens": 128, "peak_memory_mb": 1941.1328125, "mean_cpu_percent": 571.8339, "peak_cpu_percent": 602.79, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 48, "tokens_per_second": 52.391300201416016, "tpot_ms": 19.094488188976378, "total_time_ms": 2473, "total_tokens": 128, "peak_memory_mb": 1942.44140625, "mean_cpu_percent": 562.1163, "peak_cpu_percent": 594.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 49, "tokens_per_second": 52.179107666015625, "tpot_ms": 19.173228346456693, "total_time_ms": 2484, "total_tokens": 128, "peak_memory_mb": 1942.4921875, "mean_cpu_percent": 563.78796, "peak_cpu_percent": 606.39, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 71.66666666666667, "median": 49.0, "p90": 104.2, "p95": 111.1, "std_dev": 32.76515764582181, "min": 48.0, "max": 118.0 }, "tokens_per_second": { "mean": 52.171688079833984, "median": 52.179107666015625, "p90": 52.34886169433594, "p95": 52.37008094787598, "std_dev": 0.18241704099394238, "min": 51.94465637207031, "max": 52.391300201416016 }, "tpot": { "mean": 19.173228346456693, "median": 19.173228346456693, "p90": 19.236220472440944, "p95": 19.244094488188974, "std_dev": 0.06429106936438776, "min": 19.094488188976378, "max": 19.251968503937007 }, "total_time": { "mean": 2506.6666666666665, "median": 2484.0, "p90": 2547.2, "p95": 2555.1, "std_dev": 40.086018621071474, "min": 2473.0, "max": 2563.0 }, "peak_memory_mb": { "mean": 1942.0221354166667, "median": 1942.44140625, "p90": 1942.48203125, "p95": 1942.487109375, "std_dev": 0.6291879006143665, "min": 1941.1328125, "max": 1942.4921875 }, "mean_cpu_percent": { "mean": 565.9127197265625, "median": 563.7879638671875, "p90": 570.2247314453125, "p95": 571.0293273925781, "std_dev": 4.24217928428393, "min": 562.1162719726562, "max": 571.8339233398438 }, "peak_cpu_percent": { "mean": 601.1533406575521, "median": 602.7899780273438, "p90": 605.6700073242188, "p95": 606.0300109863282, "std_dev": 5.07752385281865, "min": 594.280029296875, "max": 606.3900146484375 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 132, "tokens_per_second": 52.37233352661133, "tpot_ms": 19.095652173913045, "total_time_ms": 2328, "total_tokens": 116, "peak_memory_mb": 1945.0078125, "mean_cpu_percent": 555.13763, "peak_cpu_percent": 590.99, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 53, "tokens_per_second": 52.36827087402344, "tpot_ms": 19.104347826086958, "total_time_ms": 2250, "total_tokens": 116, "peak_memory_mb": 1945.81640625, "mean_cpu_percent": 568.2383, "peak_cpu_percent": 600.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 45, "tokens_per_second": 51.60235595703125, "tpot_ms": 19.391304347826086, "total_time_ms": 2275, "total_tokens": 116, "peak_memory_mb": 1946.015625, "mean_cpu_percent": 564.9351, "peak_cpu_percent": 600.0, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 76.66666666666667, "median": 53.0, "p90": 116.2, "p95": 124.1, "std_dev": 39.262648351270904, "min": 45.0, "max": 132.0 }, "tokens_per_second": { "mean": 52.11432011922201, "median": 52.36827087402344, "p90": 52.37152099609375, "p95": 52.37192726135254, "std_dev": 0.36201713017689635, "min": 51.60235595703125, "max": 52.37233352661133 }, "tpot": { "mean": 19.19710144927536, "median": 19.104347826086958, "p90": 19.33391304347826, "p95": 19.362608695652174, "std_dev": 0.13736806505591526, "min": 19.095652173913045, "max": 19.391304347826086 }, "total_time": { "mean": 2284.3333333333335, "median": 2275.0, "p90": 2317.4, "p95": 2322.7, "std_dev": 32.52007926736274, "min": 2250.0, "max": 2328.0 }, "peak_memory_mb": { "mean": 1945.61328125, "median": 1945.81640625, "p90": 1945.97578125, "p95": 1945.9957031249999, "std_dev": 0.4357876646623617, "min": 1945.0078125, "max": 1946.015625 }, "mean_cpu_percent": { "mean": 562.7703450520834, "median": 564.9351196289062, "p90": 567.5776489257813, "p95": 567.9079650878906, "std_dev": 5.563058181844824, "min": 555.1376342773438, "max": 568.23828125 }, "peak_cpu_percent": { "mean": 596.9966634114584, "median": 600.0, "p90": 600.0, "p95": 600.0, "std_dev": 4.247359335886969, "min": 590.989990234375, "max": 600.0 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 120, "tokens_per_second": 52.3250617980957, "tpot_ms": 19.122448979591837, "total_time_ms": 1994, "total_tokens": 99, "peak_memory_mb": 1948.2578125, "mean_cpu_percent": 558.73016, "peak_cpu_percent": 600.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 48, "tokens_per_second": 52.885765075683594, "tpot_ms": 18.918367346938776, "total_time_ms": 1902, "total_tokens": 99, "peak_memory_mb": 1949.63671875, "mean_cpu_percent": 577.53467, "peak_cpu_percent": 600.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 40, "tokens_per_second": 52.372196197509766, "tpot_ms": 19.102040816326532, "total_time_ms": 1912, "total_tokens": 99, "peak_memory_mb": 1949.6953125, "mean_cpu_percent": 577.5335, "peak_cpu_percent": 600.0, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 69.33333333333333, "median": 48.0, "p90": 105.6, "p95": 112.79999999999998, "std_dev": 35.97530016861878, "min": 40.0, "max": 120.0 }, "tokens_per_second": { "mean": 52.52767435709635, "median": 52.372196197509766, "p90": 52.783051300048825, "p95": 52.83440818786621, "std_dev": 0.2539384898694634, "min": 52.3250617980957, "max": 52.885765075683594 }, "tpot": { "mean": 19.04761904761905, "median": 19.102040816326532, "p90": 19.118367346938776, "p95": 19.12040816326531, "std_dev": 0.09177372491994598, "min": 18.918367346938776, "max": 19.122448979591837 }, "total_time": { "mean": 1936.0, "median": 1912.0, "p90": 1977.6, "p95": 1985.8000000000002, "std_dev": 41.21488404286329, "min": 1902.0, "max": 1994.0 }, "peak_memory_mb": { "mean": 1949.1966145833333, "median": 1949.63671875, "p90": 1949.68359375, "p95": 1949.689453125, "std_dev": 0.6642641646403539, "min": 1948.2578125, "max": 1949.6953125 }, "mean_cpu_percent": { "mean": 571.26611328125, "median": 577.5335083007812, "p90": 577.5344360351562, "p95": 577.5345520019531, "std_dev": 8.864255059098127, "min": 558.7301635742188, "max": 577.53466796875 }, "peak_cpu_percent": { "mean": 600.0, "median": 600.0, "p90": 600.0, "p95": 600.0, "std_dev": 0.0, "min": 600.0, "max": 600.0 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 148, "tokens_per_second": 51.662750244140625, "tpot_ms": 19.36078431372549, "total_time_ms": 5085, "total_tokens": 256, "peak_memory_mb": 1953.3046875, "mean_cpu_percent": 567.51825, "peak_cpu_percent": 618.02, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 75, "tokens_per_second": 52.166709899902344, "tpot_ms": 19.172549019607843, "total_time_ms": 4964, "total_tokens": 256, "peak_memory_mb": 1955.21484375, "mean_cpu_percent": 559.29156, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 92, "tokens_per_second": 51.49485778808594, "tpot_ms": 19.423529411764704, "total_time_ms": 5045, "total_tokens": 256, "peak_memory_mb": 1955.2421875, "mean_cpu_percent": 550.80707, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 105.0, "median": 92.0, "p90": 136.8, "p95": 142.4, "std_dev": 31.187604375242845, "min": 75.0, "max": 148.0 }, "tokens_per_second": { "mean": 51.77477264404297, "median": 51.662750244140625, "p90": 52.06591796875, "p95": 52.11631393432617, "std_dev": 0.28549148101470173, "min": 51.49485778808594, "max": 52.166709899902344 }, "tpot": { "mean": 19.318954248366012, "median": 19.36078431372549, "p90": 19.410980392156862, "p95": 19.41725490196078, "std_dev": 0.10664615976141735, "min": 19.172549019607843, "max": 19.423529411764704 }, "total_time": { "mean": 5031.333333333333, "median": 5045.0, "p90": 5077.0, "p95": 5081.0, "std_dev": 50.33443707399096, "min": 4964.0, "max": 5085.0 }, "peak_memory_mb": { "mean": 1954.5872395833333, "median": 1955.21484375, "p90": 1955.23671875, "p95": 1955.239453125, "std_dev": 0.906969975619561, "min": 1953.3046875, "max": 1955.2421875 }, "mean_cpu_percent": { "mean": 559.2056274414062, "median": 559.2915649414062, "p90": 565.8729125976563, "p95": 566.6955810546875, "std_dev": 6.822581959782293, "min": 550.8070678710938, "max": 567.5182495117188 }, "peak_cpu_percent": { "mean": 606.0066731770834, "median": 600.0, "p90": 614.416015625, "p95": 616.218017578125, "std_dev": 8.494718671773938, "min": 600.0, "max": 618.02001953125 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 154, "tokens_per_second": 50.42999649047851, "tpot_ms": 19.83529411764706, "total_time_ms": 5212, "total_tokens": 256, "peak_memory_mb": 1958.4921875, "mean_cpu_percent": 544.56, "peak_cpu_percent": 585.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 82, "tokens_per_second": 51.353294372558594, "tpot_ms": 19.474509803921567, "total_time_ms": 5048, "total_tokens": 256, "peak_memory_mb": 1959.99609375, "mean_cpu_percent": 558.14746, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 85, "tokens_per_second": 51.7399787902832, "tpot_ms": 19.32941176470588, "total_time_ms": 5014, "total_tokens": 256, "peak_memory_mb": 1960.0, "mean_cpu_percent": 558.16254, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 107.0, "median": 85.0, "p90": 140.2, "p95": 147.1, "std_dev": 33.25657829663178, "min": 82.0, "max": 154.0 }, "tokens_per_second": { "mean": 51.17442321777344, "median": 51.353294372558594, "p90": 51.66264190673828, "p95": 51.70131034851074, "std_dev": 0.5495510737326309, "min": 50.42999649047851, "max": 51.7399787902832 }, "tpot": { "mean": 19.54640522875817, "median": 19.474509803921567, "p90": 19.76313725490196, "p95": 19.79921568627451, "std_dev": 0.21269062483437037, "min": 19.32941176470588, "max": 19.83529411764706 }, "total_time": { "mean": 5091.333333333333, "median": 5048.0, "p90": 5179.2, "p95": 5195.599999999999, "std_dev": 86.44587259602906, "min": 5014.0, "max": 5212.0 }, "peak_memory_mb": { "mean": 1959.49609375, "median": 1959.99609375, "p90": 1959.99921875, "p95": 1959.9996093749999, "std_dev": 0.7098707083166917, "min": 1958.4921875, "max": 1960.0 }, "mean_cpu_percent": { "mean": 553.6233317057291, "median": 558.1474609375, "p90": 558.159521484375, "p95": 558.1610290527344, "std_dev": 6.408747990886213, "min": 544.5599975585938, "max": 558.1625366210938 }, "peak_cpu_percent": { "mean": 595.2366739908854, "median": 600.0, "p90": 600.0, "p95": 600.0, "std_dev": 6.736360244094352, "min": 585.7100219726562, "max": 600.0 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 188, "tokens_per_second": 51.841251373291016, "tpot_ms": 19.294117647058822, "total_time_ms": 5108, "total_tokens": 256, "peak_memory_mb": 1967.44921875, "mean_cpu_percent": 562.9096, "peak_cpu_percent": 610.23, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 47, "tokens_per_second": 51.91971206665039, "tpot_ms": 19.262745098039215, "total_time_ms": 4959, "total_tokens": 256, "peak_memory_mb": 1969.96484375, "mean_cpu_percent": 570.709, "peak_cpu_percent": 602.69, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 45, "tokens_per_second": 50.89142227172851, "tpot_ms": 19.650980392156864, "total_time_ms": 5056, "total_tokens": 256, "peak_memory_mb": 1969.96875, "mean_cpu_percent": 564.13806, "peak_cpu_percent": 606.39, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 93.33333333333333, "median": 47.0, "p90": 159.8, "p95": 173.9, "std_dev": 66.94442139234273, "min": 45.0, "max": 188.0 }, "tokens_per_second": { "mean": 51.550795237223305, "median": 51.841251373291016, "p90": 51.90401992797852, "p95": 51.91186599731445, "std_dev": 0.4673460893311365, "min": 50.89142227172851, "max": 51.91971206665039 }, "tpot": { "mean": 19.402614379084966, "median": 19.294117647058822, "p90": 19.579607843137254, "p95": 19.615294117647057, "std_dev": 0.17608769887931633, "min": 19.262745098039215, "max": 19.650980392156864 }, "total_time": { "mean": 5041.0, "median": 5056.0, "p90": 5097.6, "p95": 5102.8, "std_dev": 61.746794788609606, "min": 4959.0, "max": 5108.0 }, "peak_memory_mb": { "mean": 1969.1276041666667, "median": 1969.96484375, "p90": 1969.9679687500002, "p95": 1969.9683593749999, "std_dev": 1.1867987809949843, "min": 1967.44921875, "max": 1969.96875 }, "mean_cpu_percent": { "mean": 565.9188842773438, "median": 564.1380615234375, "p90": 569.3947998046875, "p95": 570.0518920898437, "std_dev": 3.4240394642392635, "min": 562.9096069335938, "max": 570.708984375 }, "peak_cpu_percent": { "mean": 606.4366658528646, "median": 606.3900146484375, "p90": 609.4619873046875, "p95": 609.8459838867187, "std_dev": 3.0783598895475177, "min": 602.6900024414062, "max": 610.22998046875 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 135, "tokens_per_second": 51.8062629699707, "tpot_ms": 19.301960784313724, "total_time_ms": 5057, "total_tokens": 256, "peak_memory_mb": 1971.49609375, "mean_cpu_percent": 561.4764, "peak_cpu_percent": 606.39, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 79, "tokens_per_second": 51.51872253417969, "tpot_ms": 19.415686274509802, "total_time_ms": 5030, "total_tokens": 256, "peak_memory_mb": 1973.05859375, "mean_cpu_percent": 574.55457, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 82, "tokens_per_second": 51.8953742980957, "tpot_ms": 19.270588235294117, "total_time_ms": 4996, "total_tokens": 256, "peak_memory_mb": 1973.09375, "mean_cpu_percent": 564.1895, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 98.66666666666667, "median": 82.0, "p90": 124.39999999999999, "p95": 129.7, "std_dev": 25.72072229848057, "min": 79.0, "max": 135.0 }, "tokens_per_second": { "mean": 51.74011993408203, "median": 51.8062629699707, "p90": 51.8775520324707, "p95": 51.8864631652832, "std_dev": 0.16072297847059516, "min": 51.51872253417969, "max": 51.8953742980957 }, "tpot": { "mean": 19.32941176470588, "median": 19.301960784313724, "p90": 19.392941176470586, "p95": 19.404313725490198, "std_dev": 0.06233526283694809, "min": 19.270588235294117, "max": 19.415686274509802 }, "total_time": { "mean": 5027.666666666667, "median": 5030.0, "p90": 5051.6, "p95": 5054.299999999999, "std_dev": 24.957742063113177, "min": 4996.0, "max": 5057.0 }, "peak_memory_mb": { "mean": 1972.5494791666667, "median": 1973.05859375, "p90": 1973.08671875, "p95": 1973.090234375, "std_dev": 0.7449942360014252, "min": 1971.49609375, "max": 1973.09375 }, "mean_cpu_percent": { "mean": 566.7401529947916, "median": 564.1895141601562, "p90": 572.4815551757813, "p95": 573.5180603027344, "std_dev": 5.635545305830467, "min": 561.4763793945312, "max": 574.5545654296875 }, "peak_cpu_percent": { "mean": 602.1300048828125, "median": 600.0, "p90": 605.11201171875, "p95": 605.7510131835938, "std_dev": 3.0122817931943526, "min": 600.0, "max": 606.3900146484375 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 582, "tokens_per_second": 50.652626037597656, "tpot_ms": 19.743434343434345, "total_time_ms": 10355, "total_tokens": 496, "peak_memory_mb": 2009.55859375, "mean_cpu_percent": 567.3743, "peak_cpu_percent": 607.4, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 58, "tokens_per_second": 50.244911193847656, "tpot_ms": 19.903030303030302, "total_time_ms": 9910, "total_tokens": 496, "peak_memory_mb": 2011.08984375, "mean_cpu_percent": 560.57825, "peak_cpu_percent": 606.39, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 66, "tokens_per_second": 50.1916389465332, "tpot_ms": 19.925252525252525, "total_time_ms": 9929, "total_tokens": 496, "peak_memory_mb": 2011.1015625, "mean_cpu_percent": 563.1315, "peak_cpu_percent": 617.67, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 235.33333333333334, "median": 66.0, "p90": 478.8, "p95": 530.4, "std_dev": 245.1521069775978, "min": 58.0, "max": 582.0 }, "tokens_per_second": { "mean": 50.363058725992836, "median": 50.244911193847656, "p90": 50.571083068847656, "p95": 50.611854553222656, "std_dev": 0.20590678157200856, "min": 50.1916389465332, "max": 50.652626037597656 }, "tpot": { "mean": 19.85723905723906, "median": 19.903030303030302, "p90": 19.92080808080808, "p95": 19.923030303030302, "std_dev": 0.0809818558001435, "min": 19.743434343434345, "max": 19.925252525252525 }, "total_time": { "mean": 10064.666666666666, "median": 9929.0, "p90": 10269.8, "p95": 10312.4, "std_dev": 205.44315245071783, "min": 9910.0, "max": 10355.0 }, "peak_memory_mb": { "mean": 2010.5833333333333, "median": 2011.08984375, "p90": 2011.09921875, "p95": 2011.1003906249998, "std_dev": 0.7246161018143886, "min": 2009.55859375, "max": 2011.1015625 }, "mean_cpu_percent": { "mean": 563.6946818033854, "median": 563.1314697265625, "p90": 566.5257568359375, "p95": 566.9500427246094, "std_dev": 2.8029254916104485, "min": 560.5782470703125, "max": 567.3743286132812 }, "peak_cpu_percent": { "mean": 610.4866739908854, "median": 607.4000244140625, "p90": 615.6159912109375, "p95": 616.642987060547, "std_dev": 5.096075300589752, "min": 606.3900146484375, "max": 617.6699829101562 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 377, "tokens_per_second": 50.58578109741211, "tpot_ms": 19.76908023483366, "total_time_ms": 10479, "total_tokens": 512, "peak_memory_mb": 2013.22265625, "mean_cpu_percent": 561.38947, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 61, "tokens_per_second": 50.547245025634766, "tpot_ms": 19.78473581213307, "total_time_ms": 10171, "total_tokens": 512, "peak_memory_mb": 2014.69921875, "mean_cpu_percent": 574.0936, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 60, "tokens_per_second": 49.769344329833984, "tpot_ms": 20.09393346379648, "total_time_ms": 10328, "total_tokens": 512, "peak_memory_mb": 2014.984375, "mean_cpu_percent": 569.6794, "peak_cpu_percent": 602.79, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 166.0, "median": 61.0, "p90": 313.8, "p95": 345.4, "std_dev": 149.20008936547816, "min": 60.0, "max": 377.0 }, "tokens_per_second": { "mean": 50.30079015096029, "median": 50.547245025634766, "p90": 50.578073883056646, "p95": 50.58192749023438, "std_dev": 0.37611811336463996, "min": 49.769344329833984, "max": 50.58578109741211 }, "tpot": { "mean": 19.882583170254403, "median": 19.78473581213307, "p90": 20.032093933463795, "p95": 20.063013698630137, "std_dev": 0.14958383203716807, "min": 19.76908023483366, "max": 20.09393346379648 }, "total_time": { "mean": 10326.0, "median": 10328.0, "p90": 10448.8, "p95": 10463.9, "std_dev": 125.74842610015708, "min": 10171.0, "max": 10479.0 }, "peak_memory_mb": { "mean": 2014.3020833333333, "median": 2014.69921875, "p90": 2014.92734375, "p95": 2014.955859375, "std_dev": 0.772096989979733, "min": 2013.22265625, "max": 2014.984375 }, "mean_cpu_percent": { "mean": 568.3874918619791, "median": 569.6793823242188, "p90": 573.2107788085938, "p95": 573.6522033691407, "std_dev": 5.266287278639554, "min": 561.3894653320312, "max": 574.0936279296875 }, "peak_cpu_percent": { "mean": 600.9299926757812, "median": 600.0, "p90": 602.231982421875, "p95": 602.5109802246093, "std_dev": 1.3152082549974884, "min": 600.0, "max": 602.7899780273438 } } } ] }, { "backend": "cpu", "model_id": "qwen3-4b", "cold_start_ttft_ms": 1382, "idle_baseline": { "rss_mb": 4205.58984375, "cpu_percent": 0.0 }, "reliability": { "total_runs": 30, "successful_runs": 30, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.0, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 192, "tokens_per_second": 15.396392822265623, "tpot_ms": 64.95275590551181, "total_time_ms": 8441, "total_tokens": 128, "peak_memory_mb": 5771.94140625, "mean_cpu_percent": 580.12683, "peak_cpu_percent": 627.75, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 199, "tokens_per_second": 15.405534744262695, "tpot_ms": 64.92125984251969, "total_time_ms": 8444, "total_tokens": 128, "peak_memory_mb": 5772.0703125, "mean_cpu_percent": 572.1145, "peak_cpu_percent": 619.35, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 247, "tokens_per_second": 15.17112922668457, "tpot_ms": 65.92125984251969, "total_time_ms": 8619, "total_tokens": 128, "peak_memory_mb": 5772.19140625, "mean_cpu_percent": 565.77496, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 212.66666666666666, "median": 199.0, "p90": 237.4, "p95": 242.2, "std_dev": 24.44494948973214, "min": 192.0, "max": 247.0 }, "tokens_per_second": { "mean": 15.324352264404297, "median": 15.396392822265623, "p90": 15.403706359863282, "p95": 15.404620552062987, "std_dev": 0.10840931125598503, "min": 15.17112922668457, "max": 15.405534744262695 }, "tpot": { "mean": 65.26509186351706, "median": 64.95275590551181, "p90": 65.7275590551181, "p95": 65.8244094488189, "std_dev": 0.4641589619927602, "min": 64.92125984251969, "max": 65.92125984251969 }, "total_time": { "mean": 8501.333333333334, "median": 8444.0, "p90": 8584.0, "p95": 8601.5, "std_dev": 83.21191154048934, "min": 8441.0, "max": 8619.0 }, "peak_memory_mb": { "mean": 5772.067708333333, "median": 5772.0703125, "p90": 5772.1671875, "p95": 5772.179296875001, "std_dev": 0.10207868292978979, "min": 5771.94140625, "max": 5772.19140625 }, "mean_cpu_percent": { "mean": 572.6720987955729, "median": 572.114501953125, "p90": 578.524365234375, "p95": 579.3255981445312, "std_dev": 5.872376697999688, "min": 565.7749633789062, "max": 580.1268310546875 }, "peak_cpu_percent": { "mean": 615.6999918619791, "median": 619.3499755859375, "p90": 626.0699951171875, "p95": 626.9099975585937, "std_dev": 11.61916264593023, "min": 600.0, "max": 627.75 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 190, "tokens_per_second": 15.512235641479492, "tpot_ms": 64.4724409448819, "total_time_ms": 8378, "total_tokens": 128, "peak_memory_mb": 5778.96875, "mean_cpu_percent": 582.6748, "peak_cpu_percent": 619.35, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 168, "tokens_per_second": 15.40050983428955, "tpot_ms": 64.93700787401575, "total_time_ms": 8415, "total_tokens": 128, "peak_memory_mb": 5779.265625, "mean_cpu_percent": 580.03674, "peak_cpu_percent": 607.4, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 193, "tokens_per_second": 15.431496620178224, "tpot_ms": 64.80314960629921, "total_time_ms": 8423, "total_tokens": 128, "peak_memory_mb": 5779.3359375, "mean_cpu_percent": 586.2921, "peak_cpu_percent": 611.98, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 183.66666666666666, "median": 190.0, "p90": 192.4, "p95": 192.70000000000002, "std_dev": 11.145502331533658, "min": 168.0, "max": 193.0 }, "tokens_per_second": { "mean": 15.448080698649088, "median": 15.431496620178224, "p90": 15.496087837219239, "p95": 15.504161739349366, "std_dev": 0.04709520670660543, "min": 15.40050983428955, "max": 15.512235641479492 }, "tpot": { "mean": 64.73753280839895, "median": 64.80314960629921, "p90": 64.91023622047244, "p95": 64.9236220472441, "std_dev": 0.1952516001406514, "min": 64.4724409448819, "max": 64.93700787401575 }, "total_time": { "mean": 8405.333333333334, "median": 8415.0, "p90": 8421.4, "p95": 8422.199999999999, "std_dev": 19.601587237318874, "min": 8378.0, "max": 8423.0 }, "peak_memory_mb": { "mean": 5779.190104166667, "median": 5779.265625, "p90": 5779.321875, "p95": 5779.32890625, "std_dev": 0.15913141789012697, "min": 5778.96875, "max": 5779.3359375 }, "mean_cpu_percent": { "mean": 583.001220703125, "median": 582.6748046875, "p90": 585.5686523437499, "p95": 585.9303833007813, "std_dev": 2.5641538489429143, "min": 580.0367431640625, "max": 586.2921142578125 }, "peak_cpu_percent": { "mean": 612.9099934895834, "median": 611.97998046875, "p90": 617.8759765625, "p95": 618.6129760742188, "std_dev": 4.922670441724277, "min": 607.4000244140625, "max": 619.3499755859375 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 206, "tokens_per_second": 15.312811851501465, "tpot_ms": 65.31496062992126, "total_time_ms": 8501, "total_tokens": 128, "peak_memory_mb": 5785.78515625, "mean_cpu_percent": 579.568, "peak_cpu_percent": 623.37, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 199, "tokens_per_second": 15.392264366149902, "tpot_ms": 64.97637795275591, "total_time_ms": 8451, "total_tokens": 128, "peak_memory_mb": 5785.90234375, "mean_cpu_percent": 573.98267, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 267, "tokens_per_second": 15.42409324645996, "tpot_ms": 64.83464566929133, "total_time_ms": 8501, "total_tokens": 128, "peak_memory_mb": 5785.95703125, "mean_cpu_percent": 565.6826, "peak_cpu_percent": 622.22, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 224.0, "median": 206.0, "p90": 254.8, "p95": 260.9, "std_dev": 30.539591789456953, "min": 199.0, "max": 267.0 }, "tokens_per_second": { "mean": 15.376389821370443, "median": 15.392264366149902, "p90": 15.417727470397947, "p95": 15.420910358428955, "std_dev": 0.04679663873216691, "min": 15.312811851501465, "max": 15.42409324645996 }, "tpot": { "mean": 65.04199475065616, "median": 64.97637795275591, "p90": 65.24724409448818, "p95": 65.28110236220472, "std_dev": 0.20150233824324745, "min": 64.83464566929133, "max": 65.31496062992126 }, "total_time": { "mean": 8484.333333333334, "median": 8501.0, "p90": 8501.0, "p95": 8501.0, "std_dev": 23.570226039551585, "min": 8451.0, "max": 8501.0 }, "peak_memory_mb": { "mean": 5785.881510416667, "median": 5785.90234375, "p90": 5785.94609375, "p95": 5785.951562499999, "std_dev": 0.07169739530175107, "min": 5785.78515625, "max": 5785.95703125 }, "mean_cpu_percent": { "mean": 573.0777587890625, "median": 573.982666015625, "p90": 578.450927734375, "p95": 579.0094604492188, "std_dev": 5.704679910339026, "min": 565.6826171875, "max": 579.5679931640625 }, "peak_cpu_percent": { "mean": 615.1966552734375, "median": 622.219970703125, "p90": 623.139990234375, "p95": 623.2549926757813, "std_dev": 10.75590962101011, "min": 600.0, "max": 623.3699951171875 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 158, "tokens_per_second": 15.337129592895508, "tpot_ms": 65.20472440944881, "total_time_ms": 8439, "total_tokens": 128, "peak_memory_mb": 5788.05859375, "mean_cpu_percent": 580.83325, "peak_cpu_percent": 615.38, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 184, "tokens_per_second": 14.925369262695312, "tpot_ms": 67.00787401574803, "total_time_ms": 8694, "total_tokens": 128, "peak_memory_mb": 5788.12890625, "mean_cpu_percent": 567.5058, "peak_cpu_percent": 609.52, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 155, "tokens_per_second": 14.970383644104004, "tpot_ms": 66.81102362204724, "total_time_ms": 8640, "total_tokens": 128, "peak_memory_mb": 5788.15625, "mean_cpu_percent": 576.48987, "peak_cpu_percent": 611.98, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 165.66666666666666, "median": 158.0, "p90": 178.8, "p95": 181.4, "std_dev": 13.02134998974974, "min": 155.0, "max": 184.0 }, "tokens_per_second": { "mean": 15.077627499898275, "median": 14.970383644104004, "p90": 15.263780403137208, "p95": 15.300454998016358, "std_dev": 0.18441362175981874, "min": 14.925369262695312, "max": 15.337129592895508 }, "tpot": { "mean": 66.34120734908136, "median": 66.81102362204724, "p90": 66.96850393700788, "p95": 66.98818897637796, "std_dev": 0.8076231065837236, "min": 65.20472440944881, "max": 67.00787401574803 }, "total_time": { "mean": 8591.0, "median": 8640.0, "p90": 8683.2, "p95": 8688.6, "std_dev": 109.71781988355401, "min": 8439.0, "max": 8694.0 }, "peak_memory_mb": { "mean": 5788.114583333333, "median": 5788.12890625, "p90": 5788.15078125, "p95": 5788.153515625, "std_dev": 0.04113429426805029, "min": 5788.05859375, "max": 5788.15625 }, "mean_cpu_percent": { "mean": 574.9429728190104, "median": 576.4898681640625, "p90": 579.9645751953125, "p95": 580.3989135742187, "std_dev": 5.549769896728375, "min": 567.5057983398438, "max": 580.833251953125 }, "peak_cpu_percent": { "mean": 612.2933349609375, "median": 611.97998046875, "p90": 614.7, "p95": 615.0400024414062, "std_dev": 2.402568120222239, "min": 609.52001953125, "max": 615.3800048828125 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 215, "tokens_per_second": 14.53286361694336, "tpot_ms": 68.8156862745098, "total_time_ms": 17763, "total_tokens": 256, "peak_memory_mb": 5799.48046875, "mean_cpu_percent": 569.2745, "peak_cpu_percent": 602.59, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 100, "tokens_per_second": 14.687125205993652, "tpot_ms": 68.09019607843138, "total_time_ms": 17463, "total_tokens": 256, "peak_memory_mb": 5833.08984375, "mean_cpu_percent": 575.6728, "peak_cpu_percent": 622.22, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 97, "tokens_per_second": 14.755999565124512, "tpot_ms": 67.77254901960784, "total_time_ms": 17379, "total_tokens": 256, "peak_memory_mb": 5834.34765625, "mean_cpu_percent": 574.7276, "peak_cpu_percent": 605.6, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 137.33333333333334, "median": 100.0, "p90": 192.0, "p95": 203.49999999999997, "std_dev": 54.93228154332892, "min": 97.0, "max": 215.0 }, "tokens_per_second": { "mean": 14.658662796020508, "median": 14.687125205993652, "p90": 14.74222469329834, "p95": 14.749112129211426, "std_dev": 0.09329163745582986, "min": 14.53286361694336, "max": 14.755999565124512 }, "tpot": { "mean": 68.22614379084968, "median": 68.09019607843138, "p90": 68.67058823529412, "p95": 68.74313725490195, "std_dev": 0.43657391014193464, "min": 67.77254901960784, "max": 68.8156862745098 }, "total_time": { "mean": 17535.0, "median": 17463.0, "p90": 17703.0, "p95": 17733.0, "std_dev": 164.82718222429213, "min": 17379.0, "max": 17763.0 }, "peak_memory_mb": { "mean": 5822.305989583333, "median": 5833.08984375, "p90": 5834.09609375, "p95": 5834.221875, "std_dev": 16.148247047404034, "min": 5799.48046875, "max": 5834.34765625 }, "mean_cpu_percent": { "mean": 573.2249552408854, "median": 574.7276000976562, "p90": 575.4837524414063, "p95": 575.578271484375, "std_dev": 2.819936909297696, "min": 569.2744750976562, "max": 575.6727905273438 }, "peak_cpu_percent": { "mean": 610.1366577148438, "median": 605.5999755859375, "p90": 618.8959716796875, "p95": 620.5579711914063, "std_dev": 8.63210238661812, "min": 602.5900268554688, "max": 622.219970703125 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 265, "tokens_per_second": 14.747071266174316, "tpot_ms": 67.8156862745098, "total_time_ms": 17558, "total_tokens": 256, "peak_memory_mb": 5839.86328125, "mean_cpu_percent": 578.6841, "peak_cpu_percent": 611.98, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 111, "tokens_per_second": 14.874298095703123, "tpot_ms": 67.23529411764706, "total_time_ms": 17256, "total_tokens": 256, "peak_memory_mb": 5840.98046875, "mean_cpu_percent": 577.3756, "peak_cpu_percent": 618.02, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 116, "tokens_per_second": 14.93012523651123, "tpot_ms": 66.9843137254902, "total_time_ms": 17197, "total_tokens": 256, "peak_memory_mb": 5841.74609375, "mean_cpu_percent": 578.0255, "peak_cpu_percent": 602.69, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 164.0, "median": 116.0, "p90": 235.2, "p95": 250.1, "std_dev": 71.44695001654492, "min": 111.0, "max": 265.0 }, "tokens_per_second": { "mean": 14.85049819946289, "median": 14.874298095703123, "p90": 14.918959808349609, "p95": 14.92454252243042, "std_dev": 0.07660293862862509, "min": 14.747071266174316, "max": 14.93012523651123 }, "tpot": { "mean": 67.3450980392157, "median": 67.23529411764706, "p90": 67.69960784313724, "p95": 67.75764705882352, "std_dev": 0.34817405102403, "min": 66.9843137254902, "max": 67.8156862745098 }, "total_time": { "mean": 17337.0, "median": 17256.0, "p90": 17497.600000000002, "p95": 17527.8, "std_dev": 158.11599117947137, "min": 17197.0, "max": 17558.0 }, "peak_memory_mb": { "mean": 5840.86328125, "median": 5840.98046875, "p90": 5841.59296875, "p95": 5841.66953125, "std_dev": 0.7731086211549879, "min": 5839.86328125, "max": 5841.74609375 }, "mean_cpu_percent": { "mean": 578.0284016927084, "median": 578.0255126953125, "p90": 578.5523681640625, "p95": 578.6182250976562, "std_dev": 0.5341852324587554, "min": 577.3756103515625, "max": 578.68408203125 }, "peak_cpu_percent": { "mean": 610.8966674804688, "median": 611.97998046875, "p90": 616.81201171875, "p95": 617.416015625, "std_dev": 6.305158272750944, "min": 602.6900024414062, "max": 618.02001953125 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 275, "tokens_per_second": 15.12515640258789, "tpot_ms": 66.12156862745098, "total_time_ms": 17136, "total_tokens": 256, "peak_memory_mb": 5847.88671875, "mean_cpu_percent": 580.967, "peak_cpu_percent": 611.98, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 153, "tokens_per_second": 15.17162036895752, "tpot_ms": 65.91764705882353, "total_time_ms": 16962, "total_tokens": 256, "peak_memory_mb": 5849.546875, "mean_cpu_percent": 584.48376, "peak_cpu_percent": 626.08, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 199, "tokens_per_second": 15.139565467834473, "tpot_ms": 66.05490196078432, "total_time_ms": 17043, "total_tokens": 256, "peak_memory_mb": 5849.76171875, "mean_cpu_percent": 581.32855, "peak_cpu_percent": 619.35, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 209.0, "median": 199.0, "p90": 259.8, "p95": 267.4, "std_dev": 50.30573194643595, "min": 153.0, "max": 275.0 }, "tokens_per_second": { "mean": 15.145447413126627, "median": 15.139565467834473, "p90": 15.165209388732912, "p95": 15.168414878845216, "std_dev": 0.019419457648012544, "min": 15.12515640258789, "max": 15.17162036895752 }, "tpot": { "mean": 66.03137254901962, "median": 66.05490196078432, "p90": 66.10823529411765, "p95": 66.11490196078432, "std_dev": 0.08489690395361842, "min": 65.91764705882353, "max": 66.12156862745098 }, "total_time": { "mean": 17047.0, "median": 17043.0, "p90": 17117.4, "p95": 17126.7, "std_dev": 71.09149034870488, "min": 16962.0, "max": 17136.0 }, "peak_memory_mb": { "mean": 5849.065104166667, "median": 5849.546875, "p90": 5849.71875, "p95": 5849.740234375, "std_dev": 0.8378478787496852, "min": 5847.88671875, "max": 5849.76171875 }, "mean_cpu_percent": { "mean": 582.259765625, "median": 581.3285522460938, "p90": 583.8527221679688, "p95": 584.1682434082031, "std_dev": 1.5795173034834649, "min": 580.9669799804688, "max": 584.4837646484375 }, "peak_cpu_percent": { "mean": 619.1366577148438, "median": 619.3499755859375, "p90": 624.7340087890625, "p95": 625.4070129394531, "std_dev": 5.758291793268914, "min": 611.97998046875, "max": 626.0800170898438 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 306, "tokens_per_second": 15.11947536468506, "tpot_ms": 66.14509803921568, "total_time_ms": 17173, "total_tokens": 256, "peak_memory_mb": 5851.390625, "mean_cpu_percent": 576.6952, "peak_cpu_percent": 625.77, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 139, "tokens_per_second": 15.143769264221191, "tpot_ms": 66.03529411764706, "total_time_ms": 16978, "total_tokens": 256, "peak_memory_mb": 5853.44921875, "mean_cpu_percent": 581.7956, "peak_cpu_percent": 616.45, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 117, "tokens_per_second": 14.921745300292969, "tpot_ms": 67.01960784313725, "total_time_ms": 17207, "total_tokens": 256, "peak_memory_mb": 5853.88671875, "mean_cpu_percent": 580.42865, "peak_cpu_percent": 619.35, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 187.33333333333334, "median": 139.0, "p90": 272.6, "p95": 289.3, "std_dev": 84.38930948618761, "min": 117.0, "max": 306.0 }, "tokens_per_second": { "mean": 15.061663309733072, "median": 15.11947536468506, "p90": 15.138910484313964, "p95": 15.14133987426758, "std_dev": 0.09943284304135075, "min": 14.921745300292969, "max": 15.143769264221191 }, "tpot": { "mean": 66.39999999999999, "median": 66.14509803921568, "p90": 66.84470588235294, "p95": 66.93215686274509, "std_dev": 0.4404161931293802, "min": 66.03529411764706, "max": 67.01960784313725 }, "total_time": { "mean": 17119.333333333332, "median": 17173.0, "p90": 17200.2, "p95": 17203.600000000002, "std_dev": 100.89708728314322, "min": 16978.0, "max": 17207.0 }, "peak_memory_mb": { "mean": 5852.908854166667, "median": 5853.44921875, "p90": 5853.799218749999, "p95": 5853.842968749999, "std_dev": 1.0883064563464406, "min": 5851.390625, "max": 5853.88671875 }, "mean_cpu_percent": { "mean": 579.6398111979166, "median": 580.4286499023438, "p90": 581.5222045898438, "p95": 581.6588989257813, "std_dev": 2.1556479518464586, "min": 576.6951904296875, "max": 581.7955932617188 }, "peak_cpu_percent": { "mean": 620.5233357747396, "median": 619.3499755859375, "p90": 624.4860107421875, "p95": 625.1280151367187, "std_dev": 3.894287673877559, "min": 616.4500122070312, "max": 625.77001953125 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 1275, "tokens_per_second": 14.721823692321776, "tpot_ms": 67.92759295499022, "total_time_ms": 35986, "total_tokens": 512, "peak_memory_mb": 5926.015625, "mean_cpu_percent": 578.462, "peak_cpu_percent": 618.02, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 216, "tokens_per_second": 15.034168243408203, "tpot_ms": 66.51859099804305, "total_time_ms": 34207, "total_tokens": 512, "peak_memory_mb": 5936.1640625, "mean_cpu_percent": 582.74347, "peak_cpu_percent": 620.68, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 228, "tokens_per_second": 15.02312469482422, "tpot_ms": 66.56555772994129, "total_time_ms": 34243, "total_tokens": 512, "peak_memory_mb": 5936.25, "mean_cpu_percent": 581.93207, "peak_cpu_percent": 623.37, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 573.0, "median": 228.0, "p90": 1065.6, "p95": 1170.3, "std_dev": 496.4131343951326, "min": 216.0, "max": 1275.0 }, "tokens_per_second": { "mean": 14.926372210184732, "median": 15.02312469482422, "p90": 15.031959533691406, "p95": 15.033063888549805, "std_dev": 0.1447078945353505, "min": 14.721823692321776, "max": 15.034168243408203 }, "tpot": { "mean": 67.00391389432485, "median": 66.56555772994129, "p90": 67.65518590998043, "p95": 67.79138943248533, "std_dev": 0.6534211116934404, "min": 66.51859099804305, "max": 67.92759295499022 }, "total_time": { "mean": 34812.0, "median": 34243.0, "p90": 35637.4, "p95": 35811.7, "std_dev": 830.2734489311338, "min": 34207.0, "max": 35986.0 }, "peak_memory_mb": { "mean": 5932.809895833333, "median": 5936.1640625, "p90": 5936.232812499999, "p95": 5936.241406249999, "std_dev": 4.804403079883395, "min": 5926.015625, "max": 5936.25 }, "mean_cpu_percent": { "mean": 581.0458374023438, "median": 581.9320678710938, "p90": 582.5811889648437, "p95": 582.6623291015625, "std_dev": 1.8568524177322556, "min": 578.4619750976562, "max": 582.7434692382812 }, "peak_cpu_percent": { "mean": 620.6900024414062, "median": 620.6799926757812, "p90": 622.8319946289063, "p95": 623.100994873047, "std_dev": 2.184129855585167, "min": 618.02001953125, "max": 623.3699951171875 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 1037, "tokens_per_second": 15.101202011108398, "tpot_ms": 66.2211350293542, "total_time_ms": 34876, "total_tokens": 512, "peak_memory_mb": 5938.203125, "mean_cpu_percent": 577.2771, "peak_cpu_percent": 624.76, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 78, "tokens_per_second": 14.873873710632324, "tpot_ms": 67.23287671232876, "total_time_ms": 34434, "total_tokens": 512, "peak_memory_mb": 5938.33984375, "mean_cpu_percent": 578.06287, "peak_cpu_percent": 631.19, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 79, "tokens_per_second": 14.44057846069336, "tpot_ms": 69.25244618395304, "total_time_ms": 35467, "total_tokens": 512, "peak_memory_mb": 5938.41015625, "mean_cpu_percent": 577.5251, "peak_cpu_percent": 626.72, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 398.0, "median": 79.0, "p90": 845.4, "p95": 941.1999999999999, "std_dev": 451.8414176087299, "min": 78.0, "max": 1037.0 }, "tokens_per_second": { "mean": 14.805218060811361, "median": 14.873873710632324, "p90": 15.055736351013184, "p95": 15.078469181060791, "std_dev": 0.27403292689017866, "min": 14.44057846069336, "max": 15.101202011108398 }, "tpot": { "mean": 67.56881930854534, "median": 67.23287671232876, "p90": 68.84853228962818, "p95": 69.05048923679061, "std_dev": 1.2601203381591208, "min": 66.2211350293542, "max": 69.25244618395304 }, "total_time": { "mean": 34925.666666666664, "median": 34876.0, "p90": 35348.8, "p95": 35407.9, "std_dev": 423.1802872955634, "min": 34434.0, "max": 35467.0 }, "peak_memory_mb": { "mean": 5938.317708333333, "median": 5938.33984375, "p90": 5938.396093750001, "p95": 5938.403125000001, "std_dev": 0.08595722627134898, "min": 5938.203125, "max": 5938.41015625 }, "mean_cpu_percent": { "mean": 577.6216837565104, "median": 577.5250854492188, "p90": 577.9553100585938, "p95": 578.0090881347656, "std_dev": 0.3279793822396784, "min": 577.277099609375, "max": 578.0628662109375 }, "peak_cpu_percent": { "mean": 627.5566609700521, "median": 626.719970703125, "p90": 630.29599609375, "p95": 630.7429992675782, "std_dev": 2.6908783454141285, "min": 624.760009765625, "max": 631.1900024414062 } } } ] }, { "backend": "directml", "model_id": "qwen2.5-1.5b-instruct", "cold_start_ttft_ms": 614, "idle_baseline": { "rss_mb": 363.5625, "cpu_percent": 0.0 }, "reliability": { "total_runs": 30, "successful_runs": 30, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.3, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 147, "tokens_per_second": 104.83210483210482, "tpot_ms": 8.456692913385826, "total_time_ms": 1221, "total_tokens": 128, "peak_memory_mb": 396.4140625, "mean_cpu_percent": 97.23508, "peak_cpu_percent": 100.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 147, "tokens_per_second": 104.5751633986928, "tpot_ms": 8.48031496062992, "total_time_ms": 1224, "total_tokens": 128, "peak_memory_mb": 404.953125, "mean_cpu_percent": 100.0, "peak_cpu_percent": 107.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 147, "tokens_per_second": 104.48979591836734, "tpot_ms": 8.488188976377952, "total_time_ms": 1225, "total_tokens": 128, "peak_memory_mb": 412.20703125, "mean_cpu_percent": 99.68554, "peak_cpu_percent": 100.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 147.0, "median": 147.0, "p90": 147.0, "p95": 147.0, "std_dev": 0.0, "min": 147.0, "max": 147.0 }, "tokens_per_second": { "mean": 104.63235471638832, "median": 104.5751633986928, "p90": 104.78071654542242, "p95": 104.80641068876362, "std_dev": 0.14548077357700678, "min": 104.48979591836734, "max": 104.83210483210482 }, "tpot": { "mean": 8.4750656167979, "median": 8.48031496062992, "p90": 8.486614173228347, "p95": 8.48740157480315, "std_dev": 0.013383253316516472, "min": 8.456692913385826, "max": 8.488188976377952 }, "total_time": { "mean": 1223.3333333333333, "median": 1224.0, "p90": 1224.8, "p95": 1224.9, "std_dev": 1.699673171197595, "min": 1221.0, "max": 1225.0 }, "peak_memory_mb": { "mean": 404.5247395833333, "median": 404.953125, "p90": 410.75624999999997, "p95": 411.48164062499995, "std_dev": 6.4545643299156215, "min": 396.4140625, "max": 412.20703125 }, "mean_cpu_percent": { "mean": 98.97353871663411, "median": 99.68553924560547, "p90": 99.93710784912109, "p95": 99.96855392456054, "std_dev": 1.2359634558324002, "min": 97.23507690429688, "max": 100.0 }, "peak_cpu_percent": { "mean": 102.37999979654948, "median": 100.0, "p90": 105.71199951171876, "p95": 106.4259994506836, "std_dev": 3.3658279907254802, "min": 100.0, "max": 107.13999938964844 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 142, "tokens_per_second": 103.27455919395466, "tpot_ms": 8.598360655737705, "total_time_ms": 1191, "total_tokens": 123, "peak_memory_mb": 419.33203125, "mean_cpu_percent": 100.25824, "peak_cpu_percent": 115.38, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 146, "tokens_per_second": 103.10142497904442, "tpot_ms": 8.581967213114755, "total_time_ms": 1193, "total_tokens": 123, "peak_memory_mb": 426.953125, "mean_cpu_percent": 100.73309, "peak_cpu_percent": 110.76, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 141, "tokens_per_second": 102.8428093645485, "tpot_ms": 8.647540983606557, "total_time_ms": 1196, "total_tokens": 123, "peak_memory_mb": 434.37890625, "mean_cpu_percent": 97.67911, "peak_cpu_percent": 106.66, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 143.0, "median": 142.0, "p90": 145.20000000000002, "p95": 145.6, "std_dev": 2.160246899469287, "min": 141.0, "max": 146.0 }, "tokens_per_second": { "mean": 103.07293117918253, "median": 103.10142497904442, "p90": 103.23993235097262, "p95": 103.25724577246363, "std_dev": 0.1774089461686172, "min": 102.8428093645485, "max": 103.27455919395466 }, "tpot": { "mean": 8.609289617486338, "median": 8.598360655737705, "p90": 8.637704918032787, "p95": 8.642622950819673, "std_dev": 0.027863494609796292, "min": 8.581967213114755, "max": 8.647540983606557 }, "total_time": { "mean": 1193.3333333333333, "median": 1193.0, "p90": 1195.4, "p95": 1195.7, "std_dev": 2.0548046676563256, "min": 1191.0, "max": 1196.0 }, "peak_memory_mb": { "mean": 426.8880208333333, "median": 426.953125, "p90": 432.89375, "p95": 433.63632812500003, "std_dev": 6.143033492273681, "min": 419.33203125, "max": 434.37890625 }, "mean_cpu_percent": { "mean": 99.5568135579427, "median": 100.25823974609375, "p90": 100.63812255859375, "p95": 100.68560791015625, "std_dev": 1.3418161890409626, "min": 97.67910766601562, "max": 100.73309326171875 }, "peak_cpu_percent": { "mean": 110.93333435058594, "median": 110.76000213623047, "p90": 114.45599822998048, "p95": 114.91799774169922, "std_dev": 3.5620317330156546, "min": 106.66000366210938, "max": 115.37999725341797 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 140, "tokens_per_second": 103.3091202582728, "tpot_ms": 8.653543307086615, "total_time_ms": 1239, "total_tokens": 128, "peak_memory_mb": 441.09375, "mean_cpu_percent": 98.843834, "peak_cpu_percent": 105.66, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 144, "tokens_per_second": 103.1426269137792, "tpot_ms": 8.637795275590552, "total_time_ms": 1241, "total_tokens": 128, "peak_memory_mb": 448.73828125, "mean_cpu_percent": 97.52747, "peak_cpu_percent": 107.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 149, "tokens_per_second": 102.97666934835075, "tpot_ms": 8.614173228346457, "total_time_ms": 1243, "total_tokens": 128, "peak_memory_mb": 455.328125, "mean_cpu_percent": 95.94977, "peak_cpu_percent": 102.28, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 144.33333333333334, "median": 144.0, "p90": 148.0, "p95": 148.5, "std_dev": 3.6817870057290873, "min": 140.0, "max": 149.0 }, "tokens_per_second": { "mean": 103.14280550680091, "median": 103.1426269137792, "p90": 103.27582158937409, "p95": 103.29247092382344, "std_dev": 0.13572257439006347, "min": 102.97666934835075, "max": 103.3091202582728 }, "tpot": { "mean": 8.63517060367454, "median": 8.637795275590552, "p90": 8.650393700787403, "p95": 8.651968503937008, "std_dev": 0.016179564312254504, "min": 8.614173228346457, "max": 8.653543307086615 }, "total_time": { "mean": 1241.0, "median": 1241.0, "p90": 1242.6, "p95": 1242.8, "std_dev": 1.632993161855452, "min": 1239.0, "max": 1243.0 }, "peak_memory_mb": { "mean": 448.38671875, "median": 448.73828125, "p90": 454.01015625, "p95": 454.669140625, "std_dev": 5.81647402090023, "min": 441.09375, "max": 455.328125 }, "mean_cpu_percent": { "mean": 97.4403584798177, "median": 97.52747344970703, "p90": 98.58056182861328, "p95": 98.71219787597657, "std_dev": 1.1831021538174709, "min": 95.94976806640625, "max": 98.84383392333984 }, "peak_cpu_percent": { "mean": 105.02666727701823, "median": 105.66000366210938, "p90": 106.84400024414063, "p95": 106.99199981689453, "std_dev": 2.03400060892561, "min": 102.27999877929688, "max": 107.13999938964844 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 145, "tokens_per_second": 101.74880763116056, "tpot_ms": 8.763779527559056, "total_time_ms": 1258, "total_tokens": 128, "peak_memory_mb": 463.3984375, "mean_cpu_percent": 96.3281, "peak_cpu_percent": 100.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 150, "tokens_per_second": 102.07336523125996, "tpot_ms": 8.692913385826772, "total_time_ms": 1254, "total_tokens": 128, "peak_memory_mb": 469.8515625, "mean_cpu_percent": 94.01794, "peak_cpu_percent": 106.66, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 144, "tokens_per_second": 103.22580645161293, "tpot_ms": 8.62992125984252, "total_time_ms": 1240, "total_tokens": 128, "peak_memory_mb": 477.64453125, "mean_cpu_percent": 95.294044, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 146.33333333333334, "median": 145.0, "p90": 149.0, "p95": 149.5, "std_dev": 2.6246692913372702, "min": 144.0, "max": 150.0 }, "tokens_per_second": { "mean": 102.34932643801115, "median": 102.07336523125996, "p90": 102.99531820754234, "p95": 103.11056232957763, "std_dev": 0.6337703683133716, "min": 101.74880763116056, "max": 103.22580645161293 }, "tpot": { "mean": 8.695538057742782, "median": 8.692913385826772, "p90": 8.749606299212598, "p95": 8.756692913385828, "std_dev": 0.054678915107610554, "min": 8.62992125984252, "max": 8.763779527559056 }, "total_time": { "mean": 1250.6666666666667, "median": 1254.0, "p90": 1257.2, "p95": 1257.6, "std_dev": 7.71722460186015, "min": 1240.0, "max": 1258.0 }, "peak_memory_mb": { "mean": 470.2981770833333, "median": 469.8515625, "p90": 476.0859375, "p95": 476.865234375, "std_dev": 5.824511151045598, "min": 463.3984375, "max": 477.64453125 }, "mean_cpu_percent": { "mean": 95.21336110432942, "median": 95.2940444946289, "p90": 96.12129058837891, "p95": 96.22469635009766, "std_dev": 0.94484510442478, "min": 94.01793670654297, "max": 96.3281021118164 }, "peak_cpu_percent": { "mean": 104.78333536783855, "median": 106.66000366210938, "p90": 107.48400268554688, "p95": 107.58700256347657, "std_dev": 3.4083668916603416, "min": 100.0, "max": 107.69000244140625 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 163, "tokens_per_second": 111.20764552562989, "tpot_ms": 8.388235294117647, "total_time_ms": 2302, "total_tokens": 256, "peak_memory_mb": 485.1328125, "mean_cpu_percent": 94.86985, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 148, "tokens_per_second": 112.37928007023704, "tpot_ms": 8.352941176470589, "total_time_ms": 2278, "total_tokens": 256, "peak_memory_mb": 493.171875, "mean_cpu_percent": 100.0071, "peak_cpu_percent": 108.1, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 146, "tokens_per_second": 112.4780316344464, "tpot_ms": 8.352941176470589, "total_time_ms": 2276, "total_tokens": 256, "peak_memory_mb": 500.921875, "mean_cpu_percent": 99.993645, "peak_cpu_percent": 108.59, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 152.33333333333334, "median": 148.0, "p90": 160.0, "p95": 161.5, "std_dev": 7.586537784494028, "min": 146.0, "max": 163.0 }, "tokens_per_second": { "mean": 112.02165241010444, "median": 112.37928007023704, "p90": 112.45828132160453, "p95": 112.46815647802546, "std_dev": 0.5769999271157831, "min": 111.20764552562989, "max": 112.4780316344464 }, "tpot": { "mean": 8.364705882352942, "median": 8.352941176470589, "p90": 8.381176470588235, "p95": 8.384705882352941, "std_dev": 0.016637806616153997, "min": 8.352941176470589, "max": 8.388235294117647 }, "total_time": { "mean": 2285.3333333333335, "median": 2278.0, "p90": 2297.2, "p95": 2299.6, "std_dev": 11.813363431112899, "min": 2276.0, "max": 2302.0 }, "peak_memory_mb": { "mean": 493.0755208333333, "median": 493.171875, "p90": 499.371875, "p95": 500.146875, "std_dev": 6.446217844664269, "min": 485.1328125, "max": 500.921875 }, "mean_cpu_percent": { "mean": 98.29019927978516, "median": 99.99364471435547, "p90": 100.00441131591796, "p95": 100.00575714111328, "std_dev": 2.4185582983479366, "min": 94.8698501586914, "max": 100.0071029663086 }, "peak_cpu_percent": { "mean": 108.12666575113933, "median": 108.0999984741211, "p90": 108.49199676513672, "p95": 108.54099655151367, "std_dev": 0.36790452672304097, "min": 107.69000244140625, "max": 108.58999633789062 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 165, "tokens_per_second": 111.49825783972126, "tpot_ms": 8.35686274509804, "total_time_ms": 2296, "total_tokens": 256, "peak_memory_mb": 512.77734375, "mean_cpu_percent": 102.00031, "peak_cpu_percent": 108.1, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 148, "tokens_per_second": 112.4780316344464, "tpot_ms": 8.345098039215687, "total_time_ms": 2276, "total_tokens": 256, "peak_memory_mb": 521.53515625, "mean_cpu_percent": 96.633415, "peak_cpu_percent": 105.66, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 147, "tokens_per_second": 111.5954664341761, "tpot_ms": 8.419607843137255, "total_time_ms": 2294, "total_tokens": 256, "peak_memory_mb": 528.93359375, "mean_cpu_percent": 95.39253, "peak_cpu_percent": 105.16, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 153.33333333333334, "median": 148.0, "p90": 161.6, "p95": 163.29999999999998, "std_dev": 8.259674462242577, "min": 147.0, "max": 165.0 }, "tokens_per_second": { "mean": 111.85725196944793, "median": 111.5954664341761, "p90": 112.30151859439233, "p95": 112.38977511441936, "std_dev": 0.44074779036366957, "min": 111.49825783972126, "max": 112.4780316344464 }, "tpot": { "mean": 8.373856209150327, "median": 8.35686274509804, "p90": 8.407058823529413, "p95": 8.413333333333334, "std_dev": 0.03270587190378237, "min": 8.345098039215687, "max": 8.419607843137255 }, "total_time": { "mean": 2288.6666666666665, "median": 2294.0, "p90": 2295.6, "p95": 2295.7999999999997, "std_dev": 8.993825042154693, "min": 2276.0, "max": 2296.0 }, "peak_memory_mb": { "mean": 521.08203125, "median": 521.53515625, "p90": 527.45390625, "p95": 528.19375, "std_dev": 6.603539213424053, "min": 512.77734375, "max": 528.93359375 }, "mean_cpu_percent": { "mean": 98.00875345865886, "median": 96.63341522216797, "p90": 100.92693328857422, "p95": 101.463623046875, "std_dev": 2.867560762394082, "min": 95.39253234863281, "max": 102.00031280517578 }, "peak_cpu_percent": { "mean": 106.30666859944661, "median": 105.66000366210938, "p90": 107.61199951171875, "p95": 107.85599899291992, "std_dev": 1.2843997377633873, "min": 105.16000366210938, "max": 108.0999984741211 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 150, "tokens_per_second": 111.35276207046542, "tpot_ms": 8.427450980392157, "total_time_ms": 2299, "total_tokens": 256, "peak_memory_mb": 538.1796875, "mean_cpu_percent": 100.69931, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 155, "tokens_per_second": 111.01474414570684, "tpot_ms": 8.435294117647059, "total_time_ms": 2306, "total_tokens": 256, "peak_memory_mb": 544.51171875, "mean_cpu_percent": 98.7013, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 162, "tokens_per_second": 109.82410982410984, "tpot_ms": 8.505882352941176, "total_time_ms": 2331, "total_tokens": 256, "peak_memory_mb": 552.16796875, "mean_cpu_percent": 97.29316, "peak_cpu_percent": 106.16, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 155.66666666666666, "median": 155.0, "p90": 160.6, "p95": 161.3, "std_dev": 4.9216076867444665, "min": 150.0, "max": 162.0 }, "tokens_per_second": { "mean": 110.73053868009403, "median": 111.01474414570684, "p90": 111.2851584855137, "p95": 111.31896027798956, "std_dev": 0.6556289512547643, "min": 109.82410982410984, "max": 111.35276207046542 }, "tpot": { "mean": 8.456209150326798, "median": 8.435294117647059, "p90": 8.491764705882353, "p95": 8.498823529411764, "std_dev": 0.035269902126096715, "min": 8.427450980392157, "max": 8.505882352941176 }, "total_time": { "mean": 2312.0, "median": 2306.0, "p90": 2326.0, "p95": 2328.5, "std_dev": 13.73559851869101, "min": 2299.0, "max": 2331.0 }, "peak_memory_mb": { "mean": 544.953125, "median": 544.51171875, "p90": 550.63671875, "p95": 551.40234375, "std_dev": 5.719215137903084, "min": 538.1796875, "max": 552.16796875 }, "mean_cpu_percent": { "mean": 98.89792378743489, "median": 98.70130157470703, "p90": 100.2997085571289, "p95": 100.49950942993163, "std_dev": 1.3974884780501726, "min": 97.29315948486328, "max": 100.69931030273438 }, "peak_cpu_percent": { "mean": 107.1800028483073, "median": 107.69000244140625, "p90": 107.69000244140625, "p95": 107.69000244140625, "std_dev": 0.7212483413653068, "min": 106.16000366210938, "max": 107.69000244140625 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 148, "tokens_per_second": 111.11111111111111, "tpot_ms": 8.454901960784314, "total_time_ms": 2304, "total_tokens": 256, "peak_memory_mb": 559.6796875, "mean_cpu_percent": 99.372795, "peak_cpu_percent": 107.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 151, "tokens_per_second": 111.30434782608695, "tpot_ms": 8.427450980392157, "total_time_ms": 2300, "total_tokens": 256, "peak_memory_mb": 567.66015625, "mean_cpu_percent": 100.64311, "peak_cpu_percent": 114.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 152, "tokens_per_second": 111.15935735996528, "tpot_ms": 8.435294117647059, "total_time_ms": 2303, "total_tokens": 256, "peak_memory_mb": 574.49609375, "mean_cpu_percent": 99.97847, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 150.33333333333334, "median": 151.0, "p90": 151.8, "p95": 151.9, "std_dev": 1.699673171197595, "min": 148.0, "max": 152.0 }, "tokens_per_second": { "mean": 111.19160543238779, "median": 111.15935735996528, "p90": 111.27534973286262, "p95": 111.2898487794748, "std_dev": 0.08211804769106694, "min": 111.11111111111111, "max": 111.30434782608695 }, "tpot": { "mean": 8.43921568627451, "median": 8.435294117647059, "p90": 8.450980392156863, "p95": 8.452941176470588, "std_dev": 0.01154478544618015, "min": 8.427450980392157, "max": 8.454901960784314 }, "total_time": { "mean": 2302.3333333333335, "median": 2303.0, "p90": 2303.8, "p95": 2303.9, "std_dev": 1.699673171197595, "min": 2300.0, "max": 2304.0 }, "peak_memory_mb": { "mean": 567.2786458333334, "median": 567.66015625, "p90": 573.12890625, "p95": 573.8125, "std_dev": 6.054785225465813, "min": 559.6796875, "max": 574.49609375 }, "mean_cpu_percent": { "mean": 99.99812571207683, "median": 99.97846984863281, "p90": 100.51018371582032, "p95": 100.57664794921875, "std_dev": 0.5187909882021664, "min": 99.37279510498047, "max": 100.64311218261719 }, "peak_cpu_percent": { "mean": 109.70333353678386, "median": 107.69000244140625, "p90": 112.96199951171874, "p95": 113.62099914550782, "std_dev": 3.2439712694736578, "min": 107.13999938964844, "max": 114.27999877929688 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 206, "tokens_per_second": 112.78026905829596, "tpot_ms": 8.474103585657371, "total_time_ms": 4460, "total_tokens": 503, "peak_memory_mb": 583.21875, "mean_cpu_percent": 97.53455, "peak_cpu_percent": 114.28, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 172, "tokens_per_second": 113.69801084990958, "tpot_ms": 8.47011952191235, "total_time_ms": 4424, "total_tokens": 503, "peak_memory_mb": 590.09765625, "mean_cpu_percent": 98.46375, "peak_cpu_percent": 115.38, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 176, "tokens_per_second": 113.49277978339347, "tpot_ms": 8.47808764940239, "total_time_ms": 4432, "total_tokens": 503, "peak_memory_mb": 597.37109375, "mean_cpu_percent": 97.63135, "peak_cpu_percent": 106.16, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 184.66666666666666, "median": 176.0, "p90": 200.0, "p95": 203.0, "std_dev": 15.173075568988057, "min": 172.0, "max": 206.0 }, "tokens_per_second": { "mean": 113.32368656386633, "median": 113.49277978339347, "p90": 113.65696463660636, "p95": 113.67748774325797, "std_dev": 0.3932826690698728, "min": 112.78026905829596, "max": 113.69801084990958 }, "tpot": { "mean": 8.47410358565737, "median": 8.474103585657371, "p90": 8.477290836653387, "p95": 8.477689243027887, "std_dev": 0.003252974426006874, "min": 8.47011952191235, "max": 8.47808764940239 }, "total_time": { "mean": 4438.666666666667, "median": 4432.0, "p90": 4454.4, "p95": 4457.2, "std_dev": 15.4344492037203, "min": 4424.0, "max": 4460.0 }, "peak_memory_mb": { "mean": 590.2291666666666, "median": 590.09765625, "p90": 595.91640625, "p95": 596.64375, "std_dev": 5.778418448376677, "min": 583.21875, "max": 597.37109375 }, "mean_cpu_percent": { "mean": 97.87655131022136, "median": 97.63134765625, "p90": 98.29727172851562, "p95": 98.38051223754883, "std_dev": 0.41709025479871087, "min": 97.53455352783203, "max": 98.46375274658203 }, "peak_cpu_percent": { "mean": 111.93999989827473, "median": 114.27999877929688, "p90": 115.15999755859376, "p95": 115.26999740600586, "std_dev": 4.111671722331565, "min": 106.16000366210938, "max": 115.37999725341797 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 174, "tokens_per_second": 113.89106072452437, "tpot_ms": 8.40137614678899, "total_time_ms": 3837, "total_tokens": 437, "peak_memory_mb": 608.6640625, "mean_cpu_percent": 99.287636, "peak_cpu_percent": 114.28, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 159, "tokens_per_second": 114.15882967607106, "tpot_ms": 8.4151376146789, "total_time_ms": 3828, "total_tokens": 437, "peak_memory_mb": 615.75390625, "mean_cpu_percent": 98.34268, "peak_cpu_percent": 100.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 161, "tokens_per_second": 113.65409622886864, "tpot_ms": 8.44954128440367, "total_time_ms": 3845, "total_tokens": 437, "peak_memory_mb": 624.23828125, "mean_cpu_percent": 99.07913, "peak_cpu_percent": 107.69, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 164.66666666666666, "median": 161.0, "p90": 171.4, "p95": 172.70000000000002, "std_dev": 6.649979114420001, "min": 159.0, "max": 174.0 }, "tokens_per_second": { "mean": 113.90132887648802, "median": 113.89106072452437, "p90": 114.10527588576173, "p95": 114.1320527809164, "std_dev": 0.20618444718224607, "min": 113.65409622886864, "max": 114.15882967607106 }, "tpot": { "mean": 8.422018348623853, "median": 8.4151376146789, "p90": 8.442660550458717, "p95": 8.446100917431194, "std_dev": 0.020256332262220126, "min": 8.40137614678899, "max": 8.44954128440367 }, "total_time": { "mean": 3836.6666666666665, "median": 3837.0, "p90": 3843.3999999999996, "p95": 3844.2, "std_dev": 6.944222218666553, "min": 3828.0, "max": 3845.0 }, "peak_memory_mb": { "mean": 616.21875, "median": 615.75390625, "p90": 622.5414062499999, "p95": 623.3898437500001, "std_dev": 6.366638684169663, "min": 608.6640625, "max": 624.23828125 }, "mean_cpu_percent": { "mean": 98.9031499226888, "median": 99.07913208007812, "p90": 99.24593505859376, "p95": 99.26678543090821, "std_dev": 0.40534904036840047, "min": 98.34268188476562, "max": 99.28763580322266 }, "peak_cpu_percent": { "mean": 107.32333374023438, "median": 107.69000244140625, "p90": 112.96199951171874, "p95": 113.62099914550782, "std_dev": 5.835547717023904, "min": 100.0, "max": 114.27999877929688 } } } ] }, { "backend": "directml", "model_id": "qwen3-4b", "cold_start_ttft_ms": 490, "idle_baseline": { "rss_mb": 407.48046875, "cpu_percent": 0.0 }, "reliability": { "total_runs": 30, "successful_runs": 30, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.3, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 240, "tokens_per_second": 51.92697768762678, "tpot_ms": 17.519685039370078, "total_time_ms": 2465, "total_tokens": 128, "peak_memory_mb": 441.10546875, "mean_cpu_percent": 96.210106, "peak_cpu_percent": 107.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 240, "tokens_per_second": 51.86385737439222, "tpot_ms": 17.543307086614174, "total_time_ms": 2468, "total_tokens": 128, "peak_memory_mb": 449.953125, "mean_cpu_percent": 96.44839, "peak_cpu_percent": 106.66, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 244, "tokens_per_second": 51.75899716942985, "tpot_ms": 17.551181102362204, "total_time_ms": 2473, "total_tokens": 128, "peak_memory_mb": 461.234375, "mean_cpu_percent": 96.95516, "peak_cpu_percent": 102.28, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 241.33333333333334, "median": 240.0, "p90": 243.2, "p95": 243.6, "std_dev": 1.8856180831641267, "min": 240.0, "max": 244.0 }, "tokens_per_second": { "mean": 51.849944077149615, "median": 51.86385737439222, "p90": 51.91435362497987, "p95": 51.92066565630332, "std_dev": 0.06927986000538397, "min": 51.75899716942985, "max": 51.92697768762678 }, "tpot": { "mean": 17.538057742782154, "median": 17.543307086614174, "p90": 17.5496062992126, "p95": 17.550393700787403, "std_dev": 0.013383253316516705, "min": 17.519685039370078, "max": 17.551181102362204 }, "total_time": { "mean": 2468.6666666666665, "median": 2468.0, "p90": 2472.0, "p95": 2472.5, "std_dev": 3.299831645537222, "min": 2465.0, "max": 2473.0 }, "peak_memory_mb": { "mean": 450.7643229166667, "median": 449.953125, "p90": 458.978125, "p95": 460.10625000000005, "std_dev": 8.237586549887272, "min": 441.10546875, "max": 461.234375 }, "mean_cpu_percent": { "mean": 96.53788503011067, "median": 96.4483871459961, "p90": 96.8538070678711, "p95": 96.90448455810547, "std_dev": 0.31068158510180893, "min": 96.2101058959961, "max": 96.95516204833984 }, "peak_cpu_percent": { "mean": 105.36000061035156, "median": 106.66000366210938, "p90": 107.04400024414063, "p95": 107.09199981689454, "std_dev": 2.1866881250099217, "min": 102.27999877929688, "max": 107.13999938964844 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 240, "tokens_per_second": 50.756901157613534, "tpot_ms": 17.75221238938053, "total_time_ms": 2246, "total_tokens": 114, "peak_memory_mb": 472.2578125, "mean_cpu_percent": 96.12952, "peak_cpu_percent": 105.66, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 267, "tokens_per_second": 50.264550264550266, "tpot_ms": 17.707964601769913, "total_time_ms": 2268, "total_tokens": 114, "peak_memory_mb": 481.625, "mean_cpu_percent": 96.53854, "peak_cpu_percent": 107.14, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 250, "tokens_per_second": 50.666666666666664, "tpot_ms": 17.699115044247787, "total_time_ms": 2250, "total_tokens": 114, "peak_memory_mb": 490.31640625, "mean_cpu_percent": 97.6177, "peak_cpu_percent": 106.66, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 252.33333333333334, "median": 250.0, "p90": 263.6, "p95": 265.3, "std_dev": 11.14550233153366, "min": 240.0, "max": 267.0 }, "tokens_per_second": { "mean": 50.562706029610155, "median": 50.666666666666664, "p90": 50.738854259424166, "p95": 50.74787770851884, "std_dev": 0.21402213439727183, "min": 50.264550264550266, "max": 50.756901157613534 }, "tpot": { "mean": 17.719764011799413, "median": 17.707964601769913, "p90": 17.74336283185841, "p95": 17.74778761061947, "std_dev": 0.02322716187024165, "min": 17.699115044247787, "max": 17.75221238938053 }, "total_time": { "mean": 2254.6666666666665, "median": 2250.0, "p90": 2264.4, "p95": 2266.2, "std_dev": 9.568466729604882, "min": 2246.0, "max": 2268.0 }, "peak_memory_mb": { "mean": 481.3997395833333, "median": 481.625, "p90": 488.578125, "p95": 489.447265625, "std_dev": 7.374110511258173, "min": 472.2578125, "max": 490.31640625 }, "mean_cpu_percent": { "mean": 96.76191965738933, "median": 96.53854370117188, "p90": 97.40186767578126, "p95": 97.50978317260743, "std_dev": 0.6277441517822983, "min": 96.1295166015625, "max": 97.6176986694336 }, "peak_cpu_percent": { "mean": 106.48666890462239, "median": 106.66000366210938, "p90": 107.04400024414063, "p95": 107.09199981689454, "std_dev": 0.6165119852840235, "min": 105.66000366210938, "max": 107.13999938964844 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 241, "tokens_per_second": 46.46596858638743, "tpot_ms": 18.385714285714286, "total_time_ms": 1528, "total_tokens": 71, "peak_memory_mb": 501.64453125, "mean_cpu_percent": 97.59401, "peak_cpu_percent": 105.16, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 245, "tokens_per_second": 46.25407166123779, "tpot_ms": 18.428571428571427, "total_time_ms": 1535, "total_tokens": 71, "peak_memory_mb": 511.26953125, "mean_cpu_percent": 100.0, "peak_cpu_percent": 100.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 243, "tokens_per_second": 46.13385315139701, "tpot_ms": 18.514285714285716, "total_time_ms": 1539, "total_tokens": 71, "peak_memory_mb": 522.4453125, "mean_cpu_percent": 96.67835, "peak_cpu_percent": 100.9, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 243.0, "median": 243.0, "p90": 244.6, "p95": 244.79999999999998, "std_dev": 1.632993161855452, "min": 241.0, "max": 245.0 }, "tokens_per_second": { "mean": 46.284631133007416, "median": 46.25407166123779, "p90": 46.42358920135751, "p95": 46.444778893872474, "std_dev": 0.13729670189252632, "min": 46.13385315139701, "max": 46.46596858638743 }, "tpot": { "mean": 18.442857142857147, "median": 18.428571428571427, "p90": 18.497142857142858, "p95": 18.505714285714287, "std_dev": 0.053452248382485634, "min": 18.385714285714286, "max": 18.514285714285716 }, "total_time": { "mean": 1534.0, "median": 1535.0, "p90": 1538.2, "p95": 1538.6000000000001, "std_dev": 4.546060565661952, "min": 1528.0, "max": 1539.0 }, "peak_memory_mb": { "mean": 511.7864583333333, "median": 511.26953125, "p90": 520.21015625, "p95": 521.327734375, "std_dev": 8.499746480913437, "min": 501.64453125, "max": 522.4453125 }, "mean_cpu_percent": { "mean": 98.09078725179036, "median": 97.59400939941406, "p90": 99.5188018798828, "p95": 99.7594009399414, "std_dev": 1.4008157013570721, "min": 96.67835235595703, "max": 100.0 }, "peak_cpu_percent": { "mean": 102.02000172932942, "median": 100.9000015258789, "p90": 104.30800323486329, "p95": 104.73400344848633, "std_dev": 2.250512503119004, "min": 100.0, "max": 105.16000366210938 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 242, "tokens_per_second": 49.88452655889145, "tpot_ms": 17.97196261682243, "total_time_ms": 2165, "total_tokens": 108, "peak_memory_mb": 533.11328125, "mean_cpu_percent": 101.47397, "peak_cpu_percent": 113.74, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 243, "tokens_per_second": 49.953746530989825, "tpot_ms": 17.934579439252335, "total_time_ms": 2162, "total_tokens": 108, "peak_memory_mb": 542.88671875, "mean_cpu_percent": 97.1978, "peak_cpu_percent": 123.07, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 243, "tokens_per_second": 49.9075785582255, "tpot_ms": 17.953271028037385, "total_time_ms": 2164, "total_tokens": 108, "peak_memory_mb": 553.0703125, "mean_cpu_percent": 98.47619, "peak_cpu_percent": 100.0, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 242.66666666666666, "median": 243.0, "p90": 243.0, "p95": 243.0, "std_dev": 0.4714045207910317, "min": 242.0, "max": 243.0 }, "tokens_per_second": { "mean": 49.91528388270226, "median": 49.9075785582255, "p90": 49.944512936436965, "p95": 49.94912973371339, "std_dev": 0.028779392549693104, "min": 49.88452655889145, "max": 49.953746530989825 }, "tpot": { "mean": 17.953271028037385, "median": 17.953271028037385, "p90": 17.968224299065422, "p95": 17.970093457943925, "std_dev": 0.01526161833509859, "min": 17.934579439252335, "max": 17.97196261682243 }, "total_time": { "mean": 2163.6666666666665, "median": 2164.0, "p90": 2164.7999999999997, "p95": 2164.9, "std_dev": 1.247219128924647, "min": 2162.0, "max": 2165.0 }, "peak_memory_mb": { "mean": 543.0234375, "median": 542.88671875, "p90": 551.03359375, "p95": 552.0519531250001, "std_dev": 8.14799742639545, "min": 533.11328125, "max": 553.0703125 }, "mean_cpu_percent": { "mean": 99.04931894938152, "median": 98.47618865966797, "p90": 100.8744125366211, "p95": 101.17419052124023, "std_dev": 1.7921613948818829, "min": 97.19779968261719, "max": 101.47396850585938 }, "peak_cpu_percent": { "mean": 112.26999918619792, "median": 113.73999786376953, "p90": 121.20399932861328, "p95": 122.13699951171876, "std_dev": 9.475473376524752, "min": 100.0, "max": 123.06999969482422 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 294, "tokens_per_second": 53.815429892789574, "tpot_ms": 17.501960784313727, "total_time_ms": 4757, "total_tokens": 256, "peak_memory_mb": 566.87890625, "mean_cpu_percent": 97.21563, "peak_cpu_percent": 106.16, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 248, "tokens_per_second": 54.52609158679446, "tpot_ms": 17.43921568627451, "total_time_ms": 4695, "total_tokens": 256, "peak_memory_mb": 578.6875, "mean_cpu_percent": 97.337105, "peak_cpu_percent": 106.16, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 247, "tokens_per_second": 54.11118156837878, "tpot_ms": 17.584313725490198, "total_time_ms": 4731, "total_tokens": 256, "peak_memory_mb": 588.04296875, "mean_cpu_percent": 99.61955, "peak_cpu_percent": 114.28, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 263.0, "median": 248.0, "p90": 284.8, "p95": 289.4, "std_dev": 21.924111536540465, "min": 247.0, "max": 294.0 }, "tokens_per_second": { "mean": 54.15090101598761, "median": 54.11118156837878, "p90": 54.44310958311132, "p95": 54.48460058495289, "std_dev": 0.29148268870293165, "min": 53.815429892789574, "max": 54.52609158679446 }, "tpot": { "mean": 17.508496732026142, "median": 17.501960784313727, "p90": 17.567843137254904, "p95": 17.57607843137255, "std_dev": 0.05941604275745744, "min": 17.43921568627451, "max": 17.584313725490198 }, "total_time": { "mean": 4727.666666666667, "median": 4731.0, "p90": 4751.8, "p95": 4754.4, "std_dev": 25.42090128658349, "min": 4695.0, "max": 4757.0 }, "peak_memory_mb": { "mean": 577.8697916666666, "median": 578.6875, "p90": 586.171875, "p95": 587.107421875, "std_dev": 8.65951771423708, "min": 566.87890625, "max": 588.04296875 }, "mean_cpu_percent": { "mean": 98.05742899576823, "median": 97.33710479736328, "p90": 99.1630630493164, "p95": 99.39130783081055, "std_dev": 1.1057008945333007, "min": 97.21562957763672, "max": 99.61955261230469 }, "peak_cpu_percent": { "mean": 108.86666870117188, "median": 106.16000366210938, "p90": 112.65599975585937, "p95": 113.46799926757814, "std_dev": 3.8278024070432908, "min": 106.16000366210938, "max": 114.27999877929688 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 243, "tokens_per_second": 54.19136325148179, "tpot_ms": 17.572549019607845, "total_time_ms": 4724, "total_tokens": 256, "peak_memory_mb": 598.28515625, "mean_cpu_percent": 99.54411, "peak_cpu_percent": 109.4, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 245, "tokens_per_second": 54.30632159524819, "tpot_ms": 17.525490196078433, "total_time_ms": 4714, "total_tokens": 256, "peak_memory_mb": 608.26953125, "mean_cpu_percent": 99.05864, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 262, "tokens_per_second": 53.9629005059022, "tpot_ms": 17.576470588235296, "total_time_ms": 4744, "total_tokens": 256, "peak_memory_mb": 619.15625, "mean_cpu_percent": 97.870316, "peak_cpu_percent": 106.66, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 250.0, "median": 245.0, "p90": 258.6, "p95": 260.3, "std_dev": 8.524474568362947, "min": 243.0, "max": 262.0 }, "tokens_per_second": { "mean": 54.15352845087739, "median": 54.19136325148179, "p90": 54.28332992649491, "p95": 54.29482576087155, "std_dev": 0.14273078447314308, "min": 53.9629005059022, "max": 54.30632159524819 }, "tpot": { "mean": 17.558169934640524, "median": 17.572549019607845, "p90": 17.575686274509806, "p95": 17.57607843137255, "std_dev": 0.02316345770806443, "min": 17.525490196078433, "max": 17.576470588235296 }, "total_time": { "mean": 4727.333333333333, "median": 4724.0, "p90": 4740.0, "p95": 4742.0, "std_dev": 12.472191289246473, "min": 4714.0, "max": 4744.0 }, "peak_memory_mb": { "mean": 608.5703125, "median": 608.26953125, "p90": 616.97890625, "p95": 618.067578125, "std_dev": 8.52324236428782, "min": 598.28515625, "max": 619.15625 }, "mean_cpu_percent": { "mean": 98.82435607910156, "median": 99.05863952636719, "p90": 99.44701843261718, "p95": 99.49556579589843, "std_dev": 0.7031197896517098, "min": 97.87031555175781, "max": 99.54411315917969 }, "peak_cpu_percent": { "mean": 107.91666920979817, "median": 107.69000244140625, "p90": 109.05800170898438, "p95": 109.22900161743163, "std_dev": 1.1300237287435837, "min": 106.66000366210938, "max": 109.4000015258789 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 242, "tokens_per_second": 54.43334042100786, "tpot_ms": 17.49411764705882, "total_time_ms": 4703, "total_tokens": 256, "peak_memory_mb": 630.203125, "mean_cpu_percent": 100.067505, "peak_cpu_percent": 115.38, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 243, "tokens_per_second": 54.145516074450086, "tpot_ms": 17.58823529411765, "total_time_ms": 4728, "total_tokens": 256, "peak_memory_mb": 642.27734375, "mean_cpu_percent": 99.6949, "peak_cpu_percent": 113.27, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 248, "tokens_per_second": 54.549328787555936, "tpot_ms": 17.431372549019606, "total_time_ms": 4693, "total_tokens": 256, "peak_memory_mb": 650.93359375, "mean_cpu_percent": 98.251175, "peak_cpu_percent": 106.66, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 244.33333333333334, "median": 243.0, "p90": 247.0, "p95": 247.5, "std_dev": 2.6246692913372702, "min": 242.0, "max": 248.0 }, "tokens_per_second": { "mean": 54.37606176100463, "median": 54.43334042100786, "p90": 54.526131114246326, "p95": 54.53772995090113, "std_dev": 0.16975828008338115, "min": 54.145516074450086, "max": 54.549328787555936 }, "tpot": { "mean": 17.50457516339869, "median": 17.49411764705882, "p90": 17.569411764705883, "p95": 17.578823529411768, "std_dev": 0.06446446016176838, "min": 17.431372549019606, "max": 17.58823529411765 }, "total_time": { "mean": 4708.0, "median": 4703.0, "p90": 4723.0, "p95": 4725.5, "std_dev": 14.719601443879744, "min": 4693.0, "max": 4728.0 }, "peak_memory_mb": { "mean": 641.1380208333334, "median": 642.27734375, "p90": 649.20234375, "p95": 650.06796875, "std_dev": 8.501436199451362, "min": 630.203125, "max": 650.93359375 }, "mean_cpu_percent": { "mean": 99.33786010742188, "median": 99.69490051269531, "p90": 99.99298400878907, "p95": 100.03024444580079, "std_dev": 0.7833143326335735, "min": 98.25117492675781, "max": 100.0675048828125 }, "peak_cpu_percent": { "mean": 111.76999918619792, "median": 113.2699966430664, "p90": 114.95799713134767, "p95": 115.16899719238282, "std_dev": 3.714571876279929, "min": 106.66000366210938, "max": 115.37999725341797 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 242, "tokens_per_second": 54.56095481670929, "tpot_ms": 17.45098039215686, "total_time_ms": 4692, "total_tokens": 256, "peak_memory_mb": 662.11328125, "mean_cpu_percent": 97.972435, "peak_cpu_percent": 108.59, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 250, "tokens_per_second": 54.595862657283, "tpot_ms": 17.4078431372549, "total_time_ms": 4689, "total_tokens": 256, "peak_memory_mb": 671.5703125, "mean_cpu_percent": 98.02537, "peak_cpu_percent": 108.59, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 244, "tokens_per_second": 54.31784426055591, "tpot_ms": 17.525490196078433, "total_time_ms": 4713, "total_tokens": 256, "peak_memory_mb": 682.171875, "mean_cpu_percent": 98.11816, "peak_cpu_percent": 106.66, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 245.33333333333334, "median": 244.0, "p90": 248.79999999999998, "p95": 249.39999999999998, "std_dev": 3.39934634239519, "min": 242.0, "max": 250.0 }, "tokens_per_second": { "mean": 54.491553911516064, "median": 54.56095481670929, "p90": 54.58888108916826, "p95": 54.59237187322563, "std_dev": 0.12365522353410663, "min": 54.31784426055591, "max": 54.595862657283 }, "tpot": { "mean": 17.461437908496734, "median": 17.45098039215686, "p90": 17.51058823529412, "p95": 17.518039215686276, "std_dev": 0.04859511188350755, "min": 17.4078431372549, "max": 17.525490196078433 }, "total_time": { "mean": 4698.0, "median": 4692.0, "p90": 4708.8, "p95": 4710.900000000001, "std_dev": 10.677078252031311, "min": 4689.0, "max": 4713.0 }, "peak_memory_mb": { "mean": 671.9518229166666, "median": 671.5703125, "p90": 680.0515625, "p95": 681.11171875, "std_dev": 8.193328930921124, "min": 662.11328125, "max": 682.171875 }, "mean_cpu_percent": { "mean": 98.03865305582683, "median": 98.0253677368164, "p90": 98.09959869384765, "p95": 98.10887756347657, "std_dev": 0.06022767343100078, "min": 97.9724349975586, "max": 98.11815643310547 }, "peak_cpu_percent": { "mean": 107.94666544596355, "median": 108.58999633789062, "p90": 108.58999633789062, "p95": 108.58999633789062, "std_dev": 0.9098072724568611, "min": 106.66000366210938, "max": 108.58999633789062 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 363, "tokens_per_second": 54.22579961872484, "tpot_ms": 17.767123287671232, "total_time_ms": 9442, "total_tokens": 512, "peak_memory_mb": 696.7578125, "mean_cpu_percent": 97.1005, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 335, "tokens_per_second": 54.03123680878007, "tpot_ms": 17.888454011741683, "total_time_ms": 9476, "total_tokens": 512, "peak_memory_mb": 705.703125, "mean_cpu_percent": 97.20728, "peak_cpu_percent": 114.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 323, "tokens_per_second": 54.294803817603395, "tpot_ms": 17.82191780821918, "total_time_ms": 9430, "total_tokens": 512, "peak_memory_mb": 715.80859375, "mean_cpu_percent": 97.94937, "peak_cpu_percent": 107.69, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 340.3333333333333, "median": 335.0, "p90": 357.40000000000003, "p95": 360.20000000000005, "std_dev": 16.75974011996871, "min": 323.0, "max": 363.0 }, "tokens_per_second": { "mean": 54.183946748369436, "median": 54.22579961872484, "p90": 54.28100297782768, "p95": 54.28790339771554, "std_dev": 0.11159641303303068, "min": 54.03123680878007, "max": 54.294803817603395 }, "tpot": { "mean": 17.825831702544033, "median": 17.82191780821918, "p90": 17.875146771037183, "p95": 17.88180039138943, "std_dev": 0.04961031530810578, "min": 17.767123287671232, "max": 17.888454011741683 }, "total_time": { "mean": 9449.333333333334, "median": 9442.0, "p90": 9469.2, "p95": 9472.6, "std_dev": 19.48218559493661, "min": 9430.0, "max": 9476.0 }, "peak_memory_mb": { "mean": 706.08984375, "median": 705.703125, "p90": 713.7875, "p95": 714.798046875, "std_dev": 7.782254604870367, "min": 696.7578125, "max": 715.80859375 }, "mean_cpu_percent": { "mean": 97.41905212402344, "median": 97.20728302001953, "p90": 97.8009536743164, "p95": 97.8751625061035, "std_dev": 0.377517679277513, "min": 97.10050201416016, "max": 97.94937133789062 }, "peak_cpu_percent": { "mean": 109.8866678873698, "median": 107.69000244140625, "p90": 112.96199951171874, "p95": 113.62099914550782, "std_dev": 3.1065540656779835, "min": 107.69000244140625, "max": 114.27999877929688 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 304, "tokens_per_second": 54.81798715203426, "tpot_ms": 17.682974559686887, "total_time_ms": 9340, "total_tokens": 512, "peak_memory_mb": 724.49609375, "mean_cpu_percent": 99.531075, "peak_cpu_percent": 114.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 307, "tokens_per_second": 54.47967652692062, "tpot_ms": 17.79060665362035, "total_time_ms": 9398, "total_tokens": 512, "peak_memory_mb": 734.35546875, "mean_cpu_percent": 98.06893, "peak_cpu_percent": 114.79, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 303, "tokens_per_second": 54.64830825061373, "tpot_ms": 17.741682974559687, "total_time_ms": 9369, "total_tokens": 512, "peak_memory_mb": 743.75390625, "mean_cpu_percent": 98.51429, "peak_cpu_percent": 106.66, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 304.6666666666667, "median": 304.0, "p90": 306.4, "p95": 306.7, "std_dev": 1.699673171197595, "min": 303.0, "max": 307.0 }, "tokens_per_second": { "mean": 54.6486573098562, "median": 54.64830825061373, "p90": 54.78405137175016, "p95": 54.80101926189221, "std_dev": 0.13811495489376946, "min": 54.47967652692062, "max": 54.81798715203426 }, "tpot": { "mean": 17.738421395955644, "median": 17.741682974559687, "p90": 17.78082191780822, "p95": 17.785714285714285, "std_dev": 0.04400110098901538, "min": 17.682974559686887, "max": 17.79060665362035 }, "total_time": { "mean": 9369.0, "median": 9369.0, "p90": 9392.2, "p95": 9395.1, "std_dev": 23.678400846904054, "min": 9340.0, "max": 9398.0 }, "peak_memory_mb": { "mean": 734.2018229166666, "median": 734.35546875, "p90": 741.8742187500001, "p95": 742.8140625000001, "std_dev": 7.862719667429101, "min": 724.49609375, "max": 743.75390625 }, "mean_cpu_percent": { "mean": 98.70476531982422, "median": 98.51428985595703, "p90": 99.32771759033204, "p95": 99.42939605712891, "std_dev": 0.6119238374010316, "min": 98.06893157958984, "max": 99.53107452392578 }, "peak_cpu_percent": { "mean": 111.91000111897786, "median": 114.27999877929688, "p90": 114.68800048828125, "p95": 114.73900070190429, "std_dev": 3.7181429520289755, "min": 106.66000366210938, "max": 114.79000091552734 } } } ] }, { "backend": "openvino_npu", "model_id": "qwen2.5-1.5b-instruct", "cold_start_ttft_ms": 2298, "idle_baseline": { "rss_mb": 2150.39453125, "cpu_percent": 0.0 }, "reliability": { "total_runs": 30, "successful_runs": 30, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.2, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 2284, "tokens_per_second": 16.492097854614258, "tpot_ms": 60.645669291338585, "total_time_ms": 9986, "total_tokens": 128, "peak_memory_mb": 2610.2734375, "mean_cpu_percent": 10.90268, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2274, "tokens_per_second": 16.513307571411133, "tpot_ms": 60.56692913385827, "total_time_ms": 9966, "total_tokens": 128, "peak_memory_mb": 2610.27734375, "mean_cpu_percent": 6.8365717, "peak_cpu_percent": 28.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2273, "tokens_per_second": 16.583009719848633, "tpot_ms": 60.30708661417323, "total_time_ms": 9932, "total_tokens": 128, "peak_memory_mb": 2610.2265625, "mean_cpu_percent": 7.300208, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2277.0, "median": 2274.0, "p90": 2282.0, "p95": 2283.0, "std_dev": 4.96655480858378, "min": 2273.0, "max": 2284.0 }, "tokens_per_second": { "mean": 16.52947171529134, "median": 16.513307571411133, "p90": 16.56906929016113, "p95": 16.576039505004882, "std_dev": 0.03883470501697631, "min": 16.492097854614258, "max": 16.583009719848633 }, "tpot": { "mean": 60.506561679790025, "median": 60.56692913385827, "p90": 60.62992125984252, "p95": 60.637795275590555, "std_dev": 0.14466681133355042, "min": 60.30708661417323, "max": 60.645669291338585 }, "total_time": { "mean": 9961.333333333334, "median": 9966.0, "p90": 9982.0, "p95": 9984.0, "std_dev": 22.29100466306732, "min": 9932.0, "max": 9986.0 }, "peak_memory_mb": { "mean": 2610.2591145833335, "median": 2610.2734375, "p90": 2610.2765625, "p95": 2610.2769531249996, "std_dev": 0.02307297545139238, "min": 2610.2265625, "max": 2610.27734375 }, "mean_cpu_percent": { "mean": 8.346486727396647, "median": 7.30020809173584, "p90": 10.182185935974122, "p95": 10.542433166503907, "std_dev": 1.817385344385371, "min": 6.83657169342041, "max": 10.902680397033691 }, "peak_cpu_percent": { "mean": 35.709999084472656, "median": 35.709999084472656, "p90": 41.42199859619141, "p95": 42.13599853515625, "std_dev": 5.829785089474, "min": 28.56999969482422, "max": 42.849998474121094 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 2280, "tokens_per_second": 16.491140365600586, "tpot_ms": 60.645669291338585, "total_time_ms": 9982, "total_tokens": 128, "peak_memory_mb": 2610.26953125, "mean_cpu_percent": 7.9119854, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2274, "tokens_per_second": 16.496742248535156, "tpot_ms": 60.62204724409449, "total_time_ms": 9973, "total_tokens": 128, "peak_memory_mb": 2610.35546875, "mean_cpu_percent": 10.095728, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2277, "tokens_per_second": 16.528676986694336, "tpot_ms": 60.503937007874015, "total_time_ms": 9961, "total_tokens": 128, "peak_memory_mb": 2610.35546875, "mean_cpu_percent": 9.011104, "peak_cpu_percent": 57.14, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2277.0, "median": 2277.0, "p90": 2279.4, "p95": 2279.7000000000003, "std_dev": 2.449489742783178, "min": 2274.0, "max": 2280.0 }, "tokens_per_second": { "mean": 16.50551986694336, "median": 16.496742248535156, "p90": 16.5222900390625, "p95": 16.525483512878417, "std_dev": 0.016533489639193114, "min": 16.491140365600586, "max": 16.528676986694336 }, "tpot": { "mean": 60.59055118110236, "median": 60.62204724409449, "p90": 60.640944881889766, "p95": 60.64330708661418, "std_dev": 0.06200006200009424, "min": 60.503937007874015, "max": 60.645669291338585 }, "total_time": { "mean": 9972.0, "median": 9973.0, "p90": 9980.2, "p95": 9981.1, "std_dev": 8.602325267042627, "min": 9961.0, "max": 9982.0 }, "peak_memory_mb": { "mean": 2610.3268229166665, "median": 2610.35546875, "p90": 2610.35546875, "p95": 2610.35546875, "std_dev": 0.040511326005479284, "min": 2610.26953125, "max": 2610.35546875 }, "mean_cpu_percent": { "mean": 9.006272315979004, "median": 9.011103630065918, "p90": 9.878803062438966, "p95": 9.987265491485596, "std_dev": 0.8915156974118402, "min": 7.911985397338867, "max": 10.095727920532227 }, "peak_cpu_percent": { "mean": 52.37999979654948, "median": 50.0, "p90": 55.71199951171875, "p95": 56.4259994506836, "std_dev": 3.3658279907254802, "min": 50.0, "max": 57.13999938964844 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 2283, "tokens_per_second": 16.60100746154785, "tpot_ms": 60.244680851063826, "total_time_ms": 7946, "total_tokens": 95, "peak_memory_mb": 2610.35546875, "mean_cpu_percent": 8.849207, "peak_cpu_percent": 42.85, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 2282, "tokens_per_second": 16.50340461730957, "tpot_ms": 60.59574468085106, "total_time_ms": 7978, "total_tokens": 95, "peak_memory_mb": 2610.35546875, "mean_cpu_percent": 5.987069, "peak_cpu_percent": 21.42, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 2277, "tokens_per_second": 16.535335540771484, "tpot_ms": 60.48936170212766, "total_time_ms": 7963, "total_tokens": 95, "peak_memory_mb": 2610.35546875, "mean_cpu_percent": 8.164833, "peak_cpu_percent": 35.71, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 2280.6666666666665, "median": 2282.0, "p90": 2282.8, "p95": 2282.9, "std_dev": 2.6246692913372702, "min": 2277.0, "max": 2283.0 }, "tokens_per_second": { "mean": 16.5465825398763, "median": 16.535335540771484, "p90": 16.58787307739258, "p95": 16.594440269470216, "std_dev": 0.04063208947370191, "min": 16.50340461730957, "max": 16.60100746154785 }, "tpot": { "mean": 60.44326241134752, "median": 60.48936170212766, "p90": 60.574468085106375, "p95": 60.58510638297872, "std_dev": 0.14698143103590683, "min": 60.244680851063826, "max": 60.59574468085106 }, "total_time": { "mean": 7962.333333333333, "median": 7963.0, "p90": 7975.0, "p95": 7976.5, "std_dev": 13.072447700751718, "min": 7946.0, "max": 7978.0 }, "peak_memory_mb": { "mean": 2610.35546875, "median": 2610.35546875, "p90": 2610.35546875, "p95": 2610.35546875, "std_dev": 0.0, "min": 2610.35546875, "max": 2610.35546875 }, "mean_cpu_percent": { "mean": 7.667036374409993, "median": 8.164833068847656, "p90": 8.712332153320313, "p95": 8.780769538879394, "std_dev": 1.220330378876541, "min": 5.987069129943848, "max": 8.849206924438477 }, "peak_cpu_percent": { "mean": 33.3266658782959, "median": 35.709999084472656, "p90": 41.42199859619141, "p95": 42.13599853515625, "std_dev": 8.909598408887168, "min": 21.420000076293945, "max": 42.849998474121094 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 2277, "tokens_per_second": 16.515731811523438, "tpot_ms": 60.55905511811024, "total_time_ms": 9968, "total_tokens": 128, "peak_memory_mb": 2610.35546875, "mean_cpu_percent": 7.465029, "peak_cpu_percent": 28.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2278, "tokens_per_second": 16.51392936706543, "tpot_ms": 60.55905511811024, "total_time_ms": 9969, "total_tokens": 128, "peak_memory_mb": 2610.34765625, "mean_cpu_percent": 8.8716345, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2280, "tokens_per_second": 16.5122013092041, "tpot_ms": 60.56692913385827, "total_time_ms": 9972, "total_tokens": 128, "peak_memory_mb": 2610.41015625, "mean_cpu_percent": 7.145655, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2278.3333333333335, "median": 2278.0, "p90": 2279.6, "p95": 2279.8, "std_dev": 1.247219128924647, "min": 2277.0, "max": 2280.0 }, "tokens_per_second": { "mean": 16.513954162597656, "median": 16.51392936706543, "p90": 16.515371322631836, "p95": 16.515551567077637, "std_dev": 0.001441428173843083, "min": 16.5122013092041, "max": 16.515731811523438 }, "tpot": { "mean": 60.561679790026254, "median": 60.55905511811024, "p90": 60.565354330708665, "p95": 60.56614173228346, "std_dev": 0.0037118466204009806, "min": 60.55905511811024, "max": 60.56692913385827 }, "total_time": { "mean": 9969.666666666666, "median": 9969.0, "p90": 9971.4, "p95": 9971.7, "std_dev": 1.699673171197595, "min": 9968.0, "max": 9972.0 }, "peak_memory_mb": { "mean": 2610.37109375, "median": 2610.35546875, "p90": 2610.39921875, "p95": 2610.4046875, "std_dev": 0.02780489128133154, "min": 2610.34765625, "max": 2610.41015625 }, "mean_cpu_percent": { "mean": 7.827439467112224, "median": 7.465028762817383, "p90": 8.590313339233399, "p95": 8.7309739112854, "std_dev": 0.7497809899100113, "min": 7.145655155181885, "max": 8.871634483337402 }, "peak_cpu_percent": { "mean": 38.08999888102213, "median": 42.849998474121094, "p90": 42.849998474121094, "p95": 42.849998474121094, "std_dev": 6.7316559814509604, "min": 28.56999969482422, "max": 42.849998474121094 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 2276, "tokens_per_second": 16.578691482543945, "tpot_ms": 60.31764705882353, "total_time_ms": 17657, "total_tokens": 256, "peak_memory_mb": 2610.43359375, "mean_cpu_percent": 7.4131417, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2281, "tokens_per_second": 16.599811553955078, "tpot_ms": 60.24313725490196, "total_time_ms": 17643, "total_tokens": 256, "peak_memory_mb": 2610.47265625, "mean_cpu_percent": 7.2259665, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2278, "tokens_per_second": 16.605175018310547, "tpot_ms": 60.227450980392156, "total_time_ms": 17636, "total_tokens": 256, "peak_memory_mb": 2610.5390625, "mean_cpu_percent": 8.583451, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2278.3333333333335, "median": 2278.0, "p90": 2280.4, "p95": 2280.7, "std_dev": 2.0548046676563256, "min": 2276.0, "max": 2281.0 }, "tokens_per_second": { "mean": 16.59455935160319, "median": 16.599811553955078, "p90": 16.604102325439452, "p95": 16.604638671875, "std_dev": 0.011431933019225857, "min": 16.578691482543945, "max": 16.605175018310547 }, "tpot": { "mean": 60.26274509803921, "median": 60.24313725490196, "p90": 60.30274509803922, "p95": 60.310196078431375, "std_dev": 0.0393461880870405, "min": 60.227450980392156, "max": 60.31764705882353 }, "total_time": { "mean": 17645.333333333332, "median": 17643.0, "p90": 17654.2, "p95": 17655.600000000002, "std_dev": 8.73053390247253, "min": 17636.0, "max": 17657.0 }, "peak_memory_mb": { "mean": 2610.4817708333335, "median": 2610.47265625, "p90": 2610.52578125, "p95": 2610.532421875, "std_dev": 0.04353711848255335, "min": 2610.43359375, "max": 2610.5390625 }, "mean_cpu_percent": { "mean": 7.740853150685628, "median": 7.41314172744751, "p90": 8.349389362335206, "p95": 8.466420316696167, "std_dev": 0.600687017855184, "min": 7.225966453552246, "max": 8.583451271057129 }, "peak_cpu_percent": { "mean": 38.08999888102213, "median": 35.709999084472656, "p90": 41.42199859619141, "p95": 42.13599853515625, "std_dev": 3.3658279907254802, "min": 35.709999084472656, "max": 42.849998474121094 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 2282, "tokens_per_second": 16.542448043823242, "tpot_ms": 60.45490196078431, "total_time_ms": 17698, "total_tokens": 256, "peak_memory_mb": 2610.5703125, "mean_cpu_percent": 7.317788, "peak_cpu_percent": 28.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2259, "tokens_per_second": 16.563138961791992, "tpot_ms": 60.38039215686275, "total_time_ms": 17656, "total_tokens": 256, "peak_memory_mb": 2610.62109375, "mean_cpu_percent": 8.561351, "peak_cpu_percent": 43.83, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2286, "tokens_per_second": 16.585206985473633, "tpot_ms": 60.29803921568627, "total_time_ms": 17662, "total_tokens": 256, "peak_memory_mb": 2610.625, "mean_cpu_percent": 8.204321, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2275.6666666666665, "median": 2282.0, "p90": 2285.2000000000003, "p95": 2285.6, "std_dev": 11.897712198383164, "min": 2259.0, "max": 2286.0 }, "tokens_per_second": { "mean": 16.56359799702962, "median": 16.563138961791992, "p90": 16.580793380737305, "p95": 16.58300018310547, "std_dev": 0.017459282302496805, "min": 16.542448043823242, "max": 16.585206985473633 }, "tpot": { "mean": 60.37777777777777, "median": 60.38039215686275, "p90": 60.44, "p95": 60.447450980392155, "std_dev": 0.0640656248619232, "min": 60.29803921568627, "max": 60.45490196078431 }, "total_time": { "mean": 17672.0, "median": 17662.0, "p90": 17690.8, "p95": 17694.4, "std_dev": 18.547236990991408, "min": 17656.0, "max": 17698.0 }, "peak_memory_mb": { "mean": 2610.60546875, "median": 2610.62109375, "p90": 2610.6242187499997, "p95": 2610.6246093749996, "std_dev": 0.024910320924100247, "min": 2610.5703125, "max": 2610.625 }, "mean_cpu_percent": { "mean": 8.027819951375326, "median": 8.204320907592773, "p90": 8.48994483947754, "p95": 8.525647830963134, "std_dev": 0.5227979130601063, "min": 7.317788124084473, "max": 8.56135082244873 }, "peak_cpu_percent": { "mean": 36.03666687011719, "median": 35.709999084472656, "p90": 42.20600128173828, "p95": 43.018001556396484, "std_dev": 6.234150579994735, "min": 28.56999969482422, "max": 43.83000183105469 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 2284, "tokens_per_second": 16.462697982788086, "tpot_ms": 60.74901960784314, "total_time_ms": 17775, "total_tokens": 256, "peak_memory_mb": 2610.640625, "mean_cpu_percent": 8.398732, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2281, "tokens_per_second": 16.480562210083008, "tpot_ms": 60.67843137254902, "total_time_ms": 17754, "total_tokens": 256, "peak_memory_mb": 2610.640625, "mean_cpu_percent": 5.8269105, "peak_cpu_percent": 21.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2274, "tokens_per_second": 16.53640365600586, "tpot_ms": 60.47843137254902, "total_time_ms": 17696, "total_tokens": 256, "peak_memory_mb": 2610.796875, "mean_cpu_percent": 10.49866, "peak_cpu_percent": 51.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2279.6666666666665, "median": 2281.0, "p90": 2283.4, "p95": 2283.7000000000003, "std_dev": 4.189935029992179, "min": 2274.0, "max": 2284.0 }, "tokens_per_second": { "mean": 16.493221282958984, "median": 16.480562210083008, "p90": 16.52523536682129, "p95": 16.530819511413576, "std_dev": 0.03139342458414676, "min": 16.462697982788086, "max": 16.53640365600586 }, "tpot": { "mean": 60.63529411764706, "median": 60.67843137254902, "p90": 60.73490196078431, "p95": 60.74196078431373, "std_dev": 0.11460109173492244, "min": 60.47843137254902, "max": 60.74901960784314 }, "total_time": { "mean": 17741.666666666668, "median": 17754.0, "p90": 17770.8, "p95": 17772.9, "std_dev": 33.40991203553554, "min": 17696.0, "max": 17775.0 }, "peak_memory_mb": { "mean": 2610.6927083333335, "median": 2610.640625, "p90": 2610.765625, "p95": 2610.78125, "std_dev": 0.0736569563735987, "min": 2610.640625, "max": 2610.796875 }, "mean_cpu_percent": { "mean": 8.241434256235758, "median": 8.39873218536377, "p90": 10.078674507141113, "p95": 10.288667297363281, "std_dev": 1.9104742938279007, "min": 5.826910495758057, "max": 10.49866008758545 }, "peak_cpu_percent": { "mean": 41.089999516805015, "median": 50.0, "p90": 51.47999877929688, "p95": 51.664998626708986, "std_dev": 13.929280481078743, "min": 21.420000076293945, "max": 51.849998474121094 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 2282, "tokens_per_second": 16.525211334228516, "tpot_ms": 60.51764705882353, "total_time_ms": 17714, "total_tokens": 256, "peak_memory_mb": 2610.80078125, "mean_cpu_percent": 8.1894045, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2279, "tokens_per_second": 16.51581573486328, "tpot_ms": 60.55294117647059, "total_time_ms": 17720, "total_tokens": 256, "peak_memory_mb": 2610.81640625, "mean_cpu_percent": 7.851759, "peak_cpu_percent": 28.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2281, "tokens_per_second": 16.48301124572754, "tpot_ms": 60.67058823529412, "total_time_ms": 17752, "total_tokens": 256, "peak_memory_mb": 2610.84765625, "mean_cpu_percent": 8.905246, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2280.6666666666665, "median": 2281.0, "p90": 2281.8, "p95": 2281.9, "std_dev": 1.247219128924647, "min": 2279.0, "max": 2282.0 }, "tokens_per_second": { "mean": 16.508012771606445, "median": 16.51581573486328, "p90": 16.523332214355467, "p95": 16.524271774291993, "std_dev": 0.01809008096652624, "min": 16.48301124572754, "max": 16.525211334228516 }, "tpot": { "mean": 60.58039215686275, "median": 60.55294117647059, "p90": 60.64705882352941, "p95": 60.65882352941177, "std_dev": 0.06538561568836587, "min": 60.51764705882353, "max": 60.67058823529412 }, "total_time": { "mean": 17728.666666666668, "median": 17720.0, "p90": 17745.6, "p95": 17748.8, "std_dev": 16.679994670929073, "min": 17714.0, "max": 17752.0 }, "peak_memory_mb": { "mean": 2610.8216145833335, "median": 2610.81640625, "p90": 2610.8414062499996, "p95": 2610.84453125, "std_dev": 0.01948779888944761, "min": 2610.80078125, "max": 2610.84765625 }, "mean_cpu_percent": { "mean": 8.315469741821289, "median": 8.189404487609863, "p90": 8.762077522277831, "p95": 8.833661651611328, "std_dev": 0.43922504356629516, "min": 7.85175895690918, "max": 8.905245780944824 }, "peak_cpu_percent": { "mean": 35.709999084472656, "median": 35.709999084472656, "p90": 41.42199859619141, "p95": 42.13599853515625, "std_dev": 5.829785089474, "min": 28.56999969482422, "max": 42.849998474121094 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 2280, "tokens_per_second": 16.47112464904785, "tpot_ms": 60.713163064833005, "total_time_ms": 33183, "total_tokens": 510, "peak_memory_mb": 2610.8828125, "mean_cpu_percent": 9.8117895, "peak_cpu_percent": 71.42, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 2288, "tokens_per_second": 16.431880950927734, "tpot_ms": 60.858546168958746, "total_time_ms": 33265, "total_tokens": 510, "peak_memory_mb": 2610.8359375, "mean_cpu_percent": 9.539401, "peak_cpu_percent": 42.85, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 2279, "tokens_per_second": 16.49428939819336, "tpot_ms": 60.6286836935167, "total_time_ms": 33139, "total_tokens": 510, "peak_memory_mb": 2610.8359375, "mean_cpu_percent": 10.1209135, "peak_cpu_percent": 42.85, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 2282.3333333333335, "median": 2280.0, "p90": 2286.4, "p95": 2287.2, "std_dev": 4.027681991198191, "min": 2279.0, "max": 2288.0 }, "tokens_per_second": { "mean": 16.46576499938965, "median": 16.47112464904785, "p90": 16.489656448364258, "p95": 16.49197292327881, "std_dev": 0.025758467292272158, "min": 16.431880950927734, "max": 16.49428939819336 }, "tpot": { "mean": 60.73346430910282, "median": 60.713163064833005, "p90": 60.8294695481336, "p95": 60.84400785854617, "std_dev": 0.09493258939290687, "min": 60.6286836935167, "max": 60.858546168958746 }, "total_time": { "mean": 33195.666666666664, "median": 33183.0, "p90": 33248.6, "p95": 33256.8, "std_dev": 52.21323799787007, "min": 33139.0, "max": 33265.0 }, "peak_memory_mb": { "mean": 2610.8515625, "median": 2610.8359375, "p90": 2610.8734375, "p95": 2610.8781249999997, "std_dev": 0.02209708691207961, "min": 2610.8359375, "max": 2610.8828125 }, "mean_cpu_percent": { "mean": 9.824034690856934, "median": 9.811789512634277, "p90": 10.059088706970215, "p95": 10.090001106262207, "std_dev": 0.23755931330828944, "min": 9.539401054382324, "max": 10.1209135055542 }, "peak_cpu_percent": { "mean": 52.373331705729164, "median": 42.849998474121094, "p90": 65.70599822998047, "p95": 68.56299819946288, "std_dev": 13.468027015138532, "min": 42.849998474121094, "max": 71.41999816894531 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 2280, "tokens_per_second": 16.533782958984375, "tpot_ms": 60.483365949119374, "total_time_ms": 33187, "total_tokens": 512, "peak_memory_mb": 2610.90234375, "mean_cpu_percent": 8.781249, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2278, "tokens_per_second": 16.458824157714844, "tpot_ms": 60.759295499021526, "total_time_ms": 33326, "total_tokens": 512, "peak_memory_mb": 2611.0546875, "mean_cpu_percent": 8.3743305, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2293, "tokens_per_second": 16.50587272644043, "tpot_ms": 60.585127201565555, "total_time_ms": 33252, "total_tokens": 512, "peak_memory_mb": 2611.2421875, "mean_cpu_percent": 9.611702, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 2283.6666666666665, "median": 2280.0, "p90": 2290.4, "p95": 2291.7, "std_dev": 6.649979114420002, "min": 2278.0, "max": 2293.0 }, "tokens_per_second": { "mean": 16.49949328104655, "median": 16.50587272644043, "p90": 16.528200912475587, "p95": 16.530991935729983, "std_dev": 0.030932490627077914, "min": 16.458824157714844, "max": 16.533782958984375 }, "tpot": { "mean": 60.60926288323549, "median": 60.585127201565555, "p90": 60.72446183953033, "p95": 60.741878669275934, "std_dev": 0.11393324792630456, "min": 60.483365949119374, "max": 60.759295499021526 }, "total_time": { "mean": 33255.0, "median": 33252.0, "p90": 33311.200000000004, "p95": 33318.6, "std_dev": 56.786148545808835, "min": 33187.0, "max": 33326.0 }, "peak_memory_mb": { "mean": 2611.06640625, "median": 2611.0546875, "p90": 2611.2046875, "p95": 2611.2234375, "std_dev": 0.13898786620825934, "min": 2610.90234375, "max": 2611.2421875 }, "mean_cpu_percent": { "mean": 8.9224271774292, "median": 8.781249046325684, "p90": 9.445611381530762, "p95": 9.528656673431398, "std_dev": 0.514924247870488, "min": 8.374330520629883, "max": 9.611701965332031 }, "peak_cpu_percent": { "mean": 42.85333251953125, "median": 42.849998474121094, "p90": 48.56999969482422, "p95": 49.28499984741211, "std_dev": 5.833868920841307, "min": 35.709999084472656, "max": 50.0 } } } ] }, { "backend": "openvino_npu", "model_id": "qwen3-4b", "cold_start_ttft_ms": 4175, "idle_baseline": { "rss_mb": 5078.05859375, "cpu_percent": 0.0 }, "reliability": { "total_runs": 30, "successful_runs": 30, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.1, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 4086, "tokens_per_second": 7.841137886047363, "tpot_ms": 127.53543307086615, "total_time_ms": 20283, "total_tokens": 128, "peak_memory_mb": 5105.05078125, "mean_cpu_percent": 5.9080806, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4085, "tokens_per_second": 7.856079578399658, "tpot_ms": 127.2992125984252, "total_time_ms": 20252, "total_tokens": 128, "peak_memory_mb": 5105.171875, "mean_cpu_percent": 5.00918, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4094, "tokens_per_second": 7.802459239959717, "tpot_ms": 128.16535433070865, "total_time_ms": 20371, "total_tokens": 128, "peak_memory_mb": 5105.20703125, "mean_cpu_percent": 4.9085083, "peak_cpu_percent": 21.42, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4088.3333333333335, "median": 4086.0, "p90": 4092.4, "p95": 4093.2, "std_dev": 4.027681991198191, "min": 4085.0, "max": 4094.0 }, "tokens_per_second": { "mean": 7.833225568135579, "median": 7.841137886047363, "p90": 7.853091239929199, "p95": 7.854585409164429, "std_dev": 0.02259408115202322, "min": 7.802459239959717, "max": 7.856079578399658 }, "tpot": { "mean": 127.66666666666667, "median": 127.53543307086615, "p90": 128.03937007874015, "p95": 128.1023622047244, "std_dev": 0.36557449546414417, "min": 127.2992125984252, "max": 128.16535433070865 }, "total_time": { "mean": 20302.0, "median": 20283.0, "p90": 20353.4, "p95": 20362.2, "std_dev": 50.40502620440412, "min": 20252.0, "max": 20371.0 }, "peak_memory_mb": { "mean": 5105.143229166667, "median": 5105.171875, "p90": 5105.2, "p95": 5105.203515625, "std_dev": 0.06692758997862706, "min": 5105.05078125, "max": 5105.20703125 }, "mean_cpu_percent": { "mean": 5.275256315867106, "median": 5.009180068969727, "p90": 5.728300476074219, "p95": 5.8181905269622805, "std_dev": 0.44935777219161627, "min": 4.90850830078125, "max": 5.908080577850342 }, "peak_cpu_percent": { "mean": 38.089999516805015, "median": 42.849998474121094, "p90": 48.56999969482422, "p95": 49.28499984741211, "std_dev": 12.143511064624793, "min": 21.420000076293945, "max": 50.0 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 4080, "tokens_per_second": 7.933797836303711, "tpot_ms": 126.04724409448819, "total_time_ms": 20088, "total_tokens": 128, "peak_memory_mb": 5105.91796875, "mean_cpu_percent": 5.647044, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4101, "tokens_per_second": 7.902234077453613, "tpot_ms": 126.55905511811024, "total_time_ms": 20174, "total_tokens": 128, "peak_memory_mb": 5105.94921875, "mean_cpu_percent": 7.212716, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4086, "tokens_per_second": 7.925702571868896, "tpot_ms": 126.18110236220473, "total_time_ms": 20111, "total_tokens": 128, "peak_memory_mb": 5105.98046875, "mean_cpu_percent": 7.07751, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4089.0, "median": 4086.0, "p90": 4098.0, "p95": 4099.5, "std_dev": 8.831760866327848, "min": 4080.0, "max": 4101.0 }, "tokens_per_second": { "mean": 7.920578161875407, "median": 7.925702571868896, "p90": 7.932178783416747, "p95": 7.932988309860229, "std_dev": 0.013385624167312406, "min": 7.902234077453613, "max": 7.933797836303711 }, "tpot": { "mean": 126.26246719160105, "median": 126.18110236220473, "p90": 126.48346456692914, "p95": 126.5212598425197, "std_dev": 0.21672226092748476, "min": 126.04724409448819, "max": 126.55905511811024 }, "total_time": { "mean": 20124.333333333332, "median": 20111.0, "p90": 20161.4, "p95": 20167.7, "std_dev": 36.353205574688396, "min": 20088.0, "max": 20174.0 }, "peak_memory_mb": { "mean": 5105.94921875, "median": 5105.94921875, "p90": 5105.97421875, "p95": 5105.97734375, "std_dev": 0.02551551815399144, "min": 5105.91796875, "max": 5105.98046875 }, "mean_cpu_percent": { "mean": 6.645756721496582, "median": 7.077509880065918, "p90": 7.185674858093262, "p95": 7.199195480346679, "std_dev": 0.7083503056764303, "min": 5.6470441818237305, "max": 7.212716102600098 }, "peak_cpu_percent": { "mean": 42.849998474121094, "median": 42.849998474121094, "p90": 42.849998474121094, "p95": 42.849998474121094, "std_dev": 0.0, "min": 42.849998474121094, "max": 42.849998474121094 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 4085, "tokens_per_second": 7.909343242645264, "tpot_ms": 126.44444444444444, "total_time_ms": 14327, "total_tokens": 82, "peak_memory_mb": 5106.21875, "mean_cpu_percent": 5.8516474, "peak_cpu_percent": 37.5, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 4094, "tokens_per_second": 7.834961414337158, "tpot_ms": 127.64197530864197, "total_time_ms": 14433, "total_tokens": 82, "peak_memory_mb": 5106.3203125, "mean_cpu_percent": 5.114666, "peak_cpu_percent": 50.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 4085, "tokens_per_second": 7.854635238647461, "tpot_ms": 127.32098765432099, "total_time_ms": 14398, "total_tokens": 82, "peak_memory_mb": 5106.33203125, "mean_cpu_percent": 4.7889605, "peak_cpu_percent": 37.5, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 4088.0, "median": 4085.0, "p90": 4092.2, "p95": 4093.1, "std_dev": 4.242640687119285, "min": 4085.0, "max": 4094.0 }, "tokens_per_second": { "mean": 7.866313298543294, "median": 7.854635238647461, "p90": 7.898401641845703, "p95": 7.903872442245484, "std_dev": 0.03146899964316336, "min": 7.834961414337158, "max": 7.909343242645264 }, "tpot": { "mean": 127.1358024691358, "median": 127.32098765432099, "p90": 127.57777777777778, "p95": 127.60987654320988, "std_dev": 0.506122651330996, "min": 126.44444444444444, "max": 127.64197530864197 }, "total_time": { "mean": 14386.0, "median": 14398.0, "p90": 14426.0, "p95": 14429.5, "std_dev": 44.098374875574116, "min": 14327.0, "max": 14433.0 }, "peak_memory_mb": { "mean": 5106.290364583333, "median": 5106.3203125, "p90": 5106.3296875, "p95": 5106.330859375, "std_dev": 0.05086464839745992, "min": 5106.21875, "max": 5106.33203125 }, "mean_cpu_percent": { "mean": 5.251757939656575, "median": 5.114665985107422, "p90": 5.704251098632813, "p95": 5.777949237823487, "std_dev": 0.44453835648124024, "min": 4.7889604568481445, "max": 5.85164737701416 }, "peak_cpu_percent": { "mean": 41.666666666666664, "median": 37.5, "p90": 47.5, "p95": 48.75, "std_dev": 5.892556509887896, "min": 37.5, "max": 50.0 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 4066, "tokens_per_second": 7.858551025390625, "tpot_ms": 127.25984251968504, "total_time_ms": 20228, "total_tokens": 128, "peak_memory_mb": 5106.79296875, "mean_cpu_percent": 5.9333982, "peak_cpu_percent": 57.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4085, "tokens_per_second": 7.884751319885254, "tpot_ms": 126.83464566929133, "total_time_ms": 20193, "total_tokens": 128, "peak_memory_mb": 5106.796875, "mean_cpu_percent": 5.6625247, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4098, "tokens_per_second": 7.92183780670166, "tpot_ms": 126.24409448818898, "total_time_ms": 20131, "total_tokens": 128, "peak_memory_mb": 5106.81640625, "mean_cpu_percent": 6.070658, "peak_cpu_percent": 28.57, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4083.0, "median": 4085.0, "p90": 4095.3999999999996, "p95": 4096.7, "std_dev": 13.140268896284683, "min": 4066.0, "max": 4098.0 }, "tokens_per_second": { "mean": 7.88838005065918, "median": 7.884751319885254, "p90": 7.914420509338379, "p95": 7.91812915802002, "std_dev": 0.025963820179240756, "min": 7.858551025390625, "max": 7.92183780670166 }, "tpot": { "mean": 126.77952755905511, "median": 126.83464566929133, "p90": 127.1748031496063, "p95": 127.21732283464567, "std_dev": 0.41650491828151626, "min": 126.24409448818898, "max": 127.25984251968504 }, "total_time": { "mean": 20184.0, "median": 20193.0, "p90": 20221.0, "p95": 20224.5, "std_dev": 40.10818702792071, "min": 20131.0, "max": 20228.0 }, "peak_memory_mb": { "mean": 5106.802083333333, "median": 5106.796875, "p90": 5106.8125, "p95": 5106.814453125, "std_dev": 0.010252614419286212, "min": 5106.79296875, "max": 5106.81640625 }, "mean_cpu_percent": { "mean": 5.88886038462321, "median": 5.933398246765137, "p90": 6.043206214904785, "p95": 6.056932210922241, "std_dev": 0.16956995746343118, "min": 5.662524700164795, "max": 6.070658206939697 }, "peak_cpu_percent": { "mean": 42.85333251953125, "median": 42.849998474121094, "p90": 54.28199920654297, "p95": 55.7109992980957, "std_dev": 11.663653772223581, "min": 28.56999969482422, "max": 57.13999938964844 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 4088, "tokens_per_second": 7.893937587738037, "tpot_ms": 126.68235294117648, "total_time_ms": 36392, "total_tokens": 256, "peak_memory_mb": 5107.63671875, "mean_cpu_percent": 5.768466, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4093, "tokens_per_second": 7.85499906539917, "tpot_ms": 127.30980392156863, "total_time_ms": 36557, "total_tokens": 256, "peak_memory_mb": 5107.75, "mean_cpu_percent": 5.8766913, "peak_cpu_percent": 28.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4104, "tokens_per_second": 7.839405536651611, "tpot_ms": 127.56470588235294, "total_time_ms": 36633, "total_tokens": 256, "peak_memory_mb": 5107.7734375, "mean_cpu_percent": 6.165866, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4095.0, "median": 4093.0, "p90": 4101.8, "p95": 4102.9, "std_dev": 6.683312551921141, "min": 4088.0, "max": 4104.0 }, "tokens_per_second": { "mean": 7.862780729929606, "median": 7.85499906539917, "p90": 7.886149883270264, "p95": 7.89004373550415, "std_dev": 0.022932536944573232, "min": 7.839405536651611, "max": 7.893937587738037 }, "tpot": { "mean": 127.18562091503269, "median": 127.30980392156863, "p90": 127.51372549019608, "p95": 127.53921568627452, "std_dev": 0.3707674406191565, "min": 126.68235294117648, "max": 127.56470588235294 }, "total_time": { "mean": 36527.333333333336, "median": 36557.0, "p90": 36617.8, "p95": 36625.4, "std_dev": 100.59931521746171, "min": 36392.0, "max": 36633.0 }, "peak_memory_mb": { "mean": 5107.720052083333, "median": 5107.75, "p90": 5107.768749999999, "p95": 5107.771093750001, "std_dev": 0.059697361387227345, "min": 5107.63671875, "max": 5107.7734375 }, "mean_cpu_percent": { "mean": 5.937007745107015, "median": 5.8766913414001465, "p90": 6.108030986785889, "p95": 6.136948442459107, "std_dev": 0.16775025486876408, "min": 5.768465995788574, "max": 6.165865898132324 }, "peak_cpu_percent": { "mean": 40.47333272298177, "median": 42.849998474121094, "p90": 48.56999969482422, "p95": 49.28499984741211, "std_dev": 8.908708598498345, "min": 28.56999969482422, "max": 50.0 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 4098, "tokens_per_second": 7.817585945129395, "tpot_ms": 127.91764705882353, "total_time_ms": 36717, "total_tokens": 256, "peak_memory_mb": 5108.33203125, "mean_cpu_percent": 6.407949, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4085, "tokens_per_second": 7.894480228424072, "tpot_ms": 126.67058823529412, "total_time_ms": 36386, "total_tokens": 256, "peak_memory_mb": 5108.34765625, "mean_cpu_percent": 6.882084, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4082, "tokens_per_second": 8.018741607666016, "tpot_ms": 124.70980392156862, "total_time_ms": 35883, "total_tokens": 256, "peak_memory_mb": 5108.390625, "mean_cpu_percent": 5.7005916, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4088.3333333333335, "median": 4085.0, "p90": 4095.3999999999996, "p95": 4096.7, "std_dev": 6.944222218666553, "min": 4082.0, "max": 4098.0 }, "tokens_per_second": { "mean": 7.910269260406494, "median": 7.894480228424072, "p90": 7.993889331817627, "p95": 8.006315469741821, "std_dev": 0.08287689782982598, "min": 7.817585945129395, "max": 8.018741607666016 }, "tpot": { "mean": 126.4326797385621, "median": 126.67058823529412, "p90": 127.66823529411765, "p95": 127.7929411764706, "std_dev": 1.320357208718239, "min": 124.70980392156862, "max": 127.91764705882353 }, "total_time": { "mean": 36328.666666666664, "median": 36386.0, "p90": 36650.8, "p95": 36683.9, "std_dev": 342.8841722149851, "min": 35883.0, "max": 36717.0 }, "peak_memory_mb": { "mean": 5108.356770833333, "median": 5108.34765625, "p90": 5108.38203125, "p95": 5108.386328125001, "std_dev": 0.024773824987552668, "min": 5108.33203125, "max": 5108.390625 }, "mean_cpu_percent": { "mean": 6.33020814259847, "median": 6.407948970794678, "p90": 6.787256908416748, "p95": 6.8346704006195065, "std_dev": 0.48546455959730694, "min": 5.700591564178467, "max": 6.882083892822266 }, "peak_cpu_percent": { "mean": 40.46999867757162, "median": 42.849998474121094, "p90": 42.849998474121094, "p95": 42.849998474121094, "std_dev": 3.3658279907254802, "min": 35.709999084472656, "max": 42.849998474121094 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 4074, "tokens_per_second": 7.892005920410156, "tpot_ms": 126.71372549019608, "total_time_ms": 36386, "total_tokens": 256, "peak_memory_mb": 5108.859375, "mean_cpu_percent": 6.4101505, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4101, "tokens_per_second": 7.903649806976318, "tpot_ms": 126.52549019607844, "total_time_ms": 36365, "total_tokens": 256, "peak_memory_mb": 5108.89453125, "mean_cpu_percent": 5.718293, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4107, "tokens_per_second": 7.865466117858887, "tpot_ms": 127.13725490196079, "total_time_ms": 36527, "total_tokens": 256, "peak_memory_mb": 5108.92578125, "mean_cpu_percent": 5.6475577, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4094.0, "median": 4101.0, "p90": 4105.8, "p95": 4106.4, "std_dev": 14.352700094407323, "min": 4074.0, "max": 4107.0 }, "tokens_per_second": { "mean": 7.887040615081787, "median": 7.892005920410156, "p90": 7.901321029663086, "p95": 7.902485418319702, "std_dev": 0.015978928237048422, "min": 7.865466117858887, "max": 7.903649806976318 }, "tpot": { "mean": 126.7921568627451, "median": 126.71372549019608, "p90": 127.05254901960785, "p95": 127.09490196078433, "std_dev": 0.25583539498509855, "min": 126.52549019607844, "max": 127.13725490196079 }, "total_time": { "mean": 36426.0, "median": 36386.0, "p90": 36498.8, "p95": 36512.9, "std_dev": 71.93052203341777, "min": 36365.0, "max": 36527.0 }, "peak_memory_mb": { "mean": 5108.893229166667, "median": 5108.89453125, "p90": 5108.91953125, "p95": 5108.922656250001, "std_dev": 0.027125868041666223, "min": 5108.859375, "max": 5108.92578125 }, "mean_cpu_percent": { "mean": 5.925333817799886, "median": 5.718293190002441, "p90": 6.2717790603637695, "p95": 6.3409647941589355, "std_dev": 0.3440313047446856, "min": 5.647557735443115, "max": 6.410150527954102 }, "peak_cpu_percent": { "mean": 45.23333231608073, "median": 42.849998474121094, "p90": 48.56999969482422, "p95": 49.28499984741211, "std_dev": 3.3705430429620913, "min": 42.849998474121094, "max": 50.0 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 4086, "tokens_per_second": 7.88204288482666, "tpot_ms": 126.87450980392157, "total_time_ms": 36439, "total_tokens": 256, "peak_memory_mb": 5109.25390625, "mean_cpu_percent": 5.1761985, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4089, "tokens_per_second": 7.786911487579346, "tpot_ms": 128.42352941176472, "total_time_ms": 36837, "total_tokens": 256, "peak_memory_mb": 5109.26171875, "mean_cpu_percent": 6.76619, "peak_cpu_percent": 35.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4095, "tokens_per_second": 7.881064414978027, "tpot_ms": 126.89019607843137, "total_time_ms": 36452, "total_tokens": 256, "peak_memory_mb": 5093.265625, "mean_cpu_percent": 5.4283485, "peak_cpu_percent": 64.28, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4090.0, "median": 4089.0, "p90": 4093.7999999999997, "p95": 4094.4, "std_dev": 3.7416573867739413, "min": 4086.0, "max": 4095.0 }, "tokens_per_second": { "mean": 7.850006262461345, "median": 7.881064414978027, "p90": 7.881847190856934, "p95": 7.881945037841797, "std_dev": 0.044616531419558986, "min": 7.786911487579346, "max": 7.88204288482666 }, "tpot": { "mean": 127.39607843137254, "median": 126.89019607843137, "p90": 128.11686274509805, "p95": 128.27019607843138, "std_dev": 0.7265457786144036, "min": 126.87450980392157, "max": 128.42352941176472 }, "total_time": { "mean": 36576.0, "median": 36452.0, "p90": 36760.0, "p95": 36798.5, "std_dev": 184.63116385558172, "min": 36439.0, "max": 36837.0 }, "peak_memory_mb": { "mean": 5103.927083333333, "median": 5109.25390625, "p90": 5109.26015625, "p95": 5109.2609375, "std_dev": 7.5387901595168065, "min": 5093.265625, "max": 5109.26171875 }, "mean_cpu_percent": { "mean": 5.790245691935222, "median": 5.428348541259766, "p90": 6.49862174987793, "p95": 6.6324059009552006, "std_dev": 0.6977322593161686, "min": 5.176198482513428, "max": 6.766190052032471 }, "peak_cpu_percent": { "mean": 45.23333231608073, "median": 35.709999084472656, "p90": 58.565998840332036, "p95": 61.42299880981445, "std_dev": 13.468027015138532, "min": 35.709999084472656, "max": 64.27999877929688 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 4095, "tokens_per_second": 7.859143733978272, "tpot_ms": 127.24070450097847, "total_time_ms": 69115, "total_tokens": 512, "peak_memory_mb": 5094.3984375, "mean_cpu_percent": 6.256606, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4079, "tokens_per_second": 7.905935287475586, "tpot_ms": 126.48923679060665, "total_time_ms": 68715, "total_tokens": 512, "peak_memory_mb": 5094.43359375, "mean_cpu_percent": 6.687958, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4088, "tokens_per_second": 7.848510265350342, "tpot_ms": 127.41487279843444, "total_time_ms": 69197, "total_tokens": 512, "peak_memory_mb": 5094.44140625, "mean_cpu_percent": 6.412593, "peak_cpu_percent": 85.71, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4087.3333333333335, "median": 4088.0, "p90": 4093.6, "p95": 4094.2999999999997, "std_dev": 6.548960901462833, "min": 4079.0, "max": 4095.0 }, "tokens_per_second": { "mean": 7.871196428934733, "median": 7.859143733978272, "p90": 7.896576976776123, "p95": 7.901256132125855, "std_dev": 0.024944724002031907, "min": 7.848510265350342, "max": 7.905935287475586 }, "tpot": { "mean": 127.04827136333985, "median": 127.24070450097847, "p90": 127.38003913894326, "p95": 127.39745596868885, "std_dev": 0.401641124299561, "min": 126.48923679060665, "max": 127.41487279843444 }, "total_time": { "mean": 69009.0, "median": 69115.0, "p90": 69180.6, "p95": 69188.8, "std_dev": 210.56748720224275, "min": 68715.0, "max": 69197.0 }, "peak_memory_mb": { "mean": 5094.424479166667, "median": 5094.43359375, "p90": 5094.43984375, "p95": 5094.440624999999, "std_dev": 0.018688411581259536, "min": 5094.3984375, "max": 5094.44140625 }, "mean_cpu_percent": { "mean": 6.452385584513347, "median": 6.412592887878418, "p90": 6.632884788513184, "p95": 6.66042127609253, "std_dev": 0.17833238262318488, "min": 6.256606101989746, "max": 6.687957763671875 }, "peak_cpu_percent": { "mean": 59.519999186197914, "median": 50.0, "p90": 78.56799926757813, "p95": 82.13899917602538, "std_dev": 18.747759003122493, "min": 42.849998474121094, "max": 85.70999908447266 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 4120, "tokens_per_second": 7.873672962188721, "tpot_ms": 127.00587084148728, "total_time_ms": 69020, "total_tokens": 512, "peak_memory_mb": 4357.52734375, "mean_cpu_percent": 6.9210033, "peak_cpu_percent": 42.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4109, "tokens_per_second": 7.882505416870117, "tpot_ms": 126.86497064579257, "total_time_ms": 68937, "total_tokens": 512, "peak_memory_mb": 4357.6015625, "mean_cpu_percent": 5.7166414, "peak_cpu_percent": 57.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 4108, "tokens_per_second": 7.895064353942871, "tpot_ms": 126.66340508806262, "total_time_ms": 68833, "total_tokens": 512, "peak_memory_mb": 4357.65625, "mean_cpu_percent": 5.26183, "peak_cpu_percent": 50.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 4112.333333333333, "median": 4109.0, "p90": 4117.8, "p95": 4118.9, "std_dev": 5.436502143433363, "min": 4108.0, "max": 4120.0 }, "tokens_per_second": { "mean": 7.883747577667236, "median": 7.882505416870117, "p90": 7.89255256652832, "p95": 7.893808460235595, "std_dev": 0.008777058462427606, "min": 7.873672962188721, "max": 7.895064353942871 }, "tpot": { "mean": 126.84474885844749, "median": 126.86497064579257, "p90": 126.97769080234835, "p95": 126.99178082191781, "std_dev": 0.1405403585682835, "min": 126.66340508806262, "max": 127.00587084148728 }, "total_time": { "mean": 68930.0, "median": 68937.0, "p90": 69003.4, "p95": 69011.7, "std_dev": 76.5027232630752, "min": 68833.0, "max": 69020.0 }, "peak_memory_mb": { "mean": 4357.595052083333, "median": 4357.6015625, "p90": 4357.6453125, "p95": 4357.65078125, "std_dev": 0.05282672597682499, "min": 4357.52734375, "max": 4357.65625 }, "mean_cpu_percent": { "mean": 5.96649154027303, "median": 5.716641426086426, "p90": 6.680130958557129, "p95": 6.800567150115967, "std_dev": 0.7000157026817918, "min": 5.261829853057861, "max": 6.921003341674805 }, "peak_cpu_percent": { "mean": 49.996665954589844, "median": 50.0, "p90": 55.71199951171875, "p95": 56.4259994506836, "std_dev": 5.833868920841307, "min": 42.849998474121094, "max": 57.13999938964844 } } } ] } ], "skipped": [] }