{ "schema_version": "2.0.0", "generated_at": "2026-03-27T07:33:58.586359900+00:00", "tool_version": "0.1.0", "machine": { "label": "codex-cpu-full-20260327", "os": "windows x86_64", "hostname": "DESKTOP-EU6FFOD" }, "hardware": { "cpu_brand": "12th Gen Intel(R) Core(TM) i7-1255U", "cpu_cores_physical": 10, "cpu_cores_logical": 12, "gpus": [ { "name": "Intel(R) Iris(R) Xe Graphics", "vendor": "Intel", "vram_mb": 2047 } ], "npu": "Intel Gaussian Neural Accelerator | 1.0 TOPS", "memory_total_gb": 31.7001953125 }, "test_config": { "backends": [ "cpu" ], "models": [ "qwen2.5-1.5b-instruct", "qwen3-4b" ], "runs_per_prompt": 10, "warmup_runs": 2, "temperature": 0.0, "repetition_penalty": 1.0, "prompt_corpus_hash": "1672e3a7311c9f6f" }, "results": [ { "backend": "cpu", "model_id": "qwen2.5-1.5b-instruct", "cold_start_ttft_ms": 589, "idle_baseline": { "rss_mb": 1927.85546875, "cpu_percent": 0.0 }, "reliability": { "total_runs": 100, "successful_runs": 100, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.3, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 286, "tokens_per_second": 19.858352661132812, "tpot_ms": 50.37007874015748, "total_time_ms": 6683, "total_tokens": 128, "peak_memory_mb": 1943.22265625, "mean_cpu_percent": 419.39056, "peak_cpu_percent": 525.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 142, "tokens_per_second": 20.08650016784668, "tpot_ms": 49.79527559055118, "total_time_ms": 6466, "total_tokens": 128, "peak_memory_mb": 1943.5859375, "mean_cpu_percent": 435.63602, "peak_cpu_percent": 564.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 174, "tokens_per_second": 16.970678329467773, "tpot_ms": 58.93700787401575, "total_time_ms": 7659, "total_tokens": 128, "peak_memory_mb": 1943.62109375, "mean_cpu_percent": 243.90126, "peak_cpu_percent": 342.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 245, "tokens_per_second": 17.201261520385742, "tpot_ms": 58.1496062992126, "total_time_ms": 7630, "total_tokens": 128, "peak_memory_mb": 1944.01953125, "mean_cpu_percent": 270.76996, "peak_cpu_percent": 464.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 164, "tokens_per_second": 16.368793487548828, "tpot_ms": 61.110236220472444, "total_time_ms": 7925, "total_tokens": 128, "peak_memory_mb": 1944.25390625, "mean_cpu_percent": 238.07489, "peak_cpu_percent": 385.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 149, "tokens_per_second": 17.674068450927734, "tpot_ms": 56.59055118110236, "total_time_ms": 7336, "total_tokens": 128, "peak_memory_mb": 1944.2578125, "mean_cpu_percent": 282.21295, "peak_cpu_percent": 386.2, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 217, "tokens_per_second": 15.878557205200195, "tpot_ms": 62.99212598425197, "total_time_ms": 8217, "total_tokens": 128, "peak_memory_mb": 1944.390625, "mean_cpu_percent": 218.65863, "peak_cpu_percent": 544.82, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 189, "tokens_per_second": 15.235671997070312, "tpot_ms": 65.64566929133858, "total_time_ms": 8526, "total_tokens": 128, "peak_memory_mb": 1944.63671875, "mean_cpu_percent": 166.95972, "peak_cpu_percent": 291.32, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 179, "tokens_per_second": 14.951650619506836, "tpot_ms": 66.89763779527559, "total_time_ms": 8675, "total_tokens": 128, "peak_memory_mb": 1944.6484375, "mean_cpu_percent": 171.4517, "peak_cpu_percent": 278.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 252, "tokens_per_second": 14.495224952697754, "tpot_ms": 69.0, "total_time_ms": 9015, "total_tokens": 128, "peak_memory_mb": 1945.3984375, "mean_cpu_percent": 154.86597, "peak_cpu_percent": 275.77, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 199.7, "median": 184.0, "p90": 255.4, "p95": 270.69999999999993, "std_dev": 45.72974961663359, "min": 142.0, "max": 286.0 }, "tokens_per_second": { "mean": 16.872075939178465, "median": 16.6697359085083, "p90": 19.8811674118042, "p95": 19.98383378982544, "std_dev": 1.8239323982013291, "min": 14.495224952697754, "max": 20.08650016784668 }, "tpot": { "mean": 59.94881889763779, "median": 60.023622047244096, "p90": 67.10787401574802, "p95": 68.05393700787401, "std_dev": 6.194031565309791, "min": 49.79527559055118, "max": 69.0 }, "total_time": { "mean": 7813.2, "median": 7792.0, "p90": 8709.0, "p95": 8862.0, "std_dev": 790.4441536250363, "min": 6466.0, "max": 9015.0 }, "peak_memory_mb": { "mean": 1944.203515625, "median": 1944.255859375, "p90": 1944.7234375, "p95": 1945.0609375, "std_dev": 0.5975373911630055, "min": 1943.22265625, "max": 1945.3984375 }, "mean_cpu_percent": { "mean": 260.1921661376953, "median": 240.98807525634766, "p90": 421.01510925292973, "p95": 428.32556304931643, "std_dev": 93.19970416518736, "min": 154.865966796875, "max": 435.6360168457031 }, "peak_cpu_percent": { "mean": 405.8800048828125, "median": 385.9550018310547, "p90": 546.7660095214844, "p95": 555.5230194091796, "std_dev": 106.311985017963, "min": 275.7699890136719, "max": 564.280029296875 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 639, "tokens_per_second": 15.06468391418457, "tpot_ms": 66.39370078740157, "total_time_ms": 9071, "total_tokens": 128, "peak_memory_mb": 1949.01171875, "mean_cpu_percent": 157.2938, "peak_cpu_percent": 373.65, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 183, "tokens_per_second": 15.502601623535156, "tpot_ms": 64.51181102362204, "total_time_ms": 8376, "total_tokens": 128, "peak_memory_mb": 1950.23828125, "mean_cpu_percent": 172.29951, "peak_cpu_percent": 305.2, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 226, "tokens_per_second": 14.888517379760742, "tpot_ms": 67.18897637795276, "total_time_ms": 8759, "total_tokens": 128, "peak_memory_mb": 1952.4921875, "mean_cpu_percent": 183.31506, "peak_cpu_percent": 331.03, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 201, "tokens_per_second": 18.64931297302246, "tpot_ms": 53.62992125984252, "total_time_ms": 7012, "total_tokens": 128, "peak_memory_mb": 1952.4921875, "mean_cpu_percent": 327.484, "peak_cpu_percent": 607.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 295, "tokens_per_second": 14.814168930053713, "tpot_ms": 67.51968503937007, "total_time_ms": 8870, "total_tokens": 128, "peak_memory_mb": 1952.5, "mean_cpu_percent": 165.8908, "peak_cpu_percent": 392.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 202, "tokens_per_second": 14.743568420410156, "tpot_ms": 67.83464566929133, "total_time_ms": 8817, "total_tokens": 128, "peak_memory_mb": 1952.5390625, "mean_cpu_percent": 153.50444, "peak_cpu_percent": 228.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 212, "tokens_per_second": 14.3637056350708, "tpot_ms": 69.62992125984252, "total_time_ms": 9055, "total_tokens": 128, "peak_memory_mb": 1952.625, "mean_cpu_percent": 152.2087, "peak_cpu_percent": 202.32, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 171, "tokens_per_second": 15.343746185302734, "tpot_ms": 65.18110236220473, "total_time_ms": 8449, "total_tokens": 128, "peak_memory_mb": 1952.7890625, "mean_cpu_percent": 178.6254, "peak_cpu_percent": 300.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 219, "tokens_per_second": 17.491167068481445, "tpot_ms": 57.188976377952756, "total_time_ms": 7482, "total_tokens": 128, "peak_memory_mb": 1952.7890625, "mean_cpu_percent": 319.5517, "peak_cpu_percent": 478.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 180, "tokens_per_second": 15.54477596282959, "tpot_ms": 64.34645669291339, "total_time_ms": 8352, "total_tokens": 128, "peak_memory_mb": 1952.79296875, "mean_cpu_percent": 197.9535, "peak_cpu_percent": 350.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 252.8, "median": 207.0, "p90": 329.39999999999986, "p95": 484.1999999999996, "std_dev": 132.8847621061196, "min": 171.0, "max": 639.0 }, "tokens_per_second": { "mean": 15.640624809265137, "median": 15.204215049743652, "p90": 17.606981658935545, "p95": 18.128147315979003, "std_dev": 1.2887482687104737, "min": 14.3637056350708, "max": 18.64931297302246 }, "tpot": { "mean": 64.34251968503936, "median": 65.78740157480314, "p90": 68.01417322834645, "p95": 68.82204724409448, "std_dev": 4.7876971650354285, "min": 53.62992125984252, "max": 69.62992125984252 }, "total_time": { "mean": 8424.3, "median": 8604.0, "p90": 9056.6, "p95": 9063.8, "std_dev": 645.8777051423899, "min": 7012.0, "max": 9071.0 }, "peak_memory_mb": { "mean": 1952.026953125, "median": 1952.51953125, "p90": 1952.789453125, "p95": 1952.7912109375002, "std_dev": 1.2375767539525895, "min": 1949.01171875, "max": 1952.79296875 }, "mean_cpu_percent": { "mean": 200.81269073486328, "median": 175.46245574951172, "p90": 320.3449279785156, "p95": 323.9144683837891, "std_dev": 62.831557306406445, "min": 152.2086944580078, "max": 327.4840087890625 }, "peak_cpu_percent": { "mean": 356.93300476074216, "median": 340.51499938964844, "p90": 491.4270080566406, "p95": 549.2835113525389, "std_dev": 112.24416992054482, "min": 202.32000732421875, "max": 607.1400146484375 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 543, "tokens_per_second": 14.701067924499512, "tpot_ms": 68.04347826086956, "total_time_ms": 8368, "total_tokens": 116, "peak_memory_mb": 1955.29296875, "mean_cpu_percent": 174.12523, "peak_cpu_percent": 368.09, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 336, "tokens_per_second": 18.57219886779785, "tpot_ms": 53.85217391304348, "total_time_ms": 6529, "total_tokens": 116, "peak_memory_mb": 1956.19921875, "mean_cpu_percent": 339.51013, "peak_cpu_percent": 450.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 206, "tokens_per_second": 14.823691368103027, "tpot_ms": 67.47826086956522, "total_time_ms": 7966, "total_tokens": 116, "peak_memory_mb": 1956.23828125, "mean_cpu_percent": 169.37782, "peak_cpu_percent": 264.28, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 271, "tokens_per_second": 15.509206771850586, "tpot_ms": 64.49565217391304, "total_time_ms": 7688, "total_tokens": 116, "peak_memory_mb": 1956.37109375, "mean_cpu_percent": 179.15019, "peak_cpu_percent": 281.92, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 223, "tokens_per_second": 15.495304107666016, "tpot_ms": 64.54782608695652, "total_time_ms": 7646, "total_tokens": 116, "peak_memory_mb": 1956.6796875, "mean_cpu_percent": 181.28712, "peak_cpu_percent": 371.42, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 340, "tokens_per_second": 15.353816032409668, "tpot_ms": 65.14782608695653, "total_time_ms": 7832, "total_tokens": 116, "peak_memory_mb": 1956.75390625, "mean_cpu_percent": 197.41307, "peak_cpu_percent": 621.42, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 166, "tokens_per_second": 13.046996116638184, "tpot_ms": 76.66956521739131, "total_time_ms": 8983, "total_tokens": 116, "peak_memory_mb": 1956.8359375, "mean_cpu_percent": 337.86224, "peak_cpu_percent": 564.28, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 429, "tokens_per_second": 14.071062088012695, "tpot_ms": 71.09565217391304, "total_time_ms": 8605, "total_tokens": 116, "peak_memory_mb": 1956.8359375, "mean_cpu_percent": 372.5542, "peak_cpu_percent": 528.57, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 265, "tokens_per_second": 12.751531600952148, "tpot_ms": 78.44347826086957, "total_time_ms": 9286, "total_tokens": 116, "peak_memory_mb": 1956.87109375, "mean_cpu_percent": 198.81929, "peak_cpu_percent": 332.94, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 315, "tokens_per_second": 16.13544464111328, "tpot_ms": 62.0, "total_time_ms": 7445, "total_tokens": 116, "peak_memory_mb": 1956.921875, "mean_cpu_percent": 542.04736, "peak_cpu_percent": 685.71, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 309.4, "median": 293.0, "p90": 440.3999999999999, "p95": 491.6999999999999, "std_dev": 106.16703819924524, "min": 166.0, "max": 543.0 }, "tokens_per_second": { "mean": 15.046031951904297, "median": 15.088753700256348, "p90": 16.379120063781738, "p95": 17.475659465789793, "std_dev": 1.5656199549137797, "min": 12.751531600952148, "max": 18.57219886779785 }, "tpot": { "mean": 67.17739130434782, "median": 66.31304347826088, "p90": 76.84695652173913, "p95": 77.64521739130436, "std_dev": 6.751383195113353, "min": 53.85217391304348, "max": 78.44347826086957 }, "total_time": { "mean": 8034.8, "median": 7899.0, "p90": 9013.3, "p95": 9149.65, "std_dev": 763.7623714218971, "min": 6529.0, "max": 9286.0 }, "peak_memory_mb": { "mean": 1956.5, "median": 1956.716796875, "p90": 1956.876171875, "p95": 1956.8990234375, "std_dev": 0.476514469915723, "min": 1955.29296875, "max": 1956.921875 }, "mean_cpu_percent": { "mean": 269.21466674804685, "median": 198.11618041992188, "p90": 389.5035156249999, "p95": 465.7754394531248, "std_dev": 118.20910225253012, "min": 169.37782287597656, "max": 542.04736328125 }, "peak_cpu_percent": { "mean": 446.8630065917969, "median": 410.7100067138672, "p90": 627.8489868164063, "p95": 656.7795043945312, "std_dev": 139.07145488109802, "min": 264.2799987792969, "max": 685.7100219726562 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 764, "tokens_per_second": 12.242721557617188, "tpot_ms": 81.6938775510204, "total_time_ms": 8770, "total_tokens": 99, "peak_memory_mb": 1959.0703125, "mean_cpu_percent": 192.01486, "peak_cpu_percent": 379.31, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 267, "tokens_per_second": 12.588468551635742, "tpot_ms": 79.45918367346938, "total_time_ms": 8054, "total_tokens": 99, "peak_memory_mb": 1960.26953125, "mean_cpu_percent": 206.52126, "peak_cpu_percent": 436.99, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 382, "tokens_per_second": 13.617687225341797, "tpot_ms": 73.44897959183673, "total_time_ms": 7580, "total_tokens": 99, "peak_memory_mb": 1960.453125, "mean_cpu_percent": 241.8681, "peak_cpu_percent": 545.45, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 346, "tokens_per_second": 13.179465293884276, "tpot_ms": 75.89795918367346, "total_time_ms": 7784, "total_tokens": 99, "peak_memory_mb": 1960.65234375, "mean_cpu_percent": 225.33632, "peak_cpu_percent": 292.85, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 200, "tokens_per_second": 13.954472541809082, "tpot_ms": 71.68367346938776, "total_time_ms": 7225, "total_tokens": 99, "peak_memory_mb": 1960.71484375, "mean_cpu_percent": 326.33746, "peak_cpu_percent": 490.9, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 294, "tokens_per_second": 15.526358604431152, "tpot_ms": 64.43877551020408, "total_time_ms": 6609, "total_tokens": 99, "peak_memory_mb": 1960.8203125, "mean_cpu_percent": 454.19785, "peak_cpu_percent": 860.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 202, "tokens_per_second": 14.290830612182615, "tpot_ms": 70.0, "total_time_ms": 7062, "total_tokens": 99, "peak_memory_mb": 1960.859375, "mean_cpu_percent": 340.62396, "peak_cpu_percent": 585.71, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 207, "tokens_per_second": 13.746386528015137, "tpot_ms": 72.76530612244898, "total_time_ms": 7338, "total_tokens": 99, "peak_memory_mb": 1960.859375, "mean_cpu_percent": 299.40088, "peak_cpu_percent": 589.09, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 463, "tokens_per_second": 15.541133880615234, "tpot_ms": 64.37755102040816, "total_time_ms": 6772, "total_tokens": 99, "peak_memory_mb": 1960.875, "mean_cpu_percent": 445.44992, "peak_cpu_percent": 621.42, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 328, "tokens_per_second": 13.778046607971191, "tpot_ms": 72.60204081632654, "total_time_ms": 7443, "total_tokens": 99, "peak_memory_mb": 1960.890625, "mean_cpu_percent": 283.78815, "peak_cpu_percent": 592.85, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 345.3, "median": 311.0, "p90": 493.0999999999999, "p95": 628.5499999999997, "std_dev": 161.3524403286173, "min": 200.0, "max": 764.0 }, "tokens_per_second": { "mean": 13.846557140350342, "median": 13.762216567993164, "p90": 15.527836132049561, "p95": 15.534485006332398, "std_dev": 1.028082887064833, "min": 12.242721557617188, "max": 15.541133880615234 }, "tpot": { "mean": 72.63673469387754, "median": 72.68367346938776, "p90": 79.68265306122449, "p95": 80.68826530612245, "std_dev": 5.321391296560706, "min": 64.37755102040816, "max": 81.6938775510204 }, "total_time": { "mean": 7463.7, "median": 7390.5, "p90": 8125.6, "p95": 8447.8, "std_dev": 601.208957019105, "min": 6609.0, "max": 8770.0 }, "peak_memory_mb": { "mean": 1960.546484375, "median": 1960.767578125, "p90": 1960.8765624999999, "p95": 1960.8835937499998, "std_dev": 0.5292628563791137, "min": 1959.0703125, "max": 1960.890625 }, "mean_cpu_percent": { "mean": 301.5538757324219, "median": 291.5945129394531, "p90": 446.3247131347656, "p95": 450.261279296875, "std_dev": 87.67480224602186, "min": 192.01486206054688, "max": 454.1978454589844 }, "peak_cpu_percent": { "mean": 539.4570007324219, "median": 565.5800170898438, "p90": 645.2779846191405, "p95": 752.6389923095701, "std_dev": 147.06277353969543, "min": 292.8500061035156, "max": 860.0 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 1078, "tokens_per_second": 13.995003700256348, "tpot_ms": 71.46274509803922, "total_time_ms": 19301, "total_tokens": 256, "peak_memory_mb": 1964.84375, "mean_cpu_percent": 289.61554, "peak_cpu_percent": 485.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 426, "tokens_per_second": 14.05238151550293, "tpot_ms": 71.17254901960784, "total_time_ms": 18575, "total_tokens": 256, "peak_memory_mb": 1966.5078125, "mean_cpu_percent": 351.0138, "peak_cpu_percent": 534.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 414, "tokens_per_second": 13.943196296691896, "tpot_ms": 71.73333333333333, "total_time_ms": 18706, "total_tokens": 256, "peak_memory_mb": 1966.546875, "mean_cpu_percent": 320.44824, "peak_cpu_percent": 571.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 547, "tokens_per_second": 12.72102165222168, "tpot_ms": 78.6156862745098, "total_time_ms": 20594, "total_tokens": 256, "peak_memory_mb": 1966.5, "mean_cpu_percent": 238.25157, "peak_cpu_percent": 492.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 463, "tokens_per_second": 13.71610164642334, "tpot_ms": 72.92156862745098, "total_time_ms": 19058, "total_tokens": 256, "peak_memory_mb": 1966.53125, "mean_cpu_percent": 340.89075, "peak_cpu_percent": 535.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 409, "tokens_per_second": 12.82411003112793, "tpot_ms": 77.98823529411764, "total_time_ms": 20296, "total_tokens": 256, "peak_memory_mb": 1966.6875, "mean_cpu_percent": 208.06796, "peak_cpu_percent": 528.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 405, "tokens_per_second": 13.263876914978027, "tpot_ms": 75.4, "total_time_ms": 19632, "total_tokens": 256, "peak_memory_mb": 1966.70703125, "mean_cpu_percent": 338.21994, "peak_cpu_percent": 592.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 599, "tokens_per_second": 12.744471549987791, "tpot_ms": 78.47450980392156, "total_time_ms": 20610, "total_tokens": 256, "peak_memory_mb": 1966.70703125, "mean_cpu_percent": 244.9224, "peak_cpu_percent": 501.17, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 386, "tokens_per_second": 12.84317684173584, "tpot_ms": 77.87058823529412, "total_time_ms": 20243, "total_tokens": 256, "peak_memory_mb": 1966.70703125, "mean_cpu_percent": 196.47255, "peak_cpu_percent": 350.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 674, "tokens_per_second": 14.770703315734863, "tpot_ms": 67.71372549019608, "total_time_ms": 17941, "total_tokens": 256, "peak_memory_mb": 1966.70703125, "mean_cpu_percent": 447.8125, "peak_cpu_percent": 671.18, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 540.1, "median": 444.5, "p90": 714.3999999999999, "p95": 896.1999999999996, "std_dev": 201.20459736298275, "min": 386.0, "max": 1078.0 }, "tokens_per_second": { "mean": 13.487404346466064, "median": 13.489989280700684, "p90": 14.124213695526123, "p95": 14.447458505630493, "std_dev": 0.6729045861253113, "min": 12.72102165222168, "max": 14.770703315734863 }, "tpot": { "mean": 74.33529411764707, "median": 74.16078431372549, "p90": 78.48862745098039, "p95": 78.5521568627451, "std_dev": 3.651016722297706, "min": 67.71372549019608, "max": 78.6156862745098 }, "total_time": { "mean": 19495.6, "median": 19466.5, "p90": 20595.6, "p95": 20602.8, "std_dev": 882.2323050081537, "min": 17941.0, "max": 20610.0 }, "peak_memory_mb": { "mean": 1966.44453125, "median": 1966.6171875, "p90": 1966.70703125, "p95": 1966.70703125, "std_dev": 0.5405770164760869, "min": 1964.84375, "max": 1966.70703125 }, "mean_cpu_percent": { "mean": 297.57152404785154, "median": 305.0318908691406, "p90": 360.6936645507812, "p95": 404.2530822753905, "std_dev": 73.52999091881513, "min": 196.47254943847656, "max": 447.8125 }, "peak_cpu_percent": { "mean": 526.4309967041015, "median": 531.7099914550781, "p90": 600.6829772949218, "p95": 635.9314849853515, "std_dev": 78.80432312027251, "min": 350.0, "max": 671.1799926757812 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 1047, "tokens_per_second": 12.676794052124023, "tpot_ms": 78.89411764705882, "total_time_ms": 21165, "total_tokens": 256, "peak_memory_mb": 1970.39453125, "mean_cpu_percent": 205.73549, "peak_cpu_percent": 371.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 490, "tokens_per_second": 12.967744827270508, "tpot_ms": 77.12156862745098, "total_time_ms": 20156, "total_tokens": 256, "peak_memory_mb": 1971.875, "mean_cpu_percent": 262.8953, "peak_cpu_percent": 473.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 968, "tokens_per_second": 13.700607299804688, "tpot_ms": 73.0, "total_time_ms": 19583, "total_tokens": 256, "peak_memory_mb": 1972.140625, "mean_cpu_percent": 288.78378, "peak_cpu_percent": 457.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 440, "tokens_per_second": 12.1367769241333, "tpot_ms": 82.40392156862745, "total_time_ms": 21453, "total_tokens": 256, "peak_memory_mb": 1972.23046875, "mean_cpu_percent": 201.49673, "peak_cpu_percent": 545.45, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 695, "tokens_per_second": 13.193320274353027, "tpot_ms": 75.80392156862744, "total_time_ms": 20025, "total_tokens": 256, "peak_memory_mb": 1972.23046875, "mean_cpu_percent": 246.67819, "peak_cpu_percent": 459.88, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 493, "tokens_per_second": 13.816840171813965, "tpot_ms": 72.3843137254902, "total_time_ms": 18951, "total_tokens": 256, "peak_memory_mb": 1972.234375, "mean_cpu_percent": 333.51566, "peak_cpu_percent": 493.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 599, "tokens_per_second": 12.713808059692385, "tpot_ms": 78.66666666666667, "total_time_ms": 20659, "total_tokens": 256, "peak_memory_mb": 1972.234375, "mean_cpu_percent": 240.4229, "peak_cpu_percent": 569.23, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 505, "tokens_per_second": 12.9157133102417, "tpot_ms": 77.43529411764706, "total_time_ms": 20251, "total_tokens": 256, "peak_memory_mb": 1972.203125, "mean_cpu_percent": 277.31342, "peak_cpu_percent": 489.94, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 748, "tokens_per_second": 13.517784118652344, "tpot_ms": 74.0, "total_time_ms": 19618, "total_tokens": 256, "peak_memory_mb": 1972.203125, "mean_cpu_percent": 378.73096, "peak_cpu_percent": 770.94, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 821, "tokens_per_second": 14.261555671691896, "tpot_ms": 70.12941176470588, "total_time_ms": 18704, "total_tokens": 256, "peak_memory_mb": 1972.2578125, "mean_cpu_percent": 306.46066, "peak_cpu_percent": 407.14, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 680.6, "median": 647.0, "p90": 975.9, "p95": 1011.4499999999999, "std_dev": 202.04316370518453, "min": 440.0, "max": 1047.0 }, "tokens_per_second": { "mean": 13.190094470977783, "median": 13.080532550811768, "p90": 13.861311721801757, "p95": 14.061433696746827, "std_dev": 0.6027754012096369, "min": 12.1367769241333, "max": 14.261555671691896 }, "tpot": { "mean": 75.98392156862745, "median": 76.46274509803922, "p90": 79.24509803921568, "p95": 80.82450980392156, "std_dev": 3.46969854554027, "min": 70.12941176470588, "max": 82.40392156862745 }, "total_time": { "mean": 20056.5, "median": 20090.5, "p90": 21193.8, "p95": 21323.4, "std_dev": 840.1538252010758, "min": 18704.0, "max": 21453.0 }, "peak_memory_mb": { "mean": 1972.000390625, "median": 1972.216796875, "p90": 1972.23671875, "p95": 1972.247265625, "std_dev": 0.5457115134466226, "min": 1970.39453125, "max": 1972.2578125 }, "mean_cpu_percent": { "mean": 274.20330810546875, "median": 270.10435485839844, "p90": 338.0371856689453, "p95": 358.3840713500976, "std_dev": 52.63790585737377, "min": 201.49673461914062, "max": 378.73095703125 }, "peak_cpu_percent": { "mean": 503.79900512695315, "median": 481.5400085449219, "p90": 589.4009826660156, "p95": 680.1704925537108, "std_dev": 104.68393529054104, "min": 371.4200134277344, "max": 770.9400024414062 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 1596, "tokens_per_second": 13.787484169006348, "tpot_ms": 72.53725490196078, "total_time_ms": 20093, "total_tokens": 256, "peak_memory_mb": 1979.94140625, "mean_cpu_percent": 314.8868, "peak_cpu_percent": 689.82, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 230, "tokens_per_second": 13.65053367614746, "tpot_ms": 73.26666666666667, "total_time_ms": 18913, "total_tokens": 256, "peak_memory_mb": 1982.421875, "mean_cpu_percent": 286.52957, "peak_cpu_percent": 478.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 197, "tokens_per_second": 14.091858863830566, "tpot_ms": 70.97254901960784, "total_time_ms": 18295, "total_tokens": 256, "peak_memory_mb": 1983.4921875, "mean_cpu_percent": 332.137, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 249, "tokens_per_second": 14.588005065917969, "tpot_ms": 68.56078431372549, "total_time_ms": 17732, "total_tokens": 256, "peak_memory_mb": 1984.76171875, "mean_cpu_percent": 550.6537, "peak_cpu_percent": 840.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 163, "tokens_per_second": 13.126368522644045, "tpot_ms": 76.1921568627451, "total_time_ms": 19592, "total_tokens": 256, "peak_memory_mb": 1985.078125, "mean_cpu_percent": 342.49246, "peak_cpu_percent": 528.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 251, "tokens_per_second": 11.89555549621582, "tpot_ms": 84.07450980392157, "total_time_ms": 21690, "total_tokens": 256, "peak_memory_mb": 1985.36328125, "mean_cpu_percent": 239.33937, "peak_cpu_percent": 548.78, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 268, "tokens_per_second": 11.77799129486084, "tpot_ms": 84.91372549019607, "total_time_ms": 21921, "total_tokens": 256, "peak_memory_mb": 1985.390625, "mean_cpu_percent": 202.46494, "peak_cpu_percent": 438.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 167, "tokens_per_second": 12.19723129272461, "tpot_ms": 81.99607843137255, "total_time_ms": 21076, "total_tokens": 256, "peak_memory_mb": 1985.515625, "mean_cpu_percent": 278.61014, "peak_cpu_percent": 553.84, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 150, "tokens_per_second": 13.21729564666748, "tpot_ms": 75.67058823529412, "total_time_ms": 19446, "total_tokens": 256, "peak_memory_mb": 1985.62890625, "mean_cpu_percent": 358.78693, "peak_cpu_percent": 607.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 184, "tokens_per_second": 13.059590339660645, "tpot_ms": 76.58039215686274, "total_time_ms": 19712, "total_tokens": 256, "peak_memory_mb": 1985.6484375, "mean_cpu_percent": 386.50443, "peak_cpu_percent": 616.94, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 345.5, "median": 213.5, "p90": 400.7999999999995, "p95": 998.3999999999985, "std_dev": 418.6720076623227, "min": 150.0, "max": 1596.0 }, "tokens_per_second": { "mean": 13.139191436767579, "median": 13.171832084655762, "p90": 14.141473484039306, "p95": 14.364739274978636, "std_dev": 0.8937436227667387, "min": 11.77799129486084, "max": 14.588005065917969 }, "tpot": { "mean": 76.47647058823529, "median": 75.93137254901961, "p90": 84.15843137254902, "p95": 84.53607843137254, "std_dev": 5.281061678937997, "min": 68.56078431372549, "max": 84.91372549019607 }, "total_time": { "mean": 19847.0, "median": 19652.0, "p90": 21713.1, "p95": 21817.050000000003, "std_dev": 1314.0554782808829, "min": 17732.0, "max": 21921.0 }, "peak_memory_mb": { "mean": 1984.32421875, "median": 1985.220703125, "p90": 1985.630859375, "p95": 1985.6396484375, "std_dev": 1.7712171191042088, "min": 1979.94140625, "max": 1985.6484375 }, "mean_cpu_percent": { "mean": 329.2405319213867, "median": 323.51190185546875, "p90": 402.91935119628897, "p95": 476.7865188598631, "std_dev": 90.61340628650642, "min": 202.46493530273438, "max": 550.6536865234375 }, "peak_cpu_percent": { "mean": 590.251010131836, "median": 576.9200134277344, "p90": 704.8380065917968, "p95": 772.4190032958983, "std_dev": 107.52961694360955, "min": 438.8500061035156, "max": 840.0 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 978, "tokens_per_second": 13.565882682800291, "tpot_ms": 73.72549019607843, "total_time_ms": 19778, "total_tokens": 256, "peak_memory_mb": 1986.90234375, "mean_cpu_percent": 419.67694, "peak_cpu_percent": 614.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 551, "tokens_per_second": 12.513402938842772, "tpot_ms": 79.92549019607843, "total_time_ms": 20932, "total_tokens": 256, "peak_memory_mb": 1988.7265625, "mean_cpu_percent": 219.20769, "peak_cpu_percent": 453.48, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 639, "tokens_per_second": 12.477334022521973, "tpot_ms": 80.15294117647059, "total_time_ms": 21078, "total_tokens": 256, "peak_memory_mb": 1988.78515625, "mean_cpu_percent": 245.85649, "peak_cpu_percent": 538.92, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 657, "tokens_per_second": 12.472658157348633, "tpot_ms": 80.1843137254902, "total_time_ms": 21104, "total_tokens": 256, "peak_memory_mb": 1988.7890625, "mean_cpu_percent": 241.76015, "peak_cpu_percent": 564.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 741, "tokens_per_second": 12.287816047668455, "tpot_ms": 81.38823529411765, "total_time_ms": 21495, "total_tokens": 256, "peak_memory_mb": 1988.88671875, "mean_cpu_percent": 182.24226, "peak_cpu_percent": 335.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 629, "tokens_per_second": 11.68018913269043, "tpot_ms": 85.62745098039215, "total_time_ms": 22464, "total_tokens": 256, "peak_memory_mb": 1988.99609375, "mean_cpu_percent": 181.92264, "peak_cpu_percent": 471.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 562, "tokens_per_second": 12.178730010986328, "tpot_ms": 82.13333333333334, "total_time_ms": 21506, "total_tokens": 256, "peak_memory_mb": 1989.01953125, "mean_cpu_percent": 255.50212, "peak_cpu_percent": 467.06, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 574, "tokens_per_second": 12.759247779846191, "tpot_ms": 78.38823529411765, "total_time_ms": 20563, "total_tokens": 256, "peak_memory_mb": 1989.12109375, "mean_cpu_percent": 293.356, "peak_cpu_percent": 535.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 486, "tokens_per_second": 12.81588077545166, "tpot_ms": 78.04313725490196, "total_time_ms": 20387, "total_tokens": 256, "peak_memory_mb": 1989.13671875, "mean_cpu_percent": 327.55646, "peak_cpu_percent": 585.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 665, "tokens_per_second": 13.611586570739746, "tpot_ms": 73.47843137254903, "total_time_ms": 19402, "total_tokens": 256, "peak_memory_mb": 1989.140625, "mean_cpu_percent": 414.14377, "peak_cpu_percent": 667.41, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 648.2, "median": 634.0, "p90": 764.6999999999999, "p95": 871.3499999999997, "std_dev": 129.18420956138564, "min": 486.0, "max": 978.0 }, "tokens_per_second": { "mean": 12.636272811889649, "median": 12.495368480682373, "p90": 13.570453071594237, "p95": 13.591019821166991, "std_dev": 0.5634523170673347, "min": 11.68018913269043, "max": 13.611586570739746 }, "tpot": { "mean": 79.30470588235293, "median": 80.0392156862745, "p90": 82.48274509803922, "p95": 84.0550980392157, "std_dev": 3.488588768476471, "min": 73.47843137254903, "max": 85.62745098039215 }, "total_time": { "mean": 20870.9, "median": 21005.0, "p90": 21601.8, "p95": 22032.899999999998, "std_dev": 843.3764817683738, "min": 19402.0, "max": 22464.0 }, "peak_memory_mb": { "mean": 1988.750390625, "median": 1988.94140625, "p90": 1989.1371093750001, "p95": 1989.1388671875002, "std_dev": 0.6331583015442925, "min": 1986.90234375, "max": 1989.140625 }, "mean_cpu_percent": { "mean": 278.12245025634763, "median": 250.67930603027344, "p90": 414.69708557128905, "p95": 417.18701324462893, "std_dev": 81.24568072473971, "min": 181.92263793945312, "max": 419.67694091796875 }, "peak_cpu_percent": { "mean": 523.3980072021484, "median": 537.3150024414062, "p90": 619.5930236816406, "p95": 643.5014984130859, "std_dev": 89.96478935941413, "min": 335.7099914550781, "max": 667.4099731445312 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 3869, "tokens_per_second": 12.644841194152832, "tpot_ms": 79.08888888888889, "total_time_ms": 43018, "total_tokens": 496, "peak_memory_mb": 2025.8984375, "mean_cpu_percent": 364.86142, "peak_cpu_percent": 585.71, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 570, "tokens_per_second": 12.510787963867188, "tpot_ms": 79.93737373737373, "total_time_ms": 40139, "total_tokens": 496, "peak_memory_mb": 2027.23046875, "mean_cpu_percent": 366.9862, "peak_cpu_percent": 557.14, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 484, "tokens_per_second": 11.977217674255373, "tpot_ms": 83.4969696969697, "total_time_ms": 41815, "total_tokens": 496, "peak_memory_mb": 2027.24609375, "mean_cpu_percent": 256.45462, "peak_cpu_percent": 563.41, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 441, "tokens_per_second": 13.168807983398438, "tpot_ms": 75.94141414141414, "total_time_ms": 38032, "total_tokens": 496, "peak_memory_mb": 2027.26953125, "mean_cpu_percent": 419.9331, "peak_cpu_percent": 664.28, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 361, "tokens_per_second": 12.952144622802734, "tpot_ms": 77.21616161616161, "total_time_ms": 38583, "total_tokens": 496, "peak_memory_mb": 2027.2734375, "mean_cpu_percent": 416.20813, "peak_cpu_percent": 653.25, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 540, "tokens_per_second": 12.011402130126951, "tpot_ms": 83.25858585858586, "total_time_ms": 41753, "total_tokens": 496, "peak_memory_mb": 2027.28515625, "mean_cpu_percent": 310.14078, "peak_cpu_percent": 550.0, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 323, "tokens_per_second": 12.374990463256836, "tpot_ms": 80.81414141414142, "total_time_ms": 40326, "total_tokens": 496, "peak_memory_mb": 2027.30078125, "mean_cpu_percent": 356.58047, "peak_cpu_percent": 635.71, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 418, "tokens_per_second": 12.89484405517578, "tpot_ms": 77.55555555555556, "total_time_ms": 38808, "total_tokens": 496, "peak_memory_mb": 2027.41015625, "mean_cpu_percent": 399.30707, "peak_cpu_percent": 645.56, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 443, "tokens_per_second": 13.096707344055176, "tpot_ms": 76.36161616161615, "total_time_ms": 38242, "total_tokens": 496, "peak_memory_mb": 2027.4375, "mean_cpu_percent": 384.52063, "peak_cpu_percent": 638.46, "stop_reason": "natural_eos", "truncated": false }, { "ttft_ms": 349, "tokens_per_second": 12.652825355529783, "tpot_ms": 79.03838383838384, "total_time_ms": 39473, "total_tokens": 496, "peak_memory_mb": 2027.47265625, "mean_cpu_percent": 357.74734, "peak_cpu_percent": 643.9, "stop_reason": "natural_eos", "truncated": false } ], "stats": { "ttft": { "mean": 779.8, "median": 442.0, "p90": 899.8999999999988, "p95": 2384.449999999996, "std_dev": 1032.5183581903034, "min": 323.0, "max": 3869.0 }, "tokens_per_second": { "mean": 12.62845687866211, "median": 12.648833274841309, "p90": 13.103917407989503, "p95": 13.136362695693968, "std_dev": 0.39633271729707537, "min": 11.977217674255373, "max": 13.168807983398438 }, "tpot": { "mean": 79.2709090909091, "median": 79.06363636363636, "p90": 83.28242424242424, "p95": 83.38969696969697, "std_dev": 2.517698187400653, "min": 75.94141414141414, "max": 83.4969696969697 }, "total_time": { "mean": 40018.9, "median": 39806.0, "p90": 41935.3, "p95": 42476.649999999994, "std_dev": 1620.729863364034, "min": 38032.0, "max": 43018.0 }, "peak_memory_mb": { "mean": 2027.182421875, "median": 2027.279296875, "p90": 2027.441015625, "p95": 2027.4568359374998, "std_dev": 0.4354971244007567, "min": 2025.8984375, "max": 2027.47265625 }, "mean_cpu_percent": { "mean": 363.2739776611328, "median": 365.92381286621094, "p90": 416.58062744140625, "p95": 418.2568664550781, "std_dev": 46.99706837724568, "min": 256.4546203613281, "max": 419.93310546875 }, "peak_cpu_percent": { "mean": 613.7420104980469, "median": 637.0850219726562, "p90": 654.3530029296875, "p95": 659.3165161132813, "std_dev": 42.090821449912326, "min": 550.0, "max": 664.280029296875 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 2485, "tokens_per_second": 12.811513900756836, "tpot_ms": 78.0587084148728, "total_time_ms": 42373, "total_tokens": 512, "peak_memory_mb": 2029.73046875, "mean_cpu_percent": 391.9244, "peak_cpu_percent": 671.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 503, "tokens_per_second": 12.961871147155762, "tpot_ms": 77.1545988258317, "total_time_ms": 39929, "total_tokens": 512, "peak_memory_mb": 2031.34375, "mean_cpu_percent": 425.495, "peak_cpu_percent": 535.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 356, "tokens_per_second": 12.810489654541016, "tpot_ms": 78.0665362035225, "total_time_ms": 40248, "total_tokens": 512, "peak_memory_mb": 2031.36328125, "mean_cpu_percent": 381.89218, "peak_cpu_percent": 592.94, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 539, "tokens_per_second": 12.791495323181152, "tpot_ms": 78.18003913894324, "total_time_ms": 40489, "total_tokens": 512, "peak_memory_mb": 2031.3828125, "mean_cpu_percent": 397.69983, "peak_cpu_percent": 728.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 630, "tokens_per_second": 12.809046745300291, "tpot_ms": 78.07632093933464, "total_time_ms": 40527, "total_tokens": 512, "peak_memory_mb": 2031.4140625, "mean_cpu_percent": 418.41794, "peak_cpu_percent": 764.24, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 383, "tokens_per_second": 12.828718185424805, "tpot_ms": 77.95303326810176, "total_time_ms": 40217, "total_tokens": 512, "peak_memory_mb": 2031.55859375, "mean_cpu_percent": 263.186, "peak_cpu_percent": 500.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 437, "tokens_per_second": 13.291810035705566, "tpot_ms": 75.23679060665361, "total_time_ms": 38883, "total_tokens": 512, "peak_memory_mb": 2031.68359375, "mean_cpu_percent": 320.05844, "peak_cpu_percent": 585.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 322, "tokens_per_second": 12.990299224853516, "tpot_ms": 76.98630136986301, "total_time_ms": 39662, "total_tokens": 512, "peak_memory_mb": 2031.66796875, "mean_cpu_percent": 331.10446, "peak_cpu_percent": 541.04, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 495, "tokens_per_second": 12.041295051574709, "tpot_ms": 83.05088062622309, "total_time_ms": 42934, "total_tokens": 512, "peak_memory_mb": 2031.72265625, "mean_cpu_percent": 243.54028, "peak_cpu_percent": 521.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 513, "tokens_per_second": 12.49818515777588, "tpot_ms": 80.01761252446184, "total_time_ms": 41402, "total_tokens": 512, "peak_memory_mb": 2031.72265625, "mean_cpu_percent": 229.77344, "peak_cpu_percent": 400.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 666.3, "median": 499.0, "p90": 815.4999999999993, "p95": 1650.249999999998, "std_dev": 612.565922982988, "min": 322.0, "max": 2485.0 }, "tokens_per_second": { "mean": 12.783472442626953, "median": 12.811001777648926, "p90": 13.02045030593872, "p95": 13.156130170822143, "std_dev": 0.31136931139277757, "min": 12.041295051574709, "max": 13.291810035705566 }, "tpot": { "mean": 78.27808219178083, "median": 78.06262230919765, "p90": 80.32093933463796, "p95": 81.68590998043052, "std_dev": 1.9553718610653792, "min": 75.23679060665361, "max": 83.05088062622309 }, "total_time": { "mean": 40666.4, "median": 40368.5, "p90": 42429.1, "p95": 42681.55, "std_dev": 1172.886030268926, "min": 38883.0, "max": 42934.0 }, "peak_memory_mb": { "mean": 2031.358984375, "median": 2031.486328125, "p90": 2031.72265625, "p95": 2031.72265625, "std_dev": 0.5622380303600988, "min": 2029.73046875, "max": 2031.72265625 }, "mean_cpu_percent": { "mean": 340.3091979980469, "median": 356.4983215332031, "p90": 419.1256439208984, "p95": 422.310319519043, "std_dev": 70.04979569893491, "min": 229.7734375, "max": 425.4949951171875 }, "peak_cpu_percent": { "mean": 584.1619995117187, "median": 563.375, "p90": 732.1370056152343, "p95": 748.1884979248046, "std_dev": 104.89141412547745, "min": 400.0, "max": 764.239990234375 } } } ] }, { "backend": "cpu", "model_id": "qwen3-4b", "cold_start_ttft_ms": 1364, "idle_baseline": { "rss_mb": 7886.84765625, "cpu_percent": 0.0 }, "reliability": { "total_runs": 100, "successful_runs": 100, "failed_runs": 0, "truncation_rate": 0.0, "natural_eos_rate": 0.0, "error_breakdown": {} }, "prompts": [ { "prompt_id": "short_1", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 1429, "tokens_per_second": 3.839181661605835, "tpot_ms": 260.4803149606299, "total_time_ms": 34510, "total_tokens": 128, "peak_memory_mb": 7929.5625, "mean_cpu_percent": 227.15012, "peak_cpu_percent": 621.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1211, "tokens_per_second": 4.155392169952393, "tpot_ms": 240.66929133858267, "total_time_ms": 31776, "total_tokens": 128, "peak_memory_mb": 7929.9921875, "mean_cpu_percent": 265.12384, "peak_cpu_percent": 450.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 932, "tokens_per_second": 4.000959873199463, "tpot_ms": 249.96062992125985, "total_time_ms": 32677, "total_tokens": 128, "peak_memory_mb": 7929.9921875, "mean_cpu_percent": 213.91653, "peak_cpu_percent": 400.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1043, "tokens_per_second": 3.936795473098755, "tpot_ms": 254.0236220472441, "total_time_ms": 33304, "total_tokens": 128, "peak_memory_mb": 7929.9765625, "mean_cpu_percent": 180.74557, "peak_cpu_percent": 414.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1040, "tokens_per_second": 3.9152114391326904, "tpot_ms": 255.43307086614172, "total_time_ms": 33480, "total_tokens": 128, "peak_memory_mb": 7929.9765625, "mean_cpu_percent": 184.62839, "peak_cpu_percent": 471.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1601, "tokens_per_second": 3.7585196495056152, "tpot_ms": 266.0787401574803, "total_time_ms": 35393, "total_tokens": 128, "peak_memory_mb": 7929.9765625, "mean_cpu_percent": 160.03902, "peak_cpu_percent": 264.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1084, "tokens_per_second": 4.115024566650391, "tpot_ms": 243.0236220472441, "total_time_ms": 31948, "total_tokens": 128, "peak_memory_mb": 7929.9765625, "mean_cpu_percent": 183.13707, "peak_cpu_percent": 421.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 974, "tokens_per_second": 4.007601261138916, "tpot_ms": 249.54330708661416, "total_time_ms": 32666, "total_tokens": 128, "peak_memory_mb": 7929.9765625, "mean_cpu_percent": 207.84586, "peak_cpu_percent": 305.2, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1084, "tokens_per_second": 3.919644594192505, "tpot_ms": 255.14173228346456, "total_time_ms": 33487, "total_tokens": 128, "peak_memory_mb": 7929.9765625, "mean_cpu_percent": 172.73235, "peak_cpu_percent": 210.52, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1575, "tokens_per_second": 3.7812225818634033, "tpot_ms": 264.4803149606299, "total_time_ms": 35164, "total_tokens": 128, "peak_memory_mb": 7929.9765625, "mean_cpu_percent": 164.84169, "peak_cpu_percent": 285.71, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 1197.3, "median": 1084.0, "p90": 1577.6, "p95": 1589.3, "std_dev": 235.36696879553853, "min": 932.0, "max": 1601.0 }, "tokens_per_second": { "mean": 3.9429553270339968, "median": 3.92822003364563, "p90": 4.1190613269805905, "p95": 4.1372267484664915, "std_dev": 0.12432815073329728, "min": 3.7585196495056152, "max": 4.155392169952393 }, "tpot": { "mean": 253.88346456692912, "median": 254.5826771653543, "p90": 264.64015748031494, "p95": 265.35944881889765, "std_dev": 7.970675842320197, "min": 240.66929133858267, "max": 266.0787401574803 }, "total_time": { "mean": 33440.5, "median": 33392.0, "p90": 35186.9, "p95": 35289.95, "std_dev": 1188.6880372915343, "min": 31776.0, "max": 35393.0 }, "peak_memory_mb": { "mean": 7929.93828125, "median": 7929.9765625, "p90": 7929.9921875, "p95": 7929.9921875, "std_dev": 0.12541191894737316, "min": 7929.5625, "max": 7929.9921875 }, "mean_cpu_percent": { "mean": 196.01604309082032, "median": 183.88272857666016, "p90": 230.9474884033203, "p95": 248.03566436767574, "std_dev": 30.891165931958128, "min": 160.0390167236328, "max": 265.12384033203125 }, "peak_cpu_percent": { "mean": 384.4250015258789, "median": 407.13999938964844, "p90": 486.4200103759765, "p95": 553.9199966430663, "std_dev": 114.55845144145637, "min": 210.52000427246094, "max": 621.4199829101562 } } }, { "prompt_id": "short_2", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 1620, "tokens_per_second": 4.08176326751709, "tpot_ms": 245.0, "total_time_ms": 32735, "total_tokens": 128, "peak_memory_mb": 7935.16015625, "mean_cpu_percent": 186.55676, "peak_cpu_percent": 307.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1036, "tokens_per_second": 3.876665592193603, "tpot_ms": 257.9685039370079, "total_time_ms": 33798, "total_tokens": 128, "peak_memory_mb": 7935.16015625, "mean_cpu_percent": 174.51625, "peak_cpu_percent": 235.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1325, "tokens_per_second": 4.263755798339844, "tpot_ms": 234.55905511811022, "total_time_ms": 31114, "total_tokens": 128, "peak_memory_mb": 7935.16015625, "mean_cpu_percent": 176.51965, "peak_cpu_percent": 300.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1045, "tokens_per_second": 3.989450931549072, "tpot_ms": 250.66929133858267, "total_time_ms": 32880, "total_tokens": 128, "peak_memory_mb": 7935.16015625, "mean_cpu_percent": 177.23972, "peak_cpu_percent": 271.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1522, "tokens_per_second": 3.859240770339966, "tpot_ms": 259.1417322834646, "total_time_ms": 34433, "total_tokens": 128, "peak_memory_mb": 7935.16015625, "mean_cpu_percent": 176.37395, "peak_cpu_percent": 357.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1670, "tokens_per_second": 4.070993900299072, "tpot_ms": 245.66141732283464, "total_time_ms": 32869, "total_tokens": 128, "peak_memory_mb": 7935.16015625, "mean_cpu_percent": 267.02795, "peak_cpu_percent": 555.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1003, "tokens_per_second": 3.931708574295044, "tpot_ms": 254.36220472440945, "total_time_ms": 33307, "total_tokens": 128, "peak_memory_mb": 7935.48828125, "mean_cpu_percent": 174.2892, "peak_cpu_percent": 300.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2066, "tokens_per_second": 4.336158752441406, "tpot_ms": 230.64566929133858, "total_time_ms": 31358, "total_tokens": 128, "peak_memory_mb": 7935.48828125, "mean_cpu_percent": 316.92557, "peak_cpu_percent": 442.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1105, "tokens_per_second": 4.208045959472656, "tpot_ms": 237.66141732283464, "total_time_ms": 31288, "total_tokens": 128, "peak_memory_mb": 7935.5546875, "mean_cpu_percent": 182.14629, "peak_cpu_percent": 300.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1066, "tokens_per_second": 3.95314621925354, "tpot_ms": 252.98425196850394, "total_time_ms": 33195, "total_tokens": 128, "peak_memory_mb": 7935.5546875, "mean_cpu_percent": 176.0783, "peak_cpu_percent": 292.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 1345.8, "median": 1215.0, "p90": 1709.6, "p95": 1887.7999999999997, "std_dev": 342.15487721206017, "min": 1003.0, "max": 2066.0 }, "tokens_per_second": { "mean": 4.05709297657013, "median": 4.030222415924072, "p90": 4.27099609375, "p95": 4.303577423095703, "std_dev": 0.15715503512003606, "min": 3.859240770339966, "max": 4.336158752441406 }, "tpot": { "mean": 246.86535433070867, "median": 248.16535433070865, "p90": 258.08582677165356, "p95": 258.6137795275591, "std_dev": 9.414178925085809, "min": 230.64566929133858, "max": 259.1417322834646 }, "total_time": { "mean": 32697.7, "median": 32874.5, "p90": 33861.5, "p95": 34147.25, "std_dev": 1058.045561400831, "min": 31114.0, "max": 34433.0 }, "peak_memory_mb": { "mean": 7935.3046875, "median": 7935.16015625, "p90": 7935.5546875, "p95": 7935.5546875, "std_dev": 0.17825516077732503, "min": 7935.16015625, "max": 7935.5546875 }, "mean_cpu_percent": { "mean": 200.76736297607422, "median": 176.8796844482422, "p90": 272.01771545410156, "p95": 294.4716415405273, "std_dev": 47.08310171530242, "min": 174.28919982910156, "max": 316.9255676269531 }, "peak_cpu_percent": { "mean": 336.2530044555664, "median": 300.0, "p90": 454.10700378417965, "p95": 504.76349334716787, "std_dev": 89.89982811895193, "min": 235.7100067138672, "max": 555.4199829101562 } } }, { "prompt_id": "short_3", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 1095, "tokens_per_second": 4.529079437255859, "tpot_ms": 220.81102362204723, "total_time_ms": 29138, "total_tokens": 128, "peak_memory_mb": 7941.55078125, "mean_cpu_percent": 295.8467, "peak_cpu_percent": 471.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1755, "tokens_per_second": 3.918944358825683, "tpot_ms": 255.18897637795277, "total_time_ms": 34164, "total_tokens": 128, "peak_memory_mb": 7941.640625, "mean_cpu_percent": 185.08801, "peak_cpu_percent": 300.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1253, "tokens_per_second": 4.15130090713501, "tpot_ms": 240.92125984251967, "total_time_ms": 31850, "total_tokens": 128, "peak_memory_mb": 7941.90625, "mean_cpu_percent": 179.81499, "peak_cpu_percent": 296.47, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1076, "tokens_per_second": 4.088980197906494, "tpot_ms": 244.58267716535434, "total_time_ms": 32138, "total_tokens": 128, "peak_memory_mb": 7941.91015625, "mean_cpu_percent": 221.74289, "peak_cpu_percent": 400.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1123, "tokens_per_second": 4.0138936042785645, "tpot_ms": 249.15748031496062, "total_time_ms": 32766, "total_tokens": 128, "peak_memory_mb": 7941.91015625, "mean_cpu_percent": 246.60646, "peak_cpu_percent": 457.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1106, "tokens_per_second": 4.213672161102295, "tpot_ms": 237.33858267716536, "total_time_ms": 31248, "total_tokens": 128, "peak_memory_mb": 7941.91015625, "mean_cpu_percent": 294.76917, "peak_cpu_percent": 464.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1281, "tokens_per_second": 4.220047473907471, "tpot_ms": 236.9763779527559, "total_time_ms": 31377, "total_tokens": 128, "peak_memory_mb": 7942.0, "mean_cpu_percent": 228.0789, "peak_cpu_percent": 307.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1769, "tokens_per_second": 4.052677154541016, "tpot_ms": 246.76377952755905, "total_time_ms": 33108, "total_tokens": 128, "peak_memory_mb": 7942.02734375, "mean_cpu_percent": 160.32018, "peak_cpu_percent": 256.6, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1141, "tokens_per_second": 3.976574659347534, "tpot_ms": 251.48818897637796, "total_time_ms": 33080, "total_tokens": 128, "peak_memory_mb": 7942.02734375, "mean_cpu_percent": 175.16629, "peak_cpu_percent": 211.32, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1735, "tokens_per_second": 3.557947158813477, "tpot_ms": 281.07086614173227, "total_time_ms": 37431, "total_tokens": 128, "peak_memory_mb": 7942.02734375, "mean_cpu_percent": 158.55333, "peak_cpu_percent": 300.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 1333.4, "median": 1197.0, "p90": 1756.4, "p95": 1762.6999999999998, "std_dev": 281.79290267854515, "min": 1076.0, "max": 1769.0 }, "tokens_per_second": { "mean": 4.0723117113113405, "median": 4.070828676223755, "p90": 4.250950670242309, "p95": 4.390015053749084, "std_dev": 0.23622420094564797, "min": 3.557947158813477, "max": 4.529079437255859 }, "tpot": { "mean": 246.42992125984256, "median": 245.6732283464567, "p90": 257.7771653543307, "p95": 269.4240157480315, "std_dev": 14.737573466216999, "min": 220.81102362204723, "max": 281.07086614173227 }, "total_time": { "mean": 32630.0, "median": 32452.0, "p90": 34490.7, "p95": 35960.84999999999, "std_dev": 2058.5601278563618, "min": 29138.0, "max": 37431.0 }, "peak_memory_mb": { "mean": 7941.891015625, "median": 7941.91015625, "p90": 7942.02734375, "p95": 7942.02734375, "std_dev": 0.1572958164636082, "min": 7941.55078125, "max": 7942.02734375 }, "mean_cpu_percent": { "mean": 214.59869232177735, "median": 203.4154510498047, "p90": 294.8769195556641, "p95": 295.36181488037107, "std_dev": 48.99871302239608, "min": 158.55332946777344, "max": 295.8467102050781 }, "peak_cpu_percent": { "mean": 346.4370056152344, "median": 303.57000732421875, "p90": 464.99400024414064, "p95": 468.2070068359375, "std_dev": 89.05779560086906, "min": 211.32000732421875, "max": 471.4200134277344 } } }, { "prompt_id": "short_4", "tier": "short", "max_tokens": 128, "runs": [ { "ttft_ms": 1576, "tokens_per_second": 3.692292690277099, "tpot_ms": 270.8582677165354, "total_time_ms": 35975, "total_tokens": 128, "peak_memory_mb": 7942.72265625, "mean_cpu_percent": 163.37595, "peak_cpu_percent": 203.5, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1148, "tokens_per_second": 3.807821750640869, "tpot_ms": 262.6377952755906, "total_time_ms": 34503, "total_tokens": 128, "peak_memory_mb": 7942.72265625, "mean_cpu_percent": 194.2939, "peak_cpu_percent": 301.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1058, "tokens_per_second": 4.024382591247559, "tpot_ms": 248.50393700787401, "total_time_ms": 32618, "total_tokens": 128, "peak_memory_mb": 7942.72265625, "mean_cpu_percent": 265.72433, "peak_cpu_percent": 485.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1134, "tokens_per_second": 3.706493377685547, "tpot_ms": 269.81102362204723, "total_time_ms": 35400, "total_tokens": 128, "peak_memory_mb": 7942.76953125, "mean_cpu_percent": 178.76315, "peak_cpu_percent": 307.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1350, "tokens_per_second": 4.258372783660889, "tpot_ms": 234.84251968503938, "total_time_ms": 31175, "total_tokens": 128, "peak_memory_mb": 7942.828125, "mean_cpu_percent": 184.03352, "peak_cpu_percent": 307.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 994, "tokens_per_second": 4.218286991119385, "tpot_ms": 237.09448818897638, "total_time_ms": 31105, "total_tokens": 128, "peak_memory_mb": 7942.828125, "mean_cpu_percent": 181.19093, "peak_cpu_percent": 350.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1533, "tokens_per_second": 3.8876311779022217, "tpot_ms": 257.23622047244095, "total_time_ms": 34202, "total_tokens": 128, "peak_memory_mb": 7942.828125, "mean_cpu_percent": 171.7599, "peak_cpu_percent": 301.84, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1133, "tokens_per_second": 4.096525192260742, "tpot_ms": 244.11811023622047, "total_time_ms": 32136, "total_tokens": 128, "peak_memory_mb": 7942.828125, "mean_cpu_percent": 187.28372, "peak_cpu_percent": 300.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1194, "tokens_per_second": 3.9781298637390137, "tpot_ms": 251.39370078740157, "total_time_ms": 33121, "total_tokens": 128, "peak_memory_mb": 7942.828125, "mean_cpu_percent": 192.53928, "peak_cpu_percent": 314.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1011, "tokens_per_second": 3.85478401184082, "tpot_ms": 259.4488188976378, "total_time_ms": 33961, "total_tokens": 128, "peak_memory_mb": 7942.828125, "mean_cpu_percent": 180.404, "peak_cpu_percent": 291.32, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 1213.1, "median": 1141.0, "p90": 1537.3, "p95": 1556.65, "std_dev": 195.69233505684377, "min": 994.0, "max": 1576.0 }, "tokens_per_second": { "mean": 3.9524720430374147, "median": 3.9328805208206177, "p90": 4.222295570373535, "p95": 4.240334177017212, "std_dev": 0.1880647244908049, "min": 3.692292690277099, "max": 4.258372783660889 }, "tpot": { "mean": 253.5944881889764, "median": 254.31496062992125, "p90": 269.91574803149604, "p95": 270.38700787401575, "std_dev": 11.97821417637995, "min": 234.84251968503938, "max": 270.8582677165354 }, "total_time": { "mean": 33419.6, "median": 33541.0, "p90": 35457.5, "p95": 35716.25, "std_dev": 1590.786861901996, "min": 31105.0, "max": 35975.0 }, "peak_memory_mb": { "mean": 7942.790625, "median": 7942.828125, "p90": 7942.828125, "p95": 7942.828125, "std_dev": 0.047688257693849545, "min": 7942.72265625, "max": 7942.828125 }, "mean_cpu_percent": { "mean": 189.9368698120117, "median": 182.6122283935547, "p90": 201.4369430541992, "p95": 233.580638885498, "std_dev": 26.719821062559145, "min": 163.37594604492188, "max": 265.7243347167969 }, "peak_cpu_percent": { "mean": 316.2640014648438, "median": 304.49000549316406, "p90": 363.57099914550776, "p95": 424.64049530029286, "std_dev": 66.36475224349806, "min": 203.5, "max": 485.7099914550781 } } }, { "prompt_id": "medium_1", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 1373, "tokens_per_second": 3.915582418441773, "tpot_ms": 255.39607843137256, "total_time_ms": 66499, "total_tokens": 256, "peak_memory_mb": 7950.9765625, "mean_cpu_percent": 187.38142, "peak_cpu_percent": 600.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 419, "tokens_per_second": 3.929582357406616, "tpot_ms": 254.48627450980393, "total_time_ms": 65313, "total_tokens": 256, "peak_memory_mb": 7983.97265625, "mean_cpu_percent": 208.94173, "peak_cpu_percent": 579.54, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 372, "tokens_per_second": 4.160686492919922, "tpot_ms": 240.35294117647058, "total_time_ms": 61662, "total_tokens": 256, "peak_memory_mb": 7985.375, "mean_cpu_percent": 180.69528, "peak_cpu_percent": 416.18, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 514, "tokens_per_second": 4.221380710601807, "tpot_ms": 236.90980392156862, "total_time_ms": 60926, "total_tokens": 256, "peak_memory_mb": 7985.5625, "mean_cpu_percent": 281.64813, "peak_cpu_percent": 728.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 488, "tokens_per_second": 4.045206069946289, "tpot_ms": 247.21960784313725, "total_time_ms": 63529, "total_tokens": 256, "peak_memory_mb": 7985.5625, "mean_cpu_percent": 264.21835, "peak_cpu_percent": 783.81, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 376, "tokens_per_second": 4.270837306976318, "tpot_ms": 234.1529411764706, "total_time_ms": 60085, "total_tokens": 256, "peak_memory_mb": 7985.5625, "mean_cpu_percent": 312.97125, "peak_cpu_percent": 572.72, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 392, "tokens_per_second": 4.291295528411865, "tpot_ms": 233.03529411764706, "total_time_ms": 59816, "total_tokens": 256, "peak_memory_mb": 7985.58984375, "mean_cpu_percent": 426.0408, "peak_cpu_percent": 721.34, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 513, "tokens_per_second": 4.126792907714844, "tpot_ms": 242.3294117647059, "total_time_ms": 62307, "total_tokens": 256, "peak_memory_mb": 7985.6015625, "mean_cpu_percent": 241.04938, "peak_cpu_percent": 634.09, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 369, "tokens_per_second": 4.012204170227051, "tpot_ms": 249.24705882352941, "total_time_ms": 63927, "total_tokens": 256, "peak_memory_mb": 7985.6640625, "mean_cpu_percent": 195.04039, "peak_cpu_percent": 385.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 396, "tokens_per_second": 3.8183634281158447, "tpot_ms": 261.9019607843137, "total_time_ms": 67181, "total_tokens": 256, "peak_memory_mb": 7985.86328125, "mean_cpu_percent": 172.06097, "peak_cpu_percent": 350.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 521.2, "median": 407.5, "p90": 599.8999999999996, "p95": 986.449999999999, "std_dev": 289.15490658123025, "min": 369.0, "max": 1373.0 }, "tokens_per_second": { "mean": 4.079193139076233, "median": 4.085999488830566, "p90": 4.272883129119873, "p95": 4.28208932876587, "std_dev": 0.1528612951917533, "min": 3.8183634281158447, "max": 4.291295528411865 }, "tpot": { "mean": 245.50313725490196, "median": 244.77450980392157, "p90": 256.0466666666667, "p95": 258.97431372549016, "std_dev": 9.269132470883319, "min": 233.03529411764706, "max": 261.9019607843137 }, "total_time": { "mean": 63124.5, "median": 62918.0, "p90": 66567.2, "p95": 66874.1, "std_dev": 2475.1757210347714, "min": 59816.0, "max": 67181.0 }, "peak_memory_mb": { "mean": 7981.973046875, "median": 7985.5625, "p90": 7985.683984375, "p95": 7985.773632812499, "std_dev": 10.34413434101146, "min": 7950.9765625, "max": 7985.86328125 }, "mean_cpu_percent": { "mean": 247.00477142333983, "median": 224.99555206298828, "p90": 324.2782073974609, "p95": 375.1595046997069, "std_dev": 74.60225898265858, "min": 172.06097412109375, "max": 426.0408020019531 }, "peak_cpu_percent": { "mean": 577.1959991455078, "median": 589.7699890136719, "p90": 734.0940063476562, "p95": 758.9520019531249, "std_dev": 142.9596043771925, "min": 350.0, "max": 783.8099975585938 } } }, { "prompt_id": "medium_2", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 1632, "tokens_per_second": 4.124756813049316, "tpot_ms": 242.4470588235294, "total_time_ms": 63456, "total_tokens": 256, "peak_memory_mb": 7989.921875, "mean_cpu_percent": 273.94275, "peak_cpu_percent": 486.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 535, "tokens_per_second": 3.931850671768189, "tpot_ms": 254.34117647058824, "total_time_ms": 65392, "total_tokens": 256, "peak_memory_mb": 7990.55078125, "mean_cpu_percent": 218.76964, "peak_cpu_percent": 492.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 845, "tokens_per_second": 3.926600456237793, "tpot_ms": 254.67843137254903, "total_time_ms": 65788, "total_tokens": 256, "peak_memory_mb": 7990.546875, "mean_cpu_percent": 182.13077, "peak_cpu_percent": 306.97, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 510, "tokens_per_second": 4.278018951416016, "tpot_ms": 233.76078431372548, "total_time_ms": 60119, "total_tokens": 256, "peak_memory_mb": 7990.56640625, "mean_cpu_percent": 262.0065, "peak_cpu_percent": 400.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 529, "tokens_per_second": 4.191745281219482, "tpot_ms": 238.57254901960783, "total_time_ms": 61365, "total_tokens": 256, "peak_memory_mb": 7990.5625, "mean_cpu_percent": 278.0004, "peak_cpu_percent": 507.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 556, "tokens_per_second": 4.374453544616699, "tpot_ms": 228.6078431372549, "total_time_ms": 58851, "total_tokens": 256, "peak_memory_mb": 7990.5625, "mean_cpu_percent": 331.99255, "peak_cpu_percent": 807.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 523, "tokens_per_second": 4.403231143951416, "tpot_ms": 227.11764705882354, "total_time_ms": 58438, "total_tokens": 256, "peak_memory_mb": 7990.5625, "mean_cpu_percent": 299.18225, "peak_cpu_percent": 521.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 553, "tokens_per_second": 4.007102012634277, "tpot_ms": 249.56470588235294, "total_time_ms": 64192, "total_tokens": 256, "peak_memory_mb": 7990.5625, "mean_cpu_percent": 177.48633, "peak_cpu_percent": 342.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 558, "tokens_per_second": 4.208008289337158, "tpot_ms": 237.64705882352942, "total_time_ms": 61158, "total_tokens": 256, "peak_memory_mb": 7990.5625, "mean_cpu_percent": 275.27536, "peak_cpu_percent": 471.16, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 545, "tokens_per_second": 3.909688711166382, "tpot_ms": 255.78039215686275, "total_time_ms": 65769, "total_tokens": 256, "peak_memory_mb": 7990.5625, "mean_cpu_percent": 175.71713, "peak_cpu_percent": 291.32, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 678.6, "median": 549.0, "p90": 923.6999999999998, "p95": 1277.8499999999992, "std_dev": 330.9831415646422, "min": 510.0, "max": 1632.0 }, "tokens_per_second": { "mean": 4.1355455875396725, "median": 4.158251047134399, "p90": 4.377331304550171, "p95": 4.390281224250793, "std_dev": 0.17612939618428627, "min": 3.909688711166382, "max": 4.403231143951416 }, "tpot": { "mean": 242.25176470588235, "median": 240.50980392156862, "p90": 254.78862745098039, "p95": 255.28450980392157, "std_dev": 10.29150977984451, "min": 227.11764705882354, "max": 255.78039215686275 }, "total_time": { "mean": 62452.8, "median": 62410.5, "p90": 65770.9, "p95": 65779.45, "std_dev": 2686.798942980289, "min": 58438.0, "max": 65788.0 }, "peak_memory_mb": { "mean": 7990.49609375, "median": 7990.5625, "p90": 7990.562890625, "p95": 7990.5646484375, "std_dev": 0.1914939212481691, "min": 7989.921875, "max": 7990.56640625 }, "mean_cpu_percent": { "mean": 247.45036773681642, "median": 267.97462463378906, "p90": 302.46328124999997, "p95": 317.2279174804687, "std_dev": 52.51160722990603, "min": 175.71713256835938, "max": 331.9925537109375 }, "peak_cpu_percent": { "mean": 462.77000427246094, "median": 479.0050048828125, "p90": 549.9919860839842, "p95": 678.5660003662106, "std_dev": 140.55030727504632, "min": 291.32000732421875, "max": 807.1400146484375 } } }, { "prompt_id": "medium_3", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 1835, "tokens_per_second": 3.889988660812378, "tpot_ms": 257.078431372549, "total_time_ms": 67390, "total_tokens": 256, "peak_memory_mb": 7995.5390625, "mean_cpu_percent": 188.94542, "peak_cpu_percent": 607.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 835, "tokens_per_second": 4.116587162017822, "tpot_ms": 242.9294117647059, "total_time_ms": 62782, "total_tokens": 256, "peak_memory_mb": 7996.765625, "mean_cpu_percent": 313.7193, "peak_cpu_percent": 530.23, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1363, "tokens_per_second": 4.190303802490234, "tpot_ms": 238.65882352941176, "total_time_ms": 62221, "total_tokens": 256, "peak_memory_mb": 7996.9453125, "mean_cpu_percent": 327.1537, "peak_cpu_percent": 485.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 794, "tokens_per_second": 3.9588561058044434, "tpot_ms": 252.6078431372549, "total_time_ms": 65209, "total_tokens": 256, "peak_memory_mb": 7996.9453125, "mean_cpu_percent": 198.37357, "peak_cpu_percent": 379.31, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 891, "tokens_per_second": 4.436206340789795, "tpot_ms": 225.42745098039217, "total_time_ms": 58375, "total_tokens": 256, "peak_memory_mb": 7996.953125, "mean_cpu_percent": 255.31181, "peak_cpu_percent": 507.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 841, "tokens_per_second": 4.304633617401123, "tpot_ms": 232.31764705882352, "total_time_ms": 60082, "total_tokens": 256, "peak_memory_mb": 7996.953125, "mean_cpu_percent": 366.2293, "peak_cpu_percent": 499.99, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 797, "tokens_per_second": 4.165304660797119, "tpot_ms": 240.08627450980393, "total_time_ms": 62019, "total_tokens": 256, "peak_memory_mb": 7996.953125, "mean_cpu_percent": 300.79984, "peak_cpu_percent": 542.85, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 841, "tokens_per_second": 4.24074649810791, "tpot_ms": 235.8156862745098, "total_time_ms": 60974, "total_tokens": 256, "peak_memory_mb": 7996.953125, "mean_cpu_percent": 240.10632, "peak_cpu_percent": 393.1, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 941, "tokens_per_second": 3.7696499824523926, "tpot_ms": 265.2901960784314, "total_time_ms": 68590, "total_tokens": 256, "peak_memory_mb": 7996.953125, "mean_cpu_percent": 177.14761, "peak_cpu_percent": 535.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 929, "tokens_per_second": 3.9787557125091553, "tpot_ms": 251.34509803921569, "total_time_ms": 65022, "total_tokens": 256, "peak_memory_mb": 7997.08203125, "mean_cpu_percent": 221.44775, "peak_cpu_percent": 400.0, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 1006.7, "median": 866.0, "p90": 1410.1999999999998, "p95": 1622.5999999999995, "std_dev": 317.8647668427565, "min": 794.0, "max": 1835.0 }, "tokens_per_second": { "mean": 4.105103254318237, "median": 4.140945911407471, "p90": 4.317790889739991, "p95": 4.376998615264892, "std_dev": 0.19365201983195693, "min": 3.7696499824523926, "max": 4.436206340789795 }, "tpot": { "mean": 244.1556862745098, "median": 241.5078431372549, "p90": 257.89960784313723, "p95": 261.59490196078434, "std_dev": 11.599622294292267, "min": 225.42745098039217, "max": 265.2901960784314 }, "total_time": { "mean": 63266.4, "median": 62501.5, "p90": 67510.0, "p95": 68050.0, "std_dev": 3072.546605016757, "min": 58375.0, "max": 68590.0 }, "peak_memory_mb": { "mean": 7996.804296875, "median": 7996.953125, "p90": 7996.966015625, "p95": 7997.0240234375, "std_dev": 0.427781461449378, "min": 7995.5390625, "max": 7997.08203125 }, "mean_cpu_percent": { "mean": 258.9234619140625, "median": 247.70906829833984, "p90": 331.0612487792969, "p95": 348.64527893066406, "std_dev": 61.60209898046334, "min": 177.14761352539062, "max": 366.22930908203125 }, "peak_cpu_percent": { "mean": 488.1459991455078, "median": 503.7050018310547, "p90": 549.2789794921875, "p95": 578.2094970703124, "std_dev": 70.95909638691897, "min": 379.30999755859375, "max": 607.1400146484375 } } }, { "prompt_id": "medium_4", "tier": "medium", "max_tokens": 256, "runs": [ { "ttft_ms": 1951, "tokens_per_second": 3.831947326660156, "tpot_ms": 260.97254901960787, "total_time_ms": 68499, "total_tokens": 256, "peak_memory_mb": 7997.6484375, "mean_cpu_percent": 185.80133, "peak_cpu_percent": 428.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 638, "tokens_per_second": 4.014400005340576, "tpot_ms": 249.10980392156864, "total_time_ms": 64161, "total_tokens": 256, "peak_memory_mb": 7998.8203125, "mean_cpu_percent": 201.48714, "peak_cpu_percent": 385.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 616, "tokens_per_second": 4.379831790924072, "tpot_ms": 228.3294117647059, "total_time_ms": 58840, "total_tokens": 256, "peak_memory_mb": 7999.00390625, "mean_cpu_percent": 310.1644, "peak_cpu_percent": 473.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 645, "tokens_per_second": 4.161821365356445, "tpot_ms": 240.28627450980392, "total_time_ms": 61918, "total_tokens": 256, "peak_memory_mb": 7999.00390625, "mean_cpu_percent": 232.43584, "peak_cpu_percent": 450.86, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 716, "tokens_per_second": 4.036514759063721, "tpot_ms": 247.74901960784314, "total_time_ms": 63892, "total_tokens": 256, "peak_memory_mb": 7999.0234375, "mean_cpu_percent": 225.17374, "peak_cpu_percent": 596.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 609, "tokens_per_second": 3.806140184402466, "tpot_ms": 262.74117647058824, "total_time_ms": 67608, "total_tokens": 256, "peak_memory_mb": 7999.08984375, "mean_cpu_percent": 179.16142, "peak_cpu_percent": 378.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 956, "tokens_per_second": 4.232339859008789, "tpot_ms": 236.28627450980392, "total_time_ms": 61209, "total_tokens": 256, "peak_memory_mb": 7999.08984375, "mean_cpu_percent": 321.02386, "peak_cpu_percent": 556.09, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 877, "tokens_per_second": 3.985239267349243, "tpot_ms": 250.93725490196078, "total_time_ms": 64866, "total_tokens": 256, "peak_memory_mb": 7999.265625, "mean_cpu_percent": 196.52603, "peak_cpu_percent": 385.18, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 942, "tokens_per_second": 4.006779193878174, "tpot_ms": 249.5843137254902, "total_time_ms": 64586, "total_tokens": 256, "peak_memory_mb": 7999.265625, "mean_cpu_percent": 197.39246, "peak_cpu_percent": 378.57, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 640, "tokens_per_second": 4.79923677444458, "tpot_ms": 208.37254901960785, "total_time_ms": 53775, "total_tokens": 256, "peak_memory_mb": 7999.265625, "mean_cpu_percent": 219.87837, "peak_cpu_percent": 770.94, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 859.0, "median": 680.5, "p90": 1055.4999999999995, "p95": 1503.2499999999989, "std_dev": 386.4818236346957, "min": 609.0, "max": 1951.0 }, "tokens_per_second": { "mean": 4.125425052642822, "median": 4.025457382202148, "p90": 4.421772289276123, "p95": 4.610504531860351, "std_dev": 0.27812461068523925, "min": 3.806140184402466, "max": 4.79923677444458 }, "tpot": { "mean": 243.43686274509804, "median": 248.4294117647059, "p90": 261.1494117647059, "p95": 261.9452941176471, "std_dev": 15.3094059535225, "min": 208.37254901960785, "max": 262.74117647058824 }, "total_time": { "mean": 62935.4, "median": 64026.5, "p90": 67697.1, "p95": 68098.04999999999, "std_dev": 4078.6823901843595, "min": 53775.0, "max": 68499.0 }, "peak_memory_mb": { "mean": 7998.94765625, "median": 7999.056640625, "p90": 7999.265625, "p95": 7999.265625, "std_dev": 0.45387263214577894, "min": 7997.6484375, "max": 7999.265625 }, "mean_cpu_percent": { "mean": 226.90445861816406, "median": 210.68275451660156, "p90": 311.25034484863284, "p95": 316.13710479736324, "std_dev": 47.20418368045299, "min": 179.1614227294922, "max": 321.02386474609375 }, "peak_cpu_percent": { "mean": 480.4200042724609, "median": 439.7149963378906, "p90": 614.0070068359374, "p95": 692.4735046386718, "std_dev": 120.8392941502478, "min": 378.57000732421875, "max": 770.9400024414062 } } }, { "prompt_id": "long_1", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 5281, "tokens_per_second": 4.529773712158203, "tpot_ms": 220.7651663405088, "total_time_ms": 118092, "total_tokens": 512, "peak_memory_mb": 8070.0390625, "mean_cpu_percent": 178.53705, "peak_cpu_percent": 351.72, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 943, "tokens_per_second": 4.222125053405762, "tpot_ms": 236.853228962818, "total_time_ms": 121975, "total_tokens": 512, "peak_memory_mb": 8079.921875, "mean_cpu_percent": 187.65471, "peak_cpu_percent": 650.0, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1286, "tokens_per_second": 4.0707011222839355, "tpot_ms": 245.66536203522506, "total_time_ms": 126821, "total_tokens": 512, "peak_memory_mb": 8080.0625, "mean_cpu_percent": 285.71643, "peak_cpu_percent": 545.45, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1447, "tokens_per_second": 3.815806388854981, "tpot_ms": 262.07045009784736, "total_time_ms": 135365, "total_tokens": 512, "peak_memory_mb": 8080.09765625, "mean_cpu_percent": 214.35846, "peak_cpu_percent": 847.9, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1205, "tokens_per_second": 3.592813014984131, "tpot_ms": 278.3405088062622, "total_time_ms": 143437, "total_tokens": 512, "peak_memory_mb": 8080.09765625, "mean_cpu_percent": 173.88475, "peak_cpu_percent": 385.96, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 2010, "tokens_per_second": 3.774538993835449, "tpot_ms": 264.93737769080235, "total_time_ms": 137393, "total_tokens": 512, "peak_memory_mb": 8080.10546875, "mean_cpu_percent": 222.97957, "peak_cpu_percent": 493.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1429, "tokens_per_second": 4.291980743408203, "tpot_ms": 232.99608610567515, "total_time_ms": 120490, "total_tokens": 512, "peak_memory_mb": 8080.10546875, "mean_cpu_percent": 319.75592, "peak_cpu_percent": 614.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1243, "tokens_per_second": 4.3201422691345215, "tpot_ms": 231.47749510763208, "total_time_ms": 119528, "total_tokens": 512, "peak_memory_mb": 8080.10546875, "mean_cpu_percent": 383.25705, "peak_cpu_percent": 661.81, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1290, "tokens_per_second": 4.03558874130249, "tpot_ms": 247.79843444227006, "total_time_ms": 127915, "total_tokens": 512, "peak_memory_mb": 8080.16796875, "mean_cpu_percent": 308.99695, "peak_cpu_percent": 607.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 1219, "tokens_per_second": 3.819499731063843, "tpot_ms": 261.8180039138943, "total_time_ms": 135008, "total_tokens": 512, "peak_memory_mb": 8080.16796875, "mean_cpu_percent": 171.04932, "peak_cpu_percent": 486.85, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 1735.3, "median": 1288.0, "p90": 2337.0999999999985, "p95": 3809.0499999999965, "std_dev": 1210.0590935983248, "min": 943.0, "max": 5281.0 }, "tokens_per_second": { "mean": 4.0472969770431515, "median": 4.053144931793213, "p90": 4.341105413436889, "p95": 4.435439562797546, "std_dev": 0.28013929728215825, "min": 3.592813014984131, "max": 4.529773712158203 }, "tpot": { "mean": 248.27221135029353, "median": 246.73189823874756, "p90": 266.27769080234833, "p95": 272.3090998043053, "std_dev": 17.21970812114616, "min": 220.7651663405088, "max": 278.3405088062622 }, "total_time": { "mean": 128602.4, "median": 127368.0, "p90": 137997.4, "p95": 140717.19999999998, "std_dev": 8308.411932493478, "min": 118092.0, "max": 143437.0 }, "peak_memory_mb": { "mean": 8079.087109375, "median": 8080.1015625, "p90": 8080.16796875, "p95": 8080.16796875, "std_dev": 3.0167111922151797, "min": 8070.0390625, "max": 8080.16796875 }, "mean_cpu_percent": { "mean": 244.6190200805664, "median": 218.66901397705078, "p90": 326.1060333251953, "p95": 354.68154144287104, "std_dev": 70.78922032806989, "min": 171.04931640625, "max": 383.2570495605469 }, "peak_cpu_percent": { "mean": 564.4820068359375, "median": 576.2950134277344, "p90": 680.4190002441405, "p95": 764.1595123291013, "std_dev": 137.82118480835473, "min": 351.7200012207031, "max": 847.9000244140625 } } }, { "prompt_id": "long_2", "tier": "long", "max_tokens": 512, "runs": [ { "ttft_ms": 5427, "tokens_per_second": 3.4119436740875244, "tpot_ms": 293.0939334637965, "total_time_ms": 155198, "total_tokens": 512, "peak_memory_mb": 8081.21484375, "mean_cpu_percent": 342.0585, "peak_cpu_percent": 864.28, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 590, "tokens_per_second": 3.193352222442627, "tpot_ms": 313.1545988258317, "total_time_ms": 160612, "total_tokens": 512, "peak_memory_mb": 8081.21484375, "mean_cpu_percent": 396.96146, "peak_cpu_percent": 821.42, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 488, "tokens_per_second": 2.7407212257385254, "tpot_ms": 364.8747553816047, "total_time_ms": 186939, "total_tokens": 512, "peak_memory_mb": 8081.3515625, "mean_cpu_percent": 224.25131, "peak_cpu_percent": 757.14, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 431, "tokens_per_second": 2.781228542327881, "tpot_ms": 359.5577299412916, "total_time_ms": 184165, "total_tokens": 512, "peak_memory_mb": 8081.3515625, "mean_cpu_percent": 218.59978, "peak_cpu_percent": 773.33, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 316, "tokens_per_second": 3.775259256362915, "tpot_ms": 264.88845401174166, "total_time_ms": 135674, "total_tokens": 512, "peak_memory_mb": 8081.3515625, "mean_cpu_percent": 384.1031, "peak_cpu_percent": 781.06, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 363, "tokens_per_second": 3.3744397163391113, "tpot_ms": 296.3522504892368, "total_time_ms": 151799, "total_tokens": 512, "peak_memory_mb": 8081.3515625, "mean_cpu_percent": 305.48758, "peak_cpu_percent": 685.71, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 477, "tokens_per_second": 3.29468297958374, "tpot_ms": 303.5244618395303, "total_time_ms": 155578, "total_tokens": 512, "peak_memory_mb": 8081.3515625, "mean_cpu_percent": 411.89423, "peak_cpu_percent": 726.66, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 432, "tokens_per_second": 3.2168426513671875, "tpot_ms": 310.87084148727985, "total_time_ms": 159287, "total_tokens": 512, "peak_memory_mb": 8081.3828125, "mean_cpu_percent": 413.43262, "peak_cpu_percent": 649.18, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 435, "tokens_per_second": 3.068039894104004, "tpot_ms": 325.94716242661445, "total_time_ms": 166994, "total_tokens": 512, "peak_memory_mb": 8081.3828125, "mean_cpu_percent": 381.54987, "peak_cpu_percent": 705.88, "stop_reason": "max_tokens_exact", "truncated": false }, { "ttft_ms": 410, "tokens_per_second": 3.3740386962890625, "tpot_ms": 296.38747553816046, "total_time_ms": 151864, "total_tokens": 512, "peak_memory_mb": 8081.3828125, "mean_cpu_percent": 440.55386, "peak_cpu_percent": 771.42, "stop_reason": "max_tokens_exact", "truncated": false } ], "stats": { "ttft": { "mean": 936.9, "median": 433.5, "p90": 1073.6999999999985, "p95": 3250.3499999999945, "std_dev": 1498.319755592911, "min": 316.0, "max": 5427.0 }, "tokens_per_second": { "mean": 3.223054885864258, "median": 3.255762815475464, "p90": 3.4482752323150634, "p95": 3.611767244338989, "std_dev": 0.29056143379803173, "min": 2.7407212257385254, "max": 3.775259256362915 }, "tpot": { "mean": 312.86516634050884, "median": 307.19765166340505, "p90": 360.08943248532296, "p95": 362.48209393346383, "std_dev": 28.955998674235936, "min": 264.88845401174166, "max": 364.8747553816047 }, "total_time": { "mean": 160811.0, "median": 157432.5, "p90": 184442.4, "p95": 185690.7, "std_dev": 14562.613590973291, "min": 135674.0, "max": 186939.0 }, "peak_memory_mb": { "mean": 8081.33359375, "median": 8081.3515625, "p90": 8081.3828125, "p95": 8081.3828125, "std_dev": 0.06089742271845336, "min": 8081.21484375, "max": 8081.3828125 }, "mean_cpu_percent": { "mean": 351.8892288208008, "median": 382.82647705078125, "p90": 416.1447418212891, "p95": 428.3493026733398, "std_dev": 74.50226139239629, "min": 218.5997772216797, "max": 440.5538635253906 }, "peak_cpu_percent": { "mean": 753.6080017089844, "median": 764.2799987792969, "p90": 825.7059875488281, "p95": 844.9930084228515, "std_dev": 60.6947912829687, "min": 649.1799926757812, "max": 864.280029296875 } } } ] } ], "skipped": [] }