{ "suite": "server-failure-triage-v1", "generated_at": "2026-09-10T18:03:46", "workflow_id": "benchmark-triage-001", "summary": [ { "config": "A_full_context_256", "backend": "sglang-rtx6000", "cases": 24, "accuracy": 0.9583333333333334, "mean_latency_s": 0.5955590547916737, "median_latency_s": 0.5206998419999991, "p95_latency_s": 1.0275071900000512, "mean_prompt_tokens": 451.7916666666667, "mean_completion_tokens": 127.375, "mean_reasoning_tokens": 123.75, "total_tokens": 13900, "tokens_per_correct_answer": 604.3478260869565, "truncated": 1 }, { "config": "B_reduced_context_256", "backend": "sglang-rtx6000", "cases": 24, "accuracy": 0.9583333333333334, "mean_latency_s": 0.5553105036249804, "median_latency_s": 0.5354650080000738, "p95_latency_s": 1.0765945619998547, "mean_prompt_tokens": 190.45833333333334, "mean_completion_tokens": 123.5, "mean_reasoning_tokens": 119.95833333333333, "total_tokens": 7535, "tokens_per_correct_answer": 327.60869565217394, "truncated": 1 }, { "config": "C_reduced_context_64", "backend": "sglang-rtx6000", "cases": 24, "accuracy": 0.08333333333333333, "mean_latency_s": 0.32602416804164136, "median_latency_s": 0.32633463800004847, "p95_latency_s": 0.3739699989998826, "mean_prompt_tokens": 190.45833333333334, "mean_completion_tokens": 63.75, "mean_reasoning_tokens": 65.625, "total_tokens": 6101, "tokens_per_correct_answer": 3050.5, "truncated": 22 }, { "config": "D_specialized_prompt_64_nothink", "backend": "sglang-rtx6000", "cases": 24, "accuracy": 1.0, "mean_latency_s": 0.08014430374998938, "median_latency_s": 0.08105620699996052, "p95_latency_s": 0.09140553600013845, "mean_prompt_tokens": 159.45833333333334, "mean_completion_tokens": 2.8333333333333335, "mean_reasoning_tokens": 0.0, "total_tokens": 3895, "tokens_per_correct_answer": 162.29166666666666, "truncated": 0 }, { "config": "E_alt_model_3090_nothink", "backend": "llamacpp-3090", "cases": 24, "accuracy": 1.0, "mean_latency_s": 0.6691355095000043, "median_latency_s": 0.6749590405000845, "p95_latency_s": 0.6948294680000799, "mean_prompt_tokens": 159.45833333333334, "mean_completion_tokens": 2.8333333333333335, "mean_reasoning_tokens": 0.0, "total_tokens": 3895, "tokens_per_correct_answer": 162.29166666666666, "truncated": 0 } ], "rows": [ { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-01", "completion_tokens": 103, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.806970764000198, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 459, "reasoning_chars": 437, "reasoning_tokens": 100, "total_tokens": 562 }, { "answer": "\n\nTHERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-02", "completion_tokens": 101, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.41505831200015564, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "THERMAL", "prompt_tokens": 457, "reasoning_chars": 416, "reasoning_tokens": 96, "total_tokens": 558 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-03", "completion_tokens": 161, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.7899744530000135, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 456, "reasoning_chars": 640, "reasoning_tokens": 158, "total_tokens": 617 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-04", "completion_tokens": 97, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.44665630800000145, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 457, "reasoning_chars": 435, "reasoning_tokens": 93, "total_tokens": 554 }, { "answer": "\n\nINTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-05", "completion_tokens": 99, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.4368077940000603, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 459, "reasoning_chars": 411, "reasoning_tokens": 95, "total_tokens": 558 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-06", "completion_tokens": 88, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.4203472009999132, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 455, "reasoning_chars": 383, "reasoning_tokens": 84, "total_tokens": 543 }, { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-07", "completion_tokens": 164, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.5464556729998549, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 448, "reasoning_chars": 736, "reasoning_tokens": 161, "total_tokens": 612 }, { "answer": "\n\nTHERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-08", "completion_tokens": 199, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 1.1140807669999049, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "THERMAL", "prompt_tokens": 455, "reasoning_chars": 835, "reasoning_tokens": 194, "total_tokens": 654 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-09", "completion_tokens": 76, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.4018020109999725, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 447, "reasoning_chars": 374, "reasoning_tokens": 73, "total_tokens": 523 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-10", "completion_tokens": 99, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.5644840970001042, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 453, "reasoning_chars": 475, "reasoning_tokens": 95, "total_tokens": 552 }, { "answer": "\n\nINTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-11", "completion_tokens": 89, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.39491011900008743, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 452, "reasoning_chars": 407, "reasoning_tokens": 85, "total_tokens": 541 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-12", "completion_tokens": 107, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.5076981920001344, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 450, "reasoning_chars": 520, "reasoning_tokens": 103, "total_tokens": 557 }, { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-13", "completion_tokens": 188, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.7146771039999749, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 447, "reasoning_chars": 894, "reasoning_tokens": 185, "total_tokens": 635 }, { "answer": "\n\nTHERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-14", "completion_tokens": 95, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.5337014919998637, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "THERMAL", "prompt_tokens": 450, "reasoning_chars": 428, "reasoning_tokens": 90, "total_tokens": 545 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-15", "completion_tokens": 67, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.31145577000006597, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 445, "reasoning_chars": 326, "reasoning_tokens": 64, "total_tokens": 512 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-16", "completion_tokens": 86, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.4489611009998953, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 448, "reasoning_chars": 410, "reasoning_tokens": 82, "total_tokens": 534 }, { "answer": "\n\nINTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-17", "completion_tokens": 213, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.7904420329998629, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 445, "reasoning_chars": 918, "reasoning_tokens": 209, "total_tokens": 658 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-18", "completion_tokens": 129, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.5622826950000217, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 454, "reasoning_chars": 602, "reasoning_tokens": 125, "total_tokens": 583 }, { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-19", "completion_tokens": 182, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 1.0275071900000512, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 452, "reasoning_chars": 844, "reasoning_tokens": 179, "total_tokens": 634 }, { "answer": "\n\nTHERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-20", "completion_tokens": 156, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.8175055359999988, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "THERMAL", "prompt_tokens": 446, "reasoning_chars": 665, "reasoning_tokens": 151, "total_tokens": 602 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-21", "completion_tokens": 109, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.43744410899989816, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 451, "reasoning_chars": 490, "reasoning_tokens": 105, "total_tokens": 560 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-22", "completion_tokens": 256, "expected_phrase": "INTERCONNECT", "finish_reason": "length", "latency_seconds": 1.0255932009999924, "model": "qwen38-27b", "name": "A_full_context_256", "passed": false, "predicted": null, "prompt_tokens": 453, "reasoning_chars": 1136, "reasoning_tokens": 257, "total_tokens": 709 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-23", "completion_tokens": 86, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.3491142600000785, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 456, "reasoning_chars": 348, "reasoning_tokens": 83, "total_tokens": 542 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-24", "completion_tokens": 107, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.429487133000066, "model": "qwen38-27b", "name": "A_full_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 448, "reasoning_chars": 486, "reasoning_tokens": 103, "total_tokens": 555 }, { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-01", "completion_tokens": 95, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.48948159699989446, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 196, "reasoning_chars": 402, "reasoning_tokens": 92, "total_tokens": 291 }, { "answer": "\n\nTHERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-02", "completion_tokens": 159, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.5815049770001224, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "THERMAL", "prompt_tokens": 196, "reasoning_chars": 659, "reasoning_tokens": 154, "total_tokens": 355 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-03", "completion_tokens": 118, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.5333530650000284, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 189, "reasoning_chars": 501, "reasoning_tokens": 115, "total_tokens": 307 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-04", "completion_tokens": 60, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.32462793299987425, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 196, "reasoning_chars": 292, "reasoning_tokens": 56, "total_tokens": 256 }, { "answer": "\n\nINTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-05", "completion_tokens": 97, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.3916921889999685, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 205, "reasoning_chars": 412, "reasoning_tokens": 93, "total_tokens": 302 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-06", "completion_tokens": 88, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.4137433809999038, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 193, "reasoning_chars": 378, "reasoning_tokens": 84, "total_tokens": 281 }, { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-07", "completion_tokens": 152, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.5375769510001192, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 185, "reasoning_chars": 681, "reasoning_tokens": 149, "total_tokens": 337 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-08", "completion_tokens": 256, "expected_phrase": "THERMAL", "finish_reason": "length", "latency_seconds": 1.0860960269999396, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": false, "predicted": null, "prompt_tokens": 194, "reasoning_chars": 1130, "reasoning_tokens": 258, "total_tokens": 450 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-09", "completion_tokens": 113, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.5516248459998678, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 180, "reasoning_chars": 553, "reasoning_tokens": 110, "total_tokens": 293 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-10", "completion_tokens": 62, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.3325878670000293, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 192, "reasoning_chars": 288, "reasoning_tokens": 58, "total_tokens": 254 }, { "answer": "\n\nINTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-11", "completion_tokens": 92, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.4504374079999707, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 198, "reasoning_chars": 418, "reasoning_tokens": 88, "total_tokens": 290 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-12", "completion_tokens": 121, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.5528517320001356, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 188, "reasoning_chars": 563, "reasoning_tokens": 117, "total_tokens": 309 }, { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-13", "completion_tokens": 152, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.5543328280000424, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 184, "reasoning_chars": 689, "reasoning_tokens": 149, "total_tokens": 336 }, { "answer": "\n\nTHERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-14", "completion_tokens": 166, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.5586219340000298, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "THERMAL", "prompt_tokens": 189, "reasoning_chars": 710, "reasoning_tokens": 161, "total_tokens": 355 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-15", "completion_tokens": 66, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.37781106599982195, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 178, "reasoning_chars": 321, "reasoning_tokens": 63, "total_tokens": 244 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-16", "completion_tokens": 107, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.5006653939999524, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 187, "reasoning_chars": 537, "reasoning_tokens": 103, "total_tokens": 294 }, { "answer": "\n\nINTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-17", "completion_tokens": 92, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.43569015899993246, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 191, "reasoning_chars": 419, "reasoning_tokens": 88, "total_tokens": 283 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-18", "completion_tokens": 134, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.6429583690000982, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 192, "reasoning_chars": 626, "reasoning_tokens": 130, "total_tokens": 326 }, { "answer": "\n\nMEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-19", "completion_tokens": 125, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.5549225549998482, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "MEMORY", "prompt_tokens": 189, "reasoning_chars": 572, "reasoning_tokens": 122, "total_tokens": 314 }, { "answer": "\n\nTHERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-20", "completion_tokens": 250, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 1.0765945619998547, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "THERMAL", "prompt_tokens": 185, "reasoning_chars": 1134, "reasoning_tokens": 245, "total_tokens": 435 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-21", "completion_tokens": 73, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.4664710320000722, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 184, "reasoning_chars": 333, "reasoning_tokens": 69, "total_tokens": 257 }, { "answer": "\n\nINTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-22", "completion_tokens": 183, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.8829875769999944, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 192, "reasoning_chars": 842, "reasoning_tokens": 179, "total_tokens": 375 }, { "answer": "\n\nPOWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-23", "completion_tokens": 98, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.4808250710000266, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "POWER", "prompt_tokens": 202, "reasoning_chars": 393, "reasoning_tokens": 95, "total_tokens": 300 }, { "answer": "\n\nSTORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-24", "completion_tokens": 105, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.5499935670000013, "model": "qwen38-27b", "name": "B_reduced_context_256", "passed": true, "predicted": "STORAGE", "prompt_tokens": 186, "reasoning_chars": 482, "reasoning_tokens": 101, "total_tokens": 291 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-01", "completion_tokens": 64, "expected_phrase": "MEMORY", "finish_reason": "length", "latency_seconds": 0.35552689099995405, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 196, "reasoning_chars": 270, "reasoning_tokens": 69, "total_tokens": 260 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-02", "completion_tokens": 64, "expected_phrase": "THERMAL", "finish_reason": "length", "latency_seconds": 0.30353903299987905, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 196, "reasoning_chars": 265, "reasoning_tokens": 70, "total_tokens": 260 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-03", "completion_tokens": 64, "expected_phrase": "POWER", "finish_reason": "length", "latency_seconds": 0.3149659939999765, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 189, "reasoning_chars": 285, "reasoning_tokens": 66, "total_tokens": 253 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-04", "completion_tokens": 60, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.3146454380000705, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 196, "reasoning_chars": 292, "reasoning_tokens": 56, "total_tokens": 256 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-05", "completion_tokens": 64, "expected_phrase": "INTERCONNECT", "finish_reason": "length", "latency_seconds": 0.2914040529999511, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 205, "reasoning_chars": 277, "reasoning_tokens": 69, "total_tokens": 269 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-06", "completion_tokens": 64, "expected_phrase": "STORAGE", "finish_reason": "length", "latency_seconds": 0.3319779070000095, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 193, "reasoning_chars": 285, "reasoning_tokens": 64, "total_tokens": 257 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-07", "completion_tokens": 64, "expected_phrase": "MEMORY", "finish_reason": "length", "latency_seconds": 0.2952752200001214, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 185, "reasoning_chars": 293, "reasoning_tokens": 67, "total_tokens": 249 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-08", "completion_tokens": 64, "expected_phrase": "THERMAL", "finish_reason": "length", "latency_seconds": 0.2472051489999103, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 194, "reasoning_chars": 274, "reasoning_tokens": 64, "total_tokens": 258 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-09", "completion_tokens": 64, "expected_phrase": "POWER", "finish_reason": "length", "latency_seconds": 0.30267094700002417, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 180, "reasoning_chars": 315, "reasoning_tokens": 71, "total_tokens": 244 }, { "answer": "\n\nFIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-10", "completion_tokens": 62, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.36072664499988605, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 192, "reasoning_chars": 288, "reasoning_tokens": 58, "total_tokens": 254 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-11", "completion_tokens": 64, "expected_phrase": "INTERCONNECT", "finish_reason": "length", "latency_seconds": 0.3739699989998826, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 198, "reasoning_chars": 318, "reasoning_tokens": 65, "total_tokens": 262 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-12", "completion_tokens": 64, "expected_phrase": "STORAGE", "finish_reason": "length", "latency_seconds": 0.3283452480000051, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 188, "reasoning_chars": 313, "reasoning_tokens": 68, "total_tokens": 252 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-13", "completion_tokens": 64, "expected_phrase": "MEMORY", "finish_reason": "length", "latency_seconds": 0.27336783299983836, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 184, "reasoning_chars": 295, "reasoning_tokens": 67, "total_tokens": 248 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-14", "completion_tokens": 64, "expected_phrase": "THERMAL", "finish_reason": "length", "latency_seconds": 0.25897128099995825, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 189, "reasoning_chars": 297, "reasoning_tokens": 66, "total_tokens": 253 }, { "answer": "\n\n", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-15", "completion_tokens": 64, "expected_phrase": "POWER", "finish_reason": "length", "latency_seconds": 0.3503787849999753, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 178, "reasoning_chars": 321, "reasoning_tokens": 63, "total_tokens": 242 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-16", "completion_tokens": 64, "expected_phrase": "FIRMWARE", "finish_reason": "length", "latency_seconds": 0.3061420520000411, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 187, "reasoning_chars": 353, "reasoning_tokens": 65, "total_tokens": 251 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-17", "completion_tokens": 64, "expected_phrase": "INTERCONNECT", "finish_reason": "length", "latency_seconds": 0.3372814809999909, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 191, "reasoning_chars": 321, "reasoning_tokens": 65, "total_tokens": 255 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-18", "completion_tokens": 64, "expected_phrase": "STORAGE", "finish_reason": "length", "latency_seconds": 0.3716653999999835, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 192, "reasoning_chars": 292, "reasoning_tokens": 64, "total_tokens": 256 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-19", "completion_tokens": 64, "expected_phrase": "MEMORY", "finish_reason": "length", "latency_seconds": 0.3619956899999579, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 189, "reasoning_chars": 295, "reasoning_tokens": 69, "total_tokens": 253 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-20", "completion_tokens": 64, "expected_phrase": "THERMAL", "finish_reason": "length", "latency_seconds": 0.26737381499992807, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 185, "reasoning_chars": 277, "reasoning_tokens": 67, "total_tokens": 249 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-21", "completion_tokens": 64, "expected_phrase": "FIRMWARE", "finish_reason": "length", "latency_seconds": 0.44028525199996693, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 184, "reasoning_chars": 322, "reasoning_tokens": 65, "total_tokens": 248 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-22", "completion_tokens": 64, "expected_phrase": "INTERCONNECT", "finish_reason": "length", "latency_seconds": 0.32432402800009186, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 192, "reasoning_chars": 278, "reasoning_tokens": 67, "total_tokens": 256 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-23", "completion_tokens": 64, "expected_phrase": "POWER", "finish_reason": "length", "latency_seconds": 0.3514541330000611, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 202, "reasoning_chars": 273, "reasoning_tokens": 64, "total_tokens": 266 }, { "answer": "", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-24", "completion_tokens": 64, "expected_phrase": "STORAGE", "finish_reason": "length", "latency_seconds": 0.36108775899992906, "model": "qwen38-27b", "name": "C_reduced_context_64", "passed": false, "predicted": null, "prompt_tokens": 186, "reasoning_chars": 326, "reasoning_tokens": 66, "total_tokens": 250 }, { "answer": "MEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-01", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.08296578099998442, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 165, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 167 }, { "answer": "THERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-02", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.06809483600000021, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 165, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 169 }, { "answer": "POWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-03", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.08149678299992047, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 158, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 160 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-04", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.07166909700003998, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 165, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 168 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-05", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.08487199000001056, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 174, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 177 }, { "answer": "STORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-06", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.06838042999993377, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 162, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 165 }, { "answer": "MEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-07", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.07043322599997737, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 154, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 156 }, { "answer": "THERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-08", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.07019520900007592, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 163, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 167 }, { "answer": "POWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-09", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.08061563100000058, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 149, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 151 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-10", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.0852674879999995, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 161, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 164 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-11", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.08968185300000187, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 167, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 170 }, { "answer": "STORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-12", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.07842827900003613, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 157, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 160 }, { "answer": "MEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-13", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.09140553600013845, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 153, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 155 }, { "answer": "THERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-14", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.08330007299991848, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 158, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 162 }, { "answer": "POWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-15", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.08730913299996246, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 147, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 149 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-16", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.07401208099986434, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 156, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 159 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-17", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.07725680399994417, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 160, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 163 }, { "answer": "STORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-18", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.07534162999991167, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 161, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 164 }, { "answer": "MEMORY", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-19", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.06506846399997812, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 158, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 160 }, { "answer": "THERMAL", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-20", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.08476940499986085, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 154, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 158 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-21", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.10721606400011297, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 153, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 156 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-22", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.08800160500004495, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 161, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 164 }, { "answer": "POWER", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-23", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.07092879600008928, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 171, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 173 }, { "answer": "STORAGE", "attempt": 1, "backend": "sglang-rtx6000", "case_id": "case-24", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.08675309599993852, "model": "qwen38-27b", "name": "D_specialized_prompt_64_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 155, "reasoning_chars": 0, "reasoning_tokens": 0, "total_tokens": 158 }, { "answer": "MEMORY", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-01", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.5879698940000253, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 165, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 167 }, { "answer": "THERMAL", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-02", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.6691138670000782, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 165, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 169 }, { "answer": "POWER", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-03", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.6646554769999966, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 158, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 160 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-04", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.6896330509998734, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 165, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 168 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-05", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.6848347809998359, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 174, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 177 }, { "answer": "STORAGE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-06", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.6948294680000799, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 162, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 165 }, { "answer": "MEMORY", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-07", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.6447644360000595, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 154, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 156 }, { "answer": "THERMAL", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-08", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.6879504900000484, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 163, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 167 }, { "answer": "POWER", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-09", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.6274231370000507, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 149, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 151 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-10", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.6667481560000397, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 161, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 164 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-11", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.6753933580000648, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 167, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 170 }, { "answer": "STORAGE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-12", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.6742796700000326, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 157, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 160 }, { "answer": "MEMORY", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-13", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.6436287659998925, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 153, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 155 }, { "answer": "THERMAL", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-14", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.7004446430000826, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 158, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 162 }, { "answer": "POWER", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-15", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.651967845000172, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 147, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 149 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-16", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.6827831410000726, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 156, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 159 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-17", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.6925507500000094, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 160, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 163 }, { "answer": "STORAGE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-18", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.6801706859998831, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 161, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 164 }, { "answer": "MEMORY", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-19", "completion_tokens": 2, "expected_phrase": "MEMORY", "finish_reason": "stop", "latency_seconds": 0.6545313039998746, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "MEMORY", "prompt_tokens": 158, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 160 }, { "answer": "THERMAL", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-20", "completion_tokens": 4, "expected_phrase": "THERMAL", "finish_reason": "stop", "latency_seconds": 0.6924319909999213, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "THERMAL", "prompt_tokens": 154, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 158 }, { "answer": "FIRMWARE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-21", "completion_tokens": 3, "expected_phrase": "FIRMWARE", "finish_reason": "stop", "latency_seconds": 0.6745247230001041, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "FIRMWARE", "prompt_tokens": 153, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 156 }, { "answer": "INTERCONNECT", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-22", "completion_tokens": 3, "expected_phrase": "INTERCONNECT", "finish_reason": "stop", "latency_seconds": 0.6784132939999381, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "INTERCONNECT", "prompt_tokens": 161, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 164 }, { "answer": "POWER", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-23", "completion_tokens": 2, "expected_phrase": "POWER", "finish_reason": "stop", "latency_seconds": 0.6576203810000152, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "POWER", "prompt_tokens": 171, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 173 }, { "answer": "STORAGE", "attempt": 1, "backend": "llamacpp-3090", "case_id": "case-24", "completion_tokens": 3, "expected_phrase": "STORAGE", "finish_reason": "stop", "latency_seconds": 0.6825889189999543, "model": "Qwen3.8-27B", "name": "E_alt_model_3090_nothink", "passed": true, "predicted": "STORAGE", "prompt_tokens": 155, "reasoning_chars": 0, "reasoning_tokens": null, "total_tokens": 158 } ] }