{ "generated_at": "2026-03-05T20:14:26.181245+00:00", "models": [ { "model_id": "inception/mercury-2", "model_slug": "inception-mercury-2", "model_label": "inception/mercury-2", "overall_score": 64, "overall_mark": "F", "avg_output_tokens": 830.87, "avg_dom_elements": 17.87, "run": { "run_started_at": "2026-03-05T20:52:34.690656", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 0.35, "avg_warning_count": 3.64, "avg_output_tokens": 645.29, "avg_dom_elements": 15.69, "zero_error_count": 122, "relative_error_burden_pct": 46.85, "relative_warning_burden_pct": 99.27, "share_of_model_errors_pct": 19.48, "error_share_within_issues_pct": 8.7, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 81.33, "score": 64, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 81.33, "B": 16.67, "C": 0.67, "D": 0.67, "E": 0.0, "F": 0.67 } }, { "label": "Little guidance", "avg_error_count": 0.69, "avg_warning_count": 2.86, "avg_output_tokens": 778.1, "avg_dom_elements": 17.51, "zero_error_count": 94, "relative_error_burden_pct": 93.69, "relative_warning_burden_pct": 78.0, "share_of_model_errors_pct": 38.95, "error_share_within_issues_pct": 19.51, "improvement_vs_unguided_pct": -100.0, "zero_error_share_pct": 62.67, "score": 44, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 62.67, "B": 31.33, "C": 4.0, "D": 1.33, "E": 0.67, "F": 0.0 } }, { "label": "Expert guidance", "avg_error_count": 0.74, "avg_warning_count": 3.67, "avg_output_tokens": 1063.41, "avg_dom_elements": 20.41, "zero_error_count": 94, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 41.57, "error_share_within_issues_pct": 16.79, "improvement_vs_unguided_pct": -113.46, "zero_error_share_pct": 62.67, "score": 46, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 62.67, "B": 29.33, "C": 4.67, "D": 2.67, "E": 0.67, "F": 0.0 } } ] }, { "model_id": "openai/gpt-5-nano", "model_slug": "openai-gpt-5-nano", "model_label": "openai/gpt-5-nano", "overall_score": 62, "overall_mark": "F", "avg_output_tokens": 2270.91, "avg_dom_elements": 31.2, "run": { "run_started_at": "2026-02-20T03:40:10.028119", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 1.26, "avg_warning_count": 0.96, "avg_output_tokens": 2159.17, "avg_dom_elements": 28.39, "zero_error_count": 81, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 86.23, "share_of_model_errors_pct": 49.87, "error_share_within_issues_pct": 56.76, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 54.0, "score": 38, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 54.0, "B": 36.67, "C": 5.33, "D": 0.67, "E": 0.67, "F": 2.67 } }, { "label": "Little guidance", "avg_error_count": 0.43, "avg_warning_count": 1.11, "avg_output_tokens": 2507.94, "avg_dom_elements": 32.94, "zero_error_count": 116, "relative_error_burden_pct": 34.39, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 17.15, "error_share_within_issues_pct": 28.02, "improvement_vs_unguided_pct": 65.61, "zero_error_share_pct": 77.33, "score": 70, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 77.33, "B": 20.0, "C": 2.0, "D": 0.0, "E": 0.0, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 0.84, "avg_warning_count": 0.94, "avg_output_tokens": 2143.93, "avg_dom_elements": 32.3, "zero_error_count": 97, "relative_error_burden_pct": 67.03, "relative_warning_burden_pct": 84.36, "share_of_model_errors_pct": 32.98, "error_share_within_issues_pct": 47.35, "improvement_vs_unguided_pct": 32.97, "zero_error_share_pct": 65.54, "score": 53, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 65.54, "B": 25.0, "C": 5.41, "D": 2.7, "E": 0.68, "F": 0.68 } } ] }, { "model_id": "openai/gpt-3.5-turbo", "model_slug": "openai-gpt-3-5-turbo", "model_label": "openai/gpt-3.5-turbo", "overall_score": 49, "overall_mark": "F", "avg_output_tokens": 355.99, "avg_dom_elements": 14.69, "run": { "run_started_at": "2026-02-20T02:42:49.578237", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 1.4, "avg_warning_count": 0.25, "avg_output_tokens": 328.71, "avg_dom_elements": 13.47, "zero_error_count": 75, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 44.4, "error_share_within_issues_pct": 85.02, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 50.0, "score": 26, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 50.0, "B": 34.0, "C": 10.0, "D": 1.33, "E": 4.67, "F": 0.0 } }, { "label": "Little guidance", "avg_error_count": 0.86, "avg_warning_count": 0.21, "avg_output_tokens": 364.54, "avg_dom_elements": 15.29, "zero_error_count": 97, "relative_error_burden_pct": 61.43, "relative_warning_burden_pct": 86.49, "share_of_model_errors_pct": 27.27, "error_share_within_issues_pct": 80.12, "improvement_vs_unguided_pct": 38.57, "zero_error_share_pct": 64.67, "score": 39, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 64.67, "B": 28.0, "C": 5.33, "D": 0.0, "E": 2.0, "F": 0.0 } }, { "label": "Expert guidance", "avg_error_count": 0.91, "avg_warning_count": 0.2, "avg_output_tokens": 374.97, "avg_dom_elements": 15.31, "zero_error_count": 88, "relative_error_burden_pct": 64.67, "relative_warning_burden_pct": 79.44, "share_of_model_errors_pct": 28.33, "error_share_within_issues_pct": 82.21, "improvement_vs_unguided_pct": 35.33, "zero_error_share_pct": 59.46, "score": 36, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 59.46, "B": 32.43, "C": 5.41, "D": 0.0, "E": 2.7, "F": 0.0 } } ] }, { "model_id": "openai/gpt-5.1-codex-mini", "model_slug": "openai-gpt-5-1-codex-mini", "model_label": "openai/gpt-5.1-codex-mini", "overall_score": 46, "overall_mark": "F", "avg_output_tokens": 1273.82, "avg_dom_elements": 23.87, "run": { "run_started_at": "2026-02-19T22:57:32.235557", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 2.05, "avg_warning_count": 0.55, "avg_output_tokens": 1124.73, "avg_dom_elements": 20.95, "zero_error_count": 51, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 90.22, "share_of_model_errors_pct": 52.66, "error_share_within_issues_pct": 78.72, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 34.0, "score": 19, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 34.0, "B": 34.0, "C": 20.0, "D": 6.67, "E": 5.33, "F": 0.0 } }, { "label": "Little guidance", "avg_error_count": 0.93, "avg_warning_count": 0.61, "avg_output_tokens": 1402.59, "avg_dom_elements": 24.64, "zero_error_count": 87, "relative_error_burden_pct": 45.28, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 23.84, "error_share_within_issues_pct": 60.17, "improvement_vs_unguided_pct": 54.72, "zero_error_share_pct": 58.0, "score": 43, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 58.0, "B": 32.67, "C": 7.33, "D": 1.33, "E": 0.0, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 0.93, "avg_warning_count": 0.61, "avg_output_tokens": 1294.54, "avg_dom_elements": 26.05, "zero_error_count": 88, "relative_error_burden_pct": 45.54, "relative_warning_burden_pct": 98.71, "share_of_model_errors_pct": 23.5, "error_share_within_issues_pct": 60.62, "improvement_vs_unguided_pct": 54.46, "zero_error_share_pct": 59.86, "score": 44, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "A", "mark_distribution_pct": { "A": 59.86, "B": 31.29, "C": 6.12, "D": 2.04, "E": 0.68, "F": 0.0 } } ] }, { "model_id": "openai/gpt-5.3-codex", "model_slug": "openai-gpt-5-3-codex", "model_label": "openai/gpt-5.3-codex", "overall_score": 38, "overall_mark": "F", "avg_output_tokens": 1631.58, "avg_dom_elements": 24.75, "run": { "run_started_at": "2026-02-25T21:35:20.239366", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 1.93, "avg_warning_count": 4.15, "avg_output_tokens": 1612.89, "avg_dom_elements": 21.69, "zero_error_count": 35, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 40.28, "error_share_within_issues_pct": 31.8, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 23.33, "score": 14, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 23.33, "B": 60.67, "C": 12.0, "D": 1.33, "E": 2.0, "F": 0.67 } }, { "label": "Little guidance", "avg_error_count": 1.48, "avg_warning_count": 2.21, "avg_output_tokens": 1733.11, "avg_dom_elements": 26.45, "zero_error_count": 55, "relative_error_burden_pct": 76.55, "relative_warning_burden_pct": 53.38, "share_of_model_errors_pct": 30.83, "error_share_within_issues_pct": 40.07, "improvement_vs_unguided_pct": 23.45, "zero_error_share_pct": 36.67, "score": 26, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 36.67, "B": 50.0, "C": 10.0, "D": 2.0, "E": 0.67, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 1.39, "avg_warning_count": 3.42, "avg_output_tokens": 1548.73, "avg_dom_elements": 26.11, "zero_error_count": 47, "relative_error_burden_pct": 71.72, "relative_warning_burden_pct": 82.48, "share_of_model_errors_pct": 28.89, "error_share_within_issues_pct": 28.85, "improvement_vs_unguided_pct": 28.28, "zero_error_share_pct": 31.33, "score": 24, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 31.33, "B": 55.33, "C": 12.67, "D": 0.67, "E": 0.0, "F": 0.0 } } ] }, { "model_id": "openai/gpt-oss-20b", "model_slug": "openai-gpt-oss-20b", "model_label": "openai/gpt-oss-20b", "overall_score": 28, "overall_mark": "F", "avg_output_tokens": 616.26, "avg_dom_elements": 17.4, "run": { "run_started_at": "2026-02-20T02:20:47.395548", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 2.3, "avg_warning_count": 0.35, "avg_output_tokens": 499.3, "avg_dom_elements": 15.82, "zero_error_count": 10, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 81.71, "share_of_model_errors_pct": 39.25, "error_share_within_issues_pct": 86.68, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 6.67, "score": 4, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 6.67, "B": 51.33, "C": 33.33, "D": 4.67, "E": 3.33, "F": 0.67 } }, { "label": "Little guidance", "avg_error_count": 1.88, "avg_warning_count": 0.36, "avg_output_tokens": 652.15, "avg_dom_elements": 17.22, "zero_error_count": 17, "relative_error_burden_pct": 81.74, "relative_warning_burden_pct": 83.25, "share_of_model_errors_pct": 32.08, "error_share_within_issues_pct": 83.93, "improvement_vs_unguided_pct": 18.26, "zero_error_share_pct": 11.33, "score": 7, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 11.33, "B": 60.0, "C": 22.67, "D": 2.67, "E": 2.0, "F": 1.33 } }, { "label": "Expert guidance", "avg_error_count": 1.7, "avg_warning_count": 0.43, "avg_output_tokens": 698.43, "avg_dom_elements": 19.18, "zero_error_count": 28, "relative_error_burden_pct": 74.03, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 28.67, "error_share_within_issues_pct": 79.75, "improvement_vs_unguided_pct": 25.97, "zero_error_share_pct": 18.92, "score": 12, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 18.92, "B": 56.76, "C": 19.59, "D": 2.03, "E": 2.03, "F": 0.68 } } ] }, { "model_id": "openai/gpt-4o-2024-11-20", "model_slug": "openai-gpt-4o-2024-11-20", "model_label": "openai/gpt-4o-2024-11-20", "overall_score": 25, "overall_mark": "F", "avg_output_tokens": 616.13, "avg_dom_elements": 16.42, "run": { "run_started_at": "2026-02-20T03:11:50.598128", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 2.48, "avg_warning_count": 0.35, "avg_output_tokens": 529.45, "avg_dom_elements": 14.97, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 41.15, "error_share_within_issues_pct": 87.74, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 0, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 62.0, "C": 29.33, "D": 4.67, "E": 2.67, "F": 1.33 } }, { "label": "Little guidance", "avg_error_count": 1.73, "avg_warning_count": 0.35, "avg_output_tokens": 681.17, "avg_dom_elements": 16.41, "zero_error_count": 0, "relative_error_burden_pct": 69.62, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 28.65, "error_share_within_issues_pct": 83.28, "improvement_vs_unguided_pct": 30.38, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 80.0, "C": 16.0, "D": 2.67, "E": 1.33, "F": 0.0 } }, { "label": "Expert guidance", "avg_error_count": 1.84, "avg_warning_count": 0.24, "avg_output_tokens": 638.06, "avg_dom_elements": 17.89, "zero_error_count": 0, "relative_error_burden_pct": 74.38, "relative_warning_burden_pct": 68.22, "share_of_model_errors_pct": 30.2, "error_share_within_issues_pct": 88.64, "improvement_vs_unguided_pct": 25.62, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 77.7, "C": 16.22, "D": 2.7, "E": 3.38, "F": 0.0 } } ] }, { "model_id": "openai/gpt-5.2", "model_slug": "openai-gpt-5-2", "model_label": "openai/gpt-5.2", "overall_score": 25, "overall_mark": "F", "avg_output_tokens": 3789.77, "avg_dom_elements": 48.43, "run": { "run_started_at": "2026-02-19T21:23:42.405101", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.48, "avg_warning_count": 2.26, "avg_output_tokens": 4414.03, "avg_dom_elements": 46.06, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 44.27, "error_share_within_issues_pct": 60.58, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 4, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 52.35, "C": 29.53, "D": 9.4, "E": 4.03, "F": 4.7 } }, { "label": "Little guidance", "avg_error_count": 1.98, "avg_warning_count": 1.53, "avg_output_tokens": 3755.67, "avg_dom_elements": 45.85, "zero_error_count": 0, "relative_error_burden_pct": 56.95, "relative_warning_burden_pct": 67.5, "share_of_model_errors_pct": 25.38, "error_share_within_issues_pct": 56.46, "improvement_vs_unguided_pct": 43.05, "zero_error_share_pct": 0.0, "score": 11, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 74.67, "C": 19.33, "D": 1.33, "E": 3.33, "F": 1.33 } }, { "label": "Expert guidance", "avg_error_count": 2.4, "avg_warning_count": 2.05, "avg_output_tokens": 3195.84, "avg_dom_elements": 53.41, "zero_error_count": 0, "relative_error_burden_pct": 69.0, "relative_warning_burden_pct": 90.52, "share_of_model_errors_pct": 30.34, "error_share_within_issues_pct": 53.95, "improvement_vs_unguided_pct": 31.0, "zero_error_share_pct": 0.0, "score": 11, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 74.32, "C": 16.89, "D": 2.7, "E": 2.7, "F": 3.38 } } ] }, { "model_id": "openai/gpt-oss-120b", "model_slug": "openai-gpt-oss-120b", "model_label": "openai/gpt-oss-120b", "overall_score": 25, "overall_mark": "F", "avg_output_tokens": 850.92, "avg_dom_elements": 19.54, "run": { "run_started_at": "2026-02-20T02:18:52.415818", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 2.72, "avg_warning_count": 0.45, "avg_output_tokens": 740.85, "avg_dom_elements": 17.06, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 42.72, "error_share_within_issues_pct": 85.89, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 0, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 54.67, "C": 33.33, "D": 6.67, "E": 2.0, "F": 3.33 } }, { "label": "Little guidance", "avg_error_count": 1.83, "avg_warning_count": 0.4, "avg_output_tokens": 947.69, "avg_dom_elements": 20.09, "zero_error_count": 3, "relative_error_burden_pct": 67.16, "relative_warning_burden_pct": 89.55, "share_of_model_errors_pct": 28.69, "error_share_within_issues_pct": 82.04, "improvement_vs_unguided_pct": 32.84, "zero_error_share_pct": 2.0, "score": 3, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 2.0, "B": 66.0, "C": 28.0, "D": 1.33, "E": 2.67, "F": 0.0 } }, { "label": "Expert guidance", "avg_error_count": 1.84, "avg_warning_count": 0.36, "avg_output_tokens": 864.39, "avg_dom_elements": 21.49, "zero_error_count": 1, "relative_error_burden_pct": 67.82, "relative_warning_burden_pct": 80.17, "share_of_model_errors_pct": 28.59, "error_share_within_issues_pct": 83.74, "improvement_vs_unguided_pct": 32.18, "zero_error_share_pct": 0.68, "score": 3, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.68, "B": 66.22, "C": 29.05, "D": 0.68, "E": 3.38, "F": 0.0 } } ] }, { "model_id": "xiaomi/mimo-v2-flash", "model_slug": "xiaomi-mimo-v2-flash", "model_label": "xiaomi/mimo-v2-flash", "overall_score": 25, "overall_mark": "F", "avg_output_tokens": 2260.66, "avg_dom_elements": 32.05, "run": { "run_started_at": "2026-02-20T03:35:34.542456", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.27, "avg_warning_count": 0.5, "avg_output_tokens": 1988.14, "avg_dom_elements": 26.3, "zero_error_count": 2, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 58.14, "share_of_model_errors_pct": 46.36, "error_share_within_issues_pct": 86.75, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 1.33, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 1.33, "B": 50.0, "C": 36.0, "D": 8.67, "E": 3.33, "F": 0.67 } }, { "label": "Little guidance", "avg_error_count": 1.82, "avg_warning_count": 0.86, "avg_output_tokens": 2560.47, "avg_dom_elements": 34.39, "zero_error_count": 1, "relative_error_burden_pct": 55.6, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 25.78, "error_share_within_issues_pct": 67.91, "improvement_vs_unguided_pct": 44.4, "zero_error_share_pct": 0.67, "score": 8, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.67, "B": 71.33, "C": 23.33, "D": 3.33, "E": 1.33, "F": 0.0 } }, { "label": "Expert guidance", "avg_error_count": 1.99, "avg_warning_count": 0.4, "avg_output_tokens": 2233.0, "avg_dom_elements": 35.5, "zero_error_count": 7, "relative_error_burden_pct": 60.89, "relative_warning_burden_pct": 46.35, "share_of_model_errors_pct": 27.86, "error_share_within_issues_pct": 83.33, "improvement_vs_unguided_pct": 39.11, "zero_error_share_pct": 4.73, "score": 10, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 4.73, "B": 60.81, "C": 28.38, "D": 3.38, "E": 2.7, "F": 0.0 } } ] }, { "model_id": "deepseek/deepseek-v3.2", "model_slug": "deepseek-deepseek-v3-2", "model_label": "deepseek/deepseek-v3.2", "overall_score": 24, "overall_mark": "F", "avg_output_tokens": 2701.26, "avg_dom_elements": 37.03, "run": { "run_started_at": "2026-02-20T02:33:19.338910", "run_status": "cancelled" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.4, "avg_warning_count": 1.72, "avg_output_tokens": 2671.97, "avg_dom_elements": 35.35, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 46.58, "error_share_within_issues_pct": 66.41, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 2, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 41.33, "C": 37.33, "D": 10.67, "E": 8.0, "F": 2.67 } }, { "label": "Little guidance", "avg_error_count": 1.97, "avg_warning_count": 1.0, "avg_output_tokens": 2992.02, "avg_dom_elements": 36.09, "zero_error_count": 0, "relative_error_burden_pct": 57.84, "relative_warning_burden_pct": 58.14, "share_of_model_errors_pct": 26.76, "error_share_within_issues_pct": 66.29, "improvement_vs_unguided_pct": 42.16, "zero_error_share_pct": 0.0, "score": 8, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 75.84, "C": 18.12, "D": 3.36, "E": 2.01, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 1.99, "avg_warning_count": 0.8, "avg_output_tokens": 2436.43, "avg_dom_elements": 39.69, "zero_error_count": 0, "relative_error_burden_pct": 58.42, "relative_warning_burden_pct": 46.27, "share_of_model_errors_pct": 26.67, "error_share_within_issues_pct": 71.39, "improvement_vs_unguided_pct": 41.58, "zero_error_share_pct": 0.0, "score": 9, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 74.83, "C": 19.73, "D": 2.04, "E": 2.72, "F": 0.68 } } ] }, { "model_id": "google/gemma-3-12b-it", "model_slug": "google-gemma-3-12b-it", "model_label": "google/gemma-3-12b-it", "overall_score": 24, "overall_mark": "F", "avg_output_tokens": 813.11, "avg_dom_elements": 18.92, "run": { "run_started_at": "2026-02-20T03:02:21.339330", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 2.54, "avg_warning_count": 0.21, "avg_output_tokens": 697.49, "avg_dom_elements": 16.6, "zero_error_count": 4, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 57.41, "share_of_model_errors_pct": 37.5, "error_share_within_issues_pct": 92.25, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 2.67, "score": 2, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 2.67, "B": 58.0, "C": 33.33, "D": 5.33, "E": 0.0, "F": 0.67 } }, { "label": "Little guidance", "avg_error_count": 1.81, "avg_warning_count": 0.35, "avg_output_tokens": 881.01, "avg_dom_elements": 19.22, "zero_error_count": 3, "relative_error_burden_pct": 71.13, "relative_warning_burden_pct": 93.28, "share_of_model_errors_pct": 26.67, "error_share_within_issues_pct": 83.9, "improvement_vs_unguided_pct": 28.87, "zero_error_share_pct": 2.0, "score": 3, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 2.0, "B": 67.33, "C": 28.0, "D": 1.33, "E": 0.67, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 2.46, "avg_warning_count": 0.37, "avg_output_tokens": 861.47, "avg_dom_elements": 20.98, "zero_error_count": 0, "relative_error_burden_pct": 96.83, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 35.83, "error_share_within_issues_pct": 86.87, "improvement_vs_unguided_pct": 3.17, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 63.51, "C": 27.7, "D": 4.05, "E": 2.7, "F": 2.03 } } ] }, { "model_id": "anthropic/claude-3-haiku", "model_slug": "anthropic-claude-3-haiku", "model_label": "anthropic/claude-3-haiku", "overall_score": 23, "overall_mark": "F", "avg_output_tokens": 883.4, "avg_dom_elements": 17.04, "run": { "run_started_at": "2026-02-20T02:53:30.311116", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 2.45, "avg_warning_count": 0.3, "avg_output_tokens": 745.91, "avg_dom_elements": 14.96, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 70.48, "share_of_model_errors_pct": 36.77, "error_share_within_issues_pct": 89.08, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 0, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 56.0, "C": 38.0, "D": 3.33, "E": 1.33, "F": 1.33 } }, { "label": "Little guidance", "avg_error_count": 2.03, "avg_warning_count": 0.24, "avg_output_tokens": 944.85, "avg_dom_elements": 17.53, "zero_error_count": 0, "relative_error_burden_pct": 83.12, "relative_warning_burden_pct": 56.76, "share_of_model_errors_pct": 30.36, "error_share_within_issues_pct": 89.38, "improvement_vs_unguided_pct": 16.88, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 67.79, "C": 28.86, "D": 1.34, "E": 1.34, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 2.22, "avg_warning_count": 0.43, "avg_output_tokens": 960.87, "avg_dom_elements": 18.64, "zero_error_count": 0, "relative_error_burden_pct": 90.58, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 32.87, "error_share_within_issues_pct": 83.89, "improvement_vs_unguided_pct": 9.42, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 67.57, "C": 25.68, "D": 1.35, "E": 4.73, "F": 0.68 } } ] }, { "model_id": "google/gemini-3-flash-preview", "model_slug": "google-gemini-3-flash-preview", "model_label": "google/gemini-3-flash-preview", "overall_score": 23, "overall_mark": "F", "avg_output_tokens": 1168.64, "avg_dom_elements": 20.36, "run": { "run_started_at": "2026-02-20T01:16:52.379305", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.43, "avg_warning_count": 0.53, "avg_output_tokens": 993.9, "avg_dom_elements": 17.59, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 49.57, "error_share_within_issues_pct": 86.7, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 0, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 45.33, "C": 34.0, "D": 12.67, "E": 4.67, "F": 3.33 } }, { "label": "Little guidance", "avg_error_count": 1.73, "avg_warning_count": 0.34, "avg_output_tokens": 1356.41, "avg_dom_elements": 21.46, "zero_error_count": 0, "relative_error_burden_pct": 50.29, "relative_warning_burden_pct": 64.56, "share_of_model_errors_pct": 24.93, "error_share_within_issues_pct": 83.55, "improvement_vs_unguided_pct": 49.71, "zero_error_share_pct": 0.0, "score": 3, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 69.33, "C": 28.67, "D": 2.0, "E": 0.0, "F": 0.0 } }, { "label": "Expert guidance", "avg_error_count": 1.79, "avg_warning_count": 0.45, "avg_output_tokens": 1155.43, "avg_dom_elements": 22.07, "zero_error_count": 0, "relative_error_burden_pct": 52.15, "relative_warning_burden_pct": 85.96, "share_of_model_errors_pct": 25.51, "error_share_within_issues_pct": 79.82, "improvement_vs_unguided_pct": 47.85, "zero_error_share_pct": 0.0, "score": 3, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 71.62, "C": 22.97, "D": 4.05, "E": 1.35, "F": 0.0 } } ] }, { "model_id": "qwen/qwen3-coder-next", "model_slug": "qwen-qwen3-coder-next", "model_label": "qwen/qwen3-coder-next", "overall_score": 23, "overall_mark": "F", "avg_output_tokens": 2665.93, "avg_dom_elements": 38.35, "run": { "run_started_at": "2026-02-20T01:08:09.141791", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.43, "avg_warning_count": 0.86, "avg_output_tokens": 2486.57, "avg_dom_elements": 34.73, "zero_error_count": 1, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 44.2, "error_share_within_issues_pct": 79.94, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.67, "score": 2, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.67, "B": 51.33, "C": 37.33, "D": 5.33, "E": 4.0, "F": 1.33 } }, { "label": "Little guidance", "avg_error_count": 2.17, "avg_warning_count": 0.83, "avg_output_tokens": 3031.37, "avg_dom_elements": 40.28, "zero_error_count": 0, "relative_error_burden_pct": 63.42, "relative_warning_burden_pct": 96.12, "share_of_model_errors_pct": 28.03, "error_share_within_issues_pct": 72.44, "improvement_vs_unguided_pct": 36.58, "zero_error_share_pct": 0.0, "score": 8, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 71.33, "C": 22.0, "D": 2.67, "E": 2.67, "F": 1.33 } }, { "label": "Expert guidance", "avg_error_count": 2.18, "avg_warning_count": 0.76, "avg_output_tokens": 2477.34, "avg_dom_elements": 40.08, "zero_error_count": 0, "relative_error_burden_pct": 63.69, "relative_warning_burden_pct": 88.78, "share_of_model_errors_pct": 27.77, "error_share_within_issues_pct": 74.08, "improvement_vs_unguided_pct": 36.31, "zero_error_share_pct": 0.0, "score": 8, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 69.59, "C": 19.59, "D": 4.73, "E": 5.41, "F": 0.68 } } ] }, { "model_id": "x-ai/grok-4.1-fast", "model_slug": "x-ai-grok-4-1-fast", "model_label": "x-ai/grok-4.1-fast", "overall_score": 22, "overall_mark": "F", "avg_output_tokens": 1552.35, "avg_dom_elements": 28.48, "run": { "run_started_at": "2026-02-20T00:48:52.832567", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.18, "avg_warning_count": 0.84, "avg_output_tokens": 1337.3, "avg_dom_elements": 24.95, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 42.48, "error_share_within_issues_pct": 79.1, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 50.67, "C": 31.33, "D": 8.67, "E": 5.33, "F": 4.0 } }, { "label": "Little guidance", "avg_error_count": 2.19, "avg_warning_count": 0.63, "avg_output_tokens": 1838.4, "avg_dom_elements": 28.9, "zero_error_count": 0, "relative_error_burden_pct": 68.97, "relative_warning_burden_pct": 75.4, "share_of_model_errors_pct": 29.3, "error_share_within_issues_pct": 77.59, "improvement_vs_unguided_pct": 31.03, "zero_error_share_pct": 0.0, "score": 4, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 76.67, "C": 16.0, "D": 2.0, "E": 2.67, "F": 2.67 } }, { "label": "Expert guidance", "avg_error_count": 2.14, "avg_warning_count": 0.64, "avg_output_tokens": 1480.4, "avg_dom_elements": 31.64, "zero_error_count": 0, "relative_error_burden_pct": 67.36, "relative_warning_burden_pct": 76.42, "share_of_model_errors_pct": 28.23, "error_share_within_issues_pct": 76.94, "improvement_vs_unguided_pct": 32.64, "zero_error_share_pct": 0.0, "score": 5, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 70.95, "C": 19.59, "D": 4.05, "E": 4.73, "F": 0.68 } } ] }, { "model_id": "mistralai/devstral-2512", "model_slug": "mistralai-devstral-2512", "model_label": "mistralai/devstral-2512", "overall_score": 21, "overall_mark": "F", "avg_output_tokens": 1327.05, "avg_dom_elements": 24.61, "run": { "run_started_at": "2026-02-20T01:10:49.070322", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.57, "avg_warning_count": 0.68, "avg_output_tokens": 1036.63, "avg_dom_elements": 20.72, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 45.57, "error_share_within_issues_pct": 83.99, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 0, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 46.67, "C": 36.67, "D": 10.67, "E": 3.33, "F": 2.67 } }, { "label": "Little guidance", "avg_error_count": 1.99, "avg_warning_count": 0.51, "avg_output_tokens": 1617.75, "avg_dom_elements": 26.86, "zero_error_count": 0, "relative_error_burden_pct": 55.89, "relative_warning_burden_pct": 74.51, "share_of_model_errors_pct": 25.47, "error_share_within_issues_pct": 79.73, "improvement_vs_unguided_pct": 44.11, "zero_error_share_pct": 0.0, "score": 4, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 66.67, "C": 28.0, "D": 2.0, "E": 2.67, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 2.3, "avg_warning_count": 0.51, "avg_output_tokens": 1326.78, "avg_dom_elements": 26.26, "zero_error_count": 0, "relative_error_burden_pct": 64.41, "relative_warning_burden_pct": 74.52, "share_of_model_errors_pct": 28.96, "error_share_within_issues_pct": 81.93, "improvement_vs_unguided_pct": 35.59, "zero_error_share_pct": 0.0, "score": 2, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 62.84, "C": 26.35, "D": 6.08, "E": 4.73, "F": 0.0 } } ] }, { "model_id": "anthropic/claude-haiku-4.5", "model_slug": "anthropic-claude-haiku-4-5", "model_label": "anthropic/claude-haiku-4.5", "overall_score": 20, "overall_mark": "F", "avg_output_tokens": 2260.72, "avg_dom_elements": 33.9, "run": { "run_started_at": "2026-02-20T03:47:47.346614", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.28, "avg_warning_count": 0.97, "avg_output_tokens": 1905.26, "avg_dom_elements": 27.59, "zero_error_count": 0, "relative_error_burden_pct": 92.36, "relative_warning_burden_pct": 81.71, "share_of_model_errors_pct": 38.53, "error_share_within_issues_pct": 77.12, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 52.67, "C": 27.33, "D": 12.67, "E": 4.0, "F": 3.33 } }, { "label": "Little guidance", "avg_error_count": 2.01, "avg_warning_count": 0.85, "avg_output_tokens": 2243.37, "avg_dom_elements": 29.03, "zero_error_count": 0, "relative_error_burden_pct": 56.69, "relative_warning_burden_pct": 71.08, "share_of_model_errors_pct": 23.65, "error_share_within_issues_pct": 70.4, "improvement_vs_unguided_pct": 38.62, "zero_error_share_pct": 0.0, "score": 5, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 76.67, "C": 14.67, "D": 4.67, "E": 2.0, "F": 2.0 } }, { "label": "Expert guidance", "avg_error_count": 3.55, "avg_warning_count": 1.19, "avg_output_tokens": 2671.9, "avg_dom_elements": 46.22, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 37.82, "error_share_within_issues_pct": 74.88, "improvement_vs_unguided_pct": -8.28, "zero_error_share_pct": 0.0, "score": 3, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 70.59, "C": 14.71, "D": 8.09, "E": 2.21, "F": 4.41 } } ] }, { "model_id": "anthropic/claude-opus-4.6", "model_slug": "anthropic-claude-opus-4-6", "model_label": "anthropic/claude-opus-4.6", "overall_score": 20, "overall_mark": "F", "avg_output_tokens": 4617.76, "avg_dom_elements": 56.81, "run": { "run_started_at": "2026-02-20T01:56:59.037152", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 5.17, "avg_warning_count": 1.78, "avg_output_tokens": 3900.16, "avg_dom_elements": 43.79, "zero_error_count": 0, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 52.15, "error_share_within_issues_pct": 74.4, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.0, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 43.33, "C": 26.0, "D": 13.33, "E": 9.33, "F": 8.0 } }, { "label": "Little guidance", "avg_error_count": 2.02, "avg_warning_count": 0.99, "avg_output_tokens": 4497.89, "avg_dom_elements": 48.4, "zero_error_count": 0, "relative_error_burden_pct": 39.05, "relative_warning_burden_pct": 55.43, "share_of_model_errors_pct": 20.36, "error_share_within_issues_pct": 67.18, "improvement_vs_unguided_pct": 60.95, "zero_error_share_pct": 0.0, "score": 12, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 69.33, "C": 26.0, "D": 0.67, "E": 3.33, "F": 0.67 } }, { "label": "Expert guidance", "avg_error_count": 2.76, "avg_warning_count": 1.72, "avg_output_tokens": 5466.55, "avg_dom_elements": 78.55, "zero_error_count": 0, "relative_error_burden_pct": 53.42, "relative_warning_burden_pct": 96.8, "share_of_model_errors_pct": 27.49, "error_share_within_issues_pct": 61.6, "improvement_vs_unguided_pct": 46.58, "zero_error_share_pct": 0.0, "score": 15, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 70.27, "C": 16.89, "D": 2.7, "E": 5.41, "F": 4.73 } } ] }, { "model_id": "google/gemini-3.1-pro-preview", "model_slug": "google-gemini-3-1-pro-preview", "model_label": "google/gemini-3.1-pro-preview", "overall_score": 18, "overall_mark": "F", "avg_output_tokens": 1193.78, "avg_dom_elements": 15.79, "run": { "run_started_at": "2026-02-25T21:13:06.949976", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 3.52, "avg_warning_count": 0.25, "avg_output_tokens": 1084.17, "avg_dom_elements": 14.57, "zero_error_count": 1, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 75.51, "share_of_model_errors_pct": 39.43, "error_share_within_issues_pct": 93.45, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.67, "score": 0, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "C", "mark_distribution_pct": { "A": 0.67, "B": 38.0, "C": 47.33, "D": 8.67, "E": 4.0, "F": 1.33 } }, { "label": "Little guidance", "avg_error_count": 2.78, "avg_warning_count": 0.33, "avg_output_tokens": 1348.94, "avg_dom_elements": 15.86, "zero_error_count": 0, "relative_error_burden_pct": 78.98, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 31.14, "error_share_within_issues_pct": 89.48, "improvement_vs_unguided_pct": 21.02, "zero_error_share_pct": 0.0, "score": 0, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.0, "B": 61.33, "C": 35.33, "D": 1.33, "E": 2.0, "F": 0.0 } }, { "label": "Expert guidance", "avg_error_count": 2.68, "avg_warning_count": 0.31, "avg_output_tokens": 1147.3, "avg_dom_elements": 16.96, "zero_error_count": 1, "relative_error_burden_pct": 76.14, "relative_warning_burden_pct": 93.71, "share_of_model_errors_pct": 29.42, "error_share_within_issues_pct": 89.75, "improvement_vs_unguided_pct": 23.86, "zero_error_share_pct": 0.68, "score": 1, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 0.68, "B": 64.63, "C": 27.89, "D": 5.44, "E": 1.36, "F": 0.0 } } ] }, { "model_id": "z-ai/glm-5", "model_slug": "z-ai-glm-5", "model_label": "z-ai/glm-5", "overall_score": 14, "overall_mark": "F", "avg_output_tokens": 4314.77, "avg_dom_elements": 59.35, "run": { "run_started_at": "2026-02-20T01:15:08.811793", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 4.75, "avg_warning_count": 1.75, "avg_output_tokens": 4397.91, "avg_dom_elements": 54.73, "zero_error_count": 1, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 37.61, "error_share_within_issues_pct": 73.13, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 0.67, "score": 3, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "D", "mark_distribution_pct": { "A": 0.67, "B": 21.33, "C": 26.67, "D": 35.33, "E": 14.0, "F": 2.0 } }, { "label": "Little guidance", "avg_error_count": 4.23, "avg_warning_count": 1.24, "avg_output_tokens": 4302.16, "avg_dom_elements": 55.79, "zero_error_count": 0, "relative_error_burden_pct": 88.98, "relative_warning_burden_pct": 71.18, "share_of_model_errors_pct": 33.02, "error_share_within_issues_pct": 77.28, "improvement_vs_unguided_pct": 11.02, "zero_error_share_pct": 0.0, "score": 4, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "D", "mark_distribution_pct": { "A": 0.0, "B": 25.0, "C": 30.41, "D": 36.49, "E": 7.43, "F": 0.68 } }, { "label": "Expert guidance", "avg_error_count": 3.79, "avg_warning_count": 1.25, "avg_output_tokens": 4242.61, "avg_dom_elements": 67.65, "zero_error_count": 2, "relative_error_burden_pct": 79.71, "relative_warning_burden_pct": 71.66, "share_of_model_errors_pct": 29.38, "error_share_within_issues_pct": 75.17, "improvement_vs_unguided_pct": 20.29, "zero_error_share_pct": 1.36, "score": 8, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 1.36, "B": 36.05, "C": 31.97, "D": 23.13, "E": 6.12, "F": 1.36 } } ] }, { "model_id": "moonshotai/kimi-k2.5", "model_slug": "moonshotai-kimi-k2-5", "model_label": "moonshotai/kimi-k2.5", "overall_score": 11, "overall_mark": "F", "avg_output_tokens": 4004.76, "avg_dom_elements": 64.69, "run": { "run_started_at": "2026-02-19T22:21:53.345272", "run_status": "completed" }, "guidance": [ { "label": "Unguided", "avg_error_count": 7.11, "avg_warning_count": 4.46, "avg_output_tokens": 4312.72, "avg_dom_elements": 66.21, "zero_error_count": 2, "relative_error_burden_pct": 100.0, "relative_warning_burden_pct": 100.0, "share_of_model_errors_pct": 47.05, "error_share_within_issues_pct": 61.45, "improvement_vs_unguided_pct": 0.0, "zero_error_share_pct": 1.34, "score": 2, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 1.34, "B": 20.81, "C": 19.46, "D": 17.45, "E": 20.81, "F": 20.13 } }, { "label": "Little guidance", "avg_error_count": 4.53, "avg_warning_count": 3.86, "avg_output_tokens": 4209.81, "avg_dom_elements": 66.23, "zero_error_count": 3, "relative_error_burden_pct": 63.72, "relative_warning_burden_pct": 86.49, "share_of_model_errors_pct": 30.18, "error_share_within_issues_pct": 54.01, "improvement_vs_unguided_pct": 36.28, "zero_error_share_pct": 2.0, "score": 6, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 2.0, "B": 43.33, "C": 28.67, "D": 8.67, "E": 10.0, "F": 7.33 } }, { "label": "Expert guidance", "avg_error_count": 3.66, "avg_warning_count": 1.46, "avg_output_tokens": 3457.3, "avg_dom_elements": 61.42, "zero_error_count": 2, "relative_error_burden_pct": 51.51, "relative_warning_burden_pct": 32.65, "share_of_model_errors_pct": 22.77, "error_share_within_issues_pct": 71.55, "improvement_vs_unguided_pct": 48.49, "zero_error_share_pct": 1.43, "score": 7, "mark": "F", "dominant_mark": "F", "dominant_mark_from_distribution": "B", "mark_distribution_pct": { "A": 1.43, "B": 50.0, "C": 26.43, "D": 7.86, "E": 6.43, "F": 7.86 } } ] } ] }