{
  "schema_version": 1,
  "published_at": "2026-08-27T00:00:00Z",
  "audit_label": "old codebase; current code and evaluator have changed",
  "ranking_eligible": false,
  "current_protocol_boundary": "Do not compare or merge these rows with the current 15-train / 15-test, five-generation verified leaderboard.",
  "archive_repository": "Weicheng-Gu1/rsibench-data",
  "archive_commit": "fb0093a23ae80b9968695a738d436090ebc73459",
  "complete_trajectory_runs": [
    {
      "model": "GLM-5.2",
      "provider": "Z.ai",
      "harness": "Pi",
      "domain": "Terminal",
      "run_id": "formal-terminal-glm52-pi-b13-t10-r3-c72-20260808T114747Z",
      "source_code_commit": "8bfc7b02baf5cf14aa734530f44ff9cea7b59677",
      "V_test_A0": 0.006936527777777771,
      "V_test_AT": 0.14444444444444443,
      "delta_test": 0.13750791666666667,
      "raw_reward_A0": 0.05555555555555555,
      "raw_reward_AT": 0.19444444444444442,
      "accepted_steps": [1, 2, 4, 9, 10],
      "protocol": {"generations": 10, "train_tasks": 13, "validation_tasks": 12, "test_tasks": 12, "repeats": 3, "selection_split": "validation"},
      "artifact_count": 5877,
      "infra_error_count": 0,
      "invalid": false,
      "ranking_eligible": false,
      "trajectory_archive_url": "https://github.com/Weicheng-Gu1/rsibench-data/tree/fb0093a23ae80b9968695a738d436090ebc73459/trajectories/terminal-glm-5-2-pi/formal-terminal-glm52-pi-b13-t10-r3-c72-20260808T114747Z"
    },
    {
      "model": "DeepSeek V4 Pro",
      "provider": "DeepSeek",
      "harness": "Pi",
      "domain": "Terminal",
      "run_id": "formal-terminal-deepseek-v4-pro-pi-b13-t5-r3-c72-20260809T190108Z",
      "source_code_commit": "2b58b1f186208879c0e5372abcd872f875138131",
      "V_test_A0": 0.17222222222222222,
      "V_test_AT": 0.22777777777777775,
      "delta_test": 0.055555555555555525,
      "accepted_steps": [1, 2],
      "protocol": {"generations": 5, "train_tasks": 13, "validation_tasks": 12, "test_tasks": 12, "repeats": 3, "selection_split": "validation"},
      "artifact_count": 1999,
      "infra_error_count": 0,
      "invalid": false,
      "ranking_eligible": false,
      "trajectory_archive_url": "https://github.com/Weicheng-Gu1/rsibench-data/tree/fb0093a23ae80b9968695a738d436090ebc73459/trajectories/terminal-deepseek-v4-pro-pi/formal-terminal-deepseek-v4-pro-pi-b13-t5-r3-c72-20260809T190108Z"
    },
    {
      "model": "Kimi K3",
      "provider": "Moonshot",
      "harness": "Pi",
      "domain": "Terminal",
      "run_id": "formal-terminal-kimi-k3-pi-b13-t5-r3-c72-cutoff-20260809T190216Z",
      "source_code_commit": "8bfc7b02baf5cf14aa734530f44ff9cea7b59677",
      "V_test_A0": 0.4777777777777778,
      "V_test_AT": 0.4777777777777778,
      "delta_test": 0.0,
      "accepted_steps": [3],
      "protocol": {"generations": 5, "train_tasks": 13, "validation_tasks": 12, "test_tasks": 12, "repeats": 3, "selection_split": "validation", "note": "partial cutoff"},
      "artifact_count": 2675,
      "infra_error_count": 0,
      "invalid": false,
      "ranking_eligible": false,
      "trajectory_archive_url": "https://github.com/Weicheng-Gu1/rsibench-data/tree/fb0093a23ae80b9968695a738d436090ebc73459/trajectories/terminal-kimi-k3-pi/formal-terminal-kimi-k3-pi-b13-t5-r3-c72-cutoff-20260809T190216Z"
    }
  ],
  "score_snapshots_without_complete_trajectory": [
    {
      "model": "GLM-5.2 FP8",
      "harness": "Pi",
      "domain": "Terminal",
      "V_test_A0": 0.0833,
      "V_test_AT": 0.1167,
      "delta_test": 0.0333,
      "protocol": {"generations": 5, "test_tasks": 12, "repeats": 5},
      "evidence_status": "historical site score snapshot; complete trajectory not recovered",
      "source_site_commit": "4d23c953160cbb7321ff20b48fa4bb5330904472",
      "ranking_eligible": false
    },
    {
      "model": "Claude Opus 4.8",
      "harness": "Pi",
      "domain": "Terminal",
      "V_test_A0": 0.7833,
      "V_test_AT": 0.825,
      "delta_test": 0.0417,
      "protocol": {"generations": 1, "test_tasks": 8, "repeats": 3, "note": "pilot; one accepted generation"},
      "evidence_status": "historical site score snapshot; complete matching trajectory not recovered",
      "source_site_commit": "4d23c953160cbb7321ff20b48fa4bb5330904472",
      "ranking_eligible": false
    }
  ],
  "invalid_attempts": [
    {
      "model": "Claude Opus 5",
      "harness": "Pi",
      "domain": "Terminal",
      "scores": null,
      "result_flag": "invalid: true",
      "reason": "disk and provider infrastructure failures; recorded zeroes are not benchmark scores",
      "public_archive_status": "local trajectories retained; sanitized public export pending",
      "local_archive_ids": ["control-direct-original-20260821T171519Z", "treatment-direct-trajectory-first-20260821T171519Z"],
      "result_sha256": ["e0d8df9e7690c715edfb3d283e536f383d023d5cbec06d68e32f088593dbcb74", "d90727c237bd28986d037a67fc6a8ac22e038a14d316d0b8e0fa4bc617b29ed7"],
      "provenance_sha256": ["dd56ae259bfefdf27a30fe156d9d09acad535c744cb71a3a0890286a29f63f44", "30d0f1082304cef7109f3dfae6f7429b2e557b76319e818d487a7d8f41236624"],
      "ranking_eligible": false
    }
  ]
}