ChatDev/execorchestra/scripts/freeze_gaia_manifest.py
2026-11-05 16:22:12 +08:00

55 lines
3.4 KiB
Python

"""Bind the reader's prepared checkout to a separately acquired GAIA dataset."""
import argparse
import hashlib
import json
import subprocess
from pathlib import Path
from scripts.prepare_release import ROOT, read, write
from evaluation.gaia.finalize_experiment_manifest import finalize
from evaluation.gaia.check_manifest_identity import check_identity
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument("--output", type=Path, default=ROOT / "sgar_mvp/runtime_state/gaia_manifest.json")
args = parser.parse_args()
gaia = ROOT / "evaluation/gaia"
pending = {"method_id": "ExecOrchestra", "experiment_id": "execorchestra-gaia",
"code": {}, "models": {}, "resources": {}, "retrieval": {}, "budget": {}, "evaluation": {}}
pending["code"]["sgar_git_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
for key, name in {
"adapter_sha256": "adapter.py", "answer_adapter_sha256": "answer.py",
"scorer_wrapper_sha256": "scorer.py", "synthetic_runner_sha256": "run_synthetic.py",
"dataset_freezer_sha256": "freeze_dataset.py", "experiment_finalizer_sha256": "finalize_experiment_manifest.py",
"validation_runner_sha256": "run_validation.py",
}.items(): pending["code"][key] = digest(gaia / name)
pending["models"] = {x["role"]: x["api_model_id"] for x in read(ROOT / "sgar_mvp/config/model_profile.json")["roles"]}
pending["models"]["control_role_policy_sha256"] = digest(ROOT / "sgar_mvp/config/control_role_policy.json")
for section, fields in {
"resources": {"catalog_sha256": "Pool/resources/json/combine.json", "effective_pool_sha256": "Pool/resources/json/effective_combine.json"},
"retrieval": {"index_manifest_sha256": "Pool/index_meta/index_build_manifest.json", "retrieval_policy_sha256": "sgar_mvp/config/retrieval_policy.json",
"capability_index_sha256": "Pool/index_meta/faiss_cap.index", "constraint_index_sha256": "Pool/index_meta/faiss_con.index", "metadata_index_sha256": "Pool/index_meta/resource_meta.pkl"},
"budget": {"model_cost_policy_sha256": "sgar_mvp/config/model_cost_policy.json"},
}.items():
for key, filename in fields.items():
value = digest(ROOT / filename)
if value is None: raise RuntimeError("Preparation incomplete: " + filename)
pending[section][key] = value
pending["retrieval"]["generation_id"] = read(ROOT / "Pool/index_meta/index_build_manifest.json")["generation_id"]
pending["budget"].update(max_generation_requests_per_task=None, max_embedding_requests_per_task=None)
pending["evaluation"] = {"scorer_source": str(gaia / "scorer.py"), "scorer_sha256": digest(gaia / "scorer.py")}
stage = ROOT / "sgar_mvp/runtime_state/gaia_manifest.pending.json"
write(stage, pending)
finalize(stage, args.dataset_root.resolve() / "dataset_manifest.json", args.output.resolve())
audit = check_identity(ROOT, args.output.resolve(), gaia)
write(ROOT / "sgar_mvp/runtime_state/gaia_manifest_audit.json", audit)
if not audit["ready_for_exact_manifest_run"]:
raise RuntimeError("Manifest identity mismatch: " + ",".join(audit["mismatches"]))
print("GAIA manifest validated: " + str(args.output))
if __name__ == "__main__":
main()