mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
55 lines
3.4 KiB
Python
55 lines
3.4 KiB
Python
"""Bind the reader's prepared checkout to a separately acquired GAIA dataset."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
from scripts.prepare_release import ROOT, read, write
|
|
from evaluation.gaia.finalize_experiment_manifest import finalize
|
|
from evaluation.gaia.check_manifest_identity import check_identity
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dataset-root", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, default=ROOT / "sgar_mvp/runtime_state/gaia_manifest.json")
|
|
args = parser.parse_args()
|
|
gaia = ROOT / "evaluation/gaia"
|
|
pending = {"method_id": "ExecOrchestra", "experiment_id": "execorchestra-gaia",
|
|
"code": {}, "models": {}, "resources": {}, "retrieval": {}, "budget": {}, "evaluation": {}}
|
|
pending["code"]["sgar_git_commit"] = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
|
|
for key, name in {
|
|
"adapter_sha256": "adapter.py", "answer_adapter_sha256": "answer.py",
|
|
"scorer_wrapper_sha256": "scorer.py", "synthetic_runner_sha256": "run_synthetic.py",
|
|
"dataset_freezer_sha256": "freeze_dataset.py", "experiment_finalizer_sha256": "finalize_experiment_manifest.py",
|
|
"validation_runner_sha256": "run_validation.py",
|
|
}.items(): pending["code"][key] = digest(gaia / name)
|
|
pending["models"] = {x["role"]: x["api_model_id"] for x in read(ROOT / "sgar_mvp/config/model_profile.json")["roles"]}
|
|
pending["models"]["control_role_policy_sha256"] = digest(ROOT / "sgar_mvp/config/control_role_policy.json")
|
|
for section, fields in {
|
|
"resources": {"catalog_sha256": "Pool/resources/json/combine.json", "effective_pool_sha256": "Pool/resources/json/effective_combine.json"},
|
|
"retrieval": {"index_manifest_sha256": "Pool/index_meta/index_build_manifest.json", "retrieval_policy_sha256": "sgar_mvp/config/retrieval_policy.json",
|
|
"capability_index_sha256": "Pool/index_meta/faiss_cap.index", "constraint_index_sha256": "Pool/index_meta/faiss_con.index", "metadata_index_sha256": "Pool/index_meta/resource_meta.pkl"},
|
|
"budget": {"model_cost_policy_sha256": "sgar_mvp/config/model_cost_policy.json"},
|
|
}.items():
|
|
for key, filename in fields.items():
|
|
value = digest(ROOT / filename)
|
|
if value is None: raise RuntimeError("Preparation incomplete: " + filename)
|
|
pending[section][key] = value
|
|
pending["retrieval"]["generation_id"] = read(ROOT / "Pool/index_meta/index_build_manifest.json")["generation_id"]
|
|
pending["budget"].update(max_generation_requests_per_task=None, max_embedding_requests_per_task=None)
|
|
pending["evaluation"] = {"scorer_source": str(gaia / "scorer.py"), "scorer_sha256": digest(gaia / "scorer.py")}
|
|
stage = ROOT / "sgar_mvp/runtime_state/gaia_manifest.pending.json"
|
|
write(stage, pending)
|
|
finalize(stage, args.dataset_root.resolve() / "dataset_manifest.json", args.output.resolve())
|
|
audit = check_identity(ROOT, args.output.resolve(), gaia)
|
|
write(ROOT / "sgar_mvp/runtime_state/gaia_manifest_audit.json", audit)
|
|
if not audit["ready_for_exact_manifest_run"]:
|
|
raise RuntimeError("Manifest identity mismatch: " + ",".join(audit["mismatches"]))
|
|
print("GAIA manifest validated: " + str(args.output))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|