mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
558 lines
24 KiB
Python
558 lines
24 KiB
Python
"""Run one frozen SGAR method across the complete official GAIA validation split."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import importlib
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from typing import Any
|
|
|
|
from evaluation.gaia.adapter import AgentVisibleTask, EvaluatorRecord, GaiaTaskAdapter
|
|
|
|
|
|
def _json(path: Path) -> Any:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _jsonl(path: Path) -> list[dict[str, Any]]:
|
|
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
|
|
|
|
def _final_output(run_root: Path) -> tuple[Path | None, str]:
|
|
path = next((item for item in (run_root / "final_output.md", run_root / "final_output.txt") if item.is_file()), None)
|
|
return path, path.read_text(encoding="utf-8") if path else ""
|
|
|
|
|
|
def _failure_class(status: str, error: dict[str, Any] | None, score: float) -> str | None:
|
|
if score >= 1.0:
|
|
return None
|
|
if status == "task_timeout":
|
|
return "timeout"
|
|
if error:
|
|
error_type = str(error.get("type") or "unknown").casefold()
|
|
if "rate" in error_type or "limit" in str(error.get("message") or "").casefold():
|
|
return "provider_rate_limit"
|
|
if "timeout" in error_type or "timeout" in str(error.get("message") or "").casefold():
|
|
return "timeout"
|
|
if "connection" in error_type or "transport" in error_type:
|
|
return "provider_transport"
|
|
return "runner_exception"
|
|
if status in {"provider_rate_limit", "provider_transport", "runner_exception"}:
|
|
return status
|
|
if status == "structured_failure":
|
|
return status
|
|
return "scorer_mismatch"
|
|
|
|
|
|
def _structured_failure_class(run_root: Path) -> str | None:
|
|
events = run_root / "run_events.jsonl"
|
|
if events.is_file():
|
|
for event in reversed(_jsonl(events)):
|
|
if event.get("event_type") == "controller_session_failed":
|
|
code = event.get("failure_code")
|
|
if isinstance(code, str) and code:
|
|
return code
|
|
report = run_root / "experiment_report.md"
|
|
if not report.is_file():
|
|
return None
|
|
text = report.read_text(encoding="utf-8", errors="replace")
|
|
mappings = (
|
|
("required_input_missing", "plan_compiler_required_input_missing"),
|
|
("planner_response_schema_invalid", "planner_response_schema_invalid"),
|
|
("NodeUnrecoverableError", "node_unrecoverable"),
|
|
)
|
|
return next((label for marker, label in mappings if marker in text), None)
|
|
|
|
|
|
def _utc_now() -> str:
|
|
from datetime import datetime, timezone
|
|
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _write_json(path: Path, value: Any) -> None:
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def _write_phase_records(
|
|
task_root: Path,
|
|
*,
|
|
experiment_id: str,
|
|
trial_id: str,
|
|
task_id: str,
|
|
level: int | None,
|
|
method_id: str,
|
|
environment_started_at: str,
|
|
environment_ready_at: str | None,
|
|
environment_seconds: float | None,
|
|
agent_started_at: str | None,
|
|
agent_finished_at: str | None,
|
|
agent_seconds: float | None,
|
|
verifier_started_at: str | None,
|
|
verifier_finished_at: str | None,
|
|
verifier_seconds: float | None,
|
|
trial_started_at: str,
|
|
trial_finished_at: str,
|
|
e2e_seconds: float,
|
|
timeout_sec: float | None,
|
|
status: str,
|
|
termination_reason: str,
|
|
failure_class: str | None,
|
|
failure: dict[str, Any] | None,
|
|
reward: float | None,
|
|
cost: dict[str, Any],
|
|
execution: dict[str, Any],
|
|
final_path: Path | None,
|
|
verifier_record: dict[str, Any],
|
|
) -> None:
|
|
"""Persist canonical timing sidecars without changing native RARE logs."""
|
|
_write_json(task_root / "environment_manifest.json", {
|
|
"protocol": "sgar-gaia-environment-v1",
|
|
"task_id": task_id,
|
|
"environment_setup_started_at": environment_started_at,
|
|
"environment_ready_at": environment_ready_at,
|
|
"environment_setup_seconds": environment_seconds,
|
|
"environment_mode": "host_native_gaia",
|
|
"environment_failure": failure if failure_class == "infrastructure_failure" else None,
|
|
})
|
|
_write_json(task_root / "verifier_record.json", verifier_record)
|
|
_write_json(task_root / "artifact_manifest.json", {
|
|
"protocol": "sgar-gaia-artifacts-v1",
|
|
"task_id": task_id,
|
|
"final_output_path": str(final_path) if final_path else None,
|
|
"native_cost_summary": str(task_root / "cost_summary.json") if (task_root / "cost_summary.json").is_file() else None,
|
|
"native_execution_summary": str(task_root / "execution_summary.json") if (task_root / "execution_summary.json").is_file() else None,
|
|
})
|
|
_write_json(task_root / "trial_record.json", {
|
|
"protocol": "sgar-gaia-trial-record-v1",
|
|
"experiment_id": experiment_id,
|
|
"trial_id": trial_id,
|
|
"benchmark_id": "gaia/2023-validation",
|
|
"benchmark_version": "2023",
|
|
"task_id": task_id,
|
|
"task_level": level,
|
|
"method_id": method_id,
|
|
"harness_version": "rare-tf-gaia-runner-v1",
|
|
"trial_started_at": trial_started_at,
|
|
"trial_finished_at": trial_finished_at,
|
|
"environment_setup_seconds": environment_seconds,
|
|
"agent_start": agent_started_at,
|
|
"agent_end": agent_finished_at,
|
|
"agent_execution_seconds": agent_seconds,
|
|
"verifier_start": verifier_started_at,
|
|
"verifier_end": verifier_finished_at,
|
|
"verifier_seconds": verifier_seconds,
|
|
"e2e_wall_clock_seconds": e2e_seconds,
|
|
"timeout_limit_seconds": timeout_sec,
|
|
"timeout_scope": "agent_phase_only" if timeout_sec is not None else "none_gaia_official_timeout",
|
|
"status": status,
|
|
"termination_reason": termination_reason,
|
|
"failure_class": failure_class,
|
|
"failure": failure,
|
|
"official_reward": reward,
|
|
"cost_summary": cost,
|
|
"execution_summary": execution,
|
|
"phase_boundary_policy": {
|
|
"environment_setup_excluded_from_agent": True,
|
|
"verifier_excluded_from_agent": True,
|
|
"cleanup_excluded_from_agent": True,
|
|
"rare_owned_runtime_preparation_in_agent": True,
|
|
},
|
|
})
|
|
|
|
|
|
def _prepare_task_inputs(
|
|
*,
|
|
row: dict[str, Any],
|
|
dataset_root: Path,
|
|
output_root: Path,
|
|
sgar_root: Path,
|
|
invocation_module: Any,
|
|
trial_id: str,
|
|
) -> dict[str, Any]:
|
|
"""Prepare one GAIA task and return phase metadata without starting RARE."""
|
|
task_id = str(row["task_id"])
|
|
task_root = output_root / "tasks" / task_id
|
|
input_root = output_root / "inputs" / task_id
|
|
environment_started_at = _utc_now()
|
|
environment_started_mono = time.monotonic()
|
|
try:
|
|
task_root.mkdir(parents=True, exist_ok=False)
|
|
input_root.mkdir(parents=True, exist_ok=False)
|
|
raw_names = row.get("file_name") or []
|
|
raw_names = [raw_names] if isinstance(raw_names, str) else list(raw_names)
|
|
names = [str(name).removeprefix("2023/validation/") for name in raw_names]
|
|
for name, relative_name in zip(raw_names, names):
|
|
source = (dataset_root / "raw/2023/validation" / relative_name).resolve()
|
|
if dataset_root.resolve() not in source.parents or not source.is_file():
|
|
raise ValueError("gaia_attachment_outside_frozen_snapshot")
|
|
target = input_root / relative_name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(source, target)
|
|
task = AgentVisibleTask(task_id, str(row["Question"]), int(row["Level"]), tuple(names), "2023_validation")
|
|
evaluator = EvaluatorRecord(task_id, None, row["Final answer"], "2023_validation")
|
|
adapter = GaiaTaskAdapter(dataset_root=input_root, split="2023_validation")
|
|
prepared = invocation_module.prepare_task_invocation(
|
|
query=task.question,
|
|
input_specs=tuple(invocation_module.InputSourceSpec(logical_name=name, source_path=input_root / name) for name in names),
|
|
run_dir=task_root,
|
|
request_id=f"gaia-v1-{task_id}-{trial_id}",
|
|
allowed_public_input_roots=(input_root,),
|
|
project_root=sgar_root,
|
|
)
|
|
except Exception as exc:
|
|
task_root.mkdir(parents=True, exist_ok=True)
|
|
return {
|
|
"task_root": task_root,
|
|
"input_root": input_root,
|
|
"environment_started_at": environment_started_at,
|
|
"environment_ready_at": None,
|
|
"environment_seconds": time.monotonic() - environment_started_mono,
|
|
"error": {"type": type(exc).__name__, "message": str(exc)},
|
|
}
|
|
return {
|
|
"task_root": task_root,
|
|
"input_root": input_root,
|
|
"task": task,
|
|
"evaluator": evaluator,
|
|
"adapter": adapter,
|
|
"prepared": prepared,
|
|
"environment_started_at": environment_started_at,
|
|
"environment_ready_at": _utc_now(),
|
|
"environment_seconds": time.monotonic() - environment_started_mono,
|
|
"error": None,
|
|
}
|
|
|
|
|
|
async def run_validation(
|
|
dataset_root: Path,
|
|
sgar_root: Path,
|
|
experiment_manifest: Path,
|
|
output_root: Path,
|
|
*,
|
|
config_path: Path | None = None,
|
|
task_timeout_sec: float | None = None,
|
|
task_ids: set[str] | None = None,
|
|
max_tasks: int | None = None,
|
|
network_policy_mode: str = "disabled",
|
|
diagnostic_allow_manifest_drift: bool = False,
|
|
) -> dict[str, Any]:
|
|
manifest = _json(experiment_manifest)
|
|
if manifest.get("status") != "FROZEN" or not (manifest.get("freeze_gate") or {}).get("passed"):
|
|
raise ValueError("gaia_experiment_manifest_not_frozen")
|
|
sys.path.insert(0, str(sgar_root.resolve()))
|
|
identity_checker = importlib.import_module("evaluation.gaia.check_manifest_identity")
|
|
manifest_identity = identity_checker.check_identity(
|
|
sgar_root.resolve(), experiment_manifest.resolve(), Path(__file__).resolve().parent
|
|
)
|
|
if not manifest_identity["ready_for_exact_manifest_run"] and not diagnostic_allow_manifest_drift:
|
|
raise ValueError(
|
|
"gaia_experiment_manifest_identity_mismatch:"
|
|
+ ",".join(manifest_identity["mismatches"])
|
|
)
|
|
if not manifest_identity["ready_for_exact_manifest_run"] and diagnostic_allow_manifest_drift:
|
|
if max_tasks not in {0, 1} or (max_tasks == 1 and (not task_ids or len(task_ids) != 1)):
|
|
raise ValueError("gaia_manifest_drift_diagnostic_requires_one_explicit_task")
|
|
dataset_manifest = _json(dataset_root / "dataset_manifest.json")
|
|
if dataset_manifest.get("task_count") != 165 or not dataset_manifest.get("immutable"):
|
|
raise ValueError("gaia_dataset_snapshot_not_frozen")
|
|
visible_rows = _jsonl(dataset_root / "views/agent_visible.jsonl")
|
|
evaluator_by_id = {row["task_id"]: row for row in _jsonl(dataset_root / "views/evaluator_only.jsonl")}
|
|
if len(visible_rows) != 165 or set(evaluator_by_id) != {row["task_id"] for row in visible_rows}:
|
|
raise ValueError("gaia_dataset_views_identity_mismatch")
|
|
|
|
sgar_main = importlib.import_module("sgar_mvp.main")
|
|
invocation_module = importlib.import_module("sgar_mvp.src.task_invocation")
|
|
resolved_config = (config_path or (sgar_root / "sgar_mvp/config.json")).resolve()
|
|
config = sgar_main.load_config(str(resolved_config), resolve_secrets=True)
|
|
if config is None:
|
|
raise ValueError("gaia_sgar_config_missing")
|
|
output_root.mkdir(parents=True, exist_ok=True)
|
|
(output_root / "manifest_identity_audit.json").write_text(
|
|
json.dumps(manifest_identity, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
results: list[dict[str, Any]] = []
|
|
started = time.monotonic()
|
|
selected_rows = [row for row in visible_rows if task_ids is None or str(row["task_id"]) in task_ids]
|
|
if max_tasks is not None:
|
|
selected_rows = selected_rows[:max_tasks]
|
|
experiment_id = str(manifest.get("experiment_id") or "rare-tf-gaia")
|
|
for index, row in enumerate(selected_rows):
|
|
task_id = str(row["task_id"])
|
|
trial_id = str(uuid.uuid4())
|
|
trial_started_mono = time.monotonic()
|
|
trial_started_at = _utc_now()
|
|
prep = _prepare_task_inputs(
|
|
row={**row, "Final answer": evaluator_by_id[task_id]["Final answer"]},
|
|
dataset_root=dataset_root,
|
|
output_root=output_root,
|
|
sgar_root=sgar_root,
|
|
invocation_module=invocation_module,
|
|
trial_id=trial_id,
|
|
)
|
|
task_root = prep["task_root"]
|
|
environment_started_at = prep["environment_started_at"]
|
|
environment_ready_at = prep["environment_ready_at"]
|
|
environment_seconds = prep["environment_seconds"]
|
|
if prep["error"] is not None:
|
|
trial_finished_at = _utc_now()
|
|
e2e_seconds = time.monotonic() - trial_started_mono
|
|
_write_phase_records(
|
|
task_root,
|
|
experiment_id=experiment_id,
|
|
trial_id=trial_id,
|
|
task_id=task_id,
|
|
level=int(row["Level"]),
|
|
method_id=manifest["method_id"],
|
|
environment_started_at=environment_started_at,
|
|
environment_ready_at=environment_ready_at,
|
|
environment_seconds=environment_seconds,
|
|
agent_started_at=None,
|
|
agent_finished_at=None,
|
|
agent_seconds=None,
|
|
verifier_started_at=None,
|
|
verifier_finished_at=None,
|
|
verifier_seconds=None,
|
|
trial_started_at=trial_started_at,
|
|
trial_finished_at=trial_finished_at,
|
|
e2e_seconds=e2e_seconds,
|
|
timeout_sec=task_timeout_sec,
|
|
status="infrastructure_failure",
|
|
termination_reason="environment_setup_failure",
|
|
failure_class="infrastructure_failure",
|
|
failure=prep["error"],
|
|
reward=None,
|
|
cost={},
|
|
execution={},
|
|
final_path=None,
|
|
verifier_record={
|
|
"protocol": "sgar-gaia-verifier-v1",
|
|
"task_id": task_id,
|
|
"status": "not_started",
|
|
"reason": "environment_setup_failure",
|
|
},
|
|
)
|
|
results.append({
|
|
"task_id": task_id,
|
|
"level": int(row["Level"]),
|
|
"ordinal": index + 1,
|
|
"method_id": manifest["method_id"],
|
|
"status": "infrastructure_failure",
|
|
"failure": prep["error"],
|
|
"failure_taxonomy": "infrastructure_failure",
|
|
"score": 0.0,
|
|
"cost_usd": None,
|
|
"model_calls": 0,
|
|
"tokens": {},
|
|
"agent_delegations": 0,
|
|
"tool_calls": 0,
|
|
"environment_setup_seconds": environment_seconds,
|
|
"agent_execution_seconds": None,
|
|
"verifier_seconds": None,
|
|
"e2e_wall_clock_seconds": e2e_seconds,
|
|
"timeout_limit_seconds": task_timeout_sec,
|
|
"trial_id": trial_id,
|
|
})
|
|
continue
|
|
task = prep["task"]
|
|
evaluator = prep["evaluator"]
|
|
adapter = prep["adapter"]
|
|
prepared = prep["prepared"]
|
|
local_config = dict(config)
|
|
local_config["runtime_settings"] = {
|
|
**dict(config.get("runtime_settings") or {}),
|
|
"output_root": str(task_root),
|
|
"temporary_directory": str(task_root / "runtime_tmp"),
|
|
"pip_cache_directory": str(task_root / "pip_cache"),
|
|
}
|
|
error = None
|
|
agent_started_at = _utc_now()
|
|
agent_started_mono = time.monotonic()
|
|
agent_finished_at = None
|
|
agent_seconds = None
|
|
timeout_triggered = False
|
|
try:
|
|
pipeline_call = sgar_main.run_pipeline(
|
|
local_config, task.question, str(task_root), str(task_root / "experiment_report.md"),
|
|
run_id=f"gaia-v1-{task_id}", task_invocation=prepared,
|
|
max_generation_requests=(
|
|
None if manifest["budget"]["max_generation_requests_per_task"] is None
|
|
else int(manifest["budget"]["max_generation_requests_per_task"])
|
|
),
|
|
max_embedding_requests=(
|
|
None if manifest["budget"]["max_embedding_requests_per_task"] is None
|
|
else int(manifest["budget"]["max_embedding_requests_per_task"])
|
|
),
|
|
network_policy_mode=network_policy_mode,
|
|
)
|
|
if task_timeout_sec is None:
|
|
status = await pipeline_call
|
|
else:
|
|
status = await asyncio.wait_for(pipeline_call, timeout=task_timeout_sec)
|
|
except asyncio.TimeoutError:
|
|
timeout_triggered = True
|
|
status = "task_timeout"
|
|
error = {"type": "TaskTimeoutError", "message": f"agent phase exceeded {task_timeout_sec}s"}
|
|
except Exception as exc:
|
|
status = "runner_exception"
|
|
error = {"type": type(exc).__name__, "message": str(exc)}
|
|
finally:
|
|
agent_finished_at = _utc_now()
|
|
agent_seconds = time.monotonic() - agent_started_mono
|
|
final_path, final = _final_output(task_root)
|
|
verifier_started_at = _utc_now()
|
|
verifier_started_mono = time.monotonic()
|
|
evaluation = adapter.evaluate(task, evaluator, final)
|
|
verifier_finished_at = _utc_now()
|
|
verifier_seconds = time.monotonic() - verifier_started_mono
|
|
trial_finished_at = _utc_now()
|
|
e2e_seconds = time.monotonic() - trial_started_mono
|
|
cost = _json(task_root / "cost_summary.json") if (task_root / "cost_summary.json").exists() else {}
|
|
execution = _json(task_root / "execution_summary.json") if (task_root / "execution_summary.json").exists() else {}
|
|
failure_taxonomy = _structured_failure_class(task_root) if str(status) == "structured_failure" else None
|
|
reward = float(evaluation["scorer"]["score"])
|
|
failure_taxonomy = failure_taxonomy or _failure_class(str(status), error, reward)
|
|
termination_reason = "task_timeout" if timeout_triggered else ("finished" if reward > 0 else str(status))
|
|
verifier_record = {
|
|
"protocol": "sgar-gaia-verifier-v1",
|
|
"task_id": task_id,
|
|
"status": "ok",
|
|
"reward": reward,
|
|
"scorer": evaluation,
|
|
"verifier_started_at": verifier_started_at,
|
|
"verifier_finished_at": verifier_finished_at,
|
|
"verifier_seconds": verifier_seconds,
|
|
}
|
|
_write_phase_records(
|
|
task_root,
|
|
experiment_id=experiment_id,
|
|
trial_id=trial_id,
|
|
task_id=task_id,
|
|
level=task.level,
|
|
method_id=manifest["method_id"],
|
|
environment_started_at=environment_started_at,
|
|
environment_ready_at=environment_ready_at,
|
|
environment_seconds=environment_seconds,
|
|
agent_started_at=agent_started_at,
|
|
agent_finished_at=agent_finished_at,
|
|
agent_seconds=agent_seconds,
|
|
verifier_started_at=verifier_started_at,
|
|
verifier_finished_at=verifier_finished_at,
|
|
verifier_seconds=verifier_seconds,
|
|
trial_started_at=trial_started_at,
|
|
trial_finished_at=trial_finished_at,
|
|
e2e_seconds=e2e_seconds,
|
|
timeout_sec=task_timeout_sec,
|
|
status=str(status),
|
|
termination_reason=termination_reason,
|
|
failure_class=failure_taxonomy,
|
|
failure=error,
|
|
reward=reward,
|
|
cost=cost,
|
|
execution=execution,
|
|
final_path=final_path,
|
|
verifier_record=verifier_record,
|
|
)
|
|
result = {
|
|
"task_id": task_id,
|
|
"level": task.level,
|
|
"ordinal": index + 1,
|
|
"method_id": manifest["method_id"],
|
|
"experiment_manifest_sha256": manifest["manifest_sha256"],
|
|
"resource_pool_hash": manifest_identity["checks"]["resources.effective_pool_sha256"]["current"],
|
|
"declared_resource_pool_hash": manifest["resources"]["effective_pool_sha256"],
|
|
"formal_comparable": manifest_identity["ready_for_exact_manifest_run"],
|
|
"status": str(status),
|
|
"raw_final_output_path": str(final_path) if final_path else None,
|
|
"model_answer": evaluation["answer"],
|
|
"score": evaluation["scorer"]["score"],
|
|
"failure": error,
|
|
"failure_taxonomy": failure_taxonomy,
|
|
"model_calls": cost.get("started_call_count", 0),
|
|
"tokens": cost.get("token_usage", {}),
|
|
"cost_usd": cost.get("observed_total_model_cost_usd"),
|
|
"agent_delegations": execution.get("controller_session_count", 0),
|
|
"tool_calls": execution.get("by_resource_type", {}).get("Tool", 0),
|
|
"environment_setup_seconds": environment_seconds,
|
|
"agent_execution_seconds": agent_seconds,
|
|
"verifier_seconds": verifier_seconds,
|
|
"e2e_wall_clock_seconds": e2e_seconds,
|
|
"timeout_limit_seconds": task_timeout_sec,
|
|
"trial_id": trial_id,
|
|
}
|
|
(task_root / "benchmark_result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
results.append(result)
|
|
|
|
levels = {level: [row for row in results if row["level"] == level] for level in (1, 2, 3)}
|
|
total_cost = sum(float(row["cost_usd"] or 0) for row in results)
|
|
summary = {
|
|
"protocol": "sgar-gaia-validation-results-v1",
|
|
"experiment_manifest_sha256": manifest["manifest_sha256"],
|
|
"formal_comparable": manifest_identity["ready_for_exact_manifest_run"],
|
|
"manifest_identity_mismatches": manifest_identity["mismatches"],
|
|
"task_count": len(results),
|
|
"accuracy": sum(row["score"] for row in results) / max(len(results), 1),
|
|
"level_accuracy": {
|
|
f"L{level}": sum(row["score"] for row in rows) / max(len(rows), 1)
|
|
for level, rows in levels.items()
|
|
},
|
|
"total_cost_usd": total_cost,
|
|
"mean_cost_usd": total_cost / max(len(results), 1),
|
|
"model_calls": sum(row["model_calls"] for row in results),
|
|
"agent_delegations": sum(row["agent_delegations"] for row in results),
|
|
"tool_calls": sum(row["tool_calls"] for row in results),
|
|
"input_tokens": sum((row["tokens"] or {}).get("input_tokens", 0) for row in results),
|
|
"output_tokens": sum((row["tokens"] or {}).get("output_tokens", 0) for row in results),
|
|
"latency_seconds": time.monotonic() - started,
|
|
"failure_taxonomy": {
|
|
key: sum(1 for row in results if row["failure_taxonomy"] == key)
|
|
for key in sorted({row["failure_taxonomy"] for row in results if row["failure_taxonomy"]})
|
|
},
|
|
"results": results,
|
|
}
|
|
(output_root / "validation_results.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
return summary
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--dataset-root", type=Path, required=True)
|
|
parser.add_argument("--sgar-root", type=Path, required=True)
|
|
parser.add_argument("--experiment-manifest", type=Path, required=True)
|
|
parser.add_argument("--output-root", type=Path, required=True)
|
|
parser.add_argument("--config", type=Path, default=None)
|
|
parser.add_argument(
|
|
"--task-timeout-sec",
|
|
type=float,
|
|
default=None,
|
|
help="Optional local cap for the RARE agent phase only; GAIA has no official task wall timeout.",
|
|
)
|
|
parser.add_argument("--task-id", action="append", dest="task_ids")
|
|
parser.add_argument("--max-tasks", type=int, default=None)
|
|
parser.add_argument(
|
|
"--network-policy", choices=("disabled", "declared"), default="disabled",
|
|
help="Permit network access only for Tools that declare it.",
|
|
)
|
|
parser.add_argument(
|
|
"--diagnostic-allow-manifest-drift", action="store_true",
|
|
help="Label a single-case diagnostic as non-comparable when the frozen manifest drifts.",
|
|
)
|
|
args = parser.parse_args()
|
|
print(json.dumps(asyncio.run(run_validation(
|
|
args.dataset_root.resolve(),
|
|
args.sgar_root.resolve(),
|
|
args.experiment_manifest.resolve(),
|
|
args.output_root.resolve(),
|
|
config_path=args.config.resolve() if args.config else None,
|
|
task_timeout_sec=args.task_timeout_sec,
|
|
task_ids=set(args.task_ids) if args.task_ids else None,
|
|
max_tasks=args.max_tasks,
|
|
network_policy_mode=args.network_policy,
|
|
diagnostic_allow_manifest_drift=args.diagnostic_allow_manifest_drift,
|
|
)), ensure_ascii=False, indent=2))
|