ChatDev/execorchestra/evaluation/gaia/run_validation.py
2026-11-05 16:22:12 +08:00

558 lines
24 KiB
Python

"""Run one frozen SGAR method across the complete official GAIA validation split."""
from __future__ import annotations
import argparse
import asyncio
import importlib
import json
from pathlib import Path
import shutil
import sys
import time
import uuid
from typing import Any
from evaluation.gaia.adapter import AgentVisibleTask, EvaluatorRecord, GaiaTaskAdapter
def _json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def _jsonl(path: Path) -> list[dict[str, Any]]:
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
def _final_output(run_root: Path) -> tuple[Path | None, str]:
path = next((item for item in (run_root / "final_output.md", run_root / "final_output.txt") if item.is_file()), None)
return path, path.read_text(encoding="utf-8") if path else ""
def _failure_class(status: str, error: dict[str, Any] | None, score: float) -> str | None:
if score >= 1.0:
return None
if status == "task_timeout":
return "timeout"
if error:
error_type = str(error.get("type") or "unknown").casefold()
if "rate" in error_type or "limit" in str(error.get("message") or "").casefold():
return "provider_rate_limit"
if "timeout" in error_type or "timeout" in str(error.get("message") or "").casefold():
return "timeout"
if "connection" in error_type or "transport" in error_type:
return "provider_transport"
return "runner_exception"
if status in {"provider_rate_limit", "provider_transport", "runner_exception"}:
return status
if status == "structured_failure":
return status
return "scorer_mismatch"
def _structured_failure_class(run_root: Path) -> str | None:
events = run_root / "run_events.jsonl"
if events.is_file():
for event in reversed(_jsonl(events)):
if event.get("event_type") == "controller_session_failed":
code = event.get("failure_code")
if isinstance(code, str) and code:
return code
report = run_root / "experiment_report.md"
if not report.is_file():
return None
text = report.read_text(encoding="utf-8", errors="replace")
mappings = (
("required_input_missing", "plan_compiler_required_input_missing"),
("planner_response_schema_invalid", "planner_response_schema_invalid"),
("NodeUnrecoverableError", "node_unrecoverable"),
)
return next((label for marker, label in mappings if marker in text), None)
def _utc_now() -> str:
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def _write_json(path: Path, value: Any) -> None:
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _write_phase_records(
task_root: Path,
*,
experiment_id: str,
trial_id: str,
task_id: str,
level: int | None,
method_id: str,
environment_started_at: str,
environment_ready_at: str | None,
environment_seconds: float | None,
agent_started_at: str | None,
agent_finished_at: str | None,
agent_seconds: float | None,
verifier_started_at: str | None,
verifier_finished_at: str | None,
verifier_seconds: float | None,
trial_started_at: str,
trial_finished_at: str,
e2e_seconds: float,
timeout_sec: float | None,
status: str,
termination_reason: str,
failure_class: str | None,
failure: dict[str, Any] | None,
reward: float | None,
cost: dict[str, Any],
execution: dict[str, Any],
final_path: Path | None,
verifier_record: dict[str, Any],
) -> None:
"""Persist canonical timing sidecars without changing native RARE logs."""
_write_json(task_root / "environment_manifest.json", {
"protocol": "sgar-gaia-environment-v1",
"task_id": task_id,
"environment_setup_started_at": environment_started_at,
"environment_ready_at": environment_ready_at,
"environment_setup_seconds": environment_seconds,
"environment_mode": "host_native_gaia",
"environment_failure": failure if failure_class == "infrastructure_failure" else None,
})
_write_json(task_root / "verifier_record.json", verifier_record)
_write_json(task_root / "artifact_manifest.json", {
"protocol": "sgar-gaia-artifacts-v1",
"task_id": task_id,
"final_output_path": str(final_path) if final_path else None,
"native_cost_summary": str(task_root / "cost_summary.json") if (task_root / "cost_summary.json").is_file() else None,
"native_execution_summary": str(task_root / "execution_summary.json") if (task_root / "execution_summary.json").is_file() else None,
})
_write_json(task_root / "trial_record.json", {
"protocol": "sgar-gaia-trial-record-v1",
"experiment_id": experiment_id,
"trial_id": trial_id,
"benchmark_id": "gaia/2023-validation",
"benchmark_version": "2023",
"task_id": task_id,
"task_level": level,
"method_id": method_id,
"harness_version": "rare-tf-gaia-runner-v1",
"trial_started_at": trial_started_at,
"trial_finished_at": trial_finished_at,
"environment_setup_seconds": environment_seconds,
"agent_start": agent_started_at,
"agent_end": agent_finished_at,
"agent_execution_seconds": agent_seconds,
"verifier_start": verifier_started_at,
"verifier_end": verifier_finished_at,
"verifier_seconds": verifier_seconds,
"e2e_wall_clock_seconds": e2e_seconds,
"timeout_limit_seconds": timeout_sec,
"timeout_scope": "agent_phase_only" if timeout_sec is not None else "none_gaia_official_timeout",
"status": status,
"termination_reason": termination_reason,
"failure_class": failure_class,
"failure": failure,
"official_reward": reward,
"cost_summary": cost,
"execution_summary": execution,
"phase_boundary_policy": {
"environment_setup_excluded_from_agent": True,
"verifier_excluded_from_agent": True,
"cleanup_excluded_from_agent": True,
"rare_owned_runtime_preparation_in_agent": True,
},
})
def _prepare_task_inputs(
*,
row: dict[str, Any],
dataset_root: Path,
output_root: Path,
sgar_root: Path,
invocation_module: Any,
trial_id: str,
) -> dict[str, Any]:
"""Prepare one GAIA task and return phase metadata without starting RARE."""
task_id = str(row["task_id"])
task_root = output_root / "tasks" / task_id
input_root = output_root / "inputs" / task_id
environment_started_at = _utc_now()
environment_started_mono = time.monotonic()
try:
task_root.mkdir(parents=True, exist_ok=False)
input_root.mkdir(parents=True, exist_ok=False)
raw_names = row.get("file_name") or []
raw_names = [raw_names] if isinstance(raw_names, str) else list(raw_names)
names = [str(name).removeprefix("2023/validation/") for name in raw_names]
for name, relative_name in zip(raw_names, names):
source = (dataset_root / "raw/2023/validation" / relative_name).resolve()
if dataset_root.resolve() not in source.parents or not source.is_file():
raise ValueError("gaia_attachment_outside_frozen_snapshot")
target = input_root / relative_name
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target)
task = AgentVisibleTask(task_id, str(row["Question"]), int(row["Level"]), tuple(names), "2023_validation")
evaluator = EvaluatorRecord(task_id, None, row["Final answer"], "2023_validation")
adapter = GaiaTaskAdapter(dataset_root=input_root, split="2023_validation")
prepared = invocation_module.prepare_task_invocation(
query=task.question,
input_specs=tuple(invocation_module.InputSourceSpec(logical_name=name, source_path=input_root / name) for name in names),
run_dir=task_root,
request_id=f"gaia-v1-{task_id}-{trial_id}",
allowed_public_input_roots=(input_root,),
project_root=sgar_root,
)
except Exception as exc:
task_root.mkdir(parents=True, exist_ok=True)
return {
"task_root": task_root,
"input_root": input_root,
"environment_started_at": environment_started_at,
"environment_ready_at": None,
"environment_seconds": time.monotonic() - environment_started_mono,
"error": {"type": type(exc).__name__, "message": str(exc)},
}
return {
"task_root": task_root,
"input_root": input_root,
"task": task,
"evaluator": evaluator,
"adapter": adapter,
"prepared": prepared,
"environment_started_at": environment_started_at,
"environment_ready_at": _utc_now(),
"environment_seconds": time.monotonic() - environment_started_mono,
"error": None,
}
async def run_validation(
dataset_root: Path,
sgar_root: Path,
experiment_manifest: Path,
output_root: Path,
*,
config_path: Path | None = None,
task_timeout_sec: float | None = None,
task_ids: set[str] | None = None,
max_tasks: int | None = None,
network_policy_mode: str = "disabled",
diagnostic_allow_manifest_drift: bool = False,
) -> dict[str, Any]:
manifest = _json(experiment_manifest)
if manifest.get("status") != "FROZEN" or not (manifest.get("freeze_gate") or {}).get("passed"):
raise ValueError("gaia_experiment_manifest_not_frozen")
sys.path.insert(0, str(sgar_root.resolve()))
identity_checker = importlib.import_module("evaluation.gaia.check_manifest_identity")
manifest_identity = identity_checker.check_identity(
sgar_root.resolve(), experiment_manifest.resolve(), Path(__file__).resolve().parent
)
if not manifest_identity["ready_for_exact_manifest_run"] and not diagnostic_allow_manifest_drift:
raise ValueError(
"gaia_experiment_manifest_identity_mismatch:"
+ ",".join(manifest_identity["mismatches"])
)
if not manifest_identity["ready_for_exact_manifest_run"] and diagnostic_allow_manifest_drift:
if max_tasks not in {0, 1} or (max_tasks == 1 and (not task_ids or len(task_ids) != 1)):
raise ValueError("gaia_manifest_drift_diagnostic_requires_one_explicit_task")
dataset_manifest = _json(dataset_root / "dataset_manifest.json")
if dataset_manifest.get("task_count") != 165 or not dataset_manifest.get("immutable"):
raise ValueError("gaia_dataset_snapshot_not_frozen")
visible_rows = _jsonl(dataset_root / "views/agent_visible.jsonl")
evaluator_by_id = {row["task_id"]: row for row in _jsonl(dataset_root / "views/evaluator_only.jsonl")}
if len(visible_rows) != 165 or set(evaluator_by_id) != {row["task_id"] for row in visible_rows}:
raise ValueError("gaia_dataset_views_identity_mismatch")
sgar_main = importlib.import_module("sgar_mvp.main")
invocation_module = importlib.import_module("sgar_mvp.src.task_invocation")
resolved_config = (config_path or (sgar_root / "sgar_mvp/config.json")).resolve()
config = sgar_main.load_config(str(resolved_config), resolve_secrets=True)
if config is None:
raise ValueError("gaia_sgar_config_missing")
output_root.mkdir(parents=True, exist_ok=True)
(output_root / "manifest_identity_audit.json").write_text(
json.dumps(manifest_identity, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
results: list[dict[str, Any]] = []
started = time.monotonic()
selected_rows = [row for row in visible_rows if task_ids is None or str(row["task_id"]) in task_ids]
if max_tasks is not None:
selected_rows = selected_rows[:max_tasks]
experiment_id = str(manifest.get("experiment_id") or "rare-tf-gaia")
for index, row in enumerate(selected_rows):
task_id = str(row["task_id"])
trial_id = str(uuid.uuid4())
trial_started_mono = time.monotonic()
trial_started_at = _utc_now()
prep = _prepare_task_inputs(
row={**row, "Final answer": evaluator_by_id[task_id]["Final answer"]},
dataset_root=dataset_root,
output_root=output_root,
sgar_root=sgar_root,
invocation_module=invocation_module,
trial_id=trial_id,
)
task_root = prep["task_root"]
environment_started_at = prep["environment_started_at"]
environment_ready_at = prep["environment_ready_at"]
environment_seconds = prep["environment_seconds"]
if prep["error"] is not None:
trial_finished_at = _utc_now()
e2e_seconds = time.monotonic() - trial_started_mono
_write_phase_records(
task_root,
experiment_id=experiment_id,
trial_id=trial_id,
task_id=task_id,
level=int(row["Level"]),
method_id=manifest["method_id"],
environment_started_at=environment_started_at,
environment_ready_at=environment_ready_at,
environment_seconds=environment_seconds,
agent_started_at=None,
agent_finished_at=None,
agent_seconds=None,
verifier_started_at=None,
verifier_finished_at=None,
verifier_seconds=None,
trial_started_at=trial_started_at,
trial_finished_at=trial_finished_at,
e2e_seconds=e2e_seconds,
timeout_sec=task_timeout_sec,
status="infrastructure_failure",
termination_reason="environment_setup_failure",
failure_class="infrastructure_failure",
failure=prep["error"],
reward=None,
cost={},
execution={},
final_path=None,
verifier_record={
"protocol": "sgar-gaia-verifier-v1",
"task_id": task_id,
"status": "not_started",
"reason": "environment_setup_failure",
},
)
results.append({
"task_id": task_id,
"level": int(row["Level"]),
"ordinal": index + 1,
"method_id": manifest["method_id"],
"status": "infrastructure_failure",
"failure": prep["error"],
"failure_taxonomy": "infrastructure_failure",
"score": 0.0,
"cost_usd": None,
"model_calls": 0,
"tokens": {},
"agent_delegations": 0,
"tool_calls": 0,
"environment_setup_seconds": environment_seconds,
"agent_execution_seconds": None,
"verifier_seconds": None,
"e2e_wall_clock_seconds": e2e_seconds,
"timeout_limit_seconds": task_timeout_sec,
"trial_id": trial_id,
})
continue
task = prep["task"]
evaluator = prep["evaluator"]
adapter = prep["adapter"]
prepared = prep["prepared"]
local_config = dict(config)
local_config["runtime_settings"] = {
**dict(config.get("runtime_settings") or {}),
"output_root": str(task_root),
"temporary_directory": str(task_root / "runtime_tmp"),
"pip_cache_directory": str(task_root / "pip_cache"),
}
error = None
agent_started_at = _utc_now()
agent_started_mono = time.monotonic()
agent_finished_at = None
agent_seconds = None
timeout_triggered = False
try:
pipeline_call = sgar_main.run_pipeline(
local_config, task.question, str(task_root), str(task_root / "experiment_report.md"),
run_id=f"gaia-v1-{task_id}", task_invocation=prepared,
max_generation_requests=(
None if manifest["budget"]["max_generation_requests_per_task"] is None
else int(manifest["budget"]["max_generation_requests_per_task"])
),
max_embedding_requests=(
None if manifest["budget"]["max_embedding_requests_per_task"] is None
else int(manifest["budget"]["max_embedding_requests_per_task"])
),
network_policy_mode=network_policy_mode,
)
if task_timeout_sec is None:
status = await pipeline_call
else:
status = await asyncio.wait_for(pipeline_call, timeout=task_timeout_sec)
except asyncio.TimeoutError:
timeout_triggered = True
status = "task_timeout"
error = {"type": "TaskTimeoutError", "message": f"agent phase exceeded {task_timeout_sec}s"}
except Exception as exc:
status = "runner_exception"
error = {"type": type(exc).__name__, "message": str(exc)}
finally:
agent_finished_at = _utc_now()
agent_seconds = time.monotonic() - agent_started_mono
final_path, final = _final_output(task_root)
verifier_started_at = _utc_now()
verifier_started_mono = time.monotonic()
evaluation = adapter.evaluate(task, evaluator, final)
verifier_finished_at = _utc_now()
verifier_seconds = time.monotonic() - verifier_started_mono
trial_finished_at = _utc_now()
e2e_seconds = time.monotonic() - trial_started_mono
cost = _json(task_root / "cost_summary.json") if (task_root / "cost_summary.json").exists() else {}
execution = _json(task_root / "execution_summary.json") if (task_root / "execution_summary.json").exists() else {}
failure_taxonomy = _structured_failure_class(task_root) if str(status) == "structured_failure" else None
reward = float(evaluation["scorer"]["score"])
failure_taxonomy = failure_taxonomy or _failure_class(str(status), error, reward)
termination_reason = "task_timeout" if timeout_triggered else ("finished" if reward > 0 else str(status))
verifier_record = {
"protocol": "sgar-gaia-verifier-v1",
"task_id": task_id,
"status": "ok",
"reward": reward,
"scorer": evaluation,
"verifier_started_at": verifier_started_at,
"verifier_finished_at": verifier_finished_at,
"verifier_seconds": verifier_seconds,
}
_write_phase_records(
task_root,
experiment_id=experiment_id,
trial_id=trial_id,
task_id=task_id,
level=task.level,
method_id=manifest["method_id"],
environment_started_at=environment_started_at,
environment_ready_at=environment_ready_at,
environment_seconds=environment_seconds,
agent_started_at=agent_started_at,
agent_finished_at=agent_finished_at,
agent_seconds=agent_seconds,
verifier_started_at=verifier_started_at,
verifier_finished_at=verifier_finished_at,
verifier_seconds=verifier_seconds,
trial_started_at=trial_started_at,
trial_finished_at=trial_finished_at,
e2e_seconds=e2e_seconds,
timeout_sec=task_timeout_sec,
status=str(status),
termination_reason=termination_reason,
failure_class=failure_taxonomy,
failure=error,
reward=reward,
cost=cost,
execution=execution,
final_path=final_path,
verifier_record=verifier_record,
)
result = {
"task_id": task_id,
"level": task.level,
"ordinal": index + 1,
"method_id": manifest["method_id"],
"experiment_manifest_sha256": manifest["manifest_sha256"],
"resource_pool_hash": manifest_identity["checks"]["resources.effective_pool_sha256"]["current"],
"declared_resource_pool_hash": manifest["resources"]["effective_pool_sha256"],
"formal_comparable": manifest_identity["ready_for_exact_manifest_run"],
"status": str(status),
"raw_final_output_path": str(final_path) if final_path else None,
"model_answer": evaluation["answer"],
"score": evaluation["scorer"]["score"],
"failure": error,
"failure_taxonomy": failure_taxonomy,
"model_calls": cost.get("started_call_count", 0),
"tokens": cost.get("token_usage", {}),
"cost_usd": cost.get("observed_total_model_cost_usd"),
"agent_delegations": execution.get("controller_session_count", 0),
"tool_calls": execution.get("by_resource_type", {}).get("Tool", 0),
"environment_setup_seconds": environment_seconds,
"agent_execution_seconds": agent_seconds,
"verifier_seconds": verifier_seconds,
"e2e_wall_clock_seconds": e2e_seconds,
"timeout_limit_seconds": task_timeout_sec,
"trial_id": trial_id,
}
(task_root / "benchmark_result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
results.append(result)
levels = {level: [row for row in results if row["level"] == level] for level in (1, 2, 3)}
total_cost = sum(float(row["cost_usd"] or 0) for row in results)
summary = {
"protocol": "sgar-gaia-validation-results-v1",
"experiment_manifest_sha256": manifest["manifest_sha256"],
"formal_comparable": manifest_identity["ready_for_exact_manifest_run"],
"manifest_identity_mismatches": manifest_identity["mismatches"],
"task_count": len(results),
"accuracy": sum(row["score"] for row in results) / max(len(results), 1),
"level_accuracy": {
f"L{level}": sum(row["score"] for row in rows) / max(len(rows), 1)
for level, rows in levels.items()
},
"total_cost_usd": total_cost,
"mean_cost_usd": total_cost / max(len(results), 1),
"model_calls": sum(row["model_calls"] for row in results),
"agent_delegations": sum(row["agent_delegations"] for row in results),
"tool_calls": sum(row["tool_calls"] for row in results),
"input_tokens": sum((row["tokens"] or {}).get("input_tokens", 0) for row in results),
"output_tokens": sum((row["tokens"] or {}).get("output_tokens", 0) for row in results),
"latency_seconds": time.monotonic() - started,
"failure_taxonomy": {
key: sum(1 for row in results if row["failure_taxonomy"] == key)
for key in sorted({row["failure_taxonomy"] for row in results if row["failure_taxonomy"]})
},
"results": results,
}
(output_root / "validation_results.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return summary
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--dataset-root", type=Path, required=True)
parser.add_argument("--sgar-root", type=Path, required=True)
parser.add_argument("--experiment-manifest", type=Path, required=True)
parser.add_argument("--output-root", type=Path, required=True)
parser.add_argument("--config", type=Path, default=None)
parser.add_argument(
"--task-timeout-sec",
type=float,
default=None,
help="Optional local cap for the RARE agent phase only; GAIA has no official task wall timeout.",
)
parser.add_argument("--task-id", action="append", dest="task_ids")
parser.add_argument("--max-tasks", type=int, default=None)
parser.add_argument(
"--network-policy", choices=("disabled", "declared"), default="disabled",
help="Permit network access only for Tools that declare it.",
)
parser.add_argument(
"--diagnostic-allow-manifest-drift", action="store_true",
help="Label a single-case diagnostic as non-comparable when the frozen manifest drifts.",
)
args = parser.parse_args()
print(json.dumps(asyncio.run(run_validation(
args.dataset_root.resolve(),
args.sgar_root.resolve(),
args.experiment_manifest.resolve(),
args.output_root.resolve(),
config_path=args.config.resolve() if args.config else None,
task_timeout_sec=args.task_timeout_sec,
task_ids=set(args.task_ids) if args.task_ids else None,
max_tasks=args.max_tasks,
network_policy_mode=args.network_policy,
diagnostic_allow_manifest_drift=args.diagnostic_allow_manifest_drift,
)), ensure_ascii=False, indent=2))