2026-11-05 16:22:12 +08:00

101 lines
3.8 KiB
Python

"""Task/evaluator adapter for GAIA, deliberately separate from SGAR routing."""
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
from typing import Any, Mapping
from .answer import GaiaAnswer, extract_model_answer
from .scorer import score_answer
@dataclass(frozen=True)
class AgentVisibleTask:
task_id: str
question: str
level: int | None
attachments: tuple[str, ...] = ()
split: str = "synthetic"
@dataclass(frozen=True)
class EvaluatorRecord:
task_id: str
final_answer: Any
expected_answer: Any
split: str
class GaiaTaskAdapter:
"""Load rows while making evaluator-only fields structurally inaccessible."""
protocol = "sgar-gaia-task-adapter-v1"
def __init__(self, *, dataset_root: str | Path, split: str = "synthetic"):
self.dataset_root = Path(dataset_root).resolve()
self.split = split
def load_row(self, row: Mapping[str, Any]) -> tuple[AgentVisibleTask, EvaluatorRecord]:
task_id = str(row.get("task_id") or row.get("id") or "").strip()
if not task_id:
raise ValueError("gaia_task_id_missing")
question = str(row.get("Question") or row.get("question") or "").strip()
if not question:
raise ValueError("gaia_question_missing")
raw_attachments = row.get("file_name") or row.get("attachments") or ()
if isinstance(raw_attachments, str):
raw_attachments = (raw_attachments,)
attachments = tuple(str(item) for item in raw_attachments if str(item))
level = row.get("Level") if row.get("Level") is not None else row.get("level")
level = int(level) if level is not None else None
visible = AgentVisibleTask(task_id, question, level, attachments, self.split)
expected_answer = row.get(
"Final answer",
row.get("final_answer", row.get("expected_answer")),
)
evaluator = EvaluatorRecord(task_id, None, expected_answer, self.split)
return visible, evaluator
def visible_context(self, task: AgentVisibleTask) -> dict[str, Any]:
payload = {
"task_id": task.task_id,
"question": task.question,
"level": task.level,
"attachments": list(task.attachments),
"split": task.split,
}
if any("answer" in key.casefold() for key in payload):
raise AssertionError("gaia_answer_leakage")
return payload
def evaluate(self, task: AgentVisibleTask, evaluator: EvaluatorRecord, final_output: Any) -> dict[str, Any]:
if task.task_id != evaluator.task_id:
raise ValueError("gaia_task_evaluator_identity_mismatch")
answer: GaiaAnswer = extract_model_answer(final_output)
result = score_answer(answer.model_answer, evaluator.expected_answer)
return {
"protocol": "sgar-gaia-evaluation-v1",
"task_id": task.task_id,
"split": task.split,
"answer": {"model_answer": answer.model_answer, "status": answer.status, "extraction": answer.extraction},
"scorer": result,
}
def attachment_path(self, task: AgentVisibleTask, name: str) -> Path:
if name not in task.attachments:
raise ValueError("gaia_attachment_not_declared")
path = (self.dataset_root / name).resolve()
if self.dataset_root not in path.parents or not path.is_file():
raise ValueError("gaia_attachment_missing_or_outside_dataset")
return path
def row_without_gold(row: Mapping[str, Any]) -> dict[str, Any]:
"""Return a defensive visible projection, useful for leakage tests."""
return {
key: value for key, value in row.items()
if key.casefold() not in {"final answer", "final_answer", "expected_answer", "answer", "gold"}
}