mirror of
https://github.com/OpenBMB/ChatDev.git
synced 2026-11-18 16:00:59 +00:00
101 lines
3.8 KiB
Python
101 lines
3.8 KiB
Python
"""Task/evaluator adapter for GAIA, deliberately separate from SGAR routing."""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, Mapping
|
|
|
|
from .answer import GaiaAnswer, extract_model_answer
|
|
from .scorer import score_answer
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AgentVisibleTask:
|
|
task_id: str
|
|
question: str
|
|
level: int | None
|
|
attachments: tuple[str, ...] = ()
|
|
split: str = "synthetic"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EvaluatorRecord:
|
|
task_id: str
|
|
final_answer: Any
|
|
expected_answer: Any
|
|
split: str
|
|
|
|
|
|
class GaiaTaskAdapter:
|
|
"""Load rows while making evaluator-only fields structurally inaccessible."""
|
|
|
|
protocol = "sgar-gaia-task-adapter-v1"
|
|
|
|
def __init__(self, *, dataset_root: str | Path, split: str = "synthetic"):
|
|
self.dataset_root = Path(dataset_root).resolve()
|
|
self.split = split
|
|
|
|
def load_row(self, row: Mapping[str, Any]) -> tuple[AgentVisibleTask, EvaluatorRecord]:
|
|
task_id = str(row.get("task_id") or row.get("id") or "").strip()
|
|
if not task_id:
|
|
raise ValueError("gaia_task_id_missing")
|
|
question = str(row.get("Question") or row.get("question") or "").strip()
|
|
if not question:
|
|
raise ValueError("gaia_question_missing")
|
|
raw_attachments = row.get("file_name") or row.get("attachments") or ()
|
|
if isinstance(raw_attachments, str):
|
|
raw_attachments = (raw_attachments,)
|
|
attachments = tuple(str(item) for item in raw_attachments if str(item))
|
|
level = row.get("Level") if row.get("Level") is not None else row.get("level")
|
|
level = int(level) if level is not None else None
|
|
visible = AgentVisibleTask(task_id, question, level, attachments, self.split)
|
|
expected_answer = row.get(
|
|
"Final answer",
|
|
row.get("final_answer", row.get("expected_answer")),
|
|
)
|
|
evaluator = EvaluatorRecord(task_id, None, expected_answer, self.split)
|
|
return visible, evaluator
|
|
|
|
def visible_context(self, task: AgentVisibleTask) -> dict[str, Any]:
|
|
payload = {
|
|
"task_id": task.task_id,
|
|
"question": task.question,
|
|
"level": task.level,
|
|
"attachments": list(task.attachments),
|
|
"split": task.split,
|
|
}
|
|
if any("answer" in key.casefold() for key in payload):
|
|
raise AssertionError("gaia_answer_leakage")
|
|
return payload
|
|
|
|
def evaluate(self, task: AgentVisibleTask, evaluator: EvaluatorRecord, final_output: Any) -> dict[str, Any]:
|
|
if task.task_id != evaluator.task_id:
|
|
raise ValueError("gaia_task_evaluator_identity_mismatch")
|
|
answer: GaiaAnswer = extract_model_answer(final_output)
|
|
result = score_answer(answer.model_answer, evaluator.expected_answer)
|
|
return {
|
|
"protocol": "sgar-gaia-evaluation-v1",
|
|
"task_id": task.task_id,
|
|
"split": task.split,
|
|
"answer": {"model_answer": answer.model_answer, "status": answer.status, "extraction": answer.extraction},
|
|
"scorer": result,
|
|
}
|
|
|
|
def attachment_path(self, task: AgentVisibleTask, name: str) -> Path:
|
|
if name not in task.attachments:
|
|
raise ValueError("gaia_attachment_not_declared")
|
|
path = (self.dataset_root / name).resolve()
|
|
if self.dataset_root not in path.parents or not path.is_file():
|
|
raise ValueError("gaia_attachment_missing_or_outside_dataset")
|
|
return path
|
|
|
|
|
|
def row_without_gold(row: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""Return a defensive visible projection, useful for leakage tests."""
|
|
return {
|
|
key: value for key, value in row.items()
|
|
if key.casefold() not in {"final answer", "final_answer", "expected_answer", "answer", "gold"}
|
|
}
|