deer-flow/backend/scripts/benchmark/deermem_scope_isolation/report.py

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

from .contract import Protocol
from .grading import grade_routing_row, grade_semantic_rows
from .runner import MARKER_SCHEMA_VERSION, ROW_SCHEMA_VERSION, _atomic_write_json, _case_fingerprint, _routing_fingerprint, protocol_artifacts, row_is_intact


class RowIntegrityError(ValueError):
    pass


def _read_row(path: Path) -> dict[str, Any]:
    try:
        row = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, ValueError) as exc:
        raise RowIntegrityError(f"missing or invalid benchmark row: {path}") from exc
    if not isinstance(row, dict) or row.get("schema_version") != ROW_SCHEMA_VERSION:
        raise RowIntegrityError(f"unsupported benchmark row: {path}")
    if not row_is_intact(row):
        raise RowIntegrityError(f"benchmark row failed its result-integrity hash: {path}")
    return row


def collect_rows(protocol: Protocol, output_dir: Path, marker: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any] | None]:
    semantic: list[dict[str, Any]] = []
    for case in protocol.semantic_cases:
        row = _read_row(output_dir / "rows" / f"{case.case_id}.json")
        if row.get("row_id") != case.case_id or row.get("suite") != "semantic_model_quality" or row.get("request_fingerprint") != _case_fingerprint(case, marker):
            raise RowIntegrityError(f"row {case.case_id} does not match the current protocol")
        if row.get("update_succeeded") is not True:
            raise RowIntegrityError(f"row {case.case_id} contains a failed memory update; rerun the benchmark")
        if (
            row.get("expected_persisted_canaries") != list(case.expected_persisted_canaries)
            or row.get("expected_rejected_canaries") != list(case.expected_rejected_canaries)
            or row.get("expected_removed_canaries") != list(case.expected_removed_canaries)
        ):
            raise RowIntegrityError(f"row {case.case_id} expected outcomes were changed")
        semantic.append(row)
    routing = None
    if marker["mode"] == "offline":
        routing = _read_row(output_dir / "rows" / f"{protocol.routing_case.case_id}.json")
        if routing.get("row_id") != protocol.routing_case.case_id or routing.get("suite") != "deterministic_identity_routing" or routing.get("request_fingerprint") != _routing_fingerprint(protocol, marker):
            raise RowIntegrityError("routing row does not match the current protocol")
    return semantic, routing


def build_report(protocol: Protocol, output_dir: Path, marker: dict[str, Any]) -> dict[str, Any]:
    semantic, routing = collect_rows(protocol, output_dir, marker)
    report = {
        "schema_version": 1,
        "protocol_id": protocol.protocol_id,
        "mode": marker["mode"],
        "artifacts": marker["artifacts"],
        "model": marker["model"],
        "semantic_model_quality": grade_semantic_rows(semantic),
        "deterministic_identity_routing": grade_routing_row(routing) if routing is not None else None,
    }
    return report


def write_report(protocol: Protocol, *, manifest_path: Path, output_dir: Path) -> Path:
    marker_path = output_dir / "run.json"
    if not marker_path.exists():
        raise ValueError(f"{marker_path} is missing; run the benchmark first")
    marker = json.loads(marker_path.read_text(encoding="utf-8"))
    if marker.get("mode") not in {"offline", "live"}:
        raise ValueError(f"{marker_path} has an invalid execution mode")
    if marker.get("schema_version") != MARKER_SCHEMA_VERSION or marker.get("protocol_id") != protocol.protocol_id or marker.get("artifacts") != protocol_artifacts(manifest_path) or not isinstance(marker.get("model"), dict):
        raise ValueError(f"{marker_path} no longer matches the current protocol/source")
    target = output_dir / "report.json"
    if target.exists():
        raise FileExistsError(f"refusing to overwrite existing report: {target}")
    _atomic_write_json(target, build_report(protocol, output_dir, marker))
    return target