deer-flow/backend/tests/test_bench_deermem_eviction.py
Xinmin Zeng 3b1578fd91
test(backend): consolidate 57 fragmented single-fix files into 8 module suites (#8372)
* test(backend): consolidate 57 fragmented single-fix files into 8 module suites

* chore(tests): update benchmark README command and remap 603 shard durations

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
2026-11-19 20:05:04 +08:00

1136 lines
52 KiB
Python

"""Contract, dataset, grading, policy, protocol, QA, and reporting suite for the DeerMem eviction benchmark."""
from __future__ import annotations
import hashlib
import inspect
import json
from dataclasses import replace
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import httpx
import pytest
from scripts.benchmark.deermem_eviction.config import load_evaluation_config
from scripts.benchmark.deermem_eviction.dataset import DatasetIntegrityError, EvidenceRecord, LongMemEvalDataset, build_distractor_bank, extract_evidence, load_longmemeval
from scripts.benchmark.deermem_eviction.grading import GRADER_VERSION, OVERLAP_THRESHOLD, grade_answer, normalize_answer
from scripts.benchmark.deermem_eviction.io import sha256_file
from scripts.benchmark.deermem_eviction.manifest import OfficialManifest, load_official_manifest, load_synthetic_manifest
from scripts.benchmark.deermem_eviction.policy import PolicyResult, evaluate_case
from scripts.benchmark.deermem_eviction.pool import PreparedCase, build_case
from scripts.benchmark.deermem_eviction.protocol import _distractors, validate_official_selection
from scripts.benchmark.deermem_eviction.provider import ProviderCallError, ProviderConfigurationError, ProviderSettings, build_client, request_answer, resolve_provider_settings
from scripts.benchmark.deermem_eviction.qa import build_answer_task, render_answer_messages
from scripts.benchmark.deermem_eviction.report import AnswerRowIntegrityError, collect_answer_rows, compute_qa_statistics, grade_answer_rows, summarize_qa_rows, write_qa_report
from scripts.benchmark.deermem_eviction.results import summarize_policy_results, write_policy_run
from scripts.benchmark.deermem_eviction.runner import ensure_run_config_identity, response_path, run_answer_calls, verify_run_identity
from scripts.benchmark.deermem_eviction.stats import exact_mcnemar, paired_bootstrap_difference
BACKEND_ROOT = Path(__file__).parents[1]
EVAL_ROOT = BACKEND_ROOT / "scripts" / "benchmark" / "deermem_eviction"
def _load_config():
return load_evaluation_config(EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml")
# --- test_bench_deermem_eviction_contracts ---
def _all_keys(value: object) -> set[str]:
if isinstance(value, dict):
return set(value) | {key for child in value.values() for key in _all_keys(child)}
if isinstance(value, list):
return {key for child in value for key in _all_keys(child)}
return set()
def test_committed_protocol_is_pinned_and_does_not_copy_longmemeval_text() -> None:
config = load_evaluation_config(EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml")
official = load_official_manifest(EVAL_ROOT / "manifests" / "longmemeval-pr4789-v1.json")
synthetic = load_synthetic_manifest(EVAL_ROOT / "manifests" / "synthetic-corrections-pr4789-v1.json")
assert config.schema_version == 1
assert config.protocol_id == "deermem-hybrid-v1-pr4789-reproduction-v1"
assert config.dataset.revision == "98d7416c24c778c2fee6e6f3006e7a073259d48f"
assert config.dataset.sha256 == "821a2034d219ab45846873dd14c14f12cfe7776e73527a483f9dac095d38620c"
assert config.pool.size == 10
assert config.pool.capacities == [5, 7, 9]
assert config.pool.qa_capacity == 7
assert sum(config.policies.hybrid_v1.weights.values()) == 1.0
official_ids = [question_id for question_ids in official.scenarios.values() for question_id in question_ids]
assert len(official_ids) == 40
assert len(set(official_ids)) == 40
assert set(official.scenarios) == {
"confirmation_help",
"access_help",
"confidence_control",
"noisy_signal_control",
}
assert official.scenario_order == [
"confirmation_help",
"access_help",
"confidence_control",
"noisy_signal_control",
]
assert len(synthetic.cases) == 5
raw_official = json.loads((EVAL_ROOT / "manifests" / "longmemeval-pr4789-v1.json").read_text())
assert not ({"question", "answer", "content", "haystack_sessions"} & _all_keys(raw_official))
prompt_path = EVAL_ROOT / config.qa.answer_prompt.path
assert sha256_file(prompt_path) == config.qa.answer_prompt.sha256
def test_cli_contract_validation_is_offline(capsys) -> None:
from scripts.benchmark.deermem_eviction.cli import main
assert main(["validate-contracts"]) == 0
assert "validated 40 official and 5 synthetic cases" in capsys.readouterr().out
def test_required_policy_version_is_checked_against_production() -> None:
import pytest
from deerflow.agents.memory.backends.deermem.deermem.core.eviction import EVICTION_POLICY_HYBRID_V1
from scripts.benchmark.deermem_eviction.policy import require_production_policy
config = load_evaluation_config(EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml")
assert config.required_policy_version == EVICTION_POLICY_HYBRID_V1
require_production_policy(config.required_policy_version)
with pytest.raises(ValueError, match="production implements"):
require_production_policy("hybrid-v2")
# --- test_bench_deermem_eviction_dataset ---
def _dataset_row(question_id: str, question_type: str, content: str) -> dict[str, object]:
return {
"question_id": question_id,
"question_type": question_type,
"question": f"Question for {question_id}?",
"answer": "answer",
"question_date": "2026/08/13 (Thu) 10:00",
"haystack_session_ids": [f"session-{question_id}"],
"haystack_dates": ["2026/08/01 (Sat) 09:00"],
"haystack_sessions": [[{"role": "user", "content": content, "has_answer": True}]],
}
def test_load_longmemeval_rejects_hash_mismatch(tmp_path: Path) -> None:
dataset_path = tmp_path / "oracle.json"
dataset_path.write_text("[]")
with pytest.raises(DatasetIntegrityError, match="SHA-256 mismatch"):
load_longmemeval(dataset_path, expected_sha256="0" * 64)
def test_load_longmemeval_indexes_unique_rows(tmp_path: Path) -> None:
rows = [_dataset_row("b", "knowledge-update", "b" * 50), _dataset_row("a", "temporal-reasoning", "a" * 50)]
payload = json.dumps(rows).encode()
dataset_path = tmp_path / "oracle.json"
dataset_path.write_bytes(payload)
dataset = load_longmemeval(dataset_path, expected_sha256=hashlib.sha256(payload).hexdigest())
assert sorted(dataset.rows_by_id) == ["a", "b"]
def test_extract_evidence_uses_marked_turns_and_per_session_user_fallback() -> None:
row = {
"haystack_session_ids": ["s1", "s2"],
"haystack_dates": ["2026/01/01", "2026/01/02"],
"haystack_sessions": [
[
{"role": "user", "content": "ignore unmarked", "has_answer": False},
{"role": "assistant", "content": "marked answer", "has_answer": True},
],
[
{"role": "user", "content": "fallback user"},
{"role": "assistant", "content": "ignore fallback assistant"},
],
],
}
assert extract_evidence(row) == ("SESSION s1 AT 2026/01/01\nASSISTANT: marked answer\n\nSESSION s2 AT 2026/01/02\nUSER: fallback user")
def test_distractor_bank_is_filtered_sorted_and_bounded() -> None:
rows = [
_dataset_row("z", "single-session-user", "z" * 60),
_dataset_row("a", "single-session-preference", "a" * 60),
_dataset_row("wrong-type", "knowledge-update", "x" * 60),
_dataset_row("too-long", "single-session-user", "x" * 800),
]
bank = build_distractor_bank(rows, allowed_types={"single-session-user", "single-session-preference"}, min_evidence_chars=40, max_evidence_chars=700, limit=40)
assert [item.question_id for item in bank] == ["a", "z"]
# --- test_bench_deermem_eviction_grading ---
def test_config_pins_the_committed_grader_version() -> None:
config = load_evaluation_config(EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml")
assert config.qa.grader_version == GRADER_VERSION == "deterministic-overlap-v1"
def test_grader_is_blind_by_construction() -> None:
parameters = inspect.signature(grade_answer).parameters
assert list(parameters) == ["prediction", "reference"]
def test_normalization_lowercases_strips_and_maps_number_words() -> None:
assert normalize_answer("Seven WEEKS!") == ["7", "weeks"]
assert normalize_answer("fifteen") == ["15"]
assert normalize_answer("eleven") == ["eleven"]
assert normalize_answer("70-200mm zoom lens") == ["70", "200mm", "zoom", "lens"]
assert normalize_answer(" \t\n ") == []
def test_empty_and_insufficient_predictions_are_rejected() -> None:
empty = grade_answer("", "3 weeks")
assert not empty.correct
assert empty.rule == "empty-prediction"
assert not grade_answer(" ", "3 weeks").correct
result = grade_answer("INSUFFICIENT.", "3 weeks")
assert not result.correct
assert result.rule == "insufficient"
assert not grade_answer("insufficient", "3 weeks").correct
def test_empty_reference_is_a_contract_error() -> None:
with pytest.raises(ValueError):
grade_answer("3 weeks", " . ")
def test_exact_match_ignores_case_punctuation_and_number_words() -> None:
assert grade_answer("Every week.", "every week").rule == "exact"
assert grade_answer("NO", "no").rule == "exact"
assert grade_answer("seven", "7").rule == "exact"
def test_substring_matches_are_token_level_and_bidirectional() -> None:
assert grade_answer("132 points", "132").rule == "substring"
assert grade_answer("132", "132 points").rule == "substring"
assert grade_answer("Ford F-150 pickup truck.", "Ford F-150").rule == "substring"
assert grade_answer("Ford F-150 pickup truck.", "a Ford F-150").correct
assert not grade_answer("5", "25").correct
assert not grade_answer("no", "north").correct
def test_conflicting_numeric_answers_are_rejected() -> None:
result = grade_answer("5", "3 weeks")
assert not result.correct
assert result.rule == "numeric-conflict"
assert not grade_answer("12 weeks", "8 weeks").correct
def test_numbers_inside_an_explicit_reference_range_are_accepted() -> None:
reference = "ranging from 5 to 10 hours"
assert grade_answer("7", reference).rule == "numeric-range"
assert grade_answer("5 hours", reference).rule == "numeric-range"
assert grade_answer("10", reference).correct
assert not grade_answer("4", reference).correct
assert not grade_answer("11", reference).correct
assert grade_answer("7", "ranging from 5 dollars to 10 dollars").rule == "numeric-range"
def test_overlap_requires_sixty_percent_in_both_directions() -> None:
assert OVERLAP_THRESHOLD == 0.6
accepted = grade_answer("under my bed", "under the bed")
assert accepted.correct
assert accepted.rule == "overlap-accept"
assert grade_answer("red kite string", "red kite ribbon").correct
assert not grade_answer("red kite", "red balloon ribbon string flag").correct
assert not grade_answer("50mm prime lens", "70-200mm zoom lens").correct
assert not grade_answer("YES", "NO").correct
def test_all_stopword_predictions_cannot_pass_overlap() -> None:
result = grade_answer("of the", "3 weeks")
assert not result.correct
assert result.rule == "overlap-reject"
# --- test_bench_deermem_eviction_policy ---
NOW = datetime(2026, 8, 13, tzinfo=UTC)
def _evidence(question_id: str, question_type: str = "single-session-user") -> EvidenceRecord:
return EvidenceRecord(
question_id=question_id,
question_type=question_type,
question=f"Question for {question_id}?",
answer="answer",
question_date="2026/08/13 (Thu) 10:00",
content=f"Evidence for {question_id}",
)
def test_confirmation_scenario_is_deterministic_and_uses_production_policies() -> None:
support = _evidence("support", "knowledge-update")
distractors = [_evidence(f"d{i}") for i in range(9)]
first = build_case(
support=support,
distractors=distractors,
scenario="confirmation_help",
loss_rank=8,
evaluation_time=NOW,
)
second = build_case(
support=support,
distractors=distractors,
scenario="confirmation_help",
loss_rank=8,
evaluation_time=NOW,
)
assert first == second
assert len(first.facts) == 10
assert [fact["id"] for fact in first.facts] == sorted(fact["id"] for fact in first.facts)
confidence = evaluate_case(first, policy_name="confidence", capacity=7)
hybrid = evaluate_case(first, policy_name="hybrid-v1", capacity=7)
assert confidence.support_all_retained is False
assert hybrid.support_all_retained is True
assert confidence.kept_fact_ids != hybrid.kept_fact_ids
assert all("content" not in evicted for evicted in hybrid.evicted)
def test_correction_reserve_retains_a_low_confidence_correction() -> None:
support = _evidence("correction_case", "synthetic-correction")
distractors = [_evidence(f"d{i}") for i in range(9)]
case = build_case(
support=support,
distractors=distractors,
scenario="correction_reserve",
loss_rank=8,
evaluation_time=NOW,
)
confidence = evaluate_case(case, policy_name="confidence", capacity=7)
hybrid = evaluate_case(case, policy_name="hybrid-v1", capacity=7)
assert confidence.support_all_retained is False
assert hybrid.support_all_retained is True
assert hybrid.reserved_correction_slots == 1
def test_policy_result_contains_ids_and_scores_but_not_dataset_text() -> None:
case = build_case(
support=_evidence("support", "knowledge-update"),
distractors=[_evidence(f"d{i}") for i in range(9)],
scenario="access_help",
loss_rank=6,
evaluation_time=NOW,
)
result = evaluate_case(case, policy_name="hybrid-v1", capacity=7).to_public_dict()
assert result["case_id"] == "support"
assert result["support_fact_ids"] == ["gold_support"]
assert all(fact_id.startswith("d_support_") or fact_id == "gold_support" for fact_id in result["kept_fact_ids"])
assert "question" not in result
assert "answer" not in result
assert "facts" not in result
# --- test_bench_deermem_eviction_protocol ---
SCENARIO_ORDER = ["confirmation_help", "access_help", "confidence_control", "noisy_signal_control"]
def _protocol_row(question_id: str, question_type: str, *, answer: str = "short answer", evidence_chars: int = 120) -> dict[str, Any]:
return {
"question_id": question_id,
"question_type": question_type,
"question": f"question for {question_id}?",
"answer": answer,
"haystack_session_ids": ["s1"],
"haystack_dates": ["2023/05/20"],
"haystack_sessions": [[{"role": "user", "content": "x" * evidence_chars, "has_answer": True}]],
}
def _eligible_ids(prefix: str) -> list[str]:
return [f"{prefix}-{index:03d}" for index in range(1, 21)]
def _dataset(rows: list[dict[str, Any]]) -> LongMemEvalDataset:
return LongMemEvalDataset(path=Path("synthetic"), sha256="0" * 64, rows=tuple(rows), rows_by_id={row["question_id"]: row for row in rows})
def _rows_with_exclusions() -> list[dict[str, Any]]:
rows = [_protocol_row(question_id, "knowledge-update") for question_id in _eligible_ids("ku")]
rows += [_protocol_row(question_id, "temporal-reasoning") for question_id in _eligible_ids("tr")]
# Every excluded row sorts before the eligible IDs, so a broken exclusion changes the recomputed selection.
rows.append(_protocol_row("ku-000-pilot", "knowledge-update"))
rows.append(_protocol_row("ku-000_abs", "knowledge-update"))
rows.append(_protocol_row("ku-000-long-answer", "knowledge-update", answer="a" * 150))
rows.append(_protocol_row("ku-000-refusal", "knowledge-update", answer="there is not enough information"))
rows.append(_protocol_row("ku-000-evidence", "knowledge-update", evidence_chars=2500))
return rows
def _manifest(*, excluded_pilot_ids: list[str] | None = None, scenarios: dict[str, list[str]] | None = None) -> OfficialManifest:
if scenarios is None:
ku, tr = _eligible_ids("ku"), _eligible_ids("tr")
scenarios = {scenario: ku[index * 5 : (index + 1) * 5] + tr[index * 5 : (index + 1) * 5] for index, scenario in enumerate(SCENARIO_ORDER)}
return OfficialManifest.model_validate(
{
"schema_version": 1,
"protocol_id": "synthetic-protocol",
"selection": {
"eligible_question_types": ["knowledge-update", "temporal-reasoning"],
"excluded_pilot_ids": ["ku-000-pilot"] if excluded_pilot_ids is None else excluded_pilot_ids,
"exclude_abstention_suffix": "_abs",
"answer_min_chars": 1,
"answer_max_chars": 100,
"answer_excluded_substrings": ["not enough", "only mentioned"],
"evidence_min_chars": 1,
"evidence_max_chars": 2000,
"take_per_question_type": 20,
"cases_per_type_per_scenario": 5,
},
"scenario_order": SCENARIO_ORDER,
"loss_ranks": [6, 6, 6, 8, 8, 8, 10, 10, 10, 10],
"scenarios": scenarios,
}
)
def test_selection_recomputation_accepts_a_manifest_matching_the_published_rule() -> None:
validate_official_selection(_dataset(_rows_with_exclusions()), _manifest())
def test_selection_recomputation_rejects_ids_that_break_the_rule() -> None:
ku, tr = _eligible_ids("ku"), _eligible_ids("tr")
scenarios = {scenario: ku[index * 5 : (index + 1) * 5] + tr[index * 5 : (index + 1) * 5] for index, scenario in enumerate(SCENARIO_ORDER)}
scenarios["confirmation_help"], scenarios["access_help"] = (
scenarios["confirmation_help"][:9] + [scenarios["access_help"][9]],
scenarios["access_help"][:9] + [scenarios["confirmation_help"][9]],
)
with pytest.raises(DatasetIntegrityError, match="do not match the declared selection rule"):
validate_official_selection(_dataset(_rows_with_exclusions()), _manifest(scenarios=scenarios))
def test_selection_recomputation_applies_every_published_exclusion() -> None:
manifest = _manifest()
excluded = {"ku-000-pilot", "ku-000_abs", "ku-000-long-answer", "ku-000-refusal", "ku-000-evidence"}
pinned = {question_id for question_ids in manifest.scenarios.values() for question_id in question_ids}
assert not (excluded & pinned)
# Dropping the pilot exclusion changes the recomputed selection, so validation must fail against the pinned IDs.
with pytest.raises(DatasetIntegrityError):
validate_official_selection(_dataset(_rows_with_exclusions()), _manifest(excluded_pilot_ids=[]))
def test_selection_recomputation_requires_enough_eligible_rows() -> None:
rows = [row for row in _rows_with_exclusions() if row["question_id"] != "tr-020"]
with pytest.raises(DatasetIntegrityError, match="not enough eligible"):
validate_official_selection(_dataset(rows), _manifest())
def _bank(size: int = 5) -> list[EvidenceRecord]:
return [EvidenceRecord(question_id=f"b{index}", question_type="single-session-user", question="q?", answer="a", question_date=None, content=f"evidence {index}") for index in range(size)]
def test_distractor_offset_derivation_and_wraparound_are_pinned() -> None:
# Hardcoded expectations for sha256(f"offset-ns:{case_id}").digest()[:4] % 5;
# a regression in the digest slice, modulus, or wraparound changes these indices.
bank = _bank()
no_wrap = _distractors("case-a", bank, count=3, namespace="offset-ns")
assert [record.question_id for record in no_wrap] == ["b0", "b1", "b2"]
wraps = _distractors("case-c", bank, count=3, namespace="offset-ns")
assert [record.question_id for record in wraps] == ["b3", "b4", "b0"]
assert [record.question_id for record in _distractors("case-a", bank, count=3, namespace="alt-ns")] == ["b1", "b2", "b3"]
def test_distractor_selection_requires_a_large_enough_bank() -> None:
with pytest.raises(DatasetIntegrityError, match="required"):
_distractors("case-a", _bank(size=2), count=3, namespace="offset-ns")
# --- test_bench_deermem_eviction_published_results ---
RESULTS_ROOT = EVAL_ROOT / "results" / "pr4789-reproduction-v1"
ALLOWED_ROW_KEYS = {
"schema_version",
"row_id",
"case_id",
"source",
"scenario",
"question_type",
"policy",
"capacity",
"kept_fact_ids",
"support_all_retained",
"support_recall",
"prediction",
"grade_correct",
"grade_rule",
"grader_version",
"attempts",
"request_fingerprint",
"response_model",
"usage",
}
def _rows() -> list[dict]:
return [json.loads(line) for line in (RESULTS_ROOT / "qa.rows.jsonl").read_text(encoding="utf-8").splitlines()]
def test_published_rows_contain_only_allowed_metadata() -> None:
rows = _rows()
config = load_evaluation_config(EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml")
assert len(rows) == 90
assert {row["policy"] for row in rows} == {"confidence", "hybrid-v1"}
for row in rows:
assert set(row) == ALLOWED_ROW_KEYS
assert row["capacity"] == config.pool.qa_capacity
assert len(row["kept_fact_ids"]) == config.pool.qa_capacity
assert row["grader_version"] == GRADER_VERSION
assert isinstance(row["prediction"], str)
def test_published_statistics_are_recomputable_from_the_rows() -> None:
rows = _rows()
config = load_evaluation_config(EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml")
statistics = json.loads((RESULTS_ROOT / "qa.stats.json").read_text(encoding="utf-8"))
grades: dict[str, dict[str, bool]] = {}
sources: dict[str, str] = {}
for row in rows:
grades.setdefault(row["case_id"], {})[row["policy"]] = row["grade_correct"]
sources[row["case_id"]] = row["source"]
suites = {
"official": [case_id for case_id in sorted(grades) if sources[case_id] == "longmemeval"],
"synthetic": [case_id for case_id in sorted(grades) if sources[case_id] == "synthetic"],
"overall": sorted(grades),
}
for suite, case_ids in suites.items():
pairs = [(grades[case_id]["confidence"], grades[case_id]["hybrid-v1"]) for case_id in case_ids]
expected = exact_mcnemar(pairs)
published = statistics["suites"][suite]["mcnemar"]
assert published["p_value"] == expected.p_value
assert published["only_first_correct"] == expected.only_first_correct
assert published["only_second_correct"] == expected.only_second_correct
assert statistics["suites"][suite]["cases"] == len(pairs)
expected_bootstrap = paired_bootstrap_difference(pairs, seed=config.statistics.bootstrap_seed, iterations=config.statistics.bootstrap_iterations, alpha=config.statistics.alpha)
published_bootstrap = statistics["suites"][suite]["bootstrap"]
assert published_bootstrap["mean_difference"] == expected_bootstrap.mean_difference
assert published_bootstrap["lower"] == expected_bootstrap.lower
assert published_bootstrap["upper"] == expected_bootstrap.upper
assert (published_bootstrap["seed"], published_bootstrap["iterations"], published_bootstrap["alpha"]) == (expected_bootstrap.seed, expected_bootstrap.iterations, expected_bootstrap.alpha)
def test_published_summary_matches_the_rows_and_run_provenance_is_secret_free() -> None:
rows = _rows()
summary = json.loads((RESULTS_ROOT / "qa.summary.json").read_text(encoding="utf-8"))
for group in summary["groups"]:
matching = [row for row in rows if (row["source"], row["scenario"], row["policy"]) == (group["source"], group["scenario"], group["policy"])]
assert len(matching) == group["cases"]
assert sum(1 for row in matching if row["grade_correct"]) == group["correct"]
run = json.loads((RESULTS_ROOT / "qa_run.json").read_text(encoding="utf-8"))
serialized = json.dumps(run)
assert "sk-" not in serialized
assert run["qa"]["api_key_env"] == "DEERMEM_EVAL_ANSWER_API_KEY"
assert run["dataset"]["sha256"] == load_evaluation_config(EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml").dataset.sha256
# --- test_bench_deermem_eviction_qa ---
def _template() -> str:
return (EVAL_ROOT / "prompts" / "answer-v1.txt").read_text(encoding="utf-8")
def _qa_case(case_id: str = "case-1", question_date: str | None = "2023/05/20 (Sat) 02:21") -> PreparedCase:
config = _load_config()
facts = [
{"id": "fact-b", "content": "The cat sleeps in the study.", "category": "context", "confidence": 0.7, "createdAt": "2026-02-14T00:00:00Z", "source": "synthetic"},
{"id": "fact-a", "content": "The user moved to Lyon.", "category": "context", "confidence": 0.9, "createdAt": "2026-02-14T00:00:00Z", "source": "synthetic"},
{"id": "fact-c", "content": "The user has two bikes.", "category": "context", "confidence": 0.8, "createdAt": "2026-02-14T00:00:00Z", "source": "synthetic"},
]
return PreparedCase(
case_id=case_id,
source="synthetic",
scenario="correction_reserve",
question_type="synthetic-correction",
question="Where does the user live?",
answer="Lyon",
question_date=question_date,
evaluation_time=config.evaluation_time,
facts=facts,
usage={},
support_fact_ids=("fact-a",),
)
def _qa_policy_result(case: PreparedCase, kept: tuple[str, ...], policy: str = "hybrid-v1") -> PolicyResult:
return PolicyResult(
case_id=case.case_id,
source=case.source,
scenario=case.scenario,
question_type=case.question_type,
policy=policy, # type: ignore[arg-type]
capacity=7,
support_fact_ids=case.support_fact_ids,
kept_fact_ids=kept,
evicted=(),
scores={},
support_all_retained=True,
support_recall=1.0,
reserved_correction_slots=0,
)
def test_rendering_pins_fact_order_date_line_and_block_format() -> None:
messages = render_answer_messages(
_template(),
question="Where does the user live?",
question_date="2023/05/20 (Sat) 02:21",
retained_facts=[("fact-b", "The cat sleeps in the study."), ("fact-a", "The user moved to Lyon.")],
)
assert messages[0]["role"] == "system"
assert messages[0]["content"] == "Answer using only the stored memory below. If the answer is not supported, output exactly INSUFFICIENT. For a YES/NO question, output only YES or NO. Otherwise give only the shortest direct answer."
assert messages[1]["role"] == "user"
assert messages[1]["content"] == ("CURRENT DATE: 2023/05/20 (Sat) 02:21\nSTORED MEMORY:\n[fact-a]\nThe user moved to Lyon.\n\n[fact-b]\nThe cat sleeps in the study.\n\nQUESTION: Where does the user live?")
def test_rendering_omits_the_date_line_when_absent() -> None:
messages = render_answer_messages(_template(), question="Q?", question_date=None, retained_facts=[("fact-a", "content")])
assert messages[1]["content"].startswith("STORED MEMORY:\n")
assert "CURRENT DATE" not in messages[1]["content"]
def test_rendering_rejects_empty_or_duplicate_facts() -> None:
with pytest.raises(ValueError):
render_answer_messages(_template(), question="Q?", question_date=None, retained_facts=[])
with pytest.raises(ValueError):
render_answer_messages(_template(), question="Q?", question_date=None, retained_facts=[("fact-a", "x"), ("fact-a", "y")])
def test_build_answer_task_renders_only_kept_facts() -> None:
case = _qa_case()
task = build_answer_task(case, _qa_policy_result(case, kept=("fact-a", "fact-c")), _template())
assert task.row_id == "case-1__hybrid-v1"
assert task.kept_fact_ids == ("fact-a", "fact-c")
user = task.messages[1]["content"]
assert "[fact-a]" in user and "[fact-c]" in user
assert "fact-b" not in user
with pytest.raises(ValueError):
build_answer_task(_qa_case(case_id="other"), _qa_policy_result(case, kept=("fact-a",)), _template())
def test_provider_settings_errors_name_the_missing_variables(monkeypatch: pytest.MonkeyPatch) -> None:
config = _load_config()
monkeypatch.delenv(config.qa.api_key_env, raising=False)
monkeypatch.delenv(config.qa.base_url_env, raising=False)
with pytest.raises(ProviderConfigurationError) as excinfo:
resolve_provider_settings(config.qa)
assert config.qa.api_key_env in str(excinfo.value)
assert config.qa.base_url_env in str(excinfo.value)
monkeypatch.setenv(config.qa.api_key_env, "secret-key")
monkeypatch.setenv(config.qa.base_url_env, "https://example.invalid/v1")
settings = resolve_provider_settings(config.qa)
assert settings.api_key == "secret-key"
assert settings.base_url == "https://example.invalid/v1"
def _mock_client(handler, qa) -> httpx.Client:
return httpx.Client(base_url="https://example.invalid/v1", headers={"Authorization": "Bearer secret-key"}, transport=httpx.MockTransport(handler), timeout=qa.timeout_seconds)
def _success_body(prediction: str = "Lyon") -> dict:
return {"model": "deepseek-v4-flash", "choices": [{"message": {"role": "assistant", "content": prediction}}], "usage": {"prompt_tokens": 100, "completion_tokens": 3, "detail": "ignored"}}
def test_request_answer_parses_prediction_and_non_secret_metadata() -> None:
qa = _load_config().qa
seen: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
return httpx.Response(200, json=_success_body())
answer = request_answer(_mock_client(handler, qa), qa, ({"role": "system", "content": "s"}, {"role": "user", "content": "u"}), backoff_seconds=0)
assert answer.prediction == "Lyon"
assert answer.attempts == 1
assert answer.usage == {"prompt_tokens": 100, "completion_tokens": 3}
assert answer.response_model == "deepseek-v4-flash"
payload = json.loads(seen[0].content)
assert payload["model"] == qa.model
assert payload["temperature"] == qa.temperature
assert payload["max_tokens"] == qa.max_tokens
assert payload["stream"] is qa.stream
assert seen[0].url.path.endswith("/chat/completions")
def test_request_answer_retries_retryable_failures_and_gives_up() -> None:
qa = _load_config().qa
statuses = [429, 500]
def flaky(request: httpx.Request) -> httpx.Response:
if statuses:
return httpx.Response(statuses.pop(0), json={})
return httpx.Response(200, json=_success_body())
answer = request_answer(_mock_client(flaky, qa), qa, ({"role": "user", "content": "u"},), backoff_seconds=0)
assert answer.attempts == 3
def always_broken(request: httpx.Request) -> httpx.Response:
raise httpx.ConnectError("boom", request=request)
with pytest.raises(ProviderCallError, match="failed after 3 attempts"):
request_answer(_mock_client(always_broken, qa), qa, ({"role": "user", "content": "u"},), backoff_seconds=0)
def test_request_answer_rejects_non_retryable_and_malformed_responses() -> None:
qa = _load_config().qa
with pytest.raises(ProviderCallError, match="non-retryable status 401"):
request_answer(_mock_client(lambda request: httpx.Response(401, json={}), qa), qa, ({"role": "user", "content": "u"},), backoff_seconds=0)
with pytest.raises(ProviderCallError, match="choices"):
request_answer(_mock_client(lambda request: httpx.Response(200, json={"choices": []}), qa), qa, ({"role": "user", "content": "u"},), backoff_seconds=0)
def test_build_client_uses_configured_timeout_and_bearer_header() -> None:
config = _load_config()
with build_client(ProviderSettings(base_url="https://example.invalid/v1", api_key="secret-key"), config.qa) as client:
assert client.headers["Authorization"] == "Bearer secret-key"
assert client.timeout.read == config.qa.timeout_seconds
def _tasks(count: int = 2) -> list:
template = _template()
tasks = []
for index in range(count):
case = _qa_case(case_id=f"case-{index}")
tasks.append(build_answer_task(case, _qa_policy_result(case, kept=("fact-a", "fact-b")), template))
return tasks
def test_run_answer_calls_persists_rows_and_resumes_without_new_calls(tmp_path: Path) -> None:
config = _load_config()
calls = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(request)
return httpx.Response(200, json=_success_body())
tasks = _tasks()
with _mock_client(handler, config.qa) as client:
first = run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert (first.reused, first.called, first.failed) == (0, 2, ())
assert len(calls) == 2
row = json.loads(response_path(tmp_path, tasks[0].row_id).read_text(encoding="utf-8"))
assert set(row) == {"schema_version", "row_id", "case_id", "source", "scenario", "policy", "capacity", "kept_fact_ids", "prediction", "attempts", "request_fingerprint", "response_model", "usage", "created_at"}
assert row["prediction"] == "Lyon"
serialized = json.dumps(row)
assert "Where does the user live" not in serialized
assert "The user moved to Lyon." not in serialized
assert "Bearer" not in serialized
second = run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert (second.reused, second.called, second.failed) == (2, 0, ())
assert len(calls) == 2
response_path(tmp_path, tasks[0].row_id).write_text("{not json", encoding="utf-8")
third = run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert (third.reused, third.called, third.failed) == (1, 1, ())
assert len(calls) == 3
def test_run_answer_calls_refuses_to_reuse_a_row_bound_to_another_case(tmp_path: Path) -> None:
config = _load_config()
calls = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(request)
return httpx.Response(200, json=_success_body())
tasks = _tasks()
with _mock_client(handler, config.qa) as client:
run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert len(calls) == 2
path = response_path(tmp_path, tasks[0].row_id)
original = json.loads(path.read_text(encoding="utf-8"))
for field, value in (("case_id", tasks[1].case_id), ("source", f"not-{tasks[0].source}"), ("scenario", f"not-{tasks[0].scenario}")):
path.write_text(json.dumps(dict(original, **{field: value})), encoding="utf-8")
report = run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert (report.reused, report.called, report.failed) == (1, 1, ())
assert len(calls) == 5
def test_run_answer_calls_reports_failures_without_writing_rows(tmp_path: Path) -> None:
config = _load_config()
def broken(request: httpx.Request) -> httpx.Response:
return httpx.Response(500, json={})
tasks = _tasks(count=1)
with _mock_client(broken, config.qa) as client:
report = run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert report.called == 0
assert len(report.failed) == 1
assert tasks[0].row_id in report.failed[0]
assert not response_path(tmp_path, tasks[0].row_id).exists()
def test_run_directory_is_bound_to_the_full_protocol_identity(tmp_path: Path) -> None:
config = _load_config()
paths = {
"config_path": EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml",
"official_manifest_path": EVAL_ROOT / "manifests" / "longmemeval-pr4789-v1.json",
"synthetic_manifest_path": EVAL_ROOT / "manifests" / "synthetic-corrections-pr4789-v1.json",
"prompt_path": EVAL_ROOT / "prompts" / "answer-v1.txt",
}
dataset_path = tmp_path / "dataset.json"
dataset_path.write_text("[]", encoding="utf-8")
backend_root = Path(__file__).parents[1]
output_dir = tmp_path / "run"
ensure_run_config_identity(output_dir, config=config, dataset_path=dataset_path, backend_root=backend_root, **paths)
marker = json.loads((output_dir / "qa_run.json").read_text(encoding="utf-8"))
assert marker["qa"]["model"] == config.qa.model
assert set(marker["artifacts"]) == {"config_sha256", "official_manifest_sha256", "synthetic_manifest_sha256", "answer_prompt_sha256", "dataset_sha256"}
assert "secret" not in json.dumps(marker).lower()
ensure_run_config_identity(output_dir, config=config, dataset_path=dataset_path, backend_root=backend_root, **paths)
for changed_key, marker_field in (("config_path", "config_sha256"), ("synthetic_manifest_path", "synthetic_manifest_sha256"), ("prompt_path", "answer_prompt_sha256")):
changed_paths = dict(paths)
changed_file = tmp_path / f"changed-{changed_key}"
changed_file.write_text(paths[changed_key].read_text(encoding="utf-8") + "\n", encoding="utf-8")
changed_paths[changed_key] = changed_file
with pytest.raises(ValueError, match=f"different protocol artifacts.*{marker_field}"):
ensure_run_config_identity(output_dir, config=config, dataset_path=dataset_path, backend_root=backend_root, **changed_paths)
def test_verify_run_identity_is_read_only_and_names_the_changed_artifact(tmp_path: Path) -> None:
config = _load_config()
paths = {
"config_path": EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml",
"official_manifest_path": EVAL_ROOT / "manifests" / "longmemeval-pr4789-v1.json",
"synthetic_manifest_path": EVAL_ROOT / "manifests" / "synthetic-corrections-pr4789-v1.json",
"prompt_path": EVAL_ROOT / "prompts" / "answer-v1.txt",
}
dataset_path = tmp_path / "dataset.json"
dataset_path.write_text("[]", encoding="utf-8")
output_dir = tmp_path / "run"
with pytest.raises(ValueError, match="missing"):
verify_run_identity(output_dir, dataset_path=dataset_path, **paths)
ensure_run_config_identity(output_dir, config=config, dataset_path=dataset_path, backend_root=Path(__file__).parents[1], **paths)
verify_run_identity(output_dir, dataset_path=dataset_path, **paths)
changed_prompt = tmp_path / "changed-prompt.txt"
changed_prompt.write_text(paths["prompt_path"].read_text(encoding="utf-8") + "\n", encoding="utf-8")
with pytest.raises(ValueError, match="refusing to grade.*|answer_prompt_sha256"):
verify_run_identity(output_dir, dataset_path=dataset_path, **{**paths, "prompt_path": changed_prompt})
def test_resume_revalidates_stored_rows_against_the_current_task(tmp_path: Path) -> None:
config = _load_config()
calls: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
calls.append(request)
return httpx.Response(200, json=_success_body())
tasks = _tasks(count=1)
with _mock_client(handler, config.qa) as client:
run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert len(calls) == 1
unchanged = run_answer_calls(tasks, config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert (unchanged.reused, unchanged.called) == (1, 0)
assert len(calls) == 1
changed_message = replace(tasks[0], messages=(tasks[0].messages[0], {"role": "user", "content": "a different question"}))
after_message_change = run_answer_calls([changed_message], config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert (after_message_change.reused, after_message_change.called) == (0, 1)
assert len(calls) == 2
changed_kept = replace(changed_message, kept_fact_ids=("fact-a",))
after_kept_change = run_answer_calls([changed_kept], config=config, client=client, output_dir=tmp_path, backoff_seconds=0)
assert (after_kept_change.reused, after_kept_change.called) == (0, 1)
assert len(calls) == 3
def test_cli_run_qa_fails_fast_without_provider_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
from scripts.benchmark.deermem_eviction.cli import main
config = _load_config()
monkeypatch.delenv(config.qa.api_key_env, raising=False)
monkeypatch.delenv(config.qa.base_url_env, raising=False)
with pytest.raises(ProviderConfigurationError) as excinfo:
main(["run-qa", "--dataset", str(tmp_path / "missing.json"), "--output-dir", str(tmp_path / "out")])
assert config.qa.api_key_env in str(excinfo.value)
# --- test_bench_deermem_eviction_report ---
def _report_case(case_id: str, *, source: str, scenario: str, answer: str) -> PreparedCase:
config = _load_config()
facts = [{"id": f"{case_id}-fact", "content": "Secret memory content.", "category": "context", "confidence": 0.9, "createdAt": "2026-02-14T00:00:00Z", "source": "synthetic"}]
return PreparedCase(
case_id=case_id,
source=source, # type: ignore[arg-type]
scenario=scenario, # type: ignore[arg-type]
question_type="synthetic-shape",
question="Secret question text?",
answer=answer,
question_date=None,
evaluation_time=config.evaluation_time,
facts=facts,
usage={},
support_fact_ids=(f"{case_id}-fact",),
)
def _report_policy_result(case: PreparedCase, policy: str) -> PolicyResult:
return PolicyResult(
case_id=case.case_id,
source=case.source,
scenario=case.scenario,
question_type=case.question_type,
policy=policy, # type: ignore[arg-type]
capacity=7,
support_fact_ids=case.support_fact_ids,
kept_fact_ids=case.support_fact_ids,
evicted=(),
scores={},
support_all_retained=True,
support_recall=1.0,
reserved_correction_slots=0,
)
def _write_response(output_dir: Path, case: PreparedCase, policy: str, prediction: str, *, capacity: int = 7) -> None:
row_id = f"{case.case_id}__{policy}"
path = response_path(output_dir, row_id)
path.parent.mkdir(parents=True, exist_ok=True)
row = {
"schema_version": 1,
"row_id": row_id,
"case_id": case.case_id,
"source": case.source,
"scenario": case.scenario,
"policy": policy,
"capacity": capacity,
"kept_fact_ids": list(case.support_fact_ids),
"prediction": prediction,
"attempts": 1,
"request_fingerprint": "f" * 64,
"response_model": "deepseek-v4-flash",
"usage": {"prompt_tokens": 10, "completion_tokens": 2},
"created_at": "2026-08-17T00:00:00Z",
}
path.write_text(json.dumps(row) + "\n", encoding="utf-8")
def _setup(tmp_path: Path):
official = _report_case("case-off", source="longmemeval", scenario="access_help", answer="Lyon-Reference-Secret")
synthetic = _report_case("case-syn", source="synthetic", scenario="correction_reserve", answer="NO")
cases = [official, synthetic]
_write_response(tmp_path, official, "confidence", "Paris")
_write_response(tmp_path, official, "hybrid-v1", "Lyon-Reference-Secret")
_write_response(tmp_path, synthetic, "confidence", "NO")
_write_response(tmp_path, synthetic, "hybrid-v1", "NO")
results = {f"{case.case_id}__{policy}": _report_policy_result(case, policy) for case in cases for policy in ("confidence", "hybrid-v1")}
fingerprints = {row_id: "f" * 64 for row_id in results}
return cases, results, fingerprints
def test_exact_mcnemar_matches_the_two_sided_exact_binomial() -> None:
concordant = exact_mcnemar([(True, True), (False, False)])
assert concordant.p_value == 1.0
one_sided_shift = exact_mcnemar([(False, True)] * 5)
assert one_sided_shift.only_second_correct == 5
assert one_sided_shift.p_value == pytest.approx(2 * 0.5**5)
mixed = exact_mcnemar([(False, True)] * 8 + [(True, False)] + [(True, True)] * 3)
assert (mixed.only_first_correct, mixed.only_second_correct, mixed.both_correct) == (1, 8, 3)
assert mixed.p_value == pytest.approx(2 * (0.5**9) * (1 + 9), rel=1e-12)
def test_paired_bootstrap_is_seed_deterministic_and_signed_second_minus_first() -> None:
config = _load_config()
pairs = [(False, True)] * 4 + [(True, True)] * 4
first = paired_bootstrap_difference(pairs, seed=config.statistics.bootstrap_seed, iterations=1000, alpha=config.statistics.alpha)
second = paired_bootstrap_difference(pairs, seed=config.statistics.bootstrap_seed, iterations=1000, alpha=config.statistics.alpha)
assert first == second
assert first.mean_difference == pytest.approx(0.5)
degenerate = paired_bootstrap_difference([(False, True)] * 3, seed=1, iterations=100, alpha=0.05)
assert (degenerate.mean_difference, degenerate.lower, degenerate.upper) == (1.0, 1.0, 1.0)
def test_grading_is_blind_and_joined_by_row_id(tmp_path: Path) -> None:
cases, results, fingerprints = _setup(tmp_path)
rows = collect_answer_rows(tmp_path, cases)
graded = grade_answer_rows(cases, results, rows, expected_fingerprints=fingerprints)
by_row = {row["row_id"]: row for row in graded}
assert not by_row["case-off__confidence"]["grade_correct"]
assert by_row["case-off__hybrid-v1"]["grade_correct"]
assert by_row["case-syn__confidence"]["grade_correct"]
assert by_row["case-syn__hybrid-v1"]["grade_correct"]
assert all(row["grader_version"] == "deterministic-overlap-v1" for row in graded)
def test_collect_and_integrity_checks_reject_incomplete_or_tampered_rows(tmp_path: Path) -> None:
cases, results, fingerprints = _setup(tmp_path)
response_path(tmp_path, "case-syn__hybrid-v1").unlink()
with pytest.raises(AnswerRowIntegrityError, match="case-syn__hybrid-v1"):
collect_answer_rows(tmp_path, cases)
_write_response(tmp_path, cases[1], "hybrid-v1", "NO", capacity=5)
rows = collect_answer_rows(tmp_path, cases)
with pytest.raises(AnswerRowIntegrityError, match="capacity/policy"):
grade_answer_rows(cases, results, rows, expected_fingerprints=fingerprints)
_write_response(tmp_path, cases[1], "hybrid-v1", "NO")
rows = collect_answer_rows(tmp_path, cases)
tampered = dict(results)
tampered["case-syn__hybrid-v1"] = _report_policy_result(_report_case("case-syn", source="synthetic", scenario="correction_reserve", answer="NO"), "hybrid-v1")
object.__setattr__(tampered["case-syn__hybrid-v1"], "kept_fact_ids", ("other-fact",))
with pytest.raises(AnswerRowIntegrityError, match="kept facts"):
grade_answer_rows(cases, tampered, rows, expected_fingerprints=fingerprints)
stale = dict(fingerprints)
stale["case-syn__hybrid-v1"] = "0" * 64
with pytest.raises(AnswerRowIntegrityError, match="request fingerprint"):
grade_answer_rows(cases, results, rows, expected_fingerprints=stale)
def test_grading_rejects_a_row_reassigned_to_another_valid_case(tmp_path: Path) -> None:
cases, results, fingerprints = _setup(tmp_path)
path = response_path(tmp_path, "case-off__hybrid-v1")
original = json.loads(path.read_text(encoding="utf-8"))
for field, value in (("case_id", "case-syn"), ("source", "synthetic"), ("scenario", "correction_reserve"), ("row_id", "case-syn__hybrid-v1")):
path.write_text(json.dumps(dict(original, **{field: value})) + "\n", encoding="utf-8")
rows = collect_answer_rows(tmp_path, cases)
with pytest.raises(AnswerRowIntegrityError, match="case-off__hybrid-v1.*case identity"):
grade_answer_rows(cases, results, rows, expected_fingerprints=fingerprints)
def test_summary_and_statistics_keep_suites_separate(tmp_path: Path) -> None:
cases, results, fingerprints = _setup(tmp_path)
config = _load_config()
graded = grade_answer_rows(cases, results, collect_answer_rows(tmp_path, cases), expected_fingerprints=fingerprints)
summary = summarize_qa_rows(graded)
assert {(group["source"], group["scenario"], group["policy"]): group["accuracy"] for group in summary} == {
("longmemeval", "access_help", "confidence"): 0.0,
("longmemeval", "access_help", "hybrid-v1"): 1.0,
("synthetic", "correction_reserve", "confidence"): 1.0,
("synthetic", "correction_reserve", "hybrid-v1"): 1.0,
}
statistics = compute_qa_statistics(graded, config)
assert statistics["suites"]["official"]["cases"] == 1
assert statistics["suites"]["synthetic"]["cases"] == 1
assert statistics["suites"]["overall"]["cases"] == 2
assert statistics["suites"]["official"]["mcnemar"]["only_second_correct"] == 1
assert statistics["suites"]["official"]["bootstrap"]["seed"] == config.statistics.bootstrap_seed
def test_written_report_redacts_dataset_text_and_refuses_overwrite(tmp_path: Path) -> None:
cases, results, fingerprints = _setup(tmp_path)
config = _load_config()
graded = grade_answer_rows(cases, results, collect_answer_rows(tmp_path, cases), expected_fingerprints=fingerprints)
write_qa_report(tmp_path, graded=graded, summary=summarize_qa_rows(graded), statistics=compute_qa_statistics(graded, config), config=config)
rows_text = (tmp_path / "qa.rows.jsonl").read_text(encoding="utf-8")
assert "Secret question text" not in rows_text
assert "Secret memory content" not in rows_text
confidence_official = json.loads(next(line for line in rows_text.splitlines() if '"case-off__confidence"' in line))
assert "Lyon-Reference-Secret" not in json.dumps(confidence_official)
assert (tmp_path / "qa.summary.json").exists()
assert (tmp_path / "qa.stats.json").exists()
with pytest.raises(FileExistsError):
write_qa_report(tmp_path, graded=graded, summary=summarize_qa_rows(graded), statistics=compute_qa_statistics(graded, config), config=config)
# --- test_bench_deermem_eviction_results ---
def _results_case() -> PreparedCase:
support = EvidenceRecord("support", "knowledge-update", "private question", "private answer", None, "private support text")
distractors = [EvidenceRecord(f"d{i}", "single-session-user", "q", "a", None, f"private distractor {i}") for i in range(9)]
return build_case(
support=support,
distractors=distractors,
scenario="confirmation_help",
loss_rank=8,
evaluation_time=datetime(2026, 8, 13, tzinfo=UTC),
)
def test_summary_keeps_policy_and_scenario_groups_separate() -> None:
case = _results_case()
rows = [
evaluate_case(case, policy_name="confidence", capacity=7),
evaluate_case(case, policy_name="hybrid-v1", capacity=7),
]
summary = summarize_policy_results(rows)
assert [(item["policy"], item["support_all_retained"]) for item in summary] == [
("confidence", 0),
("hybrid-v1", 1),
]
def test_public_policy_run_omits_dataset_text_and_refuses_overwrite(tmp_path: Path) -> None:
config_path = EVAL_ROOT / "configs" / "pr4789-reproduction-v1.yaml"
official_manifest_path = EVAL_ROOT / "manifests" / "longmemeval-pr4789-v1.json"
synthetic_manifest_path = EVAL_ROOT / "manifests" / "synthetic-corrections-pr4789-v1.json"
prompt_path = EVAL_ROOT / "prompts" / "answer-v1.txt"
config = load_evaluation_config(config_path)
case = _results_case()
rows = [evaluate_case(case, policy_name="confidence", capacity=7), evaluate_case(case, policy_name="hybrid-v1", capacity=7)]
dataset_path = tmp_path / "private-dataset.json"
dataset_path.write_text("[]")
output_dir = tmp_path / "run"
write_policy_run(
output_dir,
results=rows,
config=config,
config_path=config_path,
official_manifest_path=official_manifest_path,
synthetic_manifest_path=synthetic_manifest_path,
prompt_path=prompt_path,
dataset_path=dataset_path,
backend_root=BACKEND_ROOT,
)
raw_text = (output_dir / "policy.raw.jsonl").read_text()
assert "private question" not in raw_text
assert "private answer" not in raw_text
assert "private support text" not in raw_text
assert len([json.loads(line) for line in raw_text.splitlines()]) == 2
assert json.loads((output_dir / "summary.json").read_text())["protocol_id"] == config.protocol_id
assert json.loads((output_dir / "run.json").read_text())["dataset"]["filename"] == config.dataset.filename
with pytest.raises(FileExistsError, match="refusing to overwrite"):
write_policy_run(
output_dir,
results=rows,
config=config,
config_path=config_path,
official_manifest_path=official_manifest_path,
synthetic_manifest_path=synthetic_manifest_path,
prompt_path=prompt_path,
dataset_path=dataset_path,
backend_root=BACKEND_ROOT,
)