mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-11 14:38:38 +00:00
a2808e8292
* test(checkpoint): retention deletion contract + growth baseline Six contract scenarios x memory/sqlite/postgres pin what retention deletions must never break (branch ancestors, explicit resume targets, pending writes, duration-only chain links), prove the two safe shapes (leaf sibling branches, trailing duration leaves), record the full-vs-delta growth baseline in the normalized bench shape, and add an item 4 probe showing the default ToolOutputBudgetMiddleware already externalizes oversized tool results. Refs #4189 * test(checkpoint): make the retention contract load-bearing per review Review findings from willem-bd and Ricky-7-Yan: - scenario D pins its own row: before/after stats delta plus a serde round-trip of the stored write, instead of an always-true > 0 check - _delete_checkpoint now performs the joint delete the doc mandates (checkpoint row + writes rows + blobs unreachable from surviving checkpoints), so E1/E2 exercise the shape they prescribe - E1 builds the real runtime duration shape via persist_run_durations (parent dict clone, fresh id/ts, real metadata), which surfaces the shared-version case: the leaf's blobs are the surviving parent's rows - contract doc: blob reachability must be computed from surviving checkpoints in a whole-thread pass; shared-version/duration-only hazard called out explicitly; memory data model includes saver.blobs - _stats counts memory blob rows and returns the full normalized shape (logical byte totals included) - probe: drops the unused middleware/outputs_dir graph parameters and discloses the manual-harness scope limit in the module docstring - E1/E2 assert default head resolution (protected set item 5); unused graph_for helper and DURATION_ONLY_METADATA stand-in removed Signed-off-by: zengbohan1 <310902929+zengbohan1@users.noreply.github.com> * fix(checkpoint): scope probe cleanup to owned dirs, key report by backend Second-round review findings on #5255: - [P1] bench_tool_result_probe.py removed the whole user-supplied --outputs-dir (and the shared .probe-tmp) in its finally block, so pre-existing files were deleted on success and failure alike. The run now writes into (and removes) a fresh owned probe-run-* child beneath the requested directory, and SQLite databases live in a unique mkdtemp'd temp directory that is removed with the run. Regression tests pin that unrelated pre-existing files survive both a successful and a simulated failing run. - [P2] the optional retention report keyed every backend's measurements under one shared name, so a multi-backend invocation kept only the last backend's numbers. _report() now takes the parameterized backend explicitly (saver_env.kind); regression pins that memory and sqlite entries coexist in one report file. Signed-off-by: zengbohan1 <310902929+zengbohan1@users.noreply.github.com> --------- Signed-off-by: zengbohan1 <310902929+zengbohan1@users.noreply.github.com> Co-authored-by: zengbohan1 <310902929+zengbohan1@users.noreply.github.com>
59 lines
2.4 KiB
Python
59 lines
2.4 KiB
Python
"""The probe may only clean directories it owns (review finding on #5255).
|
|
|
|
``--outputs-dir`` is user-supplied and may pre-exist with unrelated files:
|
|
on success or failure, the probe must remove only its own per-run
|
|
``probe-run-*`` child (plus its unique temp directory), never the requested
|
|
directory itself or anything beside it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
def _load_probe_module() -> object:
|
|
path = Path(__file__).resolve().parents[1] / "scripts" / "benchmark" / "checkpoint" / "bench_tool_result_probe.py"
|
|
spec = importlib.util.spec_from_file_location("bench_tool_result_probe_under_test", path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def test_successful_run_leaves_unrelated_files_alone(tmp_path: Path) -> None:
|
|
module = _load_probe_module()
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
(outputs_dir / "user-file.txt").write_text("keep me", encoding="utf-8")
|
|
tmp_dir = tmp_path / "scratch"
|
|
|
|
report = module.run_probe(20_000, outputs_dir, tmp_dir)
|
|
|
|
assert report["verdict"]["raw_content_is_full"] is True
|
|
assert report["verdict"]["wrapped_content_stays_small"] is True
|
|
assert (outputs_dir / "user-file.txt").read_text(encoding="utf-8") == "keep me", "unrelated content must survive"
|
|
assert not list(outputs_dir.glob("probe-run-*")), "the owned child must be removed"
|
|
assert not tmp_dir.exists(), "the owned temp directory must be removed with the run"
|
|
|
|
|
|
def test_failing_run_still_leaves_unrelated_files_alone(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
module = _load_probe_module()
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
(outputs_dir / "user-file.txt").write_text("keep me", encoding="utf-8")
|
|
|
|
def boom(*args: object, **kwargs: object) -> None:
|
|
raise RuntimeError("simulated probe failure")
|
|
|
|
monkeypatch.setattr(module, "_main", boom)
|
|
with pytest.raises(RuntimeError):
|
|
module.run_probe(20_000, outputs_dir, tmp_path / "scratch")
|
|
|
|
assert (outputs_dir / "user-file.txt").read_text(encoding="utf-8") == "keep me", "cleanup on failure must still be scoped to the owned child"
|
|
assert not list(outputs_dir.glob("probe-run-*"))
|