mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
fix(checkpoints): preserve channel identity in version comparisons (#8330)
* fix(checkpoints): compare channel versions by identity * fix(checkpoints): preserve channel identity in retention * test(checkpoints): cover swapped channel versions in lineage * test(checkpoints): retain swapped channel-version state * test(checkpoints): make swapped channel regression deterministic * style(checkpoints): satisfy Ruff formatting * fix(checkpoints): preserve version maps in retention paths * test(checkpoints): keep surviving blobs after sibling pruning * fix(checkpoints): prune empty-map duration leaves safely
This commit is contained in:
parent
8d60bef1b4
commit
340822f2c1
@ -76,6 +76,21 @@ def _has_postgres_stamps(metadata: dict[str, Any]) -> bool:
|
||||
return "run_durations" in metadata or "run_message_ids" in metadata
|
||||
|
||||
|
||||
def channel_versions_match(
|
||||
versions: dict[str, Any] | None,
|
||||
parent_versions: dict[str, Any] | None,
|
||||
) -> bool:
|
||||
"""Whether two checkpoint version maps match channel-for-channel.
|
||||
|
||||
LangGraph versions are scoped to their channels, not globally unique.
|
||||
Comparing only the values can mistake a state-changing checkpoint for a
|
||||
metadata-only copy when two channels exchange their versions.
|
||||
"""
|
||||
|
||||
# Empty maps are equal too; markerless inference applies its own non-empty guard.
|
||||
return isinstance(versions, dict) and isinstance(parent_versions, dict) and versions == parent_versions
|
||||
|
||||
|
||||
def is_duration_only_checkpoint(checkpoint_tuple: Any, *, parent: Any | None = None, stamp_fallback: bool = True, versions: dict[str, Any] | None = None, parent_versions: dict[str, Any] | None = None) -> bool:
|
||||
"""Return whether the tuple is a metadata-only run-duration checkpoint.
|
||||
|
||||
@ -268,7 +283,7 @@ def _copies_parent_verbatim(checkpoint_tuple: Any, parent: Any, *, versions: dic
|
||||
parent_versions = parent_versions if isinstance(parent_versions, dict) else _snapshot_versions(parent)
|
||||
if not isinstance(versions, dict) or not isinstance(parent_versions, dict) or not versions:
|
||||
return False
|
||||
return frozenset(versions.values()) == frozenset(parent_versions.values())
|
||||
return channel_versions_match(versions, parent_versions)
|
||||
|
||||
|
||||
def history_parent_index(checkpoints: Sequence[Any]) -> dict[tuple[str, str], Any]:
|
||||
|
||||
@ -53,6 +53,7 @@ from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from app.gateway.checkpoint_lineage import (
|
||||
channel_versions_match,
|
||||
checkpoint_configurable,
|
||||
is_duration_only_checkpoint,
|
||||
)
|
||||
@ -105,7 +106,7 @@ class _Node:
|
||||
duration_only: bool
|
||||
metadata_source: str | None = None
|
||||
carries_run_durations: bool = False
|
||||
versions: frozenset = frozenset()
|
||||
versions: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _thread_config(thread_id: str) -> dict[str, Any]:
|
||||
@ -156,7 +157,8 @@ def _mark_duration_leaves_without_the_marker(nodes: dict[tuple[str, str], _Node]
|
||||
parent = nodes.get((node.parent_ns, node.parent_id))
|
||||
if parent is None or node.metadata_source != "update" or not node.carries_run_durations:
|
||||
continue
|
||||
if node.versions and node.versions == parent.versions:
|
||||
# Without the authoritative writes marker, require a non-empty map as evidence.
|
||||
if node.versions and parent.versions and channel_versions_match(node.versions, parent.versions):
|
||||
node.duration_only = True
|
||||
|
||||
|
||||
@ -452,7 +454,7 @@ async def enforce_thread_retention(
|
||||
duration_only=is_duration_only_checkpoint(tuple_, stamp_fallback=False),
|
||||
metadata_source=metadata.get("source"),
|
||||
carries_run_durations=isinstance(run_durations, dict) and bool(run_durations),
|
||||
versions=frozenset(channel_versions.values()) if isinstance(channel_versions, dict) else frozenset(),
|
||||
versions=dict(channel_versions) if isinstance(channel_versions, dict) else {},
|
||||
)
|
||||
if not nodes:
|
||||
# Nothing to classify: an empty (or unknown) thread is a no-op, but
|
||||
@ -538,7 +540,7 @@ async def enforce_thread_retention(
|
||||
survivor_versions: set[Any] = set()
|
||||
for key, node in nodes.items():
|
||||
if key not in deleted_keys:
|
||||
survivor_versions.update(node.versions)
|
||||
survivor_versions.update(node.versions.values())
|
||||
|
||||
# Invalidate before touching source-of-truth rows. If cache deletion
|
||||
# fails, fail closed with every checkpoint intact; doing this after the
|
||||
@ -603,7 +605,7 @@ async def enforce_trailing_duration_retention(
|
||||
|
||||
latest_versions = latest.checkpoint.get("channel_versions")
|
||||
parent_versions = parent.checkpoint.get("channel_versions")
|
||||
if not isinstance(latest_versions, dict) or latest_versions != parent_versions:
|
||||
if not channel_versions_match(latest_versions, parent_versions):
|
||||
return report
|
||||
metadata = latest.metadata or {}
|
||||
run_durations = metadata.get("run_durations")
|
||||
@ -615,9 +617,9 @@ async def enforce_trailing_duration_retention(
|
||||
duration_only=is_duration_only_checkpoint(latest),
|
||||
metadata_source=metadata.get("source"),
|
||||
carries_run_durations=isinstance(run_durations, dict) and bool(run_durations),
|
||||
versions=frozenset(latest_versions.values()),
|
||||
versions=dict(latest_versions),
|
||||
)
|
||||
parent_node = _Node(ns="", cp_id=parent_id, parent_ns=None, parent_id=None, duration_only=False, versions=latest_node.versions)
|
||||
parent_node = _Node(ns="", cp_id=parent_id, parent_ns=None, parent_id=None, duration_only=False, versions=dict(parent_versions))
|
||||
_mark_duration_leaves_without_the_marker({("", latest_id): latest_node, ("", parent_id): parent_node})
|
||||
if not latest_node.duration_only or await _checkpoint_has_writes(storage, ("", latest_id), thread_id):
|
||||
return report
|
||||
|
||||
@ -227,6 +227,20 @@ def test_goal_leaf_inheriting_stamps_is_not_duration_only():
|
||||
assert is_duration_only_checkpoint(goal_leaf, parent=parent) is False
|
||||
|
||||
|
||||
def test_swapped_channel_versions_are_not_a_duration_checkpoint():
|
||||
"""Version values belong to channel names; a permutation is a state change."""
|
||||
parent = _snapshot("ckpt-parent", [], channel_versions={"messages": 5, "goal": 2})
|
||||
swapped_leaf = _snapshot(
|
||||
"ckpt-swapped",
|
||||
[],
|
||||
parent_id="ckpt-parent",
|
||||
metadata=dict(_POSTGRES_DURATION_METADATA),
|
||||
channel_versions={"messages": 2, "goal": 5},
|
||||
)
|
||||
|
||||
assert is_duration_only_checkpoint(swapped_leaf, parent=parent) is False
|
||||
|
||||
|
||||
def test_postgres_attribution_only_duration_checkpoint_is_recognised():
|
||||
"""A leaf stamped for message attribution alone has an empty ``run_durations`` map."""
|
||||
parent = _snapshot("ckpt-real", [], channel_versions={"messages": 5})
|
||||
|
||||
@ -298,18 +298,23 @@ async def test_postgres_round_trip_shape_pruned_by_default(saver_env: _SaverEnv)
|
||||
# aput needs the full configurable (namespace included) for the parent link.
|
||||
return {"configurable": {"thread_id": thread_id, "checkpoint_ns": "", "checkpoint_id": parent_id}}
|
||||
|
||||
# Control: one NEW channel version on an otherwise verbatim clone — a
|
||||
# client ``update_state`` can produce this, the runtime writer cannot.
|
||||
control = copy.deepcopy(dict(head.checkpoint))
|
||||
# Seed two distinct channel counters so the regression is deterministic
|
||||
# even when the real graph writes only one state channel.
|
||||
baseline = copy.deepcopy(dict(head.checkpoint))
|
||||
baseline.update(_new_checkpoint_marker())
|
||||
baseline["channel_versions"] = {"messages": "v1", "goal": "v2"}
|
||||
baseline_id = baseline["id"]
|
||||
baseline_meta = {"source": "loop", "step": 1}
|
||||
await saver_env.saver.aput(_leaf_config(head_id), baseline, baseline_meta, {})
|
||||
|
||||
# Control: exchange the channel versions without changing their value set.
|
||||
# The channel ownership changed, so this checkpoint must remain resumable.
|
||||
control = copy.deepcopy(baseline)
|
||||
control.update(_new_checkpoint_marker())
|
||||
control["channel_versions"] = dict(control["channel_versions"])
|
||||
# A version value no real channel would carry; the classifier compares the
|
||||
# frozenset of versions against the parent's, so any strictly-different
|
||||
# set is what matters.
|
||||
control["channel_versions"]["_control"] = "__control_version__"
|
||||
control["channel_versions"] = {"messages": "v2", "goal": "v1"}
|
||||
control_id = control["id"]
|
||||
control_meta = _pg_round_trip_meta(head.metadata)
|
||||
await saver_env.saver.aput(_leaf_config(head_id), control, control_meta, {})
|
||||
control_meta = _pg_round_trip_meta(baseline_meta)
|
||||
await saver_env.saver.aput(_leaf_config(baseline_id), control, control_meta, {})
|
||||
|
||||
# The Postgres round-trip shape on top: verbatim clone of the control
|
||||
# (identical ``channel_versions``), marker popped from the metadata.
|
||||
@ -374,6 +379,8 @@ async def test_regenerated_old_head_pruned_opt_in(saver_env: _SaverEnv) -> None:
|
||||
assert fork_tuple is not None
|
||||
base = await _walk(saver_env, _config_thread(thread_id, fork_head_id), "fork-turn")
|
||||
assert base is not None
|
||||
materialized = await graph.aget_state(_config(thread_id))
|
||||
assert materialized.values["messages"][-1].content.startswith("regenerated turn:")
|
||||
remaining = await _listed_checkpoint_ids(saver_env, thread_id)
|
||||
assert old_head_id not in remaining
|
||||
assert {checkpoint_ids[0], checkpoint_ids[1], fork_head_id}.issubset(remaining)
|
||||
@ -832,7 +839,7 @@ async def test_admission_retention_bounds_storage_work_independently_of_history(
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
["pending-writes", "missing-parent", "no-parent-link", "changed-versions", "swapped-channels", "markerless", "empty-durations", "empty-write-entry"],
|
||||
["pending-writes", "missing-parent", "no-parent-link", "changed-versions", "swapped-channels", "markerless", "empty-durations", "empty-write-entry", "empty-channel-versions", "markerless-empty-channel-versions"],
|
||||
)
|
||||
async def test_admission_retention_checks_only_a_proven_metadata_leaf(
|
||||
saver_env: _SaverEnv,
|
||||
@ -851,12 +858,21 @@ async def test_admission_retention_checks_only_a_proven_metadata_leaf(
|
||||
saver_env.saver.writes[(thread_id, "", duration_id)] = {}
|
||||
elif shape == "pending-writes":
|
||||
await saver_env.saver.aput_writes(head.config, [("messages", [HumanMessage(content="pending")])], task_id="pending")
|
||||
elif shape in {"markerless", "empty-durations"}:
|
||||
elif shape in {"markerless", "empty-durations", "empty-channel-versions", "markerless-empty-channel-versions"}:
|
||||
metadata = copy.deepcopy(head.metadata)
|
||||
metadata.pop("writes", None)
|
||||
if shape in {"markerless", "empty-durations", "markerless-empty-channel-versions"}:
|
||||
metadata.pop("writes", None)
|
||||
if shape == "empty-durations":
|
||||
metadata["run_durations"] = {}
|
||||
await saver_env.saver.aput(head.parent_config, head.checkpoint, metadata, {})
|
||||
if shape in {"empty-channel-versions", "markerless-empty-channel-versions"}:
|
||||
parent_checkpoint = copy.deepcopy(parent.checkpoint)
|
||||
parent_checkpoint["channel_versions"] = {}
|
||||
await saver_env.saver.aput(parent.parent_config, parent_checkpoint, parent.metadata, {})
|
||||
checkpoint = copy.deepcopy(head.checkpoint)
|
||||
checkpoint["channel_versions"] = {}
|
||||
await saver_env.saver.aput(head.parent_config, checkpoint, metadata, {})
|
||||
else:
|
||||
await saver_env.saver.aput(head.parent_config, head.checkpoint, metadata, {})
|
||||
elif shape in {"changed-versions", "swapped-channels"}:
|
||||
checkpoint = copy.deepcopy(head.checkpoint)
|
||||
versions = checkpoint["channel_versions"]
|
||||
@ -880,11 +896,15 @@ async def test_admission_retention_checks_only_a_proven_metadata_leaf(
|
||||
|
||||
monkeypatch.setattr(saver_env.saver, "aget_tuple", incomplete_chain)
|
||||
|
||||
if shape == "empty-channel-versions" and saver_env.kind == "postgres":
|
||||
pytest.skip("Postgres removes the writes marker, so this marker-backed case applies to memory and SQLite")
|
||||
|
||||
report = await enforce_completed_run_retention(checkpointer=saver_env.saver, thread_id=thread_id)
|
||||
|
||||
assert report.deleted_checkpoint_ids == ([duration_id] if shape in {"markerless", "empty-write-entry"} else [])
|
||||
should_delete = shape in {"markerless", "empty-write-entry", "empty-channel-versions"}
|
||||
assert report.deleted_checkpoint_ids == ([duration_id] if should_delete else [])
|
||||
retained = await original_get(_config_thread(thread_id, duration_id))
|
||||
assert (retained is None) == (shape in {"markerless", "empty-write-entry"})
|
||||
assert (retained is None) == should_delete
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user