fix(checkpoints): preserve channel identity in version comparisons (#8330)

* fix(checkpoints): compare channel versions by identity

* fix(checkpoints): preserve channel identity in retention

* test(checkpoints): cover swapped channel versions in lineage

* test(checkpoints): retain swapped channel-version state

* test(checkpoints): make swapped channel regression deterministic

* style(checkpoints): satisfy Ruff formatting

* fix(checkpoints): preserve version maps in retention paths

* test(checkpoints): keep surviving blobs after sibling pruning

* fix(checkpoints): prune empty-map duration leaves safely
This commit is contained in:
NanPan 2026-11-18 19:57:37 +08:00 committed by GitHub
parent 8d60bef1b4
commit 340822f2c1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 75 additions and 24 deletions

View File

@ -76,6 +76,21 @@ def _has_postgres_stamps(metadata: dict[str, Any]) -> bool:
return "run_durations" in metadata or "run_message_ids" in metadata
def channel_versions_match(
versions: dict[str, Any] | None,
parent_versions: dict[str, Any] | None,
) -> bool:
"""Whether two checkpoint version maps match channel-for-channel.
LangGraph versions are scoped to their channels, not globally unique.
Comparing only the values can mistake a state-changing checkpoint for a
metadata-only copy when two channels exchange their versions.
"""
# Empty maps are equal too; markerless inference applies its own non-empty guard.
return isinstance(versions, dict) and isinstance(parent_versions, dict) and versions == parent_versions
def is_duration_only_checkpoint(checkpoint_tuple: Any, *, parent: Any | None = None, stamp_fallback: bool = True, versions: dict[str, Any] | None = None, parent_versions: dict[str, Any] | None = None) -> bool:
"""Return whether the tuple is a metadata-only run-duration checkpoint.
@ -268,7 +283,7 @@ def _copies_parent_verbatim(checkpoint_tuple: Any, parent: Any, *, versions: dic
parent_versions = parent_versions if isinstance(parent_versions, dict) else _snapshot_versions(parent)
if not isinstance(versions, dict) or not isinstance(parent_versions, dict) or not versions:
return False
return frozenset(versions.values()) == frozenset(parent_versions.values())
return channel_versions_match(versions, parent_versions)
def history_parent_index(checkpoints: Sequence[Any]) -> dict[tuple[str, str], Any]:

View File

@ -53,6 +53,7 @@ from langgraph.checkpoint.memory import InMemorySaver
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from app.gateway.checkpoint_lineage import (
channel_versions_match,
checkpoint_configurable,
is_duration_only_checkpoint,
)
@ -105,7 +106,7 @@ class _Node:
duration_only: bool
metadata_source: str | None = None
carries_run_durations: bool = False
versions: frozenset = frozenset()
versions: dict[str, Any] = field(default_factory=dict)
def _thread_config(thread_id: str) -> dict[str, Any]:
@ -156,7 +157,8 @@ def _mark_duration_leaves_without_the_marker(nodes: dict[tuple[str, str], _Node]
parent = nodes.get((node.parent_ns, node.parent_id))
if parent is None or node.metadata_source != "update" or not node.carries_run_durations:
continue
if node.versions and node.versions == parent.versions:
# Without the authoritative writes marker, require a non-empty map as evidence.
if node.versions and parent.versions and channel_versions_match(node.versions, parent.versions):
node.duration_only = True
@ -452,7 +454,7 @@ async def enforce_thread_retention(
duration_only=is_duration_only_checkpoint(tuple_, stamp_fallback=False),
metadata_source=metadata.get("source"),
carries_run_durations=isinstance(run_durations, dict) and bool(run_durations),
versions=frozenset(channel_versions.values()) if isinstance(channel_versions, dict) else frozenset(),
versions=dict(channel_versions) if isinstance(channel_versions, dict) else {},
)
if not nodes:
# Nothing to classify: an empty (or unknown) thread is a no-op, but
@ -538,7 +540,7 @@ async def enforce_thread_retention(
survivor_versions: set[Any] = set()
for key, node in nodes.items():
if key not in deleted_keys:
survivor_versions.update(node.versions)
survivor_versions.update(node.versions.values())
# Invalidate before touching source-of-truth rows. If cache deletion
# fails, fail closed with every checkpoint intact; doing this after the
@ -603,7 +605,7 @@ async def enforce_trailing_duration_retention(
latest_versions = latest.checkpoint.get("channel_versions")
parent_versions = parent.checkpoint.get("channel_versions")
if not isinstance(latest_versions, dict) or latest_versions != parent_versions:
if not channel_versions_match(latest_versions, parent_versions):
return report
metadata = latest.metadata or {}
run_durations = metadata.get("run_durations")
@ -615,9 +617,9 @@ async def enforce_trailing_duration_retention(
duration_only=is_duration_only_checkpoint(latest),
metadata_source=metadata.get("source"),
carries_run_durations=isinstance(run_durations, dict) and bool(run_durations),
versions=frozenset(latest_versions.values()),
versions=dict(latest_versions),
)
parent_node = _Node(ns="", cp_id=parent_id, parent_ns=None, parent_id=None, duration_only=False, versions=latest_node.versions)
parent_node = _Node(ns="", cp_id=parent_id, parent_ns=None, parent_id=None, duration_only=False, versions=dict(parent_versions))
_mark_duration_leaves_without_the_marker({("", latest_id): latest_node, ("", parent_id): parent_node})
if not latest_node.duration_only or await _checkpoint_has_writes(storage, ("", latest_id), thread_id):
return report

View File

@ -227,6 +227,20 @@ def test_goal_leaf_inheriting_stamps_is_not_duration_only():
assert is_duration_only_checkpoint(goal_leaf, parent=parent) is False
def test_swapped_channel_versions_are_not_a_duration_checkpoint():
"""Version values belong to channel names; a permutation is a state change."""
parent = _snapshot("ckpt-parent", [], channel_versions={"messages": 5, "goal": 2})
swapped_leaf = _snapshot(
"ckpt-swapped",
[],
parent_id="ckpt-parent",
metadata=dict(_POSTGRES_DURATION_METADATA),
channel_versions={"messages": 2, "goal": 5},
)
assert is_duration_only_checkpoint(swapped_leaf, parent=parent) is False
def test_postgres_attribution_only_duration_checkpoint_is_recognised():
"""A leaf stamped for message attribution alone has an empty ``run_durations`` map."""
parent = _snapshot("ckpt-real", [], channel_versions={"messages": 5})

View File

@ -298,18 +298,23 @@ async def test_postgres_round_trip_shape_pruned_by_default(saver_env: _SaverEnv)
# aput needs the full configurable (namespace included) for the parent link.
return {"configurable": {"thread_id": thread_id, "checkpoint_ns": "", "checkpoint_id": parent_id}}
# Control: one NEW channel version on an otherwise verbatim clone — a
# client ``update_state`` can produce this, the runtime writer cannot.
control = copy.deepcopy(dict(head.checkpoint))
# Seed two distinct channel counters so the regression is deterministic
# even when the real graph writes only one state channel.
baseline = copy.deepcopy(dict(head.checkpoint))
baseline.update(_new_checkpoint_marker())
baseline["channel_versions"] = {"messages": "v1", "goal": "v2"}
baseline_id = baseline["id"]
baseline_meta = {"source": "loop", "step": 1}
await saver_env.saver.aput(_leaf_config(head_id), baseline, baseline_meta, {})
# Control: exchange the channel versions without changing their value set.
# The channel ownership changed, so this checkpoint must remain resumable.
control = copy.deepcopy(baseline)
control.update(_new_checkpoint_marker())
control["channel_versions"] = dict(control["channel_versions"])
# A version value no real channel would carry; the classifier compares the
# frozenset of versions against the parent's, so any strictly-different
# set is what matters.
control["channel_versions"]["_control"] = "__control_version__"
control["channel_versions"] = {"messages": "v2", "goal": "v1"}
control_id = control["id"]
control_meta = _pg_round_trip_meta(head.metadata)
await saver_env.saver.aput(_leaf_config(head_id), control, control_meta, {})
control_meta = _pg_round_trip_meta(baseline_meta)
await saver_env.saver.aput(_leaf_config(baseline_id), control, control_meta, {})
# The Postgres round-trip shape on top: verbatim clone of the control
# (identical ``channel_versions``), marker popped from the metadata.
@ -374,6 +379,8 @@ async def test_regenerated_old_head_pruned_opt_in(saver_env: _SaverEnv) -> None:
assert fork_tuple is not None
base = await _walk(saver_env, _config_thread(thread_id, fork_head_id), "fork-turn")
assert base is not None
materialized = await graph.aget_state(_config(thread_id))
assert materialized.values["messages"][-1].content.startswith("regenerated turn:")
remaining = await _listed_checkpoint_ids(saver_env, thread_id)
assert old_head_id not in remaining
assert {checkpoint_ids[0], checkpoint_ids[1], fork_head_id}.issubset(remaining)
@ -832,7 +839,7 @@ async def test_admission_retention_bounds_storage_work_independently_of_history(
@pytest.mark.anyio
@pytest.mark.parametrize(
"shape",
["pending-writes", "missing-parent", "no-parent-link", "changed-versions", "swapped-channels", "markerless", "empty-durations", "empty-write-entry"],
["pending-writes", "missing-parent", "no-parent-link", "changed-versions", "swapped-channels", "markerless", "empty-durations", "empty-write-entry", "empty-channel-versions", "markerless-empty-channel-versions"],
)
async def test_admission_retention_checks_only_a_proven_metadata_leaf(
saver_env: _SaverEnv,
@ -851,12 +858,21 @@ async def test_admission_retention_checks_only_a_proven_metadata_leaf(
saver_env.saver.writes[(thread_id, "", duration_id)] = {}
elif shape == "pending-writes":
await saver_env.saver.aput_writes(head.config, [("messages", [HumanMessage(content="pending")])], task_id="pending")
elif shape in {"markerless", "empty-durations"}:
elif shape in {"markerless", "empty-durations", "empty-channel-versions", "markerless-empty-channel-versions"}:
metadata = copy.deepcopy(head.metadata)
metadata.pop("writes", None)
if shape in {"markerless", "empty-durations", "markerless-empty-channel-versions"}:
metadata.pop("writes", None)
if shape == "empty-durations":
metadata["run_durations"] = {}
await saver_env.saver.aput(head.parent_config, head.checkpoint, metadata, {})
if shape in {"empty-channel-versions", "markerless-empty-channel-versions"}:
parent_checkpoint = copy.deepcopy(parent.checkpoint)
parent_checkpoint["channel_versions"] = {}
await saver_env.saver.aput(parent.parent_config, parent_checkpoint, parent.metadata, {})
checkpoint = copy.deepcopy(head.checkpoint)
checkpoint["channel_versions"] = {}
await saver_env.saver.aput(head.parent_config, checkpoint, metadata, {})
else:
await saver_env.saver.aput(head.parent_config, head.checkpoint, metadata, {})
elif shape in {"changed-versions", "swapped-channels"}:
checkpoint = copy.deepcopy(head.checkpoint)
versions = checkpoint["channel_versions"]
@ -880,11 +896,15 @@ async def test_admission_retention_checks_only_a_proven_metadata_leaf(
monkeypatch.setattr(saver_env.saver, "aget_tuple", incomplete_chain)
if shape == "empty-channel-versions" and saver_env.kind == "postgres":
pytest.skip("Postgres removes the writes marker, so this marker-backed case applies to memory and SQLite")
report = await enforce_completed_run_retention(checkpointer=saver_env.saver, thread_id=thread_id)
assert report.deleted_checkpoint_ids == ([duration_id] if shape in {"markerless", "empty-write-entry"} else [])
should_delete = shape in {"markerless", "empty-write-entry", "empty-channel-versions"}
assert report.deleted_checkpoint_ids == ([duration_id] if should_delete else [])
retained = await original_get(_config_thread(thread_id, duration_id))
assert (retained is None) == (shape in {"markerless", "empty-write-entry"})
assert (retained is None) == should_delete
@pytest.mark.anyio