From 340822f2c153a1db0183d6fb00d0203ae8847f0c Mon Sep 17 00:00:00 2001 From: NanPan <111261006+poijygfdyy@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:57:37 +0800 Subject: [PATCH] fix(checkpoints): preserve channel identity in version comparisons (#8330) * fix(checkpoints): compare channel versions by identity * fix(checkpoints): preserve channel identity in retention * test(checkpoints): cover swapped channel versions in lineage * test(checkpoints): retain swapped channel-version state * test(checkpoints): make swapped channel regression deterministic * style(checkpoints): satisfy Ruff formatting * fix(checkpoints): preserve version maps in retention paths * test(checkpoints): keep surviving blobs after sibling pruning * fix(checkpoints): prune empty-map duration leaves safely --- backend/app/gateway/checkpoint_lineage.py | 17 +++++- backend/app/gateway/checkpoint_retention.py | 16 +++--- backend/tests/test_checkpoint_lineage.py | 14 +++++ .../test_checkpoint_retention_service.py | 52 +++++++++++++------ 4 files changed, 75 insertions(+), 24 deletions(-) diff --git a/backend/app/gateway/checkpoint_lineage.py b/backend/app/gateway/checkpoint_lineage.py index 9e5140b4ed..360eeaba74 100644 --- a/backend/app/gateway/checkpoint_lineage.py +++ b/backend/app/gateway/checkpoint_lineage.py @@ -76,6 +76,21 @@ def _has_postgres_stamps(metadata: dict[str, Any]) -> bool: return "run_durations" in metadata or "run_message_ids" in metadata +def channel_versions_match( + versions: dict[str, Any] | None, + parent_versions: dict[str, Any] | None, +) -> bool: + """Whether two checkpoint version maps match channel-for-channel. + + LangGraph versions are scoped to their channels, not globally unique. + Comparing only the values can mistake a state-changing checkpoint for a + metadata-only copy when two channels exchange their versions. + """ + + # Empty maps are equal too; markerless inference applies its own non-empty guard. + return isinstance(versions, dict) and isinstance(parent_versions, dict) and versions == parent_versions + + def is_duration_only_checkpoint(checkpoint_tuple: Any, *, parent: Any | None = None, stamp_fallback: bool = True, versions: dict[str, Any] | None = None, parent_versions: dict[str, Any] | None = None) -> bool: """Return whether the tuple is a metadata-only run-duration checkpoint. @@ -268,7 +283,7 @@ def _copies_parent_verbatim(checkpoint_tuple: Any, parent: Any, *, versions: dic parent_versions = parent_versions if isinstance(parent_versions, dict) else _snapshot_versions(parent) if not isinstance(versions, dict) or not isinstance(parent_versions, dict) or not versions: return False - return frozenset(versions.values()) == frozenset(parent_versions.values()) + return channel_versions_match(versions, parent_versions) def history_parent_index(checkpoints: Sequence[Any]) -> dict[tuple[str, str], Any]: diff --git a/backend/app/gateway/checkpoint_retention.py b/backend/app/gateway/checkpoint_retention.py index 5ad6440afd..2b1723af0a 100644 --- a/backend/app/gateway/checkpoint_retention.py +++ b/backend/app/gateway/checkpoint_retention.py @@ -53,6 +53,7 @@ from langgraph.checkpoint.memory import InMemorySaver from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver from app.gateway.checkpoint_lineage import ( + channel_versions_match, checkpoint_configurable, is_duration_only_checkpoint, ) @@ -105,7 +106,7 @@ class _Node: duration_only: bool metadata_source: str | None = None carries_run_durations: bool = False - versions: frozenset = frozenset() + versions: dict[str, Any] = field(default_factory=dict) def _thread_config(thread_id: str) -> dict[str, Any]: @@ -156,7 +157,8 @@ def _mark_duration_leaves_without_the_marker(nodes: dict[tuple[str, str], _Node] parent = nodes.get((node.parent_ns, node.parent_id)) if parent is None or node.metadata_source != "update" or not node.carries_run_durations: continue - if node.versions and node.versions == parent.versions: + # Without the authoritative writes marker, require a non-empty map as evidence. + if node.versions and parent.versions and channel_versions_match(node.versions, parent.versions): node.duration_only = True @@ -452,7 +454,7 @@ async def enforce_thread_retention( duration_only=is_duration_only_checkpoint(tuple_, stamp_fallback=False), metadata_source=metadata.get("source"), carries_run_durations=isinstance(run_durations, dict) and bool(run_durations), - versions=frozenset(channel_versions.values()) if isinstance(channel_versions, dict) else frozenset(), + versions=dict(channel_versions) if isinstance(channel_versions, dict) else {}, ) if not nodes: # Nothing to classify: an empty (or unknown) thread is a no-op, but @@ -538,7 +540,7 @@ async def enforce_thread_retention( survivor_versions: set[Any] = set() for key, node in nodes.items(): if key not in deleted_keys: - survivor_versions.update(node.versions) + survivor_versions.update(node.versions.values()) # Invalidate before touching source-of-truth rows. If cache deletion # fails, fail closed with every checkpoint intact; doing this after the @@ -603,7 +605,7 @@ async def enforce_trailing_duration_retention( latest_versions = latest.checkpoint.get("channel_versions") parent_versions = parent.checkpoint.get("channel_versions") - if not isinstance(latest_versions, dict) or latest_versions != parent_versions: + if not channel_versions_match(latest_versions, parent_versions): return report metadata = latest.metadata or {} run_durations = metadata.get("run_durations") @@ -615,9 +617,9 @@ async def enforce_trailing_duration_retention( duration_only=is_duration_only_checkpoint(latest), metadata_source=metadata.get("source"), carries_run_durations=isinstance(run_durations, dict) and bool(run_durations), - versions=frozenset(latest_versions.values()), + versions=dict(latest_versions), ) - parent_node = _Node(ns="", cp_id=parent_id, parent_ns=None, parent_id=None, duration_only=False, versions=latest_node.versions) + parent_node = _Node(ns="", cp_id=parent_id, parent_ns=None, parent_id=None, duration_only=False, versions=dict(parent_versions)) _mark_duration_leaves_without_the_marker({("", latest_id): latest_node, ("", parent_id): parent_node}) if not latest_node.duration_only or await _checkpoint_has_writes(storage, ("", latest_id), thread_id): return report diff --git a/backend/tests/test_checkpoint_lineage.py b/backend/tests/test_checkpoint_lineage.py index c35938e5d1..1a4d7a4d83 100644 --- a/backend/tests/test_checkpoint_lineage.py +++ b/backend/tests/test_checkpoint_lineage.py @@ -227,6 +227,20 @@ def test_goal_leaf_inheriting_stamps_is_not_duration_only(): assert is_duration_only_checkpoint(goal_leaf, parent=parent) is False +def test_swapped_channel_versions_are_not_a_duration_checkpoint(): + """Version values belong to channel names; a permutation is a state change.""" + parent = _snapshot("ckpt-parent", [], channel_versions={"messages": 5, "goal": 2}) + swapped_leaf = _snapshot( + "ckpt-swapped", + [], + parent_id="ckpt-parent", + metadata=dict(_POSTGRES_DURATION_METADATA), + channel_versions={"messages": 2, "goal": 5}, + ) + + assert is_duration_only_checkpoint(swapped_leaf, parent=parent) is False + + def test_postgres_attribution_only_duration_checkpoint_is_recognised(): """A leaf stamped for message attribution alone has an empty ``run_durations`` map.""" parent = _snapshot("ckpt-real", [], channel_versions={"messages": 5}) diff --git a/backend/tests/test_checkpoint_retention_service.py b/backend/tests/test_checkpoint_retention_service.py index dbaada4ab2..34a30bca3f 100644 --- a/backend/tests/test_checkpoint_retention_service.py +++ b/backend/tests/test_checkpoint_retention_service.py @@ -298,18 +298,23 @@ async def test_postgres_round_trip_shape_pruned_by_default(saver_env: _SaverEnv) # aput needs the full configurable (namespace included) for the parent link. return {"configurable": {"thread_id": thread_id, "checkpoint_ns": "", "checkpoint_id": parent_id}} - # Control: one NEW channel version on an otherwise verbatim clone — a - # client ``update_state`` can produce this, the runtime writer cannot. - control = copy.deepcopy(dict(head.checkpoint)) + # Seed two distinct channel counters so the regression is deterministic + # even when the real graph writes only one state channel. + baseline = copy.deepcopy(dict(head.checkpoint)) + baseline.update(_new_checkpoint_marker()) + baseline["channel_versions"] = {"messages": "v1", "goal": "v2"} + baseline_id = baseline["id"] + baseline_meta = {"source": "loop", "step": 1} + await saver_env.saver.aput(_leaf_config(head_id), baseline, baseline_meta, {}) + + # Control: exchange the channel versions without changing their value set. + # The channel ownership changed, so this checkpoint must remain resumable. + control = copy.deepcopy(baseline) control.update(_new_checkpoint_marker()) - control["channel_versions"] = dict(control["channel_versions"]) - # A version value no real channel would carry; the classifier compares the - # frozenset of versions against the parent's, so any strictly-different - # set is what matters. - control["channel_versions"]["_control"] = "__control_version__" + control["channel_versions"] = {"messages": "v2", "goal": "v1"} control_id = control["id"] - control_meta = _pg_round_trip_meta(head.metadata) - await saver_env.saver.aput(_leaf_config(head_id), control, control_meta, {}) + control_meta = _pg_round_trip_meta(baseline_meta) + await saver_env.saver.aput(_leaf_config(baseline_id), control, control_meta, {}) # The Postgres round-trip shape on top: verbatim clone of the control # (identical ``channel_versions``), marker popped from the metadata. @@ -374,6 +379,8 @@ async def test_regenerated_old_head_pruned_opt_in(saver_env: _SaverEnv) -> None: assert fork_tuple is not None base = await _walk(saver_env, _config_thread(thread_id, fork_head_id), "fork-turn") assert base is not None + materialized = await graph.aget_state(_config(thread_id)) + assert materialized.values["messages"][-1].content.startswith("regenerated turn:") remaining = await _listed_checkpoint_ids(saver_env, thread_id) assert old_head_id not in remaining assert {checkpoint_ids[0], checkpoint_ids[1], fork_head_id}.issubset(remaining) @@ -832,7 +839,7 @@ async def test_admission_retention_bounds_storage_work_independently_of_history( @pytest.mark.anyio @pytest.mark.parametrize( "shape", - ["pending-writes", "missing-parent", "no-parent-link", "changed-versions", "swapped-channels", "markerless", "empty-durations", "empty-write-entry"], + ["pending-writes", "missing-parent", "no-parent-link", "changed-versions", "swapped-channels", "markerless", "empty-durations", "empty-write-entry", "empty-channel-versions", "markerless-empty-channel-versions"], ) async def test_admission_retention_checks_only_a_proven_metadata_leaf( saver_env: _SaverEnv, @@ -851,12 +858,21 @@ async def test_admission_retention_checks_only_a_proven_metadata_leaf( saver_env.saver.writes[(thread_id, "", duration_id)] = {} elif shape == "pending-writes": await saver_env.saver.aput_writes(head.config, [("messages", [HumanMessage(content="pending")])], task_id="pending") - elif shape in {"markerless", "empty-durations"}: + elif shape in {"markerless", "empty-durations", "empty-channel-versions", "markerless-empty-channel-versions"}: metadata = copy.deepcopy(head.metadata) - metadata.pop("writes", None) + if shape in {"markerless", "empty-durations", "markerless-empty-channel-versions"}: + metadata.pop("writes", None) if shape == "empty-durations": metadata["run_durations"] = {} - await saver_env.saver.aput(head.parent_config, head.checkpoint, metadata, {}) + if shape in {"empty-channel-versions", "markerless-empty-channel-versions"}: + parent_checkpoint = copy.deepcopy(parent.checkpoint) + parent_checkpoint["channel_versions"] = {} + await saver_env.saver.aput(parent.parent_config, parent_checkpoint, parent.metadata, {}) + checkpoint = copy.deepcopy(head.checkpoint) + checkpoint["channel_versions"] = {} + await saver_env.saver.aput(head.parent_config, checkpoint, metadata, {}) + else: + await saver_env.saver.aput(head.parent_config, head.checkpoint, metadata, {}) elif shape in {"changed-versions", "swapped-channels"}: checkpoint = copy.deepcopy(head.checkpoint) versions = checkpoint["channel_versions"] @@ -880,11 +896,15 @@ async def test_admission_retention_checks_only_a_proven_metadata_leaf( monkeypatch.setattr(saver_env.saver, "aget_tuple", incomplete_chain) + if shape == "empty-channel-versions" and saver_env.kind == "postgres": + pytest.skip("Postgres removes the writes marker, so this marker-backed case applies to memory and SQLite") + report = await enforce_completed_run_retention(checkpointer=saver_env.saver, thread_id=thread_id) - assert report.deleted_checkpoint_ids == ([duration_id] if shape in {"markerless", "empty-write-entry"} else []) + should_delete = shape in {"markerless", "empty-write-entry", "empty-channel-versions"} + assert report.deleted_checkpoint_ids == ([duration_id] if should_delete else []) retained = await original_get(_config_thread(thread_id, duration_id)) - assert (retained is None) == (shape in {"markerless", "empty-write-entry"}) + assert (retained is None) == should_delete @pytest.mark.anyio