mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-08-08 13:58:38 +00:00
c8cf1bf2fb
* feat(checkpoint-cache): delta-mode checkpoint history cache with recursive compose
Read-only, invalidation-free cache for LangGraph delta-channel history
({writes, seed}) at the get_delta_channel_history choke point:
- database.checkpoint_cache config (memory|redis; max_entries 0=disabled;
redis bounded by TTL, Gateway/async only)
- memory LRU backend (copy-on-read, zero-serde hit path) and redis backend
(lazy import, degrades to all-miss on outage)
- CachedHistorySaver: recursive composition from the nearest warm ancestor
(depth budget 8), caching each level; depth-0 cold chains delegate one
inner fast-path walk. Entries keyed by immutable
(db, thread, ns, checkpoint_id, channel) — no invalidation, coherent
across workers
- provider wiring: wraps in delta mode only (async + sync), full mode
untouched; sync path is memory-only
- bench opt-in: DEERFLOW_CHECKPOINT_BENCH_HISTORY_CACHE=1
sqlite bench (500 updates, payload 2KB): write phase 2.28x at f=250,
1.32x at f=10; one delegated walk per thread cold start.
* chore(config): bump config_version to 32 for database.checkpoint_cache
The checkpoint history cache feature added the database.checkpoint_cache
section to config.example.yaml; bump the schema version so existing
deployments get the outdated-config warning and can run make config-upgrade.
* chore(helm): bump config_version to 32 in chart values and README
* fix(checkpoint-cache): purge thread history entries on delete paths
Addresses review on #4638: delete_thread/prune removed source-of-truth
checkpoints but left the thread's materialized history payloads in the
cache (memory: until LRU eviction; redis: until TTL, default 1 day) — a
data-lifecycle gap for tenant offboarding / GDPR-style erasure.
- Cache contract gains thread-scoped adelete_thread/delete_thread
(lifecycle purge, not invalidation; entries remain immutable)
- Memory backend: stem scan over the LRU map; redis: SCAN MATCH + UNLINK,
outage degrades to TTL-bounded retention without raising
- CachedHistorySaver purges on delete_thread/adelete_thread and
prune/aprune (prune rewrites chains, so pre-prune histories must go);
delete_for_runs stays delegation-only (run->thread mapping unavailable,
no in-tree callers), documented in code
- ttl_seconds description documents the residual-retention window
- Tests: thread-scoped purge on both backends, saver-level delete/prune
purge, prefix-safety (t1 vs t10), redis outage degradation, and the
pinned no-purge behavior of delete_for_runs
* fix(checkpoint-cache): stable db identity, prefix-aware sync singleton, explicit zero TTL
Addresses Copilot review on #4638:
- checkpoint_cache_db_hash now hashes the credential-free postgres
identity (host:port/database + schema): credential rotation no longer
changes the cache namespace (cold cache + orphaned keys until TTL).
Unparseable URLs fall back to the raw string.
- The sync-path memory cache singleton is also keyed by its key_prefix:
a namespace change (db identity change or operator override) recreates
the cache instead of leaving stale-prefix entries unreachable and
unpurgeable.
- ttl_seconds=0 is now an explicit, documented opt-out of redis expiry
(SET without EX; redis maxmemory policy only) instead of a silent
'ttl_seconds or None' coercion.
Tests: credential-rotation hash stability, unparseable-URL fallback,
prefix-change singleton recreation, same-prefix singleton reuse, and
zero-TTL wire behavior (ex=None).
---------
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
131 lines
4.7 KiB
Python
131 lines
4.7 KiB
Python
"""Memory LRU backend for the checkpoint history cache."""
|
|
|
|
import pytest
|
|
|
|
from deerflow.runtime.checkpoint_cache.base import (
|
|
CACHE_FORMAT_VERSION,
|
|
CheckpointCacheStats,
|
|
make_history_key,
|
|
thread_key_stem,
|
|
)
|
|
from deerflow.runtime.checkpoint_cache.memory import MemoryCheckpointHistoryCache
|
|
|
|
|
|
def _entry(tag: str) -> dict:
|
|
return {"writes": [("task-1", "messages", tag)], "seed": f"seed-{tag}"}
|
|
|
|
|
|
def test_make_history_key_is_stable_and_scoped():
|
|
k1 = make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "messages")
|
|
k2 = make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "messages")
|
|
assert k1 == k2
|
|
assert k1.startswith("ckpt-hist:v1:db0:t1:")
|
|
# ns / checkpoint / channel each change the key
|
|
assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "sub", "c1", "messages")
|
|
assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "", "c2", "messages")
|
|
assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "todos")
|
|
assert k1 != make_history_key("ckpt-hist:v1:db9", "t1", "", "c1", "messages")
|
|
assert CACHE_FORMAT_VERSION == 1
|
|
|
|
|
|
def test_get_many_miss_then_hit():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
assert cache.get_many(["a"]) == {}
|
|
assert cache.stats().misses == 1
|
|
cache.set_many({"a": _entry("x")})
|
|
hit = cache.get_many(["a"])
|
|
assert hit["a"]["writes"] == [("task-1", "messages", "x")]
|
|
assert hit["a"]["seed"] == "seed-x"
|
|
assert cache.stats().hits == 1
|
|
|
|
|
|
def test_entry_without_seed_roundtrips_without_seed_key():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
cache.set_many({"a": {"writes": []}})
|
|
hit = cache.get_many(["a"])
|
|
assert hit["a"] == {"writes": []}
|
|
assert "seed" not in hit["a"]
|
|
|
|
|
|
def test_copy_on_read_returns_fresh_writes_list():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
cache.set_many({"a": _entry("x")})
|
|
first = cache.get_many(["a"])["a"]
|
|
first["writes"].append(("task-2", "messages", "MUTATION"))
|
|
second = cache.get_many(["a"])["a"]
|
|
assert second["writes"] == [("task-1", "messages", "x")]
|
|
|
|
|
|
def test_caller_mutation_after_set_does_not_leak():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
entry = _entry("x")
|
|
cache.set_many({"a": entry})
|
|
entry["writes"].append(("task-2", "messages", "MUTATION"))
|
|
assert cache.get_many(["a"])["a"]["writes"] == [("task-1", "messages", "x")]
|
|
|
|
|
|
def test_lru_evicts_oldest_and_counts():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=2)
|
|
cache.set_many({"a": _entry("a"), "b": _entry("b")})
|
|
cache.get_many(["a"]) # refresh a
|
|
cache.set_many({"c": _entry("c")}) # evicts b
|
|
assert cache.get_many(["b"]) == {}
|
|
assert cache.get_many(["a"]) != {}
|
|
assert cache.stats().evictions == 1
|
|
assert cache.stats().entries == 2
|
|
|
|
|
|
def test_zero_max_entries_disables():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=0)
|
|
assert cache.enabled is False
|
|
cache.set_many({"a": _entry("x")})
|
|
assert cache.get_many(["a"]) == {}
|
|
assert cache.stats().entries == 0
|
|
|
|
|
|
def test_delete_thread_purges_only_that_thread():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=16)
|
|
prefix = "ckpt-hist:v1:db0"
|
|
t1_keys = [make_history_key(prefix, "t1", "", f"c{i}", "messages") for i in range(3)]
|
|
t2_key = make_history_key(prefix, "t2", "", "c0", "messages")
|
|
# A thread_id that is a prefix of another must not over-match: the stem
|
|
# ends with ':' so "t1" never matches "t10"'s keys.
|
|
t10_key = make_history_key(prefix, "t10", "", "c0", "messages")
|
|
cache.set_many({k: _entry(k) for k in [*t1_keys, t2_key, t10_key]})
|
|
|
|
cache.delete_thread(prefix, "t1")
|
|
|
|
assert cache.stats().entries == 2
|
|
assert all(cache.get_many([k]) == {} for k in t1_keys)
|
|
assert cache.get_many([t2_key]) != {}
|
|
assert cache.get_many([t10_key]) != {}
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_adelete_thread_matches_sync():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
prefix = "ckpt-hist:v1:db0"
|
|
key = make_history_key(prefix, "t1", "", "c0", "messages")
|
|
await cache.aset_many({key: _entry("x")})
|
|
await cache.adelete_thread(prefix, "t1")
|
|
assert cache.get_many([key]) == {}
|
|
|
|
|
|
def test_thread_key_stem_matches_make_history_key_layout():
|
|
key = make_history_key("p", "t1", "ns", "c1", "messages")
|
|
assert key.startswith(thread_key_stem("p", "t1"))
|
|
assert not key.startswith(thread_key_stem("p", "t"))
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_async_protocol_matches_sync():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
await cache.aset_many({"a": _entry("x")})
|
|
hit = await cache.aget_many(["a"])
|
|
assert hit["a"]["seed"] == "seed-x"
|
|
stats = cache.stats()
|
|
assert isinstance(stats, CheckpointCacheStats)
|
|
assert stats.as_dict()["hits"] == 1
|
|
await cache.aclose()
|
|
assert cache.get_many(["a"]) == {}
|