"""Behavioral suite for checkpoint history cache config, provider lifecycle, payload isolation, memory LRU, and Redis backend.""" import asyncio import threading from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from queue import Queue from typing import Annotated, Any, TypedDict from unittest.mock import AsyncMock, Mock import pytest from langgraph.channels import DeltaChannel from langgraph.checkpoint.memory import InMemorySaver from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer from langgraph.graph import StateGraph from deerflow.config.app_config import AppConfig, set_app_config from deerflow.config.database_config import CheckpointCacheConfig, DatabaseConfig from deerflow.runtime.checkpoint_cache.base import ( CACHE_FORMAT_VERSION, CheckpointCacheStats, make_history_key, thread_key_stem, ) from deerflow.runtime.checkpoint_cache.memory import MemoryCheckpointHistoryCache from deerflow.runtime.checkpoint_cache.provider import ( checkpoint_cache_db_hash, checkpoint_cache_key_prefix, make_checkpoint_cache, ) from deerflow.runtime.checkpoint_cache.redis import RedisCheckpointHistoryCache from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode from deerflow.runtime.checkpoint_state import CheckpointStateAccessor from deerflow.runtime.checkpointer.async_provider import make_checkpointer from deerflow.runtime.checkpointer.cached_saver import CachedHistorySaver from deerflow.runtime.checkpointer.provider import checkpointer_context, reset_checkpointer # --- test_checkpoint_cache_config --- def test_checkpoint_cache_defaults(): cfg = DatabaseConfig() assert cfg.checkpoint_cache.type == "memory" assert cfg.checkpoint_cache.max_entries == 128 assert cfg.checkpoint_cache.redis_url is None assert cfg.checkpoint_cache.ttl_seconds == 86400 assert cfg.checkpoint_cache.key_prefix == "" def test_checkpoint_cache_from_dict_redis(): cfg = DatabaseConfig.model_validate( { "backend": "postgres", "postgres_url": "postgresql://u:p@h/db", "checkpoint_cache": { "type": "redis", "max_entries": 256, "redis_url": "redis://localhost:6379/3", "ttl_seconds": 3600, "key_prefix": "prod:", }, } ) assert cfg.checkpoint_cache.type == "redis" assert cfg.checkpoint_cache.max_entries == 256 assert cfg.checkpoint_cache.redis_url == "redis://localhost:6379/3" assert cfg.checkpoint_cache.ttl_seconds == 3600 assert cfg.checkpoint_cache.key_prefix == "prod:" def test_checkpoint_cache_zero_max_entries_means_disabled(): cfg = CheckpointCacheConfig(max_entries=0) assert cfg.max_entries == 0 def test_checkpoint_cache_rejects_negative_max_entries(): import pydantic import pytest with pytest.raises(pydantic.ValidationError): CheckpointCacheConfig(max_entries=-1) # --- test_checkpoint_cache_memory --- def _memory_entry(tag: str) -> dict: return {"writes": [("task-1", "messages", tag)], "seed": f"seed-{tag}"} def test_make_history_key_is_stable_and_scoped(): k1 = make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "messages") k2 = make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "messages") assert k1 == k2 assert k1.startswith("ckpt-hist:v1:db0:t1:") # ns / checkpoint / channel each change the key assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "sub", "c1", "messages") assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "", "c2", "messages") assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "todos") assert k1 != make_history_key("ckpt-hist:v1:db9", "t1", "", "c1", "messages") assert CACHE_FORMAT_VERSION == 1 def test_get_many_miss_then_hit(): cache = MemoryCheckpointHistoryCache(max_entries=4) assert cache.get_many(["a"]) == {} assert cache.stats().misses == 1 cache.set_many({"a": _memory_entry("x")}) hit = cache.get_many(["a"]) assert hit["a"]["writes"] == [("task-1", "messages", "x")] assert hit["a"]["seed"] == "seed-x" assert cache.stats().hits == 1 def test_entry_without_seed_roundtrips_without_seed_key(): cache = MemoryCheckpointHistoryCache(max_entries=4) cache.set_many({"a": {"writes": []}}) hit = cache.get_many(["a"]) assert hit["a"] == {"writes": []} assert "seed" not in hit["a"] def test_copy_on_read_returns_fresh_writes_list(): cache = MemoryCheckpointHistoryCache(max_entries=4) cache.set_many({"a": _memory_entry("x")}) first = cache.get_many(["a"])["a"] first["writes"].append(("task-2", "messages", "MUTATION")) second = cache.get_many(["a"])["a"] assert second["writes"] == [("task-1", "messages", "x")] def test_caller_mutation_after_set_does_not_leak(): cache = MemoryCheckpointHistoryCache(max_entries=4) entry = _memory_entry("x") cache.set_many({"a": entry}) entry["writes"].append(("task-2", "messages", "MUTATION")) assert cache.get_many(["a"])["a"]["writes"] == [("task-1", "messages", "x")] def test_lru_evicts_oldest_and_counts(): cache = MemoryCheckpointHistoryCache(max_entries=2) cache.set_many({"a": _memory_entry("a"), "b": _memory_entry("b")}) cache.get_many(["a"]) # refresh a cache.set_many({"c": _memory_entry("c")}) # evicts b assert cache.get_many(["b"]) == {} assert cache.get_many(["a"]) != {} assert cache.stats().evictions == 1 assert cache.stats().entries == 2 def test_zero_max_entries_disables(): cache = MemoryCheckpointHistoryCache(max_entries=0) assert cache.enabled is False cache.set_many({"a": _memory_entry("x")}) assert cache.get_many(["a"]) == {} assert cache.stats().entries == 0 def test_delete_thread_purges_only_that_thread(): cache = MemoryCheckpointHistoryCache(max_entries=16) prefix = "ckpt-hist:v1:db0" t1_keys = [make_history_key(prefix, "t1", "", f"c{i}", "messages") for i in range(3)] t2_key = make_history_key(prefix, "t2", "", "c0", "messages") # A thread_id that is a prefix of another must not over-match: the stem # ends with ':' so "t1" never matches "t10"'s keys. t10_key = make_history_key(prefix, "t10", "", "c0", "messages") cache.set_many({k: _memory_entry(k) for k in [*t1_keys, t2_key, t10_key]}) cache.delete_thread(prefix, "t1") assert cache.stats().entries == 2 assert all(cache.get_many([k]) == {} for k in t1_keys) assert cache.get_many([t2_key]) != {} assert cache.get_many([t10_key]) != {} @pytest.mark.anyio async def test_adelete_thread_matches_sync(): cache = MemoryCheckpointHistoryCache(max_entries=4) prefix = "ckpt-hist:v1:db0" key = make_history_key(prefix, "t1", "", "c0", "messages") await cache.aset_many({key: _memory_entry("x")}) await cache.adelete_thread(prefix, "t1") assert cache.get_many([key]) == {} def test_thread_key_stem_matches_make_history_key_layout(): key = make_history_key("p", "t1", "ns", "c1", "messages") assert key.startswith(thread_key_stem("p", "t1")) assert not key.startswith(thread_key_stem("p", "t")) @pytest.mark.anyio async def test_async_protocol_matches_sync(): cache = MemoryCheckpointHistoryCache(max_entries=4) await cache.aset_many({"a": _memory_entry("x")}) hit = await cache.aget_many(["a"]) assert hit["a"]["seed"] == "seed-x" stats = cache.stats() assert isinstance(stats, CheckpointCacheStats) assert stats.as_dict()["hits"] == 1 await cache.aclose() assert cache.get_many(["a"]) == {} # --- test_checkpoint_cache_memory_concurrency --- @pytest.mark.parametrize("mutation", ["evict", "purge", "close"]) def test_lookup_survives_concurrent_removal(mutation): cache = MemoryCheckpointHistoryCache(max_entries=1) key = make_history_key("p", "t1", "", "c1", "messages") entry = {"writes": [("task", "messages", "original")], "seed": "seed"} cache.set_many({key: entry}) looked_up = threading.Event() release_reader = threading.Event() writer_started = threading.Event() writer_finished = threading.Event() coordination = Queue() removal_reported = threading.Event() def report_removal(): if not removal_reported.is_set(): removal_reported.set() coordination.put("removed") class PausedLookup(OrderedDict): def get(self, lookup_key, default=None): value = super().get(lookup_key, default) if lookup_key == key: looked_up.set() assert release_reader.wait(5), "reader was not released" return value def __delitem__(self, deleted_key): super().__delitem__(deleted_key) if deleted_key == key: report_removal() def popitem(self, last=True): removed_key, value = super().popitem(last=last) if removed_key == key: report_removal() return removed_key, value def clear(self): contained_target = key in self super().clear() if contained_target: report_removal() class ObservedLock: """Report a contender before it blocks behind the paused reader.""" def __init__(self): self._lock = threading.Lock() def __enter__(self): if self._lock.locked(): coordination.put("writer_waiting") self._lock.acquire() return self def __exit__(self, exc_type, exc_value, traceback): self._lock.release() # Pause at the actual lookup -> LRU promotion window, not before get_many. cache._data = PausedLookup(cache._data) cache._lock = ObservedLock() def remove(): writer_started.set() try: if mutation == "evict": cache.set_many({"other": {"writes": []}}) elif mutation == "purge": cache.delete_thread("p", "t1") else: asyncio.run(cache.aclose()) finally: writer_finished.set() with ThreadPoolExecutor(max_workers=2) as pool: reader = pool.submit(cache.get_many, [key]) try: assert looked_up.wait(5) writer = pool.submit(remove) assert writer_started.wait(5) # A serialized writer reports that it is waiting on the reader's # lock. On unfixed main it instead removes the key and reports that, # deterministically reproducing the get -> move_to_end race. assert coordination.get(timeout=5) == "writer_waiting" finally: release_reader.set() assert reader.result(timeout=5) == {key: entry} writer.result(timeout=5) assert cache.get_many([key]) == {} assert cache.stats().entries == int(mutation == "evict") def test_parallel_read_write_keeps_capacity_and_accounting(): cache = MemoryCheckpointHistoryCache(max_entries=4) start = threading.Barrier(8) def exercise(worker): start.wait(timeout=5) for n in range(200): key = f"{worker}:{n}" cache.set_many({key: {"writes": [("task", "messages", key)]}}) result = cache.get_many([key]) if key in result: assert result[key]["writes"] == [("task", "messages", key)] with ThreadPoolExecutor(max_workers=8) as pool: list(pool.map(exercise, range(8))) stats = cache.stats() assert stats.entries == 4 assert stats.evictions == 8 * 200 - 4 assert stats.hits + stats.misses == 8 * 200 # --- test_checkpoint_cache_payload_isolation --- def _fold(state, writes): result = list(state) for batch in writes: result.extend(batch) return result def _stack(frequency, enabled): class State(TypedDict): items: Annotated[list[dict], DeltaChannel(_fold, snapshot_frequency=frequency)] inner = InMemorySaver() cache = MemoryCheckpointHistoryCache(max_entries=32 if enabled else 0) saver = CachedHistorySaver(inner, cache, key_prefix="payload-isolation") builder = StateGraph(State) for step in range(4): builder.add_node(str(step), lambda state: {"items": [{"nested": {"value": "saved"}}]}) if step: builder.add_edge(str(step - 1), str(step)) builder.set_entry_point("0") builder.set_finish_point("3") graph = builder.compile(checkpointer=saver) return graph, CheckpointStateAccessor.bind(graph, saver, mode="delta"), cache def _mutate(snapshot): original = deepcopy(snapshot.values) snapshot.values["items"][0]["nested"]["value"] = "tampered" snapshot.values["items"].append({"nested": {"value": "injected"}}) return original @pytest.mark.parametrize("frequency", [2, 100], ids=["snapshot-seed", "pending-writes"]) @pytest.mark.parametrize("enabled", [True, False], ids=["cached", "uncached-control"]) def test_sync_graph_snapshots_are_isolated(frequency, enabled): graph, accessor, cache = _stack(frequency, enabled) config = {"configurable": {"thread_id": "sync-payload-isolation"}} graph.invoke({"items": [{"nested": {"value": "input"}}]}, config) history = accessor.history(config) checked = 0 for checkpoint in history: snapshot = accessor.get(checkpoint.config) if not snapshot.values.get("items"): continue original = _mutate(snapshot) assert accessor.get(checkpoint.config).values == original checked += 1 assert checked >= 4 assert cache.stats().hits > 0 if enabled else cache.stats().entries == 0 @pytest.mark.asyncio @pytest.mark.parametrize("frequency", [2, 100], ids=["snapshot-seed", "pending-writes"]) @pytest.mark.parametrize("enabled", [True, False], ids=["cached", "uncached-control"]) async def test_async_graph_snapshots_are_isolated(frequency, enabled): graph, accessor, cache = _stack(frequency, enabled) config = {"configurable": {"thread_id": "async-payload-isolation"}} await graph.ainvoke({"items": [{"nested": {"value": "input"}}]}, config) history = await accessor.ahistory(config) checked = 0 for checkpoint in history: snapshot = await accessor.aget(checkpoint.config) if not snapshot.values.get("items"): continue original = _mutate(snapshot) assert (await accessor.aget(checkpoint.config)).values == original checked += 1 assert checked >= 4 assert cache.stats().hits > 0 if enabled else cache.stats().entries == 0 @pytest.mark.parametrize("direction", ["ingest", "read"]) def test_cache_copies_nested_write_payloads_and_snapshot_seeds(direction): from langchain_core.messages import HumanMessage from langgraph.checkpoint.serde.types import _DeltaSnapshot cache = MemoryCheckpointHistoryCache() seed = _DeltaSnapshot([HumanMessage(content="seed", id="seed", additional_kwargs={"nested": ["original"]})]) entry = {"seed": seed, "writes": [("task", "messages", [HumanMessage(content="write", id="write", additional_kwargs={"nested": ["original"]})])]} original = deepcopy(entry) cache.set_many({"key": entry}) exposed = entry if direction == "ingest" else cache.get_many(["key"])["key"] exposed["seed"].value[0].additional_kwargs["nested"].append("corrupted") exposed["writes"][0][2][0].content = "corrupted" assert cache.get_many(["key"])["key"] == original def test_payload_copying_runs_outside_registry_lock(): cache = MemoryCheckpointHistoryCache() copies = [] class Probe: def __deepcopy__(self, memo): assert cache._lock.acquire(blocking=False), "copy callback ran under the cache lock" cache._lock.release() copies.append(True) return Probe() cache.set_many({"key": {"writes": [("task", "channel", Probe())]}}) cache.get_many(["key"]) assert len(copies) == 2 # --- test_checkpoint_cache_provider --- def _provider_app_config(mode: str, cache: dict | None = None) -> AppConfig: database: dict = {"backend": "memory", "checkpoint_channel_mode": mode} if cache is not None: database["checkpoint_cache"] = cache return AppConfig.model_validate( { "sandbox": {"use": "deerflow.sandbox.local.provider:LocalSandboxProvider"}, "database": database, } ) @pytest.mark.anyio async def test_delta_mode_wraps_with_cached_saver(): set_app_config(_provider_app_config("delta")) freeze_checkpoint_channel_mode("delta") async with make_checkpointer() as saver: assert isinstance(saver, CachedHistorySaver) assert saver.stats()["entries"] == 0 @pytest.mark.anyio async def test_full_mode_yields_raw_saver(): set_app_config(_provider_app_config("full")) freeze_checkpoint_channel_mode("full") async with make_checkpointer() as saver: assert not isinstance(saver, CachedHistorySaver) @pytest.mark.anyio async def test_zero_max_entries_disables_but_still_wraps(): set_app_config(_provider_app_config("delta", {"max_entries": 0})) freeze_checkpoint_channel_mode("delta") async with make_checkpointer() as saver: assert isinstance(saver, CachedHistorySaver) # Disabled cache -> every history call is a full walk on the inner saver. assert saver._cache.enabled is False def test_sync_delta_mode_wraps_memory(): set_app_config(_provider_app_config("delta")) freeze_checkpoint_channel_mode("delta") reset_checkpointer() with checkpointer_context() as saver: assert isinstance(saver, CachedHistorySaver) def test_sync_redis_cache_type_is_config_error(): set_app_config(_provider_app_config("delta", {"type": "redis"})) freeze_checkpoint_channel_mode("delta") reset_checkpointer() with pytest.raises(ValueError, match="redis"): with checkpointer_context(): pass def test_sync_full_mode_unwrapped(): set_app_config(_provider_app_config("full")) freeze_checkpoint_channel_mode("full") reset_checkpointer() with checkpointer_context() as saver: assert not isinstance(saver, CachedHistorySaver) def test_sync_cache_recreated_when_key_prefix_changes(): """The singleton must not outlive its namespace: a prefix change without a process restart leaves old-prefix entries unreachable and unpurgeable.""" reset_checkpointer() set_app_config(_provider_app_config("delta", {"key_prefix": "ns-a"})) freeze_checkpoint_channel_mode("delta") with checkpointer_context() as saver: saver._cache.set_many({"ns-a:t1:x": {"writes": []}}) first_cache = saver._cache # Same prefix: singleton is reused (warm across wrappers). with checkpointer_context() as saver: assert saver._cache is first_cache assert saver._cache.stats().entries == 1 # Prefix change: fresh cache, stale namespace gone with the old instance. set_app_config(_provider_app_config("delta", {"key_prefix": "ns-b"})) with checkpointer_context() as saver: assert saver._cache is not first_cache assert saver._cache.stats().entries == 0 reset_checkpointer() # --- test_checkpoint_cache_redis --- class _FakeEncoder: def encode(self, value: str) -> bytes: return value.encode() class _FakeConnectionPool: def get_encoder(self) -> _FakeEncoder: return _FakeEncoder() class _FakeRedis: """Minimal async redis stand-in: mget / set / pipeline / scan / unlink.""" def __init__(self) -> None: self.store: dict[str, bytes] = {} self.ttls: dict[str, int | None] = {} self.unlinked: list[tuple[str, ...]] = [] self.connection_pool = _FakeConnectionPool() async def mget(self, keys: list[str]) -> list[bytes | None]: return [self.store.get(k) for k in keys] def set(self, key: str, value: bytes, ex: int | None = None) -> None: self.store[key] = value self.ttls[key] = ex async def scan(self, cursor: int = 0, match: str | None = None, count: int = 500) -> tuple[int, list[str]]: import fnmatch keys = sorted(self.store) batch = keys[cursor : cursor + count] if match is not None: batch = [k for k in batch if fnmatch.fnmatchcase(k, match)] next_cursor = cursor + count return (0 if next_cursor >= len(keys) else next_cursor), batch async def unlink(self, *keys: str) -> int: self.unlinked.append(tuple(keys)) removed = 0 for key in keys: removed += self.store.pop(key, None) is not None return removed def pipeline(self, transaction: bool = False) -> "_FakePipeline": return _FakePipeline(self) async def aclose(self) -> None: pass class _FakePipeline: def __init__(self, client: _FakeRedis) -> None: self._client = client def set(self, key: str, value: bytes, ex: int | None = None) -> "_FakePipeline": self._client.set(key, value, ex=ex) return self async def execute(self) -> None: pass class _FailingRedis(_FakeRedis): """Simulates a redis outage: every operation raises RedisError.""" async def mget(self, keys: list[str]) -> list[bytes | None]: from redis.exceptions import RedisError raise RedisError("connection refused") async def scan(self, cursor: int = 0, match: str | None = None, count: int = 500) -> tuple[int, list[str]]: from redis.exceptions import RedisError raise RedisError("connection refused") def pipeline(self, transaction: bool = False) -> "_FakePipeline": from redis.exceptions import RedisError raise RedisError("connection refused") def _make_cache( monkeypatch: pytest.MonkeyPatch, fake: _FakeRedis, ttl_seconds: int = 60, *, serde: Any | None = None, **kwargs: Any, ): import deerflow.runtime.checkpoint_cache.redis as redis_mod monkeypatch.setattr(redis_mod, "_create_client", lambda *a, **k: fake) return redis_mod.RedisCheckpointHistoryCache( "redis://unused", serde=serde if serde is not None else JsonPlusSerializer(), ttl_seconds=ttl_seconds, **kwargs, ) def _redis_entry(i: int) -> dict: # Real message-like payloads to prove serde fidelity beyond plain dicts. from langchain_core.messages import AIMessage return {"writes": [("task-1", "messages", AIMessage(content=f"m{i}", id=f"ai-{i}"))], "seed": [AIMessage(content="s", id="ai-s")]} def _redis_app_config(database: dict) -> AppConfig: return AppConfig.model_validate( { "sandbox": {"use": "deerflow.sandbox.local.provider:LocalSandboxProvider"}, "database": database, } ) @pytest.mark.anyio async def test_redis_roundtrip_preserves_types(monkeypatch: pytest.MonkeyPatch): fake = _FakeRedis() cache = _make_cache(monkeypatch, fake) await cache.aset_many({"k1": _redis_entry(1), "k2": {"writes": []}}) hit = await cache.aget_many(["k1", "k2", "k3"]) assert set(hit) == {"k1", "k2"} msg = hit["k1"]["writes"][0][2] assert msg.content == "m1" and msg.id == "ai-1" and msg.type == "ai" assert "seed" not in hit["k2"] assert cache.stats().hits == 2 and cache.stats().misses == 1 @pytest.mark.anyio async def test_redis_keys_land_verbatim_and_ttl_set(monkeypatch: pytest.MonkeyPatch): fake = _FakeRedis() cache = _make_cache(monkeypatch, fake) await cache.aset_many({"k1": _redis_entry(1)}) assert list(fake.store) == ["k1"] assert fake.ttls["k1"] == 60 @pytest.mark.anyio async def test_redis_outage_degrades_to_all_miss(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture): fake = _FailingRedis() cache = _make_cache(monkeypatch, fake) with caplog.at_level("WARNING"): assert await cache.aget_many(["k1", "k2"]) == {} assert cache.stats().misses == 2 and cache.stats().hits == 0 assert "mget failed" in caplog.text @pytest.mark.anyio async def test_redis_outage_skips_write(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture): fake = _FailingRedis() cache = _make_cache(monkeypatch, fake) with caplog.at_level("WARNING"): await cache.aset_many({"k1": _redis_entry(1)}) # must not raise assert fake.store == {} assert "write failed" in caplog.text @pytest.mark.anyio async def test_redis_serialization_failure_preserves_neighbor_writes(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None: class _SelectiveFailingSerde: def dumps_typed(self, entry: dict) -> tuple[str, bytes]: if entry.get("bad"): raise TypeError("sensitive serializer detail") return "json", b"encoded" fake = _FakeRedis() cache = _make_cache(monkeypatch, fake, serde=_SelectiveFailingSerde()) with caplog.at_level("WARNING"): await cache.aset_many( { "before": {"writes": []}, "broken": {"writes": [], "bad": True}, "after": {"writes": []}, } ) assert fake.store == { "before": b"json\x00encoded", "after": b"json\x00encoded", } assert fake.ttls == {"before": 60, "after": 60} assert "broken" in caplog.text assert "TypeError" in caplog.text assert "sensitive serializer detail" not in caplog.text assert cache.stats().write_serialization_failures == 1 assert cache.stats().as_dict()["write_serialization_failures"] == 1 @pytest.mark.anyio async def test_zero_ttl_disables_expiry_explicitly(monkeypatch: pytest.MonkeyPatch): fake = _FakeRedis() cache = _make_cache(monkeypatch, fake, ttl_seconds=0) await cache.aset_many({"k1": _redis_entry(1)}) assert cache._ttl is None assert fake.ttls["k1"] is None # SET without EX: redis maxmemory policy only @pytest.mark.anyio async def test_adelete_thread_purges_matching_keys_only(monkeypatch: pytest.MonkeyPatch): fake = _FakeRedis() cache = _make_cache(monkeypatch, fake) prefix = "ckpt-hist:v1:db0" await cache.aset_many( { f"{prefix}:t1:aaa": _redis_entry(1), f"{prefix}:t1:bbb": _redis_entry(2), f"{prefix}:t10:ccc": _redis_entry(3), # 't1' stem must not over-match 't10' f"{prefix}:t2:ddd": _redis_entry(4), } ) await cache.adelete_thread(prefix, "t1") assert sorted(fake.store) == [f"{prefix}:t10:ccc", f"{prefix}:t2:ddd"] assert fake.unlinked # UNLINK, not DEL: non-blocking on big histories @pytest.mark.anyio async def test_adelete_thread_outage_degrades_without_raising(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture): fake = _FailingRedis() cache = _make_cache(monkeypatch, fake) with caplog.at_level("WARNING"): await cache.adelete_thread("p", "t1") # must not raise assert "thread purge failed" in caplog.text @pytest.mark.anyio async def test_provider_memory_default(): from deerflow.runtime.checkpoint_cache.memory import MemoryCheckpointHistoryCache async with make_checkpoint_cache(_redis_app_config({"backend": "sqlite"}), serde=JsonPlusSerializer()) as cache: assert isinstance(cache, MemoryCheckpointHistoryCache) assert cache.enabled is True @pytest.mark.anyio async def test_provider_zero_max_entries_disables_any_type(): app_config = _redis_app_config({"backend": "sqlite", "checkpoint_cache": {"type": "redis", "max_entries": 0}}) from deerflow.runtime.checkpoint_cache.memory import MemoryCheckpointHistoryCache async with make_checkpoint_cache(app_config, serde=JsonPlusSerializer()) as cache: assert isinstance(cache, MemoryCheckpointHistoryCache) assert cache.enabled is False def test_db_hash_distinguishes_backends_and_targets(): from deerflow.config.database_config import DatabaseConfig sqlite_cfg = DatabaseConfig.model_validate({"backend": "sqlite", "sqlite_dir": "/tmp/a"}) pg_cfg = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://u:p@h/db"}) pg_cfg2 = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://u:p@h/other"}) assert checkpoint_cache_db_hash(sqlite_cfg) != checkpoint_cache_db_hash(pg_cfg) assert checkpoint_cache_db_hash(pg_cfg) != checkpoint_cache_db_hash(pg_cfg2) assert len(checkpoint_cache_db_hash(pg_cfg)) == 12 def test_db_hash_stable_across_credential_rotation(): """Same database, rotated user/password -> same cache namespace.""" from deerflow.config.database_config import DatabaseConfig before = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://alice:secret1@pg.internal:5432/deerflow"}) rotated = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://bob:secret2@pg.internal:5432/deerflow"}) driver_suffix = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql+asyncpg://alice:secret1@pg.internal:5432/deerflow"}) other_db = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://alice:secret1@pg.internal:5432/other"}) assert checkpoint_cache_db_hash(before) == checkpoint_cache_db_hash(rotated) assert checkpoint_cache_db_hash(before) == checkpoint_cache_db_hash(driver_suffix) assert checkpoint_cache_db_hash(before) != checkpoint_cache_db_hash(other_db) def test_db_hash_unparseable_url_falls_back_to_raw(): from deerflow.config.database_config import DatabaseConfig cfg = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "not-a-url"}) assert len(checkpoint_cache_db_hash(cfg)) == 12 # stable, never raises def test_key_prefix_override_wins(): app_config = _redis_app_config({"backend": "sqlite", "checkpoint_cache": {"key_prefix": "custom:"}}) assert checkpoint_cache_key_prefix(app_config) == "custom:" default = checkpoint_cache_key_prefix(_redis_app_config({"backend": "sqlite"})) assert default.startswith("ckpt-hist:v1:") # --- test_checkpoint_cache_redis_literal_purge --- @pytest.mark.anyio @pytest.mark.parametrize( ("prefix", "thread_id", "sibling_thread_id", "pattern"), [ ("tenant*", "t1", "t10", r"tenant\*:t1:*"), ("tenant?", "t1", "t10", r"tenant\?:t1:*"), ("tenant[ab]", "t1", "t10", r"tenant\[ab\]:t1:*"), (r"tenant\name", "t1", "t10", r"tenant\\name:t1:*"), ("plain", "t1", "t10", "plain:t1:*"), ("plain", "t1*", "t10", r"plain:t1\*:*"), ], ) async def test_purge_escapes_namespace_and_filters_results(prefix, thread_id, sibling_thread_id, pattern): # Exercise the actual purge method without needing a running Redis server. # The expected SCAN pattern is a wire-level Redis glob contract; returned # keys also include a foreign namespace to pin the final deletion fence. cache = RedisCheckpointHistoryCache.__new__(RedisCheckpointHistoryCache) client = AsyncMock() cache._client = client client.connection_pool = Mock() client.connection_pool.get_encoder.return_value.encode.side_effect = lambda value: value.encode("utf-8") owned = make_history_key(prefix, thread_id, "", "c1", "messages").encode() foreign = make_history_key("tenant-other", thread_id, "", "c1", "messages").encode() sibling = make_history_key(prefix, sibling_thread_id, "", "c1", "messages").encode() async def scan(*, cursor, match, count): assert match == pattern assert count == 500 # Mixed byte/string results and multiple pages exercise redis clients # with either response mode and preserve the scan cursor protocol. if cursor == 0: return 7, [owned, foreign, sibling] assert cursor == 7 return 0, [owned.decode()] client.scan.side_effect = scan await cache.adelete_thread(prefix, thread_id) assert client.unlink.await_args_list[0].args == (owned,) assert client.unlink.await_args_list[1].args == (owned.decode(),) @pytest.mark.anyio async def test_purge_does_not_unlink_empty_filtered_page(): cache = RedisCheckpointHistoryCache.__new__(RedisCheckpointHistoryCache) cache._client = AsyncMock() cache._client.connection_pool = Mock() cache._client.connection_pool.get_encoder.return_value.encode.side_effect = lambda value: value.encode("utf-8") cache._client.scan.return_value = (0, [b"other:t1:history"]) await cache.adelete_thread("owned", "t1") cache._client.unlink.assert_not_awaited() @pytest.mark.anyio async def test_purge_uses_client_encoding_for_byte_key_fence(): """A non-UTF-8 Redis client must still unlink its own non-ASCII keys.""" cache = RedisCheckpointHistoryCache.__new__(RedisCheckpointHistoryCache) client = AsyncMock() cache._client = client client.connection_pool = Mock() client.connection_pool.get_encoder.return_value.encode.side_effect = lambda value: value.encode("latin-1") owned = make_history_key("café", "t1", "", "c1", "messages").encode("latin-1") sibling = make_history_key("café", "t10", "", "c1", "messages").encode("latin-1") client.scan.return_value = (0, [owned, sibling]) await cache.adelete_thread("café", "t1") client.scan.assert_awaited_once_with(cursor=0, match="café:t1:*", count=500) client.unlink.assert_awaited_once_with(owned)