mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
* test(backend): consolidate 57 fragmented single-fix files into 8 module suites * chore(tests): update benchmark README command and remap 603 shard durations --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
875 lines
33 KiB
Python
875 lines
33 KiB
Python
"""Behavioral suite for checkpoint history cache config, provider lifecycle, payload isolation, memory LRU, and Redis backend."""
|
|
|
|
import asyncio
|
|
import threading
|
|
from collections import OrderedDict
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from copy import deepcopy
|
|
from queue import Queue
|
|
from typing import Annotated, Any, TypedDict
|
|
from unittest.mock import AsyncMock, Mock
|
|
|
|
import pytest
|
|
from langgraph.channels import DeltaChannel
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
|
|
from langgraph.graph import StateGraph
|
|
|
|
from deerflow.config.app_config import AppConfig, set_app_config
|
|
from deerflow.config.database_config import CheckpointCacheConfig, DatabaseConfig
|
|
from deerflow.runtime.checkpoint_cache.base import (
|
|
CACHE_FORMAT_VERSION,
|
|
CheckpointCacheStats,
|
|
make_history_key,
|
|
thread_key_stem,
|
|
)
|
|
from deerflow.runtime.checkpoint_cache.memory import MemoryCheckpointHistoryCache
|
|
from deerflow.runtime.checkpoint_cache.provider import (
|
|
checkpoint_cache_db_hash,
|
|
checkpoint_cache_key_prefix,
|
|
make_checkpoint_cache,
|
|
)
|
|
from deerflow.runtime.checkpoint_cache.redis import RedisCheckpointHistoryCache
|
|
from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
|
|
from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
|
|
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
|
|
from deerflow.runtime.checkpointer.cached_saver import CachedHistorySaver
|
|
from deerflow.runtime.checkpointer.provider import checkpointer_context, reset_checkpointer
|
|
|
|
# --- test_checkpoint_cache_config ---
|
|
|
|
|
|
def test_checkpoint_cache_defaults():
|
|
cfg = DatabaseConfig()
|
|
assert cfg.checkpoint_cache.type == "memory"
|
|
assert cfg.checkpoint_cache.max_entries == 128
|
|
assert cfg.checkpoint_cache.redis_url is None
|
|
assert cfg.checkpoint_cache.ttl_seconds == 86400
|
|
assert cfg.checkpoint_cache.key_prefix == ""
|
|
|
|
|
|
def test_checkpoint_cache_from_dict_redis():
|
|
cfg = DatabaseConfig.model_validate(
|
|
{
|
|
"backend": "postgres",
|
|
"postgres_url": "postgresql://u:p@h/db",
|
|
"checkpoint_cache": {
|
|
"type": "redis",
|
|
"max_entries": 256,
|
|
"redis_url": "redis://localhost:6379/3",
|
|
"ttl_seconds": 3600,
|
|
"key_prefix": "prod:",
|
|
},
|
|
}
|
|
)
|
|
assert cfg.checkpoint_cache.type == "redis"
|
|
assert cfg.checkpoint_cache.max_entries == 256
|
|
assert cfg.checkpoint_cache.redis_url == "redis://localhost:6379/3"
|
|
assert cfg.checkpoint_cache.ttl_seconds == 3600
|
|
assert cfg.checkpoint_cache.key_prefix == "prod:"
|
|
|
|
|
|
def test_checkpoint_cache_zero_max_entries_means_disabled():
|
|
cfg = CheckpointCacheConfig(max_entries=0)
|
|
assert cfg.max_entries == 0
|
|
|
|
|
|
def test_checkpoint_cache_rejects_negative_max_entries():
|
|
import pydantic
|
|
import pytest
|
|
|
|
with pytest.raises(pydantic.ValidationError):
|
|
CheckpointCacheConfig(max_entries=-1)
|
|
|
|
|
|
# --- test_checkpoint_cache_memory ---
|
|
|
|
|
|
def _memory_entry(tag: str) -> dict:
|
|
return {"writes": [("task-1", "messages", tag)], "seed": f"seed-{tag}"}
|
|
|
|
|
|
def test_make_history_key_is_stable_and_scoped():
|
|
k1 = make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "messages")
|
|
k2 = make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "messages")
|
|
assert k1 == k2
|
|
assert k1.startswith("ckpt-hist:v1:db0:t1:")
|
|
# ns / checkpoint / channel each change the key
|
|
assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "sub", "c1", "messages")
|
|
assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "", "c2", "messages")
|
|
assert k1 != make_history_key("ckpt-hist:v1:db0", "t1", "", "c1", "todos")
|
|
assert k1 != make_history_key("ckpt-hist:v1:db9", "t1", "", "c1", "messages")
|
|
assert CACHE_FORMAT_VERSION == 1
|
|
|
|
|
|
def test_get_many_miss_then_hit():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
assert cache.get_many(["a"]) == {}
|
|
assert cache.stats().misses == 1
|
|
cache.set_many({"a": _memory_entry("x")})
|
|
hit = cache.get_many(["a"])
|
|
assert hit["a"]["writes"] == [("task-1", "messages", "x")]
|
|
assert hit["a"]["seed"] == "seed-x"
|
|
assert cache.stats().hits == 1
|
|
|
|
|
|
def test_entry_without_seed_roundtrips_without_seed_key():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
cache.set_many({"a": {"writes": []}})
|
|
hit = cache.get_many(["a"])
|
|
assert hit["a"] == {"writes": []}
|
|
assert "seed" not in hit["a"]
|
|
|
|
|
|
def test_copy_on_read_returns_fresh_writes_list():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
cache.set_many({"a": _memory_entry("x")})
|
|
first = cache.get_many(["a"])["a"]
|
|
first["writes"].append(("task-2", "messages", "MUTATION"))
|
|
second = cache.get_many(["a"])["a"]
|
|
assert second["writes"] == [("task-1", "messages", "x")]
|
|
|
|
|
|
def test_caller_mutation_after_set_does_not_leak():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
entry = _memory_entry("x")
|
|
cache.set_many({"a": entry})
|
|
entry["writes"].append(("task-2", "messages", "MUTATION"))
|
|
assert cache.get_many(["a"])["a"]["writes"] == [("task-1", "messages", "x")]
|
|
|
|
|
|
def test_lru_evicts_oldest_and_counts():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=2)
|
|
cache.set_many({"a": _memory_entry("a"), "b": _memory_entry("b")})
|
|
cache.get_many(["a"]) # refresh a
|
|
cache.set_many({"c": _memory_entry("c")}) # evicts b
|
|
assert cache.get_many(["b"]) == {}
|
|
assert cache.get_many(["a"]) != {}
|
|
assert cache.stats().evictions == 1
|
|
assert cache.stats().entries == 2
|
|
|
|
|
|
def test_zero_max_entries_disables():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=0)
|
|
assert cache.enabled is False
|
|
cache.set_many({"a": _memory_entry("x")})
|
|
assert cache.get_many(["a"]) == {}
|
|
assert cache.stats().entries == 0
|
|
|
|
|
|
def test_delete_thread_purges_only_that_thread():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=16)
|
|
prefix = "ckpt-hist:v1:db0"
|
|
t1_keys = [make_history_key(prefix, "t1", "", f"c{i}", "messages") for i in range(3)]
|
|
t2_key = make_history_key(prefix, "t2", "", "c0", "messages")
|
|
# A thread_id that is a prefix of another must not over-match: the stem
|
|
# ends with ':' so "t1" never matches "t10"'s keys.
|
|
t10_key = make_history_key(prefix, "t10", "", "c0", "messages")
|
|
cache.set_many({k: _memory_entry(k) for k in [*t1_keys, t2_key, t10_key]})
|
|
|
|
cache.delete_thread(prefix, "t1")
|
|
|
|
assert cache.stats().entries == 2
|
|
assert all(cache.get_many([k]) == {} for k in t1_keys)
|
|
assert cache.get_many([t2_key]) != {}
|
|
assert cache.get_many([t10_key]) != {}
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_adelete_thread_matches_sync():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
prefix = "ckpt-hist:v1:db0"
|
|
key = make_history_key(prefix, "t1", "", "c0", "messages")
|
|
await cache.aset_many({key: _memory_entry("x")})
|
|
await cache.adelete_thread(prefix, "t1")
|
|
assert cache.get_many([key]) == {}
|
|
|
|
|
|
def test_thread_key_stem_matches_make_history_key_layout():
|
|
key = make_history_key("p", "t1", "ns", "c1", "messages")
|
|
assert key.startswith(thread_key_stem("p", "t1"))
|
|
assert not key.startswith(thread_key_stem("p", "t"))
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_async_protocol_matches_sync():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
await cache.aset_many({"a": _memory_entry("x")})
|
|
hit = await cache.aget_many(["a"])
|
|
assert hit["a"]["seed"] == "seed-x"
|
|
stats = cache.stats()
|
|
assert isinstance(stats, CheckpointCacheStats)
|
|
assert stats.as_dict()["hits"] == 1
|
|
await cache.aclose()
|
|
assert cache.get_many(["a"]) == {}
|
|
|
|
|
|
# --- test_checkpoint_cache_memory_concurrency ---
|
|
|
|
|
|
@pytest.mark.parametrize("mutation", ["evict", "purge", "close"])
|
|
def test_lookup_survives_concurrent_removal(mutation):
|
|
cache = MemoryCheckpointHistoryCache(max_entries=1)
|
|
key = make_history_key("p", "t1", "", "c1", "messages")
|
|
entry = {"writes": [("task", "messages", "original")], "seed": "seed"}
|
|
cache.set_many({key: entry})
|
|
looked_up = threading.Event()
|
|
release_reader = threading.Event()
|
|
writer_started = threading.Event()
|
|
writer_finished = threading.Event()
|
|
coordination = Queue()
|
|
removal_reported = threading.Event()
|
|
|
|
def report_removal():
|
|
if not removal_reported.is_set():
|
|
removal_reported.set()
|
|
coordination.put("removed")
|
|
|
|
class PausedLookup(OrderedDict):
|
|
def get(self, lookup_key, default=None):
|
|
value = super().get(lookup_key, default)
|
|
if lookup_key == key:
|
|
looked_up.set()
|
|
assert release_reader.wait(5), "reader was not released"
|
|
return value
|
|
|
|
def __delitem__(self, deleted_key):
|
|
super().__delitem__(deleted_key)
|
|
if deleted_key == key:
|
|
report_removal()
|
|
|
|
def popitem(self, last=True):
|
|
removed_key, value = super().popitem(last=last)
|
|
if removed_key == key:
|
|
report_removal()
|
|
return removed_key, value
|
|
|
|
def clear(self):
|
|
contained_target = key in self
|
|
super().clear()
|
|
if contained_target:
|
|
report_removal()
|
|
|
|
class ObservedLock:
|
|
"""Report a contender before it blocks behind the paused reader."""
|
|
|
|
def __init__(self):
|
|
self._lock = threading.Lock()
|
|
|
|
def __enter__(self):
|
|
if self._lock.locked():
|
|
coordination.put("writer_waiting")
|
|
self._lock.acquire()
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
self._lock.release()
|
|
|
|
# Pause at the actual lookup -> LRU promotion window, not before get_many.
|
|
cache._data = PausedLookup(cache._data)
|
|
cache._lock = ObservedLock()
|
|
|
|
def remove():
|
|
writer_started.set()
|
|
try:
|
|
if mutation == "evict":
|
|
cache.set_many({"other": {"writes": []}})
|
|
elif mutation == "purge":
|
|
cache.delete_thread("p", "t1")
|
|
else:
|
|
asyncio.run(cache.aclose())
|
|
finally:
|
|
writer_finished.set()
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
reader = pool.submit(cache.get_many, [key])
|
|
try:
|
|
assert looked_up.wait(5)
|
|
writer = pool.submit(remove)
|
|
assert writer_started.wait(5)
|
|
# A serialized writer reports that it is waiting on the reader's
|
|
# lock. On unfixed main it instead removes the key and reports that,
|
|
# deterministically reproducing the get -> move_to_end race.
|
|
assert coordination.get(timeout=5) == "writer_waiting"
|
|
finally:
|
|
release_reader.set()
|
|
assert reader.result(timeout=5) == {key: entry}
|
|
writer.result(timeout=5)
|
|
assert cache.get_many([key]) == {}
|
|
assert cache.stats().entries == int(mutation == "evict")
|
|
|
|
|
|
def test_parallel_read_write_keeps_capacity_and_accounting():
|
|
cache = MemoryCheckpointHistoryCache(max_entries=4)
|
|
start = threading.Barrier(8)
|
|
|
|
def exercise(worker):
|
|
start.wait(timeout=5)
|
|
for n in range(200):
|
|
key = f"{worker}:{n}"
|
|
cache.set_many({key: {"writes": [("task", "messages", key)]}})
|
|
result = cache.get_many([key])
|
|
if key in result:
|
|
assert result[key]["writes"] == [("task", "messages", key)]
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
list(pool.map(exercise, range(8)))
|
|
stats = cache.stats()
|
|
assert stats.entries == 4
|
|
assert stats.evictions == 8 * 200 - 4
|
|
assert stats.hits + stats.misses == 8 * 200
|
|
|
|
|
|
# --- test_checkpoint_cache_payload_isolation ---
|
|
|
|
|
|
def _fold(state, writes):
|
|
result = list(state)
|
|
for batch in writes:
|
|
result.extend(batch)
|
|
return result
|
|
|
|
|
|
def _stack(frequency, enabled):
|
|
class State(TypedDict):
|
|
items: Annotated[list[dict], DeltaChannel(_fold, snapshot_frequency=frequency)]
|
|
|
|
inner = InMemorySaver()
|
|
cache = MemoryCheckpointHistoryCache(max_entries=32 if enabled else 0)
|
|
saver = CachedHistorySaver(inner, cache, key_prefix="payload-isolation")
|
|
builder = StateGraph(State)
|
|
for step in range(4):
|
|
builder.add_node(str(step), lambda state: {"items": [{"nested": {"value": "saved"}}]})
|
|
if step:
|
|
builder.add_edge(str(step - 1), str(step))
|
|
builder.set_entry_point("0")
|
|
builder.set_finish_point("3")
|
|
graph = builder.compile(checkpointer=saver)
|
|
return graph, CheckpointStateAccessor.bind(graph, saver, mode="delta"), cache
|
|
|
|
|
|
def _mutate(snapshot):
|
|
original = deepcopy(snapshot.values)
|
|
snapshot.values["items"][0]["nested"]["value"] = "tampered"
|
|
snapshot.values["items"].append({"nested": {"value": "injected"}})
|
|
return original
|
|
|
|
|
|
@pytest.mark.parametrize("frequency", [2, 100], ids=["snapshot-seed", "pending-writes"])
|
|
@pytest.mark.parametrize("enabled", [True, False], ids=["cached", "uncached-control"])
|
|
def test_sync_graph_snapshots_are_isolated(frequency, enabled):
|
|
graph, accessor, cache = _stack(frequency, enabled)
|
|
config = {"configurable": {"thread_id": "sync-payload-isolation"}}
|
|
graph.invoke({"items": [{"nested": {"value": "input"}}]}, config)
|
|
history = accessor.history(config)
|
|
checked = 0
|
|
for checkpoint in history:
|
|
snapshot = accessor.get(checkpoint.config)
|
|
if not snapshot.values.get("items"):
|
|
continue
|
|
original = _mutate(snapshot)
|
|
assert accessor.get(checkpoint.config).values == original
|
|
checked += 1
|
|
assert checked >= 4
|
|
assert cache.stats().hits > 0 if enabled else cache.stats().entries == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("frequency", [2, 100], ids=["snapshot-seed", "pending-writes"])
|
|
@pytest.mark.parametrize("enabled", [True, False], ids=["cached", "uncached-control"])
|
|
async def test_async_graph_snapshots_are_isolated(frequency, enabled):
|
|
graph, accessor, cache = _stack(frequency, enabled)
|
|
config = {"configurable": {"thread_id": "async-payload-isolation"}}
|
|
await graph.ainvoke({"items": [{"nested": {"value": "input"}}]}, config)
|
|
history = await accessor.ahistory(config)
|
|
checked = 0
|
|
for checkpoint in history:
|
|
snapshot = await accessor.aget(checkpoint.config)
|
|
if not snapshot.values.get("items"):
|
|
continue
|
|
original = _mutate(snapshot)
|
|
assert (await accessor.aget(checkpoint.config)).values == original
|
|
checked += 1
|
|
assert checked >= 4
|
|
assert cache.stats().hits > 0 if enabled else cache.stats().entries == 0
|
|
|
|
|
|
@pytest.mark.parametrize("direction", ["ingest", "read"])
|
|
def test_cache_copies_nested_write_payloads_and_snapshot_seeds(direction):
|
|
from langchain_core.messages import HumanMessage
|
|
from langgraph.checkpoint.serde.types import _DeltaSnapshot
|
|
|
|
cache = MemoryCheckpointHistoryCache()
|
|
seed = _DeltaSnapshot([HumanMessage(content="seed", id="seed", additional_kwargs={"nested": ["original"]})])
|
|
entry = {"seed": seed, "writes": [("task", "messages", [HumanMessage(content="write", id="write", additional_kwargs={"nested": ["original"]})])]}
|
|
original = deepcopy(entry)
|
|
cache.set_many({"key": entry})
|
|
exposed = entry if direction == "ingest" else cache.get_many(["key"])["key"]
|
|
exposed["seed"].value[0].additional_kwargs["nested"].append("corrupted")
|
|
exposed["writes"][0][2][0].content = "corrupted"
|
|
assert cache.get_many(["key"])["key"] == original
|
|
|
|
|
|
def test_payload_copying_runs_outside_registry_lock():
|
|
cache = MemoryCheckpointHistoryCache()
|
|
copies = []
|
|
|
|
class Probe:
|
|
def __deepcopy__(self, memo):
|
|
assert cache._lock.acquire(blocking=False), "copy callback ran under the cache lock"
|
|
cache._lock.release()
|
|
copies.append(True)
|
|
return Probe()
|
|
|
|
cache.set_many({"key": {"writes": [("task", "channel", Probe())]}})
|
|
cache.get_many(["key"])
|
|
assert len(copies) == 2
|
|
|
|
|
|
# --- test_checkpoint_cache_provider ---
|
|
|
|
|
|
def _provider_app_config(mode: str, cache: dict | None = None) -> AppConfig:
|
|
database: dict = {"backend": "memory", "checkpoint_channel_mode": mode}
|
|
if cache is not None:
|
|
database["checkpoint_cache"] = cache
|
|
return AppConfig.model_validate(
|
|
{
|
|
"sandbox": {"use": "deerflow.sandbox.local.provider:LocalSandboxProvider"},
|
|
"database": database,
|
|
}
|
|
)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_delta_mode_wraps_with_cached_saver():
|
|
set_app_config(_provider_app_config("delta"))
|
|
freeze_checkpoint_channel_mode("delta")
|
|
async with make_checkpointer() as saver:
|
|
assert isinstance(saver, CachedHistorySaver)
|
|
assert saver.stats()["entries"] == 0
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_full_mode_yields_raw_saver():
|
|
set_app_config(_provider_app_config("full"))
|
|
freeze_checkpoint_channel_mode("full")
|
|
async with make_checkpointer() as saver:
|
|
assert not isinstance(saver, CachedHistorySaver)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_zero_max_entries_disables_but_still_wraps():
|
|
set_app_config(_provider_app_config("delta", {"max_entries": 0}))
|
|
freeze_checkpoint_channel_mode("delta")
|
|
async with make_checkpointer() as saver:
|
|
assert isinstance(saver, CachedHistorySaver)
|
|
# Disabled cache -> every history call is a full walk on the inner saver.
|
|
assert saver._cache.enabled is False
|
|
|
|
|
|
def test_sync_delta_mode_wraps_memory():
|
|
set_app_config(_provider_app_config("delta"))
|
|
freeze_checkpoint_channel_mode("delta")
|
|
reset_checkpointer()
|
|
with checkpointer_context() as saver:
|
|
assert isinstance(saver, CachedHistorySaver)
|
|
|
|
|
|
def test_sync_redis_cache_type_is_config_error():
|
|
set_app_config(_provider_app_config("delta", {"type": "redis"}))
|
|
freeze_checkpoint_channel_mode("delta")
|
|
reset_checkpointer()
|
|
with pytest.raises(ValueError, match="redis"):
|
|
with checkpointer_context():
|
|
pass
|
|
|
|
|
|
def test_sync_full_mode_unwrapped():
|
|
set_app_config(_provider_app_config("full"))
|
|
freeze_checkpoint_channel_mode("full")
|
|
reset_checkpointer()
|
|
with checkpointer_context() as saver:
|
|
assert not isinstance(saver, CachedHistorySaver)
|
|
|
|
|
|
def test_sync_cache_recreated_when_key_prefix_changes():
|
|
"""The singleton must not outlive its namespace: a prefix change without
|
|
a process restart leaves old-prefix entries unreachable and unpurgeable."""
|
|
reset_checkpointer()
|
|
set_app_config(_provider_app_config("delta", {"key_prefix": "ns-a"}))
|
|
freeze_checkpoint_channel_mode("delta")
|
|
with checkpointer_context() as saver:
|
|
saver._cache.set_many({"ns-a:t1:x": {"writes": []}})
|
|
first_cache = saver._cache
|
|
# Same prefix: singleton is reused (warm across wrappers).
|
|
with checkpointer_context() as saver:
|
|
assert saver._cache is first_cache
|
|
assert saver._cache.stats().entries == 1
|
|
# Prefix change: fresh cache, stale namespace gone with the old instance.
|
|
set_app_config(_provider_app_config("delta", {"key_prefix": "ns-b"}))
|
|
with checkpointer_context() as saver:
|
|
assert saver._cache is not first_cache
|
|
assert saver._cache.stats().entries == 0
|
|
reset_checkpointer()
|
|
|
|
|
|
# --- test_checkpoint_cache_redis ---
|
|
|
|
|
|
class _FakeEncoder:
|
|
def encode(self, value: str) -> bytes:
|
|
return value.encode()
|
|
|
|
|
|
class _FakeConnectionPool:
|
|
def get_encoder(self) -> _FakeEncoder:
|
|
return _FakeEncoder()
|
|
|
|
|
|
class _FakeRedis:
|
|
"""Minimal async redis stand-in: mget / set / pipeline / scan / unlink."""
|
|
|
|
def __init__(self) -> None:
|
|
self.store: dict[str, bytes] = {}
|
|
self.ttls: dict[str, int | None] = {}
|
|
self.unlinked: list[tuple[str, ...]] = []
|
|
self.connection_pool = _FakeConnectionPool()
|
|
|
|
async def mget(self, keys: list[str]) -> list[bytes | None]:
|
|
return [self.store.get(k) for k in keys]
|
|
|
|
def set(self, key: str, value: bytes, ex: int | None = None) -> None:
|
|
self.store[key] = value
|
|
self.ttls[key] = ex
|
|
|
|
async def scan(self, cursor: int = 0, match: str | None = None, count: int = 500) -> tuple[int, list[str]]:
|
|
import fnmatch
|
|
|
|
keys = sorted(self.store)
|
|
batch = keys[cursor : cursor + count]
|
|
if match is not None:
|
|
batch = [k for k in batch if fnmatch.fnmatchcase(k, match)]
|
|
next_cursor = cursor + count
|
|
return (0 if next_cursor >= len(keys) else next_cursor), batch
|
|
|
|
async def unlink(self, *keys: str) -> int:
|
|
self.unlinked.append(tuple(keys))
|
|
removed = 0
|
|
for key in keys:
|
|
removed += self.store.pop(key, None) is not None
|
|
return removed
|
|
|
|
def pipeline(self, transaction: bool = False) -> "_FakePipeline":
|
|
return _FakePipeline(self)
|
|
|
|
async def aclose(self) -> None:
|
|
pass
|
|
|
|
|
|
class _FakePipeline:
|
|
def __init__(self, client: _FakeRedis) -> None:
|
|
self._client = client
|
|
|
|
def set(self, key: str, value: bytes, ex: int | None = None) -> "_FakePipeline":
|
|
self._client.set(key, value, ex=ex)
|
|
return self
|
|
|
|
async def execute(self) -> None:
|
|
pass
|
|
|
|
|
|
class _FailingRedis(_FakeRedis):
|
|
"""Simulates a redis outage: every operation raises RedisError."""
|
|
|
|
async def mget(self, keys: list[str]) -> list[bytes | None]:
|
|
from redis.exceptions import RedisError
|
|
|
|
raise RedisError("connection refused")
|
|
|
|
async def scan(self, cursor: int = 0, match: str | None = None, count: int = 500) -> tuple[int, list[str]]:
|
|
from redis.exceptions import RedisError
|
|
|
|
raise RedisError("connection refused")
|
|
|
|
def pipeline(self, transaction: bool = False) -> "_FakePipeline":
|
|
from redis.exceptions import RedisError
|
|
|
|
raise RedisError("connection refused")
|
|
|
|
|
|
def _make_cache(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
fake: _FakeRedis,
|
|
ttl_seconds: int = 60,
|
|
*,
|
|
serde: Any | None = None,
|
|
**kwargs: Any,
|
|
):
|
|
import deerflow.runtime.checkpoint_cache.redis as redis_mod
|
|
|
|
monkeypatch.setattr(redis_mod, "_create_client", lambda *a, **k: fake)
|
|
return redis_mod.RedisCheckpointHistoryCache(
|
|
"redis://unused",
|
|
serde=serde if serde is not None else JsonPlusSerializer(),
|
|
ttl_seconds=ttl_seconds,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def _redis_entry(i: int) -> dict:
|
|
# Real message-like payloads to prove serde fidelity beyond plain dicts.
|
|
from langchain_core.messages import AIMessage
|
|
|
|
return {"writes": [("task-1", "messages", AIMessage(content=f"m{i}", id=f"ai-{i}"))], "seed": [AIMessage(content="s", id="ai-s")]}
|
|
|
|
|
|
def _redis_app_config(database: dict) -> AppConfig:
|
|
return AppConfig.model_validate(
|
|
{
|
|
"sandbox": {"use": "deerflow.sandbox.local.provider:LocalSandboxProvider"},
|
|
"database": database,
|
|
}
|
|
)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_redis_roundtrip_preserves_types(monkeypatch: pytest.MonkeyPatch):
|
|
fake = _FakeRedis()
|
|
cache = _make_cache(monkeypatch, fake)
|
|
await cache.aset_many({"k1": _redis_entry(1), "k2": {"writes": []}})
|
|
hit = await cache.aget_many(["k1", "k2", "k3"])
|
|
assert set(hit) == {"k1", "k2"}
|
|
msg = hit["k1"]["writes"][0][2]
|
|
assert msg.content == "m1" and msg.id == "ai-1" and msg.type == "ai"
|
|
assert "seed" not in hit["k2"]
|
|
assert cache.stats().hits == 2 and cache.stats().misses == 1
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_redis_keys_land_verbatim_and_ttl_set(monkeypatch: pytest.MonkeyPatch):
|
|
fake = _FakeRedis()
|
|
cache = _make_cache(monkeypatch, fake)
|
|
await cache.aset_many({"k1": _redis_entry(1)})
|
|
assert list(fake.store) == ["k1"]
|
|
assert fake.ttls["k1"] == 60
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_redis_outage_degrades_to_all_miss(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture):
|
|
fake = _FailingRedis()
|
|
cache = _make_cache(monkeypatch, fake)
|
|
with caplog.at_level("WARNING"):
|
|
assert await cache.aget_many(["k1", "k2"]) == {}
|
|
assert cache.stats().misses == 2 and cache.stats().hits == 0
|
|
assert "mget failed" in caplog.text
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_redis_outage_skips_write(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture):
|
|
fake = _FailingRedis()
|
|
cache = _make_cache(monkeypatch, fake)
|
|
with caplog.at_level("WARNING"):
|
|
await cache.aset_many({"k1": _redis_entry(1)}) # must not raise
|
|
assert fake.store == {}
|
|
assert "write failed" in caplog.text
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_redis_serialization_failure_preserves_neighbor_writes(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None:
|
|
class _SelectiveFailingSerde:
|
|
def dumps_typed(self, entry: dict) -> tuple[str, bytes]:
|
|
if entry.get("bad"):
|
|
raise TypeError("sensitive serializer detail")
|
|
return "json", b"encoded"
|
|
|
|
fake = _FakeRedis()
|
|
cache = _make_cache(monkeypatch, fake, serde=_SelectiveFailingSerde())
|
|
with caplog.at_level("WARNING"):
|
|
await cache.aset_many(
|
|
{
|
|
"before": {"writes": []},
|
|
"broken": {"writes": [], "bad": True},
|
|
"after": {"writes": []},
|
|
}
|
|
)
|
|
|
|
assert fake.store == {
|
|
"before": b"json\x00encoded",
|
|
"after": b"json\x00encoded",
|
|
}
|
|
assert fake.ttls == {"before": 60, "after": 60}
|
|
assert "broken" in caplog.text
|
|
assert "TypeError" in caplog.text
|
|
assert "sensitive serializer detail" not in caplog.text
|
|
assert cache.stats().write_serialization_failures == 1
|
|
assert cache.stats().as_dict()["write_serialization_failures"] == 1
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_zero_ttl_disables_expiry_explicitly(monkeypatch: pytest.MonkeyPatch):
|
|
fake = _FakeRedis()
|
|
cache = _make_cache(monkeypatch, fake, ttl_seconds=0)
|
|
await cache.aset_many({"k1": _redis_entry(1)})
|
|
assert cache._ttl is None
|
|
assert fake.ttls["k1"] is None # SET without EX: redis maxmemory policy only
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_adelete_thread_purges_matching_keys_only(monkeypatch: pytest.MonkeyPatch):
|
|
fake = _FakeRedis()
|
|
cache = _make_cache(monkeypatch, fake)
|
|
prefix = "ckpt-hist:v1:db0"
|
|
await cache.aset_many(
|
|
{
|
|
f"{prefix}:t1:aaa": _redis_entry(1),
|
|
f"{prefix}:t1:bbb": _redis_entry(2),
|
|
f"{prefix}:t10:ccc": _redis_entry(3), # 't1' stem must not over-match 't10'
|
|
f"{prefix}:t2:ddd": _redis_entry(4),
|
|
}
|
|
)
|
|
|
|
await cache.adelete_thread(prefix, "t1")
|
|
|
|
assert sorted(fake.store) == [f"{prefix}:t10:ccc", f"{prefix}:t2:ddd"]
|
|
assert fake.unlinked # UNLINK, not DEL: non-blocking on big histories
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_adelete_thread_outage_degrades_without_raising(monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture):
|
|
fake = _FailingRedis()
|
|
cache = _make_cache(monkeypatch, fake)
|
|
with caplog.at_level("WARNING"):
|
|
await cache.adelete_thread("p", "t1") # must not raise
|
|
assert "thread purge failed" in caplog.text
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_provider_memory_default():
|
|
from deerflow.runtime.checkpoint_cache.memory import MemoryCheckpointHistoryCache
|
|
|
|
async with make_checkpoint_cache(_redis_app_config({"backend": "sqlite"}), serde=JsonPlusSerializer()) as cache:
|
|
assert isinstance(cache, MemoryCheckpointHistoryCache)
|
|
assert cache.enabled is True
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_provider_zero_max_entries_disables_any_type():
|
|
app_config = _redis_app_config({"backend": "sqlite", "checkpoint_cache": {"type": "redis", "max_entries": 0}})
|
|
from deerflow.runtime.checkpoint_cache.memory import MemoryCheckpointHistoryCache
|
|
|
|
async with make_checkpoint_cache(app_config, serde=JsonPlusSerializer()) as cache:
|
|
assert isinstance(cache, MemoryCheckpointHistoryCache)
|
|
assert cache.enabled is False
|
|
|
|
|
|
def test_db_hash_distinguishes_backends_and_targets():
|
|
from deerflow.config.database_config import DatabaseConfig
|
|
|
|
sqlite_cfg = DatabaseConfig.model_validate({"backend": "sqlite", "sqlite_dir": "/tmp/a"})
|
|
pg_cfg = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://u:p@h/db"})
|
|
pg_cfg2 = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://u:p@h/other"})
|
|
assert checkpoint_cache_db_hash(sqlite_cfg) != checkpoint_cache_db_hash(pg_cfg)
|
|
assert checkpoint_cache_db_hash(pg_cfg) != checkpoint_cache_db_hash(pg_cfg2)
|
|
assert len(checkpoint_cache_db_hash(pg_cfg)) == 12
|
|
|
|
|
|
def test_db_hash_stable_across_credential_rotation():
|
|
"""Same database, rotated user/password -> same cache namespace."""
|
|
from deerflow.config.database_config import DatabaseConfig
|
|
|
|
before = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://alice:secret1@pg.internal:5432/deerflow"})
|
|
rotated = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://bob:secret2@pg.internal:5432/deerflow"})
|
|
driver_suffix = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql+asyncpg://alice:secret1@pg.internal:5432/deerflow"})
|
|
other_db = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "postgresql://alice:secret1@pg.internal:5432/other"})
|
|
assert checkpoint_cache_db_hash(before) == checkpoint_cache_db_hash(rotated)
|
|
assert checkpoint_cache_db_hash(before) == checkpoint_cache_db_hash(driver_suffix)
|
|
assert checkpoint_cache_db_hash(before) != checkpoint_cache_db_hash(other_db)
|
|
|
|
|
|
def test_db_hash_unparseable_url_falls_back_to_raw():
|
|
from deerflow.config.database_config import DatabaseConfig
|
|
|
|
cfg = DatabaseConfig.model_validate({"backend": "postgres", "postgres_url": "not-a-url"})
|
|
assert len(checkpoint_cache_db_hash(cfg)) == 12 # stable, never raises
|
|
|
|
|
|
def test_key_prefix_override_wins():
|
|
app_config = _redis_app_config({"backend": "sqlite", "checkpoint_cache": {"key_prefix": "custom:"}})
|
|
assert checkpoint_cache_key_prefix(app_config) == "custom:"
|
|
default = checkpoint_cache_key_prefix(_redis_app_config({"backend": "sqlite"}))
|
|
assert default.startswith("ckpt-hist:v1:")
|
|
|
|
|
|
# --- test_checkpoint_cache_redis_literal_purge ---
|
|
|
|
|
|
@pytest.mark.anyio
|
|
@pytest.mark.parametrize(
|
|
("prefix", "thread_id", "sibling_thread_id", "pattern"),
|
|
[
|
|
("tenant*", "t1", "t10", r"tenant\*:t1:*"),
|
|
("tenant?", "t1", "t10", r"tenant\?:t1:*"),
|
|
("tenant[ab]", "t1", "t10", r"tenant\[ab\]:t1:*"),
|
|
(r"tenant\name", "t1", "t10", r"tenant\\name:t1:*"),
|
|
("plain", "t1", "t10", "plain:t1:*"),
|
|
("plain", "t1*", "t10", r"plain:t1\*:*"),
|
|
],
|
|
)
|
|
async def test_purge_escapes_namespace_and_filters_results(prefix, thread_id, sibling_thread_id, pattern):
|
|
# Exercise the actual purge method without needing a running Redis server.
|
|
# The expected SCAN pattern is a wire-level Redis glob contract; returned
|
|
# keys also include a foreign namespace to pin the final deletion fence.
|
|
cache = RedisCheckpointHistoryCache.__new__(RedisCheckpointHistoryCache)
|
|
client = AsyncMock()
|
|
cache._client = client
|
|
client.connection_pool = Mock()
|
|
client.connection_pool.get_encoder.return_value.encode.side_effect = lambda value: value.encode("utf-8")
|
|
owned = make_history_key(prefix, thread_id, "", "c1", "messages").encode()
|
|
foreign = make_history_key("tenant-other", thread_id, "", "c1", "messages").encode()
|
|
sibling = make_history_key(prefix, sibling_thread_id, "", "c1", "messages").encode()
|
|
|
|
async def scan(*, cursor, match, count):
|
|
assert match == pattern
|
|
assert count == 500
|
|
# Mixed byte/string results and multiple pages exercise redis clients
|
|
# with either response mode and preserve the scan cursor protocol.
|
|
if cursor == 0:
|
|
return 7, [owned, foreign, sibling]
|
|
assert cursor == 7
|
|
return 0, [owned.decode()]
|
|
|
|
client.scan.side_effect = scan
|
|
await cache.adelete_thread(prefix, thread_id)
|
|
assert client.unlink.await_args_list[0].args == (owned,)
|
|
assert client.unlink.await_args_list[1].args == (owned.decode(),)
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_purge_does_not_unlink_empty_filtered_page():
|
|
cache = RedisCheckpointHistoryCache.__new__(RedisCheckpointHistoryCache)
|
|
cache._client = AsyncMock()
|
|
cache._client.connection_pool = Mock()
|
|
cache._client.connection_pool.get_encoder.return_value.encode.side_effect = lambda value: value.encode("utf-8")
|
|
cache._client.scan.return_value = (0, [b"other:t1:history"])
|
|
await cache.adelete_thread("owned", "t1")
|
|
cache._client.unlink.assert_not_awaited()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_purge_uses_client_encoding_for_byte_key_fence():
|
|
"""A non-UTF-8 Redis client must still unlink its own non-ASCII keys."""
|
|
cache = RedisCheckpointHistoryCache.__new__(RedisCheckpointHistoryCache)
|
|
client = AsyncMock()
|
|
cache._client = client
|
|
client.connection_pool = Mock()
|
|
client.connection_pool.get_encoder.return_value.encode.side_effect = lambda value: value.encode("latin-1")
|
|
owned = make_history_key("café", "t1", "", "c1", "messages").encode("latin-1")
|
|
sibling = make_history_key("café", "t10", "", "c1", "messages").encode("latin-1")
|
|
client.scan.return_value = (0, [owned, sibling])
|
|
|
|
await cache.adelete_thread("café", "t1")
|
|
|
|
client.scan.assert_awaited_once_with(cursor=0, match="café:t1:*", count=500)
|
|
client.unlink.assert_awaited_once_with(owned)
|