mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
dc6f0d7421
* fix(memory): persist the DeerMem extraction watermark across restarts and replicas
The updater's per-thread extraction watermark lived only in an in-process
LRU, so a restart or a second Gateway replica on the same storage root fed
already-extracted turns to the memory LLM again.
FileMemoryStorage now keeps it in users/{user}/agents/{agent}/.metadata/
watermarks.json, keyed by thread id, written atomically under the per-user
lock without touching memory.json and capped by watermark_max_keys. Each
update feeds after the furthest of the LRU entry and the persisted one, and
writes through after an extraction or a pre-screen skip; bypass_watermark is
unchanged and sidecar IO failures fall back to the LRU. Memory clears remove
the file with the other .metadata sidecars. Content-based message identities
are stored as SHA-256 digests so the sidecar holds no conversation text.
* docs(changelog): link the memory watermark entry to #7204 and #7205
* fix(memory): never replace an unreadable watermark sidecar
set_extraction_watermark rewrites the whole watermarks.json, and the shared
sidecar reader turns an OSError into an empty mapping, so a failed read could
replace every other thread's watermark with the current one (or delete the
file). The watermark sidecar now has its own reader: a missing file reads as
empty, malformed content reads as empty (nothing to keep; the next write
repairs it), and any other OSError propagates before anything is written. The
updater already logs that and keeps the extraction successful.
* fix(memory): keep the persisted watermark when the last message has no identity
_message_identity returns None for a last message with neither an id nor
text. The updater wrote that None through, which deleted the thread's entry
from watermarks.json, losing a watermark a peer may have advanced and forcing
a full feed on every replica. The write-through now skips None: the persisted
entry still marks an extracted prefix and the furthest-of-both read keeps
using it.
54 lines
2.0 KiB
Python
54 lines
2.0 KiB
Python
"""Regression anchor: the DeerMem watermark sidecar stays off the event loop."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage, HumanMessage
|
|
|
|
from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig
|
|
from deerflow.agents.memory.backends.deermem.deermem.core.storage import FileMemoryStorage
|
|
from deerflow.agents.memory.backends.deermem.deermem.core.updater import MemoryUpdater
|
|
|
|
pytestmark = pytest.mark.asyncio
|
|
|
|
|
|
class _Response:
|
|
content = json.dumps({"user": {}, "history": {}, "newFacts": [], "factsToRemove": []})
|
|
usage_metadata: dict[str, int] | None = None
|
|
|
|
|
|
class _LLM:
|
|
def __init__(self) -> None:
|
|
self.calls = 0
|
|
|
|
def invoke(self, prompt: Any, config: Any = None) -> _Response:
|
|
del prompt, config
|
|
self.calls += 1
|
|
return _Response()
|
|
|
|
|
|
def _updater(root: Path, llm: _LLM) -> MemoryUpdater:
|
|
# Config validation stats storage_path; the host builds DeerMem at startup.
|
|
config = DeerMemConfig(storage_path=str(root), retrieval_adapter="")
|
|
return MemoryUpdater(config, FileMemoryStorage(config), llm)
|
|
|
|
|
|
async def test_async_update_reads_and_writes_the_watermark_sidecar_off_the_loop(tmp_path: Path) -> None:
|
|
llm = _LLM()
|
|
messages = [HumanMessage(content="I prefer green tea.", id="h1"), AIMessage(content="Noted.", id="a1")]
|
|
|
|
first = await asyncio.to_thread(_updater, tmp_path, llm)
|
|
assert await first.aupdate_memory(messages, thread_id="t1", agent_name="a", user_id="u")
|
|
# A fresh updater (a restarted process) reads the sidecar and skips the batch.
|
|
restarted = await asyncio.to_thread(_updater, tmp_path, llm)
|
|
assert await restarted.aupdate_memory(messages, thread_id="t1", agent_name="a", user_id="u")
|
|
|
|
assert llm.calls == 1
|
|
sidecar = tmp_path / "users" / "u" / "agents" / "a" / ".metadata" / "watermarks.json"
|
|
assert await asyncio.to_thread(sidecar.is_file)
|