diff --git a/backend/tests/test_memory_documentation.py b/backend/tests/test_memory_documentation.py new file mode 100644 index 0000000000..15d97479f2 --- /dev/null +++ b/backend/tests/test_memory_documentation.py @@ -0,0 +1,102 @@ +"""Keep the user-facing memory guides aligned with DeerMem's real storage layout. + +The guides once described the pre-abstraction layout (a shared +``{base_dir}/memory.json`` plus one ``agents/{agent_name}/memory.json`` per +agent). DeerMem scopes the memory manifest per user and keeps agent state in a +``facts`` directory beside it, and the host loader moved every DeerMem-private +knob under ``memory.backend_config``. Both guides are prose, so nothing in CI +noticed; this test derives the documented shape from the same functions the +runtime uses. +""" + +import re +from pathlib import Path + +import pytest + +from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig +from deerflow.agents.memory.backends.deermem.deermem.core.paths import agent_facts_directory, memory_file_path +from deerflow.config.memory_config import MemoryConfig + +REPO_ROOT = Path(__file__).resolve().parents[2] +MEMORY_GUIDES = ( + Path("frontend/src/content/en/harness/memory.mdx"), + Path("frontend/src/content/zh/harness/memory.mdx"), +) + +# A root that never collides with the placeholders the guides spell out. +PROBE_ROOT = "/deer-flow-test-root" +PROBE_USER_ID = "probe-user" +PROBE_AGENT_NAME = "Probe-Agent" + +# The pre-abstraction per-agent manifest file. DeerMem writes agent state under +# ``agents/{agent_name}/facts/`` instead, so a guide naming this path is stale. +LEGACY_PER_AGENT_FILE = "agents/{agent_name}/memory.json" + + +def _guide_text(path: Path) -> str: + return (REPO_ROOT / path).read_text(encoding="utf-8") + + +def _configuration_block(path: Path) -> str: + """Return the first yaml example under the guide's Configuration heading.""" + lines = _guide_text(path).splitlines() + starts = [index for index, line in enumerate(lines) if re.fullmatch(r"## (Configuration|配置)\s*", line)] + assert len(starts) == 1, f"expected exactly one Configuration heading in {path}, found {len(starts)}" + fences = [index for index in range(starts[0], len(lines)) if lines[index].strip().startswith("```yaml")] + assert fences, f"no yaml example after the Configuration heading in {path}" + end = next(index for index in range(fences[0] + 1, len(lines)) if lines[index].strip() == "```") + return "\n".join(lines[fences[0] + 1 : end]) + + +def _top_level_memory_keys(block: str) -> set[str]: + """Keys written directly under ``memory:`` (two-space indent) in a yaml example.""" + keys = set() + inside = False + for line in block.splitlines(): + if line.startswith("memory:"): + inside = True + continue + if not inside: + continue + if line and not line.startswith((" ", " ", "\t", "#")): + break + match = re.fullmatch(r" ([A-Za-z_][A-Za-z0-9_]*):.*", line) + if match: + keys.add(match.group(1)) + return keys + + +@pytest.fixture(scope="module") +def probe_layout() -> tuple[str, str]: + """The per-user manifest and per-agent facts paths, as the runtime resolves them.""" + memory_path = memory_file_path(DeerMemConfig(storage_path=PROBE_ROOT), user_id=PROBE_USER_ID) + facts_path = agent_facts_directory(memory_path, PROBE_AGENT_NAME) + return str(memory_path.relative_to(PROBE_ROOT)), str(facts_path.relative_to(PROBE_ROOT)) + + +@pytest.mark.parametrize("path", MEMORY_GUIDES, ids=str) +def test_guide_documents_the_scoped_manifest_and_facts_paths(path: Path, probe_layout: tuple[str, str]) -> None: + content = _guide_text(path) + memory_rel, facts_rel = probe_layout + + # Guides spell the resolved layout with placeholders, so re-spell the real paths + # the same way before comparing. ``agent_facts_directory`` already folded the + # mixed-case probe name, hence the lowercase replacement. + def spelled(rel: str) -> str: + return rel.replace(PROBE_USER_ID, "{user_id}").replace(PROBE_AGENT_NAME.lower(), "{agent_name}") + + scoped_manifest = spelled(memory_rel) + agent_facts = spelled(facts_rel) + assert scoped_manifest in content, f"{path} does not document the per-user manifest at {scoped_manifest}" + assert agent_facts in content, f"{path} does not document the per-agent facts directory at {agent_facts}" + assert LEGACY_PER_AGENT_FILE not in content, f"{path} still documents the pre-abstraction per-agent file {LEGACY_PER_AGENT_FILE}" + + +@pytest.mark.parametrize("path", MEMORY_GUIDES, ids=str) +def test_guide_keeps_backend_private_knobs_out_of_the_memory_top_level(path: Path) -> None: + keys = _top_level_memory_keys(_configuration_block(path)) + assert keys, f"no keys parsed from the Configuration example in {path}" + shared = set(MemoryConfig.model_fields) + drift = sorted(keys - shared) + assert not drift, f"{path} documents DeerMem-private keys at the memory top level: {drift} (they belong under backend_config)" diff --git a/frontend/src/content/en/harness/memory.mdx b/frontend/src/content/en/harness/memory.mdx index 0d3c39ca7b..7f679e4e94 100644 --- a/frontend/src/content/en/harness/memory.mdx +++ b/frontend/src/content/en/harness/memory.mdx @@ -34,7 +34,7 @@ Memory has two operation modes: 1. **Injection in both modes**: at the start of each conversation, the agent's current memory is injected into the system prompt at a controlled token budget (`max_injection_tokens`). 2. **Middleware mode (default)**: `MemoryMiddleware` runs after each Lead Agent turn. It filters the conversation, queues a background update, and extracts new facts automatically. Updates are debounced by `debounce_seconds` to batch rapid changes. 3. **Tool mode (experimental)**: DeerFlow registers `memory_search`, `memory_get`, `memory_add`, `memory_update`, and `memory_delete` as agent tools and skips `MemoryMiddleware`. The model decides when to search, read, or write memory, so effectiveness depends on model tool-use behavior. These explicit CRUD tools are not the passive staleness-review path; operators who enable tool mode are opting into model-directed updates/deletes instead of the middleware staleness guardrails. -4. **Per-agent memory**: when a custom agent is active, its memory is stored separately from the global memory. This keeps different agents' knowledge isolated. +4. **Per-agent facts**: the default DeerMem backend stores each custom agent's learned facts in its own facts directory. Agents share one memory manifest within the same user. In tool mode, the five `memory_*` tool names are reserved. If an MCP server or custom tool already uses `memory_search`, `memory_get`, `memory_add`, `memory_update`, or `memory_delete`, the existing tool keeps that name and the colliding memory tool is skipped with a warning. @@ -295,12 +295,14 @@ guide. ## Global vs per-agent memory -DeerFlow supports two levels of memory: +DeerFlow scopes memory files per user and facts per agent: -- **Global memory**: stored at `{base_dir}/memory.json`. Used when no specific agent is active or when the agent has no per-agent memory file. -- **Per-agent memory**: stored at `{base_dir}/agents/{agent_name}/memory.json`. Used when a custom agent is active, keeping that agent's learned knowledge separate. +- **Per-user memory**: `{base_dir}/users/{user_id}/memory.json` holds that user's cross-task summary. `agent_name` does not select a different memory file. +- **Per-agent facts**: `{base_dir}/users/{user_id}/agents/{agent_name}/facts/` sits next to the same user's memory file, keeping one agent's learned facts separate from another's. The `{agent_name}` segment is lowercased, and names are restricted to `A-Z`, `a-z`, `0-9` and `-`. -The `MemoryMiddleware` automatically selects the correct memory file based on the active `agent_name` in the request configuration. +A caller without a `user_id` falls back to the unscoped legacy root `{base_dir}/memory.json` — see the `Legacy: no user_id` branch in `deermem/core/paths.py`. + +The `MemoryMiddleware` selects which agent's facts directory to read from the active `agent_name` in the request configuration. Agent names used for memory storage are validated against `AGENT_NAME_PATTERN` to ensure filesystem safety. @@ -308,11 +310,11 @@ Agent names used for memory storage are validated against `AGENT_NAME_PATTERN` t By default, memory files are stored under the backend base directory: -- Base directory: `backend/.deer-flow/` -- Global memory: `backend/.deer-flow/memory.json` -- Per-agent memory: `backend/.deer-flow/agents/{agent_name}/memory.json` +- Base directory: `backend/.deer-flow/` (the `runtime_home()` state directory, overridable with `DEER_FLOW_HOME`) +- Per-user memory: `backend/.deer-flow/users/{user_id}/memory.json` +- Per-agent facts: `backend/.deer-flow/users/{user_id}/agents/{agent_name}/facts/` -You can change the storage path with the `storage_path` field. Relative paths are resolved against the base directory. Use an absolute path to store memory in a custom location. +You can change that root with `memory.backend_config.storage_path`, and the value must be a **directory**: an absolute path is used as the root, a relative path resolves against the state directory, and an empty value keeps the default. A file-style value (the pre-abstraction `storage_path: memory.json`) is dropped with a warning by `deerflow/config/memory_config.py`, and a value pointing at an existing file makes `DeerMemConfig` raise at construction. ## Custom storage backend diff --git a/frontend/src/content/zh/harness/memory.mdx b/frontend/src/content/zh/harness/memory.mdx index c3c77a4071..676e069a5e 100644 --- a/frontend/src/content/zh/harness/memory.mdx +++ b/frontend/src/content/zh/harness/memory.mdx @@ -33,7 +33,7 @@ import { Callout } from "nextra/components"; 1. **两种模式都会注入**：每次对话开始时，Agent 当前记忆以受控的 token 预算（`max_injection_tokens`）注入到系统提示中。 2. **中间件模式（默认）**：`MemoryMiddleware` 在每次 Lead Agent 轮次结束后运行，过滤对话、排队后台更新并自动提取新事实。更新通过 `debounce_seconds` 防抖以批量处理快速变化。 3. **工具模式（实验性）**：DeerFlow 注册 `memory_search`、`memory_get`、`memory_add`、`memory_update`、`memory_delete` 五个工具，并跳过 `MemoryMiddleware`。模型自行决定何时搜索、读取或写入记忆，因此效果取决于模型的工具调用行为。这些显式 CRUD 工具不是被动的陈旧性复核路径；启用工具模式意味着运营方选择由模型主动更新/删除记忆，而不是依赖中间件的陈旧性保护栏。 -4. **按 Agent 记忆**：当自定义 Agent 激活时，其记忆独立于全局记忆存储。这保持不同 Agent 的知识隔离。 +4. **按 Agent 划分事实**：默认的 DeerMem 后端将各自定义 Agent 学到的事实保存在各自的事实目录中。同一用户的各 Agent 共用一个记忆文件。 在工具模式中，五个 `memory_*` 工具名是保留名。如果 MCP 服务或自定义工具已经使用 `memory_search`、`memory_get`、`memory_add`、`memory_update` 或 `memory_delete`，已有工具会保留该名称，冲突的记忆工具会被跳过并记录告警。 @@ -212,48 +212,61 @@ memory: ```yaml memory: enabled: true + injection_enabled: true + + # 后端选择：deermem（默认）| noop | openviking | 自定义 MemoryManager 子类的点号路径 + manager_class: deermem # 运行模式： # middleware - 默认；每轮结束后被动后台提取 # tool - 实验性；模型直接调用 memory_* 工具 mode: middleware - # 全局记忆文件的存储路径。 - # 默认：{base_dir}/memory.json（解析为 backend/.deer-flow/memory.json） - # 绝对路径按原样使用，相对路径相对于 base_dir 解析。 - storage_path: memory.json + # DeerMem 私有参数必须放在 backend_config 下（而不是顶层），因为它们只属于 DeerMem + # ——换一个后端时，由该后端自行解释自己的 backend_config。写在顶层的旧字段会在加载时 + # 被自动迁移进 backend_config，并合并打一条 WARNING；而以 .json 结尾的旧式文件路径 + # storage_path 会被直接丢弃（详见 deerflow/config/memory_config.py）。 + backend_config: + # 数据根目录，必须是目录。留空 = deer-flow 的 base_dir；按用户的记忆文件在 + # {根目录}/users/{user_id}/memory.json。绝对路径即该根目录。 + storage_path: "" - # 存储类（留空 = FileMemoryStorage，即默认的基于文件的 JSON 存储，不走 importlib； - # 自定义实现填 `module.Class` 点号路径） - storage_class: "" + # 存储类（留空 = FileMemoryStorage，即默认的可移植文件存储，不走 importlib； + # 自定义实现填 `module.Class` 点号路径） + storage_class: "" - # 处理排队记忆更新前等待的秒数（防抖） - debounce_seconds: 30 + # 处理排队记忆更新前等待的秒数（防抖） + debounce_seconds: 30 - # 记忆更新提取使用的模型（null = 使用默认模型） - model_name: null + # 记忆提取使用的模型。整段省略 = 宿主工厂注入应用默认模型（等同于旧写法 + # model_name: null）；显式填写可换用更便宜或不同的模型。 + model: + # provider: openai + # model: gpt-4o-mini + # api_key: $OPENAI_API_KEY + # base_url: # 可选，用于 OpenAI 兼容网关 + # temperature: # 可选 - # 要存储的最大事实数 - max_facts: 100 + # 要存储的最大事实数 + max_facts: 100 - # 存储事实所需的最低置信度分数（0.0–1.0） - fact_confidence_threshold: 0.7 + # 存储事实所需的最低置信度分数（0.0–1.0） + fact_confidence_threshold: 0.7 - # 是否将记忆注入到系统提示中 - injection_enabled: true - - # 注入到系统提示的最大 token 数 - max_injection_tokens: 2000 + # 注入到系统提示的最大 token 数 + max_injection_tokens: 2000 ``` ## 全局记忆与按 Agent 记忆 -DeerFlow 支持两级记忆： +DeerFlow 按用户划分记忆文件，按 Agent 划分事实目录： -- **全局记忆**：存储在 `{base_dir}/memory.json`。在没有特定 Agent 激活或 Agent 没有按 Agent 记忆文件时使用。 -- **按 Agent 记忆**：存储在 `{base_dir}/agents/{agent_name}/memory.json`。当自定义 Agent 激活时使用，使该 Agent 学到的知识保持独立。 +- **按用户记忆**：`{base_dir}/users/{user_id}/memory.json`，存放该用户跨任务的摘要。`agent_name` 不会切换到另一个记忆文件。 +- **按 Agent 事实**：`{base_dir}/users/{user_id}/agents/{agent_name}/facts/`，与同一用户的记忆文件并列存放，使各 Agent 学到的事实相互独立。`{agent_name}` 这一段会被转为小写，且仅允许 `A-Z`、`a-z`、`0-9` 和 `-`。 -`MemoryMiddleware` 根据请求配置中的活跃 `agent_name` 自动选择正确的记忆文件。 +不带 `user_id` 的调用方会退回未分用户的遗留根 `{base_dir}/memory.json`（见 `deermem/core/paths.py` 的 `Legacy: no user_id` 分支）。 + +`MemoryMiddleware` 根据请求配置中的活跃 `agent_name` 选择读取哪个 Agent 的 facts 目录。 用于记忆存储的 Agent 名称通过 `AGENT_NAME_PATTERN` 验证，以确保文件系统安全。 @@ -261,11 +274,11 @@ DeerFlow 支持两级记忆： 默认情况下，记忆文件存储在后端基础目录下： -- 基础目录：`backend/.deer-flow/` -- 全局记忆：`backend/.deer-flow/memory.json` -- 按 Agent 记忆：`backend/.deer-flow/agents/{agent_name}/memory.json` +- 基础目录：`backend/.deer-flow/`（即 `runtime_home()`，可用 `DEER_FLOW_HOME` 覆盖） +- 按用户记忆：`backend/.deer-flow/users/{user_id}/memory.json` +- 按 Agent 事实：`backend/.deer-flow/users/{user_id}/agents/{agent_name}/facts/` -你可以用 `storage_path` 字段更改存储路径。相对路径相对于基础目录解析；使用绝对路径可以将记忆存储在自定义位置。 +你可以用 `memory.backend_config.storage_path` 更改这个根目录，且它的值必须是**目录**：绝对路径即该根目录，相对路径相对于宿主状态目录解析，留空则保持默认。以文件形态给出的旧写法（`storage_path: memory.json`）会被 `deerflow/config/memory_config.py` 打一条 WARNING 后丢弃；指向已存在文件的值会让 `DeerMemConfig` 在构造时直接抛错。 ## 自定义存储后端