diff --git a/README.md b/README.md index 06bdfc5c12..62c7ed8550 100644 --- a/README.md +++ b/README.md @@ -3289,6 +3289,11 @@ become available in the next batch, when rejected additions can be retried. ### Long-Term Memory +DeerMem trims oversized user/history summaries against the configured token +counter, including the omission marker and retained facts. Mixed-language text +does not rely on an average character/token ratio; the existing guaranteed-fact +allowance remains unchanged. + Settings > Appearance > Language also supplies the preferred language for replies and new automatic DeerMem facts and summary updates on subsequent chat turns and replays. Explicit user language requests take precedence. Existing memories are diff --git a/backend/packages/harness/deerflow/agents/memory/AGENTS.md b/backend/packages/harness/deerflow/agents/memory/AGENTS.md index a23c3384d1..462fcb4a7e 100644 --- a/backend/packages/harness/deerflow/agents/memory/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/memory/AGENTS.md @@ -411,6 +411,10 @@ Checkpoint reuse requires the block to exist before the run. Gateway input handling removes forged dynamic-context markers. `prompt.py::_count_tokens` controls the injection budget. +Oversized summaries use bounded prefix search, counting each assembled candidate +with its omission marker, separators and protected Facts block. Only counted +fits replace the facts-only fallback; BPE prefix counts need not be monotone, +so the search guarantees a fitting candidate rather than the longest prefix. Default `tiktoken` mode loads and caches its encoding lazily. A failed load uses character estimation for a 600-second cooldown. Concurrent callers use character estimation while one load is active. diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompt.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompt.py index e6e84f200e..31c0248896 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompt.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompt.py @@ -734,27 +734,27 @@ def format_memory_for_injection( # equation collapses to the original prefix-truncation against # ``max_tokens``, so backward compatibility is preserved. facts_block = (facts_header + "\n".join(all_fact_lines)) if all_fact_lines else "" - facts_block_tokens = _count_tokens(facts_block, use_tiktoken=use_tiktoken) - separator_tokens = _count_tokens("\n\n", use_tiktoken=use_tiktoken) - budget_for_non_facts = max( - 0, - effective_limit - facts_block_tokens - (separator_tokens if facts_block else 0), - ) - # Build the preceding (non-facts) portion from *sections* excluding # the trailing Facts block. preceding_sections = sections[:-1] if all_fact_lines else sections preceding = "\n\n".join(preceding_sections) + result = facts_block if preceding: - preceding_tokens = _count_tokens(preceding, use_tiktoken=use_tiktoken) - if preceding_tokens > budget_for_non_facts: - char_per_token = len(preceding) / max(preceding_tokens, 1) - target_chars = int(budget_for_non_facts * char_per_token * 0.95) - preceding = preceding[:target_chars].rstrip() + "\n..." - result = (preceding + "\n\n" + facts_block) if facts_block else preceding - else: - result = facts_block + # Average density does not bound a prefix's cost, and BPE counts + # need not be monotone. Count each assembled candidate and retain + # only fitting prefixes; this need not find the longest one. + low, high = 0, len(preceding) + while low <= high: + middle = (low + high) // 2 + candidate = preceding[:middle].rstrip() + "\n..." + if facts_block: + candidate += "\n\n" + facts_block + if _count_tokens(candidate, use_tiktoken=use_tiktoken) <= effective_limit: + result = candidate + low = middle + 1 + else: + high = middle - 1 return result diff --git a/backend/tests/test_memory_prompt_injection.py b/backend/tests/test_memory_prompt_injection.py index a0611c180a..b48a728b86 100644 --- a/backend/tests/test_memory_prompt_injection.py +++ b/backend/tests/test_memory_prompt_injection.py @@ -4,6 +4,7 @@ import math import pytest +from deerflow.agents.memory.backends.deermem.deermem.core import prompt from deerflow.agents.memory.backends.deermem.deermem.core.prompt import ( FACT_EXTRACTION_PROMPT, _coerce_confidence, @@ -12,6 +13,46 @@ from deerflow.agents.memory.backends.deermem.deermem.core.prompt import ( ) +@pytest.mark.parametrize("use_tiktoken", [False, True]) +@pytest.mark.parametrize("with_guaranteed_fact", [False, True]) +@pytest.mark.parametrize("max_tokens", [100, 2000]) +def test_summary_truncation_counts_the_selected_prefix(monkeypatch, use_tiktoken, with_guaranteed_fact, max_tokens): + if use_tiktoken: + tiktoken = pytest.importorskip("tiktoken") + # Exercise real UTF-8/BPE counting without downloading a vocabulary. + encoding = tiktoken.Encoding(name="bytes", pat_str=r"[\s\S]", mergeable_ranks={bytes([i]): i for i in range(256)}, special_tokens={}) + monkeypatch.setattr(prompt, "_get_tiktoken_encoding", lambda _name: encoding) + + fact = {"content": "Always keep a rollback backup", "category": "correction", "confidence": 1.0} + memory = { + "user": {"workContext": {"summary": "数据库迁移采用事务并保留回滚记录。" * 500}}, + "history": {"recentMonths": {"summary": "Release notes retain migration records. " * 1000}}, + "facts": [fact] if with_guaranteed_fact else [], + } + result = format_memory_for_injection(memory, max_tokens=max_tokens, use_tiktoken=use_tiktoken, guaranteed_categories=["correction"], guaranteed_token_budget=100) + allowance = prompt._count_tokens(prompt._format_fact_line(fact), use_tiktoken=use_tiktoken) if with_guaranteed_fact else 0 + + assert prompt._count_tokens(result, use_tiktoken=use_tiktoken) <= max_tokens + allowance + assert result.startswith("User Context:") + assert "\ufffd" not in result + if with_guaranteed_fact: + assert fact["content"] in result + + +def test_summary_truncation_handles_nonmonotone_bpe_counts(monkeypatch): + tiktoken = pytest.importorskip("tiktoken") + ranks = {bytes([i]): i for i in range(256)} | {b"bc": 256, b"abc": 257} + encoding = tiktoken.Encoding(name="merging", pat_str=r"[\s\S]+", mergeable_ranks=ranks, special_tokens={}) + assert len(encoding.encode("ab")) == 2 + assert len(encoding.encode("abc")) == 1 + monkeypatch.setattr(prompt, "_get_tiktoken_encoding", lambda _name: encoding) + + result = format_memory_for_injection({"user": {"workContext": {"summary": "abc" * 1000}}}, max_tokens=100) + + assert result.startswith("User Context:") + assert len(encoding.encode(result)) <= 100 + + def test_format_memory_includes_facts_section() -> None: memory_data = { "user": {},