From c54883cce0750f32fa4c47c29e02f15bd89a194e Mon Sep 17 00:00:00 2001 From: wgwfn3 <43378854+wgwfn3@users.noreply.github.com> Date: Wed, 7 Oct 2026 06:58:59 +0800 Subject: [PATCH] fix(memory): drop punctuation tokens from jieba relevance ranking (#6388) * fix(memory): drop punctuation tokens from jieba relevance ranking jieba.cut emits punctuation as standalone tokens and tokenize() only filtered whitespace, so commas and full stops counted as query matches, inflated near-duplicate similarity, and consumed the 128-token budget. Keep only tokens with an alphanumeric character, matching the regex fallback and the FTS5 adapter's query filter. Document the rule in agents/memory/AGENTS.md. * docs(changelog): add the DeerMem jieba punctuation fix * docs(changelog): add the PR reference for the jieba punctuation fix (#6388) * docs(memory): document jieba relevance token filtering --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 9 +++++++ CHANGELOG_zh.md | 6 +++++ .../harness/deerflow/agents/memory/AGENTS.md | 2 ++ .../deermem/deermem/core/relevance.py | 7 ++++- .../tests/test_memory_relevance_retrieval.py | 26 +++++++++++++++++++ 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4409085ee9..badf98fe53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -623,6 +623,14 @@ This release closes that milestone with **439 merged pull requests**. ### Fixed +- **memory:** DeerMem relevance ranking no longer counts punctuation as query + terms when jieba is installed (`memory-zh` extra). `tokenize()` dropped only + whitespace from `jieba.cut`, which emits `，`, `。`, `,` and `!` as standalone + tokens, so a query and an unrelated fact that both contained a comma scored + above zero, near-duplicate similarity was inflated, and punctuation used up the + 128-token budget before later query terms. Tokens without a letter or digit are + now dropped, as the no-jieba fallback and the FTS5 query filter already did. + ([#6388]) - **middleware:** Tool-output budgeting no longer hides a failed shell exit from subagent evidence. A bash result between `externalize_min_chars` (12,000) and the sandbox limit (20,000) was replaced by a preview ending in its `Access:` @@ -8841,3 +8849,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#6350]: https://github.com/bytedance/deer-flow/pull/6350 [#6351]: https://github.com/bytedance/deer-flow/pull/6351 [#6354]: https://github.com/bytedance/deer-flow/pull/6354 +[#6388]: https://github.com/bytedance/deer-flow/pull/6388 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index f7b88dad35..4ab3e29c5f 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -558,6 +558,11 @@ ### 修复 +- **记忆：** 安装 jieba（`memory-zh` 扩展）时，DeerMem 相关度排序不再把标点计为查询词。 + `tokenize()` 只过滤了 `jieba.cut` 输出中的空白，而 jieba 会把 `，`、`。`、`,`、`!` + 切成独立 token，因此同样含逗号的查询与无关事实也会得到非零分，近似去重的相似度被抬高， + 标点还会在后续查询词之前耗尽 128 个 token 的预算。现在不含字母或数字的 token 会被丢弃， + 与无 jieba 的回退分词及 FTS5 查询过滤保持一致。([#6388]) - **中间件：** 工具输出预算不再让子智能体证据丢失失败的 shell 退出码。长度介于 `externalize_min_chars`（12,000）与沙箱上限（20,000）之间的 bash 结果会被替换为 以 `Access:` 页脚结尾的预览，结尾的 `Exit Code: N` 不再位于最后， @@ -7319,3 +7324,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#6350]: https://github.com/bytedance/deer-flow/pull/6350 [#6351]: https://github.com/bytedance/deer-flow/pull/6351 [#6354]: https://github.com/bytedance/deer-flow/pull/6354 +[#6388]: https://github.com/bytedance/deer-flow/pull/6388 diff --git a/backend/packages/harness/deerflow/agents/memory/AGENTS.md b/backend/packages/harness/deerflow/agents/memory/AGENTS.md index 23af2255af..e7e8f40d16 100644 --- a/backend/packages/harness/deerflow/agents/memory/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/memory/AGENTS.md @@ -362,6 +362,8 @@ runs by default. retrieval, while adapter indexing and warm-up remain configured. - Ranking reads at most 4096 characters and 128 tokens per query/fact. The no-jieba fallback emits both Latin words and CJK bigrams, including mixed text. + With jieba, tokens without a letter or digit (punctuation) are dropped, as in + the fallback, so they neither match nor use the 128-token budget. `DeerMem.warm()` initializes optional jieba before serving requests, even with character-based token counting. Invalid/missing confidence defaults to 0. - Search stops MMR after `top_k` picks. Injection diversifies guaranteed and diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py index 5c92a7445e..740403df46 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py @@ -52,6 +52,8 @@ def warm_tokenizer() -> None: def tokenize(text: str) -> list[str]: """Tokenize at most 4096 characters into at most 128 relevance tokens. + Jieba tokens without a letter or digit are dropped and do not use the 128-token budget. + Space-free CJK text without jieba falls back to character bigrams so Chinese queries still produce deterministic token overlap. """ @@ -59,7 +61,10 @@ def tokenize(text: str) -> list[str]: return [] lowered = text[:_TEXT_CHAR_BUDGET].strip().lower() if _jieba_available: - return list(islice((token for token in jieba.cut(lowered) if token.strip()), _SIMILARITY_TOKEN_BUDGET)) + # jieba emits punctuation as standalone tokens; drop them like the + # fallback does so they neither count as matches nor eat the budget. + tokens = (token for token in jieba.cut(lowered) if any(char.isalnum() for char in token)) + return list(islice(tokens, _SIMILARITY_TOKEN_BUDGET)) def fallback_tokens() -> Iterator[str]: for match in _WORD_RE.finditer(lowered): diff --git a/backend/tests/test_memory_relevance_retrieval.py b/backend/tests/test_memory_relevance_retrieval.py index f6e556149a..2426329002 100644 --- a/backend/tests/test_memory_relevance_retrieval.py +++ b/backend/tests/test_memory_relevance_retrieval.py @@ -12,6 +12,7 @@ confidence-based behavior. Coverage: - the DynamicContextMiddleware -> ``_get_memory_context`` query wiring. """ +import re from types import SimpleNamespace import pytest @@ -72,6 +73,31 @@ class TestLexicalRelevance: assert len(tokenize("word" * 10000)) == 128 assert seen == [4096] + @staticmethod + def _punctuation_emitting_segmenter(monkeypatch): + """Stub jieba the way it really segments: punctuation is its own token.""" + from deerflow.agents.memory.backends.deermem.deermem.core import relevance + + def cut(text): + yield from re.findall(r"\w+|[^\w\s]|\s", text) + + monkeypatch.setattr(relevance, "_jieba_available", True) + monkeypatch.setattr(relevance, "jieba", SimpleNamespace(cut=cut), raising=False) + + def test_segmenter_punctuation_is_not_a_token(self, monkeypatch): + self._punctuation_emitting_segmenter(monkeypatch) + assert tokenize("Python，数据库。SQL, rust!") == ["python", "数据库", "sql", "rust"] + + def test_segmenter_punctuation_does_not_match_unrelated_facts(self, monkeypatch): + self._punctuation_emitting_segmenter(monkeypatch) + assert lexical_relevance("PostgreSQL，数据库", "周末喜欢爬山，偶尔骑车。") == 0.0 + + def test_segmenter_punctuation_does_not_consume_token_budget(self, monkeypatch): + self._punctuation_emitting_segmenter(monkeypatch) + query = ", ".join(f"term{i:02d}" for i in range(80)) + assert tokenize(query) == [f"term{i:02d}" for i in range(80)] + assert lexical_relevance(query, "term75 is the read replica") > 0.0 + def test_mixed_cjk_without_jieba(self, monkeypatch): from deerflow.agents.memory.backends.deermem.deermem.core import relevance