From 54753636b1b6770762f8113a27d57d524851e38f Mon Sep 17 00:00:00 2001 From: shunxinyi <3515462408@qq.com> Date: Tue, 10 Nov 2026 09:50:46 +0800 Subject: [PATCH] fix(memory): recall CJK sub-phrases when jieba is not installed (#8022) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(memory): recall CJK sub-phrases when jieba is not installed Without the memory-zh extra, _tokenize fell back to whitespace splitting, so the FTS5 index kept a whole CJK run as a single unicode61 token and every CJK sub-phrase query returned no hits without any warning. The relevance ranker already emitted CJK character bigrams for the same text, so the two retrieval strategies disagreed on the same data. Reuse relevance.cjk_token for CJK runs when jieba is unavailable and route _preprocess_content through _tokenize, so the index and the query side agree. Installs with jieba keep the existing jieba path. Existing index rows are rebuilt on the first search or warm-up after the upgrade; no schema or version change is involved. * style(tests): format the CJK fallback regression test `ruff format --check .` in the backend lint job reformats the new assertion in tests/test_memory_retrieval_adapter.py: the implicitly concatenated message is joined into one line. Apply the formatter output, no behavior change. * fix(deermem): recall single CJK characters through an FTS5 prefix phrase The fallback index stores CJK runs as character bigrams, so a one-character query had no equal token and came back empty; `"重"*` is an FTS5 prefix phrase that matches the bigram `重试` and still matches a standalone `重` token, while multi-character tokens keep the plain quoted phrase. Also force the no-jieba branch with a monkeypatch so the regression test runs on installs that have the optional `memory-zh` extra, and cover the single-character and absent- character cases. * docs(deermem): scope the single-CJK prefix note to leading matches The prefix phrase only reaches indexed tokens that start with the queried character, so a character occurring only later inside a bigram (务 in 任务) is not matched. Say that in the docstring and mark the residual gap as accepted instead of widening the prefix to the run's trailing character. --- .../deermem/deermem/core/relevance.py | 36 +++++++---- .../deermem/deermem/core/retrieval.py | 64 +++++++++++++++---- .../tests/test_memory_retrieval_adapter.py | 41 +++++++++++- 3 files changed, 115 insertions(+), 26 deletions(-) diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py index 20fb40f502..a25faf3432 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py @@ -49,6 +49,27 @@ def warm_tokenizer() -> None: jieba.initialize() +def cjk_token(part: str) -> list[str]: + """Character-level tokens for one unsegmented CJK run. + + Single characters stay single; longer runs become character bigrams. + Shared with the FTS5 retrieval index so both strategies agree on Chinese. + """ + if len(part) == 1: + return [part] + return [part[index : index + 2] for index in range(len(part) - 1)] + + +def iter_fallback_tokens(text: str) -> Iterator[str]: + """Split text into words and CJK bigrams without jieba.""" + for match in _WORD_RE.finditer(text): + part = match.group() + if "\u3400" <= part[0] <= "\u9fff": + yield from cjk_token(part) + else: + yield part + + def tokenize(text: str) -> list[str]: """Tokenize at most 4096 characters into at most 128 relevance tokens. @@ -65,20 +86,7 @@ def tokenize(text: str) -> list[str]: # fallback does so they neither count as matches nor eat the budget. tokens = (token for token in jieba.cut(lowered) if any(char.isalnum() for char in token)) return list(islice(tokens, _SIMILARITY_TOKEN_BUDGET)) - - def fallback_tokens() -> Iterator[str]: - for match in _WORD_RE.finditer(lowered): - part = match.group() - if "\u3400" <= part[0] <= "\u9fff": - if len(part) == 1: - yield part - else: - for index in range(len(part) - 1): - yield part[index : index + 2] - else: - yield part - - return list(islice(fallback_tokens(), _SIMILARITY_TOKEN_BUDGET)) + return list(islice(iter_fallback_tokens(lowered), _SIMILARITY_TOKEN_BUDGET)) def build_idf(corpus: list[list[str]]) -> dict[str, float]: diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py index 92653bd8a8..4672bfd735 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py @@ -1,7 +1,7 @@ """FTS5-based retrieval engine for DeerMem. Provides BM25 full-text search over stored facts with: -- jieba Chinese tokenization (optional, falls back to whitespace) +- jieba Chinese tokenization (optional, falls back to CJK character bigrams) - FTS5 MATCH syntax support (AND/OR/NOT/phrase/prefix) with fallback - Time-decay + confidence-weighted ranking - Category filtering @@ -28,6 +28,7 @@ from pathlib import Path from typing import Any from .paths import retrieval_index_directory +from .relevance import cjk_token logger = logging.getLogger(__name__) @@ -54,13 +55,33 @@ except ImportError: _jieba_available = False +# Capturing group so ``re.split`` keeps the CJK runs it separates on. +_CJK_RUN_RE = re.compile(r"([\u3400-\u4dbf\u4e00-\u9fff]+)") + + def _tokenize(text: str) -> list[str]: - """Tokenize text: jieba for Chinese, whitespace split for English.""" + """Tokenize text: jieba for Chinese, CJK bigrams otherwise. + + Without jieba, whitespace-delimited words are kept verbatim except for + their CJK runs, which are split into character bigrams with the same + ``relevance.cjk_token`` strategy the relevance ranker uses. Indexing and + querying share this function, so unsegmented Chinese text stays matchable + in the default (dependency-free) install. + """ if not text or not text.strip(): return [] if _jieba_available: return [t for t in jieba.cut(text) if t.strip()] - return [t for t in text.split() if t.strip()] + tokens: list[str] = [] + for word in text.split(): + for part in _CJK_RUN_RE.split(word): + if not part: + continue + if _CJK_RUN_RE.fullmatch(part): + tokens.extend(cjk_token(part)) + else: + tokens.append(part) + return tokens # ── FTS5 query preprocessing ────────────────────────────────────────── @@ -78,15 +99,34 @@ def _is_advanced_query(query: str) -> bool: return bool(_FTS5_ADVANCED_RE.search(query)) +def _quote_fallback_token(token: str) -> str: + """Quote one token for the OR query, prefixing single CJK characters. + + The fallback index stores CJK runs as character bigrams, so a one-character + query (``重``) has no equal token to match. ``"重"*`` is an FTS5 prefix + phrase: it matches every indexed token that *starts with* that character, + which covers the bigram ``重试`` and still matches a standalone ``重`` token + when the content held an isolated single character. It does not reach a + character that only occurs later inside a bigram, so ``务`` does not match + the ``任务`` token; that residual gap is accepted instead of widening the + prefix. Multi-character tokens keep the plain quoted phrase so their + semantics do not change. Double quotes inside a token are the FTS5 escape + sequence for a literal quote. + """ + escaped = token.replace(chr(34), chr(34) * 2) + if len(token) == 1 and _CJK_RUN_RE.fullmatch(token): + return f'"{escaped}"*' + return f'"{escaped}"' + + def _build_fallback_query(query: str) -> str: """Convert natural-language query to FTS5 OR query (fallback strategy).""" tokens = [token for token in _tokenize(query) if any(char.isalnum() for char in token)] if not tokens: return "" # Quote each token so punctuation in natural-language input cannot become - # an FTS5 operator or syntax error. Double quotes inside a token are the - # FTS5 escape sequence for a literal quote. - return " OR ".join(f'"{token.replace(chr(34), chr(34) * 2)}"' for token in tokens) + # an FTS5 operator or syntax error. + return " OR ".join(_quote_fallback_token(token) for token in tokens) # ── Core retrieval engine ───────────────────────────────────────────── @@ -151,13 +191,15 @@ class FTS5Retrieval: # ── Index operations ─────────────────────────────────────────────── def _preprocess_content(self, content: str) -> str: - """Preprocess content for indexing: jieba tokenize for Chinese.""" + """Preprocess content for indexing: segment CJK so queries can match. + + Must stay aligned with the query side (``_build_fallback_query``); both + go through ``_tokenize`` so index and query agree on Chinese without + jieba. + """ if not content: return "" - if _jieba_available: - tokens = _tokenize(content) - return " ".join(tokens) - return content + return " ".join(_tokenize(content)) def _row_from_document(self, document: dict[str, Any]) -> tuple[Any, ...]: now = datetime.now(UTC).isoformat().replace("+00:00", "Z") diff --git a/backend/tests/test_memory_retrieval_adapter.py b/backend/tests/test_memory_retrieval_adapter.py index 1b33581dbe..92c334a9c4 100644 --- a/backend/tests/test_memory_retrieval_adapter.py +++ b/backend/tests/test_memory_retrieval_adapter.py @@ -35,8 +35,14 @@ def test_natural_language_hyphens_are_not_treated_as_fts5_syntax() -> None: assert not _is_advanced_query("real-time co-pilot node -js +python") -@pytest.mark.skipif(not _jieba_available, reason="install the optional memory-zh extra") +@pytest.mark.skipif(not _jieba_available, reason="requires the optional memory-zh extra to exercise the jieba branch") def test_chinese_subphrase_search_uses_jieba_tokenization(tmp_path: Path) -> None: + """With jieba installed, the same sub-phrase recall must still hold. + + The complementary no-jieba case is forced with a monkeypatch below, so the + two tests cover the two tokenizer branches instead of skipping each other + on the same install. + """ adapter = FTS5RetrievalAdapter(tmp_path / "facts.sqlite3") scope = {"userId": "alice", "agentName": "agent-a"} try: @@ -47,6 +53,39 @@ def test_chinese_subphrase_search_uses_jieba_tokenization(tmp_path: Path) -> Non adapter.close() +def test_chinese_fact_is_recallable_without_memory_zh_extra(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A CJK fact indexed in the default install must stay retrievable. + + ``uv sync --group dev --locked`` and the contributor CI path install no + ``memory-zh`` extra, so ``_tokenize`` has to segment CJK on its own. It + emits character bigrams (the same ``relevance.cjk_token`` strategy the + relevance ranker uses) on both the index and the query side; a whitespace + fallback would keep the whole CJK run as one FTS5 token and silently return + zero hits for every sub-phrase. ``_jieba_available`` is read at call time, + so forcing it here runs this branch even where jieba is installed. + """ + monkeypatch.setattr("deerflow.agents.memory.backends.deermem.deermem.core.retrieval._jieba_available", False) + adapter = FTS5RetrievalAdapter(tmp_path / "facts.sqlite3") + scope = {"userId": "alice", "agentName": "agent-a"} + try: + adapter.upsert(_fact("zh-default", "调度器会重试失败的任务"), scope=scope, path="") + results = adapter.search("重试", scopes=[scope], top_k=5, mode="fts5", filters=None) + assert results, "expected the stored Chinese fact to be recalled by the CJK sub-phrase '重试', got 0 hits: the no-jieba fallback must emit CJK bigrams instead of keeping '调度器会重试失败的任务' as one token" + assert [item["fact"]["id"] for item in results] == ["zh-default"] + + # A single CJK character has no equal bigram token to match; the query + # side emits the FTS5 prefix phrase "重"* so the bigram index still hits. + single = adapter.search("重", scopes=[scope], top_k=5, mode="fts5", filters=None) + assert [item["fact"]["id"] for item in single] == ["zh-default"] + + # The prefix stays anchored to that character: an absent one must not + # start matching everything in the index. + absent = adapter.search("蟹", scopes=[scope], top_k=5, mode="fts5", filters=None) + assert absent == [] + finally: + adapter.close() + + def test_score_time_decay_and_advanced_phrase_query(tmp_path: Path) -> None: engine = FTS5Retrieval(tmp_path / "facts.sqlite3") try: