diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py index 20fb40f502..a25faf3432 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/relevance.py @@ -49,6 +49,27 @@ def warm_tokenizer() -> None: jieba.initialize() +def cjk_token(part: str) -> list[str]: + """Character-level tokens for one unsegmented CJK run. + + Single characters stay single; longer runs become character bigrams. + Shared with the FTS5 retrieval index so both strategies agree on Chinese. + """ + if len(part) == 1: + return [part] + return [part[index : index + 2] for index in range(len(part) - 1)] + + +def iter_fallback_tokens(text: str) -> Iterator[str]: + """Split text into words and CJK bigrams without jieba.""" + for match in _WORD_RE.finditer(text): + part = match.group() + if "\u3400" <= part[0] <= "\u9fff": + yield from cjk_token(part) + else: + yield part + + def tokenize(text: str) -> list[str]: """Tokenize at most 4096 characters into at most 128 relevance tokens. @@ -65,20 +86,7 @@ def tokenize(text: str) -> list[str]: # fallback does so they neither count as matches nor eat the budget. tokens = (token for token in jieba.cut(lowered) if any(char.isalnum() for char in token)) return list(islice(tokens, _SIMILARITY_TOKEN_BUDGET)) - - def fallback_tokens() -> Iterator[str]: - for match in _WORD_RE.finditer(lowered): - part = match.group() - if "\u3400" <= part[0] <= "\u9fff": - if len(part) == 1: - yield part - else: - for index in range(len(part) - 1): - yield part[index : index + 2] - else: - yield part - - return list(islice(fallback_tokens(), _SIMILARITY_TOKEN_BUDGET)) + return list(islice(iter_fallback_tokens(lowered), _SIMILARITY_TOKEN_BUDGET)) def build_idf(corpus: list[list[str]]) -> dict[str, float]: diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py index 92653bd8a8..4672bfd735 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/retrieval.py @@ -1,7 +1,7 @@ """FTS5-based retrieval engine for DeerMem. Provides BM25 full-text search over stored facts with: -- jieba Chinese tokenization (optional, falls back to whitespace) +- jieba Chinese tokenization (optional, falls back to CJK character bigrams) - FTS5 MATCH syntax support (AND/OR/NOT/phrase/prefix) with fallback - Time-decay + confidence-weighted ranking - Category filtering @@ -28,6 +28,7 @@ from pathlib import Path from typing import Any from .paths import retrieval_index_directory +from .relevance import cjk_token logger = logging.getLogger(__name__) @@ -54,13 +55,33 @@ except ImportError: _jieba_available = False +# Capturing group so ``re.split`` keeps the CJK runs it separates on. +_CJK_RUN_RE = re.compile(r"([\u3400-\u4dbf\u4e00-\u9fff]+)") + + def _tokenize(text: str) -> list[str]: - """Tokenize text: jieba for Chinese, whitespace split for English.""" + """Tokenize text: jieba for Chinese, CJK bigrams otherwise. + + Without jieba, whitespace-delimited words are kept verbatim except for + their CJK runs, which are split into character bigrams with the same + ``relevance.cjk_token`` strategy the relevance ranker uses. Indexing and + querying share this function, so unsegmented Chinese text stays matchable + in the default (dependency-free) install. + """ if not text or not text.strip(): return [] if _jieba_available: return [t for t in jieba.cut(text) if t.strip()] - return [t for t in text.split() if t.strip()] + tokens: list[str] = [] + for word in text.split(): + for part in _CJK_RUN_RE.split(word): + if not part: + continue + if _CJK_RUN_RE.fullmatch(part): + tokens.extend(cjk_token(part)) + else: + tokens.append(part) + return tokens # ── FTS5 query preprocessing ────────────────────────────────────────── @@ -78,15 +99,34 @@ def _is_advanced_query(query: str) -> bool: return bool(_FTS5_ADVANCED_RE.search(query)) +def _quote_fallback_token(token: str) -> str: + """Quote one token for the OR query, prefixing single CJK characters. + + The fallback index stores CJK runs as character bigrams, so a one-character + query (``重``) has no equal token to match. ``"重"*`` is an FTS5 prefix + phrase: it matches every indexed token that *starts with* that character, + which covers the bigram ``重试`` and still matches a standalone ``重`` token + when the content held an isolated single character. It does not reach a + character that only occurs later inside a bigram, so ``务`` does not match + the ``任务`` token; that residual gap is accepted instead of widening the + prefix. Multi-character tokens keep the plain quoted phrase so their + semantics do not change. Double quotes inside a token are the FTS5 escape + sequence for a literal quote. + """ + escaped = token.replace(chr(34), chr(34) * 2) + if len(token) == 1 and _CJK_RUN_RE.fullmatch(token): + return f'"{escaped}"*' + return f'"{escaped}"' + + def _build_fallback_query(query: str) -> str: """Convert natural-language query to FTS5 OR query (fallback strategy).""" tokens = [token for token in _tokenize(query) if any(char.isalnum() for char in token)] if not tokens: return "" # Quote each token so punctuation in natural-language input cannot become - # an FTS5 operator or syntax error. Double quotes inside a token are the - # FTS5 escape sequence for a literal quote. - return " OR ".join(f'"{token.replace(chr(34), chr(34) * 2)}"' for token in tokens) + # an FTS5 operator or syntax error. + return " OR ".join(_quote_fallback_token(token) for token in tokens) # ── Core retrieval engine ───────────────────────────────────────────── @@ -151,13 +191,15 @@ class FTS5Retrieval: # ── Index operations ─────────────────────────────────────────────── def _preprocess_content(self, content: str) -> str: - """Preprocess content for indexing: jieba tokenize for Chinese.""" + """Preprocess content for indexing: segment CJK so queries can match. + + Must stay aligned with the query side (``_build_fallback_query``); both + go through ``_tokenize`` so index and query agree on Chinese without + jieba. + """ if not content: return "" - if _jieba_available: - tokens = _tokenize(content) - return " ".join(tokens) - return content + return " ".join(_tokenize(content)) def _row_from_document(self, document: dict[str, Any]) -> tuple[Any, ...]: now = datetime.now(UTC).isoformat().replace("+00:00", "Z") diff --git a/backend/tests/test_memory_retrieval_adapter.py b/backend/tests/test_memory_retrieval_adapter.py index 1b33581dbe..92c334a9c4 100644 --- a/backend/tests/test_memory_retrieval_adapter.py +++ b/backend/tests/test_memory_retrieval_adapter.py @@ -35,8 +35,14 @@ def test_natural_language_hyphens_are_not_treated_as_fts5_syntax() -> None: assert not _is_advanced_query("real-time co-pilot node -js +python") -@pytest.mark.skipif(not _jieba_available, reason="install the optional memory-zh extra") +@pytest.mark.skipif(not _jieba_available, reason="requires the optional memory-zh extra to exercise the jieba branch") def test_chinese_subphrase_search_uses_jieba_tokenization(tmp_path: Path) -> None: + """With jieba installed, the same sub-phrase recall must still hold. + + The complementary no-jieba case is forced with a monkeypatch below, so the + two tests cover the two tokenizer branches instead of skipping each other + on the same install. + """ adapter = FTS5RetrievalAdapter(tmp_path / "facts.sqlite3") scope = {"userId": "alice", "agentName": "agent-a"} try: @@ -47,6 +53,39 @@ def test_chinese_subphrase_search_uses_jieba_tokenization(tmp_path: Path) -> Non adapter.close() +def test_chinese_fact_is_recallable_without_memory_zh_extra(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A CJK fact indexed in the default install must stay retrievable. + + ``uv sync --group dev --locked`` and the contributor CI path install no + ``memory-zh`` extra, so ``_tokenize`` has to segment CJK on its own. It + emits character bigrams (the same ``relevance.cjk_token`` strategy the + relevance ranker uses) on both the index and the query side; a whitespace + fallback would keep the whole CJK run as one FTS5 token and silently return + zero hits for every sub-phrase. ``_jieba_available`` is read at call time, + so forcing it here runs this branch even where jieba is installed. + """ + monkeypatch.setattr("deerflow.agents.memory.backends.deermem.deermem.core.retrieval._jieba_available", False) + adapter = FTS5RetrievalAdapter(tmp_path / "facts.sqlite3") + scope = {"userId": "alice", "agentName": "agent-a"} + try: + adapter.upsert(_fact("zh-default", "调度器会重试失败的任务"), scope=scope, path="") + results = adapter.search("重试", scopes=[scope], top_k=5, mode="fts5", filters=None) + assert results, "expected the stored Chinese fact to be recalled by the CJK sub-phrase '重试', got 0 hits: the no-jieba fallback must emit CJK bigrams instead of keeping '调度器会重试失败的任务' as one token" + assert [item["fact"]["id"] for item in results] == ["zh-default"] + + # A single CJK character has no equal bigram token to match; the query + # side emits the FTS5 prefix phrase "重"* so the bigram index still hits. + single = adapter.search("重", scopes=[scope], top_k=5, mode="fts5", filters=None) + assert [item["fact"]["id"] for item in single] == ["zh-default"] + + # The prefix stays anchored to that character: an absent one must not + # start matching everything in the index. + absent = adapter.search("蟹", scopes=[scope], top_k=5, mode="fts5", filters=None) + assert absent == [] + finally: + adapter.close() + + def test_score_time_decay_and_advanced_phrase_query(tmp_path: Path) -> None: engine = FTS5Retrieval(tmp_path / "facts.sqlite3") try: