From 5d86ce345c9d01dde39611f777c6507705008624 Mon Sep 17 00:00:00 2001 From: wd_pan <163314032+pwd11@users.noreply.github.com> Date: Mon, 14 Sep 2026 07:22:38 +0800 Subject: [PATCH] feat(memory): add deterministic near-duplicate fact gate (#5254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(memory): add deterministic near-duplicate fact gate Add an opt-in write-side gate for DeerMem (issue #5252): a proposed NEW fact whose bounded token-Jaccard similarity to an existing fact in the same user/agent scope AND category reaches fact_dedup_similarity_threshold merges into that fact instead of being appended — the existing id/content/createdAt are kept, confidence is raised to the maximum, and the source is refreshed, with one facts_merged_dedup metric increment recording the merge. Defaults preserve the legacy behavior exactly; targeted updates by fact id and the exact-content key check are untouched. Companion write-side step to the read-side relevance/diversity work in #5251. Refs #5252 Signed-off-by: pwd11 * fix(memory): preserve corrections during fact deduplication Signed-off-by: pwd11 --------- Signed-off-by: pwd11 Co-authored-by: Willem Jiang --- README.md | 11 + .../harness/deerflow/agents/memory/AGENTS.md | 22 ++ .../memory/backends/deermem/deermem/config.py | 21 ++ .../backends/deermem/deermem/core/updater.py | 108 ++++++ backend/tests/test_memory_fact_dedup.py | 317 ++++++++++++++++++ 5 files changed, 479 insertions(+) create mode 100644 backend/tests/test_memory_fact_dedup.py diff --git a/README.md b/README.md index 4e313aa2b..1ee1194a0 100644 --- a/README.md +++ b/README.md @@ -1477,6 +1477,17 @@ request the binary capability retain the legacy JSON/base64 frame protocol. Most agents forget everything the moment a conversation ends. DeerFlow remembers. +DeerMem can optionally suppress near-duplicate extracted facts with +`memory.backend_config.fact_dedup_enabled: true` and +`fact_dedup_similarity_threshold` (default `0.7`, range `0.5`–`1.0`). +This local, deterministic word/CJK-bigram heuristic compares facts only within +the same user, agent, and category; it is not semantic equivalence detection. +It keeps the existing ID, text, and creation time, raises confidence to the +maximum, and refreshes the source only when confidence increases. Explicit +correction replacements and facts proposed for removal are protected from +near-duplicate merging. A merge does not count as user confirmation. The gate +is off by default and does not affect targeted fact updates. + DeerFlow also includes an optional `openviking` memory backend. It uses the official `langchain-openviking` package to capture completed turns into stable OpenViking Sessions and recall memory for prompt injection while leaving diff --git a/backend/packages/harness/deerflow/agents/memory/AGENTS.md b/backend/packages/harness/deerflow/agents/memory/AGENTS.md index 28150c5b6..3c1ef2125 100644 --- a/backend/packages/harness/deerflow/agents/memory/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/memory/AGENTS.md @@ -268,3 +268,25 @@ Keep these cross-component constraints in sync: - Eviction weights must total `1.0`. - `watermark_max_keys: 0` makes the conversation watermark cache unbounded. - A dropped watermark can re-extract one batch on the next turn. + +#### Write-side near-duplicate fact gate (opt-in) + +`fact_dedup_enabled` / `fact_dedup_similarity_threshold` implement the +write-side counterpart to relevance-aware retrieval (issue #5252): a proposed +NEW fact that paraphrases an existing same-category fact merges into it +(existing id/content/createdAt kept, confidence raised to the maximum, source +refreshed only when confidence increases) instead of being appended, and one `facts_merged_dedup` metric +increment records the merge. The similarity is deterministic and network-free +(bounded token-Jaccard via the updater-local tokenizer). Exact-content +duplicates keep going through the existing content-key check; targeted updates +by fact id are untouched. + +Paired replacement proposals bypass near-dedup so their content remains +available to the post-capacity replacement check. Any ID proposed for normal +or stale removal is excluded from merge targets, even if a removal guard or +cap retains it. Scope, confidence, exact-content, and capacity gates still +apply; dedup never authorizes a removal or supplies a confirmation signal. +Latin words and CJK bigrams both participate in mixed-script similarity. +Whitespace-separated CJK runs retain adjacent-character ordering. +INFO logs identify the target and proposal index without memory content and +explicitly describe a proposed merge, not a completed persistence audit. diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/config.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/config.py index 5ae6bdc37..e0d5eb83f 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/config.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/config.py @@ -73,6 +73,27 @@ class DeerMemConfig(BaseModel): default="fts5", description="Retrieval adapter factory: 'fts5' (default), an empty string to disable, or a dotted factory receiving DeerMemConfig and implementing RetrievalPort.", ) + fact_dedup_enabled: bool = Field( + default=False, + description=( + "Opt-in deterministic near-duplicate gate for NEW facts (issue " + "#5252). When true, a proposed new fact whose bounded " + "token-Jaccard similarity to an existing fact in the same " + "user/agent scope AND category reaches " + "fact_dedup_similarity_threshold merges into that fact instead " + "of being appended: the existing id/content/createdAt are kept, " + "confidence is raised to max(old, new), and source is refreshed " + "only when confidence increases. Correction replacements and " + "proposed removal targets are excluded from near-dedup. " + "False preserves the legacy behavior exactly." + ), + ) + fact_dedup_similarity_threshold: float = Field( + default=0.7, + ge=0.5, + le=1.0, + description=("Minimum bounded token-Jaccard similarity for the write-side near-duplicate merge gate. Used only when fact_dedup_enabled is true."), + ) # ── Queue ──────────────────────────────────────────────────────────── debounce_seconds: int = Field( default=30, diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py index 8d827aabd..3303c0d7b 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py @@ -472,6 +472,72 @@ def _fact_content_key(content: Any) -> str | None: return stripped.casefold() +_FACT_CJK_RANGE = "\u3400-\u4dbf\u4e00-\u9fff\U00020000-\U0002fa1f" +_FACT_TOKEN_RE = re.compile(rf"[a-zA-Z0-9_]+|[{_FACT_CJK_RANGE}]+(?:\s+[{_FACT_CJK_RANGE}]+)*") +_FACT_SIMILARITY_TOKEN_BUDGET = 128 + + +def _fact_content_tokens(content: str) -> list[str]: + """Deterministic, network-free tokenization for fact similarity. + + Latin words and CJK bigrams coexist in mixed-script text. Whitespace + between CJK runs is ignored, retaining adjacent-character order even + for spaced Chinese text, without joining across Latin words/punctuation. + """ + lowered = content.strip().lower() + if not lowered: + return [] + tokens: list[str] = [] + for match in _FACT_TOKEN_RE.finditer(lowered): + run = match.group() + if run[0].isascii(): + tokens.append(run) + else: + run = "".join(run.split()) + tokens.extend([run] if len(run) == 1 else (run[index : index + 2] for index in range(len(run) - 1))) + return tokens or lowered.split() + + +def _fact_content_similarity(left: str, right: str) -> float: + """Bounded token-Jaccard similarity over case-folded token sets.""" + left_set = set(_fact_content_tokens(left)[:_FACT_SIMILARITY_TOKEN_BUDGET]) + right_set = set(_fact_content_tokens(right)[:_FACT_SIMILARITY_TOKEN_BUDGET]) + if not left_set or not right_set: + return 0.0 + return len(left_set & right_set) / len(left_set | right_set) + + +def _find_dedup_merge_target( + content: str, + category: str, + facts: list[dict[str, Any]], + *, + threshold: float, +) -> dict[str, Any] | None: + """Return the most similar same-category fact at/above the similarity + threshold, or ``None`` (write-side near-duplicate gate, issue #5252). + + Deterministic and offline: bounded token-Jaccard similarity. Only + candidate facts whose content is a non-empty string participate; + category mismatch never merges. + """ + best_target: dict[str, Any] | None = None + best_similarity = 0.0 + for fact in facts: + if not isinstance(fact, dict): + continue + candidate_content = fact.get("content") + if not isinstance(candidate_content, str) or not candidate_content.strip(): + continue + if fact.get("category", "context") != category: + continue + similarity = _fact_content_similarity(content, candidate_content) + if similarity >= threshold and similarity > best_similarity: + best_target = fact + best_similarity = similarity + return best_target + + def _raise_if_duplicate_fact_content(memory_data: dict[str, Any], content_key: str | None) -> None: """Reject a candidate fact whose normalized content already exists. @@ -2049,6 +2115,13 @@ class MemoryUpdater: # persisted-fact count. passed_threshold = 0 replacement_fact_keys: dict[int, str] = {} + # A correction must keep its proposed content, not inherit an older + # paraphrase. Also avoid strengthening any proposed removal target, + # including stale targets retained by candidate guards or the cap. + # This only excludes dedup matches; removal validation remains below. + removal_proposals = [removal for key in ("factsToRemove", "staleFactsToRemove") for removal in (update_data.get(key) if isinstance(update_data.get(key), list) else []) if isinstance(removal, dict)] + removal_target_ids = {removal["id"] for removal in removal_proposals if isinstance(removal.get("id"), str)} + replacement_indices = {index for removal in removal_proposals if isinstance(index := removal.get("replacementFactIndex"), int) and not isinstance(index, bool) and index >= 0} for fact_index, fact in enumerate(new_facts): confidence = fact.get("confidence", 0.5) if confidence >= config.fact_confidence_threshold: @@ -2076,6 +2149,41 @@ class MemoryUpdater: if fact_key in existing_fact_keys: continue + # Write-side near-duplicate gate (issue #5252): a proposed new fact + # that paraphrases an existing same-category fact merges into it + # instead of being appended. The existing id/content/createdAt stay + # authoritative; source follows only a confidence increase, never + # an unconfirmed lower-confidence restatement. + if config.fact_dedup_enabled and fact_index not in replacement_indices: + merge_target = _find_dedup_merge_target( + normalized_content, + fact.get("category", "context"), + [candidate for candidate in current_memory.get("facts", []) if candidate.get("id") not in removal_target_ids], + threshold=config.fact_dedup_similarity_threshold, + ) + if merge_target is not None: + try: + existing_confidence = float(merge_target.get("confidence")) + if not math.isfinite(existing_confidence): + raise ValueError + except (TypeError, ValueError): + existing_confidence = 0.0 + if confidence > existing_confidence: + merge_target["confidence"] = confidence + merge_target["source"] = thread_id or "unknown" + if metrics is not None: + metrics["facts_merged_dedup"] = metrics.get("facts_merged_dedup", 0) + 1 + # New proposals have no durable ID yet. Log their batch + # index, not personal fact text; this is a proposed write, + # not a claim that persistence has already succeeded. + logger.info( + "Near-duplicate fact merge proposed: target_id=%s proposal_index=%d confidence_raised=%s", + merge_target.get("id"), + fact_index, + confidence > existing_confidence, + ) + continue + fact_entry = { "id": f"fact_{uuid.uuid4().hex[:8]}", "content": normalized_content, diff --git a/backend/tests/test_memory_fact_dedup.py b/backend/tests/test_memory_fact_dedup.py new file mode 100644 index 000000000..626e32817 --- /dev/null +++ b/backend/tests/test_memory_fact_dedup.py @@ -0,0 +1,317 @@ +"""Tests for the deterministic near-duplicate fact gate (issue #5252). + +The gate is opt-in via DeerMem-private config (``fact_dedup_enabled``) and +must never change the default behavior. A proposed NEW fact whose bounded +token-Jaccard similarity to an existing fact in the same user/agent scope +AND category reaches ``fact_dedup_similarity_threshold`` merges into that +fact (existing id/content/createdAt kept, confidence raised to the maximum, +source refreshed only on a confidence increase) instead of being appended. + +Test construction mirrors ``tests/test_memory_scope_gate.py``: a real +``MemoryUpdater`` with an in-memory storage and ``_apply_updates`` driven +directly, no LLM, no network. +""" + +from __future__ import annotations + +import copy +import logging + +import pytest + +from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig +from deerflow.agents.memory.backends.deermem.deermem.core.storage import MemoryStorage +from deerflow.agents.memory.backends.deermem.deermem.core.updater import MemoryUpdater, _fact_content_similarity, _fact_content_tokens + + +def _memory(facts: list[dict[str, object]] | None = None) -> dict[str, object]: + return { + "user": { + "workContext": {"summary": "", "updatedAt": ""}, + "personalContext": {"summary": "", "updatedAt": ""}, + "topOfMind": {"summary": "", "updatedAt": ""}, + }, + "history": { + "recentMonths": {"summary": "", "updatedAt": ""}, + "earlierContext": {"summary": "", "updatedAt": ""}, + "longTermBackground": {"summary": "", "updatedAt": ""}, + }, + "facts": copy.deepcopy(facts or []), + } + + +class _Storage(MemoryStorage): + def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, object]: + return _memory() + + def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, object]: + return self.load(agent_name, user_id=user_id) + + def save( + self, + memory_data: dict[str, object], + agent_name: str | None = None, + *, + user_id: str | None = None, + expected_revision: int | None = None, + ) -> bool: + return True + + +def _updater(**config_overrides: object) -> MemoryUpdater: + config = DeerMemConfig() + for key, value in config_overrides.items(): + setattr(config, key, value) + return MemoryUpdater(config, _Storage(), llm=None) + + +def _fact(content: str, **overrides: object) -> dict[str, object]: + fact: dict[str, object] = { + "content": content, + "category": "preference", + "confidence": 0.9, + "scope": "user", + "durability": "durable", + "authority": "descriptive", + } + fact.update(overrides) + return fact + + +def _stored_fact(fact_id: str, content: str, **overrides: object) -> dict[str, object]: + fact: dict[str, object] = { + "id": fact_id, + "content": content, + "category": "preference", + "confidence": 0.9, + "createdAt": "2026-01-01T00:00:00Z", + "source": "thread-old", + } + fact.update(overrides) + return fact + + +def _update(new_facts: list[dict[str, object]]) -> dict[str, object]: + return {"user": {}, "history": {}, "newFacts": new_facts} + + +class TestFactDedupGate: + def test_default_config_keeps_near_duplicate_facts(self): + updater = _updater() + current = _memory([_stored_fact("fact_old", "User prefers concise answers in chat")]) + + result = updater._apply_updates(current, _update([_fact("User prefers concise answers in chat, short form")])) + + assert len(result["facts"]) == 2 + + def test_enabled_merges_paraphrased_fact(self): + updater = _updater(fact_dedup_enabled=True) + current = _memory([_stored_fact("fact_old", "User prefers concise answers in chat")]) + metrics: dict[str, object] = {} + + result = updater._apply_updates( + current, + _update([_fact("User prefers concise answers in chat, short form", confidence=0.95)]), + metrics=metrics, + ) + + assert len(result["facts"]) == 1 + merged = result["facts"][0] + assert merged["id"] == "fact_old" + assert merged["content"] == "User prefers concise answers in chat" + assert merged["createdAt"] == "2026-01-01T00:00:00Z" + assert merged["confidence"] == 0.95 + assert merged["source"] == "unknown" # refreshed from the proposed fact's thread + assert metrics.get("facts_merged_dedup") == 1 + + def test_merge_keeps_higher_existing_confidence(self): + updater = _updater(fact_dedup_enabled=True) + current = _memory([_stored_fact("fact_old", "User prefers concise answers in chat", confidence=0.98)]) + + result = updater._apply_updates( + current, + _update([_fact("User prefers concise answers in chat, short form", confidence=0.8)]), + ) + + assert len(result["facts"]) == 1 + assert result["facts"][0]["confidence"] == 0.98 + + def test_different_category_does_not_merge(self): + updater = _updater(fact_dedup_enabled=True) + current = _memory([_stored_fact("fact_old", "User prefers concise answers in chat")]) + + result = updater._apply_updates( + current, + _update([_fact("User prefers concise answers in chat, short form", category="project")]), + ) + + assert len(result["facts"]) == 2 + + def test_unrelated_content_below_threshold_does_not_merge(self): + updater = _updater(fact_dedup_enabled=True) + current = _memory([_stored_fact("fact_old", "User prefers concise answers in chat")]) + + result = updater._apply_updates( + current, + _update([_fact("User works on database migrations")]), + ) + + assert len(result["facts"]) == 2 + + def test_threshold_is_respected(self): + updater = _updater(fact_dedup_enabled=True, fact_dedup_similarity_threshold=0.9) + current = _memory([_stored_fact("fact_old", "User prefers concise answers in chat")]) + + result = updater._apply_updates( + current, + _update([_fact("User prefers concise answers in chat, short form")]), + ) + + # 6/8 = 0.75 token-Jaccard is below the configured 0.9 threshold. + assert len(result["facts"]) == 2 + + def test_exact_duplicate_still_skipped_without_merge_metrics(self): + updater = _updater(fact_dedup_enabled=True) + current = _memory([_stored_fact("fact_old", "User prefers concise answers in chat")]) + metrics: dict[str, object] = {} + + result = updater._apply_updates( + current, + _update([_fact("User prefers concise answers in chat")]), + metrics=metrics, + ) + + assert len(result["facts"]) == 1 + assert "facts_merged_dedup" not in metrics + + +class TestFactDedupConfig: + def test_defaults_keep_legacy_behavior(self): + config = DeerMemConfig() + assert config.fact_dedup_enabled is False + assert config.fact_dedup_similarity_threshold == 0.7 + + def test_backend_config_accepts_new_knobs(self): + config = DeerMemConfig.from_backend_config( + { + "fact_dedup_enabled": True, + "fact_dedup_similarity_threshold": 0.8, + } + ) + assert config.fact_dedup_enabled is True + assert config.fact_dedup_similarity_threshold == 0.8 + + +@pytest.mark.parametrize("paired", [True, False]) +def test_correction_is_not_merged_into_removal_target(paired): + old = "User prefers email notifications over sms for weekly reports" + new = "User prefers push notifications over sms for weekly reports" + current = _memory([_stored_fact("fact_B", old, confidence=0.5)]) + update = _update([_fact(new, confidence=0.95)]) + removal = {"id": "fact_B", "scope": "user", "reason": "Preference changed"} + if paired: + removal["replacementFactIndex"] = 0 + update["factsToRemove"] = [removal] + metrics = {} + + result = _updater(fact_dedup_enabled=True)._apply_updates(current, update, thread_id="correction", metrics=metrics) + + assert len(result["facts"]) == 1 + assert result["facts"][0]["content"] == new + assert result["facts"][0]["id"] != "fact_B" + assert result["facts"][0]["source"] == "correction" + assert metrics.get("facts_merged_dedup", 0) == 0 + + +def test_paired_replacement_is_not_merged_into_another_similar_fact(): + other = "User prefers email notifications over sms for weekly reports" + current = _memory( + [ + _stored_fact("fact_B", "User previously disabled all notifications"), + _stored_fact("fact_other", other), + ] + ) + new = "User prefers push notifications over sms for weekly reports" + update = _update([_fact(new, confidence=0.95)]) + update["factsToRemove"] = [{"id": "fact_B", "scope": "user", "reason": "Changed", "replacementFactIndex": 0}] + + result = _updater(fact_dedup_enabled=True)._apply_updates(current, update) + + assert {fact["content"] for fact in result["facts"]} == {other, new} + assert all(fact["id"] != "fact_B" for fact in result["facts"]) + + +def test_stale_removal_proposal_is_not_strengthened_when_guard_keeps_it(): + old = _stored_fact("fact_old", "User prefers concise answers in chat") + update = _update([_fact("User prefers concise answers in chat, short form", confidence=0.95)]) + # Configure preferences as protected from staleness removal. A proposal + # still must not become evidence strengthening the proposed target. + update["staleFactsToRemove"] = [{"id": "fact_old", "reason": "Outdated"}] + + result = _updater(fact_dedup_enabled=True, staleness_protected_categories=["preference"])._apply_updates(_memory([old]), update) + + assert len(result["facts"]) == 2 + assert next(fact for fact in result["facts"] if fact["id"] == "fact_old") == old + + +@pytest.mark.parametrize("confidence", [0.8, 0.98]) +def test_nonincreasing_confidence_preserves_source_and_confirmation(confidence): + old = _stored_fact("fact_old", "User prefers concise answers in chat", confidence=0.98, lastConfirmedAt="2026-01-02T00:00:00Z", confirmationCount=2) + result = _updater(fact_dedup_enabled=True)._apply_updates( + _memory([old]), + _update([_fact("User prefers concise answers in chat, short form", confidence=confidence)]), + thread_id="restatement", + ) + assert result["facts"] == [old] + + +def test_merge_logs_metadata_at_info_without_fact_text(caplog): + old = "User prefers concise answers in chat" + new = old + ", short form" + with caplog.at_level(logging.INFO): + _updater(fact_dedup_enabled=True)._apply_updates(_memory([_stored_fact("fact_old", old)]), _update([_fact(new, confidence=0.95)])) + assert "Near-duplicate fact merge proposed" in caplog.text + assert "fact_old" in caplog.text + assert "proposal_index=0" in caplog.text + assert old not in caplog.text + assert new not in caplog.text + + +def test_mixed_script_facts_retain_different_cities(): + old = "Prefers Python, 用户在北京" + new = "Prefers Python, 用户在上海" + result = _updater(fact_dedup_enabled=True)._apply_updates(_memory([_stored_fact("fact_old", old)]), _update([_fact(new)])) + assert len(result["facts"]) == 2 + assert _fact_content_similarity(old, new) < 0.7 + + +def test_tokenizer_combines_words_and_cjk_bigrams(): + assert _fact_content_tokens("Prefers Python, 用户在北京") == ["prefers", "python", "用户", "户在", "在北", "北京"] + assert _fact_content_tokens("用户 北京") == ["用户", "户北", "北京"] + assert _fact_content_similarity("用户 北京", "北京 用户") < 0.7 + assert _fact_content_tokens("中") == ["中"] + + +@pytest.mark.parametrize("override", [{"scope": "thread"}, {"confidence": 0.1}]) +def test_rejected_replacement_does_not_delete_or_strengthen_original(override): + old = _stored_fact("fact_B", "User prefers email notifications over sms for weekly reports", confidence=0.8) + update = _update([_fact("User prefers push notifications over sms for weekly reports", **override)]) + update["factsToRemove"] = [{"id": "fact_B", "scope": "user", "reason": "Changed", "replacementFactIndex": 0}] + result = _updater(fact_dedup_enabled=True)._apply_updates(_memory([old]), update) + assert result["facts"] == [old] + + +@pytest.mark.parametrize("confirmed", [False, True]) +def test_merge_preserves_deterministic_confirmation_gate(confirmed): + old = _stored_fact("fact_old", "User prefers concise answers in chat", lastConfirmedAt="2026-01-02T00:00:00Z", confirmationCount=2) + update = _update([_fact("User prefers concise answers in chat, short form", confidence=0.95)]) + update["factsToReinforce"] = [{"id": "fact_old", "scope": "user", "reason": "Restated"}] + result = _updater(fact_dedup_enabled=True, fact_eviction_shadow_enabled=True)._apply_updates( + _memory([old]), + update, + signals=frozenset({"reinforcement"}) if confirmed else frozenset(), + ) + merged = result["facts"][0] + assert merged["confidence"] == 0.95 + assert merged["confirmationCount"] == (3 if confirmed else 2) + assert (merged["lastConfirmedAt"] != old["lastConfirmedAt"]) is confirmed