deer-flow/backend/tests/test_deermem_self_contained.py

"""Phase-2 (self-contained DeerMem) tests.

Covers: DI construction (owns storage/updater/queue/llm), zero-config defaults,
``trace_id`` threading to the optional ``callbacks`` hook, langfuse being
optional, ``hide_from_ui`` default-skip + hook-keep, empty ``storage_class``
(portable default), and portability -- ``backends/deermem/`` has exactly one
``from deerflow`` line (the ABC contract) and can be vendored into another agent
by copying the folder and repointing that one line.

Storage is isolated via ``$DEERMEM_DATA_DIR`` -> ``tmp_path``; the LLM is a fake
injected onto the updater so no network is needed.
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

import pytest
from langchain_core.messages import AIMessage, HumanMessage

from deerflow.agents.memory.backends.deermem.deer_mem import DeerMem
from deerflow.agents.memory.backends.deermem.deermem.core import storage as deermem_storage
from deerflow.agents.memory.backends.deermem.deermem.core.message_processing import (
    filter_messages_for_memory,
)
from deerflow.agents.memory.backends.deermem.deermem.core.storage import FileMemoryStorage
from deerflow.agents.memory.manager import MemoryCallbacks, MemoryCorruptionError


@pytest.fixture
def deermem_data_dir(tmp_path, monkeypatch):
    """Isolate DeerMem storage under tmp_path via $DEERMEM_DATA_DIR."""
    d = tmp_path / "deermem_data"
    d.mkdir()
    monkeypatch.setenv("DEERMEM_DATA_DIR", str(d))
    yield d


class _FakeLLM:
    """Returns a fixed memory-update JSON so no real LLM/network is needed."""

    def __init__(self, payload: str | None = None) -> None:
        self._payload = payload or '{"user":{},"history":{},"newFacts":[],"factsToRemove":[]}'

    def invoke(self, prompt, config=None):
        return type("R", (), {"content": self._payload})()


def _deermem_with_fake_llm(backend_config=None, payload=None, callbacks=None) -> DeerMem:
    dm = DeerMem(backend_config=backend_config, callbacks=callbacks)
    fake = _FakeLLM(payload)
    dm._llm = fake
    dm._updater._llm = fake
    return dm


def _legacy_alias_normalize_category(fact: dict) -> None:
    """The pre-native-category ``storage._normalize_category``.

    Historical builds parked any label outside CORE_CATEGORIES in
    ``categoryExtension`` and stored ``category="other"``; migration-retry and
    update-path regressions below replay data an interrupted such build left
    behind, so the tests need the exact historical writer.
    """
    raw_category = fact.get("category", "context")
    if not isinstance(raw_category, str):
        raise ValueError("fact.category must be a string")
    category = raw_category or "context"
    if category not in deermem_storage.CORE_CATEGORIES:
        fact.setdefault("categoryExtension", category)
        fact["category"] = "other"
    else:
        fact["category"] = category


def test_add_swallows_queue_full_so_backpressure_does_not_break_caller(deermem_data_dir, caplog) -> None:
    """Regression: QueueFull raised under backpressure is caught in
    DeerMem.add (the backend owns the queue, so it owns the degradation) so
    memory backpressure degrades to "update skipped" instead of propagating into
    MemoryMiddleware.after_agent and breaking the agent run -- peer middlewares
    self-guard the same way."""
    import logging

    dm = _deermem_with_fake_llm(backend_config={"storage_path": str(deermem_data_dir), "queue_max_depth": 1})
    # Stop the debounce timer so enqueued items stay pending (the cap persists
    # across the second add instead of being drained by a timer fire).
    dm._queue._schedule_timer = lambda *a, **k: None

    conv = [HumanMessage("Please explain quantum computing in detail"), AIMessage("Quantum computing uses qubits and superposition.")]
    # First add fills the queue to its depth cap (non-signal, new key).
    dm.add("thread-A", conv, agent_name="lead_agent", user_id="u")
    assert dm._queue.pending_count == 1

    # Second add for a different key hits the cap -> QueueFull internally. It
    # must be caught: no exception escapes DeerMem.add.
    with caplog.at_level(logging.WARNING, logger="deerflow.agents.memory.backends.deermem.deer_mem"):
        dm.add("thread-B", conv, agent_name="lead_agent", user_id="u")
    assert "rejected under backpressure" in caplog.text
    # thread-B was rejected (not enqueued); only thread-A remains.
    assert dm._queue.pending_count == 1


def test_di_construction_owns_dependencies():
    dm = DeerMem(backend_config={"max_facts": 50, "storage_path": "/tmp/x"})
    assert dm._config.max_facts == 50
    assert dm._storage is not None and dm._updater is not None and dm._queue is not None
    # dependencies are wired (DI), not globals:
    assert dm._updater._storage is dm._storage
    assert dm._queue._updater is dm._updater


def test_from_config_keeps_backend_config_pure_of_injected_hooks(deermem_data_dir):
    """Host hooks (should_keep_hidden_message / trace_context_manager / host_llm)
    arrive as from_config kwargs and are parsed into DeerMemConfig
    (self._config, PrivateAttr); the instance's backend_config field stays the
    pure data the host passed (no callables / LLM), so it remains serializable
    and matches the README contract ("host hooks ... NOT in backend_config")."""

    def _keep(ak):
        return False

    trace_cm = object()  # sentinel; trace_context_manager is typed Any
    fake_llm = _FakeLLM()
    dm = DeerMem.from_config(
        backend_config={"storage_path": str(deermem_data_dir), "max_facts": 20},
        mode="middleware",
        should_keep_hidden_message=_keep,
        trace_context_manager=trace_cm,
        host_llm_factory=lambda: fake_llm,
        callbacks=None,
    )
    # hooks reached DeerMemConfig (PrivateAttr) -- wired, not lost
    assert dm._config.should_keep_hidden_message is _keep
    assert dm._config.trace_context_manager is trace_cm
    assert dm._config.host_llm is fake_llm
    # backend_config field is the pure original data (no injected hooks)
    assert dm.backend_config == {"storage_path": str(deermem_data_dir), "max_facts": 20}
    assert "should_keep_hidden_message" not in dm.backend_config
    assert "trace_context_manager" not in dm.backend_config
    assert "host_llm" not in dm.backend_config


def test_zero_config_defaults_run_non_llm_ops(deermem_data_dir):
    dm = DeerMem(backend_config=None)  # zero config
    assert dm._llm is None  # no model -> no LLM
    dm.import_memory(
        {"version": "1.0", "lastUpdated": "", "user": {}, "history": {}, "facts": [{"id": "f", "content": "x", "category": "c", "confidence": 0.5, "createdAt": "", "source": "m"}]},
        user_id="u",
    )
    assert "x" in dm.get_context(user_id="u")
    assert dm.get_memory(user_id="u")["facts"][0]["content"] == "x"


def test_get_context_injects_facts_only_in_middleware_mode(deermem_data_dir):
    backend_config = {
        "storage_path": str(deermem_data_dir),
        "retrieval_adapter": "",
        "token_counting": "char",
    }
    middleware = DeerMem(backend_config=backend_config, mode="middleware")
    middleware.import_memory(
        {
            "user": {
                "workContext": {"summary": "Works on DeerFlow memory."},
            },
            "history": {
                "recentMonths": {"summary": "Recently redesigned storage."},
            },
            "facts": [
                {
                    "id": "fact_tool_only",
                    "content": "Use FTS5 for active fact recall.",
                    "category": "constraint",
                    "confidence": 0.9,
                    "source": "manual",
                }
            ],
        },
        user_id="u",
    )

    middleware_context = middleware.get_context(user_id="u")
    tool_context = DeerMem(backend_config=backend_config, mode="tool").get_context(user_id="u")

    assert "Works on DeerFlow memory." in middleware_context
    assert "Recently redesigned storage." in middleware_context
    assert "Use FTS5 for active fact recall." in middleware_context
    assert "Works on DeerFlow memory." in tool_context
    assert "Recently redesigned storage." in tool_context
    assert "Use FTS5 for active fact recall." not in tool_context
    assert "Facts:" not in tool_context


def test_import_without_agent_name_persists_facts_in_default_markdown_bucket(deermem_data_dir):
    dm = DeerMem(backend_config=None)
    dm.import_memory(
        {
            "user": {},
            "history": {},
            "facts": [
                {
                    "id": "fact_default_import",
                    "content": "imported through the default manager scope",
                    "category": "context",
                    "confidence": 0.8,
                    "source": "import",
                }
            ],
        },
        user_id="alice",
    )

    assert [fact["id"] for fact in dm.get_memory(user_id="alice")["facts"]] == ["fact_default_import"]
    facts_root = deermem_data_dir / "users" / "alice" / "agents" / "__default__" / "facts"
    assert [path.stem for path in facts_root.glob("**/*.md")] == ["fact_default_import"]


def test_import_empty_summary_sections_replace_existing_summaries_with_complete_defaults(deermem_data_dir):
    dm = DeerMem(backend_config=None)
    existing = dm.get_memory(user_id="alice")
    existing["user"]["workContext"] = {"summary": "old work", "updatedAt": "old"}
    existing["user"]["personalContext"] = {"summary": "old personal", "updatedAt": "old"}
    existing["history"]["recentMonths"] = {"summary": "old history", "updatedAt": "old"}
    dm.import_memory(existing, user_id="alice")

    imported = dm.import_memory({"user": {}, "history": {}, "facts": []}, user_id="alice")

    assert imported["user"] == {
        "workContext": {"summary": "", "updatedAt": ""},
        "personalContext": {"summary": "", "updatedAt": ""},
        "topOfMind": {"summary": "", "updatedAt": ""},
        "cognitiveStyle": {"summary": "", "updatedAt": ""},
    }
    assert imported["history"] == {
        "recentMonths": {"summary": "", "updatedAt": ""},
        "earlierContext": {"summary": "", "updatedAt": ""},
        "longTermBackground": {"summary": "", "updatedAt": ""},
    }


def test_trace_id_threads_through_to_callbacks(deermem_data_dir):
    """trace_id reaches the pre-LLM-call callbacks hook (on_memory_llm_call)."""
    calls = []

    class _RecordingCallbacks(MemoryCallbacks):
        def on_memory_llm_call(self, invoke_config, *, thread_id, user_id, trace_id, model_name):
            calls.append((thread_id, trace_id, model_name))

    dm = _deermem_with_fake_llm({"model": {"provider": "openai", "model": "gpt-x", "api_key": "k", "base_url": "u"}}, callbacks=_RecordingCallbacks())
    dm.add(
        thread_id="t1",
        messages=[HumanMessage(content="hi"), AIMessage(content="hello")],
        agent_name=None,
        user_id="u1",
        trace_id="trace-42",
    )
    dm._queue.flush()
    assert calls and calls[0] == ("t1", "trace-42", "gpt-x")


def test_default_passive_update_persists_fact_in_reserved_default_bucket(deermem_data_dir):
    dm = _deermem_with_fake_llm(payload='{"user":{},"history":{},"newFacts":[{"content":"Default agent fact","category":"context","confidence":0.9,"scope":"user","durability":"durable","authority":"descriptive"}],"factsToRemove":[]}')

    dm.add(
        thread_id="default-thread",
        messages=[HumanMessage(content="remember this"), AIMessage(content="understood")],
        user_id="alice",
    )
    dm._queue.flush()

    assert [fact["content"] for fact in dm.get_memory(user_id="alice")["facts"]] == ["Default agent fact"]
    facts_root = deermem_data_dir / "users" / "alice" / "agents" / "__default__" / "facts"
    assert list(facts_root.glob("**/*.md"))


def test_clear_all_memory_removes_global_summaries_and_every_agent_fact(deermem_data_dir):
    dm = DeerMem()
    imported = dm.get_memory(user_id="alice")
    imported["user"]["workContext"] = {"summary": "shared profile", "updatedAt": "now"}
    imported["facts"] = [
        {
            "id": "fact_default",
            "content": "default fact",
            "category": "context",
            "confidence": 0.9,
            "createdAt": "2026-01-01T00:00:00Z",
            "source": "manual",
        }
    ]
    dm.import_memory(imported, user_id="alice")
    dm.create_fact("custom fact", agent_name="custom-agent", user_id="alice")
    custom_dir = deermem_data_dir / "users" / "alice" / "agents" / "custom-agent"
    config_path = custom_dir / "config.yaml"
    config_path.write_text("name: custom-agent\n", encoding="utf-8")

    cleared = dm.clear_memory(user_id="alice")

    assert cleared["user"]["workContext"]["summary"] == ""
    assert cleared["facts"] == []
    assert dm.get_memory(agent_name="custom-agent", user_id="alice")["facts"] == []
    assert config_path.read_text(encoding="utf-8") == "name: custom-agent\n"


def test_scoped_clear_preserves_shared_summaries(deermem_data_dir):
    dm = DeerMem()
    imported = dm.get_memory(user_id="alice")
    imported["user"]["workContext"] = {"summary": "shared profile", "updatedAt": "now"}
    dm.import_memory(imported, user_id="alice")
    dm.create_fact("custom fact", agent_name="custom-agent", user_id="alice")

    cleared = dm.clear_memory(agent_name="custom-agent", user_id="alice")

    assert cleared["facts"] == []
    assert cleared["user"]["workContext"]["summary"] == "shared profile"
    assert dm.get_memory(user_id="alice")["user"]["workContext"]["summary"] == "shared profile"


def test_callbacks_optional_no_langfuse(deermem_data_dir):
    dm = _deermem_with_fake_llm({"model": {"provider": "openai", "model": "gpt-x", "api_key": "k", "base_url": "u"}})
    assert dm.callbacks is None  # no callbacks = no langfuse (not hard-required)
    dm.add(
        thread_id="t2",
        messages=[HumanMessage(content="hi"), AIMessage(content="hello")],
        agent_name=None,
        user_id="u2",
        trace_id="t-99",
    )
    dm._queue.flush()  # no callbacks, no error, update completes


def test_hide_from_ui_default_skip_hook_keeps():
    hidden = HumanMessage(content="secret", additional_kwargs={"hide_from_ui": True})
    normal = HumanMessage(content="hi")
    ai = AIMessage(content="hello")
    # default (no hook) -> hide_from_ui skipped
    assert hidden not in filter_messages_for_memory([hidden, normal, ai])
    # hook returns True -> hidden kept
    assert hidden in filter_messages_for_memory([hidden, normal, ai], should_keep_hidden_message=lambda ak: True)


def test_storage_class_empty_uses_filememorystorage():
    # empty storage_class (default) -> FileMemoryStorage directly, no importlib (portable, zero noise)
    dm = DeerMem(backend_config=None)
    assert dm._config.storage_class == ""
    assert isinstance(dm._storage, FileMemoryStorage)


def test_portability_only_abc_contract_imports_deerflow():
    """backends/deermem/ has exactly ONE `from deerflow` line: the ABC contract in deer_mem.py."""
    import deerflow.agents.memory.backends.deermem as pkg

    root = Path(pkg.__file__).parent
    deerflow_imports = []
    for p in root.rglob("*.py"):
        for line in p.read_text(encoding="utf-8").splitlines():
            s = line.strip()
            if s.startswith("from deerflow") or s.startswith("import deerflow"):
                deerflow_imports.append((p.relative_to(root).as_posix(), s))
    assert len(deerflow_imports) == 1, deerflow_imports
    assert deerflow_imports[0][0] == "deer_mem.py"
    assert "memory.manager import MemoryConflictError, MemoryCorruptionError, MemoryManager" in deerflow_imports[0][1]


# Minimal vendored host contract (what another agent would ship). DeerMem only
# needs this ABC -- nothing else from a host.
_VENDORED_MANAGER_PY = '''
"""Vendored host contract (pydantic BaseModel + three-tier ABC) for the portability demo."""
from abc import abstractmethod
from typing import Any, ClassVar, Literal

from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator


class MemoryManager(BaseModel):
    model_config = ConfigDict(arbitrary_types_allowed=True)

    backend_config: dict[str, Any] = Field(default_factory=dict)
    mode: Literal["middleware", "tool"] = "middleware"
    callbacks: Any = None
    supports_search: ClassVar[bool] = False

    @field_validator("backend_config", mode="before")
    @classmethod
    def _coerce_backend_config(cls, value):
        return value or {}

    @model_validator(mode="after")
    def _check_invariants(self):
        if self.mode == "tool" and not type(self).supports_search:
            raise ValueError("tool mode requires search")
        return self

    # Tier 1: abstract (every backend must implement).
    @abstractmethod
    def add(self, thread_id, messages, *, agent_name=None, user_id=None, trace_id=None) -> None: ...
    @abstractmethod
    def get_context(self, user_id, *, agent_name=None, thread_id=None) -> str: ...
    @classmethod
    @abstractmethod
    def from_config(cls, backend_config, *, mode="middleware", **host_hooks): ...

    # Tier 2: management defaults (override if supported).
    def add_nowait(self, thread_id, messages, *, agent_name=None, user_id=None) -> None:
        self.add(thread_id, messages, agent_name=agent_name, user_id=user_id)
    def search(self, query, top_k=5, *, user_id=None, agent_name=None, category=None) -> list:
        raise NotImplementedError
    def get_memory(self, *, user_id=None, agent_name=None) -> dict:
        raise NotImplementedError
    def delete_memory(self, *, user_id=None, agent_name=None) -> None:
        raise NotImplementedError
    def clear_memory(self, *, user_id=None, agent_name=None) -> dict:
        raise NotImplementedError
    def import_memory(self, memory_data, *, user_id=None, agent_name=None) -> dict:
        raise NotImplementedError
    def export_memory(self, *, user_id=None, agent_name=None) -> dict:
        raise NotImplementedError
    def shutdown_flush(self, timeout) -> bool:
        return True

    # Tier 3: optional hooks (override if supported).
    def warm(self) -> bool | None:
        return None
    def reload_memory(self, *, user_id=None, agent_name=None) -> dict:
        raise NotImplementedError
    def create_fact(self, content, category="context", confidence=0.5, *, agent_name=None, user_id=None):
        raise NotImplementedError
    def delete_fact(self, fact_id, *, agent_name=None, user_id=None) -> dict:
        raise NotImplementedError
    def update_fact(self, fact_id, content=None, category=None, confidence=None, *, agent_name=None, user_id=None) -> dict:
        raise NotImplementedError
    def on_pre_compress(self, messages) -> str:
        return ""
    def on_turn_start(self, turn_number, message, **kwargs) -> None:
        return None

class MemoryConflictError(RuntimeError): ...
class MemoryCorruptionError(RuntimeError): ...
'''


def test_portability_vendor_to_other_agent(tmp_path, monkeypatch):
    """Copy backends/deermem/ into a temp package, repoint the ONE ABC import to
    a vendored manager, import, and run a round-trip -- proves copy + 1-line +
    run portability (zero deerflow dependency at runtime)."""
    import importlib
    import shutil

    import deerflow.agents.memory.backends.deermem as pkg

    src = Path(pkg.__file__).parent
    # Vendored host package with a minimal manager.py (the contract).
    host_pkg = tmp_path / "otheragent"
    host_pkg.mkdir()
    (host_pkg / "__init__.py").write_text("", encoding="utf-8")
    (host_pkg / "manager.py").write_text(_VENDORED_MANAGER_PY, encoding="utf-8")
    # Copy the DeerMem backend folder.
    dst_pkg = tmp_path / "otheragent_deermem"
    shutil.copytree(src, dst_pkg)
    # Repoint the single ABC-contract import line to the vendored manager.
    deer_mem_file = dst_pkg / "deer_mem.py"
    text = deer_mem_file.read_text(encoding="utf-8")
    contract_import = "from deerflow.agents.memory.manager import MemoryConflictError, MemoryCorruptionError, MemoryManager"
    assert contract_import in text
    text = text.replace(
        contract_import,
        "from otheragent.manager import MemoryConflictError, MemoryCorruptionError, MemoryManager",
    )
    deer_mem_file.write_text(text, encoding="utf-8")

    monkeypatch.setenv("DEERMEM_DATA_DIR", str(tmp_path / "data"))
    monkeypatch.syspath_prepend(str(tmp_path))
    try:
        mod = importlib.import_module("otheragent_deermem.deer_mem")
        assert hasattr(mod, "DeerMem")
        dm = mod.DeerMem(backend_config=None)  # zero config, self._llm=None
        dm.import_memory(
            {"version": "1.0", "lastUpdated": "", "user": {}, "history": {}, "facts": [{"id": "f", "content": "y", "category": "c", "confidence": 0.5, "createdAt": "", "source": "m"}]},
            user_id="ua",
        )
        assert "y" in dm.get_context(user_id="ua")
    finally:
        for k in [k for k in list(sys.modules) if k.startswith("otheragent_deermem") or k == "otheragent"]:
            sys.modules.pop(k, None)


def test_per_user_memory_path_matches_host_safe_user_id(deermem_data_dir):
    """Pin the per-user memory path across the abstraction.

    DeerMem writes memory to ``{storage_path}/users/{safe_user_id}/memory.json``
    where ``safe_user_id`` is byte-identical to the host's ``make_safe_user_id``.
    The factory injects ``runtime_home()`` (= base_dir) as ``storage_path``, so
    the on-disk path is ``{base_dir}/users/{uid}/memory.json`` -- identical to
    pre-abstraction. This locks that equivalence so a future change to DeerMem's
    path / safe_user_id logic can't silently orphan existing per-user memory
    (risk:high, persistent state).
    """
    from deerflow.config.paths import make_safe_user_id

    user_id = "test-user-123@example.com"
    # storage_path mirrors what the host factory injects (runtime_home / base_dir)
    dm = DeerMem(backend_config={"storage_path": str(deermem_data_dir)})
    dm.create_fact("User prefers concise answers", category="preference", agent_name="default", user_id=user_id)

    expected_safe = make_safe_user_id(user_id)
    expected_file = deermem_data_dir / "users" / expected_safe / "memory.json"
    assert expected_file.is_file(), f"memory not at expected per-user path: {expected_file}"
    # DeerMem used the host-identical safe_user_id (not some other encoding).
    user_dirs = [p.name for p in (deermem_data_dir / "users").iterdir() if p.is_dir()]
    assert user_dirs == [expected_safe], f"safe_user_id diverged from host: {user_dirs}"


def test_create_fact_trims_to_max_and_signals_eviction(deermem_data_dir):
    """create_fact enforces max_facts and signals eviction via None fact_id.

    Regression: the vendored ``create_memory_fact`` only appended (no trim), so
    manual / tool adds could grow memory past max_facts. Now it trims (highest
    confidence wins) and returns ``None`` when the cap evicts the new fact, so
    the tool reports "not stored" instead of a dangling id + false "added".
    """
    # DeerMemConfig enforces max_facts >= 10, so fill the cap with 10 high-conf facts.
    dm = DeerMem(backend_config={"max_facts": 10, "storage_path": str(deermem_data_dir)})
    for i in range(10):
        _, fid = dm.create_fact(f"high{i}", category="context", confidence=0.9, user_id="u1")
        assert fid is not None

    # Cap is full (10 facts); a lower-confidence 11th is evicted, not stored.
    memory_data, evicted_id = dm.create_fact("low_evicted", category="context", confidence=0.1, user_id="u1")
    assert evicted_id is None
    assert "low_evicted" not in {f["content"] for f in memory_data["facts"]}
    assert len(memory_data["facts"]) == 10


def test_search_survives_non_float_confidence(deermem_data_dir):
    """DeerMem.search ranks by _coerce_source_confidence, so non-float stored
    confidence (null / string / non-numeric, reachable via import / legacy) must
    not crash the sort. Re-adds the regression guard deleted with the monolithic
    test_search_memory_facts_sort_survives_non_float_stored_confidence."""
    dm = DeerMem(backend_config={"storage_path": str(deermem_data_dir)})
    # create_fact validates confidence to float, so seed non-float via import
    # (simulating imported / legacy data that bypasses _validate_confidence).
    dm.import_memory(
        {
            "user": {},
            "history": {},
            "facts": [
                {"id": "a", "content": "alpha matching query", "confidence": None},
                {"id": "b", "content": "bravo matching query", "confidence": "0.9"},
                {"id": "c", "content": "charlie matching query", "confidence": "high"},
            ],
        },
        user_id="u1",
    )
    results = dm.search("query", top_k=10, user_id="u1")
    # No TypeError; all three match "query"; ranked by coerced confidence desc:
    # b("0.9"->0.9) > a(None->0.5)=c("high"->0.5), stable so a before c.
    assert [r["id"] for r in results] == ["b", "a", "c"]


def test_create_fact_category_round_trips_into_search_and_documents(deermem_data_dir):
    """A non-core category label must survive the write and stay searchable.

    Regression: ``storage._normalize_category`` rewrote every label outside
    CORE_CATEGORIES to "other" (parking the original in a write-only
    ``categoryExtension`` field), so a fact added via
    ``memory_add(category="personal")`` -- an example label in the memory_add
    docstring -- was persisted as "other", never matched
    ``memory_search(category="personal")`` ("Only facts with this exact category
    are returned"), and kept reporting "other" in get_memory documents.
    """
    dm = DeerMem(backend_config={"storage_path": str(deermem_data_dir)})
    memory_data, fact_id = dm.create_fact(
        "The user lives in Paris",
        category="personal",
        confidence=0.9,
        user_id="u1",
    )
    assert fact_id is not None

    assert next(f for f in memory_data["facts"] if f["id"] == fact_id)["category"] == "personal"
    document = dm.get_memory(user_id="u1")
    assert next(f for f in document["facts"] if f["id"] == fact_id)["category"] == "personal"

    results = dm.search("Paris", top_k=10, user_id="u1", category="personal")
    assert [f["id"] for f in results] == [fact_id]


def test_interrupted_legacy_migration_retry_recovers_aliased_category(deermem_data_dir, monkeypatch):
    """Regression: a migration interrupted after its Markdown commit recovers.

    An older build (pre-native-category ``_normalize_category``) committed a
    per-agent legacy JSON migration and stopped before deleting the JSON.  Its
    Markdown fact holds ``category="other"`` + ``categoryExtension="personal"``
    while the retained JSON holds ``category="personal"``.  The next build's
    migration candidate keeps "personal" natively, so the retry must recognize
    the alias as the same fact and finish recovery (delete the JSON) instead of
    raising a migration conflict that fails every read and retry for the agent
    (HTTP 500 at the Gateway).  A genuinely different label must still conflict.
    """
    dm = DeerMem(backend_config={"storage_path": str(deermem_data_dir)})
    memory_path = dm._storage._get_memory_file_path("researcher", user_id="u1")
    legacy_path = FileMemoryStorage._legacy_agent_memory_path(dm._storage, memory_path, "researcher")
    legacy_path.parent.mkdir(parents=True)
    legacy_path.write_text(
        json.dumps(
            {
                "version": "1.0",
                "lastUpdated": "2026-09-01T00:00:00Z",
                "user": {},
                "history": {},
                "facts": [
                    {
                        "id": "fact_interrupted_personal",
                        "content": "The user's sister is called Lea",
                        "category": "personal",
                        "confidence": 0.9,
                        "source": "manual",
                        "createdAt": "2026-09-01T00:00:00Z",
                        "revision": 1,
                    }
                ],
            }
        ),
        encoding="utf-8",
    )
    retained = legacy_path.read_bytes()

    native_normalize = deermem_storage._normalize_category
    monkeypatch.setattr(deermem_storage, "_normalize_category", _legacy_alias_normalize_category)
    dm.get_memory(agent_name="researcher", user_id="u1")  # the old Gateway commits the migration
    legacy_path.write_bytes(retained)  # ... and stops before the unlink
    monkeypatch.setattr(deermem_storage, "_normalize_category", native_normalize)

    # The retry must complete recovery instead of raising a conflict.
    document = dm.get_memory(agent_name="researcher", user_id="u1")
    assert not legacy_path.exists()
    fact = next(f for f in document["facts"] if f["id"] == "fact_interrupted_personal")
    # The already-committed Markdown fact is adopted as-is, not rewritten.
    assert (fact["category"], fact.get("categoryExtension")) == ("other", "personal")

    # Recovery must not relax genuine conflicts: a retained JSON label that
    # disagrees with the committed alias still stops the migration and keeps
    # the legacy file for a human decision.
    payload = json.loads(retained)
    payload["facts"][0]["category"] = "work"
    legacy_path.write_text(json.dumps(payload), encoding="utf-8")
    with pytest.raises(MemoryCorruptionError):
        dm.get_memory(agent_name="researcher", user_id="u1")
    assert legacy_path.exists()


def test_update_legacy_aliased_fact_to_native_category_clears_extension(deermem_data_dir, monkeypatch):
    """Regression: rewriting a legacy "other"+extension fact drops the stale extension.

    ``_normalize_fact`` deep-copies every key, so a legacy fact persisted as
    ``category="other"`` + ``categoryExtension="personal"`` that is updated to
    the now-native ``category="personal"`` used to keep the stale extension --
    contradicting the Gateway contract that the field is only set "when
    category is 'other'".  The update must clear it, and the rewritten fact
    must be findable by its native label.
    """
    dm = DeerMem(backend_config={"storage_path": str(deermem_data_dir)})
    native_normalize = deermem_storage._normalize_category
    monkeypatch.setattr(deermem_storage, "_normalize_category", _legacy_alias_normalize_category)
    _, fact_id = dm.create_fact(
        "The user's sister is called Lea",
        category="personal",
        agent_name="researcher",
        user_id="u1",
    )
    monkeypatch.setattr(deermem_storage, "_normalize_category", native_normalize)

    stored = next(f for f in dm.get_memory(agent_name="researcher", user_id="u1")["facts"] if f["id"] == fact_id)
    assert (stored["category"], stored.get("categoryExtension")) == ("other", "personal")

    updated = dm.update_fact(fact_id, category="personal", agent_name="researcher", user_id="u1")

    fact = next(f for f in updated["facts"] if f["id"] == fact_id)
    assert fact["category"] == "personal"
    assert "categoryExtension" not in fact
    assert fact["revision"] == 2
    results = dm.search("sister", top_k=10, user_id="u1", agent_name="researcher", category="personal")
    assert [f["id"] for f in results] == [fact_id]


def test_query_search_records_access_but_default_injection_does_not(deermem_data_dir):
    dm = DeerMem(
        backend_config={
            "storage_path": str(deermem_data_dir),
            "retrieval_adapter": "",
            "token_counting": "char",
            "fact_eviction_policy": "hybrid-v1",
        }
    )
    dm.create_fact(
        "User prefers concise answers",
        category="preference",
        confidence=0.8,
        user_id="u1",
    )

    dm.get_context(user_id="u1")
    assert dm._storage.get_fact_usage(agent_name="__default__", user_id="u1") == {}

    results = dm.search("concise", user_id="u1")
    usage = dm._storage.get_fact_usage(agent_name="__default__", user_id="u1")
    assert len(results) == 1
    fact_id = results[0]["id"]
    assert usage[fact_id]["accessCount"] == 1

    dm.delete_fact(fact_id, user_id="u1")
    dm._storage.record_fact_accesses(
        [fact_id],
        agent_name="__default__",
        user_id="u1",
    )
    assert dm._storage.get_fact_usage(agent_name="__default__", user_id="u1") == {}


def test_default_confidence_policy_does_not_collect_hybrid_usage(deermem_data_dir):
    dm = DeerMem(
        backend_config={
            "storage_path": str(deermem_data_dir),
            "retrieval_adapter": "",
            "token_counting": "char",
        }
    )
    dm.create_fact("User prefers concise answers", user_id="u1")

    assert dm.search("concise", user_id="u1")
    assert dm._storage.get_fact_usage(agent_name="__default__", user_id="u1") == {}


def test_hybrid_capacity_keeps_recent_confirmation_over_stale_confidence(deermem_data_dir):
    dm = DeerMem(
        backend_config={
            "storage_path": str(deermem_data_dir),
            "retrieval_adapter": "",
            "fact_eviction_policy": "hybrid-v1",
            "max_facts": 10,
        }
    )
    dm.import_memory(
        {
            "user": {},
            "history": {},
            "facts": [
                *[
                    {
                        "id": f"high_{index}",
                        "content": f"high {index}",
                        "category": "preference",
                        "confidence": 0.99,
                        "createdAt": "2026-08-12T00:00:00Z",
                    }
                    for index in range(8)
                ],
                {
                    "id": "stale_high",
                    "content": "stale high confidence",
                    "category": "preference",
                    "confidence": 0.9,
                    "createdAt": "2025-01-01T00:00:00Z",
                },
                {
                    "id": "confirmed_recently",
                    "content": "recently confirmed preference",
                    "category": "preference",
                    "confidence": 0.7,
                    "createdAt": "2025-01-01T00:00:00Z",
                    "lastConfirmedAt": "2026-08-12T00:00:00Z",
                },
            ],
        },
        user_id="u1",
    )

    memory, created_id = dm.create_fact(
        "new useful context",
        category="context",
        confidence=0.8,
        user_id="u1",
    )

    kept_ids = {fact["id"] for fact in memory["facts"]}
    assert created_id is not None
    assert "confirmed_recently" in kept_ids
    assert "stale_high" not in kept_ids


def test_hybrid_import_applies_same_capacity_policy(deermem_data_dir):
    dm = DeerMem(
        backend_config={
            "storage_path": str(deermem_data_dir),
            "retrieval_adapter": "",
            "fact_eviction_policy": "hybrid-v1",
            "max_facts": 10,
        }
    )
    memory = dm.import_memory(
        {
            "user": {},
            "history": {},
            "facts": [
                *[
                    {
                        "id": f"high_{index}",
                        "content": f"high {index}",
                        "category": "preference",
                        "confidence": 0.99,
                        "createdAt": "2026-08-12T00:00:00Z",
                    }
                    for index in range(8)
                ],
                {
                    "id": "stale_high",
                    "content": "stale high confidence",
                    "category": "preference",
                    "confidence": 0.9,
                    "createdAt": "2025-01-01T00:00:00Z",
                },
                {
                    "id": "confirmed_recently",
                    "content": "recently confirmed preference",
                    "category": "preference",
                    "confidence": 0.7,
                    "createdAt": "2025-01-01T00:00:00Z",
                    "lastConfirmedAt": "2026-08-12T00:00:00Z",
                },
                {
                    "id": "new_context",
                    "content": "new useful context",
                    "category": "context",
                    "confidence": 0.8,
                    "createdAt": "2026-08-12T00:00:00Z",
                },
            ],
        },
        user_id="u1",
    )

    kept_ids = {fact["id"] for fact in memory["facts"]}
    assert "confirmed_recently" in kept_ids
    assert "stale_high" not in kept_ids


def test_hybrid_capacity_reserves_correction_and_writes_audit(deermem_data_dir):
    dm = DeerMem(
        backend_config={
            "storage_path": str(deermem_data_dir),
            "retrieval_adapter": "",
            "fact_eviction_policy": "hybrid-v1",
            "max_facts": 10,
        }
    )
    dm.import_memory(
        {
            "user": {},
            "history": {},
            "facts": [
                {
                    "id": f"preference_{index}",
                    "content": f"preference {index}",
                    "category": "preference",
                    "confidence": 0.99,
                    "createdAt": "2026-08-12T00:00:00Z",
                }
                for index in range(10)
            ],
        },
        user_id="u1",
    )

    memory, correction_id = dm.create_fact(
        "Never repeat this corrected behavior",
        category="correction",
        confidence=0.1,
        user_id="u1",
    )

    assert correction_id is not None
    assert correction_id in {fact["id"] for fact in memory["facts"]}
    memory_path = dm._storage._get_memory_file_path("__default__", user_id="u1")
    audit_path = memory_path.parent / "agents" / "__default__" / ".metadata" / "eviction-audit.json"
    assert audit_path.is_file()
    assert "Never repeat this corrected behavior" not in audit_path.read_text()


def test_confidence_policy_shadow_does_not_change_actual_selection(deermem_data_dir):
    dm = DeerMem(
        backend_config={
            "storage_path": str(deermem_data_dir),
            "retrieval_adapter": "",
            "fact_eviction_shadow_enabled": True,
            "max_facts": 10,
        }
    )
    facts = [
        *[
            {
                "id": f"high_{index}",
                "content": f"high {index}",
                "category": "preference",
                "confidence": 0.99,
                "createdAt": "2026-08-12T00:00:00Z",
            }
            for index in range(8)
        ],
        {
            "id": "stale_high",
            "content": "stale high confidence",
            "category": "preference",
            "confidence": 0.9,
            "createdAt": "2025-01-01T00:00:00Z",
        },
        {
            "id": "confirmed_recently",
            "content": "recently confirmed preference",
            "category": "preference",
            "confidence": 0.7,
            "createdAt": "2025-01-01T00:00:00Z",
            "lastConfirmedAt": "2026-08-12T00:00:00Z",
        },
    ]
    dm.import_memory({"user": {}, "history": {}, "facts": facts}, user_id="u1")

    memory, _ = dm.create_fact("new useful context", confidence=0.8, user_id="u1")

    kept_ids = {fact["id"] for fact in memory["facts"]}
    assert "stale_high" in kept_ids
    assert "confirmed_recently" not in kept_ids
    memory_path = dm._storage._get_memory_file_path("__default__", user_id="u1")
    audit = (memory_path.parent / "agents" / "__default__" / ".metadata" / "eviction-audit.json").read_text()
    assert '"disagrees": true' in audit


def test_clear_memory_removes_usage_and_eviction_sidecars(deermem_data_dir):
    dm = DeerMem(
        backend_config={
            "storage_path": str(deermem_data_dir),
            "retrieval_adapter": "",
            "max_facts": 10,
        }
    )
    for index in range(10):
        dm.create_fact(f"query fact {index}", confidence=0.9, user_id="u1")
    dm.search("query", user_id="u1")
    dm.create_fact("capacity rejected", confidence=0.1, user_id="u1")
    memory_path = dm._storage._get_memory_file_path("__default__", user_id="u1")
    metadata_path = memory_path.parent / "agents" / "__default__" / ".metadata"
    assert metadata_path.is_dir()

    dm.clear_memory(user_id="u1")

    assert not metadata_path.exists()


def test_is_human_clarification_response_matches_host_read():
    """The standalone mirror must agree with the host's read_human_input_response
    so hidden-message filtering doesn't diverge between production (host hook) and
    standalone / test (mirror default). Pins drift (#5)."""

    from deerflow.agents.human_input import read_human_input_response
    from deerflow.agents.memory.backends.deermem.deermem.core.message_processing import _is_human_clarification_response

    def payload(**overrides):
        base = {"version": 1, "kind": "human_input_response", "source": "s", "request_id": "r", "value": "v", "response_kind": "text"}
        base.update(overrides)
        return {"human_input_response": base}

    cases = [
        {},
        {"human_input_response": {}},
        payload(),  # valid text response
        payload(response_kind="option", option_id="o1"),  # valid option response
        payload(response_kind="option"),  # option without option_id -> not valid
        payload(value=""),  # empty value -> not valid
        payload(source=""),  # empty source -> not valid
        payload(version=2),  # wrong version -> not valid
        payload(kind="other"),  # wrong kind -> not valid
        {"human_input_response": "not a mapping"},
        {"other_key": 1},  # no human_input_response key
    ]
    for ak in cases:
        host_keeps = read_human_input_response(ak) is not None
        mirror_keeps = _is_human_clarification_response(ak)
        assert host_keeps == mirror_keeps, f"divergence on {ak!r}: host={host_keeps} mirror={mirror_keeps}"


def test_build_llm_returns_none_when_no_model_configured():
    """Zero-config (no model_config, or model_config with no model) -> None.
    Non-LLM ops still work; an update raises at runtime."""
    from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemModelConfig
    from deerflow.agents.memory.backends.deermem.deermem.core.llm import build_llm

    assert build_llm(None) is None
    assert build_llm(DeerMemModelConfig()) is None  # model=None default


def test_build_llm_degrades_to_none_on_init_failure(caplog):
    """build_llm degrades to None (with a WARNING) when init_chat_model fails,
    mirroring _host_default_llm -- so a misconfigured explicit ``model`` does
    NOT crash app startup. Memory CRUD/read/search still work; extraction is
    disabled; an update raises at runtime with the underlying error logged."""
    from unittest.mock import patch

    from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemModelConfig
    from deerflow.agents.memory.backends.deermem.deermem.core.llm import build_llm

    model_config = DeerMemModelConfig(provider="openai", model="bogus-model", api_key="k")
    llm_logger = "deerflow.agents.memory.backends.deermem.deermem.core.llm"

    with patch("langchain.chat_models.init_chat_model", side_effect=RuntimeError("boom")):
        with caplog.at_level("WARNING", logger=llm_logger):
            result = build_llm(model_config)

    assert result is None
    assert any("build_llm failed" in r.message for r in caplog.records)


def test_from_backend_config_warns_on_unknown_keys(caplog):
    """Unknown backend_config keys log a WARNING so a typo (e.g. ``storage_pat``
    missing the ``h``) does not silently fall back to the default and write
    memory to an unintended location. Mirrors the host layer's
    load_memory_config_from_dict warning."""
    from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig

    cfg_logger = "deerflow.agents.memory.backends.deermem.deermem.config"
    with caplog.at_level("WARNING", logger=cfg_logger):
        cfg = DeerMemConfig.from_backend_config({"storage_path": "/tmp/x", "storage_pat": "/tmp/y"})

    # known key parsed; unknown key ignored but warned about
    assert cfg.storage_path == "/tmp/x"
    assert any("Unknown backend_config keys" in r.message for r in caplog.records)
    assert any("storage_pat" in r.message for r in caplog.records)


def test_from_backend_config_silent_on_known_keys(caplog):
    """No warning when every key is known (regression guard for the typo warning)."""
    from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig

    cfg_logger = "deerflow.agents.memory.backends.deermem.deermem.config"
    with caplog.at_level("WARNING", logger=cfg_logger):
        DeerMemConfig.from_backend_config({"storage_path": "/tmp/x", "max_facts": 20})
    assert not any("Unknown backend_config keys" in r.message for r in caplog.records)


def test_from_backend_config_null_values_fall_back_to_defaults():
    """Explicit YAML ``null`` values must behave like omitted keys, not crash.

    ``config.example.yaml`` ships ``backend_config.model:`` as a bare key with
    commented children, which YAML parses to ``None`` (and ``make
    config-upgrade`` writes it out as an explicit ``model: null``). Non-Optional
    fields like ``model: DeerMemModelConfig`` reject an explicit ``None`` even
    though the omitted key would use the field default — so the shipped example
    config crashed every run with a DeerMemConfig ValidationError."""
    from deerflow.agents.memory.backends.deermem.deermem.config import (
        DeerMemConfig,
        DeerMemModelConfig,
    )

    cfg = DeerMemConfig.from_backend_config({"model": None, "debounce_seconds": None, "storage_path": "/tmp/x"})

    # None entries fall back to field defaults; real values still parse
    assert isinstance(cfg.model, DeerMemModelConfig)
    assert cfg.model.model is None  # default = no extraction LLM configured
    assert cfg.debounce_seconds == DeerMemConfig().debounce_seconds
    assert cfg.storage_path == "/tmp/x"


def test_from_backend_config_null_values_do_not_warn_as_unknown(caplog):
    """Dropped ``None`` entries are known keys — they must not trip the
    unknown-key typo warning."""
    from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig

    cfg_logger = "deerflow.agents.memory.backends.deermem.deermem.config"
    with caplog.at_level("WARNING", logger=cfg_logger):
        DeerMemConfig.from_backend_config({"model": None})
    assert not any("Unknown backend_config keys" in r.message for r in caplog.records)


def test_apply_updates_cognitive_style_and_fact_category(deermem_data_dir) -> None:
    dm = DeerMem(backend_config=None)
    current_memory = dm.get_memory(user_id="cognitive-user")

    result = dm._updater._apply_updates(
        current_memory,
        {
            "user": {
                "cognitiveStyle": {
                    "summary": "Prefers conclusions first, then details.",
                    "shouldUpdate": True,
                    "scope": "user",
                    "authority": "descriptive",
                }
            },
            "newFacts": [
                {
                    "content": "User prefers conclusions before implementation details.",
                    "category": "cognitive",
                    "confidence": 0.92,
                    "scope": "user",
                    "durability": "durable",
                    "authority": "descriptive",
                }
            ],
        },
        thread_id="thread-cognitive",
    )

    assert result["user"]["cognitiveStyle"]["summary"] == "Prefers conclusions first, then details."
    assert result["user"]["cognitiveStyle"]["updatedAt"]
    assert result["facts"][0]["category"] == "cognitive"
    assert result["facts"][0]["source"] == "thread-cognitive"


def test_import_memory_persists_normalized_legacy_payload(deermem_data_dir) -> None:
    dm = DeerMem(backend_config=None)
    legacy = {
        "version": "1.0",
        "lastUpdated": "",
        "user": {},
        "history": {},
        "facts": [{"content": "User prefers conclusions first.", "category": "cognitive"}],
    }

    result = dm.import_memory(legacy, user_id="legacy-user")

    assert result["user"]["cognitiveStyle"] == {"summary": "", "updatedAt": ""}
    assert result["facts"][0]["id"].startswith("fact_")
    assert result["facts"][0]["category"] == "cognitive"
    assert result["facts"][0]["confidence"] == 0.5
    assert result["facts"][0]["createdAt"] == ""
    assert result["facts"][0]["source"] == "unknown"


@pytest.mark.parametrize("facts", [[{"id": "keep", "content": "   "}], [None], {}, None, "missing", [{"id": "keep", "content": 42}], [{"id": "new", "content": "valid"}, {}]])
def test_replacement_import_rejects_unrecoverable_facts_without_writes(deermem_data_dir, facts):
    dm = DeerMem(backend_config=None)
    before = dm.import_memory({"user": {}, "history": {}, "facts": [{"id": "keep", "content": "Saved preference"}]}, user_id="alice")
    snapshot = {str(path.relative_to(deermem_data_dir)): path.read_bytes() for path in deermem_data_dir.rglob("*") if path.is_file()}

    payload = {"user": {}, "history": {}, "facts": facts}
    if facts == "missing":
        payload.pop("facts")
    with pytest.raises(ValueError, match="facts"):
        dm.import_memory(payload, user_id="alice")

    assert {str(path.relative_to(deermem_data_dir)): path.read_bytes() for path in deermem_data_dir.rglob("*") if path.is_file()} == snapshot
    reloaded = DeerMem(backend_config=None).get_memory(user_id="alice")
    assert reloaded["facts"] == before["facts"]
    assert reloaded["revision"] == before["revision"]