diff --git a/backend/packages/harness/deerflow/agents/memory/AGENTS.md b/backend/packages/harness/deerflow/agents/memory/AGENTS.md index 10cdc1f12d..f38256225c 100644 --- a/backend/packages/harness/deerflow/agents/memory/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/memory/AGENTS.md @@ -258,6 +258,7 @@ Extraction labels proposals with `scope`, `durability`, and `authority`. Automatic writes accept only user-scoped, durable, descriptive facts. Summary prose must be user-scoped and descriptive. Missing labels reject that item without stopping unrelated updates. +An out-of-range `confidence` (including a model-emitted percentage) is rejected per item with a warning that names the raw value, and is never rescaled: a silent scale rewrite would change the confidence-threshold semantics. Storage's own `[0, 1]` validation must not fail the batch, because a failed batch keeps the extraction watermark and repeats the same paid call. A batch that also proposes `factsToRemove` still fails closed when any new fact is dropped, because removal `replacementFactIndex` values refer to newFacts positions. Contradiction removals include `id`, `scope`, `reason`, and optional `replacementFactIndex`. Task-scoped and project-scoped removals fail closed. diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py index 362d3f08f1..40aa8d2516 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py @@ -174,6 +174,14 @@ def _removal_scope_gate_reason(removal: dict[str, Any]) -> str | None: return None +def _log_dropped_confidence(raw: object) -> None: + """Make a systematic out-of-range confidence visible without logging content.""" + logger.warning( + "Dropping an extracted fact with out-of-range confidence %s; expected 0.0-1.0. A model emitting a percentage scale would lose every extracted fact.", + repr(raw)[:80], + ) + + def _normalize_memory_update_fact(fact: Any) -> dict[str, Any] | None: """Normalize a single fact entry from a model-produced memory update.""" if not isinstance(fact, dict): @@ -198,14 +206,29 @@ def _normalize_memory_update_fact(fact: Any) -> dict[str, Any] | None: return None try: raw_confidence = float(raw_confidence) - except ValueError: + except (OverflowError, ValueError): + return None + elif isinstance(raw_confidence, int): + # Range-check before float(): an unbounded JSON integer such as 10**400 + # raises OverflowError in float() and would fail the whole batch. + if not 0 <= raw_confidence <= 1: + _log_dropped_confidence(raw_confidence) return None - elif isinstance(raw_confidence, (int, float)): raw_confidence = float(raw_confidence) + elif isinstance(raw_confidence, float): + pass else: return None - if not math.isfinite(raw_confidence): + if not math.isfinite(raw_confidence) or not 0.0 <= raw_confidence <= 1.0: + # Out-of-range confidence (for example a model emitting 95 or 1.5) is + # malformed like every other shape above: drop this item instead of + # letting the storage layer reject the whole update. A rejected batch + # never advances the extraction watermark, so the same paid call would + # otherwise repeat on every turn. Values are dropped, never rescaled: + # silently rewriting the scale would change the confidence-threshold + # semantics, and the warning makes a percentage-scale model visible. + _log_dropped_confidence(raw_confidence) return None normalized_fact = { diff --git a/backend/tests/test_memory_updater.py b/backend/tests/test_memory_updater.py index 2f856f7a99..d1c268d13f 100644 --- a/backend/tests/test_memory_updater.py +++ b/backend/tests/test_memory_updater.py @@ -1,5 +1,7 @@ import asyncio import copy +import json +import logging import threading from unittest.mock import AsyncMock, MagicMock, patch @@ -16,6 +18,7 @@ from deerflow.agents.memory.backends.deermem.deermem.core.updater import ( _build_staleness_section, _coerce_source_confidence, _extract_text, + _normalize_memory_update_fact, _parse_memory_update_response, ) from deerflow.agents.memory.manager import LangfuseMemoryCallbacks @@ -1889,6 +1892,81 @@ class TestNullConfidenceDoesNotBlockUpdates: assert _coerce_source_confidence({"confidence": 0.83}) == 0.83 +class TestOutOfRangeExtractionConfidenceDoesNotBlockUpdates: + """A model-emitted confidence outside [0, 1] must not fail the whole update. + + Storage rejects an out-of-range confidence, so an unchecked proposal made + the whole extraction batch fail on every turn (the watermark never + advanced and the same paid call repeated). The extraction normalizer now + treats the range like every other malformed shape: the item is dropped with + a warning and the rest of the update still applies — except when the same + batch proposes ``factsToRemove``, where dropping a fact invalidates + ``replacementFactIndex`` positions and the parse still fails closed + (pinned below). + """ + + @pytest.mark.parametrize("confidence", [1.5, 95, -0.1, 10**400], ids=["above-one", "percent", "negative", "unbounded-int"]) + def test_normalize_rule_drops_out_of_range_confidence(self, confidence: object) -> None: + normalized = _normalize_memory_update_fact({**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": confidence}) + assert normalized is None + + @pytest.mark.parametrize("confidence", [0.0, 0.5, 1.0]) + def test_normalize_rule_keeps_in_range_confidence(self, confidence: float) -> None: + normalized = _normalize_memory_update_fact({**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": confidence}) + assert normalized is not None + assert normalized["confidence"] == confidence + + def test_update_memory_drops_the_bad_fact_and_keeps_the_rest(self, tmp_path) -> None: + from deerflow.agents.memory.backends.deermem.deermem.core.storage import FileMemoryStorage + + response = { + "user": {}, + "history": {}, + "newFacts": [ + {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "category": "preference", "confidence": 0.9}, + {**_DURABLE_USER_FACT, "content": "User lives in Berlin", "category": "context", "confidence": 1.5}, + ], + "factsToRemove": [], + } + storage = FileMemoryStorage(DeerMemConfig(storage_path=str(tmp_path))) + synthesized = MagicMock() + synthesized.content = json.dumps(response) + model = MagicMock() + model.invoke = MagicMock(return_value=synthesized) + model.ainvoke = AsyncMock(return_value=synthesized) + updater = _make_updater( + config=_memory_config(fact_confidence_threshold=0.7, max_facts=100), + storage=storage, + llm=model, + ) + msg = MagicMock(type="human", content="Hello") + ai_msg = MagicMock(type="ai", content="Hi", tool_calls=[]) + + assert updater.update_memory([msg, ai_msg], thread_id="thread-confidence", agent_name="__default__") is True + + stored = storage.load(agent_name="__default__") + assert [fact["content"] for fact in stored["facts"]] == ["User prefers dark mode"] + + def test_out_of_range_drop_logs_the_raw_confidence(self, caplog) -> None: + with caplog.at_level(logging.WARNING, logger="deerflow.agents.memory.backends.deermem.deermem.core.updater"): + normalized = _normalize_memory_update_fact({**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": 95}) + assert normalized is None + assert any("out-of-range confidence" in message and "95" in message for message in caplog.messages) + + def test_removal_batch_still_fails_closed_when_a_fact_is_dropped(self) -> None: + payload = { + "user": {}, + "history": {}, + "factsToRemove": [{"id": "fact_old", "scope": "user", "reason": "Retracted by the user"}], + "newFacts": [ + {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "confidence": 0.9}, + {**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": 1.5}, + ], + } + with pytest.raises(json.JSONDecodeError, match="Unsafe partial"): + _parse_memory_update_response(json.dumps(payload)) + + class TestParseMemoryUpdateFactsToRemoveGate: """``factsToRemove`` is optional in the memory-update JSON acceptance gate.