fix(memory): drop out-of-range extraction confidence instead of failing the batch (#6971)

* fix(memory): drop out-of-range extraction confidence instead of failing the batch

A model-emitted confidence outside [0, 1] (a percentage such as 95, or 1.5)
passed extraction normalization, which only rejected non-finite values. The
value then reached storage, whose own validation raises for out-of-range
confidence, so one malformed field failed the whole update: the extraction
watermark never advanced and the same paid call repeated on every turn, while
every valid fact, summary and removal in the batch was lost each cycle.

Treat the range like the other malformed shapes in
_normalize_memory_update_fact: drop only that item and let the rest of the
update apply. Read paths already clamp stored corrupt values
(_coerce_source_confidence), so no read-side change is needed.

* fix(memory): bound the confidence drop and make it observable (review)

Review follow-ups on the out-of-range confidence fix:
- range-check integer confidence before float(), so a model emitting 10**400
  is dropped instead of raising OverflowError and failing the batch;
- log the raw value when a fact is dropped, so a systematically
  percentage-scale model is visible instead of silently losing every fact;
- document that a batch which also proposes factsToRemove still fails closed
  when any new fact is dropped (replacementFactIndex positions shift), and
  pin that boundary with a test. Values are documented as dropped, never
  rescaled.

backend/packages/harness/deerflow/agents/memory/AGENTS.md

@@ -258,6 +258,7 @@ Extraction labels proposals with `scope`, `durability`, and `authority`.
 Automatic writes accept only user-scoped, durable, descriptive facts.
 Summary prose must be user-scoped and descriptive.
 Missing labels reject that item without stopping unrelated updates.
+An out-of-range `confidence` (including a model-emitted percentage) is rejected per item with a warning that names the raw value, and is never rescaled: a silent scale rewrite would change the confidence-threshold semantics. Storage's own `[0, 1]` validation must not fail the batch, because a failed batch keeps the extraction watermark and repeats the same paid call. A batch that also proposes `factsToRemove` still fails closed when any new fact is dropped, because removal `replacementFactIndex` values refer to newFacts positions.
 
 Contradiction removals include `id`, `scope`, `reason`, and optional `replacementFactIndex`.
 Task-scoped and project-scoped removals fail closed.

backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py

@@ -174,6 +174,14 @@ def _removal_scope_gate_reason(removal: dict[str, Any]) -> str | None:
     return None
 
 
+def _log_dropped_confidence(raw: object) -> None:
+    """Make a systematic out-of-range confidence visible without logging content."""
+    logger.warning(
+        "Dropping an extracted fact with out-of-range confidence %s; expected 0.0-1.0. A model emitting a percentage scale would lose every extracted fact.",
+        repr(raw)[:80],
+    )
+
+
 def _normalize_memory_update_fact(fact: Any) -> dict[str, Any] | None:
     """Normalize a single fact entry from a model-produced memory update."""
     if not isinstance(fact, dict):
@@ -198,14 +206,29 @@ def _normalize_memory_update_fact(fact: Any) -> dict[str, Any] | None:
             return None
         try:
             raw_confidence = float(raw_confidence)
-        except ValueError:
+        except (OverflowError, ValueError):
+            return None
+    elif isinstance(raw_confidence, int):
+        # Range-check before float(): an unbounded JSON integer such as 10**400
+        # raises OverflowError in float() and would fail the whole batch.
+        if not 0 <= raw_confidence <= 1:
+            _log_dropped_confidence(raw_confidence)
             return None
-    elif isinstance(raw_confidence, (int, float)):
         raw_confidence = float(raw_confidence)
+    elif isinstance(raw_confidence, float):
+        pass
     else:
         return None
 
-    if not math.isfinite(raw_confidence):
+    if not math.isfinite(raw_confidence) or not 0.0 <= raw_confidence <= 1.0:
+        # Out-of-range confidence (for example a model emitting 95 or 1.5) is
+        # malformed like every other shape above: drop this item instead of
+        # letting the storage layer reject the whole update. A rejected batch
+        # never advances the extraction watermark, so the same paid call would
+        # otherwise repeat on every turn. Values are dropped, never rescaled:
+        # silently rewriting the scale would change the confidence-threshold
+        # semantics, and the warning makes a percentage-scale model visible.
+        _log_dropped_confidence(raw_confidence)
         return None
 
     normalized_fact = {

backend/tests/test_memory_updater.py

@@ -1,5 +1,7 @@
 import asyncio
 import copy
+import json
+import logging
 import threading
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -16,6 +18,7 @@ from deerflow.agents.memory.backends.deermem.deermem.core.updater import (
     _build_staleness_section,
     _coerce_source_confidence,
     _extract_text,
+    _normalize_memory_update_fact,
     _parse_memory_update_response,
 )
 from deerflow.agents.memory.manager import LangfuseMemoryCallbacks
@@ -1889,6 +1892,81 @@ class TestNullConfidenceDoesNotBlockUpdates:
         assert _coerce_source_confidence({"confidence": 0.83}) == 0.83
 
 
+class TestOutOfRangeExtractionConfidenceDoesNotBlockUpdates:
+    """A model-emitted confidence outside [0, 1] must not fail the whole update.
+
+    Storage rejects an out-of-range confidence, so an unchecked proposal made
+    the whole extraction batch fail on every turn (the watermark never
+    advanced and the same paid call repeated). The extraction normalizer now
+    treats the range like every other malformed shape: the item is dropped with
+    a warning and the rest of the update still applies — except when the same
+    batch proposes ``factsToRemove``, where dropping a fact invalidates
+    ``replacementFactIndex`` positions and the parse still fails closed
+    (pinned below).
+    """
+
+    @pytest.mark.parametrize("confidence", [1.5, 95, -0.1, 10**400], ids=["above-one", "percent", "negative", "unbounded-int"])
+    def test_normalize_rule_drops_out_of_range_confidence(self, confidence: object) -> None:
+        normalized = _normalize_memory_update_fact({**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": confidence})
+        assert normalized is None
+
+    @pytest.mark.parametrize("confidence", [0.0, 0.5, 1.0])
+    def test_normalize_rule_keeps_in_range_confidence(self, confidence: float) -> None:
+        normalized = _normalize_memory_update_fact({**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": confidence})
+        assert normalized is not None
+        assert normalized["confidence"] == confidence
+
+    def test_update_memory_drops_the_bad_fact_and_keeps_the_rest(self, tmp_path) -> None:
+        from deerflow.agents.memory.backends.deermem.deermem.core.storage import FileMemoryStorage
+
+        response = {
+            "user": {},
+            "history": {},
+            "newFacts": [
+                {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "category": "preference", "confidence": 0.9},
+                {**_DURABLE_USER_FACT, "content": "User lives in Berlin", "category": "context", "confidence": 1.5},
+            ],
+            "factsToRemove": [],
+        }
+        storage = FileMemoryStorage(DeerMemConfig(storage_path=str(tmp_path)))
+        synthesized = MagicMock()
+        synthesized.content = json.dumps(response)
+        model = MagicMock()
+        model.invoke = MagicMock(return_value=synthesized)
+        model.ainvoke = AsyncMock(return_value=synthesized)
+        updater = _make_updater(
+            config=_memory_config(fact_confidence_threshold=0.7, max_facts=100),
+            storage=storage,
+            llm=model,
+        )
+        msg = MagicMock(type="human", content="Hello")
+        ai_msg = MagicMock(type="ai", content="Hi", tool_calls=[])
+
+        assert updater.update_memory([msg, ai_msg], thread_id="thread-confidence", agent_name="__default__") is True
+
+        stored = storage.load(agent_name="__default__")
+        assert [fact["content"] for fact in stored["facts"]] == ["User prefers dark mode"]
+
+    def test_out_of_range_drop_logs_the_raw_confidence(self, caplog) -> None:
+        with caplog.at_level(logging.WARNING, logger="deerflow.agents.memory.backends.deermem.deermem.core.updater"):
+            normalized = _normalize_memory_update_fact({**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": 95})
+        assert normalized is None
+        assert any("out-of-range confidence" in message and "95" in message for message in caplog.messages)
+
+    def test_removal_batch_still_fails_closed_when_a_fact_is_dropped(self) -> None:
+        payload = {
+            "user": {},
+            "history": {},
+            "factsToRemove": [{"id": "fact_old", "scope": "user", "reason": "Retracted by the user"}],
+            "newFacts": [
+                {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "confidence": 0.9},
+                {**_DURABLE_USER_FACT, "content": "User lives in Berlin", "confidence": 1.5},
+            ],
+        }
+        with pytest.raises(json.JSONDecodeError, match="Unsafe partial"):
+            _parse_memory_update_response(json.dumps(payload))
+
+
 class TestParseMemoryUpdateFactsToRemoveGate:
     """``factsToRemove`` is optional in the memory-update JSON acceptance gate.