diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/markdown_storage.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/markdown_storage.py index 443ef9e3fb..07ff73810b 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/markdown_storage.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/markdown_storage.py @@ -53,7 +53,9 @@ class MarkdownMemoryStorage(FileMemoryStorage): if not path.exists(): return None try: - raw = path.read_text(encoding="utf-8") + # utf-8-sig, matching the base loader: a BOM does not make the + # summary unreadable, so it must not trigger the quarantine path. + raw = path.read_text(encoding="utf-8-sig") except (OSError, UnicodeError) as exc: logger.warning("Cannot read memory summary %s: %s", path, exc) return None diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py index aae958b3ad..9f01eeef71 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py @@ -1143,7 +1143,11 @@ class FileMemoryStorage(MemoryStorage): if not path.exists(): return None try: - value = json.loads(path.read_text(encoding="utf-8")) + # utf-8-sig: a summary hand-edited on Windows may carry a BOM. The + # payload is still valid JSON, so raising MemoryStorageCorruption + # here would brick the backend (and the opt-in Markdown loader + # would quarantine recoverable state) over an encoding artifact. + value = json.loads(path.read_text(encoding="utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeError) as exc: raise MemoryStorageCorruption(f"Failed to load global memory JSON {path}: {exc}") from exc if not isinstance(value, dict): diff --git a/backend/tests/test_memory_storage.py b/backend/tests/test_memory_storage.py index 6b534329de..3d81057a5e 100644 --- a/backend/tests/test_memory_storage.py +++ b/backend/tests/test_memory_storage.py @@ -117,6 +117,31 @@ class TestFileMemoryStorage: assert isinstance(memory, dict) assert memory["version"] == "1.0" + def test_summary_saved_with_utf8_bom_loads(self, tmp_path, monkeypatch): + """A summary hand-edited into "UTF-8 with BOM" (Windows Notepad / + PowerShell ``Set-Content -Encoding UTF8``) is valid JSON, not corruption: + the BOM is consumed instead of raising MemoryStorageCorruption.""" + monkeypatch.setenv("DEERMEM_DATA_DIR", str(tmp_path)) + memory_file = tmp_path / "memory.json" + manifest = create_empty_memory() + manifest["revision"] = 7 + manifest["user"]["workContext"] = {"summary": "edited in Notepad"} + memory_file.write_bytes(b"\xef\xbb\xbf" + json.dumps(manifest, ensure_ascii=False).encode("utf-8")) + assert memory_file.read_bytes().startswith(b"\xef\xbb\xbf"), "fixture must carry a real BOM" + storage = FileMemoryStorage(DeerMemConfig()) + loaded = storage.load() + assert loaded["user"]["workContext"]["summary"] == "edited in Notepad" + + def test_summary_without_bom_is_unaffected(self, tmp_path, monkeypatch): + """The BOM-tolerant decoder must not alter plain UTF-8 summaries.""" + monkeypatch.setenv("DEERMEM_DATA_DIR", str(tmp_path)) + memory_file = tmp_path / "memory.json" + manifest = create_empty_memory() + manifest["user"]["workContext"] = {"summary": "plain"} + memory_file.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + loaded = FileMemoryStorage(DeerMemConfig()).load() + assert loaded["user"]["workContext"]["summary"] == "plain" + def test_save_writes_to_file(self, tmp_path, monkeypatch): monkeypatch.setenv("DEERMEM_DATA_DIR", str(tmp_path)) memory_file = tmp_path / "memory.json" @@ -323,6 +348,22 @@ class TestMarkdownMemoryStorage: assert loaded["revision"] == expected_revision assert loaded["user"]["workContext"]["summary"] == "recovered" + def test_bom_prefixed_valid_json_is_not_quarantined(self, tmp_path): + """A BOM does not make the summary unreadable, so the tolerant loader + must parse it instead of quarantining recoverable state.""" + memory_file = tmp_path / "memory.json" + manifest = create_empty_memory() + manifest["version"] = "2.0" + manifest.pop("facts") + manifest["revision"] = 7 + manifest["user"]["workContext"] = {"summary": "kept"} + memory_file.write_bytes(b"\xef\xbb\xbf" + json.dumps(manifest, ensure_ascii=False).encode("utf-8")) + storage = self._markdown_storage_at(memory_file) + loaded = storage.load() + assert loaded["revision"] == 7 + assert loaded["user"]["workContext"]["summary"] == "kept" + assert not list(tmp_path.glob("memory.json.corrupt-*")), "a readable summary must not be quarantined" + def test_hand_edited_markdown_without_fence_is_quarantined(self, tmp_path): """Markdown without a fenced block cannot be mapped onto the manifest schema losslessly: load() must not crash AND must not return an