From 18ca09b2dac76e26cbdcbbc6936be4f6f166283f Mon Sep 17 00:00:00 2001 From: yetuge <182123929+yetuge@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:27:04 +0800 Subject: [PATCH] fix(memory): parse a BOM-prefixed DeerMem fact instead of calling it corruption (#7116) * fix(memory): parse a BOM-prefixed DeerMem fact instead of calling it corruption Facts are hand-editable Markdown files, so a Windows editor can save one as "UTF-8 with BOM" -- the same exposure #7083 fixed for memory.json. The fact parser checks `text.startswith("---\n")` against a plain-utf-8 read, so the BOM makes a valid fact read as "missing YAML front matter" and raises MemoryStorageCorruption, which the module docs describe as taking down the whole agent. Read the fact with utf-8-sig, matching the summary loader and the SOUL.md / SKILL.md convention already recorded in AGENTS.md. `_parse_listed_fact` delegates here, so the directory-scan path is covered by the same change. Measured: the new BOM test fails on the base revision with exactly that "missing YAML front matter" corruption and passes at head; the no-BOM control passes both. 49 tests in test_memory_storage.py, 106 across the memory suites, ruff check/format clean. Mutation-checked: reverting to plain utf-8 fails only the BOM case. * test(memory): cover BOM facts through the public storage API --------- Co-authored-by: Willem Jiang --- README.md | 2 ++ .../harness/deerflow/agents/memory/AGENTS.md | 2 ++ .../backends/deermem/deermem/core/storage.py | 7 +++- backend/tests/test_memory_storage.py | 34 +++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ee8cb8c38b..354d216d5a 100644 --- a/README.md +++ b/README.md @@ -2900,6 +2900,8 @@ summary reads while keeping JSON writes and the existing UI. A hand-edited embedded backticks and later fenced notes are supported. Unparseable summary text is moved to `memory.json.corrupt-` for recovery before rebuilding. `storage_path` must point to the data root directory, not an existing JSON file. +Hand-edited DeerMem summaries and fact Markdown files may have a leading UTF-8 +BOM; reads accept it, while writes remain BOM-free UTF-8. Gateway shutdown drains memory updates before closing the backend, even when shutdown is cancelled. Config reload failures are logged without aborting runtime diff --git a/backend/packages/harness/deerflow/agents/memory/AGENTS.md b/backend/packages/harness/deerflow/agents/memory/AGENTS.md index f38256225c..6b1ef585a4 100644 --- a/backend/packages/harness/deerflow/agents/memory/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/memory/AGENTS.md @@ -141,6 +141,8 @@ fence so embedded backticks and later fenced notes stay lossless. Parser and storage regressions belong in `tests/test_memory_storage.py`; configure a data root directory, not the existing manifest file. Legacy v1 reads still migrate and advance the revision. Unparseable summary text is quarantined before rebuild. +Summary and canonical fact reads strip a leading UTF-8 BOM; writes use UTF-8. +Fact BOM regressions use public save/reload and preserve content and scope. `FileMemoryStorage` owns canonical storage and the retrieval adapter. Do not reach into its private adapter state from higher layers. diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py index 9f01eeef71..8dee4bfc57 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/storage.py @@ -406,7 +406,12 @@ def _render_fact_markdown(fact: dict[str, Any]) -> bytes: def _parse_fact_markdown(path: Path) -> dict[str, Any]: try: - text = path.read_text(encoding="utf-8") + # ``utf-8-sig`` drops a leading BOM, matching ``_load_memory_file``. + # Facts are hand-editable ``.md`` files, so a Windows editor can add one; + # the ``startswith("---\n")`` check below would then read a valid fact as + # "missing YAML front matter" and raise MemoryStorageCorruption over an + # encoding artifact. + text = path.read_text(encoding="utf-8-sig") if not text.startswith("---\n"): raise ValueError("missing YAML front matter") front, body = text[4:].split("\n---\n", 1) diff --git a/backend/tests/test_memory_storage.py b/backend/tests/test_memory_storage.py index 3d81057a5e..73d1dd7d20 100644 --- a/backend/tests/test_memory_storage.py +++ b/backend/tests/test_memory_storage.py @@ -430,3 +430,37 @@ class TestMarkdownMemoryStorage: ) def test_invalid_or_non_object_fenced_summary_rejected(self, body): assert mf._parse_markdown_memory(body) is None + + +class TestFactMarkdownBom: + """Hand-edited canonical facts must survive the complete storage read path.""" + + def test_fact_saved_with_utf8_bom_round_trips_through_storage(self, tmp_path): + storage = _storage_at(tmp_path / "memory.json") + memory = create_empty_memory() + memory["facts"] = [ + { + "id": "fact_bom", + "title": "Shared decision", + "content": "Use Postgres. Preserve an interior \ufeff character.", + "category": "context", + "confidence": 0.9, + "createdAt": "2026-10-19T00:00:00Z", + "source": "manual", + } + ] + assert storage.save(memory, "agent-a", user_id="alice") + before = storage.load("agent-a", user_id="alice") + (fact_path,) = tmp_path.rglob("fact_bom.md") + original = fact_path.read_bytes() + assert not original.startswith(b"\xef\xbb\xbf") + fact_path.write_bytes(b"\xef\xbb\xbf" + original) + + reloaded = storage.reload("agent-a", user_id="alice") + + assert reloaded["facts"] == before["facts"] + (fact,) = reloaded["facts"] + assert fact["id"] == "fact_bom" + assert fact["title"] == "Shared decision" + assert fact["content"] == memory["facts"][0]["content"] + assert fact["scope"] == {"userId": "alice", "agentName": "agent-a"}