fix(uploads): handle UTF-8 BOM in document summaries (#5541)

Co-authored-by: NEEDI <298523066+sherxlg-gif@users.noreply.github.com>

README.md

@@ -1532,6 +1532,8 @@ The built-in `grep` tool searches either one text file or all matching text file
 
 Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced code examples, so hashtags and code comments do not
 crowd out real document sections from the agent's heading preview.
+UTF-8 Markdown files with or without a byte-order mark (BOM) produce the same
+outlines and fallback previews, with original line numbers preserved.
 Outline titles are limited to 200 characters and fallback previews to 2,000
 characters per file, with truncation markers. Full uploaded files remain available
 for targeted reads.

backend/packages/harness/deerflow/utils/AGENTS.md

@@ -1,3 +1,10 @@
 ### Agent / Tool Assembly Off-Load
 
 Tool and agent assembly re-enters `get_available_tools()` and may block on MCP discovery, so the four async assembly entry points — `run_agent`'s `agent_factory` call, `task_tool`, durable batch `_execute_item`, and `abuild_checkpoint_state_accessor` — dispatch through `deerflow.utils.assembly_io.run_assembly`, a dedicated ContextVar-preserving bounded executor (`DEER_FLOW_ASSEMBLY_WORKERS`, default 8) rather than the loop's default executor; a hung MCP server therefore parks an assembly worker instead of queueing unrelated default-executor work, and the pool logs a warning when pending assemblies exceed the worker count. `tests/blocking_io/test_tool_assembly_offloop.py` pins all four offloads plus the ContextVar propagation.
+
+### Uploaded Document Summaries
+
+`file_outline.py` reads outlines and fallback previews as `utf-8-sig` so an
+optional leading UTF-8 BOM cannot hide a first-line heading or code fence, or
+occupy a preview line. Preserve physical line numbers, embedded U+FEFF
+characters, and the original file bytes.

backend/packages/harness/deerflow/utils/file_outline.py

@@ -121,7 +121,7 @@ def extract_outline(md_path: Path) -> list[dict]:
     fence_char = ""
     fence_length = 0
     try:
-        with md_path.open(encoding="utf-8") as f:
+        with md_path.open(encoding="utf-8-sig") as f:
             for lineno, line in enumerate(f, 1):
                 fence = _CODE_FENCE_RE.match(line.rstrip("\r\n"))
                 if fence_char:
@@ -199,7 +199,7 @@ def extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]:
     preview: list[str] = []
     remaining_chars = _OUTLINE_PREVIEW_MAX_CHARS
     try:
-        with md_path.open(encoding="utf-8") as f:
+        with md_path.open(encoding="utf-8-sig") as f:
             for line in f:
                 stripped = line.strip()
                 if stripped:

backend/tests/test_file_outline_encoding.py

@@ -0,0 +1,38 @@
+"""UTF-8 signatures must not change model-visible document summaries."""
+
+from pathlib import Path
+
+import pytest
+
+from deerflow.utils.file_outline import extract_outline, extract_outline_for_file
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
+@pytest.mark.parametrize(
+    ("heading", "title"),
+    [("# 概述", "概述"), ("**PART I**", "PART I"), ("**1** **Introduction**", "1 Introduction")],
+)
+def test_first_heading_is_preserved_with_or_without_bom(tmp_path: Path, encoding: str, heading: str, title: str) -> None:
+    document = tmp_path / "report.md"
+    document.write_text(heading + "\n\n## Next section\n", encoding=encoding)
+
+    assert extract_outline(document) == [{"title": title, "line": 1}, {"title": "Next section", "line": 3}]
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
+@pytest.mark.parametrize("fence", ["```", "~~~"])
+def test_first_code_fence_is_recognized_with_or_without_bom(tmp_path: Path, encoding: str, fence: str) -> None:
+    document = tmp_path / "guide.md"
+    document.write_text(f"{fence}python\n# Code comment\n{fence}\n# Actual section\n", encoding=encoding)
+
+    assert extract_outline_for_file(document) == ([{"title": "Actual section", "line": 4}], [])
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
+def test_preview_skips_bom_only_line_and_preserves_embedded_character(tmp_path: Path, encoding: str) -> None:
+    document = tmp_path / "notes.md"
+    document.write_text("\nFirst paragraph\nSecond\ufeffparagraph\n", encoding=encoding)
+    original = document.read_bytes()
+
+    assert extract_outline_for_file(document) == ([], ["First paragraph", "Second\ufeffparagraph"])
+    assert document.read_bytes() == original