diff --git a/README.md b/README.md index 6fef6111c..2ba961540 100644 --- a/README.md +++ b/README.md @@ -1532,6 +1532,8 @@ The built-in `grep` tool searches either one text file or all matching text file Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced code examples, so hashtags and code comments do not crowd out real document sections from the agent's heading preview. +UTF-8 Markdown files with or without a byte-order mark (BOM) produce the same +outlines and fallback previews, with original line numbers preserved. Outline titles are limited to 200 characters and fallback previews to 2,000 characters per file, with truncation markers. Full uploaded files remain available for targeted reads. diff --git a/backend/packages/harness/deerflow/utils/AGENTS.md b/backend/packages/harness/deerflow/utils/AGENTS.md index 9747410ce..01d5b0133 100644 --- a/backend/packages/harness/deerflow/utils/AGENTS.md +++ b/backend/packages/harness/deerflow/utils/AGENTS.md @@ -1,3 +1,10 @@ ### Agent / Tool Assembly Off-Load Tool and agent assembly re-enters `get_available_tools()` and may block on MCP discovery, so the four async assembly entry points — `run_agent`'s `agent_factory` call, `task_tool`, durable batch `_execute_item`, and `abuild_checkpoint_state_accessor` — dispatch through `deerflow.utils.assembly_io.run_assembly`, a dedicated ContextVar-preserving bounded executor (`DEER_FLOW_ASSEMBLY_WORKERS`, default 8) rather than the loop's default executor; a hung MCP server therefore parks an assembly worker instead of queueing unrelated default-executor work, and the pool logs a warning when pending assemblies exceed the worker count. `tests/blocking_io/test_tool_assembly_offloop.py` pins all four offloads plus the ContextVar propagation. + +### Uploaded Document Summaries + +`file_outline.py` reads outlines and fallback previews as `utf-8-sig` so an +optional leading UTF-8 BOM cannot hide a first-line heading or code fence, or +occupy a preview line. Preserve physical line numbers, embedded U+FEFF +characters, and the original file bytes. diff --git a/backend/packages/harness/deerflow/utils/file_outline.py b/backend/packages/harness/deerflow/utils/file_outline.py index 1018a68e5..8eda80fdc 100644 --- a/backend/packages/harness/deerflow/utils/file_outline.py +++ b/backend/packages/harness/deerflow/utils/file_outline.py @@ -121,7 +121,7 @@ def extract_outline(md_path: Path) -> list[dict]: fence_char = "" fence_length = 0 try: - with md_path.open(encoding="utf-8") as f: + with md_path.open(encoding="utf-8-sig") as f: for lineno, line in enumerate(f, 1): fence = _CODE_FENCE_RE.match(line.rstrip("\r\n")) if fence_char: @@ -199,7 +199,7 @@ def extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]: preview: list[str] = [] remaining_chars = _OUTLINE_PREVIEW_MAX_CHARS try: - with md_path.open(encoding="utf-8") as f: + with md_path.open(encoding="utf-8-sig") as f: for line in f: stripped = line.strip() if stripped: diff --git a/backend/tests/test_file_outline_encoding.py b/backend/tests/test_file_outline_encoding.py new file mode 100644 index 000000000..749a9dd55 --- /dev/null +++ b/backend/tests/test_file_outline_encoding.py @@ -0,0 +1,38 @@ +"""UTF-8 signatures must not change model-visible document summaries.""" + +from pathlib import Path + +import pytest + +from deerflow.utils.file_outline import extract_outline, extract_outline_for_file + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +@pytest.mark.parametrize( + ("heading", "title"), + [("# 概述", "概述"), ("**PART I**", "PART I"), ("**1** **Introduction**", "1 Introduction")], +) +def test_first_heading_is_preserved_with_or_without_bom(tmp_path: Path, encoding: str, heading: str, title: str) -> None: + document = tmp_path / "report.md" + document.write_text(heading + "\n\n## Next section\n", encoding=encoding) + + assert extract_outline(document) == [{"title": title, "line": 1}, {"title": "Next section", "line": 3}] + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_first_code_fence_is_recognized_with_or_without_bom(tmp_path: Path, encoding: str, fence: str) -> None: + document = tmp_path / "guide.md" + document.write_text(f"{fence}python\n# Code comment\n{fence}\n# Actual section\n", encoding=encoding) + + assert extract_outline_for_file(document) == ([{"title": "Actual section", "line": 4}], []) + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_preview_skips_bom_only_line_and_preserves_embedded_character(tmp_path: Path, encoding: str) -> None: + document = tmp_path / "notes.md" + document.write_text("\nFirst paragraph\nSecond\ufeffparagraph\n", encoding=encoding) + original = document.read_bytes() + + assert extract_outline_for_file(document) == ([], ["First paragraph", "Second\ufeffparagraph"]) + assert document.read_bytes() == original