From 05dc8f412308397bea980c2c2aa088451da622c8 Mon Sep 17 00:00:00 2001 From: tiammomo <26957354+tiammomo@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:06:28 +0800 Subject: [PATCH] fix(uploads): exclude fenced code from document outlines (#5281) * fix(uploads): exclude fenced code from document outlines Closes #5271 Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> * docs(uploads): keep outline guidance within instruction budget Keep the AGENTS instruction chain within the upstream hard limit. Follow-up for #5281; refs #5271. Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --------- Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --- README.md | 3 + backend/AGENTS.md | 2 +- .../harness/deerflow/utils/file_outline.py | 22 +++++ backend/tests/test_file_outline.py | 92 +++++++++++++++++++ 4 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_file_outline.py diff --git a/README.md b/README.md index 51abab6b2..583b3e4ec 100644 --- a/README.md +++ b/README.md @@ -1299,6 +1299,9 @@ Each task gets its own execution environment with a full filesystem view — ski The built-in `grep` tool searches either one text file or all matching text files below a directory, so an agent can search an uploaded document directly without first broadening the request to the entire uploads directory. +Uploaded Markdown outlines skip fenced code examples, so code comments do not +crowd out real document sections from the agent's heading preview. + Image bytes loaded for a vision-model call are transient: DeerFlow removes the hidden base64 message after the model consumes it so later checkpoints do not keep duplicating that payload. After each run, DeerFlow records a workspace change summary for the run-owned `workspace` and `outputs` directories. The Web UI shows a compact "files changed" badge on the assistant turn; opening it reveals created, modified, and deleted files with text diffs when safe to display. Uploads are excluded because they are user inputs, not agent-generated changes, and stdio MCP temporary/debug files under the DeerFlow-owned `.mcp/` namespace are excluded because they are process-internal state (like `.git/` and `node_modules/`, any directory named `.mcp` is excluded at any depth). Large, binary, or sensitive-looking files are shown as metadata only. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 29e66a6d7..c93caccad 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -320,7 +320,7 @@ engines are removed, and an empty set falls back to it. Re-check on DDGS upgrade ### File Upload -Multi-file upload with automatic document conversion: +Multi-file uploads convert documents; outlines skip fenced code: - Endpoint: `POST /api/threads/{thread_id}/uploads` - Supports: PDF, PPT, Excel, Word documents (converted via `markitdown`) - Rejects directory inputs before copying so uploads stay all-or-nothing diff --git a/backend/packages/harness/deerflow/utils/file_outline.py b/backend/packages/harness/deerflow/utils/file_outline.py index f474b1305..fd741b7a0 100644 --- a/backend/packages/harness/deerflow/utils/file_outline.py +++ b/backend/packages/harness/deerflow/utils/file_outline.py @@ -38,6 +38,10 @@ MAX_OUTLINE_ENTRIES = 50 _OUTLINE_PREVIEW_LINES = 5 +# Root-level Markdown fences allow up to three leading spaces. The rest of +# the line is an info string when opening, or whitespace only when closing. +_CODE_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") + def _clean_bold_title(raw: str) -> str: """Normalise a title string that may contain pymupdf4llm bold artefacts. @@ -88,9 +92,27 @@ def extract_outline(md_path: Path) -> list[dict]: Returns an empty list if the file cannot be read or has no headings. """ outline: list[dict] = [] + fence_char = "" + fence_length = 0 try: with md_path.open(encoding="utf-8") as f: for lineno, line in enumerate(f, 1): + fence = _CODE_FENCE_RE.match(line.rstrip("\r\n")) + if fence_char: + if fence: + marker, suffix = fence.groups() + if marker[0] == fence_char and len(marker) >= fence_length and not suffix.strip(" \t"): + fence_char = "" + continue + if fence: + marker, info = fence.groups() + # Backtick info strings cannot contain backticks; tilde + # info strings have no such restriction. + if marker[0] == "~" or "`" not in info: + fence_char = marker[0] + fence_length = len(marker) + continue + stripped = line.strip() if not stripped: continue diff --git a/backend/tests/test_file_outline.py b/backend/tests/test_file_outline.py new file mode 100644 index 000000000..6d48a2ea4 --- /dev/null +++ b/backend/tests/test_file_outline.py @@ -0,0 +1,92 @@ +"""Regression tests for fenced code in model-visible document outlines.""" + +from pathlib import Path + +import pytest + +from deerflow.utils.file_outline import MAX_OUTLINE_ENTRIES, extract_outline + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +@pytest.mark.parametrize("indent", ["", " ", " ", " "]) +def test_fenced_code_is_excluded_from_all_heading_styles(tmp_path: Path, fence: str, indent: str) -> None: + document = tmp_path / "guide.md" + document.write_text( + f"# Setup\n{indent}{fence}python\n# Code comment\n**ITEM 1. NOT A SECTION**\n**2** **Not a section**\n{indent}{fence}\n## Results\n", + encoding="utf-8", + ) + + assert extract_outline(document) == [{"title": "Setup", "line": 1}, {"title": "Results", "line": 7}] + + +@pytest.mark.parametrize( + ("opening", "non_closing"), + [ + ("````", "```"), + ("~~~~", "~~~"), + ("```", "~~~"), + ("~~~", "```"), + ("```", "```python"), + ("~~~", "~~~text"), + ("```", " ```"), + ], +) +def test_only_a_matching_closing_fence_ends_code(tmp_path: Path, opening: str, non_closing: str) -> None: + document = tmp_path / "guide.md" + document.write_text(f"{opening}\n{non_closing}\n# Still code\n{opening}\n# Real section\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Real section", "line": 5}] + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_longer_closing_fence_with_trailing_whitespace(tmp_path: Path, fence: str) -> None: + document = tmp_path / "guide.md" + document.write_text(f"{fence}\n# Code\n{fence * 2} \t\n# Section\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Section", "line": 4}] + + +def test_unclosed_code_fence_excludes_remaining_lines(tmp_path: Path) -> None: + document = tmp_path / "guide.md" + document.write_text("# Overview\n```python\n# Code\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Overview", "line": 1}] + + +@pytest.mark.parametrize("non_opening", ["``", "~~", "```bad`info", " ```"]) +def test_invalid_opening_fence_does_not_hide_subsequent_headings(tmp_path: Path, non_opening: str) -> None: + document = tmp_path / "guide.md" + document.write_text(f"{non_opening}\n# Section\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Section", "line": 2}] + + +def test_tilde_fence_allows_backticks_in_info_string(tmp_path: Path) -> None: + document = tmp_path / "guide.md" + document.write_text("~~~example `code`\n# Code\n~~~\n# Section\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Section", "line": 4}] + + +def test_code_comments_do_not_exhaust_outline_budget(tmp_path: Path) -> None: + document = tmp_path / "guide.md" + comments = "".join(f"# Code comment {index}\n" for index in range(MAX_OUTLINE_ENTRIES + 1)) + document.write_text("```python\n" + comments + "```\n# Actual findings\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Actual findings", "line": MAX_OUTLINE_ENTRIES + 4}] + + +def test_real_headings_after_code_still_obey_outline_budget(tmp_path: Path) -> None: + document = tmp_path / "guide.md" + headings = "".join(f"# Section {index}\n" for index in range(MAX_OUTLINE_ENTRIES + 1)) + document.write_text("```python\n# Code\n```\n" + headings, encoding="utf-8") + + expected = [{"title": f"Section {index}", "line": index + 4} for index in range(MAX_OUTLINE_ENTRIES)] + assert extract_outline(document) == expected + [{"truncated": True}] + + +def test_pdf_bold_headings_outside_code_remain_supported(tmp_path: Path) -> None: + document = tmp_path / "guide.md" + document.write_text("```\n**ITEM 1. CODE**\n```\n**PART I**\n**2** **Results**\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "PART I", "line": 4}, {"title": "2 Results", "line": 5}]