From e0bd70279e358d2eb554683dff926ce71eadf9d5 Mon Sep 17 00:00:00 2001 From: lilei <93921041+xihongshichaojidan8@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:10:07 +0800 Subject: [PATCH] fix(uploads): skip HTML comment blocks in document outlines (#6470) --- README.md | 2 +- .../packages/harness/deerflow/utils/AGENTS.md | 5 + .../harness/deerflow/utils/file_outline.py | 11 +++ .../tests/test_file_outline_html_comments.py | 92 +++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_file_outline_html_comments.py diff --git a/README.md b/README.md index 4ebde29d46..a844f5f58f 100644 --- a/README.md +++ b/README.md @@ -2028,7 +2028,7 @@ AIO directory listings discard missing shell sessions so the next request can re After a dropped connection, directory listings and persistent shell commands report an unknown outcome without replaying the operation; later calls use a fresh session. -Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced and indented code examples, so hashtags and code comments do not +Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip HTML comment blocks and fenced and indented code examples, so hashtags and code comments do not crowd out real document sections from the agent's heading preview. Indented bold examples are also excluded; PDF-style bold headings with up to three leading spaces remain supported. Split-bold numeric table rows, including parenthesized years, signed values, and currency-prefixed amounts, are excluded when any block after the section number diff --git a/backend/packages/harness/deerflow/utils/AGENTS.md b/backend/packages/harness/deerflow/utils/AGENTS.md index 5a736942a1..3143816145 100644 --- a/backend/packages/harness/deerflow/utils/AGENTS.md +++ b/backend/packages/harness/deerflow/utils/AGENTS.md @@ -43,6 +43,11 @@ Tool and agent assembly re-enters `get_available_tools()` and may block on MCP d ### Uploaded Document Summaries +Outlines skip root-level HTML comment blocks (`` line). Comment contents cannot open code fences; +comment markers inside fenced code cannot open comment blocks. Keep line numbers +and limits unchanged. Coverage: `tests/test_file_outline_html_comments.py`. + `file_outline.py` reads outlines and fallback previews as `utf-8-sig` so an optional leading UTF-8 BOM cannot hide a first-line heading or code fence, or occupy a preview line. Preserve physical line numbers, embedded U+FEFF diff --git a/backend/packages/harness/deerflow/utils/file_outline.py b/backend/packages/harness/deerflow/utils/file_outline.py index 349298016c..2a0e0b4859 100644 --- a/backend/packages/harness/deerflow/utils/file_outline.py +++ b/backend/packages/harness/deerflow/utils/file_outline.py @@ -47,6 +47,9 @@ _TRUNCATION_MARKER = "… (truncated)" # the line is an info string when opening, or whitespace only when closing. _CODE_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") +# HTML comment blocks end on the first line containing -->. +_HTML_COMMENT_START_RE = re.compile(r"^ {0,3}" not in line + continue + fence = _CODE_FENCE_RE.match(line.rstrip("\r\n")) if fence_char: if fence: @@ -136,6 +144,9 @@ def extract_outline(md_path: Path) -> list[dict]: if marker[0] == fence_char and len(marker) >= fence_length and not suffix.strip(" \t"): fence_char = "" continue + if _HTML_COMMENT_START_RE.match(line): + in_html_comment = "-->" not in line + continue if fence: marker, info = fence.groups() # Backtick info strings cannot contain backticks; tilde diff --git a/backend/tests/test_file_outline_html_comments.py b/backend/tests/test_file_outline_html_comments.py new file mode 100644 index 0000000000..926535d61c --- /dev/null +++ b/backend/tests/test_file_outline_html_comments.py @@ -0,0 +1,92 @@ +"""Regression tests for HTML comment blocks in uploaded-document outlines.""" + +from pathlib import Path + +import pytest + +from deerflow.uploads.companions import register_companion +from deerflow.utils.file_outline import MAX_OUTLINE_ENTRIES, extract_outline, extract_outline_for_file + + +@pytest.mark.parametrize("indent", ["", " ", " ", " "]) +def test_html_comments_exclude_all_heading_styles(tmp_path: Path, indent: str) -> None: + document = tmp_path / "guide.md" + document.write_text( + f"# Overview\n{indent}\n## Results\n", + encoding="utf-8", + ) + + assert extract_outline(document) == [{"title": "Overview", "line": 1}, {"title": "Results", "line": 7}] + + +@pytest.mark.parametrize("fence", ["```", "~~~"]) +def test_fence_inside_comment_does_not_hide_later_headings(tmp_path: Path, fence: str) -> None: + document = tmp_path / "guide.md" + document.write_text(f"\n# Results\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Results", "line": 5}] + + +@pytest.mark.parametrize("comment", ["", " # Not a heading", " # Not a heading"]) +def test_comment_ends_on_first_closing_marker_line(tmp_path: Path, comment: str) -> None: + document = tmp_path / "guide.md" + document.write_text(comment + "\n# Results\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Results", "line": comment.count("\n") + 2}] + + +def test_unclosed_comment_excludes_remaining_headings(tmp_path: Path) -> None: + document = tmp_path / "guide.md" + document.write_text("# Overview\n\n# Results\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Results", "line": MAX_OUTLINE_ENTRIES + 4}] + + +def test_real_headings_after_comment_still_obey_outline_limit(tmp_path: Path) -> None: + document = tmp_path / "guide.md" + headings = "".join(f"# Section {index}\n" for index in range(MAX_OUTLINE_ENTRIES + 1)) + document.write_text("\n" + headings, encoding="utf-8") + + expected = [{"title": f"Section {index}", "line": index + 4} for index in range(MAX_OUTLINE_ENTRIES)] + assert extract_outline(document) == expected + [{"truncated": True}] + + +@pytest.mark.parametrize("converted", [False, True]) +def test_upload_summary_ignores_comments_without_changing_file_bytes(tmp_path: Path, converted: bool) -> None: + uploads = tmp_path / "thread/user-data/uploads" + uploads.mkdir(parents=True) + markdown = uploads / "report.md" + content = b"\xef\xbb\xbf\r\n# Results\r\n" + markdown.write_bytes(content) + source = markdown + if converted: + source = uploads / "report.pdf" + source.write_bytes(b"synthetic source; conversion not invoked") + register_companion(source, markdown) + + assert extract_outline_for_file(source) == ([{"title": "Results", "line": 5}], []) + assert markdown.read_bytes() == content