fix(uploads): skip HTML comment blocks in document outlines (#6470)

README.md

@@ -2028,7 +2028,7 @@ AIO directory listings discard missing shell sessions so the next request can re
 After a dropped connection, directory listings and persistent shell commands report an
 unknown outcome without replaying the operation; later calls use a fresh session.
 
-Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced and indented code examples, so hashtags and code comments do not
+Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip HTML comment blocks and fenced and indented code examples, so hashtags and code comments do not
 crowd out real document sections from the agent's heading preview. Indented bold examples are also excluded; PDF-style bold headings with up to three leading spaces remain supported.
 Split-bold numeric table rows, including parenthesized years, signed values, and
 currency-prefixed amounts, are excluded when any block after the section number

backend/packages/harness/deerflow/utils/AGENTS.md

@@ -43,6 +43,11 @@ Tool and agent assembly re-enters `get_available_tools()` and may block on MCP d
 
 ### Uploaded Document Summaries
 
+Outlines skip root-level HTML comment blocks (`<!--` with up to three leading
+spaces through the first `-->` line). Comment contents cannot open code fences;
+comment markers inside fenced code cannot open comment blocks. Keep line numbers
+and limits unchanged. Coverage: `tests/test_file_outline_html_comments.py`.
+
 `file_outline.py` reads outlines and fallback previews as `utf-8-sig` so an
 optional leading UTF-8 BOM cannot hide a first-line heading or code fence, or
 occupy a preview line. Preserve physical line numbers, embedded U+FEFF

backend/packages/harness/deerflow/utils/file_outline.py

@@ -47,6 +47,9 @@ _TRUNCATION_MARKER = "… (truncated)"
 # the line is an info string when opening, or whitespace only when closing.
 _CODE_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
 
+# HTML comment blocks end on the first line containing -->.
+_HTML_COMMENT_START_RE = re.compile(r"^ {0,3}<!--")
+
 # ATX headings require 1-6 hashes and a space/tab separator (or end of line).
 # Match the original indentation so indented code cannot become a heading.
 _ATX_HEADING_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+(.*))?$")
@@ -126,9 +129,14 @@ def extract_outline(md_path: Path) -> list[dict]:
     outline: list[dict] = []
     fence_char = ""
     fence_length = 0
+    in_html_comment = False
     try:
         with md_path.open(encoding="utf-8-sig") as f:
             for lineno, line in enumerate(f, 1):
+                if in_html_comment:
+                    in_html_comment = "-->" not in line
+                    continue
+
                 fence = _CODE_FENCE_RE.match(line.rstrip("\r\n"))
                 if fence_char:
                     if fence:
@@ -136,6 +144,9 @@ def extract_outline(md_path: Path) -> list[dict]:
                         if marker[0] == fence_char and len(marker) >= fence_length and not suffix.strip(" \t"):
                             fence_char = ""
                     continue
+                if _HTML_COMMENT_START_RE.match(line):
+                    in_html_comment = "-->" not in line
+                    continue
                 if fence:
                     marker, info = fence.groups()
                     # Backtick info strings cannot contain backticks; tilde

backend/tests/test_file_outline_html_comments.py

@@ -0,0 +1,92 @@
+"""Regression tests for HTML comment blocks in uploaded-document outlines."""
+
+from pathlib import Path
+
+import pytest
+
+from deerflow.uploads.companions import register_companion
+from deerflow.utils.file_outline import MAX_OUTLINE_ENTRIES, extract_outline, extract_outline_for_file
+
+
+@pytest.mark.parametrize("indent", ["", " ", "  ", "   "])
+def test_html_comments_exclude_all_heading_styles(tmp_path: Path, indent: str) -> None:
+    document = tmp_path / "guide.md"
+    document.write_text(
+        f"# Overview\n{indent}<!--\n## Hidden draft\n**ITEM 1. HIDDEN**\n**2** **Hidden results**\n-->\n## Results\n",
+        encoding="utf-8",
+    )
+
+    assert extract_outline(document) == [{"title": "Overview", "line": 1}, {"title": "Results", "line": 7}]
+
+
+@pytest.mark.parametrize("fence", ["```", "~~~"])
+def test_fence_inside_comment_does_not_hide_later_headings(tmp_path: Path, fence: str) -> None:
+    document = tmp_path / "guide.md"
+    document.write_text(f"<!--\n{fence}python\n# Hidden example\n-->\n# Results\n", encoding="utf-8")
+
+    assert extract_outline(document) == [{"title": "Results", "line": 5}]
+
+
+@pytest.mark.parametrize("comment", ["<!-- note -->", "<!-- --> # Not a heading", "<!-- first --><!-- second", "<!-- note\n\n--> # Not a heading"])
+def test_comment_ends_on_first_closing_marker_line(tmp_path: Path, comment: str) -> None:
+    document = tmp_path / "guide.md"
+    document.write_text(comment + "\n# Results\n", encoding="utf-8")
+
+    assert extract_outline(document) == [{"title": "Results", "line": comment.count("\n") + 2}]
+
+
+def test_unclosed_comment_excludes_remaining_headings(tmp_path: Path) -> None:
+    document = tmp_path / "guide.md"
+    document.write_text("# Overview\n<!--\n\n## Hidden draft\n", encoding="utf-8")
+
+    assert extract_outline(document) == [{"title": "Overview", "line": 1}]
+
+
+@pytest.mark.parametrize("fence", ["```", "~~~"])
+def test_comment_markers_inside_code_do_not_open_comment_block(tmp_path: Path, fence: str) -> None:
+    document = tmp_path / "guide.md"
+    document.write_text(f"{fence}html\n<!--\n# Code\n{fence}\n# Results\n", encoding="utf-8")
+
+    assert extract_outline(document) == [{"title": "Results", "line": 5}]
+
+
+@pytest.mark.parametrize("non_opening", ["    <!--", "\t<!--", "text <!--", "&lt;!--"])
+def test_non_block_comment_marker_does_not_hide_headings(tmp_path: Path, non_opening: str) -> None:
+    document = tmp_path / "guide.md"
+    document.write_text(non_opening + "\n# Results\n", encoding="utf-8")
+
+    assert extract_outline(document) == [{"title": "Results", "line": 2}]
+
+
+def test_commented_headings_do_not_exhaust_outline_limit(tmp_path: Path) -> None:
+    document = tmp_path / "guide.md"
+    hidden = "".join(f"# Hidden {index}\n" for index in range(MAX_OUTLINE_ENTRIES + 1))
+    document.write_text("<!--\n" + hidden + "-->\n# Results\n", encoding="utf-8")
+
+    assert extract_outline(document) == [{"title": "Results", "line": MAX_OUTLINE_ENTRIES + 4}]
+
+
+def test_real_headings_after_comment_still_obey_outline_limit(tmp_path: Path) -> None:
+    document = tmp_path / "guide.md"
+    headings = "".join(f"# Section {index}\n" for index in range(MAX_OUTLINE_ENTRIES + 1))
+    document.write_text("<!--\n# Hidden\n-->\n" + headings, encoding="utf-8")
+
+    expected = [{"title": f"Section {index}", "line": index + 4} for index in range(MAX_OUTLINE_ENTRIES)]
+    assert extract_outline(document) == expected + [{"truncated": True}]
+
+
+@pytest.mark.parametrize("converted", [False, True])
+def test_upload_summary_ignores_comments_without_changing_file_bytes(tmp_path: Path, converted: bool) -> None:
+    uploads = tmp_path / "thread/user-data/uploads"
+    uploads.mkdir(parents=True)
+    markdown = uploads / "report.md"
+    content = b"\xef\xbb\xbf<!--\r\n# Hidden\r\n```python\r\n-->\r\n# Results\r\n"
+    markdown.write_bytes(content)
+    source = markdown
+    if converted:
+        source = uploads / "report.pdf"
+        source.write_bytes(b"synthetic source; conversion not invoked")
+        register_companion(source, markdown)
+
+    assert extract_outline_for_file(source) == ([{"title": "Results", "line": 5}], [])
+    assert markdown.read_bytes() == content