From e8f1e269e56d4865066f0b33e207d7c10e32ef12 Mon Sep 17 00:00:00 2001 From: siye566 <114060217+siye566@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:20:00 +0800 Subject: [PATCH] fix(uploads): exclude punctuated numeric columns from document outlines (#6266) * fix(uploads): exclude punctuated numeric columns from outlines * fix(uploads): filter currency and later numeric outline columns --------- Co-authored-by: Willem Jiang --- README.md | 3 + .../packages/harness/deerflow/utils/AGENTS.md | 6 ++ .../harness/deerflow/utils/file_outline.py | 10 +-- backend/tests/test_file_outline_split_bold.py | 74 +++++++++++++++++++ 4 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_file_outline_split_bold.py diff --git a/README.md b/README.md index 16f2a47633..41645e757d 100644 --- a/README.md +++ b/README.md @@ -1918,6 +1918,9 @@ unknown outcome without replaying the operation; later calls use a fresh session Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced and indented code examples, so hashtags and code comments do not crowd out real document sections from the agent's heading preview. Indented bold examples are also excluded; PDF-style bold headings with up to three leading spaces remain supported. +Split-bold numeric table rows, including parenthesized years, signed values, and +currency-prefixed amounts, are excluded when any block after the section number +is a numeric column, so they do not consume the outline's heading budget. UTF-8 Markdown files with or without a byte-order mark (BOM) produce the same outlines and fallback previews, with original line numbers preserved. Outline titles are limited to 200 characters and fallback previews to 2,000 diff --git a/backend/packages/harness/deerflow/utils/AGENTS.md b/backend/packages/harness/deerflow/utils/AGENTS.md index d7b103ed72..bf7e250f95 100644 --- a/backend/packages/harness/deerflow/utils/AGENTS.md +++ b/backend/packages/harness/deerflow/utils/AGENTS.md @@ -38,6 +38,12 @@ headings. Keep legitimate bold headings with up to three spaces and the shared conversion-companion path working. Regression coverage lives in `tests/test_file_outline_indented_bold.py`. +Split-bold headings reject numeric/punctuation/currency-only blocks after the +section number, including parentheses, signs, leading whitespace, and `$€£¥`. +Apply the check to the second through fourth blocks; preserve the four-block +limit, punctuated titles containing text, and non-ASCII titles. Coverage lives +in `tests/test_file_outline_split_bold.py`. + ### Active Content MIME Types `text_detection.py::_is_active_content_mime_type` is the shared download-safety diff --git a/backend/packages/harness/deerflow/utils/file_outline.py b/backend/packages/harness/deerflow/utils/file_outline.py index 8077b80f85..349298016c 100644 --- a/backend/packages/harness/deerflow/utils/file_outline.py +++ b/backend/packages/harness/deerflow/utils/file_outline.py @@ -27,12 +27,12 @@ _BOLD_HEADING_RE = re.compile(r"^\*\*((ITEM|PART|SECTION|SCHEDULE|EXHIBIT|APPEND # Requirements: # 1. Entire line consists only of **...** blocks separated by whitespace (no prose) # 2. First block is a section number (digits and dots, e.g. "1", "3.2", "A.1") -# 3. Second block must not be purely numeric/punctuation — excludes financial table -# headers like **2023** **2022** **2021** while allowing non-ASCII titles such as +# 3. Every block after the section number must contain more than numbers, punctuation, +# or currency symbols — excludes financial table columns while allowing titles such as # **1** **概述** or accented words (negative lookahead instead of [A-Za-z]) -# 4. At most two additional blocks (four total) with [^*]+ (no * inside) to keep +# 4. One to three title blocks (four total) with [^*]+ (no * inside) to keep # the regex linear and avoid ReDoS on attacker-controlled content -_SPLIT_BOLD_HEADING_RE = re.compile(r"^\*\*[\dA-Z][\d\.]*\*\*\s+\*\*(?!\d[\d\s.,\-–—/:()%]*\*\*)[^*]+\*\*(?:\s+\*\*[^*]+\*\*){0,2}\s*$") +_SPLIT_BOLD_HEADING_RE = re.compile(r"^\*\*[\dA-Z][\d\.]*\*\*(?:\s+\*\*(?![\d\s.,+\-–—/:()%$€£¥]+\*\*)[^*]+\*\*){1,3}\s*$") # Maximum number of outline entries injected into the agent context. # Keeps prompt size bounded even for very long documents. @@ -165,7 +165,7 @@ def extract_outline(md_path: Path) -> list[dict]: outline.append({"title": _truncate_outline_text(title, _OUTLINE_TITLE_MAX_CHARS), "line": lineno}) # Style 3: split-bold heading — **** **** - # Regex already enforces max 4 blocks and non-numeric second block. + # Regex enforces max 4 blocks and rejects numeric columns after the number. elif _SPLIT_BOLD_HEADING_RE.match(stripped): title = " ".join(re.findall(r"\*\*([^*]+)\*\*", stripped)) if title: diff --git a/backend/tests/test_file_outline_split_bold.py b/backend/tests/test_file_outline_split_bold.py new file mode 100644 index 0000000000..e861c3b4a3 --- /dev/null +++ b/backend/tests/test_file_outline_split_bold.py @@ -0,0 +1,74 @@ +"""Split-bold outlines must distinguish section titles from numeric table rows.""" + +from pathlib import Path + +import pytest + +from deerflow.utils.file_outline import MAX_OUTLINE_ENTRIES, extract_outline, extract_outline_for_file + + +@pytest.mark.parametrize("column", ["(2023)", "-12.5", "–12", "+12", "/2023", "(12%)", "---", "%", " 2023", "2023", "$1,234", "€100", "£1,234.50", "¥100", "(€100)", " $100", "$"]) +def test_numeric_or_punctuation_columns_are_not_headings(tmp_path: Path, column: str) -> None: + document = tmp_path / "annual.md" + document.write_text(f"# Financial Summary\n**2024** **{column}**\n## Results\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Financial Summary", "line": 1}, {"title": "Results", "line": 3}] + + +@pytest.mark.parametrize("title", ["(Introduction)", "- Results", "2023 Results", "概述", "Évaluation", "(概述)", "$ Revenue", "€ Marché", "£成本", "¥価格"]) +def test_punctuated_or_non_ascii_section_titles_remain_supported(tmp_path: Path, title: str) -> None: + document = tmp_path / "paper.md" + document.write_text(f"**3.2** **{title}**\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": f"3.2 {title}", "line": 1}] + + +@pytest.mark.parametrize( + "row", + [ + "**2024** **Revenue** **2023**", + "**2024** **Revenue** **(2023)**", + "**2024** **Revenue** **-12.5**", + "**2024** **Revenue** **$1,234**", + "**2024** **Revenue** **Growth** **€100**", + "**2024** **Revenue** **(2023)** **-12.5**", + ], +) +def test_later_numeric_columns_are_not_headings(tmp_path: Path, row: str) -> None: + document = tmp_path / "annual.md" + document.write_text(f"# Financial Summary\n{row}\n## Results\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "Financial Summary", "line": 1}, {"title": "Results", "line": 3}] + + +@pytest.mark.parametrize("blocks", ["**Results** **and discussion**", "**Results** **and** **discussion**", "**Revenue** **(2023 Results)**", "**Revenue** **$ Outlook** **概述**"]) +def test_text_bearing_later_blocks_remain_headings(tmp_path: Path, blocks: str) -> None: + document = tmp_path / "paper.md" + document.write_text(f"**3.2** {blocks}\n", encoding="utf-8") + + title = blocks.replace("**", "") + assert extract_outline(document) == [{"title": f"3.2 {title}", "line": 1}] + + +def test_split_bold_headings_keep_four_block_limit(tmp_path: Path) -> None: + document = tmp_path / "paper.md" + document.write_text("**3.2** **Results** **and** **discussion** **continued**\n", encoding="utf-8") + + assert extract_outline(document) == [] + + +@pytest.mark.parametrize("row", ["**2024** **(2023)**", "**2024** **$1,234**", "**2024** **Revenue** **(2023)** **-12.5**"]) +def test_numeric_table_rows_do_not_exhaust_outline_budget(tmp_path: Path, row: str) -> None: + document = tmp_path / "annual.md" + rows = f"{row}\n" * (MAX_OUTLINE_ENTRIES + 1) + document.write_text(rows + "**1** **Actual findings**\n", encoding="utf-8") + + assert extract_outline(document) == [{"title": "1 Actual findings", "line": MAX_OUTLINE_ENTRIES + 2}] + + +@pytest.mark.parametrize("row", ["**2024** **(2023)**", "**2024** **€100**", "**2024** **Revenue** **Growth** **¥100**"]) +def test_numeric_only_document_uses_fallback_preview(tmp_path: Path, row: str) -> None: + document = tmp_path / "annual.md" + document.write_text(f"{row}\nRevenue 100 90\n", encoding="utf-8") + + assert extract_outline_for_file(document) == ([], [row, "Revenue 100 90"])