mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
fix(uploads): exclude punctuated numeric columns from document outlines (#6266)
* fix(uploads): exclude punctuated numeric columns from outlines * fix(uploads): filter currency and later numeric outline columns --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
parent
ef76f085a3
commit
e8f1e269e5
@ -1918,6 +1918,9 @@ unknown outcome without replaying the operation; later calls use a fresh session
|
||||
|
||||
Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced and indented code examples, so hashtags and code comments do not
|
||||
crowd out real document sections from the agent's heading preview. Indented bold examples are also excluded; PDF-style bold headings with up to three leading spaces remain supported.
|
||||
Split-bold numeric table rows, including parenthesized years, signed values, and
|
||||
currency-prefixed amounts, are excluded when any block after the section number
|
||||
is a numeric column, so they do not consume the outline's heading budget.
|
||||
UTF-8 Markdown files with or without a byte-order mark (BOM) produce the same
|
||||
outlines and fallback previews, with original line numbers preserved.
|
||||
Outline titles are limited to 200 characters and fallback previews to 2,000
|
||||
|
||||
@ -38,6 +38,12 @@ headings. Keep legitimate bold headings with up to three spaces and the shared
|
||||
conversion-companion path working. Regression coverage lives in
|
||||
`tests/test_file_outline_indented_bold.py`.
|
||||
|
||||
Split-bold headings reject numeric/punctuation/currency-only blocks after the
|
||||
section number, including parentheses, signs, leading whitespace, and `$€£¥`.
|
||||
Apply the check to the second through fourth blocks; preserve the four-block
|
||||
limit, punctuated titles containing text, and non-ASCII titles. Coverage lives
|
||||
in `tests/test_file_outline_split_bold.py`.
|
||||
|
||||
### Active Content MIME Types
|
||||
|
||||
`text_detection.py::_is_active_content_mime_type` is the shared download-safety
|
||||
|
||||
@ -27,12 +27,12 @@ _BOLD_HEADING_RE = re.compile(r"^\*\*((ITEM|PART|SECTION|SCHEDULE|EXHIBIT|APPEND
|
||||
# Requirements:
|
||||
# 1. Entire line consists only of **...** blocks separated by whitespace (no prose)
|
||||
# 2. First block is a section number (digits and dots, e.g. "1", "3.2", "A.1")
|
||||
# 3. Second block must not be purely numeric/punctuation — excludes financial table
|
||||
# headers like **2023** **2022** **2021** while allowing non-ASCII titles such as
|
||||
# 3. Every block after the section number must contain more than numbers, punctuation,
|
||||
# or currency symbols — excludes financial table columns while allowing titles such as
|
||||
# **1** **概述** or accented words (negative lookahead instead of [A-Za-z])
|
||||
# 4. At most two additional blocks (four total) with [^*]+ (no * inside) to keep
|
||||
# 4. One to three title blocks (four total) with [^*]+ (no * inside) to keep
|
||||
# the regex linear and avoid ReDoS on attacker-controlled content
|
||||
_SPLIT_BOLD_HEADING_RE = re.compile(r"^\*\*[\dA-Z][\d\.]*\*\*\s+\*\*(?!\d[\d\s.,\-–—/:()%]*\*\*)[^*]+\*\*(?:\s+\*\*[^*]+\*\*){0,2}\s*$")
|
||||
_SPLIT_BOLD_HEADING_RE = re.compile(r"^\*\*[\dA-Z][\d\.]*\*\*(?:\s+\*\*(?![\d\s.,+\-–—/:()%$€£¥]+\*\*)[^*]+\*\*){1,3}\s*$")
|
||||
|
||||
# Maximum number of outline entries injected into the agent context.
|
||||
# Keeps prompt size bounded even for very long documents.
|
||||
@ -165,7 +165,7 @@ def extract_outline(md_path: Path) -> list[dict]:
|
||||
outline.append({"title": _truncate_outline_text(title, _OUTLINE_TITLE_MAX_CHARS), "line": lineno})
|
||||
|
||||
# Style 3: split-bold heading — **<num>** **<title>**
|
||||
# Regex already enforces max 4 blocks and non-numeric second block.
|
||||
# Regex enforces max 4 blocks and rejects numeric columns after the number.
|
||||
elif _SPLIT_BOLD_HEADING_RE.match(stripped):
|
||||
title = " ".join(re.findall(r"\*\*([^*]+)\*\*", stripped))
|
||||
if title:
|
||||
|
||||
74
backend/tests/test_file_outline_split_bold.py
Normal file
74
backend/tests/test_file_outline_split_bold.py
Normal file
@ -0,0 +1,74 @@
|
||||
"""Split-bold outlines must distinguish section titles from numeric table rows."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from deerflow.utils.file_outline import MAX_OUTLINE_ENTRIES, extract_outline, extract_outline_for_file
|
||||
|
||||
|
||||
@pytest.mark.parametrize("column", ["(2023)", "-12.5", "–12", "+12", "/2023", "(12%)", "---", "%", " 2023", "2023", "$1,234", "€100", "£1,234.50", "¥100", "(€100)", " $100", "$"])
|
||||
def test_numeric_or_punctuation_columns_are_not_headings(tmp_path: Path, column: str) -> None:
|
||||
document = tmp_path / "annual.md"
|
||||
document.write_text(f"# Financial Summary\n**2024** **{column}**\n## Results\n", encoding="utf-8")
|
||||
|
||||
assert extract_outline(document) == [{"title": "Financial Summary", "line": 1}, {"title": "Results", "line": 3}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("title", ["(Introduction)", "- Results", "2023 Results", "概述", "Évaluation", "(概述)", "$ Revenue", "€ Marché", "£成本", "¥価格"])
|
||||
def test_punctuated_or_non_ascii_section_titles_remain_supported(tmp_path: Path, title: str) -> None:
|
||||
document = tmp_path / "paper.md"
|
||||
document.write_text(f"**3.2** **{title}**\n", encoding="utf-8")
|
||||
|
||||
assert extract_outline(document) == [{"title": f"3.2 {title}", "line": 1}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"row",
|
||||
[
|
||||
"**2024** **Revenue** **2023**",
|
||||
"**2024** **Revenue** **(2023)**",
|
||||
"**2024** **Revenue** **-12.5**",
|
||||
"**2024** **Revenue** **$1,234**",
|
||||
"**2024** **Revenue** **Growth** **€100**",
|
||||
"**2024** **Revenue** **(2023)** **-12.5**",
|
||||
],
|
||||
)
|
||||
def test_later_numeric_columns_are_not_headings(tmp_path: Path, row: str) -> None:
|
||||
document = tmp_path / "annual.md"
|
||||
document.write_text(f"# Financial Summary\n{row}\n## Results\n", encoding="utf-8")
|
||||
|
||||
assert extract_outline(document) == [{"title": "Financial Summary", "line": 1}, {"title": "Results", "line": 3}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blocks", ["**Results** **and discussion**", "**Results** **and** **discussion**", "**Revenue** **(2023 Results)**", "**Revenue** **$ Outlook** **概述**"])
|
||||
def test_text_bearing_later_blocks_remain_headings(tmp_path: Path, blocks: str) -> None:
|
||||
document = tmp_path / "paper.md"
|
||||
document.write_text(f"**3.2** {blocks}\n", encoding="utf-8")
|
||||
|
||||
title = blocks.replace("**", "")
|
||||
assert extract_outline(document) == [{"title": f"3.2 {title}", "line": 1}]
|
||||
|
||||
|
||||
def test_split_bold_headings_keep_four_block_limit(tmp_path: Path) -> None:
|
||||
document = tmp_path / "paper.md"
|
||||
document.write_text("**3.2** **Results** **and** **discussion** **continued**\n", encoding="utf-8")
|
||||
|
||||
assert extract_outline(document) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("row", ["**2024** **(2023)**", "**2024** **$1,234**", "**2024** **Revenue** **(2023)** **-12.5**"])
|
||||
def test_numeric_table_rows_do_not_exhaust_outline_budget(tmp_path: Path, row: str) -> None:
|
||||
document = tmp_path / "annual.md"
|
||||
rows = f"{row}\n" * (MAX_OUTLINE_ENTRIES + 1)
|
||||
document.write_text(rows + "**1** **Actual findings**\n", encoding="utf-8")
|
||||
|
||||
assert extract_outline(document) == [{"title": "1 Actual findings", "line": MAX_OUTLINE_ENTRIES + 2}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("row", ["**2024** **(2023)**", "**2024** **€100**", "**2024** **Revenue** **Growth** **¥100**"])
|
||||
def test_numeric_only_document_uses_fallback_preview(tmp_path: Path, row: str) -> None:
|
||||
document = tmp_path / "annual.md"
|
||||
document.write_text(f"{row}\nRevenue 100 90\n", encoding="utf-8")
|
||||
|
||||
assert extract_outline_for_file(document) == ([], [row, "Revenue 100 90"])
|
||||
Loading…
x
Reference in New Issue
Block a user