From f4e740bc49b57bc5d18105e6a4a0c794d3a4bd26 Mon Sep 17 00:00:00 2001 From: tiammomo <26957354+tiammomo@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:28:15 +0800 Subject: [PATCH] fix(uploads): bound document outline and preview text (#5323) * fix(uploads): bound document outline and preview text Bound each document outline title to 200 characters and each fallback preview to 2000 characters across its lines, including omission markers. Refs #5322 Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> * test(uploads): cover exact preview budget and restore title guidance Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --------- Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --- README.md | 3 + backend/AGENTS.md | 2 +- .../harness/deerflow/utils/file_outline.py | 31 ++++-- .../tests/test_file_outline_text_budget.py | 97 +++++++++++++++++++ 4 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 backend/tests/test_file_outline_text_budget.py diff --git a/README.md b/README.md index 1a9863344..669822184 100644 --- a/README.md +++ b/README.md @@ -1351,6 +1351,9 @@ The built-in `grep` tool searches either one text file or all matching text file Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced code examples, so hashtags and code comments do not crowd out real document sections from the agent's heading preview. +Outline titles are limited to 200 characters and fallback previews to 2,000 +characters per file, with truncation markers. Full uploaded files remain available +for targeted reads. Image bytes loaded for a vision-model call are transient: DeerFlow removes the hidden base64 message after the model consumes it so later checkpoints do not keep duplicating that payload. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index b4fba4e35..c68c41583 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -346,7 +346,7 @@ Outlines use ATX syntax (1–6 hashes, space/tab separator, ≤3 leading spaces) - Gateway HTTP uploads stage bytes as `.upload-*.part` files and atomically replace the destination only after size validation. These staging files are hidden from upload listings, agent upload context, and sandbox listing/search tools, and swept on Gateway startup if a hard crash leaves one behind. - Gateway HTTP upload/list/delete handlers offload filesystem work through `deerflow.utils.file_io.run_file_io`, a dedicated ContextVar-preserving file IO executor. Non-mounted sandbox uploads acquire sandboxes with `SandboxProvider.acquire_async()` and offload `read_bytes()` plus `sandbox.update_file()` together. - Mounted uploads skip sandbox acquire/sync. AIO remote/provisioner requires accurate `sandbox.thread_data_mounts: true`; omission keeps backend auto-detection. -- `UploadsMiddleware` supplies file lists. Titles use original user text, not upload context; attachment-only titles use a sanitized, bounded filename or count. +- `UploadsMiddleware` caps outline titles at 200 characters and previews at 2000 including markers. Titles use `original_user_content`, not upload-prefixed content; attachment-only titles use a sanitized, bounded filename or count. See [docs/FILE_UPLOAD.md](docs/FILE_UPLOAD.md) for details. diff --git a/backend/packages/harness/deerflow/utils/file_outline.py b/backend/packages/harness/deerflow/utils/file_outline.py index b65eb14fe..1018a68e5 100644 --- a/backend/packages/harness/deerflow/utils/file_outline.py +++ b/backend/packages/harness/deerflow/utils/file_outline.py @@ -37,6 +37,9 @@ _SPLIT_BOLD_HEADING_RE = re.compile(r"^\*\*[\dA-Z][\d\.]*\*\*\s+\*\*(?!\d[\d\s., MAX_OUTLINE_ENTRIES = 50 _OUTLINE_PREVIEW_LINES = 5 +_OUTLINE_TITLE_MAX_CHARS = 200 +_OUTLINE_PREVIEW_MAX_CHARS = 2000 +_TRUNCATION_MARKER = "… (truncated)" # Root-level Markdown fences allow up to three leading spaces. The rest of # the line is an info string when opening, or whitespace only when closing. @@ -77,6 +80,15 @@ def _clean_bold_title(raw: str) -> str: return merged +def _truncate_outline_text(text: str, max_chars: int) -> str: + """Keep the omission marker inside the summary's character budget.""" + if len(text) <= max_chars: + return text + if max_chars <= len(_TRUNCATION_MARKER): + return "…"[:max_chars] + return text[: max_chars - len(_TRUNCATION_MARKER)].rstrip() + _TRUNCATION_MARKER + + def extract_outline(md_path: Path) -> list[dict]: """Extract document outline (headings) from a Markdown file. @@ -99,7 +111,7 @@ def extract_outline(md_path: Path) -> list[dict]: md_path: Path to the .md file. Returns: - List of dicts with keys: title (str), line (int, 1-based). + List of dicts with keys: title (str, at most 200 characters), line (int, 1-based). When the outline is truncated at MAX_OUTLINE_ENTRIES, a sentinel entry ``{"truncated": True}`` is appended as the last element so callers can render a "showing first N headings" hint without re-scanning the file. @@ -135,20 +147,20 @@ def extract_outline(md_path: Path) -> list[dict]: if m := _ATX_HEADING_RE.fullmatch(line.rstrip("\r\n")): title = _clean_bold_title(_strip_atx_closing_hashes(m.group(1) or "").strip()) if title: - outline.append({"title": title, "line": lineno}) + outline.append({"title": _truncate_outline_text(title, _OUTLINE_TITLE_MAX_CHARS), "line": lineno}) # Style 2: single bold block with SEC structural keyword elif m := _BOLD_HEADING_RE.match(stripped): title = m.group(1).strip() if title: - outline.append({"title": title, "line": lineno}) + outline.append({"title": _truncate_outline_text(title, _OUTLINE_TITLE_MAX_CHARS), "line": lineno}) # Style 3: split-bold heading — **** **** # Regex already enforces max 4 blocks and non-numeric second block. elif _SPLIT_BOLD_HEADING_RE.match(stripped): title = " ".join(re.findall(r"\*\*([^*]+)\*\*", stripped)) if title: - outline.append({"title": title, "line": lineno}) + outline.append({"title": _truncate_outline_text(title, _OUTLINE_TITLE_MAX_CHARS), "line": lineno}) if len(outline) > MAX_OUTLINE_ENTRIES: outline.pop() @@ -171,7 +183,7 @@ def extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]: - outline: list of ``{title, line}`` dicts (plus optional sentinel). Empty when no headings are found or no .md exists. - preview: first few non-empty lines of the .md, used as a content - anchor when outline is empty so the agent has some context. + anchor when outline is empty, capped at 2000 characters across all lines. Empty when outline is non-empty (no fallback needed). """ md_path = file_path.with_suffix(".md") @@ -185,13 +197,18 @@ def extract_outline_for_file(file_path: Path) -> tuple[list[dict], list[str]]: # outline is empty — read the first few non-empty lines as a content preview preview: list[str] = [] + remaining_chars = _OUTLINE_PREVIEW_MAX_CHARS try: with md_path.open(encoding="utf-8") as f: for line in f: stripped = line.strip() if stripped: - preview.append(stripped) - if len(preview) >= _OUTLINE_PREVIEW_LINES: + text = _truncate_outline_text(stripped, remaining_chars) + preview.append(text) + remaining_chars -= len(text) + if len(stripped) > len(text): + break + if len(preview) >= _OUTLINE_PREVIEW_LINES or remaining_chars == 0: break except Exception: logger.debug("Failed to read preview lines from %s", md_path, exc_info=True) diff --git a/backend/tests/test_file_outline_text_budget.py b/backend/tests/test_file_outline_text_budget.py new file mode 100644 index 000000000..4e3c58063 --- /dev/null +++ b/backend/tests/test_file_outline_text_budget.py @@ -0,0 +1,97 @@ +"""Uploaded document summaries must stay compact even for very long lines.""" + +from pathlib import Path + +import pytest + +from deerflow.utils.file_outline import MAX_OUTLINE_ENTRIES, extract_outline, extract_outline_for_file + + +@pytest.mark.parametrize("heading", ["# {text}", "**SECTION {text}**", "**1** **{text}**"]) +def test_long_heading_titles_are_bounded_with_original_line_numbers(tmp_path: Path, heading: str) -> None: + document = tmp_path / "report.md" + original = "Introduction\n\n" + heading.format(text="A" * 200_000) + "\n# Next section\n" + document.write_text(original, encoding="utf-8") + outline = extract_outline(document) + assert len(outline[0]["title"]) <= 200 + assert outline[0]["title"].endswith("… (truncated)") + assert outline[0]["line"] == 3 + assert outline[1] == {"title": "Next section", "line": 4} + assert document.read_text(encoding="utf-8") == original + + +def test_single_long_paragraph_has_a_bounded_preview(tmp_path: Path) -> None: + document = tmp_path / "report.md" + original = "word " * 40_000 + document.write_text(original, encoding="utf-8") + outline, preview = extract_outline_for_file(document) + assert outline == [] + assert len(preview) == 1 + assert len(preview[0]) <= 2000 + assert preview[0].endswith("… (truncated)") + assert document.read_text(encoding="utf-8") == original + + +def test_preview_budget_is_shared_across_nonempty_lines(tmp_path: Path) -> None: + document = tmp_path / "report.md" + document.write_text("\n".join(["A" * 800, "", "B" * 800, "C" * 800, "D" * 800]), encoding="utf-8") + _, preview = extract_outline_for_file(document) + assert preview[:2] == ["A" * 800, "B" * 800] + assert sum(map(len, preview)) <= 2000 + assert preview[-1].endswith("… (truncated)") + + +@pytest.mark.parametrize("length", [199, 200]) +def test_heading_at_or_below_budget_is_unchanged(tmp_path: Path, length: int) -> None: + document = tmp_path / "report.md" + document.write_text("# " + "字" * length, encoding="utf-8") + assert extract_outline(document) == [{"title": "字" * length, "line": 1}] + + +@pytest.mark.parametrize("length", [1999, 2000]) +def test_preview_at_or_below_budget_is_unchanged(tmp_path: Path, length: int) -> None: + document = tmp_path / "report.md" + document.write_text("字" * length, encoding="utf-8") + assert extract_outline_for_file(document) == ([], ["字" * length]) + + +def test_preview_still_limits_nonempty_line_count(tmp_path: Path) -> None: + document = tmp_path / "report.md" + document.write_text("\n\n".join(f"Line {i}" for i in range(7)), encoding="utf-8") + assert extract_outline_for_file(document) == ([], [f"Line {i}" for i in range(5)]) + + +def test_long_headings_preserve_entry_count_and_truncation_sentinel(tmp_path: Path) -> None: + document = tmp_path / "report.md" + document.write_text("\n".join("# " + "A" * 300 for _ in range(MAX_OUTLINE_ENTRIES + 1)), encoding="utf-8") + outline = extract_outline(document) + assert len(outline) == MAX_OUTLINE_ENTRIES + 1 + assert outline[-1] == {"truncated": True} + assert all(len(entry["title"]) <= 200 for entry in outline[:-1]) + + +def test_tiny_remaining_preview_budget_still_marks_omission(tmp_path: Path) -> None: + document = tmp_path / "report.md" + document.write_text("字" * 1999 + "\n" + "文" * 100, encoding="utf-8") + _, preview = extract_outline_for_file(document) + assert sum(map(len, preview)) <= 2000 + assert preview[-1] == "…" + + +def test_long_unicode_heading_preserves_readable_prefix(tmp_path: Path) -> None: + document = tmp_path / "report.md" + document.write_text("# " + "研究" * 200, encoding="utf-8") + title = extract_outline(document)[0]["title"] + assert title.startswith("研究研究") + assert title.endswith("… (truncated)") + assert len(title) <= 200 + + +def test_exact_fit_preview_stops_before_following_content(tmp_path: Path) -> None: + document = tmp_path / "report.md" + original = "Z" * 2000 + "\nACTUAL CONTENT\n" + document.write_text(original, encoding="utf-8") + # Markers describe truncation within an included line. Reaching the total + # budget stops the preview, just like reaching its five-line limit. + assert extract_outline_for_file(document) == ([], ["Z" * 2000]) + assert document.read_text(encoding="utf-8") == original