diff --git a/README.md b/README.md index 2bb3d161a..85a4a22fd 100644 --- a/README.md +++ b/README.md @@ -1349,7 +1349,7 @@ Each task gets its own execution environment with a full filesystem view — ski The built-in `grep` tool searches either one text file or all matching text files below a directory, so an agent can search an uploaded document directly without first broadening the request to the entire uploads directory. -Uploaded Markdown outlines skip fenced code examples, so code comments do not +Uploaded Markdown outlines recognize ATX heading syntax, clean closing markers with a linear suffix scan, and skip fenced code examples, so hashtags and code comments do not crowd out real document sections from the agent's heading preview. Image bytes loaded for a vision-model call are transient: DeerFlow removes the hidden base64 message after the model consumes it so later checkpoints do not keep duplicating that payload. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 6a914acb5..b4fba4e35 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -75,6 +75,10 @@ deer-flow/ └── custom/ # Custom skills (gitignored) ``` +ATX outline closing markers use a linear suffix scan; do not use unanchored +whitespace regex searches on unbounded uploaded headings. The long-heading +regression exercises the production extractor under a generous process deadline. + ## Important Development Guidelines ### Documentation Update Policy @@ -332,11 +336,11 @@ Title fallback: result URL, then request URL. ### File Upload -Multi-file uploads convert documents; outlines skip fenced code: +Outlines use ATX syntax (1–6 hashes, space/tab separator, ≤3 leading spaces), strip closing hashes and skip fenced code. - Endpoint: `POST /api/threads/{thread_id}/uploads` - Supports: PDF, PPT, Excel, Word documents (converted via `markitdown`) -- Rejects directory inputs before copying so uploads stay all-or-nothing -- Reuses one conversion worker per request when called from an active event loop +- Rejects directories before copying to keep uploads all-or-nothing +- One conversion worker per request when called from an active event loop - Files stored in thread-isolated directories under the resolving user's bucket (`users/{user_id}/threads/{thread_id}/user-data/uploads`). For IM channels the owner is threaded explicitly via the `user_id=` kwarg (see IM Channels → Owner-scoped file storage); HTTP/embedded callers resolve it from `get_effective_user_id()` - Duplicate filenames within one request get `_N` suffixes to prevent overwrites. - Gateway HTTP uploads stage bytes as `.upload-*.part` files and atomically replace the destination only after size validation. These staging files are hidden from upload listings, agent upload context, and sandbox listing/search tools, and swept on Gateway startup if a hard crash leaves one behind. diff --git a/backend/packages/harness/deerflow/utils/file_outline.py b/backend/packages/harness/deerflow/utils/file_outline.py index fd741b7a0..b65eb14fe 100644 --- a/backend/packages/harness/deerflow/utils/file_outline.py +++ b/backend/packages/harness/deerflow/utils/file_outline.py @@ -42,6 +42,19 @@ _OUTLINE_PREVIEW_LINES = 5 # the line is an info string when opening, or whitespace only when closing. _CODE_FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") +# ATX headings require 1-6 hashes and a space/tab separator (or end of line). +# Match the original indentation so indented code cannot become a heading. +_ATX_HEADING_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]+(.*))?$") + + +def _strip_atx_closing_hashes(raw: str) -> str: + """Remove a whitespace-separated terminal hash run in linear time.""" + trimmed = raw.rstrip(" \t") + prefix = trimmed.rstrip("#") + if len(prefix) < len(trimmed) and (not prefix or prefix[-1] in " \t"): + return prefix.rstrip(" \t") + return trimmed + def _clean_bold_title(raw: str) -> str: """Normalise a title string that may contain pymupdf4llm bold artefacts. @@ -69,7 +82,8 @@ def extract_outline(md_path: Path) -> list[dict]: Recognises three heading styles produced by pymupdf4llm: - 1. Standard Markdown headings: lines starting with one or more '#'. + 1. Standard ATX headings: up to three spaces, then 1-6 '#' characters + followed by a space/tab or end of line. Optional closing hashes are removed. Inline ``**...**`` wrappers and adjacent bold spans (``** **``) are cleaned so the title is plain text. @@ -118,8 +132,8 @@ def extract_outline(md_path: Path) -> list[dict]: continue # Style 1: standard Markdown heading - if stripped.startswith("#"): - title = _clean_bold_title(stripped.lstrip("#").strip()) + if m := _ATX_HEADING_RE.fullmatch(line.rstrip("\r\n")): + title = _clean_bold_title(_strip_atx_closing_hashes(m.group(1) or "").strip()) if title: outline.append({"title": title, "line": lineno}) diff --git a/backend/tests/test_file_outline_atx.py b/backend/tests/test_file_outline_atx.py new file mode 100644 index 000000000..7c4e3fac0 --- /dev/null +++ b/backend/tests/test_file_outline_atx.py @@ -0,0 +1,63 @@ +"""Only actual ATX headings should occupy uploaded-document outline slots.""" + +import subprocess +import sys + +import pytest + +from deerflow.utils.file_outline import MAX_OUTLINE_ENTRIES, extract_outline + + +@pytest.mark.parametrize("line", ["#tag", "##tag", "####### Too many", " # Code comment", "\t# Code comment", "#\u00a0Not a separator", "\\# Escaped"]) +def test_non_headings_do_not_hide_real_sections(tmp_path, line): + path = tmp_path / "guide.md" + path.write_text((line + "\n") * (MAX_OUTLINE_ENTRIES + 1) + "# Real section\n", encoding="utf-8") + assert extract_outline(path) == [{"title": "Real section", "line": MAX_OUTLINE_ENTRIES + 2}] + + +@pytest.mark.parametrize( + ("line", "title"), + [ + ("# Title", "Title"), + (" ###### Title", "Title"), + ("##\tTitle", "Title"), + ("## Title ### ", "Title"), + ("## Title\t###\t", "Title"), + ("# **Overview** ###", "Overview"), + ("# Title###", "Title###"), + ("# Title ### suffix", "Title ### suffix"), + ("# Title \\###", "Title \\###"), + ("# 标题 ###", "标题"), + ], +) +def test_atx_titles_and_physical_lines(tmp_path, line, title): + path = tmp_path / "guide.md" + path.write_text("Intro\n\n" + line + "\n", encoding="utf-8") + assert extract_outline(path) == [{"title": title, "line": 3}] + + +@pytest.mark.parametrize("line", ["#", "### ", "## ###", "#\t###\t"]) +def test_empty_atx_headings_do_not_create_entries(tmp_path, line): + path = tmp_path / "guide.md" + path.write_text(line + "\n", encoding="utf-8") + assert extract_outline(path) == [] + + +def test_long_whitespace_without_closing_hashes_finishes_promptly(tmp_path): + """Exercise heading recognition under a generous process deadline.""" + path = tmp_path / "long-heading.md" + path.write_text("# Title" + " " * 262144 + "suffix\n", encoding="utf-8") + # Recognition must survive long whitespace; summary budgets may clip titles. + subprocess.run( + [ + sys.executable, + "-c", + "from pathlib import Path; import sys; from deerflow.utils.file_outline import extract_outline; result = extract_outline(Path(sys.argv[1])); " + "assert len(result) == 1 and result[0]['line'] == 1 and result[0]['title'].startswith('Title')", + str(path), + ], + check=True, + timeout=10, + capture_output=True, + text=True, + )