diff --git a/README.md b/README.md index e5ac3d81dd..2d3e6fc183 100644 --- a/README.md +++ b/README.md @@ -2525,6 +2525,9 @@ Ranged `read_file` calls count lines the same way on every sandbox provider: a l Concurrent reads and writes to the same file share a gate across synchronous and asynchronous tool calls. Async callers waiting for that gate do not occupy worker threads needed to finish the current read or write. Cancelling a waiting call leaves the current operation running; a call that already started file inspection still waits for that work to finish before releasing its gate. The built-in `grep` tool searches either one text file or all matching text files below a directory, so an agent can search an uploaded document directly without first broadening the request to the entire uploads directory. +On the local sandbox, `grep(literal=True)` also searches long lines in minified text or single-line JSON, within the existing file-size limit. Case-insensitive literal matching avoids repeated-prefix rescans while preserving Unicode case behavior. Regex searches retain their long-line safety guard. + +Across sandbox providers, returned previews show only the beginning of each matched line (200 characters by default), so a match later in a long line may not appear in the preview. Use the reported path and line number to inspect the source. The `truncated` flag concerns match-count limits, not preview length. E2B's `glob` filter preserves spaces, quotes, and dollar signs in filename patterns, while wildcard matching and root-relative directory scoping remain unchanged. E2B `grep` also preserves colons in file and directory names when reporting matching paths and line numbers. `glob` and `grep` preserve non-LF separator characters, such as vertical tabs and Unicode line separators, inside returned paths on E2B, BoxLite, Tenki, and OpenSandbox; `grep` also preserves them inside matched text. diff --git a/backend/packages/harness/deerflow/sandbox/AGENTS.md b/backend/packages/harness/deerflow/sandbox/AGENTS.md index 7d4266b0e4..b95ffd7353 100644 --- a/backend/packages/harness/deerflow/sandbox/AGENTS.md +++ b/backend/packages/harness/deerflow/sandbox/AGENTS.md @@ -113,6 +113,9 @@ Fork lookups reject invalidated clients; cancelled acquires leave borrowers a pe **Sandbox Tools** (in `packages/harness/deerflow/sandbox/tools.py`): +Case-insensitive literal grep uses KMP + single-character `re.IGNORECASE` for +linear work and Unicode parity. Regression: `tests/test_local_grep_long_lines.py`. + Remote `list_dir` prints the root, then prunes ignored descendants before the 500-entry cap. Preserve root literals, host case policy, status markers and the subshell. Keep parser filtering as a backstop; test ignored roots, metacharacters, diff --git a/backend/packages/harness/deerflow/sandbox/search.py b/backend/packages/harness/deerflow/sandbox/search.py index 0129df04a9..e5f7808b9d 100644 --- a/backend/packages/harness/deerflow/sandbox/search.py +++ b/backend/packages/harness/deerflow/sandbox/search.py @@ -3,6 +3,7 @@ import fnmatch import os import re import stat +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -196,6 +197,40 @@ def find_glob_matches(root: Path, pattern: str, *, include_dirs: bool = False, m return matches, truncated +def _make_literal_matcher(pattern: str, *, case_sensitive: bool) -> Callable[[str], bool]: + if case_sensitive: + return lambda line: pattern in line + if not pattern: + return lambda _line: True + + # KMP bounds comparisons by pattern + text length. Matching one character + # at a time retains re.IGNORECASE's Unicode equivalences without rescanning + # a long escaped prefix. Full-string casefold would change cases such as ß. + by_character = {character: re.compile(re.escape(character), re.IGNORECASE).fullmatch for character in set(pattern)} + character_matches = [by_character[character] for character in pattern] + failure = [0] * len(pattern) + prefix_length = 0 + for index in range(1, len(pattern)): + while prefix_length and not character_matches[prefix_length](pattern[index]): + prefix_length = failure[prefix_length - 1] + if character_matches[prefix_length](pattern[index]): + prefix_length += 1 + failure[index] = prefix_length + + def matches_line(line: str) -> bool: + matched = 0 + for character in line: + while matched and not character_matches[matched](character): + matched = failure[matched - 1] + if character_matches[matched](character): + matched += 1 + if matched == len(pattern): + return True + return False + + return matches_line + + def find_grep_matches( root: Path, pattern: str, @@ -217,11 +252,14 @@ def find_grep_matches( if not root_is_file and not root.is_dir(): raise NotADirectoryError(root) - regex_source = re.escape(pattern) if literal else pattern - flags = 0 if case_sensitive else re.IGNORECASE - regex = re.compile(regex_source, flags) + if literal: + matches_line = _make_literal_matcher(pattern, case_sensitive=case_sensitive) + else: + regex = re.compile(pattern, 0 if case_sensitive else re.IGNORECASE) + matches_line = regex.search - # Skip lines longer than this to prevent ReDoS on minified / no-newline files. + # Only regex searches need the long-line work guard. Literal matching + # avoids repeated-prefix rescans, including case-insensitive searches. _max_line_chars = line_summary_length * 10 def handle_search_error(error: OSError) -> None: @@ -265,9 +303,9 @@ def find_grep_matches( for line_number, line in enumerate(handle, start=1): if line.endswith("\r\n"): line = line[:-2] + "\n" - if len(line) > _max_line_chars: + if not literal and len(line) > _max_line_chars: continue - if regex.search(line): + if matches_line(line): matches.append( GrepMatch( path=str(file_path), diff --git a/backend/tests/test_local_grep_long_lines.py b/backend/tests/test_local_grep_long_lines.py new file mode 100644 index 0000000000..acde6a6e09 --- /dev/null +++ b/backend/tests/test_local_grep_long_lines.py @@ -0,0 +1,157 @@ +"""Fixed-string local searches must not inherit the regex long-line guard.""" + +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping +from deerflow.sandbox.search import GrepMatch, find_grep_matches, truncate_line + + +@pytest.mark.parametrize("single_file", [False, True]) +@pytest.mark.parametrize("case_sensitive", [False, True]) +@pytest.mark.parametrize("ending", ["\n", "\r\n", ""]) +def test_literal_grep_finds_text_beyond_long_line_guard(tmp_path: Path, single_file: bool, case_sensitive: bool, ending: str) -> None: + target = tmp_path / "data.json" + line = "x" * 2500 + "Needle[0].value" + target.write_text("header\n" + line + ending, encoding="utf-8", newline="") + root = target if single_file else tmp_path + pattern = "Needle[0].value" if case_sensitive else "needle[0].VALUE" + + matches, truncated = find_grep_matches(root, pattern, literal=True, case_sensitive=case_sensitive) + + assert matches == [GrepMatch(path=str(target.resolve()), line_number=2, line=truncate_line(line))] + assert len(matches[0].line) == 200 + assert truncated is False + + +def test_regex_grep_keeps_long_line_guard(tmp_path: Path) -> None: + target = tmp_path / "data.txt" + target.write_text("x" * 2500 + "needle\nshort needle\n", encoding="utf-8", newline="") + + matches, truncated = find_grep_matches(target, "needle") + + assert matches == [GrepMatch(path=str(target.resolve()), line_number=2, line="short needle")] + assert truncated is False + + +def test_literal_grep_keeps_file_size_limit(tmp_path: Path) -> None: + target = tmp_path / "data.txt" + target.write_text("x" * 2500 + "needle", encoding="utf-8") + + matches, truncated = find_grep_matches(target, "needle", literal=True, max_file_size=2500) + + assert matches == [] + assert truncated is False + + +def test_literal_grep_keeps_glob_filter(tmp_path: Path) -> None: + target = tmp_path / "data.json" + target.write_text("x" * 2500 + "needle", encoding="utf-8") + + matches, truncated = find_grep_matches(tmp_path, "needle", literal=True, glob_pattern="*.txt") + + assert matches == [] + assert truncated is False + + +def test_literal_grep_keeps_result_limit(tmp_path: Path) -> None: + target = tmp_path / "data.txt" + line = "x" * 2500 + "needle\n" + target.write_text(line * 3, encoding="utf-8", newline="") + + matches, truncated = find_grep_matches(target, "needle", literal=True, max_results=2) + + assert [match.line_number for match in matches] == [1, 2] + assert all(len(match.line) == 200 for match in matches) + assert truncated is True + + +def test_literal_grep_keeps_binary_file_exclusion(tmp_path: Path) -> None: + target = tmp_path / "data.bin" + target.write_bytes(b"\0" + b"x" * 2500 + b"needle") + + matches, truncated = find_grep_matches(target, "needle", literal=True) + + assert matches == [] + assert truncated is False + + +@pytest.mark.parametrize("single_file", [False, True]) +def test_local_sandbox_literal_grep_keeps_virtual_paths(tmp_path: Path, single_file: bool) -> None: + target = tmp_path / "data.json" + line = "x" * 2500 + "needle" + target.write_text(line, encoding="utf-8") + virtual_root = "/mnt/user-data/workspace" + sandbox = LocalSandbox(id="local:test", path_mappings=[PathMapping(container_path=virtual_root, local_path=str(tmp_path))]) + root = f"{virtual_root}/data.json" if single_file else virtual_root + + matches, truncated = sandbox.grep(root, "needle", literal=True) + + assert matches == [GrepMatch(path=f"{virtual_root}/data.json", line_number=1, line=truncate_line(line))] + assert truncated is False + + +@pytest.mark.parametrize("case_sensitive", [False, True]) +@pytest.mark.parametrize( + ("pattern", "body"), + [ + ("aaaaab", "a" * 30 + "b"), + ("aAaB", "a" * 30 + "AAab"), + ("i", "İ"), + ("i", "ı"), + ("s", "ſ"), + ("k", "K"), + ("iıiX", "İİİİX"), + ("ΣσςX", "ςΣσςX"), + ("β", "ϐ"), + ("μ", "µ"), + ("φ", "ϕ"), + ("ss", "ß"), + ("ß", "ẞ"), + ("i\u0307", "İ"), + ("Needle[0].value", "needle[0].VALUE"), + ("", "no needle"), + ("a" * 60, "a" * 30), + ], +) +def test_literal_long_line_preserves_regex_case_semantics(tmp_path: Path, pattern: str, body: str, case_sensitive: bool) -> None: + target = tmp_path / "unicode.txt" + line = "x" * 2500 + body + target.write_text(line, encoding="utf-8") + expected = re.search(re.escape(pattern), line, 0 if case_sensitive else re.IGNORECASE) is not None + + matches, truncated = find_grep_matches(target, pattern, literal=True, case_sensitive=case_sensitive) + + assert bool(matches) is expected + assert truncated is False + + +def test_case_insensitive_literal_repeated_prefix_finishes_within_deadline(tmp_path: Path) -> None: + missing = tmp_path / "missing.txt" + present = tmp_path / "present.txt" + missing.write_text("a" * 900_000, encoding="utf-8") + present.write_text("a" * 900_000 + "b", encoding="utf-8") + env = os.environ.copy() + harness = Path(__file__).resolve().parents[1] / "packages" / "harness" + env["PYTHONPATH"] = os.pathsep.join(filter(None, (str(harness), env.get("PYTHONPATH", "")))) + script = """ +import sys +from pathlib import Path +from deerflow.sandbox.search import find_grep_matches +pattern = "a" * 10_000 + "b" +assert find_grep_matches(Path(sys.argv[1]), pattern, literal=True) == ([], False) +matches, truncated = find_grep_matches(Path(sys.argv[2]), pattern, literal=True) +assert len(matches) == 1 and matches[0].line_number == 1 +assert truncated is False +""" + + # A subprocess deadline kills and reaps a regressing regex search rather + # than leaving an uninterruptible worker running after this test fails. + result = subprocess.run([sys.executable, "-c", script, str(missing), str(present)], env=env, capture_output=True, text=True, timeout=10, check=False) + + assert result.returncode == 0, result.stderr diff --git a/backend/tests/test_sandbox_search_contract.py b/backend/tests/test_sandbox_search_contract.py index a46146709b..dcc1783d42 100644 --- a/backend/tests/test_sandbox_search_contract.py +++ b/backend/tests/test_sandbox_search_contract.py @@ -261,6 +261,24 @@ def test_grep_literal_matches_regex_metacharacters(provider, tmp_path, pattern, assert truncated is False +@pytest.mark.parametrize("single_file", [False, True], ids=["directory", "file"]) +@pytest.mark.parametrize("case_sensitive", [False, True], ids=["ignore-case", "match-case"]) +def test_grep_literal_finds_long_lines_with_bounded_prefix_summaries(provider, tmp_path, single_file, case_sensitive): + root = tmp_path / "long-lines" + root.mkdir() + target = root / "data.json" + target.write_text("header\n" + "x" * 2500 + "Needle[0].value\nNeedle0Xvalue\n", encoding="utf-8") + search_root = target if single_file else root + pattern = "Needle[0].value" if case_sensitive else "needle[0].VALUE" + + matches, truncated = provider.sandbox.grep(str(search_root), pattern, literal=True, case_sensitive=case_sensitive) + + # Search the full eligible line even when the fixed-length preview ends + # before the needle. `truncated` describes result limits, not preview text. + assert [(match.path, match.line_number, match.line) for match in matches] == [(str(target), 2, "x" * 197 + "...")] + assert truncated is False + + def test_grep_glob_filter_is_root_relative(provider, tree): matches, truncated = provider.sandbox.grep(str(tree), "needle", glob="nested/*.txt") assert [(m.path, m.line_number) for m in matches] == [(str(tree / "nested" / "result.txt"), 1)]