mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 11:20:35 +00:00
* fix(sandbox): match local grep lines without their terminator find_grep_matches ran the regex on each line with its trailing "\n", so patterns that can consume the newline (\s$, [^a-z]$, \n) matched every newline-ended line while the returned line was shown stripped. Strip the LF or CRLF terminator before matching, as grep(1) and the remote sandbox providers do. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs(sandbox): clarify local grep CRLF handling * test(sandbox): cover literal grep line terminators --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
340 lines
12 KiB
Python
340 lines
12 KiB
Python
import errno
|
|
import fnmatch
|
|
import os
|
|
import re
|
|
import stat
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
IGNORE_PATTERNS = [
|
|
".git",
|
|
".svn",
|
|
".hg",
|
|
".bzr",
|
|
"node_modules",
|
|
"__pycache__",
|
|
".venv",
|
|
"venv",
|
|
".env",
|
|
"env",
|
|
".tox",
|
|
".nox",
|
|
".eggs",
|
|
"*.egg-info",
|
|
"site-packages",
|
|
"dist",
|
|
"build",
|
|
".next",
|
|
".nuxt",
|
|
".output",
|
|
".turbo",
|
|
"target",
|
|
"out",
|
|
".idea",
|
|
".vscode",
|
|
"*.swp",
|
|
"*.swo",
|
|
"*~",
|
|
".project",
|
|
".classpath",
|
|
".settings",
|
|
".DS_Store",
|
|
"Thumbs.db",
|
|
"desktop.ini",
|
|
"*.lnk",
|
|
"*.log",
|
|
"*.tmp",
|
|
"*.temp",
|
|
".upload-*.part",
|
|
"*.bak",
|
|
"*.cache",
|
|
".cache",
|
|
"logs",
|
|
".coverage",
|
|
"coverage",
|
|
".nyc_output",
|
|
"htmlcov",
|
|
".pytest_cache",
|
|
".mypy_cache",
|
|
".ruff_cache",
|
|
]
|
|
|
|
DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000
|
|
DEFAULT_LINE_SUMMARY_LENGTH = 200
|
|
_GREP_RACE_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.ELOOP, errno.EISDIR, getattr(errno, "ESTALE", errno.ENOENT)}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GrepMatch:
|
|
path: str
|
|
line_number: int
|
|
line: str
|
|
|
|
|
|
# ``should_ignore_name`` runs once per directory entry during glob/grep tree
|
|
# walks, so we avoid ~50 ``fnmatch`` calls per name. Most ignore patterns are
|
|
# literal names (O(1) set lookup after normcase); the few glob patterns are
|
|
# pre-translated into a single combined regex. ``os.path.normcase`` keeps the
|
|
# same case behavior ``fnmatch`` applies (case-sensitive on POSIX, folded on
|
|
# Windows).
|
|
_EXACT_IGNORE_NAMES = frozenset(os.path.normcase(p) for p in IGNORE_PATTERNS if not any(c in p for c in "*?["))
|
|
_GLOB_IGNORE_PATTERNS = [p for p in IGNORE_PATTERNS if any(c in p for c in "*?[")]
|
|
_GLOB_IGNORE_RE = re.compile("|".join(fnmatch.translate(os.path.normcase(p)) for p in _GLOB_IGNORE_PATTERNS)) if _GLOB_IGNORE_PATTERNS else None
|
|
|
|
|
|
def should_ignore_name(name: str) -> bool:
|
|
normalized = os.path.normcase(name)
|
|
if normalized in _EXACT_IGNORE_NAMES:
|
|
return True
|
|
return _GLOB_IGNORE_RE is not None and _GLOB_IGNORE_RE.match(normalized) is not None
|
|
|
|
|
|
def should_ignore_path(path: str) -> bool:
|
|
return any(should_ignore_name(segment) for segment in path.replace("\\", "/").split("/") if segment)
|
|
|
|
|
|
def _match_segments(pattern_parts: tuple[str, ...], path_parts: tuple[str, ...]) -> bool:
|
|
# Positions in ``path_parts`` reachable after consuming each pattern part;
|
|
# ``**`` spans zero or more whole segments, any other part exactly one.
|
|
reachable = {0}
|
|
for part in pattern_parts:
|
|
if part == "**":
|
|
reachable = set(range(min(reachable), len(path_parts) + 1))
|
|
else:
|
|
reachable = {i + 1 for i in reachable if i < len(path_parts) and fnmatch.fnmatchcase(path_parts[i], part)}
|
|
if not reachable:
|
|
return False
|
|
return len(path_parts) in reachable
|
|
|
|
|
|
def should_ignore_path_under_root(path: str, root: str) -> bool:
|
|
"""Apply :func:`should_ignore_path` to ``path`` as seen from ``root``.
|
|
|
|
Ignore patterns are scoped to the search root, the way ``list_dir`` already
|
|
applies them: an ignored name hides its own descendants, so searching
|
|
``build`` -- or any path below an ignored ancestor -- still returns its
|
|
contents. Testing the absolute path instead silently hides every result as
|
|
soon as the search root, or one of its ancestors, is itself an ignored
|
|
name, so ``glob``/``grep`` reported "no matches" for a directory the local
|
|
walk lists without any trouble.
|
|
"""
|
|
root_clean = root.rstrip("/")
|
|
if not root_clean:
|
|
root_clean = "/"
|
|
if path == root_clean:
|
|
# The caller asked for this exact path. There is nothing relative to
|
|
# test, and an ignored name must not hide what was explicitly
|
|
# requested (a single-file ``grep`` hits this on every match).
|
|
return False
|
|
prefix = "/" if root_clean == "/" else f"{root_clean}/"
|
|
if path.startswith(prefix):
|
|
return should_ignore_path(path[len(prefix) :])
|
|
return should_ignore_path(path)
|
|
|
|
|
|
def path_matches(pattern: str, rel_path: str) -> bool:
|
|
"""Match ``rel_path`` (relative to the search root) against a glob pattern.
|
|
|
|
A pattern without ``/`` matches the basename at any depth (``*.py``). Any
|
|
other pattern is anchored at the search root, and ``**`` matches zero or
|
|
more directories (``src/**/*.py`` includes ``src/top.py``).
|
|
``PurePosixPath.match`` is not used for these: it matches from the right
|
|
and treats ``**`` as a single-segment ``*``.
|
|
"""
|
|
if "/" not in pattern:
|
|
return PurePosixPath(rel_path).match(pattern)
|
|
return _match_segments(PurePosixPath(pattern).parts, PurePosixPath(rel_path).parts)
|
|
|
|
|
|
def truncate_line(line: str, max_chars: int = DEFAULT_LINE_SUMMARY_LENGTH) -> str:
|
|
line = line.rstrip("\n\r")
|
|
if len(line) <= max_chars:
|
|
return line
|
|
return line[: max_chars - 3] + "..."
|
|
|
|
|
|
def is_binary_file(path: Path, sample_size: int = 8192) -> bool:
|
|
with path.open("rb") as handle:
|
|
return b"\0" in handle.read(sample_size)
|
|
|
|
|
|
def find_glob_matches(root: Path, pattern: str, *, include_dirs: bool = False, max_results: int = 200, relative_prefix: str = "") -> tuple[list[str], bool]:
|
|
matches: list[str] = []
|
|
truncated = False
|
|
root = root.resolve()
|
|
|
|
if not root.exists():
|
|
raise FileNotFoundError(root)
|
|
if not root.is_dir():
|
|
raise NotADirectoryError(root)
|
|
|
|
def handle_walk_error(error: OSError) -> None:
|
|
# Entries may disappear or change type during enumeration. Other
|
|
# errors, such as permission failures, must reach the caller rather
|
|
# than being reported as an incomplete successful search.
|
|
if error.errno not in _GREP_RACE_ERRNOS:
|
|
raise error
|
|
|
|
for current_root, dirs, files in os.walk(root, onerror=handle_walk_error):
|
|
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
|
|
# root is already resolved; os.walk builds current_root by joining under root,
|
|
# so relative_to() works without an extra stat()/resolve() per directory.
|
|
rel_dir = Path(current_root).relative_to(root)
|
|
|
|
if include_dirs:
|
|
for name in dirs:
|
|
rel_path = (rel_dir / name).as_posix()
|
|
if relative_prefix:
|
|
rel_path = f"{relative_prefix}/{rel_path}"
|
|
if path_matches(pattern, rel_path):
|
|
matches.append(str(Path(current_root) / name))
|
|
if len(matches) >= max_results:
|
|
truncated = True
|
|
return matches, truncated
|
|
|
|
for name in files:
|
|
if should_ignore_name(name):
|
|
continue
|
|
rel_path = (rel_dir / name).as_posix()
|
|
if relative_prefix:
|
|
rel_path = f"{relative_prefix}/{rel_path}"
|
|
if path_matches(pattern, rel_path):
|
|
matches.append(str(Path(current_root) / name))
|
|
if len(matches) >= max_results:
|
|
truncated = True
|
|
return matches, truncated
|
|
|
|
return matches, truncated
|
|
|
|
|
|
def _make_literal_matcher(pattern: str, *, case_sensitive: bool) -> Callable[[str], bool]:
|
|
if case_sensitive:
|
|
return lambda line: pattern in line
|
|
if not pattern:
|
|
return lambda _line: True
|
|
|
|
# KMP bounds comparisons by pattern + text length. Matching one character
|
|
# at a time retains re.IGNORECASE's Unicode equivalences without rescanning
|
|
# a long escaped prefix. Full-string casefold would change cases such as ß.
|
|
by_character = {character: re.compile(re.escape(character), re.IGNORECASE).fullmatch for character in set(pattern)}
|
|
character_matches = [by_character[character] for character in pattern]
|
|
failure = [0] * len(pattern)
|
|
prefix_length = 0
|
|
for index in range(1, len(pattern)):
|
|
while prefix_length and not character_matches[prefix_length](pattern[index]):
|
|
prefix_length = failure[prefix_length - 1]
|
|
if character_matches[prefix_length](pattern[index]):
|
|
prefix_length += 1
|
|
failure[index] = prefix_length
|
|
|
|
def matches_line(line: str) -> bool:
|
|
matched = 0
|
|
for character in line:
|
|
while matched and not character_matches[matched](character):
|
|
matched = failure[matched - 1]
|
|
if character_matches[matched](character):
|
|
matched += 1
|
|
if matched == len(pattern):
|
|
return True
|
|
return False
|
|
|
|
return matches_line
|
|
|
|
|
|
def find_grep_matches(
|
|
root: Path,
|
|
pattern: str,
|
|
*,
|
|
glob_pattern: str | None = None,
|
|
literal: bool = False,
|
|
case_sensitive: bool = False,
|
|
max_results: int = 100,
|
|
max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES,
|
|
line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH,
|
|
relative_prefix: str = "",
|
|
) -> tuple[list[GrepMatch], bool]:
|
|
matches: list[GrepMatch] = []
|
|
truncated = False
|
|
root = root.resolve()
|
|
|
|
if not root.exists():
|
|
raise FileNotFoundError(root)
|
|
root_is_file = root.is_file()
|
|
if not root_is_file and not root.is_dir():
|
|
raise NotADirectoryError(root)
|
|
|
|
if literal:
|
|
matches_line = _make_literal_matcher(pattern, case_sensitive=case_sensitive)
|
|
else:
|
|
regex = re.compile(pattern, 0 if case_sensitive else re.IGNORECASE)
|
|
matches_line = regex.search
|
|
|
|
# Only regex searches need the long-line work guard. Literal matching
|
|
# avoids repeated-prefix rescans, including case-insensitive searches.
|
|
_max_line_chars = line_summary_length * 10
|
|
|
|
def handle_search_error(error: OSError) -> None:
|
|
# Entries may disappear or change type during enumeration or reading.
|
|
if error.errno not in _GREP_RACE_ERRNOS:
|
|
raise error
|
|
|
|
def candidate_files():
|
|
if root_is_file:
|
|
yield root, root.name
|
|
return
|
|
|
|
for current_root, dirs, files in os.walk(root, onerror=handle_search_error):
|
|
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
|
|
rel_dir = Path(current_root).relative_to(root)
|
|
for name in files:
|
|
if should_ignore_name(name):
|
|
continue
|
|
yield Path(current_root) / name, (rel_dir / name).as_posix()
|
|
|
|
for candidate_path, rel_path in candidate_files():
|
|
if relative_prefix:
|
|
rel_path = f"{relative_prefix}/{rel_path}"
|
|
if glob_pattern is not None and not path_matches(glob_pattern, rel_path):
|
|
continue
|
|
|
|
try:
|
|
if not root_is_file and candidate_path.is_symlink():
|
|
continue
|
|
file_path = candidate_path.resolve()
|
|
if not root_is_file and not file_path.is_relative_to(root):
|
|
continue
|
|
file_stat = file_path.stat()
|
|
if not stat.S_ISREG(file_stat.st_mode):
|
|
continue
|
|
if file_stat.st_size > max_file_size or is_binary_file(file_path):
|
|
continue
|
|
# newline="\n" ends lines where LocalSandbox.read_file does, so a
|
|
# hit's line_number is the start_line that reads it back; a bare
|
|
# "\r" stays content. Like native grep on Unix, local matching
|
|
# excludes LF. LocalSandbox also strips the CR in CRLF, so $ anchors
|
|
# at the end of the line text for both line endings.
|
|
with file_path.open(encoding="utf-8", errors="replace", newline="\n") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
if line.endswith("\r\n"):
|
|
line = line[:-2]
|
|
elif line.endswith("\n"):
|
|
line = line[:-1]
|
|
if not literal and len(line) > _max_line_chars:
|
|
continue
|
|
if matches_line(line):
|
|
matches.append(
|
|
GrepMatch(
|
|
path=str(file_path),
|
|
line_number=line_number,
|
|
line=truncate_line(line, line_summary_length),
|
|
)
|
|
)
|
|
if len(matches) >= max_results:
|
|
truncated = True
|
|
return matches, truncated
|
|
except OSError as exc:
|
|
handle_search_error(exc)
|
|
|
|
return matches, truncated
|