mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
fix(sandbox): search user data instead of hiding it as repository noise (#8366)
* fix(sandbox): search user data instead of hiding it as repository noise (#8364) The ignore list was written to walk source checkouts, but glob, grep and ls ran it over every root, so /mnt/user-data/uploads hid its own *.log files and a grep over *.log answered 'No matches found'. grep also skipped any file above 1 MB during a directory walk without saying so, even when the model named the file itself. ignore_rules_for() now picks the filter per search root: user-data roots hide only repository machinery, a glob the caller spelled out wins even inside a checkout, and a file named as the search root is read at any size. The 1 MB bound still protects directory walks, and the grep tool documents it. * fix(sandbox): label user-data roots from the mount table The public ls/glob/grep tools resolve /mnt/user-data/... to host paths before they call the local provider, so checking the virtual prefix never matched a real entry point: an uploaded app.log stayed hidden whenever the host tree sat inside the checkout the Gateway runs from. Ask the provider's mount table instead, and keep in-flight .upload-*.part files hidden in user data as well as in a repository. * fix(sandbox): keep the repository filter outside user data Only a root the mount table labels as /mnt/user-data changes behaviour. A skills mount or an ad-hoc host path keeps hiding *.log as before, so the existing mounted-root tests stay valid instead of widening content visibility to every root outside a checkout. * fix(sandbox): keep an explicit grep budget on a named file The traversal default stops a directory walk from reading megabytes it was never asked about. A caller that names one file and passes its own budget still gets that budget enforced, which the literal-grep suite pins. --------- Co-authored-by: Aling33 <Aling33@users.noreply.github.com>
This commit is contained in:
parent
ccc1d1ea19
commit
6b463d1f1a
@ -78,6 +78,14 @@ Windows. The portable string assertions do not replace it.
|
||||
because `splitlines()` also splits on other Unicode separators.
|
||||
- A failed or incomplete directory traversal is an error. Returning a partial
|
||||
listing makes the model believe that files do not exist.
|
||||
- Search roots decide what counts as noise. `search.ignore_rules_for()` hides
|
||||
`*.log`, `logs/` and the other data-looking names everywhere except the user's
|
||||
own files; a glob the caller spelled out wins over those defaults. The local
|
||||
provider labels a root from its mount table, so a host path the tools resolved
|
||||
from `/mnt/user-data/...` still counts as user data, and a skills mount or any
|
||||
other root keeps the repository filter. In-flight `.upload-*.part` files stay
|
||||
hidden in both root types. A file named as the search root is read at any size;
|
||||
the 1 MB bound only skips files a directory walk discovers.
|
||||
- Same-path write serialization is scoped to `(sandbox.id, path)`. Two isolated
|
||||
sandboxes use the same virtual paths and must not contend.
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.sandbox.search import should_ignore_name
|
||||
from deerflow.sandbox.search import ignore_rules_for
|
||||
|
||||
|
||||
def list_dir(path: str, max_depth: int = 2) -> list[str]:
|
||||
def list_dir(path: str, max_depth: int = 2, *, user_data: bool = False) -> list[str]:
|
||||
"""
|
||||
List files and directories up to max_depth levels deep.
|
||||
|
||||
@ -11,11 +11,13 @@ def list_dir(path: str, max_depth: int = 2) -> list[str]:
|
||||
path: The root directory path to list.
|
||||
max_depth: Maximum depth to traverse (default: 2).
|
||||
1 = only direct children, 2 = children + grandchildren, etc.
|
||||
user_data: The path belongs to the user's own files, which the ignore
|
||||
rules must not hide outside a checkout either.
|
||||
|
||||
Returns:
|
||||
A list of absolute paths for files and directories,
|
||||
excluding items matching IGNORE_PATTERNS. An existing empty
|
||||
directory returns an empty list.
|
||||
A list of absolute paths for files and directories, excluding repository
|
||||
noise and, inside a git checkout, data-looking names such as ``*.log``.
|
||||
An existing empty directory returns an empty list.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If ``path`` does not exist or is not a directory.
|
||||
@ -24,6 +26,7 @@ def list_dir(path: str, max_depth: int = 2) -> list[str]:
|
||||
"""
|
||||
result: list[str] = []
|
||||
root_path = Path(path).resolve()
|
||||
rules = ignore_rules_for(root_path, user_data=user_data)
|
||||
|
||||
if not root_path.is_dir():
|
||||
raise FileNotFoundError(path)
|
||||
@ -41,7 +44,7 @@ def list_dir(path: str, max_depth: int = 2) -> list[str]:
|
||||
return
|
||||
|
||||
for item in current_path.iterdir():
|
||||
if should_ignore_name(item.name):
|
||||
if rules.should_ignore(item.name):
|
||||
continue
|
||||
|
||||
if item.is_symlink():
|
||||
|
||||
@ -30,6 +30,13 @@ from deerflow.utils.atomic_write import publish_bytes_atomically
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_user_data_container(container_path: str) -> bool:
|
||||
"""Whether a mount's container path belongs to the user-data tree."""
|
||||
normalized = container_path.rstrip("/")
|
||||
return normalized == VIRTUAL_PATH_PREFIX or normalized.startswith(VIRTUAL_PATH_PREFIX + "/")
|
||||
|
||||
|
||||
# Default wall-clock timeout (seconds) for a single host bash command. A
|
||||
# blocking foreground command (for example a server started without
|
||||
# backgrounding) is terminated after this long so the agent's turn cannot hang
|
||||
@ -580,6 +587,25 @@ class LocalSandbox(Sandbox):
|
||||
|
||||
return best_mapping.read_only
|
||||
|
||||
def _is_user_data_path(self, path: str) -> bool:
|
||||
"""Whether *path* is the user's own files rather than repository noise.
|
||||
|
||||
The public ls/glob/grep tools resolve ``/mnt/user-data/...`` to host
|
||||
paths before they call a provider, so the virtual prefix alone would miss
|
||||
every real entry point. Ask the mount table instead: a container path
|
||||
under ``/mnt/user-data``, or a host path inside the local root of that
|
||||
mapping.
|
||||
"""
|
||||
mapping = self._find_path_mapping(path)
|
||||
if mapping is not None:
|
||||
return _is_user_data_container(mapping[0].container_path)
|
||||
resolved = os.path.realpath(path)
|
||||
for candidate in self._mappings_by_local_specificity:
|
||||
root = self._resolved_local_paths[candidate]
|
||||
if resolved == root or resolved.startswith(root + os.sep):
|
||||
return _is_user_data_container(candidate.container_path)
|
||||
return False
|
||||
|
||||
def _find_path_mapping(self, path: str) -> tuple[PathMapping, str] | None:
|
||||
path_str = str(path)
|
||||
|
||||
@ -1232,7 +1258,7 @@ class LocalSandbox(Sandbox):
|
||||
pass
|
||||
|
||||
try:
|
||||
entries = list_dir(resolved_path, max_depth)
|
||||
entries = list_dir(resolved_path, max_depth, user_data=self._is_user_data_path(path))
|
||||
except FileNotFoundError:
|
||||
# The requested path may exist only in the container, as the
|
||||
# parent of mounted sub-directories (e.g. /mnt/skills with only
|
||||
@ -1407,6 +1433,9 @@ class LocalSandbox(Sandbox):
|
||||
return mounts
|
||||
|
||||
def glob(self, path: str, pattern: str, *, include_dirs: bool = False, max_results: int = 200) -> tuple[list[str], bool]:
|
||||
# /mnt/user-data is the user's own files: their ``*.log`` upload is
|
||||
# content, not repository noise, even if the host path sits in a checkout.
|
||||
user_data = self._is_user_data_path(path)
|
||||
try:
|
||||
mounts = self._virtual_search_mounts(path)
|
||||
if mounts is not None:
|
||||
@ -1418,13 +1447,20 @@ class LocalSandbox(Sandbox):
|
||||
if len(results) <= max_results:
|
||||
# One lookahead match proves whether the combined view
|
||||
# exceeds the cap, without collecting a cap per mount.
|
||||
matches, _ = find_glob_matches(root, pattern, include_dirs=include_dirs, max_results=max_results - len(results) + 1, relative_prefix=child)
|
||||
matches, _ = find_glob_matches(
|
||||
root,
|
||||
pattern,
|
||||
include_dirs=include_dirs,
|
||||
max_results=max_results - len(results) + 1,
|
||||
relative_prefix=child,
|
||||
user_data=user_data,
|
||||
)
|
||||
results.extend(f"{virtual_dir}/{Path(match).relative_to(root).as_posix()}" for match in matches)
|
||||
if len(results) > max_results:
|
||||
return results[:max_results], True
|
||||
return results, False
|
||||
resolved_path = Path(self._resolve_path(path))
|
||||
matches, truncated = find_glob_matches(resolved_path, pattern, include_dirs=include_dirs, max_results=max_results)
|
||||
matches, truncated = find_glob_matches(resolved_path, pattern, include_dirs=include_dirs, max_results=max_results, user_data=user_data)
|
||||
except OSError as e:
|
||||
if e.filename is None:
|
||||
raise
|
||||
@ -1444,6 +1480,7 @@ class LocalSandbox(Sandbox):
|
||||
case_sensitive: bool = False,
|
||||
max_results: int = 100,
|
||||
) -> tuple[list[GrepMatch], bool]:
|
||||
user_data = self._is_user_data_path(path)
|
||||
try:
|
||||
mounts = self._virtual_search_mounts(path)
|
||||
if mounts is not None:
|
||||
@ -1457,6 +1494,7 @@ class LocalSandbox(Sandbox):
|
||||
case_sensitive=case_sensitive,
|
||||
max_results=max_results - len(results) + 1,
|
||||
relative_prefix=child,
|
||||
user_data=user_data,
|
||||
)
|
||||
results.extend(GrepMatch(path=f"{path.rstrip('/')}/{child}/{Path(match.path).relative_to(root).as_posix()}", line_number=match.line_number, line=match.line) for match in matches)
|
||||
if len(results) > max_results:
|
||||
@ -1470,6 +1508,7 @@ class LocalSandbox(Sandbox):
|
||||
literal=literal,
|
||||
case_sensitive=case_sensitive,
|
||||
max_results=max_results,
|
||||
user_data=user_data,
|
||||
)
|
||||
except OSError as e:
|
||||
if e.filename is None:
|
||||
|
||||
@ -7,7 +7,9 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
IGNORE_PATTERNS = [
|
||||
# Repository noise: version-control metadata, dependency trees, build output
|
||||
# and editor scratch. These names are machinery, never user content.
|
||||
REPO_IGNORE_PATTERNS = [
|
||||
".git",
|
||||
".svn",
|
||||
".hg",
|
||||
@ -43,14 +45,6 @@ IGNORE_PATTERNS = [
|
||||
"Thumbs.db",
|
||||
"desktop.ini",
|
||||
"*.lnk",
|
||||
"*.log",
|
||||
"*.tmp",
|
||||
"*.temp",
|
||||
".upload-*.part",
|
||||
"*.bak",
|
||||
"*.cache",
|
||||
".cache",
|
||||
"logs",
|
||||
".coverage",
|
||||
"coverage",
|
||||
".nyc_output",
|
||||
@ -60,6 +54,21 @@ IGNORE_PATTERNS = [
|
||||
".ruff_cache",
|
||||
]
|
||||
|
||||
# Data-looking names that are ordinary user files. Uploads arrive as ``*.log``
|
||||
# and ``logs/`` is a directory the user made, so outside a git checkout these
|
||||
# stay searchable; inside one they are the repository noise the list is for.
|
||||
# ``.upload-*.part`` is the in-flight upload suffix and stays ignored everywhere.
|
||||
USER_DATA_PATTERNS = ["*.log", "*.tmp", "*.temp", "*.bak", "*.cache", ".cache", "logs"]
|
||||
|
||||
# In-flight uploads are repository machinery, not content: they stay hidden in
|
||||
# user data too, until publication renames them.
|
||||
STAGING_PATTERNS = [".upload-*.part"]
|
||||
|
||||
# ``should_ignore_name`` keeps answering with the repository defaults.
|
||||
IGNORE_PATTERNS = [*REPO_IGNORE_PATTERNS, *USER_DATA_PATTERNS, *STAGING_PATTERNS]
|
||||
|
||||
# Directory walks skip files above this so one huge log cannot stall a search;
|
||||
# a file named as the search root is read whatever its size.
|
||||
DEFAULT_MAX_FILE_SIZE_BYTES = 1_000_000
|
||||
DEFAULT_LINE_SUMMARY_LENGTH = 200
|
||||
_GREP_RACE_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.ELOOP, errno.EISDIR, getattr(errno, "ESTALE", errno.ENOENT)}
|
||||
@ -78,16 +87,54 @@ class GrepMatch:
|
||||
# pre-translated into a single combined regex. ``os.path.normcase`` keeps the
|
||||
# same case behavior ``fnmatch`` applies (case-sensitive on POSIX, folded on
|
||||
# Windows).
|
||||
_EXACT_IGNORE_NAMES = frozenset(os.path.normcase(p) for p in IGNORE_PATTERNS if not any(c in p for c in "*?["))
|
||||
_GLOB_IGNORE_PATTERNS = [p for p in IGNORE_PATTERNS if any(c in p for c in "*?[")]
|
||||
_GLOB_IGNORE_RE = re.compile("|".join(fnmatch.translate(os.path.normcase(p)) for p in _GLOB_IGNORE_PATTERNS)) if _GLOB_IGNORE_PATTERNS else None
|
||||
def _compile_ignore_patterns(patterns: list[str]) -> tuple[frozenset[str], re.Pattern[str] | None]:
|
||||
exact = frozenset(os.path.normcase(p) for p in patterns if not any(c in p for c in "*?["))
|
||||
globs = [p for p in patterns if any(c in p for c in "*?[")]
|
||||
combined = re.compile("|".join(fnmatch.translate(os.path.normcase(p)) for p in globs)) if globs else None
|
||||
return exact, combined
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IgnoreRules:
|
||||
"""Name filter for one search root."""
|
||||
|
||||
exact: frozenset[str]
|
||||
glob_re: re.Pattern[str] | None
|
||||
|
||||
def should_ignore(self, name: str) -> bool:
|
||||
normalized = os.path.normcase(name)
|
||||
if normalized in self.exact:
|
||||
return True
|
||||
return self.glob_re is not None and self.glob_re.match(normalized) is not None
|
||||
|
||||
|
||||
_REPOSITORY_RULES = IgnoreRules(*_compile_ignore_patterns(IGNORE_PATTERNS))
|
||||
_CONTENT_RULES = IgnoreRules(*_compile_ignore_patterns([*REPO_IGNORE_PATTERNS, *STAGING_PATTERNS]))
|
||||
|
||||
|
||||
def should_ignore_name(name: str) -> bool:
|
||||
normalized = os.path.normcase(name)
|
||||
if normalized in _EXACT_IGNORE_NAMES:
|
||||
return True
|
||||
return _GLOB_IGNORE_RE is not None and _GLOB_IGNORE_RE.match(normalized) is not None
|
||||
"""What a search inside a git checkout hides: the repository defaults."""
|
||||
return _REPOSITORY_RULES.should_ignore(name)
|
||||
|
||||
|
||||
def _names_user_data(pattern: str) -> bool:
|
||||
name = pattern.rsplit("/", 1)[-1]
|
||||
return any(fnmatch.fnmatchcase(name, candidate) for candidate in USER_DATA_PATTERNS)
|
||||
|
||||
|
||||
def ignore_rules_for(search_root: Path, *, glob: str | None = None, user_data: bool = False) -> IgnoreRules:
|
||||
"""Names to skip while searching ``search_root``.
|
||||
|
||||
Everything except the user's own files keeps the repository filter, so a
|
||||
skills mount or an ad-hoc host path hides ``*.log`` as it always did.
|
||||
``user_data`` marks a root the sandbox maps to ``/mnt/user-data``; there
|
||||
``*.log`` and ``logs/`` are content, so only repository noise is hidden. A
|
||||
glob the caller spelled out wins over the defaults: asking for ``*.log`` is
|
||||
a request for logs, even in a checkout.
|
||||
"""
|
||||
if user_data or (glob is not None and _names_user_data(glob)):
|
||||
return _CONTENT_RULES
|
||||
return _REPOSITORY_RULES
|
||||
|
||||
|
||||
def should_ignore_path(path: str) -> bool:
|
||||
@ -159,7 +206,15 @@ def is_binary_file(path: Path, sample_size: int = 8192) -> bool:
|
||||
return b"\0" in handle.read(sample_size)
|
||||
|
||||
|
||||
def find_glob_matches(root: Path, pattern: str, *, include_dirs: bool = False, max_results: int = 200, relative_prefix: str = "") -> tuple[list[str], bool]:
|
||||
def find_glob_matches(
|
||||
root: Path,
|
||||
pattern: str,
|
||||
*,
|
||||
include_dirs: bool = False,
|
||||
max_results: int = 200,
|
||||
relative_prefix: str = "",
|
||||
user_data: bool = False,
|
||||
) -> tuple[list[str], bool]:
|
||||
matches: list[str] = []
|
||||
truncated = False
|
||||
root = root.resolve()
|
||||
@ -169,6 +224,8 @@ def find_glob_matches(root: Path, pattern: str, *, include_dirs: bool = False, m
|
||||
if not root.is_dir():
|
||||
raise NotADirectoryError(root)
|
||||
|
||||
rules = ignore_rules_for(root, glob=pattern, user_data=user_data)
|
||||
|
||||
def handle_walk_error(error: OSError) -> None:
|
||||
# Entries may disappear or change type during enumeration. Other
|
||||
# errors, such as permission failures, must reach the caller rather
|
||||
@ -177,7 +234,7 @@ def find_glob_matches(root: Path, pattern: str, *, include_dirs: bool = False, m
|
||||
raise error
|
||||
|
||||
for current_root, dirs, files in os.walk(root, onerror=handle_walk_error):
|
||||
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
|
||||
dirs[:] = [name for name in dirs if not rules.should_ignore(name)]
|
||||
# root is already resolved; os.walk builds current_root by joining under root,
|
||||
# so relative_to() works without an extra stat()/resolve() per directory.
|
||||
rel_dir = Path(current_root).relative_to(root)
|
||||
@ -194,7 +251,7 @@ def find_glob_matches(root: Path, pattern: str, *, include_dirs: bool = False, m
|
||||
return matches, truncated
|
||||
|
||||
for name in files:
|
||||
if should_ignore_name(name):
|
||||
if rules.should_ignore(name):
|
||||
continue
|
||||
rel_path = (rel_dir / name).as_posix()
|
||||
if relative_prefix:
|
||||
@ -250,9 +307,10 @@ def find_grep_matches(
|
||||
literal: bool = False,
|
||||
case_sensitive: bool = False,
|
||||
max_results: int = 100,
|
||||
max_file_size: int = DEFAULT_MAX_FILE_SIZE_BYTES,
|
||||
max_file_size: int | None = None,
|
||||
line_summary_length: int = DEFAULT_LINE_SUMMARY_LENGTH,
|
||||
relative_prefix: str = "",
|
||||
user_data: bool = False,
|
||||
) -> tuple[list[GrepMatch], bool]:
|
||||
matches: list[GrepMatch] = []
|
||||
truncated = False
|
||||
@ -263,6 +321,9 @@ def find_grep_matches(
|
||||
root_is_file = root.is_file()
|
||||
if not root_is_file and not root.is_dir():
|
||||
raise NotADirectoryError(root)
|
||||
# The traversal budget keeps a directory walk bounded; a file the caller
|
||||
# named is read whatever its size, unless that caller sets its own budget.
|
||||
size_limit = DEFAULT_MAX_FILE_SIZE_BYTES if max_file_size is None else max_file_size
|
||||
|
||||
if literal:
|
||||
matches_line = _make_literal_matcher(pattern, case_sensitive=case_sensitive)
|
||||
@ -274,6 +335,8 @@ def find_grep_matches(
|
||||
# avoids repeated-prefix rescans, including case-insensitive searches.
|
||||
_max_line_chars = line_summary_length * 10
|
||||
|
||||
rules = ignore_rules_for(root, glob=glob_pattern, user_data=user_data)
|
||||
|
||||
def handle_search_error(error: OSError) -> None:
|
||||
# Entries may disappear or change type during enumeration or reading.
|
||||
if error.errno not in _GREP_RACE_ERRNOS:
|
||||
@ -285,10 +348,10 @@ def find_grep_matches(
|
||||
return
|
||||
|
||||
for current_root, dirs, files in os.walk(root, onerror=handle_search_error):
|
||||
dirs[:] = [name for name in dirs if not should_ignore_name(name)]
|
||||
dirs[:] = [name for name in dirs if not rules.should_ignore(name)]
|
||||
rel_dir = Path(current_root).relative_to(root)
|
||||
for name in files:
|
||||
if should_ignore_name(name):
|
||||
if rules.should_ignore(name):
|
||||
continue
|
||||
yield Path(current_root) / name, (rel_dir / name).as_posix()
|
||||
|
||||
@ -307,7 +370,9 @@ def find_grep_matches(
|
||||
file_stat = file_path.stat()
|
||||
if not stat.S_ISREG(file_stat.st_mode):
|
||||
continue
|
||||
if file_stat.st_size > max_file_size or is_binary_file(file_path):
|
||||
if is_binary_file(file_path):
|
||||
continue
|
||||
if file_stat.st_size > size_limit and (not root_is_file or max_file_size is not None):
|
||||
continue
|
||||
# newline="\n" ends lines where LocalSandbox.read_file does, so a
|
||||
# hit's line_number is the start_line that reads it back; a bare
|
||||
|
||||
@ -3232,7 +3232,8 @@ def grep_tool(
|
||||
|
||||
Args:
|
||||
pattern: The string or regex pattern to search for.
|
||||
path: The **absolute** file or root directory to search.
|
||||
path: The **absolute** file or root directory to search. Naming a file
|
||||
searches it at any size; a directory walk skips files over 1 MB.
|
||||
description: Optional short explanation of this search shown in the UI.
|
||||
glob: Optional glob filter for candidate files, for example `**/*.py`.
|
||||
literal: Whether to treat `pattern` as a plain string. Default is False.
|
||||
|
||||
167
backend/tests/test_sandbox_search_user_files.py
Normal file
167
backend/tests/test_sandbox_search_user_files.py
Normal file
@ -0,0 +1,167 @@
|
||||
"""Searches hide repository noise, not the user's own files.
|
||||
|
||||
``IGNORE_PATTERNS`` was written to walk source checkouts, but the same filter
|
||||
ran over ``/mnt/user-data/uploads``, where ``*.log`` and ``logs/`` are the
|
||||
content the agent was asked to read. A user-data root now hides only repository
|
||||
noise, an explicit glob wins even inside a checkout, and a file named as the
|
||||
search root is read whatever its size.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from deerflow.sandbox.local.list_dir import list_dir
|
||||
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping
|
||||
from deerflow.sandbox.search import (
|
||||
DEFAULT_MAX_FILE_SIZE_BYTES,
|
||||
find_glob_matches,
|
||||
find_grep_matches,
|
||||
ignore_rules_for,
|
||||
)
|
||||
from deerflow.sandbox.tools import glob_tool, grep_tool, ls_tool
|
||||
|
||||
|
||||
def _uploads(tmp_path: Path) -> Path:
|
||||
uploads = tmp_path / "uploads"
|
||||
(uploads / "logs").mkdir(parents=True)
|
||||
(uploads / "app.log").write_text("FATAL out of memory\n", encoding="utf-8")
|
||||
(uploads / "logs" / "worker.log").write_text("FATAL in worker\n", encoding="utf-8")
|
||||
(uploads / "notes.txt").write_text("FATAL in notes\n", encoding="utf-8")
|
||||
return uploads
|
||||
|
||||
|
||||
def test_a_user_data_root_lists_and_searches_data_looking_names(tmp_path: Path) -> None:
|
||||
uploads = _uploads(tmp_path)
|
||||
|
||||
listed = list_dir(str(uploads), max_depth=2, user_data=True)
|
||||
assert str(uploads / "app.log") in listed
|
||||
assert str(uploads / "logs" / "worker.log") in listed
|
||||
|
||||
glob_matches, _ = find_glob_matches(uploads, "*.log", user_data=True)
|
||||
assert {Path(match).name for match in glob_matches} == {"app.log", "worker.log"}
|
||||
|
||||
matches, _ = find_grep_matches(uploads, "FATAL", user_data=True)
|
||||
assert {Path(match.path).name for match in matches} == {"app.log", "worker.log", "notes.txt"}
|
||||
|
||||
|
||||
def test_a_checkout_keeps_hiding_data_looking_names(tmp_path: Path) -> None:
|
||||
checkout = tmp_path / "repo"
|
||||
(checkout / ".git").mkdir(parents=True)
|
||||
(checkout / "app.log").write_text("FATAL out of memory\n", encoding="utf-8")
|
||||
(checkout / "notes.txt").write_text("FATAL in notes\n", encoding="utf-8")
|
||||
|
||||
assert ignore_rules_for(checkout).should_ignore("app.log") is True
|
||||
# The user-data flag wins over the checkout: uploads inside a checkout are
|
||||
# still the user's files.
|
||||
assert ignore_rules_for(checkout, user_data=True).should_ignore("app.log") is False
|
||||
|
||||
listed = list_dir(str(checkout), max_depth=2)
|
||||
assert str(checkout / "app.log") not in listed
|
||||
|
||||
glob_matches, _ = find_glob_matches(checkout, "*")
|
||||
assert str(checkout / "app.log") not in glob_matches
|
||||
|
||||
matches, _ = find_grep_matches(checkout, "FATAL")
|
||||
assert {Path(match.path).name for match in matches} == {"notes.txt"}
|
||||
|
||||
|
||||
def test_an_explicit_glob_wins_inside_a_checkout(tmp_path: Path) -> None:
|
||||
checkout = tmp_path / "repo"
|
||||
(checkout / ".git").mkdir(parents=True)
|
||||
(checkout / "app.log").write_text("FATAL out of memory\n", encoding="utf-8")
|
||||
|
||||
glob_matches, _ = find_glob_matches(checkout, "*.log")
|
||||
assert glob_matches == [str(checkout / "app.log")]
|
||||
|
||||
matches, _ = find_grep_matches(checkout, "FATAL", glob_pattern="*.log")
|
||||
assert [Path(match.path).name for match in matches] == ["app.log"]
|
||||
|
||||
|
||||
def test_a_file_named_as_the_root_is_read_above_the_size_bound(tmp_path: Path) -> None:
|
||||
uploads = tmp_path / "uploads"
|
||||
uploads.mkdir()
|
||||
big = uploads / "server.txt"
|
||||
big.write_text("ok\n" * 400_000 + "FATAL at the end\n", encoding="utf-8")
|
||||
assert big.stat().st_size > DEFAULT_MAX_FILE_SIZE_BYTES
|
||||
|
||||
matches, truncated = find_grep_matches(big, "FATAL")
|
||||
assert [match.line for match in matches] == ["FATAL at the end"]
|
||||
assert truncated is False
|
||||
|
||||
|
||||
def test_a_directory_search_keeps_the_size_bound(tmp_path: Path) -> None:
|
||||
uploads = tmp_path / "uploads"
|
||||
uploads.mkdir()
|
||||
big = uploads / "server.txt"
|
||||
big.write_text("ok\n" * 400_000 + "FATAL at the end\n", encoding="utf-8")
|
||||
|
||||
matches, _ = find_grep_matches(uploads, "FATAL", user_data=True)
|
||||
assert matches == []
|
||||
|
||||
|
||||
def test_in_flight_upload_files_stay_hidden_in_user_data(tmp_path: Path) -> None:
|
||||
uploads = _uploads(tmp_path)
|
||||
(uploads / ".upload-active.part").write_text("partial", encoding="utf-8")
|
||||
|
||||
rules = ignore_rules_for(uploads, user_data=True)
|
||||
assert rules.should_ignore(".upload-active.part")
|
||||
assert not rules.should_ignore("app.log")
|
||||
assert not any(".upload-active.part" in name for name in list_dir(str(uploads), max_depth=2, user_data=True))
|
||||
|
||||
|
||||
def test_host_path_resolved_from_the_user_data_mount_still_counts_as_user_data(tmp_path: Path) -> None:
|
||||
# The public ls/glob/grep tools resolve /mnt/user-data/... to host paths
|
||||
# before they call the provider, so a prefix check on the argument alone
|
||||
# misses every real entry point. The mount table decides instead.
|
||||
(tmp_path / ".git").mkdir()
|
||||
uploads = _uploads(tmp_path)
|
||||
sandbox = LocalSandbox("local", [PathMapping("/mnt/user-data/uploads", str(uploads))])
|
||||
|
||||
assert any(name.endswith("uploads/app.log") for name in sandbox.list_dir(str(uploads)))
|
||||
matches, _ = sandbox.glob(str(uploads), "**/*")
|
||||
assert "/mnt/user-data/uploads/app.log" in matches
|
||||
greps, _ = sandbox.grep(str(uploads), "FATAL")
|
||||
assert sorted(match.path for match in greps) == [
|
||||
"/mnt/user-data/uploads/app.log",
|
||||
"/mnt/user-data/uploads/logs/worker.log",
|
||||
"/mnt/user-data/uploads/notes.txt",
|
||||
]
|
||||
|
||||
|
||||
def test_a_root_outside_the_user_data_mount_keeps_the_repository_filter(tmp_path: Path) -> None:
|
||||
# Any root the mount table does not label as user data -- a skills mount, an
|
||||
# ad-hoc host path, a checkout -- keeps the filter it always had.
|
||||
inbox = _uploads(tmp_path)
|
||||
sandbox = LocalSandbox("local", [])
|
||||
|
||||
assert ignore_rules_for(inbox).should_ignore("app.log") is True
|
||||
matches, _ = sandbox.glob(str(inbox), "**/*")
|
||||
assert {Path(name).name for name in matches} == {"notes.txt"}
|
||||
|
||||
|
||||
def test_public_tools_keep_uploaded_logs_when_the_host_tree_is_a_checkout(tmp_path: Path, monkeypatch) -> None:
|
||||
# The reviewer's reproduction: the real public tools, with the user-data
|
||||
# directory inside the checkout the Gateway runs from.
|
||||
(tmp_path / ".git").mkdir()
|
||||
for name in ("workspace", "uploads", "outputs"):
|
||||
(tmp_path / name).mkdir()
|
||||
(tmp_path / "uploads" / "app.log").write_text("FATAL out of memory\n", encoding="utf-8")
|
||||
sandbox = LocalSandbox("local", [PathMapping(f"/mnt/user-data/{name}", str(tmp_path / name)) for name in ("workspace", "uploads", "outputs")])
|
||||
monkeypatch.setattr("deerflow.sandbox.tools.ensure_sandbox_initialized", lambda runtime: sandbox)
|
||||
runtime = SimpleNamespace(
|
||||
state={
|
||||
"sandbox": {"sandbox_id": "local"},
|
||||
"thread_data": {
|
||||
"workspace_path": str(tmp_path / "workspace"),
|
||||
"uploads_path": str(tmp_path / "uploads"),
|
||||
"outputs_path": str(tmp_path / "outputs"),
|
||||
},
|
||||
},
|
||||
context={"thread_id": "thread-1"},
|
||||
)
|
||||
|
||||
assert "app.log" in ls_tool.func(runtime=runtime, description="list uploads", path="/mnt/user-data/uploads")
|
||||
assert "app.log" in glob_tool.func(runtime=runtime, description="all uploads", pattern="**/*", path="/mnt/user-data/uploads")
|
||||
assert "app.log" in grep_tool.func(runtime=runtime, description="find fatal", pattern="FATAL", path="/mnt/user-data/uploads")
|
||||
Loading…
x
Reference in New Issue
Block a user