mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-20 19:46:16 +00:00
ed986a10ef
* fix(sandbox): report truncated remote glob and grep results
BoxLite, Tenki, E2B, and OpenSandbox run find/grep in the sandbox, cap
the raw output with `| head`, and then filter those lines in Python:
ignored directories such as node_modules are dropped and grep's glob
scope is applied. They reported truncated only when max_results matches
survived the filter. When the capped lines were mostly filtered out, a
search with real matches past the cap came back short or empty with
truncated=False, and glob_tool/grep_tool rendered it as "No files
matched" / "No matches found". With the default max_results=200 and
1,200 files under node_modules, glob("**/*.py") reported no matches for
a workspace that has src/app.py.
remote_search_command now lets one line past its limit through, and
parse_remote_search_output(..., limit=) returns RemoteSearchOutput(text,
truncated): the first `limit` lines and whether the extra line arrived.
Exactly `limit` lines stays a complete result. Each provider passes the
cap it already computed to both calls and returns that truncated from
glob and grep when fewer than max_results results survive filtering.
The glob and grep tools now describe an empty truncated result as
incomplete instead of reporting no matches, which also covers AIO grep's
forwarded truncated flag. Sandbox.glob/grep document truncated as "the
matches may be incomplete".
* docs(changelog): reference #5427 in the remote search truncation entry
---------
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
394 lines
16 KiB
Python
394 lines
16 KiB
Python
"""``BoxliteBox`` — DeerFlow :class:`Sandbox` backed by a BoxLite micro-VM.
|
|
|
|
DeerFlow's ``Sandbox`` contract is synchronous; BoxLite's SDK is async-native and
|
|
its box handles are event-loop-affine. The provider (:mod:`.provider`) owns one
|
|
private asyncio loop on a daemon thread and injects a ``run`` callable that
|
|
marshals each coroutine onto it via ``run_coroutine_threadsafe`` — so every op
|
|
runs on the loop the box was started on, and stays safe no matter which
|
|
``asyncio.to_thread`` worker DeerFlow invokes us from.
|
|
|
|
Every operation is a shell command run inside the box (``cat`` / ``find`` /
|
|
``grep`` / chunked ``base64``), parsed with the shared ``deerflow.sandbox.search``
|
|
helpers — the same exec-driven approach as ``community/e2b_sandbox``. Commands
|
|
use only busybox-portable flags so any OCI image works.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import errno
|
|
import logging
|
|
import posixpath
|
|
import re
|
|
import shlex
|
|
import threading
|
|
from typing import TYPE_CHECKING, TypeVar
|
|
|
|
from deerflow.config.paths import VIRTUAL_PATH_PREFIX
|
|
from deerflow.sandbox.remote_list_dir import parse_remote_list_dir_output, remote_list_dir_command
|
|
from deerflow.sandbox.remote_search import parse_remote_search_output, remote_search_command
|
|
from deerflow.sandbox.sandbox import Sandbox, _validate_extra_env
|
|
from deerflow.sandbox.search import GrepMatch, path_matches, should_ignore_path, truncate_line
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable
|
|
|
|
from boxlite import SimpleBox
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
T = TypeVar("T")
|
|
|
|
_MAX_DOWNLOAD_SIZE = 100 * 1024 * 1024 # 100 MB
|
|
# One base64 chunk stays well under Linux MAX_ARG_STRLEN (128 KiB per argv entry),
|
|
# and 60000 is a multiple of 4 so each chunk is a self-contained base64 unit whose
|
|
# decoded bytes concatenate losslessly.
|
|
_B64_CHUNK = 60000
|
|
|
|
|
|
class BoxliteBox(Sandbox):
|
|
"""Adapter that delegates to a running BoxLite ``SimpleBox``.
|
|
|
|
Args:
|
|
id: DeerFlow-side sandbox id (the BoxLite box id).
|
|
box: A started async ``SimpleBox``. The provider owns its lifecycle; this
|
|
adapter stops it on :meth:`close`.
|
|
run: Runs a coroutine on the provider's private loop, returning its result
|
|
(blocking the caller thread).
|
|
default_env: Static environment merged into every command, overridden by
|
|
per-call ``env`` (request-scoped secrets).
|
|
"""
|
|
|
|
#: Every call is a fresh ``sh -lc`` exec in the box — no shell state
|
|
#: survives into the next command.
|
|
persistent_shell_sessions = False
|
|
|
|
TERMINAL_ERROR_MARKERS = (
|
|
"vsock",
|
|
"disconnected",
|
|
"broken pipe",
|
|
"connection reset",
|
|
"connection refused",
|
|
"no such box",
|
|
"box has been stopped",
|
|
"engine reported an error",
|
|
)
|
|
RETRYABLE_ERROR_MARKERS = (
|
|
"transport not ready",
|
|
"retry later",
|
|
"temporarily unavailable",
|
|
"resource busy",
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
id: str,
|
|
box: SimpleBox,
|
|
run: Callable[..., T],
|
|
*,
|
|
default_env: dict[str, str] | None = None,
|
|
on_terminal_failure: Callable[[str, str], None] | None = None,
|
|
) -> None:
|
|
super().__init__(id)
|
|
self._box = box
|
|
self._run = run
|
|
self._default_env = dict(default_env or {})
|
|
self._on_terminal_failure = on_terminal_failure
|
|
self._lock = threading.Lock()
|
|
self._closed = False
|
|
|
|
@classmethod
|
|
def _is_terminal_box_failure(cls, error: Exception) -> bool:
|
|
if isinstance(error, (BrokenPipeError, ConnectionError, EOFError)):
|
|
return True
|
|
if not isinstance(error, RuntimeError | OSError):
|
|
return False
|
|
msg = str(error).lower()
|
|
if any(marker in msg for marker in cls.RETRYABLE_ERROR_MARKERS):
|
|
return False
|
|
return any(marker in msg for marker in cls.TERMINAL_ERROR_MARKERS)
|
|
|
|
# ── bridge helpers ──────────────────────────────────────────────────
|
|
|
|
def _exec(
|
|
self,
|
|
*argv: str,
|
|
env: dict[str, str] | None = None,
|
|
timeout: float | None = None,
|
|
):
|
|
try:
|
|
with self._lock:
|
|
if self._closed:
|
|
raise RuntimeError("sandbox has been closed")
|
|
box = self._box
|
|
return self._run(box.exec(*argv, env=env, timeout=timeout), timeout=timeout)
|
|
except Exception as e:
|
|
if self._on_terminal_failure is not None and self._is_terminal_box_failure(e):
|
|
try:
|
|
self._on_terminal_failure(self.id, str(e))
|
|
except Exception:
|
|
logger.exception("Terminal BoxLite failure callback errored for %s", self.id)
|
|
raise
|
|
|
|
def _sh(
|
|
self,
|
|
script: str,
|
|
env: dict[str, str] | None = None,
|
|
timeout: float | None = None,
|
|
):
|
|
return self._exec("sh", "-lc", script, env=env, timeout=timeout)
|
|
|
|
def close(self) -> None:
|
|
with self._lock:
|
|
if self._closed:
|
|
return
|
|
self._closed = True
|
|
try:
|
|
self._run(self._box.stop())
|
|
except Exception as e:
|
|
logger.warning("Error stopping BoxLite box %s: %s", self.id, e)
|
|
|
|
@property
|
|
def is_closed(self) -> bool:
|
|
with self._lock:
|
|
return self._closed
|
|
|
|
# ── path safety (mirrors community/e2b_sandbox) ─────────────────────
|
|
|
|
@staticmethod
|
|
def _guard_traversal(path: str) -> str:
|
|
if not path:
|
|
raise ValueError("path must be a non-empty string")
|
|
normalized = path.replace("\\", "/")
|
|
for segment in normalized.split("/"):
|
|
if segment == "..":
|
|
raise PermissionError(f"Access denied: path traversal detected in '{path}'")
|
|
return normalized
|
|
|
|
def _resolve_path(self, path: str) -> str:
|
|
# The provider materialises the /mnt/user-data prefix on the box rootfs,
|
|
# so DeerFlow's virtual paths are used as-is; we only reject traversal.
|
|
return self._guard_traversal(path)
|
|
|
|
# ── command execution ───────────────────────────────────────────────
|
|
|
|
def execute_command(
|
|
self,
|
|
command: str,
|
|
env: dict[str, str] | None = None,
|
|
timeout: float | None = None,
|
|
) -> str:
|
|
"""Run ``command`` through a shell in the box and return its output.
|
|
|
|
DeerFlow passes a bash command *string*; BoxLite's ``exec`` takes argv, so
|
|
it runs through ``sh -lc``. Per-call ``env`` is layered over the static
|
|
config environment and scoped to this command only.
|
|
|
|
*timeout* bounds both layers: BoxLite's SDK ``exec(timeout=...)`` handles
|
|
command timeout inside the VM, and the event-loop bridge receives the
|
|
same value so ``run_coroutine_threadsafe(...).result(timeout)`` cannot
|
|
block the caller forever if the SDK future itself never resolves.
|
|
"""
|
|
_validate_extra_env(env) # POSIX env-var key rule; raises ValueError on a bad key
|
|
if self.is_closed:
|
|
return "Error: sandbox has been closed"
|
|
merged_env = {**self._default_env, **(env or {})} or None
|
|
try:
|
|
result = self._exec("sh", "-lc", command, env=merged_env, timeout=timeout)
|
|
except Exception as e:
|
|
logger.error("Failed to execute command in BoxLite box %s: %s", self.id, e)
|
|
return f"Error: {e}"
|
|
|
|
stdout = result.stdout or ""
|
|
stderr = result.stderr or ""
|
|
if stdout and stderr:
|
|
output = f"{stdout}\n{stderr}"
|
|
else:
|
|
output = stdout or stderr
|
|
if result.exit_code not in (0, None):
|
|
# Mirror LocalSandbox: preserve a nonzero exit in the output text
|
|
# even when the command produced output (see e2b_sandbox).
|
|
output = f"{output}\nExit Code: {result.exit_code}" if output else f"Command exited with code {result.exit_code}"
|
|
return output if output else "(no output)"
|
|
|
|
# ── file operations ─────────────────────────────────────────────────
|
|
|
|
def read_file(
|
|
self,
|
|
path: str,
|
|
start_line: int | None = None,
|
|
end_line: int | None = None,
|
|
) -> str:
|
|
resolved = self._resolve_path(path)
|
|
try:
|
|
r = self._exec("cat", "--", resolved)
|
|
except Exception as e:
|
|
logger.error("read_file %s failed: %s", resolved, e)
|
|
return f"Error: {e}"
|
|
if r.exit_code not in (0, None):
|
|
return f"Error: {(r.stderr or '').strip() or 'cannot read file'}"
|
|
content = r.stdout or ""
|
|
if start_line is None and end_line is None:
|
|
return content
|
|
lines = content.splitlines()
|
|
start = start_line or 1
|
|
end = end_line if end_line is not None else len(lines)
|
|
return "\n".join(lines[start - 1 : end])
|
|
|
|
def write_file(self, path: str, content: str, append: bool = False) -> None:
|
|
self._write_bytes(self._resolve_path(path), content.encode("utf-8"), append=append)
|
|
|
|
def update_file(self, path: str, content: bytes) -> None:
|
|
self._write_bytes(self._resolve_path(path), content, append=False)
|
|
|
|
def _write_bytes(self, resolved: str, data: bytes, *, append: bool) -> None:
|
|
parent = posixpath.dirname(resolved)
|
|
if parent:
|
|
mk = self._sh(f"mkdir -p {shlex.quote(parent)}")
|
|
if mk.exit_code not in (0, None):
|
|
raise OSError(f"cannot create parent of '{resolved}': {(mk.stderr or '').strip()}")
|
|
|
|
b64 = base64.b64encode(data).decode("ascii")
|
|
if not b64: # empty file — create/truncate without piping
|
|
r = self._sh(f": {'>>' if append else '>'} {shlex.quote(resolved)}")
|
|
if r.exit_code not in (0, None):
|
|
raise OSError(f"write '{resolved}' failed: {(r.stderr or '').strip()}")
|
|
return
|
|
|
|
first = True
|
|
for i in range(0, len(b64), _B64_CHUNK):
|
|
chunk = b64[i : i + _B64_CHUNK]
|
|
redir = ">>" if (append or not first) else ">"
|
|
r = self._sh(f"printf %s {shlex.quote(chunk)} | base64 -d {redir} {shlex.quote(resolved)}")
|
|
if r.exit_code not in (0, None):
|
|
raise OSError(f"write '{resolved}' failed: {(r.stderr or '').strip()}")
|
|
first = False
|
|
|
|
def download_file(self, path: str) -> bytes:
|
|
normalized = self._guard_traversal(path)
|
|
stripped = normalized.lstrip("/")
|
|
allowed = VIRTUAL_PATH_PREFIX.lstrip("/")
|
|
if stripped != allowed and not stripped.startswith(f"{allowed}/"):
|
|
raise PermissionError(f"Access denied: path must be under '{VIRTUAL_PATH_PREFIX}': '{path}'")
|
|
|
|
# Enforce the size cap before buffering the whole payload.
|
|
size_r = self._sh(f"wc -c < {shlex.quote(normalized)}")
|
|
if size_r.exit_code not in (0, None):
|
|
raise OSError(f"cannot read '{path}' from box: {(size_r.stderr or '').strip() or 'not found'}")
|
|
try:
|
|
size = int((size_r.stdout or "0").strip() or "0")
|
|
except ValueError:
|
|
size = 0
|
|
if size > _MAX_DOWNLOAD_SIZE:
|
|
raise OSError(errno.EFBIG, f"File exceeds maximum download size of {_MAX_DOWNLOAD_SIZE} bytes", path)
|
|
|
|
r = self._sh(f"base64 {shlex.quote(normalized)}")
|
|
if r.exit_code not in (0, None):
|
|
raise OSError(f"cannot read '{path}' from box: {(r.stderr or '').strip()}")
|
|
try:
|
|
return base64.b64decode("".join((r.stdout or "").split()))
|
|
except Exception as e:
|
|
raise OSError(f"failed to decode '{path}' from box: {e}") from e
|
|
|
|
def list_dir(self, path: str, max_depth: int = 2) -> list[str]:
|
|
resolved = self._resolve_path(path)
|
|
r = self._sh(remote_list_dir_command(resolved, max_depth))
|
|
return parse_remote_list_dir_output(r.stdout, resolved, pipeline_exit_code=r.exit_code)
|
|
|
|
def glob(
|
|
self,
|
|
path: str,
|
|
pattern: str,
|
|
*,
|
|
include_dirs: bool = False,
|
|
max_results: int = 200,
|
|
) -> tuple[list[str], bool]:
|
|
resolved = self._resolve_path(path)
|
|
types = ("f", "d") if include_dirs else ("f",)
|
|
type_expr = " -o ".join(f"-type {t}" for t in types)
|
|
hard_limit = max(max_results * 4, max_results + 50)
|
|
# -H follows a symlinked search root, as list_dir does.
|
|
search = f"find -H {shlex.quote(resolved)} \\( {type_expr} \\) -print 2>/dev/null"
|
|
r = self._sh(remote_search_command(search, resolved, limit=hard_limit))
|
|
# A missing root or a failed find must not read as "no files matched" (#5376).
|
|
output = parse_remote_search_output(r.stdout, resolved, tool="find", limit=hard_limit)
|
|
|
|
matches: list[str] = []
|
|
root = resolved.rstrip("/") or "/"
|
|
root_prefix = root if root == "/" else f"{root}/"
|
|
for entry in output.text.splitlines():
|
|
# Do NOT strip: trailing whitespace can be part of the filename.
|
|
if not entry or (entry != root and not entry.startswith(root_prefix)):
|
|
continue
|
|
if should_ignore_path(entry):
|
|
continue
|
|
rel_path = entry[len(root) :].lstrip("/")
|
|
if not rel_path:
|
|
continue
|
|
if path_matches(pattern, rel_path):
|
|
matches.append(entry)
|
|
if len(matches) >= max_results:
|
|
return matches, True
|
|
return matches, output.truncated
|
|
|
|
def grep(
|
|
self,
|
|
path: str,
|
|
pattern: str,
|
|
*,
|
|
glob: str | None = None,
|
|
literal: bool = False,
|
|
case_sensitive: bool = False,
|
|
max_results: int = 100,
|
|
) -> tuple[list[GrepMatch], bool]:
|
|
# Sanity-check a regex pattern as a Python regex at the boundary (grep uses
|
|
# POSIX ERE, but this catches gross errors); a literal needs no validation.
|
|
# grep receives the RAW pattern: -F matches it literally, -E as a regex.
|
|
if not literal:
|
|
re.compile(pattern, 0 if case_sensitive else re.IGNORECASE)
|
|
|
|
resolved = self._resolve_path(path)
|
|
# busybox+GNU-portable flags: -r recursive, -H always print the filename
|
|
# (including when path is a single file), -n line numbers, -I skip
|
|
# binary, -E/-F regex vs fixed. --include and -m are omitted for busybox
|
|
# portability; glob-scoping and the result cap are applied in Python.
|
|
flags = ["-r", "-H", "-n", "-I"]
|
|
if not case_sensitive:
|
|
flags.append("-i")
|
|
flags.append("-F" if literal else "-E")
|
|
total_cap = max(max_results * 4, max_results + 50)
|
|
search = "grep " + " ".join(flags) + f" -e {shlex.quote(pattern)} {shlex.quote(resolved)} 2>/dev/null"
|
|
r = self._sh(remote_search_command(search, resolved, limit=total_cap))
|
|
# A missing root, a missing grep or an unreadable tree must not read as "no matches" (#5376).
|
|
output = parse_remote_search_output(r.stdout, resolved, tool="grep", limit=total_cap)
|
|
|
|
root = resolved.rstrip("/") or "/"
|
|
root_prefix = root if root == "/" else f"{root}/"
|
|
matches: list[GrepMatch] = []
|
|
truncated = output.truncated
|
|
for raw in output.text.splitlines():
|
|
try:
|
|
file_path, line_no_str, line_text = raw.split(":", 2)
|
|
except ValueError:
|
|
continue
|
|
try:
|
|
line_number = int(line_no_str)
|
|
except ValueError:
|
|
continue
|
|
if should_ignore_path(file_path):
|
|
continue
|
|
if glob is not None:
|
|
# Match the caller's real directory scope: a pattern like
|
|
# "src/*.js" must not broaden to every *.js in the tree. Same
|
|
# helper, same relative-to-root semantics as glob() above.
|
|
if file_path != root and not file_path.startswith(root_prefix):
|
|
continue
|
|
rel_path = posixpath.basename(file_path) if file_path == root else file_path[len(root) :].lstrip("/")
|
|
if not path_matches(glob, rel_path):
|
|
continue
|
|
matches.append(GrepMatch(path=file_path, line_number=line_number, line=truncate_line(line_text)))
|
|
if len(matches) >= max_results:
|
|
truncated = True
|
|
break
|
|
return matches, truncated
|