From e0cccd6b376191e7017c1d4602326a96d5cde28c Mon Sep 17 00:00:00 2001 From: Michiel Voortman <135761097+MichielMAnalytics@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:29:15 +0100 Subject: [PATCH] feat(sandbox): add boxd sandbox provider (#8314) * feat(sandbox): add boxd sandbox provider Adds deerflow.community.boxd_sandbox:BoxdSandboxProvider. Each user/thread scope runs in an isolated boxd machine. New machines get the thread's files and skills before they are handed out, release mirrors outputs back to thread storage, and released machines wait in the shared warm pool. The boxd SDK is an optional extra (deerflow-harness[boxd]). Unit tests run against a fake client; tests/test_boxd_sandbox_live.py runs against real machines when DEER_FLOW_RUN_LIVE_TESTS=1 and BOXD_API_KEY are set. * fix(boxd): retain failed cleanup and bound output mirror calls --------- Co-authored-by: Willem Jiang --- .env.example | 5 + README.md | 1 + backend/docs/CONFIGURATION.md | 61 + .../deerflow/community/boxd_sandbox/AGENTS.md | 33 + .../deerflow/community/boxd_sandbox/README.md | 101 ++ .../community/boxd_sandbox/__init__.py | 38 + .../community/boxd_sandbox/boxd_sandbox.py | 521 ++++++++ .../boxd_sandbox/boxd_sandbox_provider.py | 949 ++++++++++++++ .../community/provider_credentials.py | 1 + .../harness/deerflow/config/sandbox_config.py | 8 +- backend/packages/harness/pyproject.toml | 3 + backend/tests/test_agent_guidance_check.py | 1 + backend/tests/test_boxd_sandbox_live.py | 196 +++ backend/tests/test_boxd_sandbox_provider.py | 1128 +++++++++++++++++ backend/tests/test_harness_packaging.py | 12 + backend/uv.lock | 22 +- config.example.yaml | 19 + docs/guide/sandbox.md | 6 + 18 files changed, 3103 insertions(+), 2 deletions(-) create mode 100644 backend/packages/harness/deerflow/community/boxd_sandbox/AGENTS.md create mode 100644 backend/packages/harness/deerflow/community/boxd_sandbox/README.md create mode 100644 backend/packages/harness/deerflow/community/boxd_sandbox/__init__.py create mode 100644 backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox.py create mode 100644 backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox_provider.py create mode 100644 backend/tests/test_boxd_sandbox_live.py create mode 100644 backend/tests/test_boxd_sandbox_provider.py diff --git a/.env.example b/.env.example index 84c7681211..a718076b97 100644 --- a/.env.example +++ b/.env.example @@ -92,6 +92,11 @@ EXA_BASE_URL=https://api.exa.ai # E2B cloud sandbox API key — required only when using E2BSandboxProvider. # Sign up at https://e2b.dev/dashboard # E2B_API_KEY=your-e2b-api-key + +# boxd API key, required only when using BoxdSandboxProvider. +# Create one in the boxd console: https://boxd.sh?ref=deer-flow +# BOXD_API_KEY=your-boxd-api-key + # FEISHU_APP_ID=your-feishu-app-id # FEISHU_APP_SECRET=your-feishu-app-secret diff --git a/README.md b/README.md index e8a15683d8..e1a3ff4604 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ Each guide is the full reference for its topic. Open the one you need. | DeepSeek web search, thinking mode and citation excerpts | [DeepSeek Web Search](backend/docs/CONFIGURATION.md#deepseek-web-search) | | Docker, local development, upgrades, production deployment | [Running the Application](docs/guide/deployment.md) | | Sandbox modes, file system, uploads and outputs | [Sandbox](docs/guide/sandbox.md) | +| boxd machines, bounded output copies and cleanup retries | [boxd sandbox provider](backend/packages/harness/deerflow/community/boxd_sandbox/README.md) | | Skills, built-in tools, script workflows, Claude Code integration | [Skills & Tools](docs/guide/skills-and-tools.md) | | MCP servers | [MCP Server](docs/guide/mcp.md) | | Sub-agents, task delegation and optional SQLite checkpoint history | [Sub-Agents](docs/guide/subagents.md), [Tool restrictions](docs/guide/subagents.md#tool-restrictions) | diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 23732508b3..d6a8fefb0b 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -1781,6 +1781,67 @@ Downloads are restricted to `/mnt/user-data` and all file paths reject traversal. Multi-process discovery and ownership coordination are not yet implemented, so `replicas` is a per-Gateway-process soft cap. +**boxd Machine Sandbox** (runs each sandbox as an isolated [boxd](https://boxd.sh?ref=deer-flow) machine): + +```yaml +sandbox: + use: deerflow.community.boxd_sandbox:BoxdSandboxProvider + boxd_api_key: $BOXD_API_KEY # falls back to the BOXD_API_KEY env var + # org: my-org # optional; the key's own org when omitted + # region: fra # optional; fra or nyc + # vcpu: 2 # optional; 1, 2, 4 or 8 (memory 4G, 8G, 16G or 32G follows) + # from_snapshot: my-snapshot # optional; start machines from a prepared snapshot + # auto_suspend_timeout: 300 # optional; idle seconds before a machine goes to standby + # ready_timeout: 120 # seconds to wait for a new machine to boot + # name_prefix: deer-flow- # machine names are - + # labels: # extra labels on every machine + # team: research + replicas: 3 # active + warm cap per gateway process + idle_timeout: 600 # warm seconds before the machine is deleted; 0 disables + environment: + PYTHONUNBUFFERED: "1" +``` + +Install the optional SDK before selecting this provider: + +```bash +pip install "deerflow-harness[boxd]" +``` + +Each `(user_id, thread_id)` scope gets its own Ubuntu machine, created with +`isolated: true`, so it joins no private network and holds no boxd +credentials. The API key stays in the Gateway. Machines carry the labels +`deer-flow: sandbox` and `deer-flow-scope: `, so you can find them in +the boxd console or with the CLI. The provider creates `/mnt/user-data` and the +skills root (`skills.container_path`) as real directories on the machine, so +shell commands and file tools use the same paths. + +A new machine for a thread receives the thread's uploads, workspace and outputs +files and the skills projection before `acquire()` returns, each as one archive +of at most 5,000 files and 128 MiB; files past that limit are skipped with a +warning, and a failed transfer deletes the machine and raises. An Agent with a +skill policy gets its own skills projection, and the machine sees only what was +uploaded. Work outside `/mnt/user-data` is lost when the idle reaper deletes a +machine. At the end of a turn, release copies +`/mnt/user-data/outputs` back to the thread's output directory (bounded by file +count, bytes and one shared 120-second deadline), then parks the machine in an +in-process warm pool. A stopped machine's output copy is skipped until a normal +sandbox call restarts it. The same scope reclaims it after a health check; +the idle reaper deletes it after +`idle_timeout`. Shutdown deletes every machine the provider tracks and can be +retried when a delete fails. +Machines discovered after a failed create use the same tracked deletion retries. + +Commands run through `bash -lc`. Per-call environment variables travel as exec +options, never inside the command string. The command deadline (the caller's +timeout, else `bash_command_timeout`) is enforced in the machine, so a command +that runs over it ends with `Exit Code: 124` and keeps its partial output. A +stopped machine is started on the next call; a command that hit the stopped +machine is not repeated and returns an error. Machines in standby or +hibernation wake on the call. Downloads are restricted to `/mnt/user-data` and +capped while streaming. Like the other remote providers, `replicas` is a +per-Gateway-process soft cap. + Choose between local execution or Docker-based isolation: **Option 1: Local Sandbox** (default, simpler setup): diff --git a/backend/packages/harness/deerflow/community/boxd_sandbox/AGENTS.md b/backend/packages/harness/deerflow/community/boxd_sandbox/AGENTS.md new file mode 100644 index 0000000000..09d13cf694 --- /dev/null +++ b/backend/packages/harness/deerflow/community/boxd_sandbox/AGENTS.md @@ -0,0 +1,33 @@ +# boxd sandbox provider (`deerflow/community/boxd_sandbox/`) + +Runs sandboxes as isolated boxd machines. General sandbox rules are in the +[sandbox guide](../../sandbox/AGENTS.md); user docs are in [README.md](README.md). + +```bash +cd backend && python -m pytest tests -q -k "boxd" +``` + +- **Address machines by id, never by name.** Names are unique per owner and + are not free again right after a delete, so every machine generation gets a + random suffix. +- **Every machine is created with `isolated=True`.** The agent runs untrusted + code; an isolated machine holds no boxd credentials and joins no private + network. A test pins it. +- **A stopped machine answers with a conflict.** Start it; repeat file calls + once, but never a command, which may have been running when it stopped. + Standby and hibernated machines wake on the call. +- **Every SDK call has a deadline.** Commands run under coreutils `timeout`, + so exit code 124 and partial output come back. `machines.fs` read streams take + no deadline, so reads run as `head -c` through exec. +- **Release copies share one deadline.** Pass it through listing and downloads. + Budgeted calls skip stopped-machine recovery and failure probes because those + control-plane calls have separate timeouts. +- **Failed-create cleanup stays owned.** A machine found by generation label + goes through `_close_quietly`, so a failed delete is retried by the reaper + and shutdown. +- **`MachineFileNotFoundError` is also a `NotFoundError`.** Test for the + filesystem error, and confirm with `machines.get` before treating a machine + as gone. +- **Claims decide when a cancelled acquire releases.** Each caller that + receives a sandbox holds a claim; a cancelled caller gives its claim back + under the scope serializer, and the last one releases the sandbox. diff --git a/backend/packages/harness/deerflow/community/boxd_sandbox/README.md b/backend/packages/harness/deerflow/community/boxd_sandbox/README.md new file mode 100644 index 0000000000..62b16e2c62 --- /dev/null +++ b/backend/packages/harness/deerflow/community/boxd_sandbox/README.md @@ -0,0 +1,101 @@ +# boxd sandbox provider + +Runs DeerFlow sandboxes on [boxd](https://boxd.sh?ref=deer-flow) machines through +the `boxd` Python SDK. Each user/thread scope gets an isolated Ubuntu microVM. +The machine lives while the thread is active and while it waits in the warm +pool; after `idle_timeout` it is deleted, and a later turn gets a new machine +with the thread's files restored (see below). Anything the agent left outside +`/mnt/user-data` is gone at that point. + +## Installation and configuration + +Install the optional SDK, then select the provider: + +```bash +pip install "deerflow-harness[boxd]" +export BOXD_API_KEY=... # an API key from the boxd console +``` + +```yaml +sandbox: + use: deerflow.community.boxd_sandbox:BoxdSandboxProvider + # boxd_api_key: $BOXD_API_KEY # falls back to the BOXD_API_KEY env var + # org: my-org # optional; the key's own org when omitted + # region: fra # optional; fra or nyc + # vcpu: 2 # optional; 1, 2, 4 or 8 (memory 4G, 8G, 16G or 32G follows) + # memory: 8G # optional; give either vcpu or memory, or a matching pair + # from_snapshot: my-snapshot # optional; start machines from a prepared snapshot + # auto_suspend_timeout: 300 # optional; idle seconds before a machine goes to standby + # ready_timeout: 120 # seconds to wait for a new machine to boot + # name_prefix: deer-flow- # machine names are - + # labels: # extra labels on every machine + # team: research + # replicas: 3 # active + warm machines per gateway process + # idle_timeout: 600 # warm seconds before the machine is deleted; 0 disables + # environment: # merged into every command's environment + # PYTHONUNBUFFERED: "1" +``` + +Invalid values (an unknown region, a size boxd does not offer, a malformed +environment name) fail at provider startup, before any machine is created. + +`from_snapshot` is the way to give every sandbox extra tools: prepare one +machine, save it as a snapshot in the boxd console or with the CLI, and name the +snapshot here. New machines start from it. + +## Lifecycle and contract + +- **One machine per scope.** The local sandbox id comes from + `derive_sandbox_scope_token(user_id, thread_id)`. Every SDK call addresses the + machine by the id that `create` returned. +- **Isolated.** Machines are created with `isolated: true`: no private network, + no boxd credentials inside the machine. The API key stays in the Gateway, and + per-call secrets travel as exec environment, not in the command string. +- **Bootstrap.** `/mnt/user-data/{workspace,uploads,outputs}` and the skills root + are created as directories owned by the machine user, so the paths the agent + sees are the real paths. +- **Thread files and skills.** A new machine receives the thread's uploads, + workspace and outputs files (in that order) and the skills projection, each + as one archive of at most 5,000 files and 128 MiB. Files past that limit are + skipped with a warning. A failed transfer deletes the machine. An Agent with + a skill policy gets its own projection through `sync_agent_skills`, which + replaces the managed skill categories; the machine sees only what was + uploaded. +- **Release.** Output files are copied back to the thread's output directory + (at most 2,000 files, 512 MiB and 120 seconds per release), then the machine is + parked in the warm pool. Names that differ only by case or Unicode + normalization, unsafe names and linked destinations are skipped. A copy + failure keeps the previous copy. Listing and downloads share the remaining + 120-second budget. A stopped machine's copy is skipped, and recovery waits + for the next normal sandbox call. +- **Reuse.** Only the same scope can reclaim a warm machine, after a health + check. A failed check deletes the machine and creates a new one. +- **Stopped machines** are started on the next call. A file call then runs + once more; a command does not, and returns an error that asks to run it + again, so a command never runs twice. Machines in standby or hibernation + wake on the call. +- **Commands** run through `bash -lc` with a deadline: the caller's timeout, + else `bash_command_timeout`. The deadline ends the command inside the machine + with exit code 124 and keeps its partial output. +- **Reads** run as a bounded `head -c` with a deadline, so a large or stalled + file cannot hold a Gateway thread or its memory. +- **Cleanup.** The idle reaper deletes warm machines after `idle_timeout`; + shutdown deletes every tracked machine. A failed delete is kept and retried + by the reaper (when `idle_timeout` is above 0) and by the next shutdown. If + `create` raises after the server made the machine, the provider finds it by a + per-create label and tracks it until deletion succeeds. When every caller + that acquired a machine was cancelled, the machine is released to the warm pool. +- **Leftovers.** Machines carry the labels `deer-flow: sandbox` and + `deer-flow-scope: `. A Gateway that crashes cannot delete its + machines; find them by these labels in the boxd console or with the CLI. + +## Tests + +```bash +cd backend +python -m pytest tests -q -k "boxd" +DEER_FLOW_RUN_LIVE_TESTS=1 BOXD_API_KEY=... PYTHONPATH=. uv run pytest tests/test_boxd_sandbox_live.py -v -s +``` + +The live test creates one machine, runs the checks against it and deletes it +at the end. diff --git a/backend/packages/harness/deerflow/community/boxd_sandbox/__init__.py b/backend/packages/harness/deerflow/community/boxd_sandbox/__init__.py new file mode 100644 index 0000000000..71157468b9 --- /dev/null +++ b/backend/packages/harness/deerflow/community/boxd_sandbox/__init__.py @@ -0,0 +1,38 @@ +"""boxd machine sandbox provider for DeerFlow. + +This package implements DeerFlow's :class:`Sandbox` / :class:`SandboxProvider` +contract on top of `boxd `_ machines through the +``boxd`` Python SDK. Each sandbox is an isolated Ubuntu microVM. + +Configuration example (``config.yaml``):: + + sandbox: + use: deerflow.community.boxd_sandbox:BoxdSandboxProvider + # boxd specific options (read via SandboxConfig's ``extra="allow"``): + boxd_api_key: $BOXD_API_KEY # falls back to the BOXD_API_KEY env var + # org: my-org # optional; the key's own org when omitted + # region: fra # optional; fra or nyc + # vcpu: 2 # optional; 1, 2, 4 or 8 (memory follows: 4G, 8G, 16G, 32G) + # from_snapshot: my-snapshot # optional; start machines from a prepared snapshot + # auto_suspend_timeout: 300 # optional; idle seconds before a machine goes to standby + # ready_timeout: 120 # seconds to wait for a new machine to boot + # name_prefix: deer-flow- # machine names: - + # labels: # extra machine labels + # team: research + replicas: 3 # active + warm machine cap per Gateway process + idle_timeout: 600 # warm seconds before the machine is deleted; 0 disables + environment: # merged into every command's environment + PYTHONUNBUFFERED: "1" + +Install the optional SDK before selecting this provider:: + + pip install "deerflow-harness[boxd]" +""" + +from .boxd_sandbox import BoxdSandbox +from .boxd_sandbox_provider import BoxdSandboxProvider + +__all__ = [ + "BoxdSandbox", + "BoxdSandboxProvider", +] diff --git a/backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox.py b/backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox.py new file mode 100644 index 0000000000..c7f1937314 --- /dev/null +++ b/backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox.py @@ -0,0 +1,521 @@ +"""``BoxdSandbox``: DeerFlow :class:`Sandbox` backed by one boxd machine. + +A boxd machine is a full Ubuntu microVM with a persistent disk. The provider +bootstraps ``/mnt/user-data`` and the skills root as real directories owned by +the machine user, so virtual paths need no remapping: the agent's shell and the +file APIs see the same paths. + +Commands run through ``bash -lc`` with the per-call environment passed as +structured exec options, never spliced into the command string. Writes use the +SDK's streaming ``machines.fs`` API; reads run as a bounded ``head -c`` exec +with a deadline. Search (``list_dir`` / ``glob`` / +``grep``) shells out to ``find`` / ``grep`` and is parsed with the shared +``deerflow.sandbox`` helpers, like ``community/e2b_sandbox``. + +The ``boxd`` SDK is never imported here. Errors are classified by the names in +their class hierarchy, so this module imports without the optional dependency. +""" + +from __future__ import annotations + +import errno +import logging +import math +import re +import shlex +import threading +import time +from typing import TYPE_CHECKING, Any + +from deerflow.config.paths import VIRTUAL_PATH_PREFIX +from deerflow.sandbox.download import DEFAULT_DOWNLOAD_LIMIT as _MAX_DOWNLOAD_SIZE +from deerflow.sandbox.download import download_limit +from deerflow.sandbox.read_file_contract import split_file_lines +from deerflow.sandbox.remote_list_dir import parse_remote_list_dir_output, remote_list_dir_command +from deerflow.sandbox.remote_search import parse_remote_search_output, remote_search_command +from deerflow.sandbox.sandbox import Sandbox, _validate_extra_env +from deerflow.sandbox.search import GrepMatch, path_matches, should_ignore_path_under_root, truncate_line + +if TYPE_CHECKING: + from collections.abc import Callable + + from boxd import Boxd + +logger = logging.getLogger(__name__) + +# Extra seconds the SDK call waits after the in-machine ``timeout`` fired, so +# the command's own exit code 124 and its partial output come back first. +_TIMEOUT_GRACE_SECONDS = 30.0 +# Seconds between SIGTERM and SIGKILL for a command that hit its deadline. +_KILL_AFTER_SECONDS = 5 +# Bound for internal helper commands (search, listing, health probes). +_HELPER_TIMEOUT_SECONDS = 120.0 +# How long a started machine may take before commands reach it again. +_START_PROBE_SECONDS = 60.0 +_START_PROBE_INTERVAL = 1.0 +# Upper bound for one read_file call, so a huge file cannot exhaust Gateway memory. +_MAX_READ_FILE_SIZE = _MAX_DOWNLOAD_SIZE +# Deadlines for file reads, which run as a bounded ``head -c`` exec. +_READ_TIMEOUT_SECONDS = 120.0 +_DOWNLOAD_TIMEOUT_SECONDS = 300.0 +# Command deadline when the caller gives none (SandboxConfig.bash_command_timeout default). +DEFAULT_COMMAND_TIMEOUT = 600.0 + + +def _error_names(error: BaseException) -> set[str]: + return {cls.__name__ for cls in type(error).__mro__} + + +def is_machine_gone_error(error: BaseException) -> bool: + """The machine itself no longer exists (not a missing file inside it).""" + names = _error_names(error) + return "NotFoundError" in names and "MachineFilesystemError" not in names + + +def is_stopped_machine_error(error: BaseException) -> bool: + """A conflict that a later ``start`` can clear (the machine is stopped).""" + names = _error_names(error) + return "ConflictError" in names and "MachineFilesystemError" not in names + + +def is_missing_file_error(error: BaseException) -> bool: + return isinstance(error, FileNotFoundError) or "MachineFileNotFoundError" in _error_names(error) + + +class BoxdSandbox(Sandbox): + """DeerFlow Sandbox adapter that delegates to a live boxd machine. + + Args: + id: DeerFlow-side sandbox id (the provider's cache key). + client: A ``boxd.Boxd`` client. It is shared by every sandbox of one + provider; this adapter never closes it. + machine_id: The boxd machine id. Every call addresses the machine by id. + region: Region the machine lives in, forwarded to every call. + default_env: Static environment merged into every command, overridden + per call by the ``env`` passed to :meth:`execute_command`. + default_command_timeout: Deadline in seconds for commands whose caller + gives none. + on_terminal_failure: Optional callback ``(sandbox_id, reason)`` invoked + when the machine is gone, so the provider can evict this sandbox. + """ + + #: Every call is a fresh ``bash -lc`` exec; no shell state survives. + persistent_shell_sessions = False + artifact_root = VIRTUAL_PATH_PREFIX + + def __init__( + self, + id: str, + client: Boxd, + machine_id: str, + *, + region: str | None = None, + default_env: dict[str, str] | None = None, + default_command_timeout: float = DEFAULT_COMMAND_TIMEOUT, + on_terminal_failure: Callable[[str, str], None] | None = None, + ) -> None: + super().__init__(id) + self._client = client + self._machine_id = machine_id + self._region = region + self._default_env = dict(default_env or {}) + self._default_command_timeout = default_command_timeout + self._on_terminal_failure = on_terminal_failure + self._lock = threading.Lock() + # Serializes the append read-modify-write. Distinct from _lock so the + # eviction callback, which reaches back into the provider, never runs + # under it. + self._write_lock = threading.Lock() + self._closed = False + + # ── Properties / lifecycle ─────────────────────────────────────────── + + @property + def machine_id(self) -> str: + return self._machine_id + + @property + def is_closed(self) -> bool: + with self._lock: + return self._closed + + def close(self) -> None: + """Delete the machine (idempotent). + + The sandbox is marked closed only after the delete was accepted or the + machine is confirmed gone. Any other failure is raised, so the provider + keeps the handle and retries instead of leaving a machine running. + """ + with self._lock: + if self._closed: + return + try: + self._client.machines.delete(self._machine_id, region=self._region) + except Exception as e: + if not is_machine_gone_error(e): + logger.error("Error deleting boxd machine for sandbox %s: %s", self.id, e) + raise + logger.info("boxd machine for sandbox %s was already gone at close", self.id) + self._closed = True + + def ping(self) -> bool: + """Return True when the machine answers a trivial command.""" + try: + result = self._run(["bash", "-lc", "echo ok"], timeout=30.0) + except Exception as e: + logger.warning("boxd sandbox %s health check error: %s", self.id, e) + return False + return result.exit_code == 0 and "ok" in _text(result.stdout).split() + + # ── SDK bridge ─────────────────────────────────────────────────────── + + def _note_failure(self, error: Exception) -> None: + if self._on_terminal_failure is None or not is_machine_gone_error(error): + return + # Confirm before the provider drops the sandbox: a not-found answer + # from one call is not proof that the machine itself is gone. + try: + self._client.machines.get(self._machine_id, region=self._region) + except Exception as e: + if not is_machine_gone_error(e): + return + else: + return + try: + self._on_terminal_failure(self.id, str(error)) + except Exception: + logger.exception("Terminal boxd failure callback errored for %s", self.id) + + def _ensure_open(self) -> None: + with self._lock: + if self._closed: + raise RuntimeError("sandbox has been closed") + + def _start_machine(self) -> None: + """Start a stopped machine and wait until it runs commands again. + + ``wait_until_ready`` reports the machine state; a no-op probe confirms + that commands reach it before the original call is repeated. + """ + logger.info("boxd machine for sandbox %s is stopped; starting it", self.id) + machines = self._client.machines + machines.start(self._machine_id, region=self._region) + machines.wait_until_ready(self._machine_id, region=self._region) + deadline = time.monotonic() + _START_PROBE_SECONDS + while True: + try: + machines.exec(self._machine_id, ["true"], timeout=_START_PROBE_INTERVAL * 5, region=self._region) + return + except Exception: + if time.monotonic() >= deadline: + raise + time.sleep(_START_PROBE_INTERVAL) + + def _call(self, op: Callable[[], Any], *, repeat: bool = True, recover: bool = True) -> Any: + """Run one SDK call, starting the machine if it is stopped. + + A machine in standby or hibernation wakes on the call itself. A stopped + machine is started, and the call is repeated once when ``repeat`` is + true. Commands pass ``repeat=False``: a machine stopped while the + command ran must not run it a second time, so the caller gets an error + and decides. Budgeted mirror calls pass ``recover=False`` to skip the + control-plane calls used for recovery and failure confirmation. + """ + self._ensure_open() + if not recover: + # Budgeted output copies must not enter control-plane recovery or + # failure probes, whose SDK calls have separate request deadlines. + return op() + try: + try: + return op() + except Exception as first: + if not is_stopped_machine_error(first): + raise + try: + status = str(getattr(self._client.machines.get(self._machine_id, region=self._region), "status", "")) + except Exception: + raise first from None + if status.rsplit(".", 1)[-1].lower() != "stopped": + raise + self._start_machine() + if not repeat: + raise RuntimeError("The sandbox machine was stopped and has been started again; the command did not run to completion, run it again") from first + return op() + except Exception as e: + self._note_failure(e) + raise + + def _run(self, argv: list[str], *, env: dict[str, str] | None = None, timeout: float | None = None, binary: bool = False, repeat: bool = True, deadline: float | None = None) -> Any: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("boxd output synchronization deadline exceeded") + timeout = min(timeout, remaining) if timeout is not None else remaining + return self._call( + lambda: self._client.machines.exec( + self._machine_id, + argv, + env=env or None, + timeout=timeout, + encoding="buffer" if binary else "utf-8", + region=self._region, + ), + repeat=repeat, + recover=deadline is None, + ) + + def run_shell(self, script: str, *, timeout: float = _HELPER_TIMEOUT_SECONDS, binary: bool = False, deadline: float | None = None) -> Any: + """Run an internal helper script with the configured environment.""" + return self._run(["bash", "-lc", script], env=self._default_env, timeout=timeout, binary=binary, deadline=deadline) + + # ── Paths ──────────────────────────────────────────────────────────── + + @staticmethod + def _guard_path(path: str) -> str: + if not path: + raise ValueError("path must be a non-empty string") + normalized = path.replace("\\", "/") + if any(segment == ".." for segment in normalized.split("/")): + raise PermissionError(f"Access denied: path traversal detected in '{path}'") + if not normalized.startswith("/"): + raise ValueError(f"path must be absolute: '{path}'") + return normalized + + # ── Command execution ──────────────────────────────────────────────── + + def execute_command( + self, + command: str, + env: dict[str, str] | None = None, + timeout: float | None = None, + ) -> str: + """Run ``command`` through ``bash -lc`` and return combined output. + + The deadline (``timeout``, else the provider default) is enforced + inside the machine with coreutils ``timeout``, which ends the whole + command with exit code 124 and keeps the output it produced so far. + The SDK call gets a slightly longer deadline as a backstop. + """ + _validate_extra_env(env) + if self.is_closed: + return "Error: sandbox has been closed" + merged_env = {**self._default_env, **(env or {})} + if timeout is None or not math.isfinite(timeout) or timeout <= 0: + timeout = self._default_command_timeout + seconds = max(1, math.ceil(timeout)) + argv = ["timeout", "-k", str(_KILL_AFTER_SECONDS), str(seconds), "bash", "-lc", command] + call_timeout = seconds + _KILL_AFTER_SECONDS + _TIMEOUT_GRACE_SECONDS + try: + result = self._run(argv, env=merged_env, timeout=call_timeout, repeat=False) + except Exception as e: + logger.error("Failed to execute command in boxd sandbox %s: %s", self.id, e) + return f"Error: {e}" + + stdout = _text(result.stdout) + stderr = _text(result.stderr) + if stdout and stderr: + output = f"{stdout}\n{stderr}" + else: + output = stdout or stderr + exit_code = result.exit_code + if exit_code not in (0, None): + output = f"{output}\nExit Code: {exit_code}" if output else f"Command exited with code {exit_code}\nExit Code: {exit_code}" + return output if output else "(no output)" + + # ── File operations ────────────────────────────────────────────────── + + def _read_bytes(self, path: str, limit: int, *, timeout: float = _READ_TIMEOUT_SECONDS, deadline: float | None = None) -> bytes: + """Read at most ``limit`` bytes with a deadline. + + ``head -c limit+1`` bounds what the machine sends, so an oversized + file is detected without buffering more than one byte past the cap, + and the exec deadline bounds a stalled transfer. + """ + result = self._run(["head", "-c", str(limit + 1), "--", path], timeout=timeout, binary=True, deadline=deadline) + data = result.stdout if isinstance(result.stdout, bytes) else _text(result.stdout).encode("utf-8") + if result.exit_code != 0: + message = _text(result.stderr).strip() or f"head exited with code {result.exit_code}" + if "No such file" in message: + raise FileNotFoundError(errno.ENOENT, "No such file or directory", path) + if "Is a directory" in message: + raise IsADirectoryError(errno.EISDIR, "Is a directory", path) + if "Permission denied" in message: + raise PermissionError(errno.EACCES, "Permission denied", path) + raise OSError(f"cannot read '{path}': {message}") + if len(data) > limit: + raise OSError(errno.EFBIG, f"File exceeds maximum download size of {limit} bytes", path) + return data + + def read_file( + self, + path: str, + start_line: int | None = None, + end_line: int | None = None, + ) -> str: + try: + resolved = self._guard_path(path) + content = self._read_bytes(resolved, _MAX_READ_FILE_SIZE).decode("utf-8", errors="replace") + except Exception as e: + logger.error("read_file %s failed: %s", path, e) + return f"Error: {e}" + if start_line is None and end_line is None: + return content + lines = split_file_lines(content) + # Clamp like LocalSandbox.read_file: a negative start must not wrap + # around through negative-index slicing. + start = max(start_line or 1, 1) + end = max(end_line, 0) if end_line is not None else len(lines) + return "\n".join(lines[start - 1 : end]) + + def write_file(self, path: str, content: str, append: bool = False) -> None: + self._write_bytes(self._guard_path(path), content.encode("utf-8"), append=append) + + def update_file(self, path: str, content: bytes) -> None: + self._write_bytes(self._guard_path(path), content, append=False) + + def _write_bytes(self, resolved: str, data: bytes, *, append: bool) -> None: + fs = self._client.machines.fs + if not append: + # fs.write creates missing parent directories. + self._call(lambda: fs.write(self._machine_id, resolved, data, region=self._region)) + return + with self._write_lock: + try: + existing = self._read_bytes(resolved, _MAX_READ_FILE_SIZE) + except Exception as e: + if not is_missing_file_error(e): + raise + existing = b"" + payload = existing + data + self._call(lambda: fs.write(self._machine_id, resolved, payload, region=self._region)) + + def download_file(self, path: str, *, max_bytes: int | None = None, deadline: float | None = None) -> bytes: + """Download under the size cap and optional monotonic mirror deadline.""" + limit = download_limit(max_bytes, _MAX_DOWNLOAD_SIZE) + normalized = self._guard_path(path) + if normalized != VIRTUAL_PATH_PREFIX and not normalized.startswith(f"{VIRTUAL_PATH_PREFIX}/"): + raise PermissionError(f"Access denied: path must be under '{VIRTUAL_PATH_PREFIX}': '{path}'") + # head -c stops one byte past the cap, so a file that grows during + # the transfer still cannot exceed it. + try: + return self._read_bytes(normalized, limit, timeout=_DOWNLOAD_TIMEOUT_SECONDS, deadline=deadline) + except OSError as e: + if e.errno == errno.EFBIG or is_missing_file_error(e): + raise + raise OSError(f"cannot read '{path}' from sandbox: {e}") from e + except Exception as e: + raise OSError(f"cannot read '{path}' from sandbox: {e}") from e + + def list_dir(self, path: str, max_depth: int = 2) -> list[str]: + resolved = self._guard_path(path) + result = self.run_shell(remote_list_dir_command(resolved, max_depth)) + return parse_remote_list_dir_output(_text(result.stdout), resolved, pipeline_exit_code=result.exit_code) + + def glob( + self, + path: str, + pattern: str, + *, + include_dirs: bool = False, + max_results: int = 200, + ) -> tuple[list[str], bool]: + resolved = self._guard_path(path) + types = ("f", "d") if include_dirs else ("f",) + type_expr = " -o ".join(f"-type {t}" for t in types) + hard_limit = max(max_results * 4, max_results + 50) + # -H follows a symlinked search root, as list_dir does. + search = f"find -H {shlex.quote(resolved)} \\( {type_expr} \\) -print 2>/dev/null" + result = self.run_shell(remote_search_command(search, resolved, limit=hard_limit)) + # A missing root or a failed find must not read as "no files matched". + output = parse_remote_search_output(_text(result.stdout), resolved, tool="find", limit=hard_limit) + + matches: list[str] = [] + root = resolved.rstrip("/") or "/" + root_prefix = root if root == "/" else f"{root}/" + # Records are LF-delimited; other splitlines() boundaries can be path characters. + for entry in output.text.split("\n"): + # Do not strip: trailing whitespace can be part of a file name. + if not entry or (entry != root and not entry.startswith(root_prefix)): + continue + if should_ignore_path_under_root(entry, root): + continue + rel_path = entry[len(root) :].lstrip("/") + if not rel_path: + continue + if path_matches(pattern, rel_path): + matches.append(entry) + # Look one match past the cap so an exhausted tree that held + # exactly max_results is not reported as truncated. + if len(matches) > max_results: + return matches[:max_results], True + return matches, output.truncated + + def grep( + self, + path: str, + pattern: str, + *, + glob: str | None = None, + literal: bool = False, + case_sensitive: bool = False, + max_results: int = 100, + ) -> tuple[list[GrepMatch], bool]: + # grep gets the raw pattern: -F matches it literally, -E as a regex. + # The Python compile only catches gross errors at the boundary. + if not literal: + re.compile(pattern, 0 if case_sensitive else re.IGNORECASE) + + resolved = self._guard_path(path) + # --null ends the file name with NUL, so a colon in a path cannot be + # mistaken for the line-number separator. + flags = ["-r", "-n", "-H", "-I", "--null"] + if not case_sensitive: + flags.append("-i") + flags.append("-F" if literal else "-E") + if glob is not None: + # --include matches basenames only; it is a coarse pre-filter and + # path_matches below enforces the real directory scope. + include_pattern = glob.split("/")[-1] or glob + flags.append(shlex.quote(f"--include={include_pattern}")) + per_file_cap = max(max_results + 1, 50) + total_cap = max(max_results * 4, max_results + 50) + flags.append(f"-m{per_file_cap}") + search = "grep " + " ".join(flags) + f" -- {shlex.quote(pattern)} {shlex.quote(resolved)} 2>/dev/null" + result = self.run_shell(remote_search_command(search, resolved, limit=total_cap)) + # A missing root, a missing grep or an unreadable tree must not read as "no matches". + output = parse_remote_search_output(_text(result.stdout), resolved, tool="grep", limit=total_cap) + + root = resolved.rstrip("/") or "/" + root_prefix = root if root == "/" else f"{root}/" + matches: list[GrepMatch] = [] + for raw in output.text.split("\n"): + try: + file_path, rest = raw.split("\0", 1) + line_no_str, line_text = rest.split(":", 1) + line_number = int(line_no_str) + except ValueError: + continue + if should_ignore_path_under_root(file_path, root): + continue + if glob is not None: + if file_path != root and not file_path.startswith(root_prefix): + continue + rel_path = file_path.rsplit("/", 1)[-1] if file_path == root else file_path[len(root) :].lstrip("/") + if not path_matches(glob, rel_path): + continue + matches.append(GrepMatch(path=file_path, line_number=line_number, line=truncate_line(line_text))) + if len(matches) > max_results: + return matches[:max_results], True + return matches, output.truncated + + +def _text(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value + + +__all__ = ["BoxdSandbox"] diff --git a/backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox_provider.py b/backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox_provider.py new file mode 100644 index 0000000000..20a98fccd3 --- /dev/null +++ b/backend/packages/harness/deerflow/community/boxd_sandbox/boxd_sandbox_provider.py @@ -0,0 +1,949 @@ +"""``BoxdSandboxProvider``: DeerFlow :class:`SandboxProvider` backed by boxd. + +Runs each DeerFlow sandbox as an isolated `boxd `_ +machine: a full Ubuntu microVM with a persistent disk. The provider creates +one machine per ``(user, thread)``, reuses it within the process, and parks +released machines in a warm pool (shared :class:`WarmPoolLifecycleMixin` +machinery) for fast reclaim. The idle reaper deletes warm machines after +``idle_timeout``. + +A fresh machine for a thread gets the thread's Gateway-side files (uploads, +outputs, workspace) and the skills projection before it is handed out. +Release mirrors ``/mnt/user-data/outputs`` back to the Gateway, so artifacts +stay downloadable after the machine is gone. + +Config is read off :class:`SandboxConfig` (``extra="allow"``); see this +package's ``__init__`` docstring for the full set. The ``boxd`` SDK is imported +lazily, so the harness installs without it. +""" + +from __future__ import annotations + +import asyncio +import atexit +import logging +import math +import os +import posixpath +import shlex +import stat +import tarfile +import tempfile +import threading +import time +import unicodedata +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from functools import partial +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Any + +from deerflow.community.provider_credentials import resolve_api_key +from deerflow.config import get_app_config +from deerflow.sandbox.acquire_serialization import AcquireSerializer, AcquireSerializerClosed +from deerflow.sandbox.download import DEFAULT_DOWNLOAD_LIMIT +from deerflow.sandbox.identity import derive_sandbox_scope_token +from deerflow.sandbox.sandbox import Sandbox, _validate_extra_env +from deerflow.sandbox.sandbox_provider import SandboxProvider + +from ..warm_pool_lifecycle import WarmPoolLifecycleMixin +from .boxd_sandbox import DEFAULT_COMMAND_TIMEOUT, BoxdSandbox + +if TYPE_CHECKING: + from boxd import Boxd + + from deerflow.skills.projection import SkillProjectionPaths + +logger = logging.getLogger(__name__) + +DEFAULT_NAME_PREFIX = "deer-flow-" +DEFAULT_READY_TIMEOUT = 120.0 +USER_DATA_ROOT = "/mnt/user-data" +OUTPUTS_ROOT = f"{USER_DATA_ROOT}/outputs" +SKILL_CATEGORIES = ("public", "custom", "legacy", "integrations") +# Machine sizes boxd offers, as vCPU -> memory in GiB. +MACHINE_SIZES = {1: 4, 2: 8, 4: 16, 8: 32} +VALID_REGIONS = frozenset({"fra", "nyc"}) +_BOOTSTRAP_TIMEOUT = 60.0 +_UPLOAD_EXTRACT_TIMEOUT = 300.0 +# Deadline for each unary SDK call; an upload of up to 128 MiB is one call. +_CLIENT_TIMEOUT = 180.0 +_LABEL_APP = "deer-flow" +_LABEL_SCOPE = "deer-flow-scope" +_LABEL_GENERATION = "deer-flow-generation" +_PROTECTED_ROOTS = ("/bin", "/boot", "/dev", "/etc", "/lib", "/lib64", "/proc", "/root", "/run", "/sbin", "/sys", "/usr", "/var") +_WINDOWS_RESERVED_NAMES = frozenset({"CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$"} | {f"{prefix}{digit}" for prefix in ("COM", "LPT") for digit in "123456789¹²³"}) + + +def _import_client() -> type[Boxd]: + """Import boxd's ``Boxd`` client lazily (optional dependency).""" + try: + from boxd import Boxd + except ImportError as e: # pragma: no cover - depends on the optional dependency + raise ImportError("BoxdSandboxProvider requires the optional 'boxd' dependency. Install it with: pip install 'deerflow-harness[boxd]' or pip install 'boxd>=0.3.2'.") from e + return Boxd + + +def _validate_skills_root(container_path: str) -> str: + """The skills categories are replaced recursively, so the root must be a dedicated path.""" + normalized = posixpath.normpath(container_path or "") + if not normalized.startswith("/") or normalized == "/" or normalized != (container_path or "").rstrip("/"): + raise ValueError(f"skills.container_path {container_path!r} is not a safe boxd skills root: use a normalized absolute path such as /mnt/skills") + if normalized in {"/home", "/home/boxd", "/mnt", "/tmp"} or any(normalized == root or normalized.startswith(f"{root}/") for root in _PROTECTED_ROOTS): + raise ValueError(f"skills.container_path {container_path!r} is not a safe boxd skills root: it is a system directory") + if normalized == USER_DATA_ROOT or normalized.startswith(f"{USER_DATA_ROOT}/") or USER_DATA_ROOT.startswith(f"{normalized}/"): + raise ValueError(f"skills.container_path {container_path!r} must not overlap {USER_DATA_ROOT}") + return normalized + + +class _ShutdownIncompleteError(RuntimeError): + """Cleanup still owns acquisitions or machine deletions that need a retry.""" + + +class _AcquireTicket: + """Hands an acquire's result to a caller that may have been cancelled.""" + + def __init__(self) -> None: + self.lock = threading.Lock() + self.abandoned = False + self.sandbox_id: str | None = None + + +class _UploadBudget: + """File-count and byte limits for one tar upload into a machine.""" + + def __init__(self, max_files: int, max_bytes: int) -> None: + self.max_files = max_files + self.max_bytes = max_bytes + self.files = 0 + self.bytes = 0 + + def admit(self, size: int) -> bool: + if self.files + 1 > self.max_files or self.bytes + size > self.max_bytes: + return False + self.files += 1 + self.bytes += size + return True + + +class BoxdSandboxProvider(WarmPoolLifecycleMixin[BoxdSandbox], SandboxProvider): + """Run each DeerFlow sandbox as an isolated boxd machine.""" + + uses_thread_data_mounts = False + needs_upload_permission_adjustment = True + # The machine sees only the skills uploaded into it, so a per-Agent + # projection from sync_agent_skills cannot be bypassed from the shell. + supports_agent_skill_isolation = True + _idle_checker_thread_name = "boxd-idle-reaper" + _MAX_SYNC_FILES = 2000 + _MAX_SYNC_TOTAL_BYTES = 512 * 1024 * 1024 + _MAX_SYNC_LISTING_BYTES = 512 * 1024 + _SYNC_DEADLINE_SECONDS = 120.0 + _MAX_UPLOAD_FILES = 5000 + _MAX_UPLOAD_BYTES = 128 * 1024 * 1024 + _UPLOAD_CHUNK_BYTES = 1024 * 1024 + + def __init__(self) -> None: + self._lock = threading.Lock() + self._sandboxes: dict[str, BoxdSandbox] = {} + self._thread_sandboxes: dict[tuple[str, str], str] = {} + self._warm_pool: dict[str, tuple[BoxdSandbox, float]] = {} + self._acquire_serializer: AcquireSerializer[str] = AcquireSerializer(thread_name_prefix="boxd-acquire-wait") + self._idle_checker_stop = threading.Event() + self._idle_checker_thread: threading.Thread | None = None + self._shutdown_called = False + self._shutdown_complete = False + self._shutdown_lock = threading.Lock() + self._accepting = True + self._acquires = 0 + # Keyed by object identity: a replacement can reuse the logical id + # while an older machine still needs deletion. + self._pending_closes: dict[int, BoxdSandbox] = {} + self._closing: set[int] = set() + # Callers that received each active sandbox and have not released it. + self._claims: dict[str, int] = {} + self._client: Boxd | None = None + self._config = self._load_config() + atexit.register(self._shutdown_at_exit) + self._start_idle_checker() + + # ── Config ─────────────────────────────────────────────────────────── + + @staticmethod + def _optional_int(name: str, value: Any, *, minimum: int) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"sandbox.{name} must be an integer >= {minimum}") + return value + + @staticmethod + def _memory_gib(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, int) and not isinstance(value, bool): + return value + text = str(value).strip().upper().removesuffix("IB").removesuffix("B").removesuffix("G") + if text.isdigit(): + return int(text) + raise ValueError("sandbox.memory must be a size such as 8G") + + def _load_config(self) -> dict[str, Any]: + app_config = get_app_config() + sandbox_config = app_config.sandbox + + def option(name: str) -> Any: + return getattr(sandbox_config, name, None) + + # $VARS in config.yaml are already resolved by AppConfig; resolving + # again would rewrite a secret that itself starts with $. + environment = dict(option("environment") or {}) + _validate_extra_env(environment) + + region = option("region") + if region is not None and region not in VALID_REGIONS: + raise ValueError(f"sandbox.region must be one of {sorted(VALID_REGIONS)}") + + vcpu = self._optional_int("vcpu", option("vcpu"), minimum=1) + memory = self._memory_gib(option("memory")) + if vcpu is not None and vcpu not in MACHINE_SIZES: + raise ValueError(f"sandbox.vcpu must be one of {sorted(MACHINE_SIZES)}") + if memory is not None and memory not in MACHINE_SIZES.values(): + raise ValueError(f"sandbox.memory must be one of {[f'{gib}G' for gib in MACHINE_SIZES.values()]}") + if vcpu is not None and memory is not None and MACHINE_SIZES[vcpu] != memory: + raise ValueError(f"sandbox.vcpu={vcpu} goes with sandbox.memory={MACHINE_SIZES[vcpu]}G") + if vcpu is not None and memory is None: + memory = MACHINE_SIZES[vcpu] + if memory is not None and vcpu is None: + vcpu = next(cpu for cpu, gib in MACHINE_SIZES.items() if gib == memory) + + labels = option("labels") or {} + if not isinstance(labels, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in labels.items()): + raise ValueError("sandbox.labels must map strings to strings") + + name_prefix = option("name_prefix") or DEFAULT_NAME_PREFIX + if not isinstance(name_prefix, str) or not name_prefix.replace("-", "").isalnum() or name_prefix != name_prefix.lower(): + raise ValueError("sandbox.name_prefix may hold lowercase letters, digits and '-'") + + ready_timeout = option("ready_timeout") + try: + ready_timeout = float(DEFAULT_READY_TIMEOUT if ready_timeout is None else ready_timeout) + except (TypeError, ValueError) as exc: + raise ValueError("sandbox.ready_timeout must be positive") from exc + if not math.isfinite(ready_timeout) or ready_timeout <= 0: + raise ValueError("sandbox.ready_timeout must be positive") + + command_timeout = option("bash_command_timeout") + try: + command_timeout = float(DEFAULT_COMMAND_TIMEOUT if command_timeout is None else command_timeout) + except (TypeError, ValueError) as exc: + raise ValueError("sandbox.bash_command_timeout must be positive") from exc + if not math.isfinite(command_timeout) or command_timeout <= 0: + raise ValueError("sandbox.bash_command_timeout must be positive") + + replicas = option("replicas") + idle_timeout = option("idle_timeout") + return { + "command_timeout": command_timeout, + # Blank, absent or template values read as None; the SDK then + # reads BOXD_API_KEY from the environment. + # A boxd-specific name, so a key configured for another provider + # under sandbox.api_key is never sent to boxd. + "api_key": resolve_api_key(option("boxd_api_key")), + "org": option("org"), + "region": region, + "vcpu": vcpu, + "memory": f"{memory}G" if memory is not None else None, + "from_snapshot": option("from_snapshot"), + "auto_suspend_timeout": self._optional_int("auto_suspend_timeout", option("auto_suspend_timeout"), minimum=0), + "ready_timeout": ready_timeout, + "labels": dict(labels), + "name_prefix": name_prefix, + "environment": environment, + "skills_container_path": _validate_skills_root(app_config.skills.container_path), + "replicas": replicas if replicas is not None else self.DEFAULT_REPLICAS, + "idle_timeout": idle_timeout if idle_timeout is not None else self.DEFAULT_IDLE_TIMEOUT, + } + + def _get_client(self) -> Boxd: + with self._lock: + if self._client is not None: + return self._client + client_cls = _import_client() + if self._config["api_key"] is None and not os.environ.get("BOXD_API_KEY"): + raise ValueError("BoxdSandboxProvider needs an API key: set sandbox.boxd_api_key in config.yaml or the BOXD_API_KEY environment variable") + client = client_cls(api_key=self._config["api_key"], region=self._config["region"], timeout=_CLIENT_TIMEOUT) + with self._lock: + if self._client is None: + self._client = client + return self._client + + # ── Identity ───────────────────────────────────────────────────────── + + @staticmethod + def _sandbox_id(thread_id: str, user_id: str) -> str: + return derive_sandbox_scope_token(user_id=user_id, thread_id=thread_id) + + @staticmethod + def _thread_key(thread_id: str, user_id: str | None) -> tuple[str, str]: + return (user_id or "", thread_id) + + def _machine_name(self, sandbox_id: str) -> str: + # Names are unique per owner and are not free again the moment a + # machine is deleted, so each generation gets a random suffix. + return f"{self._config['name_prefix']}{sandbox_id}-{uuid.uuid4().hex[:6]}" + + # ── Warm pool plumbing ─────────────────────────────────────────────── + + def _start_idle_checker(self) -> None: + if self._config["idle_timeout"] <= 0: + return + super()._start_idle_checker() + + def _active_count_locked(self) -> int: + return len(self._sandboxes) + len(self._pending_closes) + + def _cleanup_idle_resources(self, idle_timeout: float) -> None: + self._retry_pending_closes() + super()._cleanup_idle_resources(idle_timeout) + + def _destroy_warm_entry(self, sandbox_id: str, entry: BoxdSandbox, *, reason: str) -> None: + self._close_quietly(entry, context=f"warm pool, reason={reason}") + + def _close_quietly(self, sandbox: BoxdSandbox, *, context: str) -> None: + """Own deletion until it is accepted; keep failures for a later retry.""" + token = id(sandbox) + with self._lock: + self._pending_closes[token] = sandbox + if token in self._closing: + return + self._closing.add(token) + closed = False + try: + sandbox.close() + except Exception as e: + logger.warning("Error deleting boxd sandbox %s (%s): %s", sandbox.id, context, e) + else: + closed = True + finally: + with self._lock: + if closed: + self._pending_closes.pop(token, None) + self._closing.discard(token) + + def _retry_pending_closes(self) -> None: + with self._lock: + pending = list(self._pending_closes.values()) + for sandbox in pending: + self._close_quietly(sandbox, context="retry deletion") + + def _invalidate_sandbox(self, sandbox_id: str, reason: str, *, expected_sandbox: BoxdSandbox | None = None) -> None: + """Deregister a sandbox whose machine is gone, and finish its cleanup.""" + with self._lock: + active = self._sandboxes.get(sandbox_id) + warm_entry = self._warm_pool.get(sandbox_id) + current = active or (warm_entry[0] if warm_entry is not None else None) + if expected_sandbox is not None and current is not expected_sandbox: + logger.debug("Ignoring late terminal failure from replaced boxd sandbox %s: %s", sandbox_id, reason) + return + self._sandboxes.pop(sandbox_id, None) + self._claims.pop(sandbox_id, None) + self._warm_pool.pop(sandbox_id, None) + for key in [k for k, sid in self._thread_sandboxes.items() if sid == sandbox_id]: + self._thread_sandboxes.pop(key, None) + if current is None: + return + self._pending_closes[id(current)] = current + logger.warning("Invalidating boxd sandbox %s: %s", sandbox_id, reason) + self._close_quietly(current, context="terminal failure") + + # ── Acquire ────────────────────────────────────────────────────────── + + @contextmanager + def _acquiring(self) -> Iterator[None]: + with self._lock: + if not self._accepting: + raise RuntimeError("boxd provider is retired or shutting down") + self._acquires += 1 + try: + yield + finally: + with self._lock: + self._acquires -= 1 + + def acquire(self, thread_id: str | None = None, *, user_id: str | None = None) -> str: + return self._acquire(thread_id, user_id) + + def _acquire(self, thread_id: str | None, user_id: str | None, ticket: _AcquireTicket | None = None) -> str: + with self._acquiring(): + if thread_id is None: + sandbox_id = str(uuid.uuid4())[:8] + sandbox_id = self._publish_sandbox(self._create_sandbox(sandbox_id, thread_id=None, user_id=user_id)) + else: + key = self._thread_key(thread_id, user_id) + scope_id = self._sandbox_id(thread_id, user_id or "") + with self._acquire_serializer.hold(scope_id): + sandbox_id = self._acquire_scope_locked(key, scope_id, thread_id=thread_id, user_id=user_id) + if ticket is not None: + with ticket.lock: + ticket.sandbox_id = sandbox_id + abandoned = ticket.abandoned + if abandoned: + self._drop_claim(sandbox_id) + return sandbox_id + + async def acquire_async(self, thread_id: str | None = None, *, user_id: str | None = None) -> str: + """Acquire without blocking the event loop. + + The whole synchronous acquire (serializer wait and body) runs on the + serializer's dedicated executor. A cancelled caller never receives the + sandbox id, so it gives back the claim that the worker took for it, + either right away or when the worker finishes. A sandbox whose last + claim is given back is released to the warm pool, where the idle + reaper deletes it. Neither path depends on the event loop. + """ + ticket = _AcquireTicket() + try: + return await self._acquire_serializer.run_on_executor(partial(self._acquire, thread_id, user_id, ticket)) + except asyncio.CancelledError: + with ticket.lock: + ticket.abandoned = True + sandbox_id = ticket.sandbox_id + if sandbox_id is not None: + # release() can transfer files and wait on the scope lock. + threading.Thread(target=self._drop_claim, args=(sandbox_id,), name="boxd-abandoned-release", daemon=True).start() + raise + + def _drop_claim(self, sandbox_id: str) -> None: + """Give back the claim of a caller that never received ``sandbox_id``.""" + try: + with self._acquire_serializer.hold(sandbox_id): + with self._lock: + if sandbox_id not in self._sandboxes: + return + remaining = self._claims.get(sandbox_id, 1) - 1 + self._claims[sandbox_id] = remaining + if remaining > 0: + return + logger.info("Releasing boxd sandbox %s: every caller that acquired it was cancelled", sandbox_id) + self._release_scope_locked(sandbox_id) + except AcquireSerializerClosed: + pass # reset/shutdown owns the sandbox now + except Exception: + logger.warning("Could not release abandoned boxd sandbox %s", sandbox_id, exc_info=True) + + def _publish_sandbox(self, sandbox: BoxdSandbox, key: tuple[str, str] | None = None) -> str: + with self._lock: + if self._accepting: + self._sandboxes[sandbox.id] = sandbox + self._claims[sandbox.id] = 1 + if key is not None: + self._thread_sandboxes[key] = sandbox.id + return sandbox.id + # The worker that created the machine owns its cleanup, even when the + # caller abandoned the await or shutdown closed admission meanwhile. + self._close_quietly(sandbox, context="acquired after provider retirement") + raise RuntimeError("boxd provider retired or shut down during acquire") + + def _acquire_scope_locked(self, key: tuple[str, str], sandbox_id: str, *, thread_id: str, user_id: str | None) -> str: + with self._lock: + if not self._accepting: + raise RuntimeError("boxd provider is retired or shutting down") + existing = self._thread_sandboxes.get(key) + if existing is not None and existing in self._sandboxes: + self._claims[existing] = self._claims.get(existing, 0) + 1 + return existing + + reclaimed = self._reclaim_warm_pool(sandbox_id) + if reclaimed is not None: + with self._lock: + if not self._accepting: + raise RuntimeError("boxd provider retired or shut down during acquire") + self._thread_sandboxes[key] = reclaimed + return reclaimed + + sandbox = self._create_sandbox(sandbox_id, thread_id=thread_id, user_id=user_id) + return self._publish_sandbox(sandbox, key) + + def _create_sandbox(self, sandbox_id: str, *, thread_id: str | None, user_id: str | None) -> BoxdSandbox: + replicas, total = self._replica_count() + if total >= replicas: + evicted = self._evict_oldest_warm() + self._log_replicas_soft_cap(replicas, sandbox_id, evicted) + + client = self._get_client() + generation = uuid.uuid4().hex + labels = {**self._config["labels"], _LABEL_APP: "sandbox", _LABEL_SCOPE: sandbox_id, _LABEL_GENERATION: generation} + create_kwargs: dict[str, Any] = { + "org": self._config["org"], + # The agent runs untrusted code: an isolated machine joins no + # private network and holds no boxd credentials. + "isolated": True, + "labels": labels, + } + for name in ("region", "vcpu", "memory", "from_snapshot", "auto_suspend_timeout"): + if self._config[name] is not None: + create_kwargs[name] = self._config[name] + try: + machine = client.machines.create(self._machine_name(sandbox_id), **create_kwargs) + except BaseException: + self._delete_generation(client, generation, sandbox_id) + raise + sandbox = BoxdSandbox( + sandbox_id, + client, + machine.id, + region=self._config["region"], + default_env=self._config["environment"], + default_command_timeout=self._config["command_timeout"], + ) + try: + client.machines.wait_until_ready(machine.id, timeout=self._config["ready_timeout"], region=self._config["region"]) + self._bootstrap(sandbox) + if thread_id is not None and user_id: + self._restore_thread_files(sandbox, thread_id=thread_id, user_id=user_id) + self._upload_shared_skills(sandbox, thread_id=thread_id, user_id=user_id) + except BaseException: + # Never hand out a half-prepared machine, and never leave one running. + self._close_quietly(sandbox, context="preparation failure") + raise + sandbox._on_terminal_failure = lambda failed_id, reason: self._invalidate_sandbox(failed_id, reason, expected_sandbox=sandbox) + logger.info("Created boxd sandbox %s (machine %s)", sandbox_id, machine.id) + return sandbox + + def _delete_generation(self, client: Boxd, generation: str, sandbox_id: str) -> None: + """Delete a machine whose create raised after the server made it. + + The generation label is unique to one create call, so a match can + only be the machine that this call asked for. + """ + try: + found = client.machines.list(org=self._config["org"], labels={_LABEL_GENERATION: generation}, region=self._config["region"]) + except Exception: + logger.warning("Could not look for a boxd machine left by a failed create", exc_info=True) + return + for machine in found: + if (getattr(machine, "labels", None) or {}).get(_LABEL_GENERATION) != generation: + continue + sandbox = BoxdSandbox(sandbox_id, client, machine.id, region=self._config["region"]) + self._close_quietly(sandbox, context="failed create") + + def _bootstrap(self, sandbox: BoxdSandbox) -> None: + skills_root = shlex.quote(self._config["skills_container_path"]) + dirs = " ".join(shlex.quote(f"{USER_DATA_ROOT}/{name}") for name in ("workspace", "uploads", "outputs")) + script = f'set -e; sudo -n mkdir -p -- {dirs} {skills_root}; sudo -n chown "$(id -u):$(id -g)" -- {shlex.quote(USER_DATA_ROOT)} {dirs} {skills_root}; echo BOOTSTRAP_OK' + result = sandbox.run_shell(script, timeout=_BOOTSTRAP_TIMEOUT) + stdout = result.stdout if isinstance(result.stdout, str) else bytes(result.stdout).decode("utf-8", "replace") + if result.exit_code != 0 or "BOOTSTRAP_OK" not in stdout: + raise RuntimeError(f"boxd sandbox bootstrap failed with exit code {result.exit_code}") + + # ── Host to machine transfer ───────────────────────────────────────── + + def _add_tree(self, archive: tarfile.TarFile, source: Path, arcname: str, budget: _UploadBudget) -> bool: + """Add regular files below ``source``; skip links and special files. + + Returns False when the budget stopped the walk early. + """ + + def scan_error(error: OSError) -> None: + raise error + + for directory, dirnames, filenames in os.walk(source, followlinks=False, onerror=scan_error): + parent = Path(directory) + dirnames[:] = sorted(name for name in dirnames if not (parent / name).is_symlink()) + for name in sorted(filenames): + path = parent / name + st = path.lstat() + if not stat.S_ISREG(st.st_mode): + continue + if not budget.admit(st.st_size): + return False + rel = path.relative_to(source).as_posix() + info = tarfile.TarInfo(f"{arcname}/{rel}") + info.size = st.st_size + info.mode = st.st_mode & 0o755 + info.mtime = int(st.st_mtime) + with path.open("rb") as fh: + archive.addfile(info, fh) + return True + + def _upload_trees(self, sandbox: BoxdSandbox, sources: list[tuple[Path, str]], *, dest_root: str, replace: tuple[str, ...] = ()) -> bool: + """Upload host directories into ``dest_root`` as one tar archive. + + ``sources`` pairs a host directory with its sub-path below + ``dest_root``. ``replace`` lists sub-paths removed before extraction. + Returns False when the upload stopped at its size limit. + """ + budget = _UploadBudget(self._MAX_UPLOAD_FILES, self._MAX_UPLOAD_BYTES) + complete = True + with tempfile.TemporaryFile() as spool: + with tarfile.open(fileobj=spool, mode="w:gz") as archive: + for source, arcname in sources: + for part in (source, *source.parents): + if part.is_symlink(): + raise ValueError(f"Refusing symlinked upload root for {arcname}") + if not source.is_dir(): + continue + if not self._add_tree(archive, source, arcname, budget): + complete = False + break + if budget.files == 0 and not replace: + return complete + remote_tar = f"/tmp/deerflow-upload-{uuid.uuid4().hex}.tar.gz" + + def chunks() -> Iterator[bytes]: + spool.seek(0) + while chunk := spool.read(self._UPLOAD_CHUNK_BYTES): + yield chunk + + sandbox._call(lambda: sandbox._client.machines.fs.write(sandbox.machine_id, remote_tar, chunks(), region=sandbox._region)) + root = shlex.quote(dest_root) + removals = " ".join(shlex.quote(f"{dest_root}/{name}") for name in replace) + script = ( + f"set -e; trap 'rm -f {shlex.quote(remote_tar)}' EXIT; " + f"if [ -L {root} ]; then echo 'refusing symlinked destination' >&2; exit 2; fi; " + + (f"rm -rf -- {removals}; " if removals else "") + + f"mkdir -p -- {root}; tar -xzf {shlex.quote(remote_tar)} -C {root} --no-same-owner; echo UPLOAD_OK" + ) + result = sandbox.run_shell(script, timeout=_UPLOAD_EXTRACT_TIMEOUT) + stdout = result.stdout if isinstance(result.stdout, str) else bytes(result.stdout).decode("utf-8", "replace") + if result.exit_code != 0 or "UPLOAD_OK" not in stdout: + raise RuntimeError(f"boxd upload into {dest_root} failed with exit code {result.exit_code}") + return complete + + def _restore_thread_files(self, sandbox: BoxdSandbox, *, thread_id: str, user_id: str) -> None: + """Give a fresh machine the thread's Gateway-side files. + + Uploads go first, then the workspace, then outputs (which the Gateway + already serves). Files past the size limit are skipped with a warning, + so a large thread still gets a sandbox. A transfer failure raises: the + caller discards the machine instead of handing out a half-restored one. + """ + from deerflow.config.paths import get_paths + + paths = get_paths() + sources = [ + (paths.sandbox_uploads_dir(thread_id, user_id=user_id), "uploads"), + (paths.sandbox_work_dir(thread_id, user_id=user_id), "workspace"), + (paths.sandbox_outputs_dir(thread_id, user_id=user_id), "outputs"), + ] + if not self._upload_trees(sandbox, sources, dest_root=USER_DATA_ROOT): + logger.warning( + "Thread files for boxd sandbox %s exceed %d files or %d bytes; the rest were not copied into the new machine", + sandbox.id, + self._MAX_UPLOAD_FILES, + self._MAX_UPLOAD_BYTES, + ) + + def _upload_shared_skills(self, sandbox: BoxdSandbox, *, thread_id: str, user_id: str) -> None: + """Best effort: upload the user's shared skills projection. + + A thread with an Agent skill policy view gets its projection from the + sandbox middleware through :meth:`sync_agent_skills`, so it never sees + the shared view first. + """ + from deerflow.config.paths import get_paths + from deerflow.skills.projection import ensure_skill_projections + from deerflow.skills.storage import get_or_new_user_skill_storage + + try: + if get_paths().thread_skills_view_dir(thread_id, user_id=user_id).exists(): + return + storage = get_or_new_user_skill_storage(user_id, app_config=get_app_config()) + projection = ensure_skill_projections(storage) + self._upload_skills(sandbox, projection) + except Exception as exc: + logger.warning("Could not upload the skills projection for boxd sandbox %s: %s", sandbox.id, exc) + + def _upload_skills(self, sandbox: BoxdSandbox, projection: SkillProjectionPaths) -> None: + sources = [(Path(getattr(projection, category)), category) for category in SKILL_CATEGORIES] + if not self._upload_trees(sandbox, sources, dest_root=self._config["skills_container_path"], replace=SKILL_CATEGORIES): + raise RuntimeError("Skills projection exceeds the boxd upload limit") + + def sync_agent_skills( + self, + sandbox_id: str, + *, + thread_id: str, + user_id: str, + projection: SkillProjectionPaths, + ) -> None: + """Replace the machine's skill categories with this thread's projection. + + Holds the same scope serializer as acquire and release, so no other + lifecycle step sees a half-written skills tree. + """ + with self._acquire_serializer.hold(sandbox_id): + with self._lock: + sandbox = self._sandboxes.get(sandbox_id) + if sandbox is None: + raise RuntimeError(f"boxd sandbox {sandbox_id} is not available for skill synchronization") + self._upload_skills(sandbox, projection) + + # ── Reuse ──────────────────────────────────────────────────────────── + + def _reclaim_warm_pool(self, sandbox_id: str) -> str | None: + with self._lock: + warm_entry = self._warm_pool.get(sandbox_id) + if warm_entry is None: + return None + sandbox = warm_entry[0] + if not sandbox.ping(): + with self._lock: + removed = self._warm_pool.pop(sandbox_id, None) + if removed is not None: + self._destroy_warm_entry(sandbox_id, removed[0], reason="health_check_failed") + return None + with self._lock: + removed = self._warm_pool.pop(sandbox_id, None) + if removed is None: + return None + admitted = self._accepting + if admitted: + self._sandboxes[sandbox_id] = removed[0] + self._claims[sandbox_id] = 1 + if not admitted: + self._close_quietly(removed[0], context="reclaimed after provider retirement") + raise RuntimeError("boxd provider retired or shut down during acquire") + logger.info("Reclaimed warm boxd sandbox %s", sandbox_id) + return sandbox_id + + def get(self, sandbox_id: str) -> Sandbox | None: + with self._lock: + return self._sandboxes.get(sandbox_id) + + # ── Release and output mirroring ───────────────────────────────────── + + def release(self, sandbox_id: str) -> None: + """Mirror outputs to the Gateway, then park the machine in the warm pool. + + Holds the acquire serializer, so the next turn cannot reuse the + machine while its outputs are still being copied, and keeps the active + entry during the transfer so the reaper cannot delete it. + """ + with self._lock: + if sandbox_id not in self._sandboxes: + return + try: + with self._acquire_serializer.hold(sandbox_id): + self._release_scope_locked(sandbox_id) + except AcquireSerializerClosed: + # reset/shutdown closed the serializer after the check above. + with self._lock: + untracked = sandbox_id not in self._sandboxes + if not untracked: + raise + + def _release_scope_locked(self, sandbox_id: str) -> None: + with self._lock: + sandbox = self._sandboxes.get(sandbox_id) + scope = next((key for key, value in self._thread_sandboxes.items() if value == sandbox_id), None) + if sandbox is None: + return + if scope is not None and scope[0]: + try: + self._sync_outputs_to_host(sandbox, user_id=scope[0], thread_id=scope[1]) + except Exception: + logger.warning("Failed to synchronize boxd outputs for %s", sandbox_id, exc_info=True) + with self._lock: + # A terminal failure or shutdown may have untracked the sandbox + # during the transfer. Never resurrect it in the warm pool. + if self._sandboxes.get(sandbox_id) is not sandbox: + return + self._sandboxes.pop(sandbox_id) + self._claims.pop(sandbox_id, None) + for key in [key for key, value in self._thread_sandboxes.items() if value == sandbox_id]: + self._thread_sandboxes.pop(key, None) + close_now = self._shutdown_called + if close_now: + self._pending_closes[id(sandbox)] = sandbox + else: + self._warm_pool[sandbox_id] = (sandbox, time.monotonic()) + if close_now: + self._close_quietly(sandbox, context="released during shutdown") + else: + logger.info("Released boxd sandbox %s to the warm pool", sandbox_id) + + @staticmethod + def _output_parts(remote_path: str) -> tuple[str, ...]: + prefix = f"{OUTPUTS_ROOT}/" + if not remote_path.startswith(prefix) or "\\" in remote_path: + raise ValueError("Output path is outside the outputs directory") + parts = remote_path[len(prefix) :].split("/") + if any(part in {"", ".", ".."} or ":" in part or part.partition(".")[0].rstrip(" ").upper() in _WINDOWS_RESERVED_NAMES or part.endswith((" ", ".")) for part in parts): + raise ValueError("Unsafe output path") + return tuple(parts) + + @staticmethod + def _output_name_key(name: str) -> str: + return unicodedata.normalize("NFC", name.casefold()) + + @classmethod + def _ambiguous_output_paths(cls, remote_paths: list[str | None]) -> set[str]: + # Names that differ only by case or Unicode normalization collide on + # some hosts. A directory alias rejects all of its children. + components: dict[tuple[int, str], tuple[str, int]] = {} + ambiguous: set[int] = set() + path_nodes: dict[str, list[int]] = {} + for remote_path in remote_paths: + if remote_path is None: + continue + try: + parts = cls._output_parts(remote_path) + except ValueError: + continue + parent = 0 + nodes = path_nodes.setdefault(remote_path, []) + for part in parts: + key = (parent, cls._output_name_key(part)) + spelling, node = components.setdefault(key, (part, len(components) + 1)) + if spelling != part: + ambiguous.add(node) + nodes.append(node) + parent = node + return {path for path, nodes in path_nodes.items() if ambiguous.intersection(nodes)} + + @classmethod + def _output_destination(cls, root: Path, remote_path: str) -> Path: + parts = cls._output_parts(remote_path) + target = root.joinpath(*PurePosixPath(*parts).parts) + # Reject links in existing parents, including the owner/thread bucket. + for component in reversed((target, *target.parents)): + if component.is_symlink() or component.is_junction(): + raise ValueError("Symlinked output destination") + parent = root + for part in parts: + if parent.is_dir(): + key = cls._output_name_key(part) + with os.scandir(parent) as entries: + if any(entry.name != part and cls._output_name_key(entry.name) == key for entry in entries): + raise ValueError("Output destination aliases an existing path") + parent = parent / part + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and not stat.S_ISREG(target.stat().st_mode): + raise ValueError("Output destination is not a regular file") + return target + + def _list_remote_outputs(self, sandbox: BoxdSandbox, *, deadline: float) -> list[str | None] | None: + """NUL-framed listing of output files; None when it is over the limit.""" + root = shlex.quote(OUTPUTS_ROOT) + script = f"set -o pipefail; [ -d {root} ] && [ ! -L {root} ] || exit 0; find {root} -type f -print0 2>/dev/null | head -c {self._MAX_SYNC_LISTING_BYTES + 1}" + result = sandbox.run_shell(script, timeout=self._SYNC_DEADLINE_SECONDS, binary=True, deadline=deadline) + raw = result.stdout if isinstance(result.stdout, bytes) else str(result.stdout).encode("utf-8") + if len(raw) > self._MAX_SYNC_LISTING_BYTES: + logger.warning("boxd output listing exceeded %d bytes; skipping synchronization", self._MAX_SYNC_LISTING_BYTES) + return None + if result.exit_code != 0: + # find hit an unreadable directory: keep the complete records. + logger.warning("boxd output listing had traversal errors; synchronizing complete records") + if raw and not raw.endswith(b"\0"): + raw = raw[: raw.rfind(b"\0") + 1] + paths: list[str | None] = [] + for record in filter(None, raw.split(b"\0")): + try: + paths.append(record.decode("utf-8")) + except UnicodeDecodeError: + logger.warning("Skipping non-UTF-8 boxd output %r", record) + paths.append(None) + return paths + + def _sync_outputs_to_host(self, sandbox: BoxdSandbox, *, thread_id: str, user_id: str) -> None: + """Best-effort bounded mirror of the outputs directory to the Gateway.""" + from deerflow.config.paths import get_paths + + root = get_paths().sandbox_outputs_dir(thread_id, user_id=user_id) + deadline = time.monotonic() + self._SYNC_DEADLINE_SECONDS + remote_paths = self._list_remote_outputs(sandbox, deadline=deadline) + if not remote_paths: + return + # Include records beyond the transfer cap: publishing the first of two + # aliases would otherwise expose its bytes through both artifact URLs. + ambiguous_paths = self._ambiguous_output_paths(remote_paths) + total = 0 + for index, remote_path in enumerate(remote_paths): + if index >= self._MAX_SYNC_FILES or total >= self._MAX_SYNC_TOTAL_BYTES or time.monotonic() >= deadline: + logger.warning("boxd output synchronization stopped at its file, byte or time limit") + break + if remote_path is None: + continue + if remote_path in ambiguous_paths: + logger.warning("Skipping ambiguous boxd output %r", remote_path) + continue + staged: Path | None = None + try: + target = self._output_destination(root, remote_path) + data = sandbox.download_file(remote_path, max_bytes=min(DEFAULT_DOWNLOAD_LIMIT, self._MAX_SYNC_TOTAL_BYTES - total), deadline=deadline) + total += len(data) + with tempfile.NamedTemporaryFile(prefix=".boxd-sync-", dir=target.parent, delete=False) as output: + staged = Path(output.name) + output.write(data) + # Check confinement again after the network call, keeping any + # previous complete version on error. + self._output_destination(root, remote_path) + staged.replace(target) + except (OSError, ValueError): + logger.warning("Could not synchronize boxd output %r", remote_path, exc_info=True) + finally: + if staged is not None: + staged.unlink(missing_ok=True) + + # ── Teardown ───────────────────────────────────────────────────────── + + def reset(self) -> None: + """Park tracked sandboxes in this instance's warm pool. + + ``reset_sandbox_provider()`` drops the singleton and calls this so a + config change applies on the next construction. The machines stay + visible to this instance's idle reaper and atexit shutdown. + """ + with self._lock: + self._accepting = False + now = time.monotonic() + for sandbox_id, sandbox in self._sandboxes.items(): + self._warm_pool.setdefault(sandbox_id, (sandbox, now)) + self._sandboxes.clear() + self._thread_sandboxes.clear() + self._acquire_serializer.close() + + def _shutdown_at_exit(self) -> None: + try: + self.shutdown() + except _ShutdownIncompleteError as e: + logger.warning("%s", e) + + def shutdown(self) -> None: + # Shutdown retries cleanup but never reopens admission. An SDK call in + # flight cannot be cancelled: its worker keeps ownership and deletes a + # late machine itself. Report unfinished work instead of hiding it. + with self._shutdown_lock: + with self._lock: + if self._shutdown_complete: + return + self._shutdown_called = True + self._accepting = False + self._acquire_serializer.close() + reaper_stopped = True + try: + self._stop_idle_checker() + except RuntimeError: + # The reaper may be inside a slow delete. Delete everything + # else now; _closing keeps the two from deleting one machine twice. + reaper_stopped = False + with self._lock: + active = list(self._sandboxes.values()) + warm = [sandbox for sandbox, _ in self._warm_pool.values()] + self._sandboxes.clear() + self._claims.clear() + self._warm_pool.clear() + self._thread_sandboxes.clear() + previous_pending = list(self._pending_closes.values()) + for sandbox in {id(box): box for box in active + warm + previous_pending}.values(): + self._close_quietly(sandbox, context="shutdown") + with self._lock: + if self._acquires or self._pending_closes or not reaper_stopped: + raise _ShutdownIncompleteError(f"boxd shutdown incomplete: {self._acquires} acquires and {len(self._pending_closes)} deletions pending, reaper stopped: {reaper_stopped}; retry shutdown") + self._shutdown_complete = True + + +__all__ = ["BoxdSandboxProvider"] diff --git a/backend/packages/harness/deerflow/community/provider_credentials.py b/backend/packages/harness/deerflow/community/provider_credentials.py index 888c0fbcfd..cb8e9b9945 100644 --- a/backend/packages/harness/deerflow/community/provider_credentials.py +++ b/backend/packages/harness/deerflow/community/provider_credentials.py @@ -38,6 +38,7 @@ URL_USERINFO_ERROR = "base_url must not contain username or password information ENV_TEMPLATE_PLACEHOLDERS = frozenset( { "your-anthropic-auth-token", + "your-boxd-api-key", "your-claude-code-oauth-token", "your-deepseek-api-key", "your-dingtalk-client-id", diff --git a/backend/packages/harness/deerflow/config/sandbox_config.py b/backend/packages/harness/deerflow/config/sandbox_config.py index 92765c211e..7d9f138f83 100644 --- a/backend/packages/harness/deerflow/config/sandbox_config.py +++ b/backend/packages/harness/deerflow/config/sandbox_config.py @@ -234,6 +234,12 @@ class SandboxConfig(BaseModel): ready_timeout: Create/readiness deadline in seconds (default: 30). sandbox_timeout: Remote lifetime in seconds (default: 14400); 0 requires explicit provider cleanup. + + BoxdSandboxProvider specific options: + boxd_api_key, org, region: boxd credentials and placement + (``BOXD_API_KEY`` is the fallback for ``boxd_api_key``). + vcpu, memory, from_snapshot, auto_suspend_timeout, ready_timeout, + name_prefix, labels: machine shape, readiness deadline and naming. """ use: str = Field( @@ -365,7 +371,7 @@ class SandboxConfig(BaseModel): "`hard_timeout`; the frozen legacy `all-in-one-sandbox:latest` image " "only gets the bounded host-side request. `bash_command_timeout` is " "forwarded to each provider's command API. LocalSandbox, AioSandbox, " - "OpenSandbox, E2B, BoxLite, and Tenki apply it to bash commands; " + "OpenSandbox, E2B, BoxLite, Tenki, and boxd apply it to bash commands; " "providers without per-command timeout support retain their own " "defaults." ), diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml index eca5cbc2d7..dbb9236705 100644 --- a/backend/packages/harness/pyproject.toml +++ b/backend/packages/harness/pyproject.toml @@ -98,6 +98,9 @@ tenki = ["tenki>=1.4.0"] # OpenSandbox remote sandbox provider (deerflow.community.opensandbox). The # sync SDK is loaded only when this provider is selected. opensandbox = ["opensandbox>=0.1.15,<0.2.0"] +# boxd machine sandbox provider (deerflow.community.boxd_sandbox). The SDK is +# loaded only when this provider is selected. +boxd = ["boxd>=0.3.2"] # Agent observability (Monocle). Optional so a default install stays free of the # OpenTelemetry stack; only pulled in when MONOCLE_TRACING is used. monocle = ["monocle_apptrace>=0.8.8"] diff --git a/backend/tests/test_agent_guidance_check.py b/backend/tests/test_agent_guidance_check.py index 3d50d067e4..c5f5f78364 100644 --- a/backend/tests/test_agent_guidance_check.py +++ b/backend/tests/test_agent_guidance_check.py @@ -35,6 +35,7 @@ EXPECTED_GUIDANCE_PATHS = { "backend/packages/harness/deerflow/community/boxlite/AGENTS.md", "backend/packages/harness/deerflow/community/browser_automation/AGENTS.md", "backend/packages/harness/deerflow/community/opensandbox/AGENTS.md", + "backend/packages/harness/deerflow/community/boxd_sandbox/AGENTS.md", "backend/packages/harness/deerflow/config/AGENTS.md", "backend/packages/harness/deerflow/extensions/AGENTS.md", "backend/packages/harness/deerflow/runtime/AGENTS.md", diff --git a/backend/tests/test_boxd_sandbox_live.py b/backend/tests/test_boxd_sandbox_live.py new file mode 100644 index 0000000000..33475ca4e0 --- /dev/null +++ b/backend/tests/test_boxd_sandbox_live.py @@ -0,0 +1,196 @@ +"""Live tests for the boxd sandbox provider against real boxd machines. + +Skipped unless explicitly enabled; they create and delete real machines: + + DEER_FLOW_RUN_LIVE_TESTS=1 BOXD_API_KEY=... PYTHONPATH=. uv run pytest tests/test_boxd_sandbox_live.py -v -s + +Optional: ``BOXD_ORG``, ``BOXD_TEST_REGION`` (fra or nyc), ``BOXD_TEST_NAME_PREFIX`` +(default ``deer-flow-test-``) and ``BOXD_TEST_LABELS`` (``key=value,key=value``). +The module shares one provider; its shutdown deletes every machine the tests +created and checks that each one is gone. +""" + +from __future__ import annotations + +import hashlib +import os +import time +import types +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.live + +_skip_reason = None +if os.environ.get("CI"): + _skip_reason = "Live tests skipped in CI" +elif os.environ.get("DEER_FLOW_RUN_LIVE_TESTS") != "1": + _skip_reason = "Set DEER_FLOW_RUN_LIVE_TESTS=1 to run live tests with real external APIs" +elif not os.environ.get("BOXD_API_KEY"): + _skip_reason = "Set BOXD_API_KEY to run the boxd live tests" +if _skip_reason: + pytest.skip(_skip_reason, allow_module_level=True) + +pytest.importorskip("boxd") + +from deerflow.community.boxd_sandbox import BoxdSandboxProvider # noqa: E402 +from deerflow.config.paths import Paths # noqa: E402 + +THREAD = "live-thread" +USER = "live-user" + + +def _labels() -> dict[str, str]: + raw = os.environ.get("BOXD_TEST_LABELS", "") + return dict(item.split("=", 1) for item in raw.split(",") if "=" in item) + + +@pytest.fixture(scope="module") +def live(tmp_path_factory: pytest.TempPathFactory): + paths = Paths(base_dir=tmp_path_factory.mktemp("deer-flow-home")) + sandbox_options = { + "use": "deerflow.community.boxd_sandbox:BoxdSandboxProvider", + "org": os.environ.get("BOXD_ORG"), + "region": os.environ.get("BOXD_TEST_REGION", "fra"), + "vcpu": 1, + "name_prefix": os.environ.get("BOXD_TEST_NAME_PREFIX", "deer-flow-test-"), + "labels": _labels(), + "idle_timeout": 0, + "environment": {"DEERFLOW_LIVE": "static"}, + } + config = types.SimpleNamespace(sandbox=types.SimpleNamespace(**sandbox_options), skills=types.SimpleNamespace(container_path="/mnt/skills")) + patch = pytest.MonkeyPatch() + patch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox_provider.get_app_config", lambda: config) + patch.setattr("deerflow.config.paths.get_paths", lambda: paths) + + def no_shared_skills(*args, **kwargs): + raise RuntimeError("shared skills are not part of this test") + + patch.setattr("deerflow.skills.storage.get_or_new_user_skill_storage", no_shared_skills) + uploads = paths.sandbox_uploads_dir(THREAD, user_id=USER) + uploads.mkdir(parents=True) + (uploads / "input.csv").write_text("a,b\n1,2\n") + + provider = BoxdSandboxProvider() + started = time.monotonic() + sandbox_id = provider.acquire(THREAD, user_id=USER) + sandbox = provider.get(sandbox_id) + print(f"\nacquire (create, ready, bootstrap, restore): {time.monotonic() - started:.1f}s, machine {sandbox.machine_id}") + machine_ids = {sandbox.machine_id} + yield types.SimpleNamespace(provider=provider, paths=paths, sandbox_id=sandbox_id, machine_ids=machine_ids) + + client = provider._get_client() + provider.shutdown() + for machine_id in machine_ids: + with pytest.raises(Exception) as excinfo: + client.machines.get(machine_id, region=sandbox_options["region"]) + assert type(excinfo.value).__name__ == "NotFoundError" + print(f"machine {machine_id} deleted by shutdown") + patch.undo() + + +def _sandbox(live): + return live.provider.get(live.sandbox_id) + + +def test_machine_is_isolated(live) -> None: + client = live.provider._get_client() + machine = client.machines.get(_sandbox(live).machine_id) + assert machine.networking.isolated is True + assert machine.labels["deer-flow-scope"] == live.sandbox_id + + +def test_restored_upload_is_visible(live) -> None: + assert _sandbox(live).read_file("/mnt/user-data/uploads/input.csv") == "a,b\n1,2\n" + + +def test_exec_env_streams_and_exit_code(live) -> None: + sandbox = _sandbox(live) + started = time.monotonic() + output = sandbox.execute_command('echo "$DEERFLOW_LIVE $REQUEST_TOKEN"; echo warn >&2; exit 3', env={"REQUEST_TOKEN": "per-call"}) + print(f"exec: {time.monotonic() - started:.2f}s") + assert output == "static per-call\n\nwarn\n\nExit Code: 3" + assert sandbox.execute_command("cat") == "(no output)" # stdin is closed + + +def test_timeout_keeps_partial_output(live) -> None: + output = _sandbox(live).execute_command("echo before; sleep 30; echo after", timeout=2) + assert output.startswith("before") and "after" not in output + assert output.endswith("Exit Code: 124") + + +def test_background_process_survives_between_calls(live) -> None: + sandbox = _sandbox(live) + sandbox.execute_command("nohup sleep 300 >/dev/null 2>&1 & echo $! > /mnt/user-data/workspace/bg.pid") + output = sandbox.execute_command("kill -0 $(cat /mnt/user-data/workspace/bg.pid) && echo alive") + assert output.strip() == "alive" + + +def test_five_mib_round_trip(live) -> None: + sandbox = _sandbox(live) + data = os.urandom(5 * 1024 * 1024) + started = time.monotonic() + sandbox.update_file("/mnt/user-data/workspace/blob.bin", data) + got = sandbox.download_file("/mnt/user-data/workspace/blob.bin") + print(f"5 MiB up + down: {time.monotonic() - started:.2f}s") + assert hashlib.sha256(got).hexdigest() == hashlib.sha256(data).hexdigest() + with pytest.raises(OSError): + sandbox.download_file("/mnt/user-data/workspace/blob.bin", max_bytes=1024) + + +def test_write_append_and_read(live) -> None: + sandbox = _sandbox(live) + sandbox.write_file("/mnt/user-data/workspace/new/appended.txt", "first\n", append=True) + sandbox.write_file("/mnt/user-data/workspace/new/appended.txt", "second\n", append=True) + assert sandbox.read_file("/mnt/user-data/workspace/new/appended.txt") == "first\nsecond\n" + assert sandbox.read_file("/mnt/user-data/workspace/new/appended.txt", start_line=2, end_line=2) == "second" + assert sandbox.read_file("/mnt/user-data/workspace/missing.txt").startswith("Error:") + + +def test_search_tools(live) -> None: + sandbox = _sandbox(live) + sandbox.write_file("/mnt/user-data/workspace/src/report:2026.txt", "alpha\nneedle here\n") + sandbox.write_file("/mnt/user-data/workspace/src/main.py", "needle = 1\n") + matches, _ = sandbox.grep("/mnt/user-data/workspace/src", "needle") + assert sorted((Path(m.path).name, m.line_number) for m in matches) == [("main.py", 1), ("report:2026.txt", 2)] + found, _ = sandbox.glob("/mnt/user-data/workspace", "**/*.py") + assert found == ["/mnt/user-data/workspace/src/main.py"] + assert "/mnt/user-data/workspace/src/main.py" in sandbox.list_dir("/mnt/user-data/workspace") + + +def test_release_mirrors_outputs_and_reclaims_same_machine(live) -> None: + sandbox = _sandbox(live) + machine_id = sandbox.machine_id + sandbox.execute_command("mkdir -p /mnt/user-data/outputs/charts && echo '# Report' > /mnt/user-data/outputs/report.md && printf png > /mnt/user-data/outputs/charts/a.png") + live.provider.release(live.sandbox_id) + outputs = live.paths.sandbox_outputs_dir(THREAD, user_id=USER) + assert (outputs / "report.md").read_text() == "# Report\n" + assert (outputs / "charts" / "a.png").read_bytes() == b"png" + started = time.monotonic() + assert live.provider.acquire(THREAD, user_id=USER) == live.sandbox_id + print(f"warm reclaim: {time.monotonic() - started:.2f}s") + assert _sandbox(live).machine_id == machine_id + + +def test_stopped_machine_is_started_on_next_call(live) -> None: + sandbox = _sandbox(live) + client = live.provider._get_client() + client.machines.stop(sandbox.machine_id) + started = time.monotonic() + # A command that meets a stopped machine starts it but is not repeated. + assert "run it again" in sandbox.execute_command("echo once >> /mnt/user-data/workspace/ran.txt") + print(f"start after stop: {time.monotonic() - started:.1f}s") + assert sandbox.execute_command("cat /mnt/user-data/workspace/new/appended.txt") == "first\nsecond\n" + assert sandbox.read_file("/mnt/user-data/workspace/ran.txt").startswith("Error:") + client.machines.stop(sandbox.machine_id) + # A file read is repeated once the machine runs again. + assert sandbox.read_file("/mnt/user-data/workspace/new/appended.txt") == "first\nsecond\n" + + +def test_wrong_api_key_fails_clearly(live) -> None: + from boxd import Boxd + + with pytest.raises(Exception) as excinfo: + Boxd(api_key="not-a-real-key", region=os.environ.get("BOXD_TEST_REGION", "fra")).machines.get(_sandbox(live).machine_id) + assert type(excinfo.value).__name__ == "AuthenticationError" diff --git a/backend/tests/test_boxd_sandbox_provider.py b/backend/tests/test_boxd_sandbox_provider.py new file mode 100644 index 0000000000..25957fc3b1 --- /dev/null +++ b/backend/tests/test_boxd_sandbox_provider.py @@ -0,0 +1,1128 @@ +"""Unit tests for the boxd community sandbox provider. + +They run without the ``boxd`` SDK installed: a fake client replaces the SDK's +``Boxd`` object and raises errors with the SDK's class names, which is how the +adapter classifies them. Search and command tests run the generated shell +scripts with the local ``bash`` (POSIX only). The live test is +``test_boxd_sandbox_live.py``. +""" + +from __future__ import annotations + +import asyncio +import errno +import io +import os +import re +import subprocess +import sys +import tarfile +import threading +import types +from pathlib import Path +from typing import Any + +import pytest + +from deerflow.community.boxd_sandbox.boxd_sandbox import BoxdSandbox +from deerflow.community.boxd_sandbox.boxd_sandbox_provider import BoxdSandboxProvider, _import_client, _ShutdownIncompleteError +from deerflow.config.paths import Paths + +posix_only = pytest.mark.skipif(os.name != "posix", reason="runs generated scripts with the local bash") + +# ── Fake boxd SDK ──────────────────────────────────────────────────────── + + +class BoxdError(Exception): + pass + + +class NotFoundError(BoxdError): + pass + + +class ConflictError(BoxdError): + pass + + +class MachineFilesystemError(BoxdError, OSError): + pass + + +class MachineFileNotFoundError(MachineFilesystemError, NotFoundError, FileNotFoundError): + pass + + +class _Result: + def __init__(self, exit_code: int = 0, stdout: str | bytes = "", stderr: str | bytes = "") -> None: + self.exit_code = exit_code + self.stdout = stdout + self.stderr = stderr + + +class _Machine: + def __init__(self, machine_id: str, name: str) -> None: + self.id = machine_id + self.name = name + self.status = "running" + + +class _FakeFs: + _CHUNK = 64 + + def __init__(self, client: _FakeClient) -> None: + self._client = client + + def _files(self, machine_id: str) -> dict[str, bytes]: + return self._client.files.setdefault(machine_id, {}) + + def read_stream(self, machine_id: str, path: str, *, region: str | None = None): + self._client.check(machine_id) + files = self._files(machine_id) + if path not in files: + raise MachineFileNotFoundError(f"{path}: No such file or directory") + data = files[path] + for i in range(0, len(data), self._CHUNK): + yield data[i : i + self._CHUNK] + + def write(self, machine_id: str, path: str, data: bytes | str, *, region: str | None = None) -> None: + self._client.check(machine_id) + if isinstance(data, str): + data = data.encode() + elif not isinstance(data, (bytes, bytearray, memoryview)): + data = b"".join(data) # an iterable of chunks, like the SDK accepts + self._files(machine_id)[path] = bytes(data) + + +class _FakeMachines: + def __init__(self, client: _FakeClient) -> None: + self._client = client + self.fs = _FakeFs(client) + + def create(self, name: str, **kwargs: Any) -> _Machine: + self._client.create_gate.wait() + if self._client.create_error is not None and not self._client.create_error_after_create: + raise self._client.create_error + machine = _Machine(f"m-{len(self._client.created) + 1}", name) + machine.labels = dict(kwargs.get("labels") or {}) + self._client.created.append((name, kwargs)) + self._client.machines[machine.id] = machine + if self._client.create_error is not None: + raise self._client.create_error + return machine + + def list(self, *, org=None, labels=None, region=None) -> list[_Machine]: + return [m for m in self._client.machines.values() if all(m.labels.get(k) == v for k, v in (labels or {}).items())] + + def wait_until_ready(self, machine_id: str, *, timeout: float = 90.0, region: str | None = None) -> _Machine: + if self._client.ready_error is not None: + raise self._client.ready_error + return self._client.machines[machine_id] + + def get(self, machine_id: str, *, region: str | None = None) -> _Machine: + if machine_id not in self._client.machines: + raise NotFoundError(f"machine {machine_id} not found") + return self._client.machines[machine_id] + + def start(self, machine_id: str, *, region: str | None = None) -> None: + self._client.starts.append(machine_id) + self._client.machines[machine_id].status = "running" + + def delete(self, machine_id: str, *, region: str | None = None) -> None: + self._client.delete_calls.append(machine_id) + if self._client.delete_errors: + raise self._client.delete_errors.pop(0) + if machine_id not in self._client.machines: + raise NotFoundError(f"machine {machine_id} not found") + del self._client.machines[machine_id] + self._client.deleted.append(machine_id) + + def exec(self, machine_id: str, command: list[str], *, env=None, timeout=None, encoding="utf-8", region=None) -> _Result: + self._client.check(machine_id) + self._client.execs.append({"machine": machine_id, "argv": list(command), "env": dict(env or {}), "timeout": timeout, "encoding": encoding}) + return self._client.handle_exec(machine_id, list(command), env=dict(env or {}), encoding=encoding) + + +class _FakeClient: + """In-memory stand-in for ``boxd.Boxd`` with a scripted shell.""" + + def __init__(self) -> None: + self.machines: dict[str, _Machine] = {} + self.files: dict[str, dict[str, bytes]] = {} + self.created: list[tuple[str, dict[str, Any]]] = [] + self.delete_calls: list[str] = [] + self.delete_errors: list[Exception] = [] + self.deleted: list[str] = [] + self.starts: list[str] = [] + self.execs: list[dict[str, Any]] = [] + self.create_error: Exception | None = None + self.create_error_after_create = False + self.ready_error: Exception | None = None + self.create_gate = threading.Event() + self.create_gate.set() + self.exec_override = None + self.health_ok = True + self.bootstrap_exit = 0 + self.local_shell = False + self.machines_api = _FakeMachines(self) + + def check(self, machine_id: str) -> None: + machine = self.machines.get(machine_id) + if machine is None: + raise NotFoundError(f"machine {machine_id} not found") + if machine.status == "stopped": + raise ConflictError(f"VM {machine.name} is stopped") + + def handle_exec(self, machine_id: str, argv: list[str], *, env: dict[str, str], encoding: str) -> _Result: + if self.exec_override is not None: + result = self.exec_override(machine_id, argv, env) + if result is not None: + return result + script = argv[-1] + files = self.files.setdefault(machine_id, {}) + if argv[0] == "head": + limit, path = int(argv[2]), argv[4] + if path not in files: + if any(name.startswith(f"{path}/") for name in files): + return _Result(1, b"", f"head: error reading '{path}': Is a directory") + return _Result(1, b"", f"head: cannot open '{path}' for reading: No such file or directory") + return _Result(0, files[path][:limit]) + if "BOOTSTRAP_OK" in script: + return _Result(self.bootstrap_exit, "BOOTSTRAP_OK\n" if self.bootstrap_exit == 0 else "") + if "UPLOAD_OK" in script: + return self._extract(files, script) + if script == "echo ok": + return _Result(0, "ok\n") if self.health_ok else _Result(1, "", "boom") + if "-print0" in script and "/mnt/user-data/outputs" in script: + listing = b"".join(path.encode() + b"\0" for path in sorted(files) if path.startswith("/mnt/user-data/outputs/")) + return _Result(0, listing if encoding == "buffer" else listing.decode()) + if self.local_shell: + completed = subprocess.run(argv, env={**os.environ, **env}, capture_output=True, timeout=60) + if encoding == "buffer": + return _Result(completed.returncode, completed.stdout, completed.stderr) + return _Result(completed.returncode, completed.stdout.decode("utf-8", "replace"), completed.stderr.decode("utf-8", "replace")) + return _Result(0, "") + + @staticmethod + def _extract(files: dict[str, bytes], script: str) -> _Result: + match = re.search(r"tar -xzf (\S+) -C (\S+) ", script) + assert match is not None + tar_path, dest = match.group(1).strip("'"), match.group(2).strip("'") + removal = re.search(r"rm -rf -- (.*?);", script) + if removal: + for target in removal.group(1).split(): + target = target.strip("'") + for path in [p for p in files if p.startswith(f"{target}/")]: + del files[path] + with tarfile.open(fileobj=io.BytesIO(files.pop(tar_path)), mode="r:gz") as archive: + for member in archive.getmembers(): + files[f"{dest}/{member.name}"] = archive.extractfile(member).read() + return _Result(0, "UPLOAD_OK\n") + + +def _boxd_class(fake: _FakeClient): + def factory(*, api_key=None, region=None, timeout=None): + fake.client_kwargs = {"api_key": api_key, "region": region} + fake.client_timeout = timeout + return types.SimpleNamespace(machines=fake.machines_api) + + return factory + + +def _stub_config(attrs: dict[str, Any] | None = None, *, skills_root: str = "/mnt/skills") -> types.SimpleNamespace: + values = {"idle_timeout": 0, "boxd_api_key": "test-key", **(attrs or {})} + return types.SimpleNamespace(sandbox=types.SimpleNamespace(**values), skills=types.SimpleNamespace(container_path=skills_root)) + + +@pytest.fixture +def paths(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Paths: + result = Paths(base_dir=tmp_path / "home") + monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: result) + return result + + +@pytest.fixture +def no_skills(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def fail(*args, **kwargs): + raise RuntimeError("no skills in this test") + + monkeypatch.setattr("deerflow.skills.storage.get_or_new_user_skill_storage", fail) + + +def _install(monkeypatch: pytest.MonkeyPatch, config: dict[str, Any] | None = None, *, fake: _FakeClient | None = None, skills_root: str = "/mnt/skills") -> tuple[BoxdSandboxProvider, _FakeClient]: + fake = fake or _FakeClient() + monkeypatch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox_provider.get_app_config", lambda: _stub_config(config, skills_root=skills_root)) + monkeypatch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox_provider._import_client", lambda: _boxd_class(fake)) + return BoxdSandboxProvider(), fake + + +@pytest.fixture +def installed(monkeypatch: pytest.MonkeyPatch, paths: Paths, no_skills: None): + provider, fake = _install(monkeypatch) + yield provider, fake + provider.shutdown() + + +def _box(fake: _FakeClient, *, env: dict[str, str] | None = None, on_terminal_failure=None) -> BoxdSandbox: + machine = fake.machines_api.create("t") + return BoxdSandbox("sandbox-id", types.SimpleNamespace(machines=fake.machines_api), machine.id, default_env=env, on_terminal_failure=on_terminal_failure) + + +# ── SDK import and config ──────────────────────────────────────────────── + + +def test_missing_sdk_has_actionable_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "boxd", None) + with pytest.raises(ImportError, match=r"deerflow-harness\[boxd\]"): + _import_client() + + +def test_package_imports_without_the_sdk() -> None: + code = "import sys; sys.modules['boxd'] = None; import deerflow.community.boxd_sandbox as m; print(m.BoxdSandboxProvider.__name__)" + completed = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=False) + assert completed.returncode == 0, completed.stderr + assert completed.stdout.strip() == "BoxdSandboxProvider" + + +def test_missing_api_key_fails_with_remedy(monkeypatch: pytest.MonkeyPatch, paths: Paths) -> None: + monkeypatch.delenv("BOXD_API_KEY", raising=False) + provider, fake = _install(monkeypatch, {"boxd_api_key": None}) + with pytest.raises(ValueError, match="BOXD_API_KEY"): + provider.acquire("thread", user_id="owner") + assert fake.created == [] + + +@pytest.mark.parametrize( + ("config", "message"), + [ + ({"region": "mars"}, "sandbox.region"), + ({"vcpu": 3}, "sandbox.vcpu"), + ({"vcpu": True}, "sandbox.vcpu"), + ({"memory": "12G"}, "sandbox.memory"), + ({"vcpu": 2, "memory": "16G"}, "goes with"), + ({"environment": {"bad-key": "1"}}, "bad-key"), + ({"auto_suspend_timeout": -1}, "auto_suspend_timeout"), + ({"labels": {"team": 1}}, "sandbox.labels"), + ({"name_prefix": "Deer_"}, "name_prefix"), + ({"ready_timeout": 0}, "ready_timeout"), + ({"bash_command_timeout": 0}, "bash_command_timeout"), + ], +) +def test_invalid_config_fails_at_startup(monkeypatch: pytest.MonkeyPatch, config: dict[str, Any], message: str) -> None: + with pytest.raises(ValueError, match=message): + _install(monkeypatch, config) + + +@pytest.mark.parametrize("root", ["/", "/etc/skills", "/home/boxd", "/mnt/user-data/skills", "/mnt", "skills", "/mnt//skills"]) +def test_unsafe_skills_root_is_rejected(monkeypatch: pytest.MonkeyPatch, root: str) -> None: + with pytest.raises(ValueError, match="skills"): + _install(monkeypatch, skills_root=root) + + +def test_create_is_isolated_labelled_and_sized(monkeypatch: pytest.MonkeyPatch, paths: Paths, no_skills: None) -> None: + provider, fake = _install( + monkeypatch, + {"org": "acme", "region": "fra", "memory": "16G", "from_snapshot": "prepared", "auto_suspend_timeout": 300, "labels": {"team": "research"}}, + ) + try: + sandbox_id = provider.acquire("thread", user_id="owner") + name, kwargs = fake.created[0] + assert kwargs["isolated"] is True + assert kwargs["org"] == "acme" + assert kwargs["region"] == "fra" + assert (kwargs["vcpu"], kwargs["memory"]) == (4, "16G") + assert kwargs["from_snapshot"] == "prepared" + assert kwargs["auto_suspend_timeout"] == 300 + generation = kwargs["labels"].pop("deer-flow-generation") + assert re.fullmatch(r"[0-9a-f]{32}", generation) + assert kwargs["labels"] == {"team": "research", "deer-flow": "sandbox", "deer-flow-scope": sandbox_id} + assert re.fullmatch(rf"deer-flow-{sandbox_id}-[0-9a-f]{{6}}", name) + assert fake.client_kwargs == {"api_key": "test-key", "region": "fra"} + # Every later call addresses the machine by the id create returned. + assert {call["machine"] for call in fake.execs} == {"m-1"} + finally: + provider.shutdown() + + +def test_create_that_raises_after_the_machine_exists_deletes_it(installed) -> None: + provider, fake = installed + fake.machines_api.create("someone-else", labels={"deer-flow": "sandbox"}) # m-1 must survive: no matching generation + fake.create_error = BoxdError("deadline exceeded") + fake.create_error_after_create = True + with pytest.raises(BoxdError): + provider.acquire("thread", user_id="owner") + assert fake.deleted == ["m-2"] + assert "m-1" in fake.machines + + +@pytest.mark.parametrize("retry", ["reaper", "shutdown"]) +def test_failed_create_delete_remains_tracked_until_retry(installed, retry: str) -> None: + provider, fake = installed + fake.machines_api.create("someone-else", labels={"deer-flow": "sandbox"}) + fake.create_error = BoxdError("create response lost") + fake.create_error_after_create = True + fake.delete_errors = [BoxdError("temporary delete failure")] + with pytest.raises(BoxdError, match="create response lost"): + provider.acquire("thread", user_id="owner") + + assert fake.delete_calls == ["m-2"] + assert set(fake.machines) == {"m-1", "m-2"} + assert [box.machine_id for box in provider._pending_closes.values()] == ["m-2"] + assert provider._sandboxes == {} + assert provider._warm_pool == {} + if retry == "reaper": + provider._cleanup_idle_resources(0) + else: + provider.shutdown() + assert fake.deleted == ["m-2"] + assert fake.delete_calls == ["m-2", "m-2"] + assert set(fake.machines) == {"m-1"} + assert provider._pending_closes == {} + + +def test_provider_enforces_agent_skill_isolation(installed) -> None: + provider, _ = installed + assert provider.supports_agent_skill_isolation is True + + +def test_each_generation_gets_a_new_machine_name(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + provider._invalidate_sandbox(sandbox_id, "gone") + provider.acquire("thread", user_id="owner") + assert len({name for name, _ in fake.created}) == 2 + + +# ── Acquire, reuse and release ─────────────────────────────────────────── + + +def test_scope_reuse_and_user_thread_isolation(installed) -> None: + provider, fake = installed + first = provider.acquire("thread", user_id="alice") + assert provider.acquire("thread", user_id="alice") == first + other_user = provider.acquire("thread", user_id="bob") + other_thread = provider.acquire("other", user_id="alice") + assert len({first, other_user, other_thread}) == 3 + assert len(fake.created) == 3 + + +def test_release_parks_and_same_scope_reclaims_without_create(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + provider.release(sandbox_id) + assert provider.get(sandbox_id) is None + assert provider.acquire("thread", user_id="owner") == sandbox_id + assert len(fake.created) == 1 + + +def test_unhealthy_warm_machine_is_deleted_and_replaced(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + provider.release(sandbox_id) + fake.health_ok = False + provider.acquire("thread", user_id="owner") + assert fake.deleted == ["m-1"] + assert len(fake.created) == 2 + + +def test_readiness_failure_deletes_machine_by_id(installed) -> None: + provider, fake = installed + fake.ready_error = TimeoutError("not ready") + with pytest.raises(TimeoutError): + provider.acquire("thread", user_id="owner") + assert fake.delete_calls == ["m-1"] + assert fake.machines == {} + + +def test_bootstrap_failure_deletes_machine(installed) -> None: + provider, fake = installed + fake.bootstrap_exit = 1 + with pytest.raises(RuntimeError, match="bootstrap"): + provider.acquire("thread", user_id="owner") + assert fake.deleted == ["m-1"] + + +def test_bootstrap_creates_user_data_and_skills_root(installed) -> None: + provider, fake = installed + provider.acquire("thread", user_id="owner") + script = next(call["argv"][-1] for call in fake.execs if "BOOTSTRAP_OK" in call["argv"][-1]) + for path in ("/mnt/user-data/workspace", "/mnt/user-data/uploads", "/mnt/user-data/outputs", "/mnt/skills"): + assert path in script + assert "sudo -n" in script + + +def test_replica_cap_evicts_oldest_warm_machine(monkeypatch: pytest.MonkeyPatch, paths: Paths, no_skills: None) -> None: + provider, fake = _install(monkeypatch, {"replicas": 1}) + try: + provider.release(provider.acquire("one", user_id="owner")) + provider.acquire("two", user_id="owner") + assert fake.deleted == ["m-1"] + finally: + provider.shutdown() + + +def test_acquire_without_thread_creates_unscoped_machine(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire() + assert provider.get(sandbox_id) is not None + assert len(fake.created) == 1 + + +# ── Thread files and skills ────────────────────────────────────────────── + + +def test_fresh_machine_restores_thread_files(installed, paths: Paths) -> None: + provider, fake = installed + uploads = paths.sandbox_uploads_dir("thread", user_id="owner") + uploads.mkdir(parents=True) + (uploads / "data.csv").write_bytes(b"a,b\n") + outputs = paths.sandbox_outputs_dir("thread", user_id="owner") + (outputs / "nested").mkdir(parents=True) + (outputs / "nested" / "report.md").write_text("# report") + (outputs / "link").symlink_to(uploads / "data.csv") + + provider.acquire("thread", user_id="owner") + files = fake.files["m-1"] + assert files["/mnt/user-data/uploads/data.csv"] == b"a,b\n" + assert files["/mnt/user-data/outputs/nested/report.md"] == b"# report" + assert "/mnt/user-data/outputs/link" not in files + assert not any(path.startswith("/tmp/") for path in files) + + +def test_restore_over_limit_is_partial_and_keeps_machine(installed, paths: Paths, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None: + provider, fake = installed + monkeypatch.setattr(BoxdSandboxProvider, "_MAX_UPLOAD_FILES", 1) + uploads = paths.sandbox_uploads_dir("thread", user_id="owner") + uploads.mkdir(parents=True) + (uploads / "a.txt").write_text("a") + outputs = paths.sandbox_outputs_dir("thread", user_id="owner") + outputs.mkdir(parents=True) + (outputs / "big.txt").write_text("b") + sandbox_id = provider.acquire("thread", user_id="owner") + files = fake.files["m-1"] + # Uploads come first; the rest is skipped instead of locking the thread out. + assert files["/mnt/user-data/uploads/a.txt"] == b"a" + assert "/mnt/user-data/outputs/big.txt" not in files + assert provider.get(sandbox_id) is not None + assert fake.deleted == [] + assert "were not copied" in caplog.text + + +def test_restore_transfer_failure_discards_machine(installed, paths: Paths) -> None: + provider, fake = installed + uploads = paths.sandbox_uploads_dir("thread", user_id="owner") + uploads.mkdir(parents=True) + (uploads / "a.txt").write_text("a") + fake.exec_override = lambda machine_id, argv, env: _Result(2, "", "tar: broken") if "UPLOAD_OK" in argv[-1] else None + with pytest.raises(RuntimeError, match="upload"): + provider.acquire("thread", user_id="owner") + assert fake.deleted == ["m-1"] + + +def _projection(root: Path, files: dict[str, str]) -> Any: + categories = {} + for category in ("public", "custom", "legacy", "integrations"): + path = root / category + path.mkdir(parents=True, exist_ok=True) + categories[category] = path + for rel, text in files.items(): + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + return types.SimpleNamespace(**categories) + + +def test_fresh_machine_gets_shared_skills_projection(monkeypatch: pytest.MonkeyPatch, paths: Paths, tmp_path: Path) -> None: + projection = _projection(tmp_path / "proj", {"public/research/SKILL.md": "# research"}) + monkeypatch.setattr("deerflow.skills.storage.get_or_new_user_skill_storage", lambda user_id, app_config=None: object()) + monkeypatch.setattr("deerflow.skills.projection.ensure_skill_projections", lambda storage: projection) + provider, fake = _install(monkeypatch) + try: + provider.acquire("thread", user_id="owner") + assert fake.files["m-1"]["/mnt/skills/public/research/SKILL.md"] == b"# research" + finally: + provider.shutdown() + + +def test_policy_scoped_thread_skips_shared_skills(monkeypatch: pytest.MonkeyPatch, paths: Paths, tmp_path: Path) -> None: + projection = _projection(tmp_path / "proj", {"public/research/SKILL.md": "# research"}) + monkeypatch.setattr("deerflow.skills.storage.get_or_new_user_skill_storage", lambda user_id, app_config=None: object()) + monkeypatch.setattr("deerflow.skills.projection.ensure_skill_projections", lambda storage: projection) + paths.thread_skills_view_dir("thread", user_id="owner").mkdir(parents=True) + provider, fake = _install(monkeypatch) + try: + provider.acquire("thread", user_id="owner") + assert not any(path.startswith("/mnt/skills/") for path in fake.files.get("m-1", {})) + finally: + provider.shutdown() + + +def test_sync_agent_skills_replaces_managed_categories(installed, tmp_path: Path) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + fake.files["m-1"]["/mnt/skills/public/old/SKILL.md"] = b"old" + fake.files["m-1"]["/mnt/skills/notes.txt"] = b"unmanaged" + projection = _projection(tmp_path / "proj", {"custom/mine/SKILL.md": "# mine"}) + provider.sync_agent_skills(sandbox_id, thread_id="thread", user_id="owner", projection=projection) + files = fake.files["m-1"] + assert files["/mnt/skills/custom/mine/SKILL.md"] == b"# mine" + assert "/mnt/skills/public/old/SKILL.md" not in files + assert files["/mnt/skills/notes.txt"] == b"unmanaged" + + +def test_sync_agent_skills_on_unknown_sandbox_raises(installed, tmp_path: Path) -> None: + provider, _ = installed + with pytest.raises(RuntimeError, match="not available"): + provider.sync_agent_skills("missing", thread_id="thread", user_id="owner", projection=_projection(tmp_path / "p", {})) + + +# ── Output mirroring ───────────────────────────────────────────────────── + + +def test_release_mirrors_outputs_to_gateway(installed, paths: Paths) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + fake.files["m-1"]["/mnt/user-data/outputs/report # 中文.txt"] = b"first" + fake.files["m-1"]["/mnt/user-data/outputs/charts/a.png"] = b"\x89PNG" + provider.release(sandbox_id) + outputs = paths.sandbox_outputs_dir("thread", user_id="owner") + assert (outputs / "report # 中文.txt").read_bytes() == b"first" + assert (outputs / "charts" / "a.png").read_bytes() == b"\x89PNG" + assert not list(outputs.rglob(".boxd-sync-*")) + # The listing is read as bytes, so file names are never decoded lossily. + listing = next(call for call in fake.execs if "-print0" in call["argv"][-1]) + assert listing["encoding"] == "buffer" + + +@pytest.mark.parametrize("first_transfer_fails", [False, True]) +def test_output_mirror_passes_remaining_deadline_to_downloads(installed, paths: Paths, monkeypatch: pytest.MonkeyPatch, first_transfer_fails: bool) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + files = fake.files["m-1"] + files["/mnt/user-data/outputs/a.txt"] = b"a" + files["/mnt/user-data/outputs/b.txt"] = b"b" + outputs = paths.sandbox_outputs_dir("thread", user_id="owner") + outputs.mkdir(parents=True, exist_ok=True) + (outputs / "a.txt").write_bytes(b"previous") + now = [0.0] + monkeypatch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox_provider.time.monotonic", lambda: now[0]) + + def transfer(machine_id, argv, env): + if "-print0" in argv[-1]: + now[0] += provider._SYNC_DEADLINE_SECONDS - 1 + elif argv[0] == "head": + now[0] += 0.75 + if first_transfer_fails and argv[-1].endswith("a.txt"): + raise BoxdError("temporary download failure") + return None + + fake.exec_override = transfer + provider.release(sandbox_id) + downloads = [call for call in fake.execs if call["argv"][0] == "head"] + assert [call["timeout"] for call in downloads] == [1.0, 0.25] + assert (outputs / "a.txt").read_bytes() == (b"previous" if first_transfer_fails else b"a") + assert (outputs / "b.txt").read_bytes() == b"b" + assert sandbox_id in provider._warm_pool + + +def test_expired_output_mirror_starts_no_download(installed, monkeypatch: pytest.MonkeyPatch) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + fake.files["m-1"]["/mnt/user-data/outputs/a.txt"] = b"a" + now = [0.0] + monkeypatch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox_provider.time.monotonic", lambda: now[0]) + + def listing(machine_id, argv, env): + if "-print0" in argv[-1]: + now[0] += provider._SYNC_DEADLINE_SECONDS + return None + + fake.exec_override = listing + provider.release(sandbox_id) + assert all(call["argv"][0] != "head" for call in fake.execs) + assert sandbox_id in provider._warm_pool + + +@pytest.mark.parametrize("stop_during_listing", [False, True]) +def test_output_mirror_defers_stopped_machine_recovery(installed, paths: Paths, stop_during_listing: bool) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + fake.files["m-1"]["/mnt/user-data/outputs/a.txt"] = b"new" + outputs = paths.sandbox_outputs_dir("thread", user_id="owner") + outputs.mkdir(parents=True, exist_ok=True) + (outputs / "a.txt").write_bytes(b"previous") + + def stop(machine_id, argv, env): + if "-print0" in argv[-1]: + fake.machines[machine_id].status = "stopped" + return None + + if stop_during_listing: + fake.exec_override = stop + else: + fake.machines["m-1"].status = "stopped" + provider.release(sandbox_id) + assert fake.starts == [] + assert (outputs / "a.txt").read_bytes() == b"previous" + assert sandbox_id in provider._warm_pool + fake.exec_override = None + provider.acquire("thread", user_id="owner") + assert fake.starts == ["m-1"] + + +def test_output_mirror_rechecks_deadline_after_local_preflight(installed, monkeypatch: pytest.MonkeyPatch) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + fake.files["m-1"]["/mnt/user-data/outputs/a.txt"] = b"a" + now = [0.0] + monkeypatch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox_provider.time.monotonic", lambda: now[0]) + destination = provider._output_destination + + def preflight(root, remote_path): + now[0] += provider._SYNC_DEADLINE_SECONDS + return destination(root, remote_path) + + monkeypatch.setattr(provider, "_output_destination", preflight) + provider.release(sandbox_id) + assert all(call["argv"][0] != "head" for call in fake.execs) + assert sandbox_id in provider._warm_pool + + +def test_output_sync_skips_ambiguous_and_unsafe_names(installed, paths: Paths) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + files = fake.files["m-1"] + files["/mnt/user-data/outputs/Report.txt"] = b"upper" + files["/mnt/user-data/outputs/report.txt"] = b"lower" + files["/mnt/user-data/outputs/a:b.txt"] = b"colon" + files["/mnt/user-data/outputs/ok.txt"] = b"ok" + provider.release(sandbox_id) + outputs = paths.sandbox_outputs_dir("thread", user_id="owner") + assert sorted(p.name for p in outputs.iterdir()) == ["ok.txt"] + + +def test_output_sync_failure_still_parks_machine(installed, monkeypatch: pytest.MonkeyPatch) -> None: + provider, _ = installed + sandbox_id = provider.acquire("thread", user_id="owner") + monkeypatch.setattr(provider, "_sync_outputs_to_host", lambda *a, **k: (_ for _ in ()).throw(OSError("listing failed"))) + provider.release(sandbox_id) + assert sandbox_id in provider._warm_pool + + +def test_invalidated_during_sync_is_not_parked(installed, monkeypatch: pytest.MonkeyPatch) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + sandbox = provider.get(sandbox_id) + + def invalidate(*args, **kwargs): + provider._invalidate_sandbox(sandbox_id, "gone", expected_sandbox=sandbox) + + monkeypatch.setattr(provider, "_sync_outputs_to_host", invalidate) + provider.release(sandbox_id) + assert sandbox_id not in provider._warm_pool + assert fake.deleted == ["m-1"] + + +# ── Adapter: commands ──────────────────────────────────────────────────── + + +def test_env_goes_with_every_exec_not_in_the_command(installed) -> None: + provider, fake = installed + fake.local_shell = False + sandbox = BoxdSandbox("s", types.SimpleNamespace(machines=fake.machines_api), fake.machines_api.create("t").id, default_env={"STATIC": "1", "TOKEN": "old"}) + sandbox.execute_command("echo hi", env={"TOKEN": "secret-value"}) + call = fake.execs[-1] + assert call["env"] == {"STATIC": "1", "TOKEN": "secret-value"} + assert "secret-value" not in " ".join(call["argv"]) + assert call["argv"][-3:] == ["bash", "-lc", "echo hi"] + + +def test_execute_command_rejects_invalid_env_key(installed) -> None: + _, fake = installed + with pytest.raises(ValueError): + _box(fake).execute_command("true", env={"bad-key": "x"}) + + +def test_command_without_timeout_gets_the_configured_deadline(monkeypatch: pytest.MonkeyPatch, paths: Paths, no_skills: None) -> None: + provider, fake = _install(monkeypatch, {"bash_command_timeout": 42}) + try: + provider.get(provider.acquire("thread", user_id="owner")).execute_command("true") + call = fake.execs[-1] + assert call["argv"][:4] == ["timeout", "-k", "5", "42"] + assert call["timeout"] is not None + finally: + provider.shutdown() + + +def test_every_sdk_call_has_a_deadline(installed) -> None: + provider, fake = installed + sandbox = provider.get(provider.acquire("thread", user_id="owner")) + sandbox.write_file("/mnt/user-data/workspace/a.txt", "x") + sandbox.read_file("/mnt/user-data/workspace/a.txt") + sandbox.download_file("/mnt/user-data/workspace/a.txt") + sandbox.execute_command("true") + assert all(call["timeout"] for call in fake.execs) + assert fake.client_timeout is not None + + +def test_timeout_runs_inside_the_machine(installed) -> None: + _, fake = installed + _box(fake).execute_command("sleep 1", timeout=2.5) + call = fake.execs[-1] + assert call["argv"][:4] == ["timeout", "-k", "5", "3"] + assert call["timeout"] > 3 + + +@posix_only +def test_timed_out_command_keeps_partial_output_and_exit_124(installed) -> None: + _, fake = installed + fake.local_shell = True + output = _box(fake).execute_command("echo before; sleep 30; echo after", timeout=1) + assert output.startswith("before") + assert "after" not in output + assert output.endswith("Exit Code: 124") + + +@posix_only +def test_exit_code_and_streams_are_reported(installed) -> None: + _, fake = installed + fake.local_shell = True + box = _box(fake) + assert box.execute_command("echo out; echo err >&2; exit 3") == "out\n\nerr\n\nExit Code: 3" + assert box.execute_command("exit 2") == "Command exited with code 2\nExit Code: 2" + assert box.execute_command("true") == "(no output)" + + +def test_sdk_error_is_returned_as_text(installed) -> None: + _, fake = installed + box = _box(fake) + fake.exec_override = lambda *a: (_ for _ in ()).throw(BoxdError("transport down")) + assert box.execute_command("echo hi") == "Error: transport down" + + +def test_stopped_machine_command_is_not_repeated(installed) -> None: + _, fake = installed + box = _box(fake) + fake.machines[box.machine_id].status = "stopped" + output = box.execute_command("echo hi") + assert output.startswith("Error:") and "run it again" in output + assert fake.starts == [box.machine_id] + # Only the readiness probe ran; the command itself never ran twice. + assert [call["argv"] for call in fake.execs] == [["true"]] + + +def test_stopped_machine_file_read_is_repeated_once(installed) -> None: + _, fake = installed + box = _box(fake) + fake.files.setdefault(box.machine_id, {})["/mnt/user-data/workspace/a.txt"] = b"data" + fake.machines[box.machine_id].status = "stopped" + assert box.read_file("/mnt/user-data/workspace/a.txt") == "data" + assert fake.starts == [box.machine_id] + assert [call["argv"][0] for call in fake.execs] == ["true", "head"] + + +def test_started_machine_is_probed_before_the_call_repeats(installed, monkeypatch: pytest.MonkeyPatch) -> None: + _, fake = installed + box = _box(fake) + fake.files.setdefault(box.machine_id, {})["/mnt/user-data/workspace/a.txt"] = b"done" + fake.machines[box.machine_id].status = "stopped" + monkeypatch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox._START_PROBE_INTERVAL", 0) + probes = [] + + def flaky_probe(machine_id, argv, env): + if argv == ["true"]: + probes.append(argv) + if len(probes) < 3: + raise BoxdError("not reachable yet") + return _Result(0) + return None + + fake.exec_override = flaky_probe + assert box.read_file("/mnt/user-data/workspace/a.txt") == "done" + assert len(probes) == 3 + + +def test_other_conflicts_are_not_retried(installed) -> None: + _, fake = installed + box = _box(fake) + calls = [] + + def conflict(*args): + calls.append(args) + raise ConflictError("machine is busy") + + fake.exec_override = conflict + assert box.execute_command("echo hi").startswith("Error:") + assert len(calls) == 1 + assert fake.starts == [] + + +def test_gone_machine_invalidates_sandbox(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + sandbox = provider.get(sandbox_id) + del fake.machines["m-1"] + assert sandbox.execute_command("echo hi").startswith("Error:") + assert provider.get(sandbox_id) is None + assert not provider._pending_closes + + +def test_not_found_from_a_live_machine_does_not_invalidate(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + sandbox = provider.get(sandbox_id) + fake.exec_override = lambda *a: (_ for _ in ()).throw(NotFoundError("session not found")) + assert sandbox.execute_command("echo hi").startswith("Error:") + assert provider.get(sandbox_id) is sandbox + assert fake.deleted == [] + + +def test_missing_file_does_not_invalidate_sandbox(installed) -> None: + provider, _ = installed + sandbox_id = provider.acquire("thread", user_id="owner") + sandbox = provider.get(sandbox_id) + assert sandbox.read_file("/mnt/user-data/workspace/missing.txt").startswith("Error:") + assert provider.get(sandbox_id) is sandbox + + +# ── Adapter: files ─────────────────────────────────────────────────────── + + +def test_write_read_append_and_line_ranges(installed) -> None: + _, fake = installed + box = _box(fake) + box.write_file("/mnt/user-data/workspace/a.txt", "one\ntwo\n") + box.write_file("/mnt/user-data/workspace/a.txt", "three\n", append=True) + box.write_file("/mnt/user-data/workspace/new.txt", "fresh", append=True) + assert box.read_file("/mnt/user-data/workspace/a.txt") == "one\ntwo\nthree\n" + assert box.read_file("/mnt/user-data/workspace/a.txt", start_line=2, end_line=3) == "two\nthree" + assert box.read_file("/mnt/user-data/workspace/a.txt", start_line=-5, end_line=1) == "one" + assert box.read_file("/mnt/user-data/workspace/new.txt") == "fresh" + box.update_file("/mnt/user-data/outputs/b.bin", b"\x00\x01") + assert fake.files[box.machine_id]["/mnt/user-data/outputs/b.bin"] == b"\x00\x01" + + +@pytest.mark.parametrize("path", ["/mnt/user-data/../etc/passwd", "relative.txt", ""]) +def test_paths_are_guarded(installed, path: str) -> None: + _, fake = installed + box = _box(fake) + assert box.read_file(path).startswith("Error:") + with pytest.raises((PermissionError, ValueError)): + box.write_file(path, "x") + + +def test_download_is_bounded_and_confined(installed) -> None: + _, fake = installed + box = _box(fake) + files = fake.files.setdefault(box.machine_id, {}) + files["/mnt/user-data/outputs/big.bin"] = b"x" * 1000 + files["/etc/passwd"] = b"root" + assert box.download_file("/mnt/user-data/outputs/big.bin", max_bytes=1000) == b"x" * 1000 + with pytest.raises(OSError) as excinfo: + box.download_file("/mnt/user-data/outputs/big.bin", max_bytes=999) + assert excinfo.value.errno == errno.EFBIG + with pytest.raises(PermissionError): + box.download_file("/etc/passwd") + with pytest.raises(FileNotFoundError): + box.download_file("/mnt/user-data/outputs/missing.bin") + + +@posix_only +def test_grep_glob_and_list_dir_through_the_shell(installed, tmp_path: Path) -> None: + _, fake = installed + fake.local_shell = True + root = tmp_path / "ws" + (root / "src").mkdir(parents=True) + (root / "src" / "report:2026.txt").write_text("alpha\nneedle here\n") + (root / "src" / "main.py").write_text("needle = 1\n") + (root / "notes.md").write_text("nothing\n") + box = _box(fake) + + matches, truncated = box.grep(str(root), "needle") + assert not truncated + assert sorted((Path(m.path).name, m.line_number) for m in matches) == [("main.py", 1), ("report:2026.txt", 2)] + scoped, _ = box.grep(str(root), "needle", glob="src/*.py") + assert [Path(m.path).name for m in scoped] == ["main.py"] + + found, truncated = box.glob(str(root), "**/*.txt") + assert [Path(p).name for p in found] == ["report:2026.txt"] + assert not truncated + + listing = box.list_dir(str(root)) + assert str(root / "src" / "main.py") in listing + + +# ── Teardown ───────────────────────────────────────────────────────────── + + +def test_failed_delete_is_retried_by_shutdown(monkeypatch: pytest.MonkeyPatch, paths: Paths, no_skills: None) -> None: + provider, fake = _install(monkeypatch) + sandbox_id = provider.acquire("thread", user_id="owner") + provider.release(sandbox_id) + fake.delete_errors = [BoxdError("transport down")] + with pytest.raises(_ShutdownIncompleteError): + provider.shutdown() + assert "m-1" in fake.machines + provider.shutdown() + assert fake.deleted == ["m-1"] + provider.shutdown() # complete: a no-op + assert fake.delete_calls == ["m-1", "m-1"] + + +def test_shutdown_deletes_machines_even_when_the_reaper_is_slow(monkeypatch: pytest.MonkeyPatch, paths: Paths, no_skills: None) -> None: + provider, fake = _install(monkeypatch) + provider.acquire("thread", user_id="owner") + + def slow_reaper() -> None: + raise RuntimeError("warm-pool idle checker is still running after stop timeout") + + monkeypatch.setattr(provider, "_stop_idle_checker", slow_reaper) + with pytest.raises(_ShutdownIncompleteError, match="reaper"): + provider.shutdown() + assert fake.deleted == ["m-1"] + monkeypatch.setattr(provider, "_stop_idle_checker", lambda: None) + provider.shutdown() + + +def test_delete_of_gone_machine_counts_as_closed(installed) -> None: + _, fake = installed + box = _box(fake) + del fake.machines[box.machine_id] + box.close() + assert box.is_closed + + +def test_shutdown_deletes_active_and_warm_and_rejects_acquire(monkeypatch: pytest.MonkeyPatch, paths: Paths, no_skills: None) -> None: + provider, fake = _install(monkeypatch) + active = provider.acquire("one", user_id="owner") + provider.release(provider.acquire("two", user_id="owner")) + provider.shutdown() + assert sorted(fake.deleted) == ["m-1", "m-2"] + assert provider.get(active) is None + with pytest.raises(RuntimeError, match="shutting down"): + provider.acquire("three", user_id="owner") + + +def test_reset_parks_active_machines_for_later_cleanup(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + provider.reset() + assert provider.get(sandbox_id) is None + assert sandbox_id in provider._warm_pool + assert fake.deleted == [] + + +def test_machine_created_after_shutdown_is_deleted(installed) -> None: + provider, fake = installed + fake.create_gate.clear() + errors: list[BaseException] = [] + + def acquire() -> None: + try: + provider.acquire("thread", user_id="owner") + except BaseException as e: + errors.append(e) + + worker = threading.Thread(target=acquire) + worker.start() + while not provider._acquires: + threading.Event().wait(0.01) + provider._accepting = False + fake.create_gate.set() + worker.join(timeout=10) + assert errors and "retired" in str(errors[0]) + assert fake.deleted == ["m-1"] + + +def test_cancelled_acquire_releases_its_new_machine(installed) -> None: + provider, fake = installed + fake.create_gate.clear() + + async def scenario() -> None: + task = asyncio.create_task(provider.acquire_async("thread", user_id="owner")) + while not provider._acquires: + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + fake.create_gate.set() + for _ in range(500): + if provider._warm_pool: + return + await asyncio.sleep(0.01) + + asyncio.run(scenario()) + assert list(provider._warm_pool) == [provider._sandbox_id("thread", "owner")] + assert provider._sandboxes == {} + + +def test_cancelled_acquire_does_not_release_a_borrowed_machine(installed) -> None: + provider, fake = installed + sandbox_id = provider.acquire("thread", user_id="owner") + gate = threading.Event() + original = provider._acquire + + def slow_acquire(thread_id, user_id, ticket=None): + gate.wait(10) + return original(thread_id, user_id, ticket) + + provider._acquire = slow_acquire + + async def scenario() -> None: + task = asyncio.create_task(provider.acquire_async("thread", user_id="owner")) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + gate.set() + await asyncio.sleep(0.3) + + asyncio.run(scenario()) + assert provider.get(sandbox_id) is not None + assert provider._warm_pool == {} + + +def test_two_cancelled_acquires_release_the_machine(installed) -> None: + provider, fake = installed + fake.create_gate.clear() + + async def scenario() -> None: + first = asyncio.create_task(provider.acquire_async("thread", user_id="owner")) + second = asyncio.create_task(provider.acquire_async("thread", user_id="owner")) + while not provider._acquires: + await asyncio.sleep(0.01) + await asyncio.sleep(0.05) + first.cancel() + second.cancel() + for task in (first, second): + with pytest.raises(asyncio.CancelledError): + await task + fake.create_gate.set() + for _ in range(500): + if provider._warm_pool and not provider._acquires: + return + await asyncio.sleep(0.01) + + asyncio.run(scenario()) + assert list(provider._warm_pool) == [provider._sandbox_id("thread", "owner")] + assert provider._sandboxes == {} + assert len(fake.created) == 1 + + +def test_acquire_async_returns_sandbox_id(installed) -> None: + provider, _ = installed + sandbox_id = asyncio.run(provider.acquire_async("thread", user_id="owner")) + assert provider.get(sandbox_id) is not None + + +def test_real_sdk_errors_are_classified() -> None: + errors = pytest.importorskip("boxd.errors") + from deerflow.community.boxd_sandbox.boxd_sandbox import is_machine_gone_error, is_missing_file_error, is_stopped_machine_error + + missing_file = errors.MachineFileNotFoundError.__new__(errors.MachineFileNotFoundError) + assert is_missing_file_error(missing_file) + assert not is_machine_gone_error(missing_file) + is_dir = errors.MachineIsADirectoryError.__new__(errors.MachineIsADirectoryError) + assert not is_stopped_machine_error(is_dir) + assert is_machine_gone_error(errors.NotFoundError.__new__(errors.NotFoundError)) + assert is_stopped_machine_error(errors.ConflictError.__new__(errors.ConflictError)) diff --git a/backend/tests/test_harness_packaging.py b/backend/tests/test_harness_packaging.py index c63688f74b..37883cc208 100644 --- a/backend/tests/test_harness_packaging.py +++ b/backend/tests/test_harness_packaging.py @@ -26,3 +26,15 @@ def test_opensandbox_is_optional_harness_dependency() -> None: assert not any(dep.startswith("opensandbox") for dep in core_dependencies) assert optional_dependencies["opensandbox"] == ["opensandbox>=0.1.15,<0.2.0"] + + +def test_boxd_is_optional_harness_dependency() -> None: + """The boxd SDK should be installed only when its provider is selected.""" + pyproject_path = Path(__file__).resolve().parents[1] / "packages" / "harness" / "pyproject.toml" + pyproject = tomllib.loads(pyproject_path.read_text(encoding="utf-8")) + + core_dependencies = pyproject["project"]["dependencies"] + optional_dependencies = pyproject["project"]["optional-dependencies"] + + assert not any(dep.startswith("boxd") for dep in core_dependencies) + assert optional_dependencies["boxd"] == ["boxd>=0.3.2"] diff --git a/backend/uv.lock b/backend/uv.lock index 16d21d01b0..5d03037b23 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -529,6 +529,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/c1/84fc6811122f54b20de2e5afb312ee07a3a47a328755587d1e505475239b/blockbuster-1.5.26-py3-none-any.whl", hash = "sha256:f8e53fb2dd4b6c6ec2f04907ddbd063ca7cd1ef587d24448ef4e50e81e3a79bb", size = 13226, upload-time = "2025-12-05T10:43:48.778Z" }, ] +[[package]] +name = "boxd" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "httpx" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/37/ad184af822d5b1c8171330986aef06e521d0e3dd0c0c47b3c37f5a1a85b7/boxd-0.3.2.tar.gz", hash = "sha256:03aa9471d1ddd663a379655b19bc7a1f20b6199a5aadfb51238889552450bd75", size = 308049, upload-time = "2026-11-13T01:14:49.059Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f0/badded7811ce0c26841e0a6145c98413189a138dc2edca2eb61f9abf5787/boxd-0.3.2-py3-none-any.whl", hash = "sha256:b78c9609ea8bbad942a27447718902744ce8904bed2185c79283eea2bfd72c42", size = 199336, upload-time = "2026-11-13T01:14:47.777Z" }, +] + [[package]] name = "boxlite" version = "0.13.1" @@ -1030,6 +1046,9 @@ dependencies = [ ] [package.optional-dependencies] +boxd = [ + { name = "boxd" }, +] boxlite = [ { name = "boxlite" }, ] @@ -1075,6 +1094,7 @@ requires-dist = [ { name = "alembic", specifier = ">=1.13" }, { name = "asyncpg", marker = "extra == 'postgres'", specifier = ">=0.29" }, { name = "beautifulsoup4", specifier = ">=4.12" }, + { name = "boxd", marker = "extra == 'boxd'", specifier = ">=0.3.2" }, { name = "boxlite", marker = "extra == 'boxlite'", specifier = ">=0.9.7" }, { name = "croniter", specifier = ">=6.0.0" }, { name = "cryptography", specifier = ">=50.0.0" }, @@ -1128,7 +1148,7 @@ requires-dist = [ { name = "tiktoken", specifier = ">=0.8.0" }, { name = "tzdata", specifier = ">=2025.1" }, ] -provides-extras = ["tui", "groundroute", "ollama", "postgres", "redis", "pymupdf", "boxlite", "tenki", "opensandbox", "monocle", "browser", "memory-zh"] +provides-extras = ["tui", "groundroute", "ollama", "postgres", "redis", "pymupdf", "boxlite", "tenki", "opensandbox", "boxd", "monocle", "browser", "memory-zh"] [[package]] name = "defusedxml" diff --git a/config.example.yaml b/config.example.yaml index f1f1d1bdb9..7c48adc739 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -2034,6 +2034,25 @@ sandbox: # # environment: # create-time and per-command defaults # # PYTHONUNBUFFERED: "1" +# Option 7: boxd machine Sandbox +# Runs each sandbox as an isolated boxd machine (an Ubuntu microVM). Released +# machines stay in an in-process warm pool and are reclaimed only by the same +# user/thread. Requires the optional SDK: pip install "deerflow-harness[boxd]". +# sandbox: +# use: deerflow.community.boxd_sandbox:BoxdSandboxProvider +# # boxd_api_key: $BOXD_API_KEY # falls back to the BOXD_API_KEY env var +# # org: my-org # optional; the key's own org when omitted +# # region: fra # optional; fra or nyc +# # vcpu: 2 # optional; 1, 2, 4 or 8 (memory 4G, 8G, 16G or 32G follows) +# # from_snapshot: my-snapshot # optional; start machines from a prepared snapshot +# # auto_suspend_timeout: 300 # optional; idle seconds before a machine goes to standby +# # ready_timeout: 120 # seconds to wait for a new machine to boot +# # name_prefix: deer-flow- # machine names are - +# # replicas: 3 # active + warm cap per gateway process +# # idle_timeout: 600 # warm seconds before the machine is deleted; 0 disables +# # environment: # merged into every command's environment +# # PYTHONUNBUFFERED: "1" + # ============================================================================ # Subagents Configuration # ============================================================================ diff --git a/docs/guide/sandbox.md b/docs/guide/sandbox.md index 0990cf14cf..203486c985 100644 --- a/docs/guide/sandbox.md +++ b/docs/guide/sandbox.md @@ -84,6 +84,12 @@ Outputs with case- or Unicode-equivalent file or directory names are skipped before copying, and aliases of existing local paths are rejected, preserving previous complete copies. +The optional [boxd provider](../../backend/packages/harness/deerflow/community/boxd_sandbox/README.md) +runs each user/thread scope in an isolated boxd machine. A new machine receives +the thread's files and skills before it is handed out, and release copies +generated outputs back to thread storage with the same alias and path checks. +See its [configuration](../../backend/docs/CONFIGURATION.md#sandbox). + The [sandbox benchmark](../../backend/scripts/benchmark/sandbox/README.md#opensandbox) also supports OpenSandbox cold acquisition, warm reuse and concurrent scopes, using the configured service connection and reporting unconfirmed cleanup as a