deer-flow/backend/tests/test_boxd_sandbox_live.py
Michiel Voortman e0cccd6b37
feat(sandbox): add boxd sandbox provider (#8314)
* feat(sandbox): add boxd sandbox provider

Adds deerflow.community.boxd_sandbox:BoxdSandboxProvider. Each user/thread
scope runs in an isolated boxd machine. New machines get the thread's files
and skills before they are handed out, release mirrors outputs back to
thread storage, and released machines wait in the shared warm pool.

The boxd SDK is an optional extra (deerflow-harness[boxd]). Unit tests run
against a fake client; tests/test_boxd_sandbox_live.py runs against real
machines when DEER_FLOW_RUN_LIVE_TESTS=1 and BOXD_API_KEY are set.

* fix(boxd): retain failed cleanup and bound output mirror calls

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
2026-11-18 21:29:15 +08:00

197 lines
8.6 KiB
Python

"""Live tests for the boxd sandbox provider against real boxd machines.
Skipped unless explicitly enabled; they create and delete real machines:
DEER_FLOW_RUN_LIVE_TESTS=1 BOXD_API_KEY=... PYTHONPATH=. uv run pytest tests/test_boxd_sandbox_live.py -v -s
Optional: ``BOXD_ORG``, ``BOXD_TEST_REGION`` (fra or nyc), ``BOXD_TEST_NAME_PREFIX``
(default ``deer-flow-test-``) and ``BOXD_TEST_LABELS`` (``key=value,key=value``).
The module shares one provider; its shutdown deletes every machine the tests
created and checks that each one is gone.
"""
from __future__ import annotations
import hashlib
import os
import time
import types
from pathlib import Path
import pytest
pytestmark = pytest.mark.live
_skip_reason = None
if os.environ.get("CI"):
_skip_reason = "Live tests skipped in CI"
elif os.environ.get("DEER_FLOW_RUN_LIVE_TESTS") != "1":
_skip_reason = "Set DEER_FLOW_RUN_LIVE_TESTS=1 to run live tests with real external APIs"
elif not os.environ.get("BOXD_API_KEY"):
_skip_reason = "Set BOXD_API_KEY to run the boxd live tests"
if _skip_reason:
pytest.skip(_skip_reason, allow_module_level=True)
pytest.importorskip("boxd")
from deerflow.community.boxd_sandbox import BoxdSandboxProvider # noqa: E402
from deerflow.config.paths import Paths # noqa: E402
THREAD = "live-thread"
USER = "live-user"
def _labels() -> dict[str, str]:
raw = os.environ.get("BOXD_TEST_LABELS", "")
return dict(item.split("=", 1) for item in raw.split(",") if "=" in item)
@pytest.fixture(scope="module")
def live(tmp_path_factory: pytest.TempPathFactory):
paths = Paths(base_dir=tmp_path_factory.mktemp("deer-flow-home"))
sandbox_options = {
"use": "deerflow.community.boxd_sandbox:BoxdSandboxProvider",
"org": os.environ.get("BOXD_ORG"),
"region": os.environ.get("BOXD_TEST_REGION", "fra"),
"vcpu": 1,
"name_prefix": os.environ.get("BOXD_TEST_NAME_PREFIX", "deer-flow-test-"),
"labels": _labels(),
"idle_timeout": 0,
"environment": {"DEERFLOW_LIVE": "static"},
}
config = types.SimpleNamespace(sandbox=types.SimpleNamespace(**sandbox_options), skills=types.SimpleNamespace(container_path="/mnt/skills"))
patch = pytest.MonkeyPatch()
patch.setattr("deerflow.community.boxd_sandbox.boxd_sandbox_provider.get_app_config", lambda: config)
patch.setattr("deerflow.config.paths.get_paths", lambda: paths)
def no_shared_skills(*args, **kwargs):
raise RuntimeError("shared skills are not part of this test")
patch.setattr("deerflow.skills.storage.get_or_new_user_skill_storage", no_shared_skills)
uploads = paths.sandbox_uploads_dir(THREAD, user_id=USER)
uploads.mkdir(parents=True)
(uploads / "input.csv").write_text("a,b\n1,2\n")
provider = BoxdSandboxProvider()
started = time.monotonic()
sandbox_id = provider.acquire(THREAD, user_id=USER)
sandbox = provider.get(sandbox_id)
print(f"\nacquire (create, ready, bootstrap, restore): {time.monotonic() - started:.1f}s, machine {sandbox.machine_id}")
machine_ids = {sandbox.machine_id}
yield types.SimpleNamespace(provider=provider, paths=paths, sandbox_id=sandbox_id, machine_ids=machine_ids)
client = provider._get_client()
provider.shutdown()
for machine_id in machine_ids:
with pytest.raises(Exception) as excinfo:
client.machines.get(machine_id, region=sandbox_options["region"])
assert type(excinfo.value).__name__ == "NotFoundError"
print(f"machine {machine_id} deleted by shutdown")
patch.undo()
def _sandbox(live):
return live.provider.get(live.sandbox_id)
def test_machine_is_isolated(live) -> None:
client = live.provider._get_client()
machine = client.machines.get(_sandbox(live).machine_id)
assert machine.networking.isolated is True
assert machine.labels["deer-flow-scope"] == live.sandbox_id
def test_restored_upload_is_visible(live) -> None:
assert _sandbox(live).read_file("/mnt/user-data/uploads/input.csv") == "a,b\n1,2\n"
def test_exec_env_streams_and_exit_code(live) -> None:
sandbox = _sandbox(live)
started = time.monotonic()
output = sandbox.execute_command('echo "$DEERFLOW_LIVE $REQUEST_TOKEN"; echo warn >&2; exit 3', env={"REQUEST_TOKEN": "per-call"})
print(f"exec: {time.monotonic() - started:.2f}s")
assert output == "static per-call\n\nwarn\n\nExit Code: 3"
assert sandbox.execute_command("cat") == "(no output)" # stdin is closed
def test_timeout_keeps_partial_output(live) -> None:
output = _sandbox(live).execute_command("echo before; sleep 30; echo after", timeout=2)
assert output.startswith("before") and "after" not in output
assert output.endswith("Exit Code: 124")
def test_background_process_survives_between_calls(live) -> None:
sandbox = _sandbox(live)
sandbox.execute_command("nohup sleep 300 >/dev/null 2>&1 & echo $! > /mnt/user-data/workspace/bg.pid")
output = sandbox.execute_command("kill -0 $(cat /mnt/user-data/workspace/bg.pid) && echo alive")
assert output.strip() == "alive"
def test_five_mib_round_trip(live) -> None:
sandbox = _sandbox(live)
data = os.urandom(5 * 1024 * 1024)
started = time.monotonic()
sandbox.update_file("/mnt/user-data/workspace/blob.bin", data)
got = sandbox.download_file("/mnt/user-data/workspace/blob.bin")
print(f"5 MiB up + down: {time.monotonic() - started:.2f}s")
assert hashlib.sha256(got).hexdigest() == hashlib.sha256(data).hexdigest()
with pytest.raises(OSError):
sandbox.download_file("/mnt/user-data/workspace/blob.bin", max_bytes=1024)
def test_write_append_and_read(live) -> None:
sandbox = _sandbox(live)
sandbox.write_file("/mnt/user-data/workspace/new/appended.txt", "first\n", append=True)
sandbox.write_file("/mnt/user-data/workspace/new/appended.txt", "second\n", append=True)
assert sandbox.read_file("/mnt/user-data/workspace/new/appended.txt") == "first\nsecond\n"
assert sandbox.read_file("/mnt/user-data/workspace/new/appended.txt", start_line=2, end_line=2) == "second"
assert sandbox.read_file("/mnt/user-data/workspace/missing.txt").startswith("Error:")
def test_search_tools(live) -> None:
sandbox = _sandbox(live)
sandbox.write_file("/mnt/user-data/workspace/src/report:2026.txt", "alpha\nneedle here\n")
sandbox.write_file("/mnt/user-data/workspace/src/main.py", "needle = 1\n")
matches, _ = sandbox.grep("/mnt/user-data/workspace/src", "needle")
assert sorted((Path(m.path).name, m.line_number) for m in matches) == [("main.py", 1), ("report:2026.txt", 2)]
found, _ = sandbox.glob("/mnt/user-data/workspace", "**/*.py")
assert found == ["/mnt/user-data/workspace/src/main.py"]
assert "/mnt/user-data/workspace/src/main.py" in sandbox.list_dir("/mnt/user-data/workspace")
def test_release_mirrors_outputs_and_reclaims_same_machine(live) -> None:
sandbox = _sandbox(live)
machine_id = sandbox.machine_id
sandbox.execute_command("mkdir -p /mnt/user-data/outputs/charts && echo '# Report' > /mnt/user-data/outputs/report.md && printf png > /mnt/user-data/outputs/charts/a.png")
live.provider.release(live.sandbox_id)
outputs = live.paths.sandbox_outputs_dir(THREAD, user_id=USER)
assert (outputs / "report.md").read_text() == "# Report\n"
assert (outputs / "charts" / "a.png").read_bytes() == b"png"
started = time.monotonic()
assert live.provider.acquire(THREAD, user_id=USER) == live.sandbox_id
print(f"warm reclaim: {time.monotonic() - started:.2f}s")
assert _sandbox(live).machine_id == machine_id
def test_stopped_machine_is_started_on_next_call(live) -> None:
sandbox = _sandbox(live)
client = live.provider._get_client()
client.machines.stop(sandbox.machine_id)
started = time.monotonic()
# A command that meets a stopped machine starts it but is not repeated.
assert "run it again" in sandbox.execute_command("echo once >> /mnt/user-data/workspace/ran.txt")
print(f"start after stop: {time.monotonic() - started:.1f}s")
assert sandbox.execute_command("cat /mnt/user-data/workspace/new/appended.txt") == "first\nsecond\n"
assert sandbox.read_file("/mnt/user-data/workspace/ran.txt").startswith("Error:")
client.machines.stop(sandbox.machine_id)
# A file read is repeated once the machine runs again.
assert sandbox.read_file("/mnt/user-data/workspace/new/appended.txt") == "first\nsecond\n"
def test_wrong_api_key_fails_clearly(live) -> None:
from boxd import Boxd
with pytest.raises(Exception) as excinfo:
Boxd(api_key="not-a-real-key", region=os.environ.get("BOXD_TEST_REGION", "fra")).machines.get(_sandbox(live).machine_id)
assert type(excinfo.value).__name__ == "AuthenticationError"