From 2254ec6ed5ff1b86d056a23819eacd45c3771034 Mon Sep 17 00:00:00 2001 From: ShiYuan Xiong <2757609992@qq.com> Date: Mon, 19 Oct 2026 07:45:59 +0800 Subject: [PATCH] feat(benchmark): add OpenSandbox provider coverage (#7059) --- README.md | 5 + backend/AGENTS.md | 4 + backend/scripts/benchmark/sandbox/README.md | 57 ++++ .../benchmark/sandbox/bench_provider.py | 88 +++++- backend/tests/test_bench_opensandbox.py | 287 ++++++++++++++++++ 5 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_bench_opensandbox.py diff --git a/README.md b/README.md index 7abd4f1196..8a5bf279d3 100644 --- a/README.md +++ b/README.md @@ -868,6 +868,11 @@ Outputs with case- or Unicode-equivalent file or directory names are skipped before copying, and aliases of existing local paths are rejected, preserving previous complete copies. +The [sandbox benchmark](backend/scripts/benchmark/sandbox/README.md#opensandbox) +also supports OpenSandbox cold acquisition, warm reuse and concurrent scopes, +using the configured service connection and reporting unconfirmed cleanup as a +failed run. Live runs require a disposable OpenSandbox service and runtime home. + BoxLite shutdown rejects late VM registration and keeps its SDK loop open while in-flight acquisitions finish. If they cannot drain within five seconds, shutdown fails with resources still owned and can be retried. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index af2eddabcb..2e4cd2a83c 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -74,6 +74,10 @@ vars; `summarize` and pytest are offline. See its README for the protocol. Benchmarks in `scripts/benchmark/` must be standalone and reproducible. Import production functions; never duplicate them or introduce an alternative runtime. +OpenSandbox benchmarks reuse the production lifecycle with scoped config overrides. +Export only non-secret dimensions; final cleanup failures must fail both the runner +and JSONL summary. See the sandbox benchmark README for supported options and tests. + - Pin every external dataset by immutable revision and SHA-256. Callers provide the local dataset path; evaluation commands must not silently download data. - Never commit upstream dataset text, credentials, complete provider requests, diff --git a/backend/scripts/benchmark/sandbox/README.md b/backend/scripts/benchmark/sandbox/README.md index a1777ee07e..f6f104a337 100644 --- a/backend/scripts/benchmark/sandbox/README.md +++ b/backend/scripts/benchmark/sandbox/README.md @@ -49,6 +49,63 @@ explicit destruction. `warm_hit` comes from the actual reclaim path. Keep differ images, pool settings, replica limits, user counts and concurrency levels separate; the summary's default grouping does so. Percentiles use inclusive interpolation. +## OpenSandbox + +`--provider opensandbox` uses the production OpenSandbox provider and reads its +connection, timeouts and environment from the selected DeerFlow configuration. +Point `DEER_FLOW_CONFIG_PATH` at a disposable benchmark configuration with the +[OpenSandbox settings](../../../packages/harness/deerflow/community/opensandbox/README.md) +and set `DEER_FLOW_HOME` to a disposable local directory. Use a test OpenSandbox +service; these commands create and destroy real remote resources. Install the +optional `deerflow-harness[opensandbox]` dependency before running. + +```sh +DEER_FLOW_CONFIG_PATH=/absolute/path/opensandbox-benchmark.yaml \ +DEER_FLOW_HOME=/absolute/path/disposable-home \ +uv run --no-sync python scripts/benchmark/sandbox/bench_provider.py \ + --provider opensandbox --image \ + --scenario warm_same_thread --workload state_reuse --iterations 10 \ + --output opensandbox-warm.jsonl +``` + +All five scenarios are supported. For a cold baseline, use +`--scenario cold_unique_thread --no-warmpool`; for simultaneous scopes, use +`--scenario replica_pressure --users 4 --concurrency 4`. Use an image with the +Python and POSIX utilities required by the selected workload and output sync. +The normal warm-up creates and destroys a separate remote before measurement. + +CLI `--replicas` and `--idle-timeout` override their YAML values, including their +CLI defaults (3 and 600). `--image` overrides YAML only when supplied. Connection +settings, including HTTPS and `use_server_proxy`, are retained. The exported +configuration dimensions contain only image, replicas and idle timeout; credentials, +endpoint and environment are not copied into result rows. Record non-secret SDK, +server and deployment versions separately for reproducibility. + +`release_ms` includes production best-effort output synchronization. Successful +command/release timing does not prove that every artifact was mirrored. Warm hits +come from the actual provider reclaim path, including its health probe. The replica +limit is a soft cap, not global admission control. `--fault-inject` and nonzero +`--health-check-skip-seconds` are rejected before provisioning: OpenSandbox does not +implement the existing BoxLite/AIO fault and health-skip adapters. + +The benchmark retains production lifecycle behavior but records reported cleanup +failures instead of accepting them as success. A final cleanup failure exits nonzero +and appends a failed `shutdown` / `cleanup` row, so `summarize.py` also exits nonzero. +These rows are cleanup diagnostics, not command latency samples. Inspect the remote +IDs in the error and clean up unconfirmed resources through the test service. +The benchmark cannot establish physical deletion beyond the provider's destroy +contract. The Docker-prefix resource sampler below does not measure remote +OpenSandbox resources; collect server-side metrics separately. + +Offline regressions use the production provider with an SDK transport double: + +```sh +uv run --no-sync pytest tests/test_bench_opensandbox.py tests/test_bench_sandbox_provider.py -q +``` + +They check lifecycle and reporting contracts, not live SDK/server compatibility or +performance. Keep real-service runs opt-in and report their evidence separately. + ## Resource samples Run the sampler beside an explicit benchmark run ID: diff --git a/backend/scripts/benchmark/sandbox/bench_provider.py b/backend/scripts/benchmark/sandbox/bench_provider.py index 6b4ef9c5f4..97d34f0ea6 100755 --- a/backend/scripts/benchmark/sandbox/bench_provider.py +++ b/backend/scripts/benchmark/sandbox/bench_provider.py @@ -25,6 +25,7 @@ Providers --------- ``boxlite`` BoxLite micro-VM sandbox (requires ``pip install boxlite``). ``aio-docker`` AIO Docker sandbox (requires Docker daemon + ``deerflow-harness`` extras). +``opensandbox`` OpenSandbox service (requires ``deerflow-harness[opensandbox]`` and sandbox config). Scenarios --------- @@ -256,9 +257,64 @@ def _make_aio_provider(config: dict[str, Any]) -> tuple[Any, dict[str, Any]]: return provider, {**sandbox_attrs, "image": provider._config["image"]} +def _make_opensandbox_provider(config: dict[str, Any]) -> tuple[Any, dict[str, Any]]: + """Use the configured connection and the production lifecycle, with strict cleanup accounting.""" + from deerflow.community.opensandbox.provider import OpenSandboxProvider + from deerflow.config.app_config import get_app_config, pop_current_app_config, push_current_app_config + + class BenchmarkOpenSandboxProvider(OpenSandboxProvider): + # Production cleanup logs failures and continues. A benchmark must still + # attempt every cleanup, but cannot report a successful run afterwards. + def __init__(self): + self._cleanup_failures: set[str] = set() + super().__init__() + + def _destroy_quietly(self, sandbox, *, context: str) -> None: + try: + sandbox.destroy() + except Exception: + with self._lock: + self._cleanup_failures.add(sandbox.remote_id) + + def _check_cleanup(self) -> None: + with self._lock: + failed = sorted(self._cleanup_failures) + if failed: + # Report resource identities for manual cleanup, never arbitrary + # SDK exception text that might contain connection credentials. + raise RuntimeError(f"OpenSandbox benchmark cleanup unconfirmed for remote IDs: {', '.join(failed)}") + + def destroy(self, sandbox_id: str) -> None: + # The runner calls this after release. Reuse the provider's owner of + # identity removal instead of separately editing its three maps. + with self._acquire_serializer.hold(sandbox_id): + self._invalidate_sandbox(sandbox_id, reason="benchmark_no_warmpool") + self._check_cleanup() + + def shutdown(self) -> None: + if self._shutdown_called: + return + super().shutdown() + self._check_cleanup() + + app_config = get_app_config() + overrides = {key: config[key] for key in ("replicas", "idle_timeout", "image") if config.get(key) is not None} + sandbox_config = app_config.sandbox.model_copy(update=overrides) + # Preserve resolved connection/environment settings without mutating either + # the caller's config or a module-global getter used by another provider. + push_current_app_config(app_config.model_copy(update={"sandbox": sandbox_config})) + try: + provider = BenchmarkOpenSandboxProvider() + finally: + pop_current_app_config() + # Only non-secret benchmark dimensions leave the factory. + return provider, {key: provider._config[key] for key in ("image", "replicas", "idle_timeout")} + + PROVIDER_FACTORIES: dict[str, Callable] = { "boxlite": _make_boxlite_provider, "aio-docker": _make_aio_provider, + "opensandbox": _make_opensandbox_provider, } @@ -695,6 +751,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) + if args.provider == "opensandbox" and (args.fault_inject is not None or args.health_check_skip_seconds != 0): + raise SystemExit("opensandbox benchmarks do not support --fault-inject or nonzero --health-check-skip-seconds") if min(args.concurrency, args.users, args.replicas) < 1 or min(args.iterations, args.warmup_iterations) < 0: raise SystemExit("concurrency, users and replicas must be positive; iteration counts must be nonnegative") if args.fault_inject is not None and (args.concurrency != 1 or args.no_warmpool or args.scenario != "warm_same_thread"): @@ -786,7 +844,35 @@ def main(argv: list[str] | None = None) -> int: return int(any(not result.success for result in results)) finally: - provider.shutdown() + cleanup_started = time.perf_counter() + try: + provider.shutdown() + except Exception as exc: + if args.provider == "opensandbox": + # Preserve failed final cleanup in the existing provider-row + # format so summarize.py cannot turn a failed run green. + elapsed = (time.perf_counter() - cleanup_started) * 1000 + failure = BenchResult( + provider=args.provider, + scenario="shutdown", + workload="cleanup", + iteration=-1, + concurrency=args.concurrency, + thread_id="", + user_id="", + acquire_ms=0, + run_ms=0, + release_ms=0, + total_ms=elapsed, + cleanup_ms=elapsed, + success=False, + error=str(exc), + run_id=args.run_id, + users=args.users, + no_warmpool=args.no_warmpool, + ) + _write_results([failure], output_path, config_used) + raise def _run_idle_timeout_scenario( diff --git a/backend/tests/test_bench_opensandbox.py b/backend/tests/test_bench_opensandbox.py new file mode 100644 index 0000000000..ca8c6b57e6 --- /dev/null +++ b/backend/tests/test_bench_opensandbox.py @@ -0,0 +1,287 @@ +"""Exercise the benchmark through the real provider with an offline SDK transport.""" + +from __future__ import annotations + +import json +import threading +from datetime import timedelta +from types import SimpleNamespace + +import pytest + +from deerflow.community.opensandbox import provider as provider_module +from deerflow.config.app_config import AppConfig, get_app_config, pop_current_app_config, push_current_app_config +from deerflow.config.sandbox_config import SandboxConfig +from scripts.benchmark.sandbox import bench_provider as bench +from scripts.benchmark.sandbox import summarize + + +class Remote: + def __init__(self, number): + self.id = f"offline-remote-{number}" + self.destroy_calls = 0 + self.destroy_error = False + self.command_error = False + self.commands = SimpleNamespace(run=self.run) + self.state = "" + + def run(self, command, **kwargs): + if self.command_error: + raise RuntimeError("offline command failure") + if "__DF_OUTPUT_SYNC__" in command: + output = "\n__DF_OUTPUT_SYNC__:0:0:0" + elif 'write_text("benchmark-state-42")' in command: + self.state = "benchmark-state-42" + output = "written" + elif 'Path("/tmp/warm_state.txt").read_text()' in command: + output = self.state + else: + output = "" + return SimpleNamespace(exit_code=0, logs=SimpleNamespace(stdout=[SimpleNamespace(text=output)], stderr=[]), result=[]) + + def renew(self, timeout): + pass + + def destroy(self): + self.destroy_calls += 1 + if self.destroy_error: + raise RuntimeError("sentinel-private-sdk-error") + + +@pytest.fixture +def transport(monkeypatch, tmp_path): + remotes = [] + requests = [] + lock = threading.Lock() + + def create(image, **kwargs): + with lock: + remote = Remote(len(remotes) + 1) + remotes.append(remote) + requests.append({"image": image, **kwargs}) + return remote + + monkeypatch.setattr(provider_module, "_import_sdk", lambda: (SimpleNamespace(create=create), SimpleNamespace, SimpleNamespace)) + monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "home")) + config = AppConfig( + sandbox=SandboxConfig( + use="deerflow.community.opensandbox:OpenSandboxProvider", + image="test-image@sha256:fixture", + api_key="sentinel-api-key", + domain="sandbox.invalid:443", + protocol="https", + request_timeout=17, + ready_timeout=19, + use_server_proxy=True, + environment={"BENCH_SECRET": "sentinel-environment"}, + ) + ) + push_current_app_config(config) + try: + yield SimpleNamespace(remotes=remotes, requests=requests, config=config) + finally: + pop_current_app_config() + + +def test_factory_preserves_connection_config_without_exporting_secrets(transport): + provider, snapshot = bench._make_opensandbox_provider({"replicas": 2, "idle_timeout": 0, "image": None}) + try: + assert get_app_config() is transport.config + assert transport.config.sandbox.image == "test-image@sha256:fixture" + sid = provider.acquire("thread", user_id="owner") + request = transport.requests[0] + assert request["image"] == "test-image@sha256:fixture" + assert request["env"] == {"BENCH_SECRET": "sentinel-environment"} + assert vars(request["connection_config"]) == { + "api_key": "sentinel-api-key", + "domain": "sandbox.invalid:443", + "protocol": "https", + "request_timeout": timedelta(seconds=17), + "use_server_proxy": True, + } + assert request["ready_timeout"].total_seconds() == 19 + assert snapshot == {"image": request["image"], "replicas": 2, "idle_timeout": 0} + assert "sentinel" not in json.dumps(snapshot) + provider.release(sid) + finally: + provider.shutdown() + + +@pytest.mark.parametrize("cold", [False, True]) +def test_cli_measures_real_reclaim_and_destroys_all_created_remotes(transport, tmp_path, cold): + output = tmp_path / "results.jsonl" + args = ["--provider", "opensandbox", "--iterations", "2", "--idle-timeout", "0", "--output", str(output)] + if cold: + args.append("--no-warmpool") + assert bench.main(args) == 0 + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert [row["warm_hit"] for row in rows] == [False, not cold] + assert all(row["success"] and row["provider"] == "opensandbox" for row in rows) + assert all(row["health_check_skip_seconds"] is None for row in rows) + # One separate warm-up remote, then two cold remotes or one reused remote. + assert len(transport.remotes) == (3 if cold else 2) + assert all(remote.destroy_calls == 1 for remote in transport.remotes) + assert "sentinel" not in output.read_text(encoding="utf-8") + + +def test_state_reuse_preserves_remote_files(transport, tmp_path): + output = tmp_path / "state.jsonl" + assert bench.main(["--provider", "opensandbox", "--workload", "state_reuse", "--iterations", "2", "--idle-timeout", "0", "--output", str(output)]) == 0 + assert transport.remotes[-1].state == "benchmark-state-42" + + +def test_replica_pressure_keeps_concurrent_scopes_separate(transport, tmp_path): + output = tmp_path / "pressure.jsonl" + assert ( + bench.main( + [ + "--provider", + "opensandbox", + "--scenario", + "replica_pressure", + "--iterations", + "4", + "--concurrency", + "2", + "--users", + "2", + "--replicas", + "1", + "--idle-timeout", + "0", + "--output", + str(output), + ] + ) + == 0 + ) + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert all(row["active_at_acquire"] == 2 and row["success"] for row in rows) + assert len({row["sandbox_id"] for row in rows}) == 2 + assert all(remote.destroy_calls == 1 for remote in transport.remotes) + + +def test_failed_command_still_releases_and_destroys_remote(transport): + provider, _ = bench._make_opensandbox_provider({"idle_timeout": 0}) + sid = provider.acquire("thread", user_id="owner") + transport.remotes[0].command_error = True + try: + row = bench._run_one_turn(provider, "opensandbox", "warm_same_thread", "noop", "true", 0, 1, "owner", "thread", True) + assert not row.success + assert row.sandbox_id == sid + assert transport.remotes[0].destroy_calls == 1 + assert not provider._warm_pool and not provider._sandboxes + finally: + provider.shutdown() + + +def test_failed_destroy_invalidates_measurement_and_fails_shutdown(transport): + provider, _ = bench._make_opensandbox_provider({"idle_timeout": 0}) + provider.acquire("thread", user_id="owner") + transport.remotes[0].destroy_error = True + try: + row = bench._run_one_turn(provider, "opensandbox", "warm_same_thread", "noop", "true", 0, 1, "owner", "thread", True) + assert not row.success + assert "offline-remote-1" in row.error + assert "sentinel-private-sdk-error" not in row.error + finally: + with pytest.raises(RuntimeError, match="cleanup"): + provider.shutdown() + # atexit must not repeat the error or pretend that a closed SDK client retried it. + provider.shutdown() + assert transport.remotes[0].destroy_calls == 1 + + +def test_shutdown_attempts_every_remote_before_reporting_failure(transport): + provider, _ = bench._make_opensandbox_provider({"idle_timeout": 0}) + provider.acquire("first", user_id="owner") + provider.acquire("second", user_id="owner") + transport.remotes[0].destroy_error = True + with pytest.raises(RuntimeError, match="cleanup"): + provider.shutdown() + assert [remote.destroy_calls for remote in transport.remotes] == [1, 1] + + +def test_shutdown_failure_is_also_visible_in_jsonl(transport, monkeypatch, tmp_path): + original_factory = bench.PROVIDER_FACTORIES["opensandbox"] + + def factory(config): + provider, snapshot = original_factory(config) + original_create = provider._create_sandbox + + def create(*args, **kwargs): + sandbox = original_create(*args, **kwargs) + transport.remotes[-1].destroy_error = True + return sandbox + + provider._create_sandbox = create + return provider, snapshot + + monkeypatch.setitem(bench.PROVIDER_FACTORIES, "opensandbox", factory) + output = tmp_path / "cleanup.jsonl" + with pytest.raises(RuntimeError, match="cleanup"): + bench.main(["--provider", "opensandbox", "--iterations", "1", "--warmup-iterations", "0", "--idle-timeout", "0", "--output", str(output)]) + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert rows[0]["success"] + assert rows[-1]["scenario"] == "shutdown" + assert rows[-1]["success"] is False + assert "sentinel-private-sdk-error" not in output.read_text(encoding="utf-8") + assert summarize.main([str(output), "--json"]) == 1 + + +def test_image_override_and_failed_construction_restore_config(transport): + provider, snapshot = bench._make_opensandbox_provider({"image": "override-image", "idle_timeout": 0}) + try: + assert snapshot["image"] == "override-image" + assert get_app_config() is transport.config + assert transport.config.sandbox.image == "test-image@sha256:fixture" + finally: + provider.shutdown() + bad = transport.config.model_copy(update={"sandbox": transport.config.sandbox.model_copy(update={"request_timeout": -1})}) + push_current_app_config(bad) + try: + with pytest.raises(ValueError, match="request_timeout"): + bench._make_opensandbox_provider({}) + assert get_app_config() is bad + finally: + pop_current_app_config() + + +def test_idle_reaping_forces_a_new_remote(transport, monkeypatch, tmp_path): + from deerflow.community import warm_pool_lifecycle + + clock = [100.0] + monkeypatch.setattr(warm_pool_lifecycle.time, "time", lambda: clock[0]) + monkeypatch.setattr(bench.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds)) + output = tmp_path / "idle.jsonl" + assert ( + bench.main( + [ + "--provider", + "opensandbox", + "--scenario", + "idle_timeout", + "--iterations", + "1", + "--warmup-iterations", + "0", + "--idle-timeout", + "1", + "--output", + str(output), + ] + ) + == 0 + ) + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert [row["warm_hit"] for row in rows] == [False, False] + assert all(row["success"] for row in rows) + assert len(transport.remotes) == 2 + assert all(remote.destroy_calls == 1 for remote in transport.remotes) + + +@pytest.mark.parametrize("extra", [["--fault-inject", "0"], ["--health-check-skip-seconds", "1"]]) +def test_unsupported_options_fail_before_creating_resources(transport, extra): + with pytest.raises(SystemExit, match="opensandbox"): + bench.main(["--provider", "opensandbox", *extra]) + assert not transport.remotes