diff --git a/backend/scripts/benchmark/sandbox/README.md b/backend/scripts/benchmark/sandbox/README.md index f6f104a337..456775d7db 100644 --- a/backend/scripts/benchmark/sandbox/README.md +++ b/backend/scripts/benchmark/sandbox/README.md @@ -75,8 +75,10 @@ Python and POSIX utilities required by the selected workload and output sync. The normal warm-up creates and destroys a separate remote before measurement. CLI `--replicas` and `--idle-timeout` override their YAML values, including their -CLI defaults (3 and 600). `--image` overrides YAML only when supplied. Connection -settings, including HTTPS and `use_server_proxy`, are retained. The exported +CLI defaults (3 and 600). All providers reject negative `--idle-timeout` values +before provider creation. Zero disables idle reaping, except that the +`idle_timeout` scenario requires a positive value. `--image` overrides YAML only +when supplied. Connection settings, including HTTPS and `use_server_proxy`, are retained. The exported configuration dimensions contain only image, replicas and idle timeout; credentials, endpoint and environment are not copied into result rows. Record non-secret SDK, server and deployment versions separately for reproducibility. diff --git a/backend/scripts/benchmark/sandbox/bench_provider.py b/backend/scripts/benchmark/sandbox/bench_provider.py index 97d34f0ea6..8e1a55b491 100755 --- a/backend/scripts/benchmark/sandbox/bench_provider.py +++ b/backend/scripts/benchmark/sandbox/bench_provider.py @@ -755,6 +755,8 @@ def main(argv: list[str] | None = None) -> int: raise SystemExit("opensandbox benchmarks do not support --fault-inject or nonzero --health-check-skip-seconds") if min(args.concurrency, args.users, args.replicas) < 1 or min(args.iterations, args.warmup_iterations) < 0: raise SystemExit("concurrency, users and replicas must be positive; iteration counts must be nonnegative") + if args.idle_timeout < 0: + raise SystemExit("idle timeout must be nonnegative") if args.fault_inject is not None and (args.concurrency != 1 or args.no_warmpool or args.scenario != "warm_same_thread"): raise SystemExit("fault injection requires warm_same_thread, concurrency=1 and an enabled warm pool") if args.fault_inject is not None and not 0 <= args.fault_inject < args.iterations: diff --git a/backend/tests/test_bench_opensandbox.py b/backend/tests/test_bench_opensandbox.py index 30d4257b26..2f882ea616 100644 --- a/backend/tests/test_bench_opensandbox.py +++ b/backend/tests/test_bench_opensandbox.py @@ -288,3 +288,15 @@ def test_unsupported_options_fail_before_creating_resources(transport, extra): with pytest.raises(SystemExit, match="opensandbox"): bench.main(["--provider", "opensandbox", *extra]) assert not transport.remotes + + +@pytest.mark.parametrize("provider", ["opensandbox", "boxlite", "aio-docker"]) +def test_negative_idle_timeout_fails_before_provider_creation(monkeypatch, provider): + assert provider in bench.PROVIDER_FACTORIES + + def unexpected_provider(*args, **kwargs): + pytest.fail("invalid idle timeout must be rejected before provisioning") + + monkeypatch.setitem(bench.PROVIDER_FACTORIES, provider, unexpected_provider) + with pytest.raises(SystemExit, match="idle timeout must be nonnegative"): + bench.main(["--provider", provider, "--scenario", "warm_same_thread", "--idle-timeout", "-5"])