mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 11:20:35 +00:00
* fix(sandbox): run scoped env commands concurrently * test(sandbox): gate concurrent command admission deterministically * fix(sandbox): keep releasing scopes visible to client close * docs(sandbox): clarify scoped env ownership and capacity * fix(sandbox): retain cancellation ownership while scopes drain * fix(sandbox): bound Bash admission to provisioned session capacity
533 lines
24 KiB
Python
533 lines
24 KiB
Python
"""Scoped env commands run independently and drain before client teardown."""
|
|
|
|
import asyncio
|
|
import importlib
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from types import SimpleNamespace
|
|
|
|
import httpx
|
|
import pytest
|
|
from agent_sandbox import Sandbox as SandboxClient
|
|
from support.shell import require_script_bash
|
|
|
|
from deerflow.runtime.secret_context import ACTIVE_SECRETS_CONTEXT_KEY
|
|
from deerflow.sandbox.command_scope import command_owner
|
|
from deerflow.sandbox.lease import run_sync_lifecycle_operation
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ["plain", "active-skill-env", "lark-cli"])
|
|
def test_independent_subagent_commands_enter_distinct_sessions_concurrently(monkeypatch, tmp_path, mode):
|
|
aio = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox")
|
|
tools = importlib.import_module("deerflow.sandbox.tools")
|
|
app_config = importlib.import_module("deerflow.config.app_config")
|
|
native_bash = require_script_bash()
|
|
shim_bin = tmp_path / "bin"
|
|
shim_bin.mkdir()
|
|
shim = shim_bin / "lark-cli"
|
|
shim.write_text("#!/bin/sh\nsleep 0.4\nprintf done\n", encoding="utf-8")
|
|
shim.chmod(0o755)
|
|
bookkeeping = threading.Lock()
|
|
sessions = {}
|
|
in_flight = peak = 0
|
|
entered = threading.Event()
|
|
requests = []
|
|
|
|
def complete(session):
|
|
nonlocal in_flight
|
|
if session.get("finished"):
|
|
return
|
|
try:
|
|
# Keep each service-side operation admitted until all peers have
|
|
# entered. Correctness must not depend on their sleep windows
|
|
# overlapping on a busy host.
|
|
assert entered.wait(5), "Independent commands did not all reach the service before any completion"
|
|
stdout, stderr = session["process"].communicate(timeout=5)
|
|
session.update(stdout=stdout, stderr=stderr, finished=True)
|
|
finally:
|
|
with bookkeeping:
|
|
in_flight -= 1
|
|
|
|
def transport(request):
|
|
nonlocal in_flight, peak
|
|
path = request.url.path
|
|
payload = json.loads(request.content) if request.content else {}
|
|
with bookkeeping:
|
|
requests.append((request.method, path, payload))
|
|
if path == "/v1/shell/sessions/create":
|
|
return httpx.Response(200, json={"success": True, "data": {"session_id": payload["id"], "working_dir": "/mnt/user-data/workspace"}})
|
|
if path == "/v1/bash/sessions/create":
|
|
return httpx.Response(
|
|
200, json={"success": True, "data": {"session_id": payload["session_id"], "status": "ready", "working_dir": "/mnt/user-data/workspace", "created_at": "2026-11-03T00:00:00Z", "last_used_at": "2026-11-03T00:00:00Z"}}
|
|
)
|
|
if path in ("/v1/shell/exec", "/v1/bash/exec"):
|
|
session_id = payload.get("id") or payload.get("session_id")
|
|
assert session_id is not None
|
|
command = payload["command"].replace("cd /mnt/user-data/workspace;", f"cd {shlex.quote(tmp_path.as_posix())};")
|
|
environment = {**os.environ, **payload.get("env", {})}
|
|
if "PATH" in payload.get("env", {}):
|
|
environment["PATH"] = environment["PATH"].replace("/mnt/integrations/lark-cli/runtime/bin", shim_bin.as_posix())
|
|
process = subprocess.Popen([native_bash, "--noprofile", "--norc", "-c", command], cwd=tmp_path, env=environment, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8")
|
|
session = {"process": process, "command": payload["command"], "id": session_id}
|
|
with bookkeeping:
|
|
sessions[session_id] = session
|
|
in_flight += 1
|
|
peak = max(peak, in_flight)
|
|
if in_flight == 3:
|
|
entered.set()
|
|
data = {"session_id": session_id, "command": payload["command"], "status": "running"}
|
|
if path == "/v1/bash/exec":
|
|
data["command_id"] = session_id
|
|
return httpx.Response(200, json={"success": True, "data": data})
|
|
if path == "/v1/shell/wait":
|
|
complete(sessions[payload["id"]])
|
|
return httpx.Response(200, json={"success": True, "data": {"status": "completed"}})
|
|
if path == "/v1/shell/view":
|
|
session = sessions[payload["id"]]
|
|
assert session["finished"]
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"success": True,
|
|
"data": {
|
|
"session_id": session["id"],
|
|
"status": "completed",
|
|
"command": session["command"],
|
|
"output": session["stdout"],
|
|
"exit_code": session["process"].returncode,
|
|
"console": [{"ps1": "$ ", "command": session["command"], "output": session["stdout"]}],
|
|
},
|
|
},
|
|
)
|
|
if path == "/v1/bash/output":
|
|
session = sessions[payload["session_id"]]
|
|
complete(session)
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"success": True,
|
|
"data": {
|
|
"session_id": session["id"],
|
|
"stdout": session["stdout"],
|
|
"stderr": session["stderr"],
|
|
"command": {"command_id": session["id"], "command": session["command"], "status": "completed", "exit_code": session["process"].returncode},
|
|
},
|
|
},
|
|
)
|
|
assert path.startswith("/v1/shell/sessions/") or (path.startswith("/v1/bash/sessions/") and path.endswith("/close"))
|
|
return httpx.Response(200, json={"success": True})
|
|
|
|
with httpx.Client(transport=httpx.MockTransport(transport)) as http_client:
|
|
sdk = SandboxClient(base_url="http://sandbox.invalid", httpx_client=http_client)
|
|
monkeypatch.setattr(aio, "AioSandboxClient", lambda **_kwargs: sdk)
|
|
monkeypatch.setattr(aio, "sandbox_http_trust_env", lambda _url: True)
|
|
sandbox = aio.AioSandbox("parallel-public-tool", "http://sandbox.invalid", lark_cli_broker=False)
|
|
monkeypatch.setattr(tools, "ensure_sandbox_initialized", lambda _runtime: sandbox)
|
|
monkeypatch.setattr(tools, "is_local_sandbox", lambda _runtime: False)
|
|
monkeypatch.setattr(app_config, "get_app_config", lambda: SimpleNamespace(sandbox=SimpleNamespace(bash_output_max_chars=20000, bash_command_timeout=30)))
|
|
start = threading.Barrier(3)
|
|
|
|
def worker(index):
|
|
owner = f"subagent:{index}"
|
|
context = {"user_id": "test-user", "sandbox_lease_owner_id": owner, "sandbox_command_scope_id": owner}
|
|
if mode == "active-skill-env":
|
|
context[ACTIVE_SECRETS_CONTEXT_KEY] = {"REQUEST_TEST_VALUE": "ordinary-test-value"}
|
|
runtime = SimpleNamespace(context=context, state={})
|
|
start.wait(timeout=5)
|
|
command = "lark-cli audit-wait" if mode == "lark-cli" else "sleep 0.4; printf done"
|
|
return tools.bash_tool.func(runtime, command)
|
|
|
|
started = time.perf_counter()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=3) as executor:
|
|
futures = [executor.submit(worker, index) for index in range(3)]
|
|
results = [future.result(timeout=20) for future in futures]
|
|
elapsed = time.perf_counter() - started
|
|
assert results == ["done"] * 3
|
|
assert len(sessions) == 3, "Each operation received its own real SDK session identity"
|
|
print(
|
|
"AIO_ADMISSION_REGRESSION "
|
|
+ json.dumps(
|
|
{
|
|
"mode": mode,
|
|
"commands": 3,
|
|
"sleep_seconds": 0.4,
|
|
# Diagnostic only: this includes the admission gate and
|
|
# is not the separate ungated audit benchmark.
|
|
"gated_elapsed_seconds": round(elapsed, 4),
|
|
"peak_admitted_commands": peak,
|
|
"all_entered_together": entered.is_set(),
|
|
"exec_requests": sum(path in ("/v1/shell/exec", "/v1/bash/exec") for _, path, _ in requests),
|
|
}
|
|
)
|
|
)
|
|
assert entered.is_set() and peak == 3, "Independent fresh sessions were serialized before reaching the AIO service"
|
|
finally:
|
|
for session in sessions.values():
|
|
if session["process"].poll() is None:
|
|
session["process"].terminate()
|
|
session["process"].communicate(timeout=5)
|
|
sandbox.close()
|
|
|
|
|
|
class _ObservedScopeLock:
|
|
def __init__(self, lock):
|
|
self.lock = lock
|
|
self.queued = threading.Event()
|
|
self.teardown = threading.Event()
|
|
|
|
def acquire(self, *args, **kwargs):
|
|
if threading.current_thread().name == "queued-env":
|
|
self.queued.set()
|
|
if threading.current_thread().name == "scope-teardown":
|
|
self.teardown.set()
|
|
return self.lock.acquire(*args, **kwargs)
|
|
|
|
def release(self):
|
|
self.lock.release()
|
|
|
|
def __enter__(self):
|
|
self.acquire()
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
self.release()
|
|
|
|
|
|
@pytest.fixture
|
|
def paused_env_command(monkeypatch, request):
|
|
aio = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox")
|
|
entered = threading.Event()
|
|
resume = threading.Event()
|
|
client_closed = threading.Event()
|
|
requests = []
|
|
executions = []
|
|
control_sessions = set()
|
|
request_lock = threading.Lock()
|
|
|
|
def transport(request):
|
|
assert not client_closed.is_set(), "Client closed while an admitted command still owned it"
|
|
path = request.url.path
|
|
payload = json.loads(request.content) if request.content else {}
|
|
with request_lock:
|
|
requests.append((request.method, path, payload))
|
|
if path == "/v1/bash/sessions/create":
|
|
return httpx.Response(200, json={"success": True, "data": {"session_id": payload["session_id"], "status": "ready", "working_dir": "/home/user", "created_at": "2026-11-03T00:00:00Z", "last_used_at": "2026-11-03T00:00:00Z"}})
|
|
if path == "/v1/bash/exec":
|
|
with request_lock:
|
|
executions.append(payload)
|
|
return httpx.Response(200, json={"success": True, "data": {"session_id": payload["session_id"], "command_id": payload["session_id"], "command": payload["command"], "status": "running"}})
|
|
if path == "/v1/bash/output":
|
|
execution = next(item for item in executions if item["session_id"] == payload["session_id"])
|
|
if execution is executions[0]:
|
|
entered.set()
|
|
assert resume.wait(5), "Test never released the first command"
|
|
assert not client_closed.is_set()
|
|
return httpx.Response(
|
|
200, json={"success": True, "data": {"session_id": payload["session_id"], "stdout": "done", "command": {"command_id": payload["session_id"], "command": execution["command"], "status": "completed", "exit_code": 0}}}
|
|
)
|
|
if path.startswith("/v1/bash/sessions/") and path.endswith("/close"):
|
|
return httpx.Response(200, json={"success": True})
|
|
if path == "/v1/shell/sessions/create":
|
|
control_sessions.add(payload["id"])
|
|
return httpx.Response(200, json={"success": True, "data": {"session_id": payload["id"], "working_dir": "/home/user"}})
|
|
if path == "/v1/shell/exec":
|
|
assert payload["id"] in control_sessions
|
|
assert payload["command"].startswith("python3 -c ")
|
|
return httpx.Response(200, json={"success": True, "data": {"session_id": payload["id"], "command": payload["command"], "status": "running"}})
|
|
if path == "/v1/shell/view":
|
|
assert payload["id"] in control_sessions
|
|
return httpx.Response(200, json={"success": True, "data": {"session_id": payload["id"], "output": '{"remaining":0}', "status": "completed", "exit_code": 0}})
|
|
assert request.method == "DELETE" and path.removeprefix("/v1/shell/sessions/") in control_sessions
|
|
return httpx.Response(200, json={"success": True})
|
|
|
|
http_transport = httpx.MockTransport(transport)
|
|
real_close = http_transport.close
|
|
|
|
def close():
|
|
client_closed.set()
|
|
real_close()
|
|
|
|
monkeypatch.setattr(http_transport, "close", close)
|
|
http_client = httpx.Client(transport=http_transport)
|
|
sdk = SandboxClient(base_url="http://sandbox.invalid", httpx_client=http_client)
|
|
monkeypatch.setattr(aio, "AioSandboxClient", lambda **_kwargs: sdk)
|
|
monkeypatch.setattr(aio, "sandbox_http_trust_env", lambda _url: True)
|
|
options = {"max_bash_sessions": request.param} if hasattr(request, "param") else {}
|
|
sandbox = aio.AioSandbox("scoped-env-lifecycle", "http://sandbox.invalid", **options)
|
|
|
|
def run(owner="first", value="one"):
|
|
with command_owner(sandbox, owner):
|
|
return sandbox.execute_command_in_scope("ordinary-command", env={"REQUEST_VALUE": value}, scope_id=owner)
|
|
|
|
yield SimpleNamespace(sandbox=sandbox, entered=entered, resume=resume, client_closed=client_closed, http_client=http_client, requests=requests, executions=executions, controls=control_sessions, run=run)
|
|
resume.set()
|
|
sandbox.close()
|
|
http_client.close()
|
|
|
|
|
|
def _start_observed_thread(function, *, name):
|
|
results = []
|
|
errors = []
|
|
|
|
def run():
|
|
try:
|
|
results.append(function())
|
|
except BaseException as error:
|
|
errors.append(error)
|
|
|
|
thread = threading.Thread(target=run, name=name)
|
|
thread.start()
|
|
return SimpleNamespace(thread=thread, results=results, errors=errors)
|
|
|
|
|
|
def _join_observed_thread(worker):
|
|
worker.thread.join(5)
|
|
assert not worker.thread.is_alive()
|
|
assert not worker.errors
|
|
|
|
|
|
def test_env_commands_in_one_scope_remain_serialized(paused_env_command):
|
|
case = paused_env_command
|
|
first = _start_observed_thread(case.run, name="first-env")
|
|
second = None
|
|
try:
|
|
assert case.entered.wait(5)
|
|
scoped = case.sandbox._scoped_shell_sessions["first"]
|
|
observed = _ObservedScopeLock(scoped.lock)
|
|
scoped.lock = observed
|
|
second = _start_observed_thread(lambda: case.run(value="two"), name="queued-env")
|
|
assert observed.queued.wait(5)
|
|
assert len(case.executions) == 1
|
|
case.resume.set()
|
|
_join_observed_thread(first)
|
|
_join_observed_thread(second)
|
|
assert first.results == second.results == ["done"]
|
|
assert [item["env"] for item in case.executions] == [{"REQUEST_VALUE": "one"}, {"REQUEST_VALUE": "two"}]
|
|
assert len({item["session_id"] for item in case.executions}) == 2
|
|
assert not case.controls
|
|
finally:
|
|
case.resume.set()
|
|
_join_observed_thread(first)
|
|
if second is not None:
|
|
_join_observed_thread(second)
|
|
|
|
|
|
@pytest.mark.parametrize("teardown", ["close", "release"])
|
|
def test_teardown_drains_env_and_rejects_already_queued_commands(paused_env_command, teardown):
|
|
case = paused_env_command
|
|
first = _start_observed_thread(case.run, name="first-env")
|
|
queued = stopping = None
|
|
try:
|
|
assert case.entered.wait(5)
|
|
scoped = case.sandbox._scoped_shell_sessions["first"]
|
|
observed = _ObservedScopeLock(scoped.lock)
|
|
scoped.lock = observed
|
|
queued = _start_observed_thread(case.run, name="queued-env")
|
|
assert observed.queued.wait(5)
|
|
action = case.sandbox.close if teardown == "close" else lambda: case.sandbox.release_command_scope("first")
|
|
stopping = _start_observed_thread(action, name="scope-teardown")
|
|
assert observed.teardown.wait(5)
|
|
assert not case.client_closed.is_set()
|
|
assert stopping.thread.is_alive()
|
|
if teardown == "close":
|
|
assert case.run(owner="new") == "Error: sandbox client is closed"
|
|
case.resume.set()
|
|
for worker in (first, queued, stopping):
|
|
_join_observed_thread(worker)
|
|
assert first.results == ["done"]
|
|
assert queued.results == ["Error: sandbox command scope is no longer active"]
|
|
assert len(case.executions) == 1
|
|
assert case.client_closed.is_set() is (teardown == "close")
|
|
assert not case.sandbox._scoped_shell_sessions
|
|
finally:
|
|
case.resume.set()
|
|
for worker in (first, queued, stopping):
|
|
if worker is not None:
|
|
_join_observed_thread(worker)
|
|
|
|
|
|
def test_cancelled_env_waiter_never_submits_a_command(paused_env_command):
|
|
case = paused_env_command
|
|
first = _start_observed_thread(case.run, name="first-env")
|
|
queued = None
|
|
try:
|
|
assert case.entered.wait(5)
|
|
scoped = case.sandbox._scoped_shell_sessions["first"]
|
|
observed = _ObservedScopeLock(scoped.lock)
|
|
scoped.lock = observed
|
|
queued = _start_observed_thread(case.run, name="queued-env")
|
|
assert observed.queued.wait(5)
|
|
case.sandbox.request_command_cancel("first")
|
|
_join_observed_thread(queued)
|
|
assert "cancelled" in queued.results[0]
|
|
assert len(case.executions) == 1
|
|
assert not case.controls
|
|
finally:
|
|
case.resume.set()
|
|
_join_observed_thread(first)
|
|
if queued is not None:
|
|
_join_observed_thread(queued)
|
|
|
|
|
|
@pytest.mark.parametrize("close_first", [False, True])
|
|
@pytest.mark.parametrize("release_count", [1, 2])
|
|
def test_cancel_reaches_env_command_while_scope_drains(paused_env_command, close_first, release_count):
|
|
case = paused_env_command
|
|
first = _start_observed_thread(case.run, name="first-env")
|
|
workers = []
|
|
progress = {name: threading.Event() for name in ["close", "release-0", "release-1"]}
|
|
|
|
class DrainLock(_ObservedScopeLock):
|
|
def acquire(self, *args, **kwargs):
|
|
if event := progress.get(threading.current_thread().name):
|
|
event.set()
|
|
return super().acquire(*args, **kwargs)
|
|
|
|
try:
|
|
assert case.entered.wait(5)
|
|
owner = case.sandbox._command_scopes.find("first")
|
|
scoped = case.sandbox._scoped_shell_sessions["first"]
|
|
scoped.lock = DrainLock(scoped.lock)
|
|
if close_first:
|
|
workers.append(_start_observed_thread(case.sandbox.close, name="close"))
|
|
assert progress["close"].wait(5)
|
|
for index in range(release_count):
|
|
workers.append(_start_observed_thread(lambda: case.sandbox.release_command_scope("first"), name=f"release-{index}"))
|
|
assert progress[f"release-{index}"].wait(5), "Release must join the scope even when close is already draining it"
|
|
case.sandbox.request_command_cancel("first")
|
|
assert owner.cancelled.is_set(), "Cancellation was routed to a replacement owner instead of the running command"
|
|
assert case.sandbox._command_scopes.find("first") is owner
|
|
case.resume.set()
|
|
for worker in [first, *workers]:
|
|
_join_observed_thread(worker)
|
|
assert "cancelled" in first.results[0]
|
|
assert owner.cleaned.is_set()
|
|
assert len(case.controls) == 1
|
|
assert sum(path.endswith("/close") for _, path, _ in case.requests) == 1
|
|
assert case.sandbox._command_scopes.find("first") is None
|
|
assert not case.sandbox._scoped_shell_sessions
|
|
finally:
|
|
case.resume.set()
|
|
for worker in [first, *workers]:
|
|
_join_observed_thread(worker)
|
|
|
|
|
|
def test_cancelling_one_env_scope_preserves_another_scopes_command(paused_env_command):
|
|
case = paused_env_command
|
|
first = _start_observed_thread(case.run, name="first-env")
|
|
second = None
|
|
try:
|
|
assert case.entered.wait(5)
|
|
second = _start_observed_thread(lambda: case.run(owner="second", value="two"), name="second-env")
|
|
_join_observed_thread(second)
|
|
assert second.results == ["done"]
|
|
case.sandbox.request_command_cancel("first")
|
|
case.resume.set()
|
|
_join_observed_thread(first)
|
|
assert "cancelled" in first.results[0]
|
|
assert len(case.executions) == 2
|
|
assert len(case.controls) == 1
|
|
assert not case.sandbox.requires_container_recycle
|
|
finally:
|
|
case.resume.set()
|
|
_join_observed_thread(first)
|
|
if second is not None:
|
|
_join_observed_thread(second)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_repeated_async_cancellation_drains_scoped_env_cleanup(paused_env_command):
|
|
case = paused_env_command
|
|
task = asyncio.create_task(run_sync_lifecycle_operation(case.run, _on_cancel=lambda: case.sandbox.request_command_cancel("first")))
|
|
try:
|
|
assert await asyncio.to_thread(case.entered.wait, 5)
|
|
task.cancel()
|
|
await asyncio.sleep(0)
|
|
task.cancel()
|
|
assert not task.done()
|
|
assert not case.client_closed.is_set()
|
|
case.resume.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
assert len(case.controls) == 1
|
|
assert sum(path.startswith("/v1/bash/sessions/") and path.endswith("/close") for _, path, _ in case.requests) == 1
|
|
assert case.run(owner="second", value="two") == "done"
|
|
assert len(case.executions) == 2
|
|
assert not case.sandbox.requires_container_recycle
|
|
finally:
|
|
case.resume.set()
|
|
await asyncio.gather(task, return_exceptions=True)
|
|
|
|
|
|
@pytest.mark.parametrize("release_count", [1, 2])
|
|
def test_releasing_env_scope_stays_visible_to_concurrent_close(paused_env_command, monkeypatch, release_count):
|
|
case = paused_env_command
|
|
first = _start_observed_thread(case.run, name="first-env")
|
|
queued = closing = other = None
|
|
releases = []
|
|
progress = {f"release-{index}": threading.Event() for index in range(release_count)}
|
|
progress["close"] = threading.Event()
|
|
|
|
class OverlapLock(_ObservedScopeLock):
|
|
def acquire(self, *args, **kwargs):
|
|
name = threading.current_thread().name
|
|
if name in progress:
|
|
progress[name].set()
|
|
return super().acquire(*args, **kwargs)
|
|
|
|
real_client_close = case.http_client.close
|
|
|
|
def client_close():
|
|
# On the unfixed path close skips the popped releasing scope, detaches
|
|
# _client, and reaches this callback instead of waiting on its lock.
|
|
progress["close"].set()
|
|
real_client_close()
|
|
|
|
monkeypatch.setattr(case.http_client, "close", client_close)
|
|
try:
|
|
assert case.entered.wait(5)
|
|
scoped = case.sandbox._scoped_shell_sessions["first"]
|
|
observed = OverlapLock(scoped.lock)
|
|
scoped.lock = observed
|
|
# A prior no-env call can leave a persistent shell in the same scope.
|
|
scoped.session_id = "retained-shell"
|
|
case.controls.add("retained-shell")
|
|
queued = _start_observed_thread(case.run, name="queued-env")
|
|
assert observed.queued.wait(5)
|
|
for index in range(release_count):
|
|
release = _start_observed_thread(lambda: case.sandbox.release_command_scope("first"), name=f"release-{index}")
|
|
releases.append(release)
|
|
assert progress[f"release-{index}"].wait(5)
|
|
|
|
assert case.run() == "Error: sandbox command scope is no longer active"
|
|
other = _start_observed_thread(lambda: case.run(owner="other", value="two"), name="other-env")
|
|
_join_observed_thread(other)
|
|
assert other.results == ["done"], "Releasing one scope must not serialize an independent scope"
|
|
closing = _start_observed_thread(case.sandbox.close, name="close")
|
|
assert progress["close"].wait(5)
|
|
assert case.sandbox._client is not None, "close detached the client while a releasing scope still had an admitted command"
|
|
assert not case.client_closed.is_set()
|
|
assert closing.thread.is_alive()
|
|
|
|
case.resume.set()
|
|
for worker in (first, queued, *releases, closing):
|
|
_join_observed_thread(worker)
|
|
assert first.results == ["done"]
|
|
assert queued.results == ["Error: sandbox command scope is no longer active"]
|
|
assert len(case.executions) == 2
|
|
assert sum(method == "DELETE" and path.endswith("/retained-shell") for method, path, _ in case.requests) == 1
|
|
assert sum(path.endswith("/close") for _, path, _ in case.requests) == 2
|
|
assert not case.sandbox._scoped_shell_sessions
|
|
assert case.client_closed.is_set()
|
|
finally:
|
|
case.resume.set()
|
|
for worker in (first, queued, other, *releases, closing):
|
|
if worker is not None:
|
|
_join_observed_thread(worker)
|