mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 11:20:35 +00:00
* fix(sandbox): reconcile ports after restricted creation failures * fix(sandbox): guard creation ports through concurrent teardown * fix(sandbox): narrow restricted startup port cleanup --------- Co-authored-by: Totoro <msy626836554@gmail.com>
666 lines
30 KiB
Python
666 lines
30 KiB
Python
"""Restricted creation releases only reservations whose proxy is known absent.
|
|
|
|
Docker CLI responses and readiness HTTP are offline. Creation, rollback, discovery,
|
|
cleanup and PortAllocator remain real. Ownership assertions use recorded allocation
|
|
and release events, so another process occupying the requested base port is harmless.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import socket
|
|
import subprocess
|
|
from dataclasses import dataclass, field
|
|
from types import SimpleNamespace
|
|
from urllib.parse import urlparse
|
|
|
|
import pytest
|
|
|
|
from deerflow.community.aio_sandbox import backend as sandbox_backend
|
|
from deerflow.community.aio_sandbox import local_backend
|
|
from deerflow.utils import network
|
|
|
|
|
|
@dataclass
|
|
class DockerCLI:
|
|
start_error: BaseException | None
|
|
proxy_create_error: BaseException | None = None
|
|
fail_proxy_stop: bool = False
|
|
fail_network_remove: bool = False
|
|
inspect_error: BaseException | None = None
|
|
containers: dict = field(default_factory=dict)
|
|
networks: dict = field(default_factory=dict)
|
|
calls: list = field(default_factory=list)
|
|
proxy_names: list = field(default_factory=list)
|
|
published_ports: list = field(default_factory=list)
|
|
|
|
@staticmethod
|
|
def _options(cmd, flag):
|
|
return [cmd[index + 1] for index, value in enumerate(cmd) if value == flag]
|
|
|
|
@staticmethod
|
|
def _response(cmd, kwargs, *, stdout="", stderr="", returncode=0):
|
|
if kwargs.get("check") and returncode:
|
|
raise subprocess.CalledProcessError(returncode, cmd, output=stdout, stderr=stderr)
|
|
return SimpleNamespace(stdout=stdout, stderr=stderr, returncode=returncode)
|
|
|
|
def record_main_container(self, cmd):
|
|
name = cmd[cmd.index("--name") + 1]
|
|
entry = {
|
|
"Name": f"/{name}",
|
|
"Id": f"generation-{name}",
|
|
"State": {"Running": True},
|
|
"Config": {
|
|
"Labels": dict(item.split("=", 1) for item in self._options(cmd, "--label")),
|
|
"Env": self._options(cmd, "-e"),
|
|
"Image": cmd[-1],
|
|
},
|
|
"HostConfig": {"PortBindings": {}},
|
|
"NetworkSettings": {
|
|
"Networks": {cmd[cmd.index("--network") + 1]: {}},
|
|
"Ports": {},
|
|
},
|
|
}
|
|
self.containers[name] = entry
|
|
return entry
|
|
|
|
def run(self, cmd, **kwargs):
|
|
self.calls.append(tuple(cmd))
|
|
assert cmd[0] == "docker", cmd
|
|
if cmd[1] == "inspect":
|
|
if self.inspect_error is not None:
|
|
raise self.inspect_error
|
|
if cmd[2:4] == ["-f", "{{.State.Running}}"]:
|
|
entry = self.containers.get(cmd[-1])
|
|
return self._response(
|
|
cmd,
|
|
kwargs,
|
|
stdout=str(entry["State"]["Running"]).lower() if entry else "",
|
|
stderr="" if entry else f"Error: No such container: {cmd[-1]}",
|
|
returncode=0 if entry else 1,
|
|
)
|
|
names = cmd[2:]
|
|
found = [self.containers[name] for name in names if name in self.containers]
|
|
missing = [f"Error: No such object: {name}" for name in names if name not in self.containers]
|
|
return self._response(
|
|
cmd,
|
|
kwargs,
|
|
stdout=json.dumps(found),
|
|
stderr="\n".join(missing),
|
|
returncode=int(bool(missing)),
|
|
)
|
|
if cmd[1:3] == ["network", "inspect"]:
|
|
name = cmd[-1]
|
|
if name in self.networks:
|
|
return self._response(cmd, kwargs, stdout=json.dumps([self.networks[name]]))
|
|
return self._response(
|
|
cmd,
|
|
kwargs,
|
|
stdout="[]",
|
|
stderr=f"Error: No such network: {name}",
|
|
returncode=1,
|
|
)
|
|
if cmd[1:3] == ["network", "create"]:
|
|
self.networks[cmd[-1]] = {
|
|
"Driver": "bridge",
|
|
"Internal": "--internal" in cmd,
|
|
"Labels": dict(item.split("=", 1) for item in self._options(cmd, "--label")),
|
|
"Options": dict(item.split("=", 1) for item in self._options(cmd, "--opt")),
|
|
}
|
|
return self._response(cmd, kwargs, stdout="network-id\n")
|
|
if cmd[1] == "create":
|
|
if self.proxy_create_error is not None:
|
|
raise self.proxy_create_error
|
|
name = cmd[cmd.index("--name") + 1]
|
|
port = int(cmd[cmd.index("-p") + 1].split(":")[-2])
|
|
self.proxy_names.append(name)
|
|
self.published_ports.append(port)
|
|
self.containers[name] = {
|
|
"Name": f"/{name}",
|
|
"Id": f"generation-{name}",
|
|
"State": {"Running": False},
|
|
"Config": {
|
|
"Labels": dict(item.split("=", 1) for item in self._options(cmd, "--label")),
|
|
"Env": self._options(cmd, "-e"),
|
|
"Image": cmd[-4],
|
|
},
|
|
"HostConfig": {"PortBindings": {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}]}},
|
|
"NetworkSettings": {
|
|
"Networks": {cmd[cmd.index("--network") + 1]: {}},
|
|
"Ports": {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}]},
|
|
},
|
|
}
|
|
return self._response(cmd, kwargs, stdout=f"generation-{name}\n")
|
|
if cmd[1:3] == ["network", "connect"]:
|
|
self.containers[cmd[-1]]["NetworkSettings"]["Networks"][cmd[-2]] = {}
|
|
return self._response(cmd, kwargs)
|
|
if cmd[1] == "start":
|
|
self.containers[cmd[-1]]["State"]["Running"] = True
|
|
return self._response(cmd, kwargs, stdout=f"{cmd[-1]}\n")
|
|
if cmd[1] == "exec":
|
|
assert cmd[2] == "-i" and cmd[3] in self.containers
|
|
assert kwargs.get("input"), "Proxy installation must use its actual script bytes"
|
|
return self._response(cmd, kwargs, stdout=b"", stderr=b"")
|
|
if cmd[1] == "run":
|
|
assert "-p" not in cmd, "Only the proxy publishes the host port"
|
|
if self.start_error is not None:
|
|
raise self.start_error
|
|
entry = self.record_main_container(cmd)
|
|
return self._response(cmd, kwargs, stdout=f"{entry['Id']}\n")
|
|
if cmd[1] == "stop":
|
|
target = cmd[-1]
|
|
name = next(
|
|
(name for name, entry in self.containers.items() if entry["Id"] == target),
|
|
target,
|
|
)
|
|
if name in self.proxy_names and self.fail_proxy_stop:
|
|
return self._response(cmd, kwargs, stderr="daemon denied proxy stop", returncode=1)
|
|
if self.containers.pop(name, None) is None:
|
|
return self._response(
|
|
cmd,
|
|
kwargs,
|
|
stderr=f"Error: No such container: {name}",
|
|
returncode=1,
|
|
)
|
|
return self._response(cmd, kwargs, stdout=f"{name}\n")
|
|
if cmd[1:3] == ["rm", "-f"]:
|
|
name = cmd[-1]
|
|
if self.containers.pop(name, None) is None:
|
|
return self._response(
|
|
cmd,
|
|
kwargs,
|
|
stderr=f"Error: No such container: {name}",
|
|
returncode=1,
|
|
)
|
|
return self._response(cmd, kwargs, stdout=f"{name}\n")
|
|
if cmd[1:3] == ["network", "rm"]:
|
|
if self.fail_network_remove:
|
|
raise subprocess.TimeoutExpired(cmd, timeout=15)
|
|
name = cmd[-1]
|
|
if any(name in entry["NetworkSettings"]["Networks"] for entry in self.containers.values()):
|
|
return self._response(cmd, kwargs, stderr="network has active endpoints", returncode=1)
|
|
if self.networks.pop(name, None) is None:
|
|
return self._response(cmd, kwargs, stderr=f"Error: No such network: {name}", returncode=1)
|
|
return self._response(cmd, kwargs, stdout=f"{name}\n")
|
|
raise AssertionError(f"Unexpected Docker command: {cmd!r}")
|
|
|
|
|
|
@dataclass
|
|
class PortHistory:
|
|
allocated: list[int] = field(default_factory=list)
|
|
released: list[int] = field(default_factory=list)
|
|
|
|
|
|
@pytest.fixture(params=[False, True], ids=["available-base", "occupied-base"])
|
|
def restricted_backend(monkeypatch, request):
|
|
"""Record real allocations; also exercise a base port occupied by a socket."""
|
|
occupied = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
occupied.bind(("0.0.0.0", 0))
|
|
base_port = occupied.getsockname()[1]
|
|
if not request.param:
|
|
occupied.close()
|
|
allocator = network.PortAllocator()
|
|
history = PortHistory()
|
|
allocate, release = allocator.allocate, allocator.release
|
|
|
|
def record_allocation(*args, **kwargs):
|
|
port = allocate(*args, **kwargs)
|
|
history.allocated.append(port)
|
|
if request.param:
|
|
assert port != base_port
|
|
return port
|
|
|
|
def record_release(port):
|
|
history.released.append(port)
|
|
release(port)
|
|
|
|
monkeypatch.setattr(allocator, "allocate", record_allocation)
|
|
monkeypatch.setattr(allocator, "release", record_release)
|
|
monkeypatch.setattr(network, "_global_port_allocator", allocator)
|
|
monkeypatch.setattr(local_backend.LocalContainerBackend, "_detect_runtime", lambda self: "docker")
|
|
monkeypatch.setattr(
|
|
local_backend.LocalContainerBackend,
|
|
"_require_restricted_network_support",
|
|
lambda self: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
local_backend.LocalContainerBackend,
|
|
"_docker_server_is_desktop",
|
|
lambda self: False,
|
|
)
|
|
monkeypatch.setattr(local_backend, "_resolve_docker_bind_host", lambda: "127.0.0.1")
|
|
monkeypatch.setenv("DEER_FLOW_SANDBOX_HOST", "localhost")
|
|
monkeypatch.delenv("DEER_FLOW_SANDBOX_NETWORK", raising=False)
|
|
backend = local_backend.LocalContainerBackend(
|
|
image="offline-sandbox:unused",
|
|
base_port=base_port,
|
|
container_prefix="offline-restricted",
|
|
config_mounts=[],
|
|
environment={},
|
|
network_config={"mode": "allowlist"},
|
|
)
|
|
|
|
def scenario(start_error, **options):
|
|
docker = DockerCLI(start_error, **options)
|
|
monkeypatch.setattr(local_backend.subprocess, "run", docker.run)
|
|
return backend, allocator, history, docker
|
|
|
|
try:
|
|
yield scenario
|
|
finally:
|
|
occupied.close()
|
|
|
|
|
|
def assert_reserved(allocator, *ports):
|
|
assert set(ports) <= allocator._reserved_ports
|
|
|
|
|
|
def assert_released_once(allocator, history, port):
|
|
assert port not in allocator._reserved_ports
|
|
assert history.released.count(port) == 1
|
|
|
|
|
|
def rejected_start(message="daemon rejected sandbox start"):
|
|
return subprocess.CalledProcessError(1, ["docker", "run"], stderr=message)
|
|
|
|
|
|
def allow_readiness_http(monkeypatch):
|
|
requests = []
|
|
|
|
def healthy_response(session, url, **kwargs):
|
|
requests.append(url)
|
|
return SimpleNamespace(status_code=200)
|
|
|
|
monkeypatch.setattr(sandbox_backend.requests.Session, "get", healthy_response)
|
|
return requests
|
|
|
|
|
|
def test_runtime_error_with_failed_proxy_cleanup_preserves_port(restricted_backend):
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), fail_proxy_stop=True)
|
|
with pytest.raises(RuntimeError, match="daemon rejected sandbox start"):
|
|
backend.create("thread", "sandbox")
|
|
assert docker.published_ports == history.allocated
|
|
assert docker.proxy_names[0] in docker.containers
|
|
assert_reserved(allocator, history.allocated[0])
|
|
assert history.released == []
|
|
|
|
|
|
@pytest.mark.parametrize("error_type", [FileNotFoundError, PermissionError])
|
|
def test_unstarted_main_cli_releases_port_after_proxy_is_removed(restricted_backend, error_type):
|
|
original = error_type("sandbox executable cannot start")
|
|
backend, allocator, history, docker = restricted_backend(original)
|
|
with pytest.raises(error_type) as caught:
|
|
backend.create("thread", "sandbox")
|
|
assert caught.value is original
|
|
assert docker.published_ports == history.allocated
|
|
assert docker.proxy_names[0] not in docker.containers
|
|
assert_released_once(allocator, history, history.allocated[0])
|
|
|
|
|
|
def test_network_cleanup_timeout_releases_proxy_port_and_preserves_start_error(restricted_backend, caplog):
|
|
original = FileNotFoundError("sandbox executable cannot start")
|
|
backend, allocator, history, docker = restricted_backend(original, fail_network_remove=True)
|
|
with (
|
|
caplog.at_level(logging.WARNING, logger=local_backend.__name__),
|
|
pytest.raises(FileNotFoundError) as caught,
|
|
):
|
|
backend.create("thread", "sandbox")
|
|
assert caught.value is original
|
|
assert docker.proxy_names[0] not in docker.containers
|
|
assert any(cmd[:3] == ("docker", "network", "rm") for cmd in docker.calls)
|
|
warnings = [record for record in caplog.records if record.name == local_backend.__name__ and record.levelno >= logging.WARNING]
|
|
assert any("rollback" in record.getMessage().lower() and "sandbox" in record.getMessage() and record.exc_info is not None for record in warnings)
|
|
assert_released_once(allocator, history, history.allocated[0])
|
|
|
|
|
|
def test_preflight_failure_releases_port_before_any_proxy_attempt(restricted_backend):
|
|
original = FileNotFoundError("runtime unavailable during preflight")
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), inspect_error=original)
|
|
with pytest.raises(RuntimeError, match="Failed to inspect containers") as caught:
|
|
backend.create("thread", "sandbox")
|
|
assert caught.value.__cause__ is original
|
|
assert docker.published_ports == []
|
|
assert all(cmd[:2] != ("docker", "create") for cmd in docker.calls)
|
|
assert_released_once(allocator, history, history.allocated[0])
|
|
|
|
|
|
def test_uncertain_proxy_cleanup_does_not_retry_port_collision(restricted_backend):
|
|
backend, allocator, history, docker = restricted_backend(rejected_start("address already in use"), fail_proxy_stop=True)
|
|
with pytest.raises(RuntimeError, match="address already in use"):
|
|
backend.create("thread", "sandbox")
|
|
assert len(history.allocated) == 1
|
|
assert docker.published_ports == history.allocated
|
|
assert_reserved(allocator, history.allocated[0])
|
|
assert history.released == []
|
|
|
|
|
|
def test_confirmed_absence_releases_failed_creation_port_once(restricted_backend):
|
|
original = FileNotFoundError("sandbox executable cannot start")
|
|
backend, allocator, history, docker = restricted_backend(original, fail_proxy_stop=True)
|
|
with pytest.raises(FileNotFoundError) as caught:
|
|
backend.create("thread", "sandbox")
|
|
assert caught.value is original
|
|
port = history.allocated[0]
|
|
assert_reserved(allocator, port)
|
|
|
|
# The caller supplies the provider's teardown fences and authoritative
|
|
# absence proof; cleanup must also remove the proxy/network leftovers.
|
|
docker.fail_proxy_stop = False
|
|
assert backend.is_absent("sandbox") is True
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert docker.containers == {}
|
|
assert_released_once(allocator, history, port)
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert_released_once(allocator, history, port)
|
|
|
|
|
|
@pytest.mark.parametrize("error_type", [FileNotFoundError, PermissionError])
|
|
def test_proxy_cli_spawn_failure_returns_its_unused_port(restricted_backend, error_type):
|
|
original = error_type("proxy executable cannot start")
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), proxy_create_error=original)
|
|
with pytest.raises(error_type) as caught:
|
|
backend.create("thread", "sandbox")
|
|
assert caught.value is original
|
|
assert any(cmd[:2] == ("docker", "create") for cmd in docker.calls)
|
|
assert docker.containers == {}
|
|
assert_released_once(allocator, history, history.allocated[0])
|
|
|
|
|
|
def test_same_id_create_does_not_reclaim_an_earlier_uncertain_reservation(
|
|
restricted_backend,
|
|
):
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), fail_proxy_stop=True)
|
|
with pytest.raises(RuntimeError, match="daemon rejected sandbox start"):
|
|
backend.create("thread", "sandbox")
|
|
old_port = history.allocated[0]
|
|
|
|
# External removal is not the provider's fenced absence-cleanup path.
|
|
# A later create must not silently reclaim the earlier reservation.
|
|
docker.containers.clear()
|
|
docker.networks.clear()
|
|
docker.fail_proxy_stop = False
|
|
docker.start_error = None
|
|
replacement = backend.create("thread", "sandbox")
|
|
replacement_port = urlparse(replacement.sandbox_url).port
|
|
assert replacement_port != old_port
|
|
assert_reserved(allocator, old_port, replacement_port)
|
|
assert history.released == []
|
|
|
|
backend.destroy(replacement)
|
|
assert_released_once(allocator, history, replacement_port)
|
|
assert_reserved(allocator, old_port)
|
|
assert backend.is_absent("sandbox") is True
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert_released_once(allocator, history, old_port)
|
|
|
|
|
|
def test_first_failure_probe_cannot_release_a_successors_reservation(restricted_backend, monkeypatch):
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), fail_proxy_stop=True)
|
|
successors = []
|
|
interleaved = False
|
|
|
|
def first_probe_then_transfer(cmd, **kwargs):
|
|
nonlocal interleaved
|
|
if not interleaved and docker.proxy_names and cmd == ["docker", "inspect", docker.proxy_names[0]]:
|
|
interleaved = True
|
|
# Keep a stale absence result while a fenced cleanup retires the
|
|
# original reservation and a new create obtains its own port.
|
|
del docker.containers[docker.proxy_names[0]]
|
|
response = docker.run(cmd, **kwargs)
|
|
docker.fail_proxy_stop = False
|
|
assert backend.is_absent("sandbox") is True
|
|
backend.complete_absent_teardown("sandbox")
|
|
docker.start_error = None
|
|
successors.append(backend.create("successor-thread", "successor"))
|
|
return response
|
|
return docker.run(cmd, **kwargs)
|
|
|
|
monkeypatch.setattr(local_backend.subprocess, "run", first_probe_then_transfer)
|
|
with pytest.raises(RuntimeError, match="daemon rejected sandbox start"):
|
|
backend.create("thread", "sandbox")
|
|
assert interleaved
|
|
assert len(history.allocated) == 2
|
|
assert history.released == history.allocated[:1]
|
|
successor_port = urlparse(successors[0].sandbox_url).port
|
|
assert allocator._reserved_ports == {successor_port}
|
|
|
|
|
|
class CleanupInterrupted(BaseException):
|
|
"""Model a lifecycle interruption without stopping the pytest session."""
|
|
|
|
|
|
@pytest.mark.parametrize("interrupted_stage", ["rollback", "settlement"])
|
|
def test_cleanup_interruption_keeps_the_reservation_recoverable(restricted_backend, monkeypatch, interrupted_stage):
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), fail_proxy_stop=True)
|
|
interruption = CleanupInterrupted("cleanup interrupted")
|
|
interrupted = False
|
|
|
|
def interrupt_cleanup(cmd, **kwargs):
|
|
nonlocal interrupted
|
|
is_rollback = cmd[1] == "stop"
|
|
is_settlement = docker.proxy_names and cmd == [
|
|
"docker",
|
|
"inspect",
|
|
docker.proxy_names[0],
|
|
]
|
|
if not interrupted and ((interrupted_stage == "rollback" and is_rollback) or (interrupted_stage == "settlement" and is_settlement)):
|
|
interrupted = True
|
|
raise interruption
|
|
return docker.run(cmd, **kwargs)
|
|
|
|
monkeypatch.setattr(local_backend.subprocess, "run", interrupt_cleanup)
|
|
with pytest.raises(CleanupInterrupted) as caught:
|
|
backend.create("thread", "sandbox")
|
|
assert caught.value is interruption
|
|
port = history.allocated[0]
|
|
assert_reserved(allocator, port)
|
|
docker.fail_proxy_stop = False
|
|
assert backend.is_absent("sandbox") is True
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert_released_once(allocator, history, port)
|
|
|
|
|
|
def test_destroying_discovered_failed_creation_clears_its_reservation(restricted_backend, monkeypatch):
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), fail_proxy_stop=True)
|
|
with pytest.raises(RuntimeError, match="daemon rejected sandbox start"):
|
|
backend.create("thread", "sandbox")
|
|
port = history.allocated[0]
|
|
main_command = list(next(cmd for cmd in docker.calls if cmd[:2] == ("docker", "run")))
|
|
docker.record_main_container(main_command)
|
|
readiness_requests = allow_readiness_http(monkeypatch)
|
|
info = backend.discover("sandbox")
|
|
assert info is not None and not info.requires_replacement
|
|
assert readiness_requests == [f"http://localhost:{port}/v1/sandbox"]
|
|
docker.fail_proxy_stop = False
|
|
backend.destroy(info)
|
|
assert docker.containers == {}
|
|
assert_released_once(allocator, history, port)
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert_released_once(allocator, history, port)
|
|
|
|
|
|
@pytest.mark.parametrize("conflict_stage", ["proxy", "main"])
|
|
@pytest.mark.parametrize("inspection_fails", [False, True])
|
|
def test_name_conflict_only_retains_a_port_from_this_attempts_proxy(restricted_backend, monkeypatch, caplog, conflict_stage, inspection_fails):
|
|
conflict = 'Conflict. The container name is already in use by container "peer"'
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(conflict))
|
|
peer_ports = []
|
|
|
|
def conflicting_runtime(cmd, **kwargs):
|
|
if conflict_stage == "proxy" and cmd[1] == "create":
|
|
docker.run(cmd, **kwargs)
|
|
name = cmd[cmd.index("--name") + 1]
|
|
peer_port = allocator.allocate(history.allocated[0] + 1)
|
|
peer_ports.append(peer_port)
|
|
entry = docker.containers[name]
|
|
for bindings in (
|
|
entry["HostConfig"]["PortBindings"],
|
|
entry["NetworkSettings"]["Ports"],
|
|
):
|
|
bindings["8080/tcp"][0]["HostPort"] = str(peer_port)
|
|
if inspection_fails:
|
|
docker.inspect_error = PermissionError("daemon unavailable after conflict")
|
|
return docker._response(cmd, kwargs, stderr=conflict, returncode=1)
|
|
if conflict_stage == "main" and cmd[1] == "run":
|
|
entry = docker.record_main_container(cmd)
|
|
entry["Config"]["Labels"]["deerflow.sandbox_id"] = "peer"
|
|
if inspection_fails:
|
|
docker.inspect_error = PermissionError("daemon unavailable after conflict")
|
|
return docker.run(cmd, **kwargs)
|
|
|
|
monkeypatch.setattr(local_backend.subprocess, "run", conflicting_runtime)
|
|
with pytest.raises(RuntimeError, match="already in use"):
|
|
backend.create("thread", "sandbox")
|
|
assert docker.proxy_names[0] in docker.containers
|
|
assert all(cmd[1] not in {"stop", "rm"} for cmd in docker.calls)
|
|
if inspection_fails:
|
|
assert any("Could not probe conflicting sandbox sandbox" in record.getMessage() and record.exc_info is not None for record in caplog.records)
|
|
if conflict_stage == "proxy":
|
|
assert_released_once(allocator, history, history.allocated[0])
|
|
assert_reserved(allocator, peer_ports[0])
|
|
else:
|
|
assert_reserved(allocator, history.allocated[0])
|
|
assert history.released == []
|
|
|
|
|
|
def test_main_name_conflict_adoption_keeps_proxy_port_until_destroy(restricted_backend, monkeypatch):
|
|
conflict = 'Conflict. The container name is already in use by container "peer"'
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(conflict))
|
|
|
|
def compatible_main_wins_name(cmd, **kwargs):
|
|
if cmd[1] == "run":
|
|
docker.record_main_container(cmd)
|
|
return docker.run(cmd, **kwargs)
|
|
|
|
monkeypatch.setattr(local_backend.subprocess, "run", compatible_main_wins_name)
|
|
readiness_requests = allow_readiness_http(monkeypatch)
|
|
info = backend.create("thread", "sandbox")
|
|
port = history.allocated[0]
|
|
assert info.sandbox_url == f"http://localhost:{port}"
|
|
assert readiness_requests == [f"http://localhost:{port}/v1/sandbox"]
|
|
assert_reserved(allocator, port)
|
|
assert history.released == []
|
|
backend.destroy(info)
|
|
assert_released_once(allocator, history, port)
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert_released_once(allocator, history, port)
|
|
|
|
|
|
@pytest.mark.parametrize("outcome", ["timeout", "rejected"])
|
|
def test_late_creation_failure_cannot_release_a_destroyed_instances_successor(restricted_backend, monkeypatch, outcome):
|
|
delayed_failure = subprocess.TimeoutExpired(["docker", "run"], timeout=1) if outcome == "timeout" else rejected_start("daemon rejected late sandbox start")
|
|
expected_error = subprocess.TimeoutExpired if outcome == "timeout" else RuntimeError
|
|
backend, allocator, history, docker = restricted_backend(None)
|
|
allow_readiness_http(monkeypatch)
|
|
successors = []
|
|
interleaved = False
|
|
|
|
def destroy_before_create_returns(cmd, **kwargs):
|
|
nonlocal interleaved
|
|
if not interleaved and cmd[1] == "run":
|
|
interleaved = True
|
|
docker.calls.append(tuple(cmd))
|
|
docker.record_main_container(cmd)
|
|
info = backend.discover("sandbox")
|
|
assert info is not None and not info.requires_replacement
|
|
backend.destroy(info)
|
|
successors.append(backend.create("successor-thread", "successor"))
|
|
raise delayed_failure
|
|
return docker.run(cmd, **kwargs)
|
|
|
|
monkeypatch.setattr(local_backend.subprocess, "run", destroy_before_create_returns)
|
|
with pytest.raises(expected_error) as caught:
|
|
backend.create("thread", "sandbox")
|
|
if outcome == "timeout":
|
|
assert caught.value is delayed_failure
|
|
else:
|
|
assert "daemon rejected late sandbox start" in str(caught.value)
|
|
assert interleaved
|
|
assert len(history.allocated) == 2
|
|
assert history.released == history.allocated[:1]
|
|
successor_port = urlparse(successors[0].sandbox_url).port
|
|
assert allocator._reserved_ports == {successor_port}
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert allocator._reserved_ports == {successor_port}
|
|
assert history.released == history.allocated[:1]
|
|
|
|
|
|
def test_empty_url_destroy_keeps_unknown_port_until_fenced_absent_cleanup(restricted_backend, monkeypatch):
|
|
backend, allocator, history, docker = restricted_backend(rejected_start(), fail_proxy_stop=True)
|
|
with pytest.raises(RuntimeError, match="daemon rejected sandbox start"):
|
|
backend.create("thread", "sandbox")
|
|
port = history.allocated[0]
|
|
main_command = list(next(cmd for cmd in docker.calls if cmd[:2] == ("docker", "run")))
|
|
docker.record_main_container(main_command)
|
|
docker.containers[docker.proxy_names[0]]["Config"]["Labels"]["deerflow.network_policy_digest"] = "older-policy"
|
|
info = backend.discover("sandbox")
|
|
assert info is not None and info.requires_replacement
|
|
assert info.sandbox_url == ""
|
|
docker.fail_proxy_stop = False
|
|
backend.destroy(info)
|
|
assert docker.containers == {}
|
|
assert_reserved(allocator, port)
|
|
assert history.released == []
|
|
assert backend.is_absent("sandbox") is True
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert_released_once(allocator, history, port)
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert_released_once(allocator, history, port)
|
|
|
|
|
|
def test_failed_proxy_probe_logs_its_error_without_replacing_creation_error(restricted_backend, monkeypatch, caplog):
|
|
original = FileNotFoundError("sandbox executable cannot start")
|
|
backend, allocator, history, docker = restricted_backend(original, fail_proxy_stop=True)
|
|
probe_error = PermissionError("proxy inspection denied")
|
|
|
|
def failed_proxy_probe(cmd, **kwargs):
|
|
if docker.proxy_names and cmd == ["docker", "inspect", docker.proxy_names[0]]:
|
|
raise probe_error
|
|
return docker.run(cmd, **kwargs)
|
|
|
|
monkeypatch.setattr(local_backend.subprocess, "run", failed_proxy_probe)
|
|
with (
|
|
caplog.at_level(logging.WARNING, logger=local_backend.__name__),
|
|
pytest.raises(FileNotFoundError) as caught,
|
|
):
|
|
backend.create("thread", "sandbox")
|
|
assert caught.value is original
|
|
warnings = [record for record in caplog.records if record.name == local_backend.__name__ and record.levelno >= logging.WARNING]
|
|
assert any("proxy" in record.getMessage().lower() and record.exc_info and probe_error in (record.exc_info[1], record.exc_info[1].__cause__) for record in warnings)
|
|
assert_reserved(allocator, history.allocated[0])
|
|
assert history.released == []
|
|
|
|
|
|
def test_known_destroy_keeps_release_ownership_when_unknown_destroy_overlaps(restricted_backend, monkeypatch):
|
|
backend, allocator, history, docker = restricted_backend(None)
|
|
info = backend.create("thread", "sandbox")
|
|
port = history.allocated[0]
|
|
unknown_info = local_backend.SandboxInfo(
|
|
sandbox_id=info.sandbox_id,
|
|
sandbox_url=info.sandbox_url,
|
|
container_name=info.container_name,
|
|
container_id=None,
|
|
)
|
|
successors = []
|
|
interleaved = False
|
|
|
|
def unknown_destroy_before_known_stop_returns(cmd, **kwargs):
|
|
nonlocal interleaved
|
|
if not interleaved and cmd[1] == "stop" and cmd[-1] == info.container_id:
|
|
interleaved = True
|
|
backend.destroy(unknown_info)
|
|
# The known-generation owner still waits for its stop receipt.
|
|
assert_reserved(allocator, port)
|
|
assert history.released == []
|
|
successors.append(backend.create("successor-thread", "successor"))
|
|
return docker.run(cmd, **kwargs)
|
|
|
|
monkeypatch.setattr(local_backend.subprocess, "run", unknown_destroy_before_known_stop_returns)
|
|
backend.destroy(info)
|
|
assert interleaved
|
|
assert_released_once(allocator, history, port)
|
|
successor_port = urlparse(successors[0].sandbox_url).port
|
|
assert allocator._reserved_ports == {successor_port}
|
|
backend.complete_absent_teardown("sandbox")
|
|
assert allocator._reserved_ports == {successor_port}
|
|
assert history.released == [port]
|