From eae908c00d99fc8a998902bdbcd01144a761eff4 Mon Sep 17 00:00:00 2001 From: Totoro Date: Sun, 18 Oct 2026 09:57:12 +0800 Subject: [PATCH] fix(sandbox): make failed AIO teardown safely retryable (#7001) * fix(sandbox): make failed AIO teardown safely retryable * docs(sandbox): add sanitized teardown retry verification * fix(sandbox): clear stale teardown fence on warm reclaim --------- Co-authored-by: Willem Jiang --- README.md | 5 + .../deerflow/community/aio_sandbox/AGENTS.md | 8 + .../aio_sandbox/aio_sandbox_provider.py | 221 +++++++--- .../community/aio_sandbox/local_backend.py | 5 +- .../harness/deerflow/sandbox/AGENTS.md | 1 + .../deerflow/sandbox/sandbox_provider.py | 21 +- .../tests/test_aio_sandbox_local_backend.py | 9 +- backend/tests/test_aio_sandbox_provider.py | 378 +++++++++++++++++- .../tests/test_sandbox_provider_lifecycle.py | 42 ++ .../aio-teardown-retry-20261017/README.md | 185 +++++++++ .../aio-teardown-retry-20261017/evidence.json | 347 ++++++++++++++++ .../aio-teardown-retry-20261017/probe.py | 265 ++++++++++++ 12 files changed, 1412 insertions(+), 75 deletions(-) create mode 100644 docs/experiments/aio-teardown-retry-20261017/README.md create mode 100644 docs/experiments/aio-teardown-retry-20261017/evidence.json create mode 100644 docs/experiments/aio-teardown-retry-20261017/probe.py diff --git a/README.md b/README.md index 2798ed4a65..4171cdb049 100644 --- a/README.md +++ b/README.md @@ -836,6 +836,11 @@ Local AIO sandbox port allocation stops at TCP port 65535. If the remaining ports from the configured starting port are occupied, allocation reports that no port is available instead of attempting an out-of-range bind. +AIO teardown failures retain a generation-bound cleanup record. Partial shutdown +reports failure and can be retried; retries verify the runtime identity and absence +before completing cleanup. Healthy replacements with the same logical ID remain +reusable, including after background recovery into the warm pool. + See the [Sandbox Configuration Guide](backend/docs/CONFIGURATION.md#sandbox) to configure your preferred mode. Managed async AIO runs can opt into bounded sandbox prewarming to overlap startup diff --git a/backend/packages/harness/deerflow/community/aio_sandbox/AGENTS.md b/backend/packages/harness/deerflow/community/aio_sandbox/AGENTS.md index 4498f841ba..cffeb7e4ad 100644 --- a/backend/packages/harness/deerflow/community/aio_sandbox/AGENTS.md +++ b/backend/packages/harness/deerflow/community/aio_sandbox/AGENTS.md @@ -42,6 +42,8 @@ Local absence matching compares Docker's fixed error scaffolding case-insensitively — CLI v29 lowercases `Error: No such object:` to `error: no such object:` — while the resource name stays byte-exact and the whole message must match; Apple Container forms remain exact-match. +Missing-network errors may end with Docker CLI v29's exact `exit status 1` line; +other trailing diagnostics still fail closed. Remote proofs use `/api/sandboxes/{id}/presence`, which inspects the Pod directly; the ordinary status GET can return 404 merely because its Service is missing. This covers runtimes without a recoverable generation, including Apple Container. @@ -73,6 +75,12 @@ under those same fences after confirmed absence, including a timed-out stop that actually removed the container. It also clears deterministic local proxy/network leftovers before recreation; unknown cleanup errors retain the quarantine record. +Explicit stop failures use that same generation-bound quarantine before dropping +tracking. Failed record writes retain metadata and fence cached lookups in memory. +Warm-to-active promotion clears an obsolete in-memory fence under the metadata +lock, only for the entry that passed reuse checks; a changed entry must retry. +Partial shutdown raises and remains retryable; successful resources are not retried, +and cleanup infrastructure closes only after every owned teardown completes. Broker capability failures are unknown, never confirmed negatives. Cache only successful observations per provisioner endpoint/auth identity; a failed refresh diff --git a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py index 6dd985cbde..b8e130cb77 100644 --- a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py @@ -286,6 +286,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): self._acquire_epoch_counter = 0 self._acquire_inflight: dict[str, int] = {} self._shutdown_called = False + self._cleanup_retry_ids: set[str] = set() self._idle_checker_stop = threading.Event() self._idle_checker_thread: threading.Thread | None = None self._renewal_stop = threading.Event() @@ -1887,11 +1888,12 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): logger.info("Warm-pool sandbox %s was claimed for teardown while publishing ownership; not reclaiming it", sandbox_id) return None self._assert_warm_identity_available_locked(sandbox_id, key) - warm_item = self._warm_pool.pop(sandbox_id, None) - if warm_item is None: + warm_item = self._warm_pool.get(sandbox_id) + if warm_item is None or warm_item[0] is not info: + # A different entry has not passed the reuse checks above. return None + self._warm_pool.pop(sandbox_id) self._warm_pool_identity.pop(sandbox_id, None) - info, _ = warm_item sandbox = AioSandbox( id=sandbox_id, base_url=info.sandbox_url, @@ -1901,6 +1903,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): ) self._sandboxes[sandbox_id] = sandbox self._sandbox_infos[sandbox_id] = info + getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id) self._active_sandbox_identity[sandbox_id] = key self._last_activity[sandbox_id] = time.time() self._thread_sandboxes[key] = sandbox_id @@ -1980,6 +1983,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): self._warm_pool_identity.pop(info.sandbox_id, None) self._sandboxes[info.sandbox_id] = sandbox self._sandbox_infos[info.sandbox_id] = info + getattr(self, "_cleanup_retry_ids", set()).discard(info.sandbox_id) self._active_sandbox_identity[info.sandbox_id] = key self._last_activity[info.sandbox_id] = time.time() self._thread_sandboxes[key] = info.sandbox_id @@ -2026,6 +2030,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): self._warm_pool_identity.pop(sandbox_id, None) self._sandboxes[sandbox_id] = sandbox self._sandbox_infos[sandbox_id] = info + getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id) self._active_sandbox_identity[sandbox_id] = key self._last_activity[sandbox_id] = time.time() if key is not None: @@ -2187,7 +2192,19 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): # release matters just as much — the container is probably still up, # so a marker left behind would block its thread from re-acquiring it. with self._held_teardown_lease(sandbox_id): - self._backend.destroy(entry) + try: + self._backend.destroy(entry) + except Exception: + if reason == "shutdown": + # Shutdown has no later idle tick. Persist before + # releasing the ownership fence or dropping metadata. + self._remember_failed_teardown(sandbox_id, entry) + with self._lock: + current = self._warm_pool.get(sandbox_id) + if current is not None and current[0] is entry: + self._warm_pool.pop(sandbox_id, None) + self._warm_pool_identity.pop(sandbox_id, None) + raise except Exception as e: if reason == "idle_timeout": logger.error(f"Failed to destroy idle warm-pool sandbox {sandbox_id}: {e}") @@ -2498,41 +2515,58 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): reason="is quarantined and owned by a peer; its cleanup is deferred until that owner releases it", ) with self._held_teardown_lease(sandbox_id): - if self._backend.is_absent(sandbox_id) is not True: - inspect_runtime = getattr(self._backend, "inspect_runtime", None) - if not callable(inspect_runtime): - raise RuntimeError(f"Runtime inspection is unavailable for sandbox {sandbox_id}") - current = inspect_runtime(sandbox_id) - if current is not None: - if not isinstance(current, SandboxInfo) or current.sandbox_id != sandbox_id: - raise RuntimeError(f"Runtime inspection returned invalid identity for sandbox {sandbox_id}") - # A replacement can share the logical ID while its Service - # is still unavailable. Do not stop that unquarantined - # runtime; create can repair its endpoint and validate the - # actual provisioning mode. The replacement itself proves - # the fenced generations are absent (logical IDs are unique - # per Pod/container), so prune only those stale records — - # a full retire() would also wipe a fence a peer may have - # just written for this live generation. - if not store.contains(current): - store.retire_replaced_generations(current) - return - # Inspection recovers runtime identity, but replacement - # cleanup must also remove its previous policy resources. - current.requires_replacement = True - self._backend.destroy(current) - # DELETE acceptance cannot prove the Pod/container vanished. - # Also recheck when it disappeared between our observations. - if self._backend.is_absent(sandbox_id) is not True: - raise SandboxPolicyReplacementDeferredError( - sandbox_id, - reason="is quarantined and its previous runtime is still present; the fence is retained", - ) - self._backend.complete_absent_teardown(sandbox_id) - store.retire(info) + self._complete_quarantined_teardown(sandbox_id, store, retire_replaced=True) finally: self._finish_local_teardown(sandbox_id) + def _remember_failed_teardown(self, sandbox_id: str, info: SandboxInfo) -> None: + """Fence an uncertain stop before dropping its retry metadata.""" + with self._lock: + self._cleanup_retry_ids = getattr(self, "_cleanup_retry_ids", set()) | {sandbox_id} + info.requires_replacement = True + self._quarantine_store().mark(info) + + def _complete_quarantined_teardown(self, sandbox_id: str, store: SandboxQuarantine, *, retire_replaced: bool) -> bool: + """Retry under the caller's local reservation and ownership heartbeat.""" + if self._backend.is_absent(sandbox_id) is not True: + inspect_runtime = getattr(self._backend, "inspect_runtime", None) + if not callable(inspect_runtime): + raise RuntimeError(f"Runtime inspection is unavailable for sandbox {sandbox_id}") + current = inspect_runtime(sandbox_id) + if current is not None: + if not isinstance(current, SandboxInfo) or current.sandbox_id != sandbox_id: + raise RuntimeError(f"Runtime inspection returned invalid identity for sandbox {sandbox_id}") + # A replacement can share the logical ID while its Service + # is still unavailable. Do not stop that unquarantined + # runtime; create can repair its endpoint and validate the + # actual provisioning mode. The replacement itself proves + # the fenced generations are absent (logical IDs are unique + # per Pod/container), so prune only those stale records — + # a full retire() would also wipe a fence a peer may have + # just written for this live generation. + if not store.contains(current): + if retire_replaced: + store.retire_replaced_generations(current) + with self._lock: + getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id) + return False + # Inspection recovers runtime identity, but replacement + # cleanup must also remove its previous policy resources. + current.requires_replacement = True + self._backend.destroy(current) + # DELETE acceptance cannot prove the Pod/container vanished. + # Also recheck when it disappeared between our observations. + if self._backend.is_absent(sandbox_id) is not True: + raise SandboxPolicyReplacementDeferredError( + sandbox_id, + reason="is quarantined and its previous runtime is still present; the fence is retained", + ) + self._backend.complete_absent_teardown(sandbox_id) + store.retire(SandboxInfo(sandbox_id, "")) + with self._lock: + getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id) + return True + def _create_sandbox(self, thread_id: str | None, sandbox_id: str, *, user_id: str | None = None) -> str: """Create a new sandbox via the backend. @@ -2669,6 +2703,8 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): The sandbox instance if found, None otherwise. """ with self._lock: + if sandbox_id in getattr(self, "_cleanup_retry_ids", set()): + return None sandbox = self._sandboxes.get(sandbox_id) if sandbox is not None: self._last_activity[sandbox_id] = time.time() @@ -2684,6 +2720,8 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): """Return a cached client only for its recorded user/thread identity.""" key = self._thread_key(thread_id, user_id) with self._lock: + if sandbox_id in getattr(self, "_cleanup_retry_ids", set()): + return None if self._thread_sandboxes.get(key) != sandbox_id: return None if self._active_sandbox_identity.get(sandbox_id) != key: @@ -2828,32 +2866,40 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): with self._lock: sandbox = self._sandboxes.get(sandbox_id) info = self._sandbox_infos.get(sandbox_id) - if getattr(sandbox, "has_pending_session_creates", False) is True or getattr(sandbox, "requires_container_recycle", False) is True: + if info is None and (warm := self._warm_pool.get(sandbox_id)) is not None: + info = warm[0] + store = self._quarantine_store() + with self._lock: + retry_cleanup = sandbox_id in getattr(self, "_cleanup_retry_ids", set()) + retry_cleanup = retry_cleanup or (info is None and store.contains(SandboxInfo(sandbox_id, ""))) + if getattr(sandbox, "has_pending_session_creates", False) is True or getattr(sandbox, "requires_container_recycle", False) is True or (retry_cleanup and info is not None): if info is None: raise RuntimeError(f"Cannot quarantine sandbox {sandbox_id} without its runtime metadata") # All destroy paths must persist before claiming/untracking. If # storage fails, retain the dirty client and its renewal lease so # a later idle/shutdown/explicit destroy retries this same fence. info.requires_replacement = True - self._quarantine_store().mark(info) + store.mark(info) - # Claim before untracking. The reverse order loses the container on a - # refused claim: still running, and no longer in any of our maps, so - # nothing here would ever reap or reclaim it. + # Keep both the local reservation and ownership fence through the stop. + # A failed stop must preserve a generation-bound cleanup target before + # losing its in-process metadata, or a second destroy falsely succeeds. if not self._claim_ownership(sandbox_id, for_destroy=True): logger.warning("Refusing to destroy sandbox %s: owned by another instance", sandbox_id) return False - sandbox, info, _ = self._remove_tracked_sandbox(sandbox_id) - - if sandbox is not None: - # Defense-in-depth: close() already swallows its own errors; this - # guard only protects against a future close() that misbehaves, so - # host-side client cleanup can never block container destruction. - try: - sandbox.close() - except Exception as e: - logger.warning(f"Error closing sandbox {sandbox_id} during destroy: {e}") + if retry_cleanup: + # A previous failed stop may have lost its endpoint or been replaced + # under the same logical ID. Use the existing authoritative probes. + with self._held_teardown_lease(sandbox_id): + if info is not None: + self._remove_tracked_sandbox(sandbox_id, expected_info=info) + if sandbox is not None: + try: + sandbox.close() + except Exception as e: + logger.warning(f"Error closing sandbox {sandbox_id} during destroy: {e}") + return self._complete_quarantined_teardown(sandbox_id, store, retire_replaced=False) if info: # The marker must outlast the stop, not the TTL it was written with, @@ -2863,13 +2909,32 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): # the error still propagates out of the `with` (`shutdown()` logs per # sandbox off it), it is just no longer this method's job to release. with self._held_teardown_lease(sandbox_id): - self._backend.destroy(info) + try: + self._backend.destroy(info) + except Exception: + self._remember_failed_teardown(sandbox_id, info) + self._remove_tracked_sandbox(sandbox_id, expected_info=info) + if sandbox is not None: + try: + sandbox.close() + except Exception as e: + logger.warning(f"Error closing sandbox {sandbox_id} after failed destroy: {e}") + sandbox = None + raise + # A stale completion must not remove a replacement generation. + self._remove_tracked_sandbox(sandbox_id, expected_info=info) logger.info(f"Destroyed sandbox {sandbox_id}") else: + self._remove_tracked_sandbox(sandbox_id) # No container to stop, so no teardown lease was held: clear the # marker the claim above wrote, so an untracked id cannot leave a # lease stuck in `del:`. self._release_ownership(sandbox_id) + if sandbox is not None: + try: + sandbox.close() + except Exception as e: + logger.warning(f"Error closing sandbox {sandbox_id} during destroy: {e}") return True def reset(self) -> None: @@ -2901,28 +2966,56 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): # known stopped. If either bounded join fails, a retry must still own # every active and warm entry. with self._lock: - sandbox_ids = list(self._sandboxes.keys()) - warm_items = list(self._warm_pool.items()) - self._warm_pool.clear() - self._warm_pool_identity.clear() + pending = getattr(self, "_cleanup_retry_ids", set()) + sandbox_ids = list(set(self._sandboxes) | pending) + warm_items = [(sid, item) for sid, item in self._warm_pool.items() if sid not in pending] logger.info(f"Shutting down {len(sandbox_ids)} active + {len(warm_items)} warm-pool sandbox(es)") + failed: set[str] = set() for sandbox_id in sandbox_ids: try: # Shutdown has stopped the maintenance workers and closes the - # entire provider. It must clean up even unfinished executions. - self._destroy_tracked(sandbox_id, still_reapable=lambda: True, force=True) + # entire provider. Retries must also clean unfinished executions + # whose client was already detached after a failed stop. + destroyed = self._destroy_tracked(sandbox_id, still_reapable=lambda: True, force=True) + if not destroyed: + failed.add(sandbox_id) except Exception as e: + failed.add(sandbox_id) logger.error(f"Failed to destroy sandbox {sandbox_id} during shutdown: {e}") for sandbox_id, (info, _) in warm_items: - # Route through _destroy_warm_entry so the ownership claim and the - # container stop stay together, as on the idle path. Unconditional - # here: the entries were removed from `_warm_pool` under the lock - # above, so the pool-membership predicate the other callers use would - # refuse every one of them. - self._destroy_warm_entry(sandbox_id, info, reason="shutdown", still_reapable=lambda: True) + # The helper removes successful entries under its reservation and + # preserves failed ones in tracking or quarantine for the next attempt. + try: + destroyed = self._destroy_warm_entry(sandbox_id, info, reason="shutdown", still_reapable=lambda sid=sandbox_id, expected=info: self._warm_pool.get(sid, (None,))[0] is expected) + except Exception as e: + logger.error(f"Failed to destroy warm-pool sandbox {sandbox_id} during shutdown: {e}") + destroyed = False + if not destroyed: + failed.add(sandbox_id) + + # A known peer owns its own cleanup. Unknown ownership or a failed stop + # must retain the record, manager and backend for an explicit retry. + for sandbox_id in list(failed): + epoch = self._acquire_epoch_of(sandbox_id) + try: + owner = self._ownership.owner(sandbox_id) + except OwnershipBackendError: + continue + if owner is not None and owner != self._owner_id: + self._forget_lost_sandbox(sandbox_id, expected_epoch=epoch) + with self._lock: + if sandbox_id not in self._sandboxes and sandbox_id not in self._warm_pool: + failed.remove(sandbox_id) + getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id) + + if failed: + with self._lock: + self._shutdown_called = False + self._cleanup_retry_ids = getattr(self, "_cleanup_retry_ids", set()) | failed + raise RuntimeError(f"Failed to destroy {len(failed)} sandbox(es) during AIO shutdown; retry shutdown") try: self._ownership.close() diff --git a/backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py b/backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py index 3753c012cb..ecc6f6147d 100644 --- a/backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py @@ -552,7 +552,7 @@ _DOCKER_NO_SUCH_CONTAINER_RE = re.compile( re.IGNORECASE, ) _DOCKER_NO_SUCH_NETWORK_RE = re.compile( - r"(?:error(?: response from daemon)?: )?(?:no such network: (?P.+)|network (?P.+) not found)", + r"(?:error(?: response from daemon)?: )?(?:no such network: (?P.+)|network (?P.+) not found)(?:\nexit status 1)?", re.IGNORECASE, ) @@ -585,7 +585,8 @@ def _is_no_such_network_error(stderr: str, network_name: str) -> bool: """Accept Docker's complete missing-network response for this exact name. Same contract as ``_is_no_such_container_error``: case-insensitive - scaffolding, byte-exact network name, full-message match. + scaffolding, byte-exact network name, full-message match. Docker CLI v29 + may append exactly one ``exit status 1`` line; other diagnostics still fail. """ message = (stderr or "").strip() match = _DOCKER_NO_SUCH_NETWORK_RE.fullmatch(message) diff --git a/backend/packages/harness/deerflow/sandbox/AGENTS.md b/backend/packages/harness/deerflow/sandbox/AGENTS.md index 31b6386644..f1565c18d2 100644 --- a/backend/packages/harness/deerflow/sandbox/AGENTS.md +++ b/backend/packages/harness/deerflow/sandbox/AGENTS.md @@ -18,6 +18,7 @@ cancellation before propagating it. **Authorization gate** (`sandbox:execute`, RFC #4063 Phase 3): every sandbox-backed tool call passes through the gate in `deerflow/authz/sandbox_authz.py` - a binary `authorize(principal, "sandbox", "execute", target="*")` check before either reusing a persisted sandbox id or calling `provider.acquire`. Rechecking reuse is required because authorization config and user roles can change while the sandbox remains cached. Sync tool invocations call `authorize_sandbox_execution`; async tool invocations await `authorize_sandbox_execution_async` exactly once. A task-local `ContextVar` scopes that single decision across the complete composed tool invocation, including `ReadBeforeWriteMiddleware`'s pre-write inspection, tool body, and post-read mark; the value is copied into `asyncio.to_thread` workers. Authorization denial is converted to the normal error `ToolMessage` at the composed middleware boundary and is explicitly excluded from the gate's generic fail-open handlers. Async config loading and provider class discovery/import are offloaded before `aauthorize()` so reused sandbox calls do not hash config files or import custom modules on the event loop; provider construction remains on the running event loop because async providers may initialize loop-affine clients. The gate lives at the single tool initialization entry point (`ensure_sandbox_initialized` / `ensure_sandbox_initialized_async` in `tools.py`), while `SandboxMiddleware.before_agent` / `abefore_agent` apply the matching sync/async check to eager acquisition. Deny raises `SandboxAuthorizationError` (`sandbox/exceptions.py`), which propagates out of ordinary tool execution as a friendly error `ToolMessage` ("sandbox execution is not permitted for your role") - the eager path catches it and skips acquisition instead, deferring the deny to the first sandbox-touching tool call so both paths share the same semantics. Provider errors (authorization calls and provider resolution) follow `authorization.fail_closed` / `fail_open`; no readable `config.yaml` or `authorization.enabled: false` makes the gate a no-op (`safe_app_config` tolerates missing config). Gateway upload/artifact sync calls `try_acquire_sandbox_for_request` (`app/gateway/authz.py`), which gates, returns a request lease, and skips sync on deny while preserving the primary operation. Callers release after their last sandbox operation; artifacts request normal parking, uploads do not. Tests: `tests/test_sandbox_authorization.py` and `tests/blocking_io/test_sandbox_authorization.py`. **Environment policy** (`sandbox/env_policy.py`): `build_sandbox_env()` scrubs inherited credential names via `is_blocked_env_name`, preserving benign vars (`PATH`, `HOME`, `LANG`, `VIRTUAL_ENV`). Explicit `sandbox.environment` injection survives scrubbing; request secrets override it. Local sandboxes validate keys at construction; generic and per-thread instances receive resolved config values verbatim. Local `bash` masks effective operator credentials and request secrets in output and command errors, preserving benign config values. Holders stay bound through scope cleanup; AIO reserves idle teardown atomically. +Failed provider shutdown retains the singleton and lease manager for retry unless replaced concurrently. Fork lookups reject invalidated clients; cancelled acquires leave borrowers a pending release. **Implementations**: - `LocalSandboxProvider` - Local filesystem execution. `acquire(thread_id)` returns a per-user/thread `LocalSandbox` (id `local:{user_id}:{thread_id}`) whose `path_mappings` resolve `/mnt/user-data` (file tools: `ls` lists only these three; search/read reject the root), its `{workspace,uploads,outputs}`, and `/mnt/acp-workspace` to that thread's host directories, so the public `Sandbox` API honours the `/mnt/user-data` contract uniformly with AIO. `acquire()` / `acquire(None)` keeps the legacy generic singleton (id `local`) for callers without a thread context. Per-thread sandboxes are held in an LRU cache (default 256 entries) guarded by a `threading.Lock`. Shared runs use category mappings; a policy-scoped run replaces them with one `/mnt/skills` root mapping to the coherent thread view, so structured file tools resolve through one managed boundary. This is not a host filesystem security boundary: an enabled host `bash` subprocess can use canonical paths without `PathMapping`, so `supports_agent_skill_isolation` is dynamic and explicit Agent policies fail closed while host bash is enabled; unrestricted runs then get the shared mappings, never a leftover thread view. Host-to-virtual output masking scans dynamic per-user/per-thread roots directly instead of compiling path-specific regexes, so evicted thread IDs do not remain in Python's global regex caches; a separate 256-entry root cache prevents repeated `realpath()` walks for every glob/grep match while bounding dynamic-path retention, and only the small process-stable skill/integration source set uses a bounded compiled cache. The shared `path_patterns.py` tail stops at `:`, so `$PATH`-style lists mask every entry; a mount symlink resolving outside all mounts keeps its mount path. On Windows, Git Bash/MSYS argument-conversion exclusions are limited to safe non-root virtual path prefixes; do not restore a blanket conversion disable, because host-native CLI launchers need normal MSYS path conversion for their own installation paths. diff --git a/backend/packages/harness/deerflow/sandbox/sandbox_provider.py b/backend/packages/harness/deerflow/sandbox/sandbox_provider.py index 9a5e1b2a68..b608dec188 100644 --- a/backend/packages/harness/deerflow/sandbox/sandbox_provider.py +++ b/backend/packages/harness/deerflow/sandbox/sandbox_provider.py @@ -253,11 +253,11 @@ def reset_sandbox_provider() -> None: def shutdown_sandbox_provider() -> None: - """Shutdown and reset the sandbox provider. + """Shutdown the provider, keeping failed cleanup reachable for retry. - This properly shuts down the provider (releasing all sandboxes) - before clearing the singleton. Call this when the application - is shutting down or when you need to completely reset the sandbox system. + A provider installed concurrently takes precedence over a failed instance. + Provider callbacks run outside the registry lock; discard execution leases + only after successful shutdown. """ global _default_sandbox_provider # Detach the reference under the lock, then run the (potentially slow) @@ -268,8 +268,17 @@ def shutdown_sandbox_provider() -> None: if provider is not None and hasattr(provider, "shutdown"): from deerflow.sandbox.lease import discard_sandbox_lease_manager - discard_sandbox_lease_manager(provider) - provider.shutdown() + try: + provider.shutdown() + except BaseException: + # Keep a failed cleanup reachable for a later shutdown call, without + # replacing a new provider installed while this callback ran. + with _provider_lock: + if _default_sandbox_provider is None: + _default_sandbox_provider = provider + raise + else: + discard_sandbox_lease_manager(provider) def set_sandbox_provider(provider: SandboxProvider) -> None: diff --git a/backend/tests/test_aio_sandbox_local_backend.py b/backend/tests/test_aio_sandbox_local_backend.py index c1ac23f39f..a0aa5f8849 100644 --- a/backend/tests/test_aio_sandbox_local_backend.py +++ b/backend/tests/test_aio_sandbox_local_backend.py @@ -2254,7 +2254,7 @@ def test_absent_teardown_preserves_pending_port_on_unknown_network_cleanup_error assert backend._pending_cleanup_ports == {"old-generation": ("existing", 18080)} -@pytest.mark.parametrize("error_style", ["legacy", "lowercase-scaffolding", "uppercase-scaffolding"]) +@pytest.mark.parametrize("error_style", ["legacy", "lowercase-scaffolding", "uppercase-scaffolding", "cli-exit-status"]) @pytest.mark.parametrize("sandbox_id", ["existing", "Mixed-Case"]) def test_absent_teardown_accepts_only_exact_missing_networks(monkeypatch, error_style, sandbox_id): backend = _restricted_backend() @@ -2269,6 +2269,7 @@ def test_absent_teardown_accepts_only_exact_missing_networks(monkeypatch, error_ "legacy": ("Error: No such container: {}", "Error response from daemon: network {} not found"), "lowercase-scaffolding": ("error: no such object: {}", "error: network {} not found"), "uppercase-scaffolding": ("ERROR: NO SUCH OBJECT: {}", "ERROR: NETWORK {} NOT FOUND"), + "cli-exit-status": ("Error: No such container: {}", "Error response from daemon: network {} not found\nexit status 1\n"), } container_error, network_error = error_formats[error_style] @@ -2296,6 +2297,12 @@ def test_absent_teardown_accepts_only_exact_missing_networks(monkeypatch, error_ ("error: no such network: sandbox-net", True), ("No such network: sandbox-net", True), ("network sandbox-net not found", True), + ("Error response from daemon: network sandbox-net not found\nexit status 1\n", True), + ("Error response from daemon: network sandbox-net-extra not found\nexit status 1", False), + ("Error response from daemon: network Sandbox-Net not found\nexit status 1", False), + ("Error response from daemon: network sandbox-net not found\nexit status 2", False), + ("Error response from daemon: network sandbox-net not found\npermission denied\nexit status 1", False), + ("Error response from daemon: network sandbox-net not found\nexit status 1\nunknown failure", False), ("error: network sandbox-net-extra not found", False), ("error: network Sandbox-Net not found", False), ("error: no such network: sandbox-other", False), diff --git a/backend/tests/test_aio_sandbox_provider.py b/backend/tests/test_aio_sandbox_provider.py index 534efa5d7f..e27378e327 100644 --- a/backend/tests/test_aio_sandbox_provider.py +++ b/backend/tests/test_aio_sandbox_provider.py @@ -7,6 +7,7 @@ import importlib import os import stat import threading +import time from types import SimpleNamespace from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -3286,6 +3287,377 @@ def test_destroy_swallows_close_errors_and_still_destroys_backend(tmp_path, capl provider._backend.destroy.assert_called_once() +def _observe_destroy_target(provider, info): + running = {info.sandbox_id: info} + provider._backend.is_absent = MagicMock(side_effect=lambda sid: sid not in running) + provider._backend.inspect_runtime = MagicMock(side_effect=running.get) + provider._backend.complete_absent_teardown = MagicMock() + return running + + +def test_destroy_failure_keeps_durable_runtime_target_for_cleanup_retry(tmp_path): + provider, sandbox, _ = _make_provider_with_active_sandbox(tmp_path, "retry-stop") + info = provider._sandbox_infos["retry-stop"] + info.container_id = "generation-one" + running = _observe_destroy_target(provider, info) + + def stop(target): + if provider._backend.destroy.call_count == 1: + raise RuntimeError("stop failed") + running.pop(target.sandbox_id) + + provider._backend.destroy.side_effect = stop + try: + with pytest.raises(RuntimeError, match="stop failed"): + provider.destroy(info.sandbox_id) + assert provider._quarantine_store().contains(info) + assert info.sandbox_id not in provider._sandboxes + sandbox.close.assert_called_once() + assert provider.destroy(info.sandbox_id) is True + assert provider._backend.destroy.call_count == 2 + assert all(call.args[0] is info for call in provider._backend.destroy.call_args_list) + assert info.sandbox_id not in provider._sandboxes + assert info.sandbox_id not in provider._sandbox_infos + assert not provider._quarantine_store().contains(info) + sandbox.close.assert_called_once() + finally: + provider.reset() + + +def test_destroy_repeated_failure_never_turns_into_false_success(tmp_path): + provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "still-running") + _observe_destroy_target(provider, provider._sandbox_infos["still-running"]) + provider._backend.destroy.side_effect = RuntimeError("stop still failing") + try: + for _ in range(2): + with pytest.raises(RuntimeError, match="stop still failing"): + provider.destroy("still-running") + assert provider._backend.destroy.call_count == 2 + finally: + provider.reset() + + +def test_destroy_retry_preserves_a_different_runtime_generation(tmp_path): + provider, _, module = _make_provider_with_active_sandbox(tmp_path, "replacement") + old = provider._sandbox_infos["replacement"] + old.container_id = "old-generation" + running = _observe_destroy_target(provider, old) + provider._backend.destroy.side_effect = RuntimeError("stop failed") + try: + with pytest.raises(RuntimeError, match="stop failed"): + provider.destroy("replacement") + new = module.SandboxInfo("replacement", "http://new", container_id="new-generation") + running["replacement"] = new + for _ in range(2): + assert provider.destroy("replacement") is False + assert provider._backend.destroy.call_count == 1 + assert running["replacement"] is new + assert provider._quarantine_store().contains(old) + assert not provider._quarantine_store().contains(new) + finally: + provider.reset() + + +@pytest.mark.parametrize("admission", ["discovery", "orphan-adoption"]) +@pytest.mark.parametrize("via_lease", [False, True], ids=["provider", "lease-manager"]) +def test_failed_destroy_allows_replacement_acquisition(failure_recovery_lifecycle, admission, via_lease): + from deerflow.sandbox.lease import SandboxLeaseManager + + provider, _, _, old = failure_recovery_lifecycle + running = _observe_destroy_target(provider, old) + provider._backend.destroy.side_effect = RuntimeError("stop failed") + with pytest.raises(RuntimeError, match="stop failed"): + provider.destroy(old.sandbox_id) + + replacement = type(old)(old.sandbox_id, old.sandbox_url, container_id="replacement-generation") + running[old.sandbox_id] = replacement + provider._backend.discover = MagicMock(return_value=replacement) + provider._backend.list_running = MagicMock(return_value=[replacement]) + if admission == "orphan-adoption": + provider._reconcile_orphans() + assert provider._warm_pool[old.sandbox_id][0] is replacement + + manager = SandboxLeaseManager(provider) + try: + if via_lease: + acquired = manager.acquire("new-execution", "thread-failure-recovery", user_id="alice") + else: + acquired = provider.acquire("thread-failure-recovery", user_id="alice") + assert acquired == old.sandbox_id + assert provider._sandbox_infos[acquired] is replacement + assert provider.get(acquired) is not None + assert provider.get_scoped(acquired, thread_id="thread-failure-recovery", user_id="alice") is not None + assert acquired not in provider._cleanup_retry_ids + assert provider._quarantine_store().contains(old) + assert not provider._quarantine_store().contains(replacement) + provider._backend.destroy.assert_called_once_with(old) + + if via_lease: + manager.release("new-execution") + reacquired = manager.acquire("next-execution", "thread-failure-recovery", user_id="alice") + else: + provider.release(acquired) + reacquired = provider.acquire("thread-failure-recovery", user_id="alice") + assert reacquired == acquired + assert provider.get(reacquired) is not None + assert provider._sandbox_infos[reacquired] is replacement + provider._backend.destroy.assert_called_once_with(old) + finally: + manager.close() + + +@pytest.mark.parametrize("generation", ["container-generation-1", None], ids=["quarantined", "unknown"]) +def test_warm_reclaim_keeps_failed_stop_fence_for_unsafe_generation(failure_recovery_lifecycle, generation): + provider, _, _, old = failure_recovery_lifecycle + _observe_destroy_target(provider, old) + provider._backend.destroy.side_effect = RuntimeError("stop failed") + with pytest.raises(RuntimeError, match="stop failed"): + provider.destroy(old.sandbox_id) + candidate = type(old)(old.sandbox_id, old.sandbox_url, container_id=generation) + provider._warm_pool[old.sandbox_id] = (candidate, time.time()) + provider._warm_pool_identity[old.sandbox_id] = None + provider_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider") + + with pytest.raises(provider_mod.SandboxPolicyReplacementDeferredError): + provider._reclaim_warm_pool_sandbox("thread-failure-recovery", old.sandbox_id, user_id="alice") + + assert old.sandbox_id in provider._cleanup_retry_ids + assert provider._quarantine_store().contains(candidate) + assert provider.get(old.sandbox_id) is None + assert old.sandbox_id not in provider._sandboxes + + +@pytest.mark.parametrize("generation", ["container-generation-1", None], ids=["quarantined", "unknown"]) +def test_warm_reclaim_does_not_clear_fence_if_entry_changes(failure_recovery_lifecycle, monkeypatch, generation): + provider, _, _, old = failure_recovery_lifecycle + _observe_destroy_target(provider, old) + provider._backend.destroy.side_effect = RuntimeError("stop failed") + with pytest.raises(RuntimeError, match="stop failed"): + provider.destroy(old.sandbox_id) + replacement = type(old)(old.sandbox_id, old.sandbox_url, container_id="replacement-generation") + provider._backend.list_running = MagicMock(return_value=[replacement]) + provider._reconcile_orphans() + assert provider._warm_pool[old.sandbox_id][0] is replacement + unvalidated = type(old)(old.sandbox_id, old.sandbox_url, container_id=generation) + publish = provider._publish_ownership + + def swap_during_publish(sandbox_id): + publish(sandbox_id) + with provider._lock: + provider._warm_pool[sandbox_id] = (unvalidated, time.time()) + + monkeypatch.setattr(provider, "_publish_ownership", swap_during_publish) + assert provider._reclaim_warm_pool_sandbox("thread-failure-recovery", old.sandbox_id, user_id="alice") is None + assert provider._warm_pool[old.sandbox_id][0] is unvalidated + assert old.sandbox_id in provider._cleanup_retry_ids + assert provider._quarantine_store().contains(unvalidated) + assert provider.get(old.sandbox_id) is None + assert old.sandbox_id not in provider._sandboxes + + +def test_failed_destroy_record_write_blocks_cached_reuse_until_retry(tmp_path, monkeypatch): + provider, client, _ = _make_provider_with_active_sandbox(tmp_path, "record-write") + info = provider._sandbox_infos["record-write"] + info.container_id = "generation-one" + provider._thread_sandboxes[("alice", "thread")] = info.sandbox_id + provider._active_sandbox_identity[info.sandbox_id] = ("alice", "thread") + running = _observe_destroy_target(provider, info) + + def stop(target): + if provider._backend.destroy.call_count == 1: + raise RuntimeError("stop failed") + running.pop(target.sandbox_id) + + provider._backend.destroy.side_effect = stop + store = provider._quarantine_store() + try: + with monkeypatch.context() as patcher: + patcher.setattr(store, "mark", MagicMock(side_effect=PermissionError("record unavailable"))) + with pytest.raises(PermissionError, match="record unavailable"): + provider.destroy(info.sandbox_id) + assert provider._sandbox_infos[info.sandbox_id] is info + client.close.assert_not_called() + assert provider.get(info.sandbox_id) is None + assert provider.get_scoped(info.sandbox_id, thread_id="thread", user_id="alice") is None + assert provider.destroy(info.sandbox_id) is True + assert provider._backend.destroy.call_count == 2 + assert not provider._cleanup_retry_ids + client.close.assert_called_once() + finally: + provider.reset() + + +def test_destroy_retry_needs_confirmed_absence_after_accepted_stop(tmp_path): + provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "terminating") + info = provider._sandbox_infos["terminating"] + info.container_id = "pod-uid" + running = _observe_destroy_target(provider, info) + provider._backend.destroy.side_effect = [RuntimeError("delete failed"), None] + try: + with pytest.raises(RuntimeError, match="delete failed"): + provider.destroy(info.sandbox_id) + with pytest.raises(RuntimeError, match="still present"): + provider.destroy(info.sandbox_id) + assert provider._quarantine_store().contains(info) + running.clear() + assert provider.destroy(info.sandbox_id) is True + assert provider._backend.destroy.call_count == 2 + assert not provider._quarantine_store().contains(info) + finally: + provider.reset() + + +def test_destroy_keeps_reservation_while_stop_is_pending(tmp_path): + from deerflow.sandbox.lease import get_sandbox_lease_manager + + provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "slow-stop") + info = provider._sandbox_infos["slow-stop"] + entered, finish = threading.Event(), threading.Event() + errors = [] + + def stop(_info): + entered.set() + assert finish.wait(5) + + def destroy(): + try: + provider.destroy("slow-stop") + except BaseException as exc: + errors.append(exc) + + provider._backend.destroy.side_effect = stop + worker = threading.Thread(target=destroy) + worker.start() + try: + assert entered.wait(5) + assert provider._sandbox_infos.get("slow-stop") is info + assert "slow-stop" in provider._local_teardown + assert provider._lock.acquire(blocking=False), "container I/O held the metadata lock" + provider._lock.release() + with pytest.raises(RuntimeError, match="teardown"): + get_sandbox_lease_manager(provider).retain("late-holder", "slow-stop", thread_id="thread", user_id="alice") + finally: + finish.set() + worker.join(5) + provider.reset() + assert not worker.is_alive() + assert not errors + + +def test_destroy_completion_does_not_remove_a_replacement_runtime(tmp_path): + provider, old_client, module = _make_provider_with_active_sandbox(tmp_path, "same-id") + old_info = provider._sandbox_infos["same-id"] + old_info.container_id = "old-generation" + replacement = module.SandboxInfo(sandbox_id="same-id", sandbox_url="http://new", container_id="new-generation") + new_client = MagicMock() + + def stop(info): + assert info is old_info + # A replacement published by a stale/external callback must survive the + # completion of destruction of the old immutable generation. + with provider._lock: + provider._sandbox_infos["same-id"] = replacement + provider._sandboxes["same-id"] = new_client + + provider._backend.destroy.side_effect = stop + try: + assert provider.destroy("same-id") is True + assert provider._sandbox_infos["same-id"] is replacement + assert provider._sandboxes["same-id"] is new_client + old_client.close.assert_called_once() + new_client.close.assert_not_called() + finally: + provider.reset() + + +@pytest.mark.parametrize("failed_state", ["active", "warm"]) +@pytest.mark.parametrize("journal_failure", [False, True], ids=["recorded", "record-write-failed"]) +def test_shutdown_retains_only_failed_resources_and_retries_them(tmp_path, monkeypatch, failed_state, journal_failure): + provider, client, module = _make_provider_with_active_sandbox(tmp_path, "active") + warm = module.SandboxInfo(sandbox_id="warm", sandbox_url="http://warm", container_id="warm-generation") + provider._warm_pool["warm"] = (warm, 1.0) + provider._warm_pool_identity["warm"] = ("alice", "thread") + provider._stop_idle_checker = MagicMock() + provider._stop_lease_renewal = MagicMock() + provider._ownership.close = MagicMock() + active = provider._sandbox_infos["active"] + active.container_id = "active-generation" + running = _observe_destroy_target(provider, active) + running["warm"] = warm + failed_id = failed_state + failed = False + store = provider._quarantine_store() + mark = store.mark + mark_calls = 0 + + def record(info): + nonlocal mark_calls + mark_calls += 1 + if mark_calls == 1: + assert info.sandbox_id in provider._local_teardown + assert provider._ownership.take(info.sandbox_id) is False, "cleanup record escaped the teardown lease" + if journal_failure: + raise PermissionError("record storage unavailable") + mark(info) + + monkeypatch.setattr(store, "mark", record) + + def stop(info): + nonlocal failed + if info.sandbox_id == failed_id and not failed: + failed = True + raise RuntimeError("temporary stop failure") + running.pop(info.sandbox_id) + + provider._backend.destroy.side_effect = stop + try: + with pytest.raises(RuntimeError, match="shutdown"): + provider.shutdown() + assert provider._shutdown_called is False + provider._ownership.close.assert_not_called() + assert provider._cleanup_retry_ids == {failed_id} + assert store.contains(running[failed_id]) is (not journal_failure) + assert provider._backend.destroy.call_count == 2, "one failed resource skipped the others" + provider.shutdown() + assert provider._backend.destroy.call_count == 3 + assert provider._backend.destroy.call_args.args[0].sandbox_id == failed_id + assert not provider._sandboxes and not provider._warm_pool + provider._ownership.close.assert_called_once() + client.close.assert_called_once() + provider.shutdown() + assert provider._backend.destroy.call_count == 3 + finally: + provider.reset() + + +def test_shutdown_retry_still_closes_unfinished_executions(tmp_path): + from deerflow.sandbox.lease import get_sandbox_lease_manager + + provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "unfinished") + info = provider._sandbox_infos["unfinished"] + info.container_id = "generation-one" + running = _observe_destroy_target(provider, info) + manager = get_sandbox_lease_manager(provider) + manager.retain("unfinished-run", info.sandbox_id, thread_id="thread", user_id="alice") + + def stop(target): + if provider._backend.destroy.call_count == 1: + raise RuntimeError("stop failed") + running.pop(target.sandbox_id) + + provider._backend.destroy.side_effect = stop + try: + with pytest.raises(RuntimeError, match="shutdown"): + provider.shutdown() + assert manager.binding_for("unfinished-run") == info.sandbox_id + provider.shutdown() + assert provider._backend.destroy.call_count == 2 + assert not running and not provider._cleanup_retry_ids + finally: + manager.close() + provider.reset() + + def test_shutdown_keeps_aio_warm_entries_owned_when_idle_checker_stop_times_out(tmp_path): """A failed reaper join must leave AIO warm entries available to a retry.""" aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider") @@ -3301,7 +3673,8 @@ def test_shutdown_keeps_aio_warm_entries_owned_when_idle_checker_stop_times_out( provider._warm_pool_identity = {"warm-retry": ("default", "thread-retry")} provider._stop_idle_checker = MagicMock(side_effect=[RuntimeError("reaper still alive"), None]) provider._stop_lease_renewal = MagicMock() - provider._destroy_warm_entry = MagicMock() + provider._destroy_warm_entry = MagicMock(wraps=provider._destroy_warm_entry) + provider._backend.destroy = MagicMock() provider._ownership.close = MagicMock() with pytest.raises(RuntimeError, match="reaper still alive"): @@ -3338,7 +3711,8 @@ def test_shutdown_keeps_aio_warm_entries_owned_when_lease_renewal_stop_times_out provider._warm_pool = {"warm-renewal-retry": (warm_info, 1.0)} provider._warm_pool_identity = {"warm-renewal-retry": ("default", "thread-retry")} provider._stop_idle_checker = MagicMock() - provider._destroy_warm_entry = MagicMock() + provider._destroy_warm_entry = MagicMock(wraps=provider._destroy_warm_entry) + provider._backend.destroy = MagicMock() provider._ownership.close = MagicMock() renewal_thread = MagicMock() diff --git a/backend/tests/test_sandbox_provider_lifecycle.py b/backend/tests/test_sandbox_provider_lifecycle.py index 2afb455c27..37b7d03db1 100644 --- a/backend/tests/test_sandbox_provider_lifecycle.py +++ b/backend/tests/test_sandbox_provider_lifecycle.py @@ -12,6 +12,9 @@ exit, so tests never leak a provider into one another. import threading import time +from unittest.mock import Mock + +import pytest import deerflow.sandbox.sandbox_provider as sandbox_provider from deerflow.sandbox.sandbox import Sandbox @@ -291,3 +294,42 @@ def test_losing_cold_start_racer_shuts_down_its_orphan(monkeypatch): assert all(inst.shutdown_calls == 1 for inst in losers) finally: sandbox_provider.reset_sandbox_provider() + + +def test_failed_shutdown_remains_reachable_with_its_execution_manager(): + from deerflow.sandbox.lease import get_sandbox_lease_manager + + provider = ShutdownSandboxProvider() + provider.shutdown = Mock(side_effect=[RuntimeError("cleanup failed"), None]) + sandbox_provider.set_sandbox_provider(provider) + manager = get_sandbox_lease_manager(provider) + try: + with pytest.raises(RuntimeError, match="cleanup failed"): + sandbox_provider.shutdown_sandbox_provider() + assert sandbox_provider.get_initialized_sandbox_provider() is provider + assert get_sandbox_lease_manager(provider) is manager + sandbox_provider.shutdown_sandbox_provider() + assert provider.shutdown.call_count == 2 + assert sandbox_provider.get_initialized_sandbox_provider() is None + finally: + sandbox_provider.reset_sandbox_provider() + + +def test_failed_shutdown_does_not_replace_a_concurrently_installed_provider(): + from deerflow.sandbox.lease import discard_sandbox_lease_manager + + old, replacement = ShutdownSandboxProvider(), ShutdownSandboxProvider() + + def stop(): + sandbox_provider.set_sandbox_provider(replacement) + raise RuntimeError("old cleanup failed") + + old.shutdown = Mock(side_effect=stop) + sandbox_provider.set_sandbox_provider(old) + try: + with pytest.raises(RuntimeError, match="old cleanup failed"): + sandbox_provider.shutdown_sandbox_provider() + assert sandbox_provider.get_initialized_sandbox_provider() is replacement + finally: + sandbox_provider.reset_sandbox_provider() + discard_sandbox_lease_manager(old) diff --git a/docs/experiments/aio-teardown-retry-20261017/README.md b/docs/experiments/aio-teardown-retry-20261017/README.md new file mode 100644 index 0000000000..a15b3feacd --- /dev/null +++ b/docs/experiments/aio-teardown-retry-20261017/README.md @@ -0,0 +1,185 @@ +# AIO teardown retry verification + +## Scope and revisions + +This report verifies recovery after a failed AIO stop, independently of the +opt-in prewarming work in [#6998](https://github.com/bytedance/deer-flow/pull/6998). +The existing behavior also reproduces without prewarming enabled or installed. + +- Baseline: `34316550dffbe8fbaed5678a6be87c5d3c65155b`. +- Initial teardown fix and Docker probe revision: `997cb9bc30a0aec5804d1cd55bda3b87f36f5c31`. +- Environment: macOS arm64, Python 3.12.13, Docker client/server 29.7.2. +- Image: `enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0`. +- Local Docker image ID: `sha256:6328d7fd2f0ff0b4c147c3d05b3df1ce331f4a482eb6e550ecd64ed1fcf906e7` + (an image ID, not a portable registry manifest digest). + +[evidence.json](evidence.json) contains allowlisted observations and production +source hashes. Host paths, account names, random container prefixes, container +IDs, Pod identifiers, and raw exception messages are omitted. The local raw +observations were checked against the production source hashes from that initial +probe revision. + +## Method + +[probe.py](probe.py) constructs a complete typed production AppConfig and uses +the real AioSandboxProvider. Each case has a separate process, home directory, +and randomly generated container prefix. No model is configured or called. + +The fault is **one injected TimeoutExpired before stopping a particular test +container**. This leaves a real container running. Docker inspection, subsequent +stops, and cleanup use the real backend. This is controlled fault injection, +not a claim about production failure frequency or performance. + +The replacement case removes the old container after the injected failure, +creates a new generation under the same logical ID, and checks whether retry +preserves it. The shutdown cases also include an unaffected resource, so a +failure cannot silently skip the rest of the cleanup. + +## Observed results + +| Scenario | Baseline | Candidate | +| --- | --- | --- | +| Explicit destroy | First stop raises; second destroy returns True without another stop; target still exists | Generation is fenced; second destroy stops it; absence confirmed; record retired | +| Active shutdown failure | First call reports success; failed container remains; singleton detached; second call does nothing | First call raises and retains provider; second call retries only the failed resource and completes | +| Warm shutdown failure | Failed warm resource loses tracking; second call does nothing | Failed warm resource remains retryable and is removed on the second call | +| Shutdown with unfinished execution | Failed resource remains after both calls | Execution binding survives the first failure; full shutdown retry still removes the resource | +| Replacement under the same logical ID | Second destroy reports True without identifying the old cleanup intent | Retry returns False and preserves the observed replacement and the old fence | + +All five candidate scenarios matched their expected observations. Explicit +recovery makes two main-container stop attempts; partial shutdown recovery +makes three in total: one failure, one unaffected resource, one retry. The +replacement retry makes no additional stop attempt against the replacement. +All owned test containers were removed during final probe cleanup. + +Normalized observations from the explicit case (not verbatim raw logs): + +```text +baseline: first=TimeoutExpired; after_first: absent=false, quarantined=false, stops=1 +baseline: second=True; after_second: absent=false, quarantined=false, stops=1 +candidate: first=TimeoutExpired; after_first: absent=false, quarantined=true, stops=1 +candidate: second=True; after_second: absent=true, quarantined=false, stops=2 +``` + +## Related Docker error format + +Real Docker 29.7.2 returned this exact missing-network shape (resource name +normalized here): + +```text +Error response from daemon: network not found +exit status 1 +``` + +The strict parser previously rejected the second line and prevented cleanup +completion even after the container was absent. Three new regressions failed +before the parser fix. The fix accepts only this optional terminal line; +wrong targets, different exit codes, and additional diagnostics remain errors. + +## Initial teardown verification + +| Check | Result | +| --- | --- | +| Seven relevant offline test files | 633 passed, 3 live tests deselected | +| Strict blocking-I/O suite | 256 passed, 1 skipped | +| `make lint` | Passed, including Ruff formatting | +| Final standard four-shard `make test` | 32,407 passed, 564 skipped, **3 failed**, exit code 2 | +| Clean baseline standard `make test` | 32,384 passed, 565 skipped, **3 failed** | + +The final full-suite failures match the baseline: two frontend/AGENTS.md +hard-budget assertions (32,805 bytes against 32,768) and the native Cargo +acceptance case `unit-with-empty-doc`, affected by inherited GNU Make jobserver +flags. The full suite is **not green**. Those files are unchanged by this fix. + +An earlier full run also had one unchanged Docker orphan test miss `multi00`. +Its isolated rerun passed all three tests; the baseline's two-shard rerun also +passed; the final full run did not repeat the failure. Shared-prefix cleanup +between shards is a possible explanation, not a proven diagnosis. + +An additional restricted-network live test failed with HTTPS CONNECT 403 +(281 passed, 1 failed in that invocation). It is not counted as successful +network acceptance, and no network policy was relaxed. The offline selection +and the controlled Docker teardown probes above do not require external traffic. + +## Reproduction + +Prepare the selected checkout's own backend virtual environment with +`uv sync --locked --extra postgres` and make the stated AIO image available. +Run the same probe separately against the baseline and the fix, using each +checkout's interpreter. Keep the probe path absolute when selecting a different +checkout: + +```bash +REPO_ROOT="$PWD" +PROBE="$REPO_ROOT/docs/experiments/aio-teardown-retry-20261017/probe.py" +RESULTS_DIR="$(mktemp -d)" +"$REPO_ROOT/backend/.venv/bin/python" "$PROBE" \ + --root "$REPO_ROOT" --output "$RESULTS_DIR" +``` + +`--case explicit`, `shutdown-active`, `shutdown-warm`, `shutdown-held`, or +`replacement` runs one scenario. A zero exit means the probe ran and its final +container cleanup completed; compare the observations above to determine +behavioral acceptance. Generated outputs are local raw diagnostics and can +contain paths and resource IDs; this committed report includes only allowlisted +observations. + +The scoped offline command, from `backend/`, was: + +```bash +uv run --no-sync pytest -m 'not live' -q --tb=short \ + tests/test_aio_sandbox_provider.py tests/test_sandbox_orphan_reconciliation.py \ + tests/test_sandbox_provider_lifecycle.py tests/test_sandbox_quarantine.py \ + tests/test_sandbox_leases.py tests/test_sandbox_lease_rebind_cancellation.py \ + tests/test_aio_sandbox_local_backend.py +``` + +Full backend tests used the standard `make test` with Python 3.12 and Node 24 on +PATH, `NO_COLOR` unset, and no `UV_NO_SYNC` override. No acceptance assertions +were weakened. Kubernetes, Apple Container, and Redis multi-Gateway deployments +were not exercised as live environments; their contracts retain existing unit +coverage. No new authorization, prewarming, or configuration behavior is claimed. + +## Replacement admission follow-up + +[Review #5557122845](https://github.com/bytedance/deer-flow/pull/7001#discussion_r4302123092) +identified a regression in the initial fix: after a failed stop, a healthy new +generation with the same logical ID could be adopted into the warm pool, then +promoted without clearing the old in-memory retry marker. Acquisition returned +an ID, but cached client lookup failed; the execution lease manager therefore +raised `SandboxClientUnavailableError`. + +Warm-to-active promotion now clears that obsolete marker in the same metadata +critical section as client registration, after reuse and ownership checks. If +the warm entry changes during those checks, promotion defers rather than +admitting an unvalidated entry. Generation-bound quarantine records are retained. + +Eight offline regressions cover: + +- Direct discovery and orphan adoption, each through the provider and the real + execution lease manager. All four cases also release and reacquire the new + generation and verify that no extra stop occurs. +- Retaining the retry fence for quarantined and unknown generations. +- Retaining that fence when either unsafe entry replaces the validated warm + entry during ownership publication. + +Before the follow-up, four cases failed and four passed; afterward all eight +passed. These tests use real provider, quarantine and lease-manager methods with +mocked backend and SDK transport. The original Docker results above remain tied +to their recorded revision; Docker was unavailable for this follow-up. + +Follow-up validation against the working tree based on `673304c7`: + +| Check | Result | +| --- | --- | +| Eight new regression cases | 8 passed | +| Seven relevant offline files | 641 passed, 3 live tests deselected | +| Strict blocking-I/O suite, before and after | 256 passed, 1 skipped in each run | +| `make format`, `make lint`, frontend `pnpm check` | Passed | +| Agent guidance size check | 0 errors, 0 warnings | +| Standard `make test`, before follow-up | 32,520 passed, 565 skipped, 1 failed | +| Standard `make test`, after follow-up | 32,528 passed, 565 skipped, 1 failed | + +Both full runs fail only the unchanged native Cargo acceptance case +`unit-with-empty-doc`, already observed in the initial local verification. Its +isolated run without inherited Make flags passes. The full local suite is not +green; there are no additional failures after the follow-up. diff --git a/docs/experiments/aio-teardown-retry-20261017/evidence.json b/docs/experiments/aio-teardown-retry-20261017/evidence.json new file mode 100644 index 0000000000..fea402a183 --- /dev/null +++ b/docs/experiments/aio-teardown-retry-20261017/evidence.json @@ -0,0 +1,347 @@ +{ + "schema_version": 1, + "baseline_revision": "34316550dffbe8fbaed5678a6be87c5d3c65155b", + "fix_revision": "997cb9bc30a0aec5804d1cd55bda3b87f36f5c31", + "environment": { + "os": "macOS", + "architecture": "arm64", + "python": "3.12.13", + "docker_client": "29.7.2", + "docker_server": "29.7.2", + "aio_image": "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0", + "local_image_id": "sha256:6328d7fd2f0ff0b4c147c3d05b3df1ce331f4a482eb6e550ecd64ed1fcf906e7" + }, + "method": { + "stop_fault": "One injected TimeoutExpired for an owned test container; inspection and later stops use real Docker", + "models_called": 0, + "redaction": "Allowlisted fields only; no host paths, account names, raw errors, container prefixes or resource IDs" + }, + "candidate_production_sha256": { + "backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py": "09657b79e1bd6ca272945018ad8bc085635e26538a1b2bb330a19af5303794e7", + "backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py": "7d6af6d83f52829c8be03d3451a2db3f34c6b0293e6d10827165909d8b8190f1", + "backend/packages/harness/deerflow/sandbox/sandbox_provider.py": "de39a17853c5275fe8d2735427151f5859196c533b6dc807d3a4196589329cee" + }, + "scenarios": [ + { + "case": "explicit", + "baseline": { + "first": { + "raised": "TimeoutExpired" + }, + "second": { + "returned": true + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 1, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 1, + "owned_container_count": 1 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 1 + }, + "candidate": { + "first": { + "raised": "TimeoutExpired" + }, + "second": { + "returned": true + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": true, + "singleton_retained": false, + "stop_attempts": 1, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": true, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 2, + "owned_container_count": 0 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 0 + } + }, + { + "case": "shutdown-active", + "baseline": { + "first": { + "returned": null + }, + "second": { + "returned": null + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 2, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 2, + "owned_container_count": 1 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 2 + }, + "candidate": { + "first": { + "raised": "RuntimeError" + }, + "second": { + "returned": null + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": true, + "singleton_retained": true, + "stop_attempts": 2, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": true, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 3, + "owned_container_count": 0 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 0 + } + }, + { + "case": "shutdown-warm", + "baseline": { + "first": { + "returned": null + }, + "second": { + "returned": null + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 2, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 2, + "owned_container_count": 1 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 2 + }, + "candidate": { + "first": { + "raised": "RuntimeError" + }, + "second": { + "returned": null + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": true, + "singleton_retained": true, + "stop_attempts": 2, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": true, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 3, + "owned_container_count": 0 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 0 + } + }, + { + "case": "shutdown-held", + "baseline": { + "first": { + "returned": null + }, + "second": { + "returned": null + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 2, + "execution_still_bound": true, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 2, + "owned_container_count": 1 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 2 + }, + "candidate": { + "first": { + "raised": "RuntimeError" + }, + "second": { + "returned": null + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": true, + "singleton_retained": true, + "stop_attempts": 2, + "execution_still_bound": true, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": true, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 3, + "owned_container_count": 0 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 0 + } + }, + { + "case": "replacement", + "baseline": { + "first": { + "raised": "TimeoutExpired" + }, + "second": { + "returned": true + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 1, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": false, + "quarantined": false, + "singleton_retained": false, + "stop_attempts": 1, + "owned_container_count": 1 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 2, + "replacement_survived": true + }, + "candidate": { + "first": { + "raised": "TimeoutExpired" + }, + "second": { + "returned": false + }, + "final_shutdown": { + "returned": null + }, + "after_first": { + "bad_absent": false, + "quarantined": true, + "singleton_retained": false, + "stop_attempts": 1, + "owned_container_count": 1 + }, + "after_second": { + "bad_absent": false, + "quarantined": true, + "singleton_retained": false, + "stop_attempts": 1, + "owned_container_count": 1 + }, + "final_owned_container_count": 0, + "auxiliary_cleanup_error_count": 0, + "replacement_survived": true + } + } + ], + "checks": { + "scoped": { + "passed": 633, + "live_deselected": 3 + }, + "blocking_io": { + "passed": 256, + "skipped": 1 + }, + "full_backend": { + "passed": 32407, + "skipped": 564, + "failed": 3, + "exit_code": 2 + }, + "baseline_full_backend": { + "passed": 32384, + "skipped": 565, + "failed": 3 + }, + "lint": "passed" + }, + "known_full_failures": [ + "tests/test_agent_guidance_check.py::test_repository_guidance_chains_fit_hard_budget", + "tests/test_agent_guidance_check.py::test_repository_guidance_stays_below_hard_budgets_and_avoids_doc_indexes", + "tests/test_cargo_acceptance.py::test_native_cargo_output_reaches_parent_acceptance_through_tool_node[unit-with-empty-doc]" + ], + "limitations": [ + "Controlled single-host Docker fault injection; no live Kubernetes, Apple Container or Redis multi-Gateway run", + "One earlier unchanged orphan E2E failure did not repeat in isolated, baseline split, or final full reruns; cause not proven", + "Supplemental restricted-network live invocation: 281 passed, 1 failed with HTTPS CONNECT 403; not counted as successful network acceptance" + ] +} diff --git a/docs/experiments/aio-teardown-retry-20261017/probe.py b/docs/experiments/aio-teardown-retry-20261017/probe.py new file mode 100644 index 0000000000..961b427efd --- /dev/null +++ b/docs/experiments/aio-teardown-retry-20261017/probe.py @@ -0,0 +1,265 @@ +"""Real AIO containers, with one simulated stop timeout per isolated case. + +No models, network credentials or daemon-wide mutations. Cleanup is restricted +to immutable IDs created by this invocation. Run once per selected checkout. +""" + +from __future__ import annotations + +import argparse +import atexit +import hashlib +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import uuid + +parser = argparse.ArgumentParser() +parser.add_argument("--root", type=Path, required=True) +parser.add_argument("--output", type=Path, required=True) +parser.add_argument( + "--case", + choices=[ + "explicit", + "shutdown-active", + "shutdown-warm", + "shutdown-held", + "replacement", + ], +) +args = parser.parse_args() +args.output.mkdir(parents=True, exist_ok=True) + +if args.case is None: + results = [] + for case in [ + "explicit", + "shutdown-active", + "shutdown-warm", + "shutdown-held", + "replacement", + ]: + result = subprocess.run( + [ + sys.executable, + __file__, + "--root", + str(args.root), + "--output", + str(args.output), + "--case", + case, + ], + cwd=args.output, + capture_output=True, + text=True, + timeout=240, + ) + (args.output / f"{case}.log").write_text( + result.stdout + result.stderr, encoding="utf-8" + ) + print(case, result.returncode, result.stdout, flush=True) + if result.returncode: + print(result.stderr, flush=True) + raise SystemExit(result.returncode) + results.append(json.loads((args.output / f"{case}.json").read_text())) + (args.output / "results.json").write_text( + json.dumps(results, indent=2), encoding="utf-8" + ) + raise SystemExit(0) + +root = args.root.resolve() +backend = root / "backend" +sys.path[:0] = [ + str(backend), + str(backend / "packages/harness"), + str(backend / "packages/extension-api"), +] +case_dir = args.output / args.case +case_dir.mkdir(exist_ok=True) +os.environ["DEER_FLOW_HOME"] = str(case_dir / "home") +os.environ["DEER_FLOW_CONFIG_PATH"] = str(case_dir / "config.yaml") +(case_dir / ".env").write_text("", encoding="utf-8") +os.chdir(case_dir) + +# Import only after selecting the checkout and isolated configuration above. +from deerflow.config.app_config import AppConfig, set_app_config # noqa: E402 +from deerflow.community.aio_sandbox import aio_sandbox_provider as aio # noqa: E402 +from deerflow.sandbox import sandbox_provider as registry # noqa: E402 + +assert Path(aio.__file__).resolve().is_relative_to(root) +prefix = "df-retry-" + args.case + "-" + uuid.uuid4().hex[:8] +image = ( + "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0" +) +config = { + "models": [], + "tools": [], + "tool_groups": [], + "memory": {"enabled": False}, + "sandbox": { + "use": "deerflow.community.aio_sandbox:AioSandboxProvider", + "image": image, + "container_prefix": prefix, + "port": 18800, + "idle_timeout": 0, + "replicas": 4, + "ownership": {"type": "memory"}, + }, + "skills": {"path": str(root / "skills"), "container_path": "/mnt/skills"}, +} +# Construct the full typed production configuration, including network defaults. +set_app_config(AppConfig.model_validate(config)) +(case_dir / "config.yaml").write_text(json.dumps(config, indent=2), encoding="utf-8") +provider = aio.AioSandboxProvider() +saved = [] +original_stop = provider._backend._stop_container +attempts = [] +result = { + "case": args.case, + "checkout": str(root), + "imported": aio.__file__, + "prefix": prefix, + "revision": subprocess.check_output( + ["git", "-C", str(root), "rev-parse", "HEAD"], text=True + ).strip(), + "injection": "one simulated TimeoutExpired at real LocalContainerBackend._stop_container", + "models_called": 0, +} +result["source_sha256"] = { + str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in [ + Path(aio.__file__), + Path(aio.__file__).with_name("local_backend.py"), + Path(registry.__file__), + ] +} + + +def call(action): + try: + return {"returned": action()} + except Exception as exc: + return {"raised": type(exc).__name__, "message": str(exc)} + + +def inventory(): + proc = subprocess.run( + [ + "docker", + "ps", + "-a", + "--filter", + "name=" + prefix + "-", + "--format", + "{{.ID}} {{.Names}} {{.Status}}", + ], + capture_output=True, + text=True, + check=True, + ) + return proc.stdout.strip().splitlines() + + +try: + bad_id = provider.acquire("bad", user_id="probe") + bad = provider._sandbox_infos[bad_id] + saved.append(bad) + if args.case.startswith("shutdown"): + good_id = provider.acquire("good", user_id="probe") + good = provider._sandbox_infos[good_id] + saved.append(good) + provider.release(bad_id if args.case == "shutdown-warm" else good_id) + assert len(provider._warm_pool) == 1 + registry.set_sandbox_provider(provider) + if args.case == "shutdown-held": + from deerflow.sandbox.lease import get_sandbox_lease_manager + + manager = get_sandbox_lease_manager(provider) + manager.retain("unfinished", bad_id, thread_id="bad", user_id="probe") + + injected = False + + def fail_once(target): + global injected + if target in {info.container_id for info in saved}: + attempts.append(target) + if target == bad.container_id and not injected: + injected = True + raise subprocess.TimeoutExpired(["docker", "stop", target], 0.001) + original_stop(target) + + provider._backend._stop_container = fail_once + action = ( + registry.shutdown_sandbox_provider + if args.case.startswith("shutdown") + else lambda: provider.destroy(bad_id) + ) + result["first"] = call(action) + assert injected, "probe never entered the target stop operation" + result["after_first"] = { + "bad_absent": provider._backend.is_absent(bad_id), + "inventory": inventory(), + "quarantined": provider._quarantine_store().contains(bad), + "singleton_retained": registry.get_initialized_sandbox_provider() is provider, + "stop_attempts": len(attempts), + } + assert result["after_first"]["bad_absent"] is False + if args.case == "shutdown-held": + result["after_first"]["execution_still_bound"] = ( + manager.binding_for("unfinished") == bad_id + ) + if args.case == "replacement": + original_stop(bad.container_id) + replacement = provider._backend.create(None, bad_id) + saved.append(replacement) + result["replacement_generation"] = replacement.container_id + assert replacement.container_id != bad.container_id + result["second"] = call(action) + result["after_second"] = { + "bad_absent": provider._backend.is_absent(bad_id), + "inventory": inventory(), + "quarantined": provider._quarantine_store().contains(bad), + "singleton_retained": registry.get_initialized_sandbox_provider() is provider, + "stop_attempts": len(attempts), + } + if args.case == "replacement": + current = provider._backend.inspect_runtime(bad_id) + result["replacement_survived"] = ( + current is not None and current.container_id == replacement.container_id + ) + assert result["replacement_survived"] + result["attempted_generations"] = attempts +finally: + provider._backend._stop_container = original_stop + for info in reversed(saved): + try: + original_stop(info.container_id) + except Exception: + if not provider._backend.is_absent(info.sandbox_id): + raise + cleanup = call( + lambda: provider._backend.complete_absent_teardown(info.sandbox_id) + ) + if "raised" in cleanup: + result.setdefault("cleanup_errors", []).append(cleanup) + result["final_shutdown"] = call(provider.shutdown) + atexit.unregister(provider.shutdown) + registry._default_sandbox_provider = None + # This isolated process owns the signal registrations; restore originals. + signal.signal(signal.SIGTERM, provider._original_sigterm) + signal.signal(signal.SIGINT, provider._original_sigint) + if hasattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, provider._original_sighup) + result["final_inventory"] = inventory() + assert result["final_inventory"] == [], ( + "owned containers leaked after probe cleanup" + ) + +(args.output / f"{args.case}.json").write_text( + json.dumps(result, indent=2), encoding="utf-8" +) +print(json.dumps(result), flush=True)