github/deer-flow
1
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-10-26 05:06:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(sandbox): make failed AIO teardown safely retryable (#7001)

Browse Source 
* fix(sandbox): make failed AIO teardown safely retryable

* docs(sandbox): add sanitized teardown retry verification

* fix(sandbox): clear stale teardown fence on warm reclaim

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
Totoro 2026-10-18 09:57:12 +08:00 committed by GitHub
parent 6e0ab091e7
commit eae908c00d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 1412 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -836,6 +836,11 @@ Local AIO sandbox port allocation stops at TCP port 65535. If the remaining
ports from the configured starting port are occupied, allocation reports
that no port is available instead of attempting an out-of-range bind.
AIO teardown failures retain a generation-bound cleanup record. Partial shutdown
reports failure and can be retried; retries verify the runtime identity and absence
before completing cleanup. Healthy replacements with the same logical ID remain
reusable, including after background recovery into the warm pool.
See the [Sandbox Configuration Guide](backend/docs/CONFIGURATION.md#sandbox) to configure your preferred mode.
Managed async AIO runs can opt into bounded sandbox prewarming to overlap startup

8
backend/packages/harness/deerflow/community/aio_sandbox/AGENTS.md
View File

@ -42,6 +42,8 @@ Local absence matching compares Docker's fixed error scaffolding
case-insensitively — CLI v29 lowercases `Error: No such object:` to
`error: no such object:` — while the resource name stays byte-exact and the
whole message must match; Apple Container forms remain exact-match.
Missing-network errors may end with Docker CLI v29's exact `exit status 1` line;
other trailing diagnostics still fail closed.
Remote proofs use `/api/sandboxes/{id}/presence`, which inspects the Pod directly;
the ordinary status GET can return 404 merely because its Service is missing.
This covers runtimes without a recoverable generation, including Apple Container.
@ -73,6 +75,12 @@ under those same fences after confirmed absence, including a timed-out stop that
actually removed the container.
It also clears deterministic local proxy/network leftovers before recreation;
unknown cleanup errors retain the quarantine record.
Explicit stop failures use that same generation-bound quarantine before dropping
tracking. Failed record writes retain metadata and fence cached lookups in memory.
Warm-to-active promotion clears an obsolete in-memory fence under the metadata
lock, only for the entry that passed reuse checks; a changed entry must retry.
Partial shutdown raises and remains retryable; successful resources are not retried,
and cleanup infrastructure closes only after every owned teardown completes.
Broker capability failures are unknown, never confirmed negatives. Cache only
successful observations per provisioner endpoint/auth identity; a failed refresh

221
backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py
View File

@ -286,6 +286,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
self._acquire_epoch_counter = 0
self._acquire_inflight: dict[str, int] = {}
self._shutdown_called = False
self._cleanup_retry_ids: set[str] = set()
self._idle_checker_stop = threading.Event()
self._idle_checker_thread: threading.Thread | None = None
self._renewal_stop = threading.Event()
@ -1887,11 +1888,12 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
logger.info("Warm-pool sandbox %s was claimed for teardown while publishing ownership; not reclaiming it", sandbox_id)
return None
self._assert_warm_identity_available_locked(sandbox_id, key)
warm_item = self._warm_pool.pop(sandbox_id, None)
if warm_item is None:
warm_item = self._warm_pool.get(sandbox_id)
if warm_item is None or warm_item[0] is not info:
# A different entry has not passed the reuse checks above.
return None
self._warm_pool.pop(sandbox_id)
self._warm_pool_identity.pop(sandbox_id, None)
info, _ = warm_item
sandbox = AioSandbox(
id=sandbox_id,
base_url=info.sandbox_url,
@ -1901,6 +1903,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
)
self._sandboxes[sandbox_id] = sandbox
self._sandbox_infos[sandbox_id] = info
getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id)
self._active_sandbox_identity[sandbox_id] = key
self._last_activity[sandbox_id] = time.time()
self._thread_sandboxes[key] = sandbox_id
@ -1980,6 +1983,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
self._warm_pool_identity.pop(info.sandbox_id, None)
self._sandboxes[info.sandbox_id] = sandbox
self._sandbox_infos[info.sandbox_id] = info
getattr(self, "_cleanup_retry_ids", set()).discard(info.sandbox_id)
self._active_sandbox_identity[info.sandbox_id] = key
self._last_activity[info.sandbox_id] = time.time()
self._thread_sandboxes[key] = info.sandbox_id
@ -2026,6 +2030,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
self._warm_pool_identity.pop(sandbox_id, None)
self._sandboxes[sandbox_id] = sandbox
self._sandbox_infos[sandbox_id] = info
getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id)
self._active_sandbox_identity[sandbox_id] = key
self._last_activity[sandbox_id] = time.time()
if key is not None:
@ -2187,7 +2192,19 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
# release matters just as much — the container is probably still up,
# so a marker left behind would block its thread from re-acquiring it.
with self._held_teardown_lease(sandbox_id):
self._backend.destroy(entry)
try:
self._backend.destroy(entry)
except Exception:
if reason == "shutdown":
# Shutdown has no later idle tick. Persist before
# releasing the ownership fence or dropping metadata.
self._remember_failed_teardown(sandbox_id, entry)
with self._lock:
current = self._warm_pool.get(sandbox_id)
if current is not None and current[0] is entry:
self._warm_pool.pop(sandbox_id, None)
self._warm_pool_identity.pop(sandbox_id, None)
raise
except Exception as e:
if reason == "idle_timeout":
logger.error(f"Failed to destroy idle warm-pool sandbox {sandbox_id}: {e}")
@ -2498,41 +2515,58 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
reason="is quarantined and owned by a peer; its cleanup is deferred until that owner releases it",
)
with self._held_teardown_lease(sandbox_id):
if self._backend.is_absent(sandbox_id) is not True:
inspect_runtime = getattr(self._backend, "inspect_runtime", None)
if not callable(inspect_runtime):
raise RuntimeError(f"Runtime inspection is unavailable for sandbox {sandbox_id}")
current = inspect_runtime(sandbox_id)
if current is not None:
if not isinstance(current, SandboxInfo) or current.sandbox_id != sandbox_id:
raise RuntimeError(f"Runtime inspection returned invalid identity for sandbox {sandbox_id}")
# A replacement can share the logical ID while its Service
# is still unavailable. Do not stop that unquarantined
# runtime; create can repair its endpoint and validate the
# actual provisioning mode. The replacement itself proves
# the fenced generations are absent (logical IDs are unique
# per Pod/container), so prune only those stale records —
# a full retire() would also wipe a fence a peer may have
# just written for this live generation.
if not store.contains(current):
store.retire_replaced_generations(current)
return
# Inspection recovers runtime identity, but replacement
# cleanup must also remove its previous policy resources.
current.requires_replacement = True
self._backend.destroy(current)
# DELETE acceptance cannot prove the Pod/container vanished.
# Also recheck when it disappeared between our observations.
if self._backend.is_absent(sandbox_id) is not True:
raise SandboxPolicyReplacementDeferredError(
sandbox_id,
reason="is quarantined and its previous runtime is still present; the fence is retained",
)
self._backend.complete_absent_teardown(sandbox_id)
store.retire(info)
self._complete_quarantined_teardown(sandbox_id, store, retire_replaced=True)
finally:
self._finish_local_teardown(sandbox_id)
def _remember_failed_teardown(self, sandbox_id: str, info: SandboxInfo) -> None:
"""Fence an uncertain stop before dropping its retry metadata."""
with self._lock:
self._cleanup_retry_ids = getattr(self, "_cleanup_retry_ids", set()) | {sandbox_id}
info.requires_replacement = True
self._quarantine_store().mark(info)
def _complete_quarantined_teardown(self, sandbox_id: str, store: SandboxQuarantine, *, retire_replaced: bool) -> bool:
"""Retry under the caller's local reservation and ownership heartbeat."""
if self._backend.is_absent(sandbox_id) is not True:
inspect_runtime = getattr(self._backend, "inspect_runtime", None)
if not callable(inspect_runtime):
raise RuntimeError(f"Runtime inspection is unavailable for sandbox {sandbox_id}")
current = inspect_runtime(sandbox_id)
if current is not None:
if not isinstance(current, SandboxInfo) or current.sandbox_id != sandbox_id:
raise RuntimeError(f"Runtime inspection returned invalid identity for sandbox {sandbox_id}")
# A replacement can share the logical ID while its Service
# is still unavailable. Do not stop that unquarantined
# runtime; create can repair its endpoint and validate the
# actual provisioning mode. The replacement itself proves
# the fenced generations are absent (logical IDs are unique
# per Pod/container), so prune only those stale records —
# a full retire() would also wipe a fence a peer may have
# just written for this live generation.
if not store.contains(current):
if retire_replaced:
store.retire_replaced_generations(current)
with self._lock:
getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id)
return False
# Inspection recovers runtime identity, but replacement
# cleanup must also remove its previous policy resources.
current.requires_replacement = True
self._backend.destroy(current)
# DELETE acceptance cannot prove the Pod/container vanished.
# Also recheck when it disappeared between our observations.
if self._backend.is_absent(sandbox_id) is not True:
raise SandboxPolicyReplacementDeferredError(
sandbox_id,
reason="is quarantined and its previous runtime is still present; the fence is retained",
)
self._backend.complete_absent_teardown(sandbox_id)
store.retire(SandboxInfo(sandbox_id, ""))
with self._lock:
getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id)
return True
def _create_sandbox(self, thread_id: str | None, sandbox_id: str, *, user_id: str | None = None) -> str:
"""Create a new sandbox via the backend.
@ -2669,6 +2703,8 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
The sandbox instance if found, None otherwise.
"""
with self._lock:
if sandbox_id in getattr(self, "_cleanup_retry_ids", set()):
return None
sandbox = self._sandboxes.get(sandbox_id)
if sandbox is not None:
self._last_activity[sandbox_id] = time.time()
@ -2684,6 +2720,8 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
"""Return a cached client only for its recorded user/thread identity."""
key = self._thread_key(thread_id, user_id)
with self._lock:
if sandbox_id in getattr(self, "_cleanup_retry_ids", set()):
return None
if self._thread_sandboxes.get(key) != sandbox_id:
return None
if self._active_sandbox_identity.get(sandbox_id) != key:
@ -2828,32 +2866,40 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
with self._lock:
sandbox = self._sandboxes.get(sandbox_id)
info = self._sandbox_infos.get(sandbox_id)
if getattr(sandbox, "has_pending_session_creates", False) is True or getattr(sandbox, "requires_container_recycle", False) is True:
if info is None and (warm := self._warm_pool.get(sandbox_id)) is not None:
info = warm[0]
store = self._quarantine_store()
with self._lock:
retry_cleanup = sandbox_id in getattr(self, "_cleanup_retry_ids", set())
retry_cleanup = retry_cleanup or (info is None and store.contains(SandboxInfo(sandbox_id, "")))
if getattr(sandbox, "has_pending_session_creates", False) is True or getattr(sandbox, "requires_container_recycle", False) is True or (retry_cleanup and info is not None):
if info is None:
raise RuntimeError(f"Cannot quarantine sandbox {sandbox_id} without its runtime metadata")
# All destroy paths must persist before claiming/untracking. If
# storage fails, retain the dirty client and its renewal lease so
# a later idle/shutdown/explicit destroy retries this same fence.
info.requires_replacement = True
self._quarantine_store().mark(info)
store.mark(info)
# Claim before untracking. The reverse order loses the container on a
# refused claim: still running, and no longer in any of our maps, so
# nothing here would ever reap or reclaim it.
# Keep both the local reservation and ownership fence through the stop.
# A failed stop must preserve a generation-bound cleanup target before
# losing its in-process metadata, or a second destroy falsely succeeds.
if not self._claim_ownership(sandbox_id, for_destroy=True):
logger.warning("Refusing to destroy sandbox %s: owned by another instance", sandbox_id)
return False
sandbox, info, _ = self._remove_tracked_sandbox(sandbox_id)
if sandbox is not None:
# Defense-in-depth: close() already swallows its own errors; this
# guard only protects against a future close() that misbehaves, so
# host-side client cleanup can never block container destruction.
try:
sandbox.close()
except Exception as e:
logger.warning(f"Error closing sandbox {sandbox_id} during destroy: {e}")
if retry_cleanup:
# A previous failed stop may have lost its endpoint or been replaced
# under the same logical ID. Use the existing authoritative probes.
with self._held_teardown_lease(sandbox_id):
if info is not None:
self._remove_tracked_sandbox(sandbox_id, expected_info=info)
if sandbox is not None:
try:
sandbox.close()
except Exception as e:
logger.warning(f"Error closing sandbox {sandbox_id} during destroy: {e}")
return self._complete_quarantined_teardown(sandbox_id, store, retire_replaced=False)
if info:
# The marker must outlast the stop, not the TTL it was written with,
@ -2863,13 +2909,32 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
# the error still propagates out of the `with` (`shutdown()` logs per
# sandbox off it), it is just no longer this method's job to release.
with self._held_teardown_lease(sandbox_id):
self._backend.destroy(info)
try:
self._backend.destroy(info)
except Exception:
self._remember_failed_teardown(sandbox_id, info)
self._remove_tracked_sandbox(sandbox_id, expected_info=info)
if sandbox is not None:
try:
sandbox.close()
except Exception as e:
logger.warning(f"Error closing sandbox {sandbox_id} after failed destroy: {e}")
sandbox = None
raise
# A stale completion must not remove a replacement generation.
self._remove_tracked_sandbox(sandbox_id, expected_info=info)
logger.info(f"Destroyed sandbox {sandbox_id}")
else:
self._remove_tracked_sandbox(sandbox_id)
# No container to stop, so no teardown lease was held: clear the
# marker the claim above wrote, so an untracked id cannot leave a
# lease stuck in `del:`.
self._release_ownership(sandbox_id)
if sandbox is not None:
try:
sandbox.close()
except Exception as e:
logger.warning(f"Error closing sandbox {sandbox_id} during destroy: {e}")
return True
def reset(self) -> None:
@ -2901,28 +2966,56 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider):
# known stopped. If either bounded join fails, a retry must still own
# every active and warm entry.
with self._lock:
sandbox_ids = list(self._sandboxes.keys())
warm_items = list(self._warm_pool.items())
self._warm_pool.clear()
self._warm_pool_identity.clear()
pending = getattr(self, "_cleanup_retry_ids", set())
sandbox_ids = list(set(self._sandboxes) | pending)
warm_items = [(sid, item) for sid, item in self._warm_pool.items() if sid not in pending]
logger.info(f"Shutting down {len(sandbox_ids)} active + {len(warm_items)} warm-pool sandbox(es)")
failed: set[str] = set()
for sandbox_id in sandbox_ids:
try:
# Shutdown has stopped the maintenance workers and closes the
# entire provider. It must clean up even unfinished executions.
self._destroy_tracked(sandbox_id, still_reapable=lambda: True, force=True)
# entire provider. Retries must also clean unfinished executions
# whose client was already detached after a failed stop.
destroyed = self._destroy_tracked(sandbox_id, still_reapable=lambda: True, force=True)
if not destroyed:
failed.add(sandbox_id)
except Exception as e:
failed.add(sandbox_id)
logger.error(f"Failed to destroy sandbox {sandbox_id} during shutdown: {e}")
for sandbox_id, (info, _) in warm_items:
# Route through _destroy_warm_entry so the ownership claim and the
# container stop stay together, as on the idle path. Unconditional
# here: the entries were removed from `_warm_pool` under the lock
# above, so the pool-membership predicate the other callers use would
# refuse every one of them.
self._destroy_warm_entry(sandbox_id, info, reason="shutdown", still_reapable=lambda: True)
# The helper removes successful entries under its reservation and
# preserves failed ones in tracking or quarantine for the next attempt.
try:
destroyed = self._destroy_warm_entry(sandbox_id, info, reason="shutdown", still_reapable=lambda sid=sandbox_id, expected=info: self._warm_pool.get(sid, (None,))[0] is expected)
except Exception as e:
logger.error(f"Failed to destroy warm-pool sandbox {sandbox_id} during shutdown: {e}")
destroyed = False
if not destroyed:
failed.add(sandbox_id)
# A known peer owns its own cleanup. Unknown ownership or a failed stop
# must retain the record, manager and backend for an explicit retry.
for sandbox_id in list(failed):
epoch = self._acquire_epoch_of(sandbox_id)
try:
owner = self._ownership.owner(sandbox_id)
except OwnershipBackendError:
continue
if owner is not None and owner != self._owner_id:
self._forget_lost_sandbox(sandbox_id, expected_epoch=epoch)
with self._lock:
if sandbox_id not in self._sandboxes and sandbox_id not in self._warm_pool:
failed.remove(sandbox_id)
getattr(self, "_cleanup_retry_ids", set()).discard(sandbox_id)
if failed:
with self._lock:
self._shutdown_called = False
self._cleanup_retry_ids = getattr(self, "_cleanup_retry_ids", set()) | failed
raise RuntimeError(f"Failed to destroy {len(failed)} sandbox(es) during AIO shutdown; retry shutdown")
try:
self._ownership.close()

5
backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py
View File

@ -552,7 +552,7 @@ _DOCKER_NO_SUCH_CONTAINER_RE = re.compile(
re.IGNORECASE,
)
_DOCKER_NO_SUCH_NETWORK_RE = re.compile(
r"(?:error(?: response from daemon)?: )?(?:no such network: (?P<named>.+)|network (?P<embedded>.+) not found)",
r"(?:error(?: response from daemon)?: )?(?:no such network: (?P<named>.+)|network (?P<embedded>.+) not found)(?:\nexit status 1)?",
re.IGNORECASE,
)
@ -585,7 +585,8 @@ def _is_no_such_network_error(stderr: str, network_name: str) -> bool:
"""Accept Docker's complete missing-network response for this exact name.
Same contract as ``_is_no_such_container_error``: case-insensitive
scaffolding, byte-exact network name, full-message match.
scaffolding, byte-exact network name, full-message match. Docker CLI v29
may append exactly one ``exit status 1`` line; other diagnostics still fail.
"""
message = (stderr or "").strip()
match = _DOCKER_NO_SUCH_NETWORK_RE.fullmatch(message)

1
backend/packages/harness/deerflow/sandbox/AGENTS.md
View File

@ -18,6 +18,7 @@ cancellation before propagating it.
**Authorization gate** (`sandbox:execute`, RFC #4063 Phase 3): every sandbox-backed tool call passes through the gate in `deerflow/authz/sandbox_authz.py` - a binary `authorize(principal, "sandbox", "execute", target="*")` check before either reusing a persisted sandbox id or calling `provider.acquire`. Rechecking reuse is required because authorization config and user roles can change while the sandbox remains cached. Sync tool invocations call `authorize_sandbox_execution`; async tool invocations await `authorize_sandbox_execution_async` exactly once. A task-local `ContextVar` scopes that single decision across the complete composed tool invocation, including `ReadBeforeWriteMiddleware`'s pre-write inspection, tool body, and post-read mark; the value is copied into `asyncio.to_thread` workers. Authorization denial is converted to the normal error `ToolMessage` at the composed middleware boundary and is explicitly excluded from the gate's generic fail-open handlers. Async config loading and provider class discovery/import are offloaded before `aauthorize()` so reused sandbox calls do not hash config files or import custom modules on the event loop; provider construction remains on the running event loop because async providers may initialize loop-affine clients. The gate lives at the single tool initialization entry point (`ensure_sandbox_initialized` / `ensure_sandbox_initialized_async` in `tools.py`), while `SandboxMiddleware.before_agent` / `abefore_agent` apply the matching sync/async check to eager acquisition. Deny raises `SandboxAuthorizationError` (`sandbox/exceptions.py`), which propagates out of ordinary tool execution as a friendly error `ToolMessage` ("sandbox execution is not permitted for your role") - the eager path catches it and skips acquisition instead, deferring the deny to the first sandbox-touching tool call so both paths share the same semantics. Provider errors (authorization calls and provider resolution) follow `authorization.fail_closed` / `fail_open`; no readable `config.yaml` or `authorization.enabled: false` makes the gate a no-op (`safe_app_config` tolerates missing config). Gateway upload/artifact sync calls `try_acquire_sandbox_for_request` (`app/gateway/authz.py`), which gates, returns a request lease, and skips sync on deny while preserving the primary operation. Callers release after their last sandbox operation; artifacts request normal parking, uploads do not. Tests: `tests/test_sandbox_authorization.py` and `tests/blocking_io/test_sandbox_authorization.py`.
**Environment policy** (`sandbox/env_policy.py`): `build_sandbox_env()` scrubs inherited credential names via `is_blocked_env_name`, preserving benign vars (`PATH`, `HOME`, `LANG`, `VIRTUAL_ENV`). Explicit `sandbox.environment` injection survives scrubbing; request secrets override it. Local sandboxes validate keys at construction; generic and per-thread instances receive resolved config values verbatim. Local `bash` masks effective operator credentials and request secrets in output and command errors, preserving benign config values.
Holders stay bound through scope cleanup; AIO reserves idle teardown atomically.
Failed provider shutdown retains the singleton and lease manager for retry unless replaced concurrently.
Fork lookups reject invalidated clients; cancelled acquires leave borrowers a pending release.
**Implementations**:
- `LocalSandboxProvider` - Local filesystem execution. `acquire(thread_id)` returns a per-user/thread `LocalSandbox` (id `local:{user_id}:{thread_id}`) whose `path_mappings` resolve `/mnt/user-data` (file tools: `ls` lists only these three; search/read reject the root), its `{workspace,uploads,outputs}`, and `/mnt/acp-workspace` to that thread's host directories, so the public `Sandbox` API honours the `/mnt/user-data` contract uniformly with AIO. `acquire()` / `acquire(None)` keeps the legacy generic singleton (id `local`) for callers without a thread context. Per-thread sandboxes are held in an LRU cache (default 256 entries) guarded by a `threading.Lock`. Shared runs use category mappings; a policy-scoped run replaces them with one `/mnt/skills` root mapping to the coherent thread view, so structured file tools resolve through one managed boundary. This is not a host filesystem security boundary: an enabled host `bash` subprocess can use canonical paths without `PathMapping`, so `supports_agent_skill_isolation` is dynamic and explicit Agent policies fail closed while host bash is enabled; unrestricted runs then get the shared mappings, never a leftover thread view. Host-to-virtual output masking scans dynamic per-user/per-thread roots directly instead of compiling path-specific regexes, so evicted thread IDs do not remain in Python's global regex caches; a separate 256-entry root cache prevents repeated `realpath()` walks for every glob/grep match while bounding dynamic-path retention, and only the small process-stable skill/integration source set uses a bounded compiled cache. The shared `path_patterns.py` tail stops at `:`, so `$PATH`-style lists mask every entry; a mount symlink resolving outside all mounts keeps its mount path. On Windows, Git Bash/MSYS argument-conversion exclusions are limited to safe non-root virtual path prefixes; do not restore a blanket conversion disable, because host-native CLI launchers need normal MSYS path conversion for their own installation paths.

21
backend/packages/harness/deerflow/sandbox/sandbox_provider.py
View File

@ -253,11 +253,11 @@ def reset_sandbox_provider() -> None:
def shutdown_sandbox_provider() -> None:
"""Shutdown and reset the sandbox provider.
"""Shutdown the provider, keeping failed cleanup reachable for retry.
This properly shuts down the provider (releasing all sandboxes)
before clearing the singleton. Call this when the application
is shutting down or when you need to completely reset the sandbox system.
A provider installed concurrently takes precedence over a failed instance.
Provider callbacks run outside the registry lock; discard execution leases
only after successful shutdown.
"""
global _default_sandbox_provider
# Detach the reference under the lock, then run the (potentially slow)
@ -268,8 +268,17 @@ def shutdown_sandbox_provider() -> None:
if provider is not None and hasattr(provider, "shutdown"):
from deerflow.sandbox.lease import discard_sandbox_lease_manager
discard_sandbox_lease_manager(provider)
provider.shutdown()
try:
provider.shutdown()
except BaseException:
# Keep a failed cleanup reachable for a later shutdown call, without
# replacing a new provider installed while this callback ran.
with _provider_lock:
if _default_sandbox_provider is None:
_default_sandbox_provider = provider
raise
else:
discard_sandbox_lease_manager(provider)
def set_sandbox_provider(provider: SandboxProvider) -> None:

9
backend/tests/test_aio_sandbox_local_backend.py
View File

@ -2254,7 +2254,7 @@ def test_absent_teardown_preserves_pending_port_on_unknown_network_cleanup_error
assert backend._pending_cleanup_ports == {"old-generation": ("existing", 18080)}
@pytest.mark.parametrize("error_style", ["legacy", "lowercase-scaffolding", "uppercase-scaffolding"])
@pytest.mark.parametrize("error_style", ["legacy", "lowercase-scaffolding", "uppercase-scaffolding", "cli-exit-status"])
@pytest.mark.parametrize("sandbox_id", ["existing", "Mixed-Case"])
def test_absent_teardown_accepts_only_exact_missing_networks(monkeypatch, error_style, sandbox_id):
backend = _restricted_backend()
@ -2269,6 +2269,7 @@ def test_absent_teardown_accepts_only_exact_missing_networks(monkeypatch, error_
"legacy": ("Error: No such container: {}", "Error response from daemon: network {} not found"),
"lowercase-scaffolding": ("error: no such object: {}", "error: network {} not found"),
"uppercase-scaffolding": ("ERROR: NO SUCH OBJECT: {}", "ERROR: NETWORK {} NOT FOUND"),
"cli-exit-status": ("Error: No such container: {}", "Error response from daemon: network {} not found\nexit status 1\n"),
}
container_error, network_error = error_formats[error_style]
@ -2296,6 +2297,12 @@ def test_absent_teardown_accepts_only_exact_missing_networks(monkeypatch, error_
("error: no such network: sandbox-net", True),
("No such network: sandbox-net", True),
("network sandbox-net not found", True),
("Error response from daemon: network sandbox-net not found\nexit status 1\n", True),
("Error response from daemon: network sandbox-net-extra not found\nexit status 1", False),
("Error response from daemon: network Sandbox-Net not found\nexit status 1", False),
("Error response from daemon: network sandbox-net not found\nexit status 2", False),
("Error response from daemon: network sandbox-net not found\npermission denied\nexit status 1", False),
("Error response from daemon: network sandbox-net not found\nexit status 1\nunknown failure", False),
("error: network sandbox-net-extra not found", False),
("error: network Sandbox-Net not found", False),
("error: no such network: sandbox-other", False),

378
backend/tests/test_aio_sandbox_provider.py
View File

@ -7,6 +7,7 @@ import importlib
import os
import stat
import threading
import time
from types import SimpleNamespace
from unittest.mock import ANY, AsyncMock, MagicMock, patch
@ -3286,6 +3287,377 @@ def test_destroy_swallows_close_errors_and_still_destroys_backend(tmp_path, capl
provider._backend.destroy.assert_called_once()
def _observe_destroy_target(provider, info):
running = {info.sandbox_id: info}
provider._backend.is_absent = MagicMock(side_effect=lambda sid: sid not in running)
provider._backend.inspect_runtime = MagicMock(side_effect=running.get)
provider._backend.complete_absent_teardown = MagicMock()
return running
def test_destroy_failure_keeps_durable_runtime_target_for_cleanup_retry(tmp_path):
provider, sandbox, _ = _make_provider_with_active_sandbox(tmp_path, "retry-stop")
info = provider._sandbox_infos["retry-stop"]
info.container_id = "generation-one"
running = _observe_destroy_target(provider, info)
def stop(target):
if provider._backend.destroy.call_count == 1:
raise RuntimeError("stop failed")
running.pop(target.sandbox_id)
provider._backend.destroy.side_effect = stop
try:
with pytest.raises(RuntimeError, match="stop failed"):
provider.destroy(info.sandbox_id)
assert provider._quarantine_store().contains(info)
assert info.sandbox_id not in provider._sandboxes
sandbox.close.assert_called_once()
assert provider.destroy(info.sandbox_id) is True
assert provider._backend.destroy.call_count == 2
assert all(call.args[0] is info for call in provider._backend.destroy.call_args_list)
assert info.sandbox_id not in provider._sandboxes
assert info.sandbox_id not in provider._sandbox_infos
assert not provider._quarantine_store().contains(info)
sandbox.close.assert_called_once()
finally:
provider.reset()
def test_destroy_repeated_failure_never_turns_into_false_success(tmp_path):
provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "still-running")
_observe_destroy_target(provider, provider._sandbox_infos["still-running"])
provider._backend.destroy.side_effect = RuntimeError("stop still failing")
try:
for _ in range(2):
with pytest.raises(RuntimeError, match="stop still failing"):
provider.destroy("still-running")
assert provider._backend.destroy.call_count == 2
finally:
provider.reset()
def test_destroy_retry_preserves_a_different_runtime_generation(tmp_path):
provider, _, module = _make_provider_with_active_sandbox(tmp_path, "replacement")
old = provider._sandbox_infos["replacement"]
old.container_id = "old-generation"
running = _observe_destroy_target(provider, old)
provider._backend.destroy.side_effect = RuntimeError("stop failed")
try:
with pytest.raises(RuntimeError, match="stop failed"):
provider.destroy("replacement")
new = module.SandboxInfo("replacement", "http://new", container_id="new-generation")
running["replacement"] = new
for _ in range(2):
assert provider.destroy("replacement") is False
assert provider._backend.destroy.call_count == 1
assert running["replacement"] is new
assert provider._quarantine_store().contains(old)
assert not provider._quarantine_store().contains(new)
finally:
provider.reset()
@pytest.mark.parametrize("admission", ["discovery", "orphan-adoption"])
@pytest.mark.parametrize("via_lease", [False, True], ids=["provider", "lease-manager"])
def test_failed_destroy_allows_replacement_acquisition(failure_recovery_lifecycle, admission, via_lease):
from deerflow.sandbox.lease import SandboxLeaseManager
provider, _, _, old = failure_recovery_lifecycle
running = _observe_destroy_target(provider, old)
provider._backend.destroy.side_effect = RuntimeError("stop failed")
with pytest.raises(RuntimeError, match="stop failed"):
provider.destroy(old.sandbox_id)
replacement = type(old)(old.sandbox_id, old.sandbox_url, container_id="replacement-generation")
running[old.sandbox_id] = replacement
provider._backend.discover = MagicMock(return_value=replacement)
provider._backend.list_running = MagicMock(return_value=[replacement])
if admission == "orphan-adoption":
provider._reconcile_orphans()
assert provider._warm_pool[old.sandbox_id][0] is replacement
manager = SandboxLeaseManager(provider)
try:
if via_lease:
acquired = manager.acquire("new-execution", "thread-failure-recovery", user_id="alice")
else:
acquired = provider.acquire("thread-failure-recovery", user_id="alice")
assert acquired == old.sandbox_id
assert provider._sandbox_infos[acquired] is replacement
assert provider.get(acquired) is not None
assert provider.get_scoped(acquired, thread_id="thread-failure-recovery", user_id="alice") is not None
assert acquired not in provider._cleanup_retry_ids
assert provider._quarantine_store().contains(old)
assert not provider._quarantine_store().contains(replacement)
provider._backend.destroy.assert_called_once_with(old)
if via_lease:
manager.release("new-execution")
reacquired = manager.acquire("next-execution", "thread-failure-recovery", user_id="alice")
else:
provider.release(acquired)
reacquired = provider.acquire("thread-failure-recovery", user_id="alice")
assert reacquired == acquired
assert provider.get(reacquired) is not None
assert provider._sandbox_infos[reacquired] is replacement
provider._backend.destroy.assert_called_once_with(old)
finally:
manager.close()
@pytest.mark.parametrize("generation", ["container-generation-1", None], ids=["quarantined", "unknown"])
def test_warm_reclaim_keeps_failed_stop_fence_for_unsafe_generation(failure_recovery_lifecycle, generation):
provider, _, _, old = failure_recovery_lifecycle
_observe_destroy_target(provider, old)
provider._backend.destroy.side_effect = RuntimeError("stop failed")
with pytest.raises(RuntimeError, match="stop failed"):
provider.destroy(old.sandbox_id)
candidate = type(old)(old.sandbox_id, old.sandbox_url, container_id=generation)
provider._warm_pool[old.sandbox_id] = (candidate, time.time())
provider._warm_pool_identity[old.sandbox_id] = None
provider_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
with pytest.raises(provider_mod.SandboxPolicyReplacementDeferredError):
provider._reclaim_warm_pool_sandbox("thread-failure-recovery", old.sandbox_id, user_id="alice")
assert old.sandbox_id in provider._cleanup_retry_ids
assert provider._quarantine_store().contains(candidate)
assert provider.get(old.sandbox_id) is None
assert old.sandbox_id not in provider._sandboxes
@pytest.mark.parametrize("generation", ["container-generation-1", None], ids=["quarantined", "unknown"])
def test_warm_reclaim_does_not_clear_fence_if_entry_changes(failure_recovery_lifecycle, monkeypatch, generation):
provider, _, _, old = failure_recovery_lifecycle
_observe_destroy_target(provider, old)
provider._backend.destroy.side_effect = RuntimeError("stop failed")
with pytest.raises(RuntimeError, match="stop failed"):
provider.destroy(old.sandbox_id)
replacement = type(old)(old.sandbox_id, old.sandbox_url, container_id="replacement-generation")
provider._backend.list_running = MagicMock(return_value=[replacement])
provider._reconcile_orphans()
assert provider._warm_pool[old.sandbox_id][0] is replacement
unvalidated = type(old)(old.sandbox_id, old.sandbox_url, container_id=generation)
publish = provider._publish_ownership
def swap_during_publish(sandbox_id):
publish(sandbox_id)
with provider._lock:
provider._warm_pool[sandbox_id] = (unvalidated, time.time())
monkeypatch.setattr(provider, "_publish_ownership", swap_during_publish)
assert provider._reclaim_warm_pool_sandbox("thread-failure-recovery", old.sandbox_id, user_id="alice") is None
assert provider._warm_pool[old.sandbox_id][0] is unvalidated
assert old.sandbox_id in provider._cleanup_retry_ids
assert provider._quarantine_store().contains(unvalidated)
assert provider.get(old.sandbox_id) is None
assert old.sandbox_id not in provider._sandboxes
def test_failed_destroy_record_write_blocks_cached_reuse_until_retry(tmp_path, monkeypatch):
provider, client, _ = _make_provider_with_active_sandbox(tmp_path, "record-write")
info = provider._sandbox_infos["record-write"]
info.container_id = "generation-one"
provider._thread_sandboxes[("alice", "thread")] = info.sandbox_id
provider._active_sandbox_identity[info.sandbox_id] = ("alice", "thread")
running = _observe_destroy_target(provider, info)
def stop(target):
if provider._backend.destroy.call_count == 1:
raise RuntimeError("stop failed")
running.pop(target.sandbox_id)
provider._backend.destroy.side_effect = stop
store = provider._quarantine_store()
try:
with monkeypatch.context() as patcher:
patcher.setattr(store, "mark", MagicMock(side_effect=PermissionError("record unavailable")))
with pytest.raises(PermissionError, match="record unavailable"):
provider.destroy(info.sandbox_id)
assert provider._sandbox_infos[info.sandbox_id] is info
client.close.assert_not_called()
assert provider.get(info.sandbox_id) is None
assert provider.get_scoped(info.sandbox_id, thread_id="thread", user_id="alice") is None
assert provider.destroy(info.sandbox_id) is True
assert provider._backend.destroy.call_count == 2
assert not provider._cleanup_retry_ids
client.close.assert_called_once()
finally:
provider.reset()
def test_destroy_retry_needs_confirmed_absence_after_accepted_stop(tmp_path):
provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "terminating")
info = provider._sandbox_infos["terminating"]
info.container_id = "pod-uid"
running = _observe_destroy_target(provider, info)
provider._backend.destroy.side_effect = [RuntimeError("delete failed"), None]
try:
with pytest.raises(RuntimeError, match="delete failed"):
provider.destroy(info.sandbox_id)
with pytest.raises(RuntimeError, match="still present"):
provider.destroy(info.sandbox_id)
assert provider._quarantine_store().contains(info)
running.clear()
assert provider.destroy(info.sandbox_id) is True
assert provider._backend.destroy.call_count == 2
assert not provider._quarantine_store().contains(info)
finally:
provider.reset()
def test_destroy_keeps_reservation_while_stop_is_pending(tmp_path):
from deerflow.sandbox.lease import get_sandbox_lease_manager
provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "slow-stop")
info = provider._sandbox_infos["slow-stop"]
entered, finish = threading.Event(), threading.Event()
errors = []
def stop(_info):
entered.set()
assert finish.wait(5)
def destroy():
try:
provider.destroy("slow-stop")
except BaseException as exc:
errors.append(exc)
provider._backend.destroy.side_effect = stop
worker = threading.Thread(target=destroy)
worker.start()
try:
assert entered.wait(5)
assert provider._sandbox_infos.get("slow-stop") is info
assert "slow-stop" in provider._local_teardown
assert provider._lock.acquire(blocking=False), "container I/O held the metadata lock"
provider._lock.release()
with pytest.raises(RuntimeError, match="teardown"):
get_sandbox_lease_manager(provider).retain("late-holder", "slow-stop", thread_id="thread", user_id="alice")
finally:
finish.set()
worker.join(5)
provider.reset()
assert not worker.is_alive()
assert not errors
def test_destroy_completion_does_not_remove_a_replacement_runtime(tmp_path):
provider, old_client, module = _make_provider_with_active_sandbox(tmp_path, "same-id")
old_info = provider._sandbox_infos["same-id"]
old_info.container_id = "old-generation"
replacement = module.SandboxInfo(sandbox_id="same-id", sandbox_url="http://new", container_id="new-generation")
new_client = MagicMock()
def stop(info):
assert info is old_info
# A replacement published by a stale/external callback must survive the
# completion of destruction of the old immutable generation.
with provider._lock:
provider._sandbox_infos["same-id"] = replacement
provider._sandboxes["same-id"] = new_client
provider._backend.destroy.side_effect = stop
try:
assert provider.destroy("same-id") is True
assert provider._sandbox_infos["same-id"] is replacement
assert provider._sandboxes["same-id"] is new_client
old_client.close.assert_called_once()
new_client.close.assert_not_called()
finally:
provider.reset()
@pytest.mark.parametrize("failed_state", ["active", "warm"])
@pytest.mark.parametrize("journal_failure", [False, True], ids=["recorded", "record-write-failed"])
def test_shutdown_retains_only_failed_resources_and_retries_them(tmp_path, monkeypatch, failed_state, journal_failure):
provider, client, module = _make_provider_with_active_sandbox(tmp_path, "active")
warm = module.SandboxInfo(sandbox_id="warm", sandbox_url="http://warm", container_id="warm-generation")
provider._warm_pool["warm"] = (warm, 1.0)
provider._warm_pool_identity["warm"] = ("alice", "thread")
provider._stop_idle_checker = MagicMock()
provider._stop_lease_renewal = MagicMock()
provider._ownership.close = MagicMock()
active = provider._sandbox_infos["active"]
active.container_id = "active-generation"
running = _observe_destroy_target(provider, active)
running["warm"] = warm
failed_id = failed_state
failed = False
store = provider._quarantine_store()
mark = store.mark
mark_calls = 0
def record(info):
nonlocal mark_calls
mark_calls += 1
if mark_calls == 1:
assert info.sandbox_id in provider._local_teardown
assert provider._ownership.take(info.sandbox_id) is False, "cleanup record escaped the teardown lease"
if journal_failure:
raise PermissionError("record storage unavailable")
mark(info)
monkeypatch.setattr(store, "mark", record)
def stop(info):
nonlocal failed
if info.sandbox_id == failed_id and not failed:
failed = True
raise RuntimeError("temporary stop failure")
running.pop(info.sandbox_id)
provider._backend.destroy.side_effect = stop
try:
with pytest.raises(RuntimeError, match="shutdown"):
provider.shutdown()
assert provider._shutdown_called is False
provider._ownership.close.assert_not_called()
assert provider._cleanup_retry_ids == {failed_id}
assert store.contains(running[failed_id]) is (not journal_failure)
assert provider._backend.destroy.call_count == 2, "one failed resource skipped the others"
provider.shutdown()
assert provider._backend.destroy.call_count == 3
assert provider._backend.destroy.call_args.args[0].sandbox_id == failed_id
assert not provider._sandboxes and not provider._warm_pool
provider._ownership.close.assert_called_once()
client.close.assert_called_once()
provider.shutdown()
assert provider._backend.destroy.call_count == 3
finally:
provider.reset()
def test_shutdown_retry_still_closes_unfinished_executions(tmp_path):
from deerflow.sandbox.lease import get_sandbox_lease_manager
provider, _, _ = _make_provider_with_active_sandbox(tmp_path, "unfinished")
info = provider._sandbox_infos["unfinished"]
info.container_id = "generation-one"
running = _observe_destroy_target(provider, info)
manager = get_sandbox_lease_manager(provider)
manager.retain("unfinished-run", info.sandbox_id, thread_id="thread", user_id="alice")
def stop(target):
if provider._backend.destroy.call_count == 1:
raise RuntimeError("stop failed")
running.pop(target.sandbox_id)
provider._backend.destroy.side_effect = stop
try:
with pytest.raises(RuntimeError, match="shutdown"):
provider.shutdown()
assert manager.binding_for("unfinished-run") == info.sandbox_id
provider.shutdown()
assert provider._backend.destroy.call_count == 2
assert not running and not provider._cleanup_retry_ids
finally:
manager.close()
provider.reset()
def test_shutdown_keeps_aio_warm_entries_owned_when_idle_checker_stop_times_out(tmp_path):
"""A failed reaper join must leave AIO warm entries available to a retry."""
aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
@ -3301,7 +3673,8 @@ def test_shutdown_keeps_aio_warm_entries_owned_when_idle_checker_stop_times_out(
provider._warm_pool_identity = {"warm-retry": ("default", "thread-retry")}
provider._stop_idle_checker = MagicMock(side_effect=[RuntimeError("reaper still alive"), None])
provider._stop_lease_renewal = MagicMock()
provider._destroy_warm_entry = MagicMock()
provider._destroy_warm_entry = MagicMock(wraps=provider._destroy_warm_entry)
provider._backend.destroy = MagicMock()
provider._ownership.close = MagicMock()
with pytest.raises(RuntimeError, match="reaper still alive"):
@ -3338,7 +3711,8 @@ def test_shutdown_keeps_aio_warm_entries_owned_when_lease_renewal_stop_times_out
provider._warm_pool = {"warm-renewal-retry": (warm_info, 1.0)}
provider._warm_pool_identity = {"warm-renewal-retry": ("default", "thread-retry")}
provider._stop_idle_checker = MagicMock()
provider._destroy_warm_entry = MagicMock()
provider._destroy_warm_entry = MagicMock(wraps=provider._destroy_warm_entry)
provider._backend.destroy = MagicMock()
provider._ownership.close = MagicMock()
renewal_thread = MagicMock()

42
backend/tests/test_sandbox_provider_lifecycle.py
View File

@ -12,6 +12,9 @@ exit, so tests never leak a provider into one another.
import threading
import time
from unittest.mock import Mock
import pytest
import deerflow.sandbox.sandbox_provider as sandbox_provider
from deerflow.sandbox.sandbox import Sandbox
@ -291,3 +294,42 @@ def test_losing_cold_start_racer_shuts_down_its_orphan(monkeypatch):
assert all(inst.shutdown_calls == 1 for inst in losers)
finally:
sandbox_provider.reset_sandbox_provider()
def test_failed_shutdown_remains_reachable_with_its_execution_manager():
from deerflow.sandbox.lease import get_sandbox_lease_manager
provider = ShutdownSandboxProvider()
provider.shutdown = Mock(side_effect=[RuntimeError("cleanup failed"), None])
sandbox_provider.set_sandbox_provider(provider)
manager = get_sandbox_lease_manager(provider)
try:
with pytest.raises(RuntimeError, match="cleanup failed"):
sandbox_provider.shutdown_sandbox_provider()
assert sandbox_provider.get_initialized_sandbox_provider() is provider
assert get_sandbox_lease_manager(provider) is manager
sandbox_provider.shutdown_sandbox_provider()
assert provider.shutdown.call_count == 2
assert sandbox_provider.get_initialized_sandbox_provider() is None
finally:
sandbox_provider.reset_sandbox_provider()
def test_failed_shutdown_does_not_replace_a_concurrently_installed_provider():
from deerflow.sandbox.lease import discard_sandbox_lease_manager
old, replacement = ShutdownSandboxProvider(), ShutdownSandboxProvider()
def stop():
sandbox_provider.set_sandbox_provider(replacement)
raise RuntimeError("old cleanup failed")
old.shutdown = Mock(side_effect=stop)
sandbox_provider.set_sandbox_provider(old)
try:
with pytest.raises(RuntimeError, match="old cleanup failed"):
sandbox_provider.shutdown_sandbox_provider()
assert sandbox_provider.get_initialized_sandbox_provider() is replacement
finally:
sandbox_provider.reset_sandbox_provider()
discard_sandbox_lease_manager(old)

185
docs/experiments/aio-teardown-retry-20261017/README.md Normal file
View File

@ -0,0 +1,185 @@
# AIO teardown retry verification
## Scope and revisions
This report verifies recovery after a failed AIO stop, independently of the
opt-in prewarming work in [#6998](https://github.com/bytedance/deer-flow/pull/6998).
The existing behavior also reproduces without prewarming enabled or installed.
- Baseline: `34316550dffbe8fbaed5678a6be87c5d3c65155b`.
- Initial teardown fix and Docker probe revision: `997cb9bc30a0aec5804d1cd55bda3b87f36f5c31`.
- Environment: macOS arm64, Python 3.12.13, Docker client/server 29.7.2.
- Image: `enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0`.
- Local Docker image ID: `sha256:6328d7fd2f0ff0b4c147c3d05b3df1ce331f4a482eb6e550ecd64ed1fcf906e7`
(an image ID, not a portable registry manifest digest).
[evidence.json](evidence.json) contains allowlisted observations and production
source hashes. Host paths, account names, random container prefixes, container
IDs, Pod identifiers, and raw exception messages are omitted. The local raw
observations were checked against the production source hashes from that initial
probe revision.
## Method
[probe.py](probe.py) constructs a complete typed production AppConfig and uses
the real AioSandboxProvider. Each case has a separate process, home directory,
and randomly generated container prefix. No model is configured or called.
The fault is **one injected TimeoutExpired before stopping a particular test
container**. This leaves a real container running. Docker inspection, subsequent
stops, and cleanup use the real backend. This is controlled fault injection,
not a claim about production failure frequency or performance.
The replacement case removes the old container after the injected failure,
creates a new generation under the same logical ID, and checks whether retry
preserves it. The shutdown cases also include an unaffected resource, so a
failure cannot silently skip the rest of the cleanup.
## Observed results
| Scenario | Baseline | Candidate |
| --- | --- | --- |
| Explicit destroy | First stop raises; second destroy returns True without another stop; target still exists | Generation is fenced; second destroy stops it; absence confirmed; record retired |
| Active shutdown failure | First call reports success; failed container remains; singleton detached; second call does nothing | First call raises and retains provider; second call retries only the failed resource and completes |
| Warm shutdown failure | Failed warm resource loses tracking; second call does nothing | Failed warm resource remains retryable and is removed on the second call |
| Shutdown with unfinished execution | Failed resource remains after both calls | Execution binding survives the first failure; full shutdown retry still removes the resource |
| Replacement under the same logical ID | Second destroy reports True without identifying the old cleanup intent | Retry returns False and preserves the observed replacement and the old fence |
All five candidate scenarios matched their expected observations. Explicit
recovery makes two main-container stop attempts; partial shutdown recovery
makes three in total: one failure, one unaffected resource, one retry. The
replacement retry makes no additional stop attempt against the replacement.
All owned test containers were removed during final probe cleanup.
Normalized observations from the explicit case (not verbatim raw logs):
```text
baseline: first=TimeoutExpired; after_first: absent=false, quarantined=false, stops=1
baseline: second=True; after_second: absent=false, quarantined=false, stops=1
candidate: first=TimeoutExpired; after_first: absent=false, quarantined=true, stops=1
candidate: second=True; after_second: absent=true, quarantined=false, stops=2
```
## Related Docker error format
Real Docker 29.7.2 returned this exact missing-network shape (resource name
normalized here):
```text
Error response from daemon: network <network-name> not found
exit status 1
```
The strict parser previously rejected the second line and prevented cleanup
completion even after the container was absent. Three new regressions failed
before the parser fix. The fix accepts only this optional terminal line;
wrong targets, different exit codes, and additional diagnostics remain errors.
## Initial teardown verification
| Check | Result |
| --- | --- |
| Seven relevant offline test files | 633 passed, 3 live tests deselected |
| Strict blocking-I/O suite | 256 passed, 1 skipped |
| `make lint` | Passed, including Ruff formatting |
| Final standard four-shard `make test` | 32,407 passed, 564 skipped, **3 failed**, exit code 2 |
| Clean baseline standard `make test` | 32,384 passed, 565 skipped, **3 failed** |
The final full-suite failures match the baseline: two frontend/AGENTS.md
hard-budget assertions (32,805 bytes against 32,768) and the native Cargo
acceptance case `unit-with-empty-doc`, affected by inherited GNU Make jobserver
flags. The full suite is **not green**. Those files are unchanged by this fix.
An earlier full run also had one unchanged Docker orphan test miss `multi00`.
Its isolated rerun passed all three tests; the baseline's two-shard rerun also
passed; the final full run did not repeat the failure. Shared-prefix cleanup
between shards is a possible explanation, not a proven diagnosis.
An additional restricted-network live test failed with HTTPS CONNECT 403
(281 passed, 1 failed in that invocation). It is not counted as successful
network acceptance, and no network policy was relaxed. The offline selection
and the controlled Docker teardown probes above do not require external traffic.
## Reproduction
Prepare the selected checkout's own backend virtual environment with
`uv sync --locked --extra postgres` and make the stated AIO image available.
Run the same probe separately against the baseline and the fix, using each
checkout's interpreter. Keep the probe path absolute when selecting a different
checkout:
```bash
REPO_ROOT="$PWD"
PROBE="$REPO_ROOT/docs/experiments/aio-teardown-retry-20261017/probe.py"
RESULTS_DIR="$(mktemp -d)"
"$REPO_ROOT/backend/.venv/bin/python" "$PROBE" \
--root "$REPO_ROOT" --output "$RESULTS_DIR"
```
`--case explicit`, `shutdown-active`, `shutdown-warm`, `shutdown-held`, or
`replacement` runs one scenario. A zero exit means the probe ran and its final
container cleanup completed; compare the observations above to determine
behavioral acceptance. Generated outputs are local raw diagnostics and can
contain paths and resource IDs; this committed report includes only allowlisted
observations.
The scoped offline command, from `backend/`, was:
```bash
uv run --no-sync pytest -m 'not live' -q --tb=short \
tests/test_aio_sandbox_provider.py tests/test_sandbox_orphan_reconciliation.py \
tests/test_sandbox_provider_lifecycle.py tests/test_sandbox_quarantine.py \
tests/test_sandbox_leases.py tests/test_sandbox_lease_rebind_cancellation.py \
tests/test_aio_sandbox_local_backend.py
```
Full backend tests used the standard `make test` with Python 3.12 and Node 24 on
PATH, `NO_COLOR` unset, and no `UV_NO_SYNC` override. No acceptance assertions
were weakened. Kubernetes, Apple Container, and Redis multi-Gateway deployments
were not exercised as live environments; their contracts retain existing unit
coverage. No new authorization, prewarming, or configuration behavior is claimed.
## Replacement admission follow-up
[Review #5557122845](https://github.com/bytedance/deer-flow/pull/7001#discussion_r4302123092)
identified a regression in the initial fix: after a failed stop, a healthy new
generation with the same logical ID could be adopted into the warm pool, then
promoted without clearing the old in-memory retry marker. Acquisition returned
an ID, but cached client lookup failed; the execution lease manager therefore
raised `SandboxClientUnavailableError`.
Warm-to-active promotion now clears that obsolete marker in the same metadata
critical section as client registration, after reuse and ownership checks. If
the warm entry changes during those checks, promotion defers rather than
admitting an unvalidated entry. Generation-bound quarantine records are retained.
Eight offline regressions cover:
- Direct discovery and orphan adoption, each through the provider and the real
execution lease manager. All four cases also release and reacquire the new
generation and verify that no extra stop occurs.
- Retaining the retry fence for quarantined and unknown generations.
- Retaining that fence when either unsafe entry replaces the validated warm
entry during ownership publication.
Before the follow-up, four cases failed and four passed; afterward all eight
passed. These tests use real provider, quarantine and lease-manager methods with
mocked backend and SDK transport. The original Docker results above remain tied
to their recorded revision; Docker was unavailable for this follow-up.
Follow-up validation against the working tree based on `673304c7`:
| Check | Result |
| --- | --- |
| Eight new regression cases | 8 passed |
| Seven relevant offline files | 641 passed, 3 live tests deselected |
| Strict blocking-I/O suite, before and after | 256 passed, 1 skipped in each run |
| `make format`, `make lint`, frontend `pnpm check` | Passed |
| Agent guidance size check | 0 errors, 0 warnings |
| Standard `make test`, before follow-up | 32,520 passed, 565 skipped, 1 failed |
| Standard `make test`, after follow-up | 32,528 passed, 565 skipped, 1 failed |
Both full runs fail only the unchanged native Cargo acceptance case
`unit-with-empty-doc`, already observed in the initial local verification. Its
isolated run without inherited Make flags passes. The full local suite is not
green; there are no additional failures after the follow-up.

347
docs/experiments/aio-teardown-retry-20261017/evidence.json Normal file
View File

@ -0,0 +1,347 @@
{
"schema_version": 1,
"baseline_revision": "34316550dffbe8fbaed5678a6be87c5d3c65155b",
"fix_revision": "997cb9bc30a0aec5804d1cd55bda3b87f36f5c31",
"environment": {
"os": "macOS",
"architecture": "arm64",
"python": "3.12.13",
"docker_client": "29.7.2",
"docker_server": "29.7.2",
"aio_image": "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0",
"local_image_id": "sha256:6328d7fd2f0ff0b4c147c3d05b3df1ce331f4a482eb6e550ecd64ed1fcf906e7"
},
"method": {
"stop_fault": "One injected TimeoutExpired for an owned test container; inspection and later stops use real Docker",
"models_called": 0,
"redaction": "Allowlisted fields only; no host paths, account names, raw errors, container prefixes or resource IDs"
},
"candidate_production_sha256": {
"backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py": "09657b79e1bd6ca272945018ad8bc085635e26538a1b2bb330a19af5303794e7",
"backend/packages/harness/deerflow/community/aio_sandbox/local_backend.py": "7d6af6d83f52829c8be03d3451a2db3f34c6b0293e6d10827165909d8b8190f1",
"backend/packages/harness/deerflow/sandbox/sandbox_provider.py": "de39a17853c5275fe8d2735427151f5859196c533b6dc807d3a4196589329cee"
},
"scenarios": [
{
"case": "explicit",
"baseline": {
"first": {
"raised": "TimeoutExpired"
},
"second": {
"returned": true
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 1,
"owned_container_count": 1
},
"after_second": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 1,
"owned_container_count": 1
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 1
},
"candidate": {
"first": {
"raised": "TimeoutExpired"
},
"second": {
"returned": true
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": true,
"singleton_retained": false,
"stop_attempts": 1,
"owned_container_count": 1
},
"after_second": {
"bad_absent": true,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 2,
"owned_container_count": 0
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 0
}
},
{
"case": "shutdown-active",
"baseline": {
"first": {
"returned": null
},
"second": {
"returned": null
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 2,
"owned_container_count": 1
},
"after_second": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 2,
"owned_container_count": 1
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 2
},
"candidate": {
"first": {
"raised": "RuntimeError"
},
"second": {
"returned": null
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": true,
"singleton_retained": true,
"stop_attempts": 2,
"owned_container_count": 1
},
"after_second": {
"bad_absent": true,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 3,
"owned_container_count": 0
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 0
}
},
{
"case": "shutdown-warm",
"baseline": {
"first": {
"returned": null
},
"second": {
"returned": null
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 2,
"owned_container_count": 1
},
"after_second": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 2,
"owned_container_count": 1
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 2
},
"candidate": {
"first": {
"raised": "RuntimeError"
},
"second": {
"returned": null
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": true,
"singleton_retained": true,
"stop_attempts": 2,
"owned_container_count": 1
},
"after_second": {
"bad_absent": true,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 3,
"owned_container_count": 0
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 0
}
},
{
"case": "shutdown-held",
"baseline": {
"first": {
"returned": null
},
"second": {
"returned": null
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 2,
"execution_still_bound": true,
"owned_container_count": 1
},
"after_second": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 2,
"owned_container_count": 1
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 2
},
"candidate": {
"first": {
"raised": "RuntimeError"
},
"second": {
"returned": null
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": true,
"singleton_retained": true,
"stop_attempts": 2,
"execution_still_bound": true,
"owned_container_count": 1
},
"after_second": {
"bad_absent": true,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 3,
"owned_container_count": 0
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 0
}
},
{
"case": "replacement",
"baseline": {
"first": {
"raised": "TimeoutExpired"
},
"second": {
"returned": true
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 1,
"owned_container_count": 1
},
"after_second": {
"bad_absent": false,
"quarantined": false,
"singleton_retained": false,
"stop_attempts": 1,
"owned_container_count": 1
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 2,
"replacement_survived": true
},
"candidate": {
"first": {
"raised": "TimeoutExpired"
},
"second": {
"returned": false
},
"final_shutdown": {
"returned": null
},
"after_first": {
"bad_absent": false,
"quarantined": true,
"singleton_retained": false,
"stop_attempts": 1,
"owned_container_count": 1
},
"after_second": {
"bad_absent": false,
"quarantined": true,
"singleton_retained": false,
"stop_attempts": 1,
"owned_container_count": 1
},
"final_owned_container_count": 0,
"auxiliary_cleanup_error_count": 0,
"replacement_survived": true
}
}
],
"checks": {
"scoped": {
"passed": 633,
"live_deselected": 3
},
"blocking_io": {
"passed": 256,
"skipped": 1
},
"full_backend": {
"passed": 32407,
"skipped": 564,
"failed": 3,
"exit_code": 2
},
"baseline_full_backend": {
"passed": 32384,
"skipped": 565,
"failed": 3
},
"lint": "passed"
},
"known_full_failures": [
"tests/test_agent_guidance_check.py::test_repository_guidance_chains_fit_hard_budget",
"tests/test_agent_guidance_check.py::test_repository_guidance_stays_below_hard_budgets_and_avoids_doc_indexes",
"tests/test_cargo_acceptance.py::test_native_cargo_output_reaches_parent_acceptance_through_tool_node[unit-with-empty-doc]"
],
"limitations": [
"Controlled single-host Docker fault injection; no live Kubernetes, Apple Container or Redis multi-Gateway run",
"One earlier unchanged orphan E2E failure did not repeat in isolated, baseline split, or final full reruns; cause not proven",
"Supplemental restricted-network live invocation: 281 passed, 1 failed with HTTPS CONNECT 403; not counted as successful network acceptance"
]
}

265
docs/experiments/aio-teardown-retry-20261017/probe.py Normal file
View File

@ -0,0 +1,265 @@
"""Real AIO containers, with one simulated stop timeout per isolated case.
No models, network credentials or daemon-wide mutations. Cleanup is restricted
to immutable IDs created by this invocation. Run once per selected checkout.
"""
from __future__ import annotations
import argparse
import atexit
import hashlib
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import uuid
parser = argparse.ArgumentParser()
parser.add_argument("--root", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument(
"--case",
choices=[
"explicit",
"shutdown-active",
"shutdown-warm",
"shutdown-held",
"replacement",
],
)
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
if args.case is None:
results = []
for case in [
"explicit",
"shutdown-active",
"shutdown-warm",
"shutdown-held",
"replacement",
]:
result = subprocess.run(
[
sys.executable,
__file__,
"--root",
str(args.root),
"--output",
str(args.output),
"--case",
case,
],
cwd=args.output,
capture_output=True,
text=True,
timeout=240,
)
(args.output / f"{case}.log").write_text(
result.stdout + result.stderr, encoding="utf-8"
)
print(case, result.returncode, result.stdout, flush=True)
if result.returncode:
print(result.stderr, flush=True)
raise SystemExit(result.returncode)
results.append(json.loads((args.output / f"{case}.json").read_text()))
(args.output / "results.json").write_text(
json.dumps(results, indent=2), encoding="utf-8"
)
raise SystemExit(0)
root = args.root.resolve()
backend = root / "backend"
sys.path[:0] = [
str(backend),
str(backend / "packages/harness"),
str(backend / "packages/extension-api"),
]
case_dir = args.output / args.case
case_dir.mkdir(exist_ok=True)
os.environ["DEER_FLOW_HOME"] = str(case_dir / "home")
os.environ["DEER_FLOW_CONFIG_PATH"] = str(case_dir / "config.yaml")
(case_dir / ".env").write_text("", encoding="utf-8")
os.chdir(case_dir)
# Import only after selecting the checkout and isolated configuration above.
from deerflow.config.app_config import AppConfig, set_app_config # noqa: E402
from deerflow.community.aio_sandbox import aio_sandbox_provider as aio # noqa: E402
from deerflow.sandbox import sandbox_provider as registry # noqa: E402
assert Path(aio.__file__).resolve().is_relative_to(root)
prefix = "df-retry-" + args.case + "-" + uuid.uuid4().hex[:8]
image = (
"enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:1.11.0"
)
config = {
"models": [],
"tools": [],
"tool_groups": [],
"memory": {"enabled": False},
"sandbox": {
"use": "deerflow.community.aio_sandbox:AioSandboxProvider",
"image": image,
"container_prefix": prefix,
"port": 18800,
"idle_timeout": 0,
"replicas": 4,
"ownership": {"type": "memory"},
},
"skills": {"path": str(root / "skills"), "container_path": "/mnt/skills"},
}
# Construct the full typed production configuration, including network defaults.
set_app_config(AppConfig.model_validate(config))
(case_dir / "config.yaml").write_text(json.dumps(config, indent=2), encoding="utf-8")
provider = aio.AioSandboxProvider()
saved = []
original_stop = provider._backend._stop_container
attempts = []
result = {
"case": args.case,
"checkout": str(root),
"imported": aio.__file__,
"prefix": prefix,
"revision": subprocess.check_output(
["git", "-C", str(root), "rev-parse", "HEAD"], text=True
).strip(),
"injection": "one simulated TimeoutExpired at real LocalContainerBackend._stop_container",
"models_called": 0,
}
result["source_sha256"] = {
str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in [
Path(aio.__file__),
Path(aio.__file__).with_name("local_backend.py"),
Path(registry.__file__),
]
}
def call(action):
try:
return {"returned": action()}
except Exception as exc:
return {"raised": type(exc).__name__, "message": str(exc)}
def inventory():
proc = subprocess.run(
[
"docker",
"ps",
"-a",
"--filter",
"name=" + prefix + "-",
"--format",
"{{.ID}} {{.Names}} {{.Status}}",
],
capture_output=True,
text=True,
check=True,
)
return proc.stdout.strip().splitlines()
try:
bad_id = provider.acquire("bad", user_id="probe")
bad = provider._sandbox_infos[bad_id]
saved.append(bad)
if args.case.startswith("shutdown"):
good_id = provider.acquire("good", user_id="probe")
good = provider._sandbox_infos[good_id]
saved.append(good)
provider.release(bad_id if args.case == "shutdown-warm" else good_id)
assert len(provider._warm_pool) == 1
registry.set_sandbox_provider(provider)
if args.case == "shutdown-held":
from deerflow.sandbox.lease import get_sandbox_lease_manager
manager = get_sandbox_lease_manager(provider)
manager.retain("unfinished", bad_id, thread_id="bad", user_id="probe")
injected = False
def fail_once(target):
global injected
if target in {info.container_id for info in saved}:
attempts.append(target)
if target == bad.container_id and not injected:
injected = True
raise subprocess.TimeoutExpired(["docker", "stop", target], 0.001)
original_stop(target)
provider._backend._stop_container = fail_once
action = (
registry.shutdown_sandbox_provider
if args.case.startswith("shutdown")
else lambda: provider.destroy(bad_id)
)
result["first"] = call(action)
assert injected, "probe never entered the target stop operation"
result["after_first"] = {
"bad_absent": provider._backend.is_absent(bad_id),
"inventory": inventory(),
"quarantined": provider._quarantine_store().contains(bad),
"singleton_retained": registry.get_initialized_sandbox_provider() is provider,
"stop_attempts": len(attempts),
}
assert result["after_first"]["bad_absent"] is False
if args.case == "shutdown-held":
result["after_first"]["execution_still_bound"] = (
manager.binding_for("unfinished") == bad_id
)
if args.case == "replacement":
original_stop(bad.container_id)
replacement = provider._backend.create(None, bad_id)
saved.append(replacement)
result["replacement_generation"] = replacement.container_id
assert replacement.container_id != bad.container_id
result["second"] = call(action)
result["after_second"] = {
"bad_absent": provider._backend.is_absent(bad_id),
"inventory": inventory(),
"quarantined": provider._quarantine_store().contains(bad),
"singleton_retained": registry.get_initialized_sandbox_provider() is provider,
"stop_attempts": len(attempts),
}
if args.case == "replacement":
current = provider._backend.inspect_runtime(bad_id)
result["replacement_survived"] = (
current is not None and current.container_id == replacement.container_id
)
assert result["replacement_survived"]
result["attempted_generations"] = attempts
finally:
provider._backend._stop_container = original_stop
for info in reversed(saved):
try:
original_stop(info.container_id)
except Exception:
if not provider._backend.is_absent(info.sandbox_id):
raise
cleanup = call(
lambda: provider._backend.complete_absent_teardown(info.sandbox_id)
)
if "raised" in cleanup:
result.setdefault("cleanup_errors", []).append(cleanup)
result["final_shutdown"] = call(provider.shutdown)
atexit.unregister(provider.shutdown)
registry._default_sandbox_provider = None
# This isolated process owns the signal registrations; restore originals.
signal.signal(signal.SIGTERM, provider._original_sigterm)
signal.signal(signal.SIGINT, provider._original_sigint)
if hasattr(signal, "SIGHUP"):
signal.signal(signal.SIGHUP, provider._original_sighup)
result["final_inventory"] = inventory()
assert result["final_inventory"] == [], (
"owned containers leaked after probe cleanup"
)
(args.output / f"{args.case}.json").write_text(
json.dumps(result, indent=2), encoding="utf-8"
)
print(json.dumps(result), flush=True)