From aa978d1b32f9c74a6177868084b247729b58e3bd Mon Sep 17 00:00:00 2001 From: Hyeonsang Cho Date: Fri, 23 Oct 2026 09:08:35 +0900 Subject: [PATCH] fix(sandbox): release the E2B ownership lease when acquire finds the VM gone (#7277) * fix(sandbox): release the E2B ownership lease when acquire finds the VM gone The in-process reuse path (dead or unreachable cached client) and the warm-pool reclaim path (reconnect reports "sandbox not found") dropped their local bookkeeping but kept the VM's ownership lease. Nothing later sees that ID: the warm sweep only walks the warm pool, reconciliation only walks the remote inventory, and the lease renewer keeps renewing (and re-claims a lapsed lease) until shutdown. Every reaped VM left one entry in _owned_sandbox_ids and one renewed ownership key behind. Call _release_ownership in both branches. Both act only on E2B's trusted not-found signal, the store's release is a no-op for a peer's lease, and a backend error still drops the local ID so the shared lease expires on its TTL. * docs(changelog): reference #7277 for the E2B dead-VM lease fix * test(sandbox): pin that transient E2B errors keep the ownership lease The dead-VM release is only safe because the gone signatures match narrowly. Pin the other side: a non-gone ping error keeps reusing the cached VM, and a non-gone warm reconnect error leaves the lease to eviction. --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 6 + CHANGELOG_zh.md | 5 + .../deerflow/community/e2b_sandbox/AGENTS.md | 3 + .../e2b_sandbox/e2b_sandbox_provider.py | 5 + backend/tests/test_e2b_sandbox_provider.py | 108 ++++++++++++++++++ 5 files changed, 127 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 588d4578c6..5e567af2df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1397,6 +1397,11 @@ As of 2026-10-21, the milestone includes **829 merged pull requests**. ### Fixed +- **sandbox:** Release an E2B VM's ownership lease when acquire finds it gone + (a dead cached client or a warm-pool VM reaped while parked). The lease was + kept and renewed until Gateway shutdown, so every reaped VM left a tracked + ID and a renewed ownership key behind. ([#7277]) + - **auth:** SSO login and callback no longer fail with a `500` when the OIDC provider answers `200` with a body that is not the expected JSON object, such as a maintenance HTML page. Discovery, token, JWKS and UserInfo bodies, a @@ -13014,3 +13019,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7217]: https://github.com/bytedance/deer-flow/pull/7217 [#7255]: https://github.com/bytedance/deer-flow/pull/7255 [#7258]: https://github.com/bytedance/deer-flow/pull/7258 +[#7277]: https://github.com/bytedance/deer-flow/pull/7277 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index d06035d15a..bce6271462 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -1112,6 +1112,10 @@ ### 修复 +- **沙箱：** acquire 发现 E2B VM 已不存在（缓存客户端已失效，或暖池中的 VM 在停放期间被 + 回收）时，释放该 VM 的所有权租约。此前租约会一直保留并续约到 Gateway 关闭，每个被回收的 + VM 都会留下一个跟踪 ID 和一个持续续约的所有权键。([#7277]) + - **认证：** OIDC 提供方以 `200` 返回非预期 JSON 对象的响应体（例如维护中的 HTML 页面）时，SSO 登录与回调不再以 `500` 失败。discovery、token、JWKS 与 UserInfo 响应体、格式错误的 ID token 头以及缺少可用端点的 discovery 文档，现在都走既有的 @@ -10388,3 +10392,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7217]: https://github.com/bytedance/deer-flow/pull/7217 [#7255]: https://github.com/bytedance/deer-flow/pull/7255 [#7258]: https://github.com/bytedance/deer-flow/pull/7258 +[#7277]: https://github.com/bytedance/deer-flow/pull/7277 diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md b/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md index 80dd012422..191d44d3dd 100644 --- a/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md @@ -45,6 +45,9 @@ Release only needs the VM lock while leaving active state; do not hold it during output sync, which must not prevent ownership heartbeats. Track that release in `_remote_ops_in_progress` until it completes so reconciliation cannot probe or re-adopt a VM between active and warm states. +Acquire drops the lease of a VM it finds gone (dead cached client, reaped +warm entry): no later path sees that ID, and the renewer re-claims any lapsed +lease it still tracks. Shutdown owns both maintenance workers as well as sandbox registries. After signalling the lease-renewal and reconciliation threads, a bounded join is only diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py index 39adf386ca..418010a84e 100644 --- a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py @@ -671,6 +671,9 @@ class E2BSandboxProvider(SandboxProvider): self._sandboxes.pop(sid, None) self._thread_sandboxes.pop(key, None) self._forget_mount_result(sid) + # The lease renewer would otherwise keep re-claiming this lease + # until shutdown: no later path sees the ID again. + self._release_ownership(sid) try: sandbox.close() except Exception: @@ -745,6 +748,8 @@ class E2BSandboxProvider(SandboxProvider): target_id, ) self._forget_mount_result(target_id) + # The warm entry's lease would outlive it; see the in-process reuse path. + self._release_ownership(target_id) self._complete_transition_remote_op(target_id, remote_destroyed=True) return None diff --git a/backend/tests/test_e2b_sandbox_provider.py b/backend/tests/test_e2b_sandbox_provider.py index 2d00ebdd39..b6f57d32f6 100644 --- a/backend/tests/test_e2b_sandbox_provider.py +++ b/backend/tests/test_e2b_sandbox_provider.py @@ -2219,6 +2219,114 @@ def test_reclaim_warm_pool_sandbox_returns_none_on_seed_mismatch(monkeypatch): assert "sb-other" in p._warm_pool +def _assert_lease_dropped(p, sandbox_id: str) -> None: + assert sandbox_id not in p._owned_sandbox_ids + assert p._ownership.owner(sandbox_id) is None + # The renewer re-claims a lapsed lease it still tracks, so a single + # leftover ID would hold the shared lease until shutdown. + p._refresh_owned_leases() + assert sandbox_id not in p._owned_sandbox_ids + assert p._ownership.owner(sandbox_id) is None + + +@pytest.mark.parametrize("dead_signal", ["is_dead", "ping"]) +def test_acquire_releases_lease_of_dead_in_process_sandbox(monkeypatch, tmp_path, dead_signal): + _setup_paths(monkeypatch, tmp_path) + p = _make_provider() + _install_fake_sdk(monkeypatch, p) + dead_id = p.acquire("t1", user_id="u1") + assert p._ownership.owner(dead_id) == p._owner_id + if dead_signal == "is_dead": + p._sandboxes[dead_id]._dead = True + else: + p._sandboxes[dead_id].client.commands = FakeCommandsAPI([FakeCommandsAPI.GONE]) + + fresh_id = p.acquire("t1", user_id="u1") + + assert fresh_id != dead_id + _assert_lease_dropped(p, dead_id) + assert p._ownership.owner(fresh_id) == p._owner_id + assert fresh_id in p._owned_sandbox_ids + + +def test_acquire_releases_lease_of_reaped_warm_pool_sandbox(monkeypatch, tmp_path): + _setup_paths(monkeypatch, tmp_path) + p = _make_provider() + fake_cls = _install_fake_sdk(monkeypatch, p) + dead_id = p.acquire("t1", user_id="u1") + p.release(dead_id) + assert dead_id in p._warm_pool + assert p._ownership.owner(dead_id) == p._owner_id + + def missing_sandbox(_sid, **_kw): + raise RuntimeError(FakeCommandsAPI.NOT_FOUND_MSG) + + fake_cls.connect_factory = missing_sandbox + + fresh_id = p.acquire("t1", user_id="u1") + + assert fresh_id != dead_id + _assert_lease_dropped(p, dead_id) + assert p._ownership.owner(fresh_id) == p._owner_id + assert p._transitioning_slots == 0 + + +def _transient_error(*_args, **_kwargs): + # Not one of the trusted gone signatures: the VM may still be alive. + raise RuntimeError("connection reset by peer") + + +def test_acquire_keeps_lease_when_in_process_ping_fails_transiently(monkeypatch, tmp_path): + _setup_paths(monkeypatch, tmp_path) + p = _make_provider() + _install_fake_sdk(monkeypatch, p) + sandbox_id = p.acquire("t1", user_id="u1") + p._sandboxes[sandbox_id].client.commands = FakeCommandsAPI([_transient_error]) + + assert p.acquire("t1", user_id="u1") == sandbox_id + assert sandbox_id in p._owned_sandbox_ids + assert p._ownership.owner(sandbox_id) == p._owner_id + + +def test_acquire_keeps_lease_when_warm_pool_reconnect_fails_transiently(monkeypatch, tmp_path): + _setup_paths(monkeypatch, tmp_path) + p = _make_provider() + fake_cls = _install_fake_sdk(monkeypatch, p) + warm_id = p.acquire("t1", user_id="u1") + p.release(warm_id) + fake_cls.connect_factory = _transient_error + + fresh_id = p.acquire("t1", user_id="u1") + + assert fresh_id != warm_id + assert warm_id not in p._warm_pool + assert warm_id in p._owned_sandbox_ids + assert p._ownership.owner(warm_id) == p._owner_id + # Eviction releases the lease once the VM's state is known. + assert warm_id in p._eviction_tombstones + + +def test_dead_sandbox_lease_release_failure_does_not_block_rebuild(monkeypatch, tmp_path): + from deerflow.community.aio_sandbox.ownership import OwnershipBackendError + + _setup_paths(monkeypatch, tmp_path) + p = _make_provider() + _install_fake_sdk(monkeypatch, p) + dead_id = p.acquire("t1", user_id="u1") + p._sandboxes[dead_id]._dead = True + + def unavailable(_sid): + raise OwnershipBackendError("redis down") + + monkeypatch.setattr(p._ownership, "release", unavailable) + + fresh_id = p.acquire("t1", user_id="u1") + + assert fresh_id != dead_id + # Without a renewer the shared lease expires on its own TTL. + assert dead_id not in p._owned_sandbox_ids + + class _FakePaginator: """Mirror of e2b SDK's ``SandboxPaginator``: items via ``next_items``."""