mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-25 14:06:18 +00:00
74f006951c
* fix(sandbox): bound remaining shell session cleanup requests release_command_scope(), close()'s scoped-session drain, and close()'s recovery-session cleanup called shell.cleanup_session() without a request budget, so a stalled cleanup could hold the thread-key serializer, scoped.lock, or the sandbox lock for the SDK's default transport budget. Pass the existing _bounded_cleanup_request_options() (5s, max_retries=0) at those three sites; _cleanup_session_best_effort() and its swallow-and-log contract are unchanged, as are create_session and list_dir lifetimes. * fix(sandbox): bound AIO list_dir with a directory deadline Give AioSandbox.list_dir its own 60s directory deadline with a 65s no-retry host envelope, independent of bash_command_timeout. Preserve #5634's shell-generation selection: list_dir runs on the current recovery session once the implicit shell is fenced, and an ambiguous list_dir outcome - transport timeout or an ambiguous returned status - fences whichever generation actually executed the request, dropping local recovery ownership and attempting bounded best-effort cleanup instead of leaving that session reusable. hard_timeout remains definite termination and keeps the targeted session reusable. Only completed/None results are parsed, so a partial find is never returned as a complete listing. Session creation RPC lifetime remains out of scope. * fix(sandbox): recover sessions after transport failures --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
127 lines
6.5 KiB
Python
127 lines
6.5 KiB
Python
"""AIO transport failures must not leave a shell generation reusable."""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
from agent_sandbox.core.api_error import ApiError
|
|
|
|
from deerflow.community.aio_sandbox.aio_sandbox import AioSandbox, _ScopedShellSession
|
|
|
|
|
|
@pytest.fixture
|
|
def sandbox():
|
|
with patch("deerflow.community.aio_sandbox.aio_sandbox.AioSandboxClient"):
|
|
yield AioSandbox(id="recovery-test", base_url="http://localhost:8080")
|
|
|
|
|
|
def completed(output="/test\n/test/file\n\n__DF_FIND_STATUS__:0\n"):
|
|
return SimpleNamespace(data=SimpleNamespace(output=output, exit_code=0, status="completed"))
|
|
|
|
|
|
@pytest.mark.parametrize("session_id", [None, "old-session"])
|
|
@pytest.mark.parametrize("error_type", [httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError])
|
|
def test_list_dir_transport_failure_fences_before_cleanup_and_recovers(sandbox, session_id, error_type):
|
|
sandbox._default_shell_corrupted = session_id is not None
|
|
sandbox._recovery_session_id = session_id
|
|
client = sandbox._client
|
|
client.shell.exec_command.side_effect = [error_type("connection lost"), completed()]
|
|
|
|
cleanup_states = []
|
|
|
|
def cleanup(*args, **kwargs):
|
|
cleanup_states.append((sandbox._recovery_session_id, sandbox._default_shell_corrupted))
|
|
raise httpx.ReadError("cleanup also failed")
|
|
|
|
client.shell.cleanup_session.side_effect = cleanup
|
|
with pytest.raises(OSError, match="unknown"):
|
|
sandbox.list_dir("/test")
|
|
assert client.shell.exec_command.call_count == 1
|
|
assert sandbox._default_shell_corrupted
|
|
assert sandbox._recovery_session_id is None
|
|
assert sandbox._lock.acquire(blocking=False)
|
|
sandbox._lock.release()
|
|
if session_id:
|
|
assert cleanup_states == [(None, True)]
|
|
client.shell.cleanup_session.assert_called_once_with(session_id, request_options={"timeout_in_seconds": 5, "max_retries": 0})
|
|
else:
|
|
client.shell.cleanup_session.assert_not_called()
|
|
assert sandbox.list_dir("/test") == ["/test", "/test/file"]
|
|
replacement = client.shell.create_session.call_args.kwargs["id"]
|
|
assert replacement != session_id
|
|
assert client.shell.exec_command.call_args.kwargs["id"] == replacement
|
|
|
|
|
|
@pytest.mark.parametrize("session_id", [None, "missing-session"])
|
|
def test_list_dir_missing_session_is_forgotten_without_replaying(sandbox, session_id):
|
|
sandbox._default_shell_corrupted = session_id is not None
|
|
sandbox._recovery_session_id = session_id
|
|
client = sandbox._client
|
|
client.shell.exec_command.side_effect = [ApiError(status_code=404, body={"message": "Shell session not found"}), completed()]
|
|
with pytest.raises(OSError):
|
|
sandbox.list_dir("/test")
|
|
assert client.shell.exec_command.call_count == 1
|
|
assert sandbox._default_shell_corrupted
|
|
assert sandbox._recovery_session_id is None
|
|
client.shell.cleanup_session.assert_not_called()
|
|
assert sandbox.list_dir("/test") == ["/test", "/test/file"]
|
|
replacement = client.shell.create_session.call_args.kwargs["id"]
|
|
assert replacement != session_id
|
|
assert client.shell.exec_command.call_args.kwargs["id"] == replacement
|
|
|
|
|
|
@pytest.mark.parametrize("status,body", [(404, {"message": "route not found"}), (503, {"message": "session not found"}), (404, "session not found")])
|
|
def test_list_dir_other_api_errors_do_not_discard_recovery_session(sandbox, status, body):
|
|
sandbox._default_shell_corrupted = True
|
|
sandbox._recovery_session_id = "keep-session"
|
|
sandbox._client.shell.exec_command.side_effect = ApiError(status_code=status, body=body)
|
|
with pytest.raises(OSError):
|
|
sandbox.list_dir("/test")
|
|
assert sandbox._recovery_session_id == "keep-session"
|
|
sandbox._client.shell.create_session.assert_not_called()
|
|
sandbox._client.shell.cleanup_session.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize("scoped", [False, True])
|
|
@pytest.mark.parametrize("error_type", [httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError])
|
|
def test_command_transport_failure_fences_without_replay(sandbox, scoped, error_type):
|
|
client = sandbox._client
|
|
sandbox._default_shell_corrupted = True
|
|
sandbox._recovery_session_id = "default-session"
|
|
sandbox._scoped_shell_sessions["task"] = _ScopedShellSession(session_id="scoped-session")
|
|
client.shell.exec_command.side_effect = [error_type("connection lost"), completed("recovered")]
|
|
invoke = (lambda: sandbox.execute_command_in_scope("echo test", scope_id="task")) if scoped else (lambda: sandbox.execute_command("echo test"))
|
|
result = invoke()
|
|
assert "unknown" in result
|
|
assert "not retried" in result
|
|
assert client.shell.exec_command.call_count == 1
|
|
if scoped:
|
|
assert sandbox._scoped_shell_sessions["task"].session_id is None
|
|
assert sandbox._recovery_session_id == "default-session"
|
|
else:
|
|
assert sandbox._recovery_session_id is None
|
|
assert sandbox._scoped_shell_sessions["task"].session_id == "scoped-session"
|
|
client.shell.cleanup_session.assert_called_once_with("scoped-session" if scoped else "default-session", request_options={"timeout_in_seconds": 5, "max_retries": 0})
|
|
assert invoke() == "recovered"
|
|
assert client.shell.exec_command.call_args.kwargs["id"] == client.shell.create_session.call_args.kwargs["id"]
|
|
|
|
|
|
@pytest.mark.parametrize("scoped", [False, True])
|
|
@pytest.mark.parametrize("missing", [False, True])
|
|
def test_replacement_transport_failure_is_cleaned_up_without_third_attempt(sandbox, scoped, missing):
|
|
client = sandbox._client
|
|
sandbox._default_shell_corrupted = True
|
|
sandbox._recovery_session_id = "default-session"
|
|
sandbox._scoped_shell_sessions["task"] = _ScopedShellSession(session_id="scoped-session")
|
|
first = ApiError(status_code=404, body={"message": "Session not found"}) if missing else completed("'ErrorObservation' object has no attribute 'exit_code'")
|
|
client.shell.exec_command.side_effect = [first, httpx.RemoteProtocolError("connection dropped during recovery")]
|
|
result = sandbox.execute_command_in_scope("echo test", scope_id="task") if scoped else sandbox.execute_command("echo test")
|
|
assert "unknown" in result
|
|
assert "not retried" in result
|
|
assert client.shell.exec_command.call_count == 2
|
|
replacement = client.shell.create_session.call_args.kwargs["id"]
|
|
assert client.shell.cleanup_session.call_args.args == (replacement,)
|
|
assert client.shell.cleanup_session.call_args.kwargs == {"request_options": {"timeout_in_seconds": 5, "max_retries": 0}}
|
|
assert (sandbox._scoped_shell_sessions["task"].session_id if scoped else sandbox._recovery_session_id) is None
|