diff --git a/README.md b/README.md index 726a419f1e..a8ac76ea73 100644 --- a/README.md +++ b/README.md @@ -625,6 +625,12 @@ Remote directory listings report traversal failures (for example, unreadable directories) as incomplete results, even when no entries were returned. A missing start path is reported separately as “Directory not found.” +The optional [Tenki cloud sandbox provider](backend/packages/harness/deerflow/community/tenki/README.md) +uses Tenki SDK 1.4.0 or newer. Timed-out commands preserve partial output and +report `Exit Code: 124`; unsuccessful health checks cannot reclaim a warm sandbox. +Health probes tolerate login-shell output around the `ok` line, and failures log +the sandbox ID and probe output before replacing the sandbox. + #### MCP Server In the chat UI, enable **Token Usage → Debug** to inspect generic/MCP tool calls. diff --git a/backend/packages/harness/deerflow/community/tenki/README.md b/backend/packages/harness/deerflow/community/tenki/README.md index 52769099bc..e1ccbcdeb9 100644 --- a/backend/packages/harness/deerflow/community/tenki/README.md +++ b/backend/packages/harness/deerflow/community/tenki/README.md @@ -37,7 +37,7 @@ Install the optional SDK before selecting this provider: pip install "deerflow-harness[tenki]" ``` -The `tenki` package (which provides the `tenki_sandbox` module) is an optional +The `tenki>=1.4.0` package (which provides the `tenki_sandbox` module) is an optional DeerFlow harness extra, not part of the default install. Get an API key from . @@ -67,6 +67,9 @@ API; directory and content search shell out and reuse `deerflow.sandbox.search`, mirroring `e2b_sandbox`: - `execute_command` — `sh -lc`, with per-call env and timeout. + Timed-out commands preserve partial output and report `Error: command timed out` + with `Exit Code: 124`, even if the SDK returned exit code zero. They are not + retried, and the sandbox remains available for subsequent commands. - `read_file` / `write_file` / `update_file` — native `fs.read_text` / `fs.mkdir` / `fs.write_stream` (binary-safe, streamed). - `download_file` — native `fs.read_stream`, restricted to the `/mnt/user-data` prefix; the 100 MB cap is enforced on bytes actually received, so a file growing mid-transfer cannot slip past it. - `list_dir` / `glob` / `grep` — `find` / `grep` with busybox-portable flags (the fs API is single-level and has no content search); results filtered/capped in Python and reported back under `/mnt/user-data`. @@ -83,6 +86,11 @@ sandboxes. `sandbox.idle_timeout` controls how long released warm sandboxes stay running; `0` disables idle reaping. Active sandboxes are never evicted to satisfy the cap. +The warm-pool health probe accepts an exact `ok` line alongside login-shell +profile output or stderr warnings. Timeout, nonzero-exit, and command-error +diagnostics still prevent reuse. Failed probes log the sandbox ID and returned +output before the unhealthy microVM is terminated and replaced. + ## Scope: stable features only Only the stable Tenki surface is used — sandbox create/terminate plus diff --git a/backend/packages/harness/deerflow/community/tenki/provider.py b/backend/packages/harness/deerflow/community/tenki/provider.py index 8b6b8f7d6e..27ba2deaa6 100644 --- a/backend/packages/harness/deerflow/community/tenki/provider.py +++ b/backend/packages/harness/deerflow/community/tenki/provider.py @@ -330,11 +330,12 @@ class TenkiSandboxProvider(WarmPoolLifecycleMixin[TenkiSandbox], SandboxProvider # Best-effort: on failure the file APIs still work via the home remap. try: result = remote.exec("sh", "-lc", _bootstrap_script(self._config["home_dir"]), timeout=_BOOTSTRAP_TIMEOUT) - if result.exit_code not in (0, None) or "BOOTSTRAP_OK" not in (result.stdout_text or ""): + if result.timed_out or result.exit_code not in (0, None) or "BOOTSTRAP_OK" not in (result.stdout_text or ""): logger.warning( - "Tenki bootstrap for %s exited code=%s stderr=%s", + "Tenki bootstrap for %s exited code=%s timed_out=%s stderr=%s", sandbox_id, result.exit_code, + result.timed_out, (result.stderr_text or "").strip(), ) except Exception as e: @@ -407,7 +408,12 @@ class TenkiSandboxProvider(WarmPoolLifecycleMixin[TenkiSandbox], SandboxProvider try: result = sandbox.execute_command("echo ok", timeout=10) - healthy = "ok" in result + # Login-shell profiles can add stdout/stderr around the probe marker. + # Keep rejecting adapter failure diagnostics even if "ok" was printed. + lines = result.splitlines() + healthy = "ok" in lines and not any(line.startswith(("Error:", "Exit Code:")) for line in lines) + if not healthy: + logger.warning("Tenki warm-pool sandbox %s health check failed: %s", sandbox_id, result) except Exception as e: logger.warning("Tenki warm-pool sandbox %s health check error: %s", sandbox_id, e) healthy = False diff --git a/backend/packages/harness/deerflow/community/tenki/sandbox.py b/backend/packages/harness/deerflow/community/tenki/sandbox.py index 4a29841ce5..59a3471395 100644 --- a/backend/packages/harness/deerflow/community/tenki/sandbox.py +++ b/backend/packages/harness/deerflow/community/tenki/sandbox.py @@ -274,6 +274,9 @@ class TenkiSandbox(Sandbox): output = f"{stdout}\n{stderr}" else: output = stdout or stderr + if result.timed_out: + message = "Error: command timed out\nExit Code: 124" + return f"{output}\n{message}" if output else message if result.exit_code not in (0, None): # Mirror LocalSandbox: preserve a nonzero exit in the output text # even when the command produced output (see e2b_sandbox). diff --git a/backend/packages/harness/deerflow/sandbox/AGENTS.md b/backend/packages/harness/deerflow/sandbox/AGENTS.md index ce5fdd8691..71a49eaf97 100644 --- a/backend/packages/harness/deerflow/sandbox/AGENTS.md +++ b/backend/packages/harness/deerflow/sandbox/AGENTS.md @@ -92,7 +92,8 @@ cancellation before propagating it. - `BoxliteProvider` (`packages/harness/deerflow/community/boxlite/`) - optional BoxLite micro-VM isolation. Loop-affine handles live on one private daemon-thread asyncio loop; sync `Sandbox` calls marshal through `run_coroutine_threadsafe`. Boxes use deterministic `user_id:thread_id` IDs, park in a same-owner warm pool, and are reclaimed only by that owner. Health-check timeouts reach both BoxLite `exec(timeout=...)` and private-loop `.result(timeout)`; a bridge timeout must cancel its coroutine so work cannot continue or retain acquire ownership in the background. `sandbox.replicas` caps active + warm VMs per gateway process; if capacity is exhausted, only warm-pool VMs are evicted. `sandbox.idle_timeout` stops idle warm VMs after the configured seconds. `reset()` is intentionally a lightweight registry clear for `reset_sandbox_provider()` and does not close boxes, stop the idle reaper, or close the private loop; full teardown remains `shutdown()`. -- `TenkiSandboxProvider` (`packages/harness/deerflow/community/tenki/`) - Tenki cloud microVM isolation. The `tenki` SDK is optional (`deerflow-harness[tenki]`, importable as `tenki_sandbox`) and lazy-imported (`_import_client`) only when this provider is selected. Unlike Boxlite, the SDK is synchronous, so the adapter calls it directly with no event-loop bridge. File transport uses Tenki's native `sandbox.fs` API (`read_text`/`read_stream`/`write_stream`/`mkdir`/`stat`) — binary-safe and streaming, no base64/shell hop; only directory/content *search* (`list_dir`/`glob`/`grep`) shells out to busybox-portable `find`/`grep`, parsed with the shared `deerflow.sandbox.search` helpers like `community/e2b_sandbox`. Sandboxes run as the unprivileged `tenki` user, so DeerFlow's `/mnt/user-data` prefix is remapped under a writable HOME (`_resolve_path`) and best-effort `sudo`-symlinked at bootstrap. Boxes are named deterministically from `sha256(user_id:thread_id)[:16]` (64-bit, matching E2B; the warm pool is keyed by this id alone with no full-seed fallback), released into an in-process warm pool, and reclaimed only by the same user/thread after a liveness check. A terminal session error (named SDK errors plus builtin `ConnectionError`/`BrokenPipeError`/`EOFError`) routes through `_invalidate_sandbox` to evict the dead microVM. Cross-process orphan reconciliation is a follow-up (single-process warm pool today). +- `TenkiSandboxProvider` (`community/tenki/`): cloud microVMs. Optional `tenki` SDK (`deerflow-harness[tenki]`, module `tenki_sandbox`) loads via `_import_client` only on provider selection; calls are synchronous, without an event-loop bridge. Native `sandbox.fs` (`read_text`/`read_stream`/`write_stream`/`mkdir`/`stat`) streams binary-safe files without shell/base64. Only search (`list_dir`/`glob`/`grep`) uses busybox-portable `find`/`grep` and shared `deerflow.sandbox.search` parsing, like E2B. The unprivileged `tenki` user needs `/mnt/user-data` remapped to writable HOME (`_resolve_path`), with a best-effort bootstrap `sudo` symlink. Deterministic names and warm-pool keys use `sha256(user_id:thread_id)[:16]` (64-bit, matching E2B), with no full-seed fallback. Release parks VMs in-process; only the same user/thread reclaims them after a health check. Named terminal SDK errors and builtin `ConnectionError`/`BrokenPipeError`/`EOFError` evict via `_invalidate_sandbox`. Cross-process orphan reconciliation remains a follow-up. + Health probes require an exact `ok` line and reject `Error:`/`Exit Code:` diagnostic lines; other login-shell stdout/stderr is tolerated. Failures log sandbox id and output before eviction. **Shared warm-pool lifecycle:** community sandbox providers that keep released sandboxes alive for fast reuse share `deerflow.community.warm_pool_lifecycle.WarmPoolLifecycleMixin`. The mixin owns the common `DEFAULT_IDLE_TIMEOUT=600`, `IDLE_CHECK_INTERVAL=60`, `DEFAULT_REPLICAS=3`, idle-checker loop, warm-pool expiry, oldest-warm eviction, replica counting, and soft-cap logging. Providers remain responsible for their own active registries, creation/discovery, health checks, and destroy hook (`_destroy_warm_entry`): AIO destroys `SandboxInfo` through its backend; Boxlite closes loop-affine `BoxliteBox` handles; Tenki closes the microVM session (`TenkiSandbox.close`, which terminates the remote sandbox). AIO keeps active-idle cleanup outside the mixin and delegates only warm-pool expiry to the shared helper. diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml index 923fb946bf..242ff903a3 100644 --- a/backend/packages/harness/pyproject.toml +++ b/backend/packages/harness/pyproject.toml @@ -95,7 +95,7 @@ boxlite = ["boxlite>=0.9.7"] # install stays free of the Tenki SDK; only pulled in when the provider is used. # The distribution is ``tenki``; it still ships the ``tenki_sandbox`` module that # provider.py and sandbox.py import. -tenki = ["tenki>=1.0.0"] +tenki = ["tenki>=1.4.0"] # OpenSandbox remote sandbox provider (deerflow.community.opensandbox). The # sync SDK is loaded only when this provider is selected. opensandbox = ["opensandbox>=0.1.15,<0.2.0"] diff --git a/backend/tests/test_tenki_provider.py b/backend/tests/test_tenki_provider.py index 5f3b552207..54e96b6d8a 100644 --- a/backend/tests/test_tenki_provider.py +++ b/backend/tests/test_tenki_provider.py @@ -31,10 +31,11 @@ from deerflow.config.app_config import AppConfig class _FakeResult: - def __init__(self, exit_code: int = 0, stdout: bytes = b"", stderr: bytes = b"") -> None: + def __init__(self, exit_code: int = 0, stdout: bytes = b"", stderr: bytes = b"", timed_out: bool = False) -> None: self.exit_code = exit_code self.stdout = stdout self.stderr = stderr + self.timed_out = timed_out @property def stdout_text(self) -> str: @@ -388,6 +389,26 @@ def test_execute_command_returns_error_as_text() -> None: assert box.execute_command("echo hi") == "Error: boom" +@pytest.mark.parametrize("exit_code", [0, 143]) +@pytest.mark.parametrize("stdout,stderr", [(b"partial output", b"diagnostic"), (b"", b"")]) +def test_execute_command_reports_timeout_and_preserves_output(monkeypatch, exit_code, stdout, stderr) -> None: + fake = _FakeSandbox() + monkeypatch.setattr(fake, "_run_script", lambda script: _FakeResult(exit_code=exit_code, stdout=stdout, stderr=stderr, timed_out=True)) + invalidated = [] + box = TenkiSandbox("sb", fake, on_terminal_failure=lambda *args: invalidated.append(args)) + + output = box.execute_command("sleep 10", timeout=1) + + assert "Error: command timed out" in output + assert output.endswith("Exit Code: 124") + assert stdout.decode() in output + assert stderr.decode() in output + assert len(fake.exec_calls) == 1 + assert fake.exec_calls[0]["timeout"] == 1 + assert not invalidated + assert not box.is_closed + + def test_execute_command_closed_returns_error() -> None: box = TenkiSandbox("sb", _FakeSandbox()) box.close() @@ -845,6 +866,127 @@ def test_bootstrap_is_non_interactive_and_time_bounded(monkeypatch): provider.shutdown() +def test_timed_out_bootstrap_warns_even_with_success_marker(monkeypatch, caplog): + class _TimedOutBootstrap(_FakeSandbox): + def _run_script(self, script: str) -> _FakeResult: + if "BOOTSTRAP_OK" in script: + return _FakeResult(stdout=b"BOOTSTRAP_OK\n", timed_out=True) + return super()._run_script(script) + + provider = _install(monkeypatch, client=_FakeClient(sandbox_factory=_TimedOutBootstrap)) + try: + with caplog.at_level("WARNING"): + sid = provider.acquire("thread-1", user_id="u1") + assert provider.get(sid) is not None + assert any("bootstrap" in r.message and "timed_out=True" in r.message for r in caplog.records) + finally: + provider.shutdown() + + +@pytest.mark.parametrize("timed_out,exit_code", [(True, 0), (False, 1)]) +def test_warm_pool_rejects_unsuccessful_probe_with_ok_output(monkeypatch, caplog, timed_out, exit_code): + client = _FakeClient() + provider = _install(monkeypatch, client=client) + try: + sid = provider.acquire("thread-1", user_id="u1") + first = client.last_sandbox + provider.release(sid) + monkeypatch.setattr( + first, + "_run_script", + lambda script: _FakeResult(stdout=b"ok\n", exit_code=exit_code, timed_out=timed_out), + ) + + with caplog.at_level(logging.WARNING): + assert provider.acquire("thread-1", user_id="u1") == sid + assert client.last_sandbox is not first + assert first.closed + failure = next(record.message for record in caplog.records if "health check failed" in record.message) + assert sid in failure + assert "ok\n" in failure + assert f"Exit Code: {124 if timed_out else exit_code}" in failure + finally: + provider.shutdown() + + +@pytest.mark.parametrize( + "stdout,stderr", + [ + (b"ok\n", b"sh: warning: setlocale: LC_ALL: cannot change locale\n"), + (b"profile loaded\nok\n", b""), + ( + b"profile loaded\nok\n", + b"sh: warning: setlocale: LC_ALL: cannot change locale\n", + ), + ], +) +def test_warm_pool_reclaims_healthy_probe_with_shell_noise(monkeypatch, caplog, stdout, stderr): + client = _FakeClient() + provider = _install(monkeypatch, client=client) + try: + sid = provider.acquire("thread-1", user_id="u1") + first = client.last_sandbox + provider.release(sid) + monkeypatch.setattr( + first, + "_run_script", + lambda script: _FakeResult(stdout=stdout, stderr=stderr), + ) + calls_before = len(first.exec_calls) + + with caplog.at_level(logging.WARNING): + assert provider.acquire("thread-1", user_id="u1") == sid + assert client.create_count == 1 + assert client.last_sandbox is first + assert not first.closed + assert len(first.exec_calls) == calls_before + 1 + assert first.exec_calls[-1]["argv"] == ("sh", "-lc", "echo ok") + assert first.exec_calls[-1]["timeout"] == 10 + assert not any("health check" in record.message for record in caplog.records) + finally: + provider.shutdown() + + +@pytest.mark.parametrize("stdout", [b"not ok\n", b"ok-ish\n", b""]) +def test_warm_pool_logs_probe_without_exact_success_line(monkeypatch, caplog, stdout): + client = _FakeClient() + provider = _install(monkeypatch, client=client) + try: + sid = provider.acquire("thread-1", user_id="u1") + first = client.last_sandbox + provider.release(sid) + monkeypatch.setattr(first, "_run_script", lambda script: _FakeResult(stdout=stdout)) + + with caplog.at_level(logging.WARNING): + assert provider.acquire("thread-1", user_id="u1") == sid + assert client.create_count == 2 + assert client.last_sandbox is not first + assert first.closed + output = stdout.decode() if stdout else "(no output)" + assert any(f"Tenki warm-pool sandbox {sid} health check failed: {output}" == record.message for record in caplog.records) + finally: + provider.shutdown() + + +def test_warm_pool_rejects_probe_error_containing_success_line(monkeypatch, caplog): + client = _FakeClient() + provider = _install(monkeypatch, client=client) + try: + sid = provider.acquire("thread-1", user_id="u1") + first = client.last_sandbox + provider.release(sid) + first.exec_error = RuntimeError("probe transport failed\nok") + + with caplog.at_level(logging.WARNING): + assert provider.acquire("thread-1", user_id="u1") == sid + assert client.create_count == 2 + assert client.last_sandbox is not first + assert first.closed + assert any(f"Tenki warm-pool sandbox {sid} health check failed: Error: probe transport failed\nok" == record.message for record in caplog.records) + finally: + provider.shutdown() + + def test_release_parks_in_warm_pool(monkeypatch): client = _FakeClient() provider = _install(monkeypatch, client=client) diff --git a/backend/uv.lock b/backend/uv.lock index cf1812f73f..67a264b1fc 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1064,7 +1064,7 @@ requires-dist = [ { name = "redis", marker = "extra == 'redis'", specifier = ">=5.0.0" }, { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0,<3.0" }, { name = "tavily-python", specifier = ">=0.7.17" }, - { name = "tenki", marker = "extra == 'tenki'", specifier = ">=1.0.0" }, + { name = "tenki", marker = "extra == 'tenki'", specifier = ">=1.4.0" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=0.80" }, { name = "tiktoken", specifier = ">=0.8.0" }, { name = "tzdata", specifier = ">=2025.1" }, @@ -4653,16 +4653,16 @@ wheels = [ [[package]] name = "tenki" -version = "1.0.2" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio" }, { name = "protobuf" }, { name = "websocket-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/68/d9/1669b9ffe9b729a854985918ec84f39abf1ba032764a00bd105459e7e97d/tenki-1.0.2.tar.gz", hash = "sha256:131947e54758674f814a66699d3b65518adc9f093a62ea6dddbd4394cbc771c4", size = 240024, upload-time = "2026-08-27T19:40:32.85Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/64/232ea63ec1473c8ee8050881cb547e4176ea63fe1c1591a3bf19e111e379/tenki-1.4.0.tar.gz", hash = "sha256:eec11071341c2da5c439ad4316896c04bf29abe92e8ab4fd229011dd5499930e", size = 288846, upload-time = "2026-09-30T04:51:36.806Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/ef/06486b129744c3f7293f9b1f1ea60055104db4d4ad03483c6857c57c0bbb/tenki-1.0.2-py3-none-any.whl", hash = "sha256:6259651ed243e071e88d789fa86a7c570954ec5f0372a40bcccc2d4b9ae2e342", size = 164788, upload-time = "2026-08-27T19:40:30.915Z" }, + { url = "https://files.pythonhosted.org/packages/28/5b/c435195220f11ee46cb0a585f4abc4770f69a32626211523acb9b1a43798/tenki-1.4.0-py3-none-any.whl", hash = "sha256:8a2c4da58d807313d6f4da7d0bc0aa7c3db4001bbd751f84aac0130437624b2f", size = 198601, upload-time = "2026-09-30T04:51:35.24Z" }, ] [[package]]