fix(tenki): upgrade sdk and preserve timeout failures (#6162)

* fix(tenki): upgrade sdk and preserve timeout failures

* fix(tenki): tolerate noisy warm-pool health probes

* docs(tenki): compact sandbox guidance within chain budget

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

README.md

@@ -625,6 +625,12 @@ Remote directory listings report traversal failures (for example, unreadable
 directories) as incomplete results, even when no entries were returned. A
 missing start path is reported separately as “Directory not found.”
 
+The optional [Tenki cloud sandbox provider](backend/packages/harness/deerflow/community/tenki/README.md)
+uses Tenki SDK 1.4.0 or newer. Timed-out commands preserve partial output and
+report `Exit Code: 124`; unsuccessful health checks cannot reclaim a warm sandbox.
+Health probes tolerate login-shell output around the `ok` line, and failures log
+the sandbox ID and probe output before replacing the sandbox.
+
 #### MCP Server
 
 In the chat UI, enable **Token Usage → Debug** to inspect generic/MCP tool calls.

backend/packages/harness/deerflow/community/tenki/README.md

@@ -37,7 +37,7 @@ Install the optional SDK before selecting this provider:
 pip install "deerflow-harness[tenki]"
 ```
 
-The `tenki` package (which provides the `tenki_sandbox` module) is an optional
+The `tenki>=1.4.0` package (which provides the `tenki_sandbox` module) is an optional
 DeerFlow harness extra, not part of the default install. Get an API key from
 <https://tenki.cloud/docs/sandbox/sdk>.
 
@@ -67,6 +67,9 @@ API; directory and content search shell out and reuse `deerflow.sandbox.search`,
 mirroring `e2b_sandbox`:
 
 - `execute_command` — `sh -lc`, with per-call env and timeout.
+  Timed-out commands preserve partial output and report `Error: command timed out`
+  with `Exit Code: 124`, even if the SDK returned exit code zero. They are not
+  retried, and the sandbox remains available for subsequent commands.
 - `read_file` / `write_file` / `update_file` — native `fs.read_text` / `fs.mkdir` / `fs.write_stream` (binary-safe, streamed).
 - `download_file` — native `fs.read_stream`, restricted to the `/mnt/user-data` prefix; the 100 MB cap is enforced on bytes actually received, so a file growing mid-transfer cannot slip past it.
 - `list_dir` / `glob` / `grep` — `find` / `grep` with busybox-portable flags (the fs API is single-level and has no content search); results filtered/capped in Python and reported back under `/mnt/user-data`.
@@ -83,6 +86,11 @@ sandboxes. `sandbox.idle_timeout` controls how long released warm sandboxes stay
 running; `0` disables idle reaping. Active sandboxes are never evicted to satisfy
 the cap.
 
+The warm-pool health probe accepts an exact `ok` line alongside login-shell
+profile output or stderr warnings. Timeout, nonzero-exit, and command-error
+diagnostics still prevent reuse. Failed probes log the sandbox ID and returned
+output before the unhealthy microVM is terminated and replaced.
+
 ## Scope: stable features only
 
 Only the stable Tenki surface is used — sandbox create/terminate plus

backend/packages/harness/deerflow/community/tenki/provider.py

@@ -330,11 +330,12 @@ class TenkiSandboxProvider(WarmPoolLifecycleMixin[TenkiSandbox], SandboxProvider
         # Best-effort: on failure the file APIs still work via the home remap.
         try:
             result = remote.exec("sh", "-lc", _bootstrap_script(self._config["home_dir"]), timeout=_BOOTSTRAP_TIMEOUT)
-            if result.exit_code not in (0, None) or "BOOTSTRAP_OK" not in (result.stdout_text or ""):
+            if result.timed_out or result.exit_code not in (0, None) or "BOOTSTRAP_OK" not in (result.stdout_text or ""):
                 logger.warning(
-                    "Tenki bootstrap for %s exited code=%s stderr=%s",
+                    "Tenki bootstrap for %s exited code=%s timed_out=%s stderr=%s",
                     sandbox_id,
                     result.exit_code,
+                    result.timed_out,
                     (result.stderr_text or "").strip(),
                 )
         except Exception as e:
@@ -407,7 +408,12 @@ class TenkiSandboxProvider(WarmPoolLifecycleMixin[TenkiSandbox], SandboxProvider
 
         try:
             result = sandbox.execute_command("echo ok", timeout=10)
-            healthy = "ok" in result
+            # Login-shell profiles can add stdout/stderr around the probe marker.
+            # Keep rejecting adapter failure diagnostics even if "ok" was printed.
+            lines = result.splitlines()
+            healthy = "ok" in lines and not any(line.startswith(("Error:", "Exit Code:")) for line in lines)
+            if not healthy:
+                logger.warning("Tenki warm-pool sandbox %s health check failed: %s", sandbox_id, result)
         except Exception as e:
             logger.warning("Tenki warm-pool sandbox %s health check error: %s", sandbox_id, e)
             healthy = False

backend/packages/harness/deerflow/community/tenki/sandbox.py

@@ -274,6 +274,9 @@ class TenkiSandbox(Sandbox):
             output = f"{stdout}\n{stderr}"
         else:
             output = stdout or stderr
+        if result.timed_out:
+            message = "Error: command timed out\nExit Code: 124"
+            return f"{output}\n{message}" if output else message
         if result.exit_code not in (0, None):
             # Mirror LocalSandbox: preserve a nonzero exit in the output text
             # even when the command produced output (see e2b_sandbox).

backend/packages/harness/deerflow/sandbox/AGENTS.md

@@ -92,7 +92,8 @@ cancellation before propagating it.
 - `BoxliteProvider` (`packages/harness/deerflow/community/boxlite/`) - optional BoxLite micro-VM isolation. Loop-affine handles live on one private daemon-thread asyncio loop; sync `Sandbox` calls marshal through `run_coroutine_threadsafe`.
   Boxes use deterministic `user_id:thread_id` IDs, park in a same-owner warm pool, and are reclaimed only by that owner. Health-check timeouts reach both BoxLite `exec(timeout=...)` and private-loop `.result(timeout)`; a bridge timeout must cancel its coroutine so work cannot continue or retain acquire ownership in the background.
   `sandbox.replicas` caps active + warm VMs per gateway process; if capacity is exhausted, only warm-pool VMs are evicted. `sandbox.idle_timeout` stops idle warm VMs after the configured seconds. `reset()` is intentionally a lightweight registry clear for `reset_sandbox_provider()` and does not close boxes, stop the idle reaper, or close the private loop; full teardown remains `shutdown()`.
-- `TenkiSandboxProvider` (`packages/harness/deerflow/community/tenki/`) - Tenki cloud microVM isolation. The `tenki` SDK is optional (`deerflow-harness[tenki]`, importable as `tenki_sandbox`) and lazy-imported (`_import_client`) only when this provider is selected. Unlike Boxlite, the SDK is synchronous, so the adapter calls it directly with no event-loop bridge. File transport uses Tenki's native `sandbox.fs` API (`read_text`/`read_stream`/`write_stream`/`mkdir`/`stat`) — binary-safe and streaming, no base64/shell hop; only directory/content *search* (`list_dir`/`glob`/`grep`) shells out to busybox-portable `find`/`grep`, parsed with the shared `deerflow.sandbox.search` helpers like `community/e2b_sandbox`. Sandboxes run as the unprivileged `tenki` user, so DeerFlow's `/mnt/user-data` prefix is remapped under a writable HOME (`_resolve_path`) and best-effort `sudo`-symlinked at bootstrap. Boxes are named deterministically from `sha256(user_id:thread_id)[:16]` (64-bit, matching E2B; the warm pool is keyed by this id alone with no full-seed fallback), released into an in-process warm pool, and reclaimed only by the same user/thread after a liveness check. A terminal session error (named SDK errors plus builtin `ConnectionError`/`BrokenPipeError`/`EOFError`) routes through `_invalidate_sandbox` to evict the dead microVM. Cross-process orphan reconciliation is a follow-up (single-process warm pool today).
+- `TenkiSandboxProvider` (`community/tenki/`): cloud microVMs. Optional `tenki` SDK (`deerflow-harness[tenki]`, module `tenki_sandbox`) loads via `_import_client` only on provider selection; calls are synchronous, without an event-loop bridge. Native `sandbox.fs` (`read_text`/`read_stream`/`write_stream`/`mkdir`/`stat`) streams binary-safe files without shell/base64. Only search (`list_dir`/`glob`/`grep`) uses busybox-portable `find`/`grep` and shared `deerflow.sandbox.search` parsing, like E2B. The unprivileged `tenki` user needs `/mnt/user-data` remapped to writable HOME (`_resolve_path`), with a best-effort bootstrap `sudo` symlink. Deterministic names and warm-pool keys use `sha256(user_id:thread_id)[:16]` (64-bit, matching E2B), with no full-seed fallback. Release parks VMs in-process; only the same user/thread reclaims them after a health check. Named terminal SDK errors and builtin `ConnectionError`/`BrokenPipeError`/`EOFError` evict via `_invalidate_sandbox`. Cross-process orphan reconciliation remains a follow-up.
+  Health probes require an exact `ok` line and reject `Error:`/`Exit Code:` diagnostic lines; other login-shell stdout/stderr is tolerated. Failures log sandbox id and output before eviction.
 
 
 **Shared warm-pool lifecycle:** community sandbox providers that keep released sandboxes alive for fast reuse share `deerflow.community.warm_pool_lifecycle.WarmPoolLifecycleMixin`. The mixin owns the common `DEFAULT_IDLE_TIMEOUT=600`, `IDLE_CHECK_INTERVAL=60`, `DEFAULT_REPLICAS=3`, idle-checker loop, warm-pool expiry, oldest-warm eviction, replica counting, and soft-cap logging. Providers remain responsible for their own active registries, creation/discovery, health checks, and destroy hook (`_destroy_warm_entry`): AIO destroys `SandboxInfo` through its backend; Boxlite closes loop-affine `BoxliteBox` handles; Tenki closes the microVM session (`TenkiSandbox.close`, which terminates the remote sandbox). AIO keeps active-idle cleanup outside the mixin and delegates only warm-pool expiry to the shared helper.

backend/packages/harness/pyproject.toml

@@ -95,7 +95,7 @@ boxlite = ["boxlite>=0.9.7"]
 # install stays free of the Tenki SDK; only pulled in when the provider is used.
 # The distribution is ``tenki``; it still ships the ``tenki_sandbox`` module that
 # provider.py and sandbox.py import.
-tenki = ["tenki>=1.0.0"]
+tenki = ["tenki>=1.4.0"]
 # OpenSandbox remote sandbox provider (deerflow.community.opensandbox). The
 # sync SDK is loaded only when this provider is selected.
 opensandbox = ["opensandbox>=0.1.15,<0.2.0"]

backend/tests/test_tenki_provider.py

@@ -31,10 +31,11 @@ from deerflow.config.app_config import AppConfig
 
 
 class _FakeResult:
-    def __init__(self, exit_code: int = 0, stdout: bytes = b"", stderr: bytes = b"") -> None:
+    def __init__(self, exit_code: int = 0, stdout: bytes = b"", stderr: bytes = b"", timed_out: bool = False) -> None:
         self.exit_code = exit_code
         self.stdout = stdout
         self.stderr = stderr
+        self.timed_out = timed_out
 
     @property
     def stdout_text(self) -> str:
@@ -388,6 +389,26 @@ def test_execute_command_returns_error_as_text() -> None:
     assert box.execute_command("echo hi") == "Error: boom"
 
 
+@pytest.mark.parametrize("exit_code", [0, 143])
+@pytest.mark.parametrize("stdout,stderr", [(b"partial output", b"diagnostic"), (b"", b"")])
+def test_execute_command_reports_timeout_and_preserves_output(monkeypatch, exit_code, stdout, stderr) -> None:
+    fake = _FakeSandbox()
+    monkeypatch.setattr(fake, "_run_script", lambda script: _FakeResult(exit_code=exit_code, stdout=stdout, stderr=stderr, timed_out=True))
+    invalidated = []
+    box = TenkiSandbox("sb", fake, on_terminal_failure=lambda *args: invalidated.append(args))
+
+    output = box.execute_command("sleep 10", timeout=1)
+
+    assert "Error: command timed out" in output
+    assert output.endswith("Exit Code: 124")
+    assert stdout.decode() in output
+    assert stderr.decode() in output
+    assert len(fake.exec_calls) == 1
+    assert fake.exec_calls[0]["timeout"] == 1
+    assert not invalidated
+    assert not box.is_closed
+
+
 def test_execute_command_closed_returns_error() -> None:
     box = TenkiSandbox("sb", _FakeSandbox())
     box.close()
@@ -845,6 +866,127 @@ def test_bootstrap_is_non_interactive_and_time_bounded(monkeypatch):
     provider.shutdown()
 
 
+def test_timed_out_bootstrap_warns_even_with_success_marker(monkeypatch, caplog):
+    class _TimedOutBootstrap(_FakeSandbox):
+        def _run_script(self, script: str) -> _FakeResult:
+            if "BOOTSTRAP_OK" in script:
+                return _FakeResult(stdout=b"BOOTSTRAP_OK\n", timed_out=True)
+            return super()._run_script(script)
+
+    provider = _install(monkeypatch, client=_FakeClient(sandbox_factory=_TimedOutBootstrap))
+    try:
+        with caplog.at_level("WARNING"):
+            sid = provider.acquire("thread-1", user_id="u1")
+        assert provider.get(sid) is not None
+        assert any("bootstrap" in r.message and "timed_out=True" in r.message for r in caplog.records)
+    finally:
+        provider.shutdown()
+
+
+@pytest.mark.parametrize("timed_out,exit_code", [(True, 0), (False, 1)])
+def test_warm_pool_rejects_unsuccessful_probe_with_ok_output(monkeypatch, caplog, timed_out, exit_code):
+    client = _FakeClient()
+    provider = _install(monkeypatch, client=client)
+    try:
+        sid = provider.acquire("thread-1", user_id="u1")
+        first = client.last_sandbox
+        provider.release(sid)
+        monkeypatch.setattr(
+            first,
+            "_run_script",
+            lambda script: _FakeResult(stdout=b"ok\n", exit_code=exit_code, timed_out=timed_out),
+        )
+
+        with caplog.at_level(logging.WARNING):
+            assert provider.acquire("thread-1", user_id="u1") == sid
+        assert client.last_sandbox is not first
+        assert first.closed
+        failure = next(record.message for record in caplog.records if "health check failed" in record.message)
+        assert sid in failure
+        assert "ok\n" in failure
+        assert f"Exit Code: {124 if timed_out else exit_code}" in failure
+    finally:
+        provider.shutdown()
+
+
+@pytest.mark.parametrize(
+    "stdout,stderr",
+    [
+        (b"ok\n", b"sh: warning: setlocale: LC_ALL: cannot change locale\n"),
+        (b"profile loaded\nok\n", b""),
+        (
+            b"profile loaded\nok\n",
+            b"sh: warning: setlocale: LC_ALL: cannot change locale\n",
+        ),
+    ],
+)
+def test_warm_pool_reclaims_healthy_probe_with_shell_noise(monkeypatch, caplog, stdout, stderr):
+    client = _FakeClient()
+    provider = _install(monkeypatch, client=client)
+    try:
+        sid = provider.acquire("thread-1", user_id="u1")
+        first = client.last_sandbox
+        provider.release(sid)
+        monkeypatch.setattr(
+            first,
+            "_run_script",
+            lambda script: _FakeResult(stdout=stdout, stderr=stderr),
+        )
+        calls_before = len(first.exec_calls)
+
+        with caplog.at_level(logging.WARNING):
+            assert provider.acquire("thread-1", user_id="u1") == sid
+        assert client.create_count == 1
+        assert client.last_sandbox is first
+        assert not first.closed
+        assert len(first.exec_calls) == calls_before + 1
+        assert first.exec_calls[-1]["argv"] == ("sh", "-lc", "echo ok")
+        assert first.exec_calls[-1]["timeout"] == 10
+        assert not any("health check" in record.message for record in caplog.records)
+    finally:
+        provider.shutdown()
+
+
+@pytest.mark.parametrize("stdout", [b"not ok\n", b"ok-ish\n", b""])
+def test_warm_pool_logs_probe_without_exact_success_line(monkeypatch, caplog, stdout):
+    client = _FakeClient()
+    provider = _install(monkeypatch, client=client)
+    try:
+        sid = provider.acquire("thread-1", user_id="u1")
+        first = client.last_sandbox
+        provider.release(sid)
+        monkeypatch.setattr(first, "_run_script", lambda script: _FakeResult(stdout=stdout))
+
+        with caplog.at_level(logging.WARNING):
+            assert provider.acquire("thread-1", user_id="u1") == sid
+        assert client.create_count == 2
+        assert client.last_sandbox is not first
+        assert first.closed
+        output = stdout.decode() if stdout else "(no output)"
+        assert any(f"Tenki warm-pool sandbox {sid} health check failed: {output}" == record.message for record in caplog.records)
+    finally:
+        provider.shutdown()
+
+
+def test_warm_pool_rejects_probe_error_containing_success_line(monkeypatch, caplog):
+    client = _FakeClient()
+    provider = _install(monkeypatch, client=client)
+    try:
+        sid = provider.acquire("thread-1", user_id="u1")
+        first = client.last_sandbox
+        provider.release(sid)
+        first.exec_error = RuntimeError("probe transport failed\nok")
+
+        with caplog.at_level(logging.WARNING):
+            assert provider.acquire("thread-1", user_id="u1") == sid
+        assert client.create_count == 2
+        assert client.last_sandbox is not first
+        assert first.closed
+        assert any(f"Tenki warm-pool sandbox {sid} health check failed: Error: probe transport failed\nok" == record.message for record in caplog.records)
+    finally:
+        provider.shutdown()
+
+
 def test_release_parks_in_warm_pool(monkeypatch):
     client = _FakeClient()
     provider = _install(monkeypatch, client=client)

backend/uv.lock

@@ -1064,7 +1064,7 @@ requires-dist = [
     { name = "redis", marker = "extra == 'redis'", specifier = ">=5.0.0" },
     { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0,<3.0" },
     { name = "tavily-python", specifier = ">=0.7.17" },
-    { name = "tenki", marker = "extra == 'tenki'", specifier = ">=1.0.0" },
+    { name = "tenki", marker = "extra == 'tenki'", specifier = ">=1.4.0" },
     { name = "textual", marker = "extra == 'tui'", specifier = ">=0.80" },
     { name = "tiktoken", specifier = ">=0.8.0" },
     { name = "tzdata", specifier = ">=2025.1" },
@@ -4653,16 +4653,16 @@ wheels = [
 
 [[package]]
 name = "tenki"
-version = "1.0.2"
+version = "1.4.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "grpcio" },
     { name = "protobuf" },
     { name = "websocket-client" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/68/d9/1669b9ffe9b729a854985918ec84f39abf1ba032764a00bd105459e7e97d/tenki-1.0.2.tar.gz", hash = "sha256:131947e54758674f814a66699d3b65518adc9f093a62ea6dddbd4394cbc771c4", size = 240024, upload-time = "2026-08-27T19:40:32.85Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/62/64/232ea63ec1473c8ee8050881cb547e4176ea63fe1c1591a3bf19e111e379/tenki-1.4.0.tar.gz", hash = "sha256:eec11071341c2da5c439ad4316896c04bf29abe92e8ab4fd229011dd5499930e", size = 288846, upload-time = "2026-09-30T04:51:36.806Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/98/ef/06486b129744c3f7293f9b1f1ea60055104db4d4ad03483c6857c57c0bbb/tenki-1.0.2-py3-none-any.whl", hash = "sha256:6259651ed243e071e88d789fa86a7c570954ec5f0372a40bcccc2d4b9ae2e342", size = 164788, upload-time = "2026-08-27T19:40:30.915Z" },
+    { url = "https://files.pythonhosted.org/packages/28/5b/c435195220f11ee46cb0a585f4abc4770f69a32626211523acb9b1a43798/tenki-1.4.0-py3-none-any.whl", hash = "sha256:8a2c4da58d807313d6f4da7d0bc0aa7c3db4001bbd751f84aac0130437624b2f", size = 198601, upload-time = "2026-09-30T04:51:35.24Z" },
 ]
 
 [[package]]