deer-flow/backend/tests/test_local_sandbox_command_timeout.py

"""Regression tests for blocking-command timeout handling in LocalSandbox.

These pin the fix for the "starting a server hangs the whole turn" bug:
a backgrounded long-lived process must not keep the bash tool blocked until
the timeout, and a genuinely blocking foreground command must be terminated
(process group and all) once it exceeds the timeout.

The platform-specific cases exercise real subprocess and process-tree/group
semantics, while the shared tests pin the user-facing timeout notice.
"""

import os
import shlex
import signal
import subprocess
import sys
import time
from pathlib import Path

import pytest

from deerflow.config.sandbox_config import SandboxConfig
from deerflow.sandbox.local import local_sandbox
from deerflow.sandbox.local.local_sandbox import LocalSandbox

posix_only = pytest.mark.skipif(os.name == "nt", reason="POSIX process-group semantics")
linux_proc_fd_only = pytest.mark.skipif(not Path("/proc/self/fd").exists(), reason="requires Linux /proc fd links")


@posix_only
def test_backgrounded_process_returns_promptly():
    """A backgrounded long-lived process (e.g. a dev server started with `&`)
    must return as soon as the foreground command finishes, instead of
    blocking the bash tool until the timeout because it inherited the
    captured pipe."""
    sandbox = LocalSandbox("t")
    start = time.monotonic()
    output = sandbox.execute_command("sleep 5 & echo serving", timeout=10)
    elapsed = time.monotonic() - start

    assert elapsed < 3, f"expected prompt return, took {elapsed:.1f}s"
    assert "serving" in output


@posix_only
@linux_proc_fd_only
def test_backgrounded_process_does_not_inherit_deleted_temp_capture(tmp_path):
    """A backgrounded process that forgets to redirect output must not inherit
    an anonymous deleted temp file for fd 1. That would be an invisible,
    unbounded disk leak for long-lived processes that keep writing."""
    marker = tmp_path / "fd1"
    script = f"import os, pathlib, time; pathlib.Path({str(marker)!r}).write_text(os.readlink('/proc/self/fd/1')); time.sleep(2)"
    sandbox = LocalSandbox("t")

    output = sandbox.execute_command(f"{shlex.quote(sys.executable)} -c {shlex.quote(script)} & echo launched", timeout=10)

    assert "launched" in output
    for _ in range(50):
        if marker.exists():
            break
        time.sleep(0.1)
    assert marker.exists()
    assert " (deleted)" not in marker.read_text()


@posix_only
def test_foreground_blocking_command_times_out_with_notice():
    """A foreground command that never exits is terminated at the timeout and
    the agent receives an explanatory notice instead of a generic error."""
    sandbox = LocalSandbox("t")
    start = time.monotonic()
    output = sandbox.execute_command("while true; do sleep 0.2; done", timeout=1)
    elapsed = time.monotonic() - start

    assert elapsed < 5, f"timeout not enforced, took {elapsed:.1f}s"
    assert "timed out" in output.lower()


def test_timeout_notice_formats_fractional_and_singular_timeouts(monkeypatch):
    monkeypatch.setattr(LocalSandbox, "_get_shell", lambda self: "/bin/sh")
    runner = "_run_windows_command" if os.name == "nt" else "_run_posix_command"
    monkeypatch.setattr(LocalSandbox, runner, staticmethod(lambda args, timeout, env=None: ("", "", 0, True)))

    assert "after 1.5 seconds" in LocalSandbox("t").execute_command("wait", timeout=1.5)
    assert "after 1 second" in LocalSandbox("t").execute_command("wait", timeout=1)


def test_timeout_output_carries_authoritative_failure_marker(monkeypatch):
    """A timed-out command is a failed execution: the output must carry an
    exit marker so exit-status evidence (acceptance checklist) cannot read a
    partial passing summary as success."""
    canned = ("12 passed\n", "", 0, True)
    monkeypatch.setattr(LocalSandbox, "_get_shell", lambda self: "/bin/sh")
    monkeypatch.setattr(LocalSandbox, "_run_posix_command", staticmethod(lambda args, timeout, env=None: canned))
    # Windows hosts dispatch timed-out commands through the MSYS/PowerShell
    # runner instead of the POSIX one; stub it too so the marker contract is
    # hermetic and does not require make(1) or a resolvable shell path.
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(lambda *args, **kwargs: canned))

    output = LocalSandbox("t").execute_command("make test", timeout=1)

    assert "timed out" in output.lower()
    assert output.endswith("Exit Code: 124")


def test_windows_timeout_returns_notice(monkeypatch):
    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(LocalSandbox, "_get_shell", lambda self: "cmd.exe")
    monkeypatch.setattr(
        LocalSandbox,
        "_run_windows_command",
        staticmethod(lambda args, timeout, env, **kwargs: ("partial out", "partial err", 0, True)),
    )

    output = LocalSandbox("t").execute_command("wait", timeout=1.5)

    assert "partial out" in output
    assert "Std Error:" in output
    assert "partial err" in output
    assert "after 1.5 seconds" in output
    assert "Unexpected error" not in output


@pytest.mark.skipif(os.name != "nt", reason="Windows process-tree semantics")
def test_windows_foreground_timeout_is_wall_clock_bound(monkeypatch):
    monkeypatch.setattr(LocalSandbox, "_get_shell", lambda self: r"C:\Windows\System32\cmd.exe")
    sandbox = LocalSandbox("t")

    start = time.monotonic()
    output = sandbox.execute_command("ping -n 5 127.0.0.1", timeout=0.2)
    elapsed = time.monotonic() - start

    assert elapsed < 2, f"timeout not enforced for process tree, took {elapsed:.1f}s"
    assert "after 0.2 seconds" in output


@pytest.mark.skipif(os.name != "nt", reason="Windows bounded-capture semantics")
def test_windows_command_output_is_bounded(monkeypatch):
    monkeypatch.setattr(
        LocalSandbox,
        "_get_shell",
        lambda self: r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe",
    )
    emitted_chars = local_sandbox._COMMAND_CAPTURE_LIMIT_BYTES + 1024

    output = LocalSandbox("t").execute_command(f"[Console]::Out.Write('x' * {emitted_chars})", timeout=20)

    assert len(output) < emitted_chars
    assert "output truncated after 10485760 of 10486784 bytes" in output


@posix_only
def test_foreground_timeout_kills_whole_process_group(tmp_path):
    """On timeout the entire process group is killed, not just the direct
    child, so child processes spawned by the command do not survive."""
    marker = tmp_path / "alive"
    sandbox = LocalSandbox("t")
    sandbox.execute_command(f"while true; do touch {marker}; sleep 0.2; done", timeout=1)

    assert marker.exists()
    first_mtime = marker.stat().st_mtime
    time.sleep(1.5)
    assert marker.stat().st_mtime == first_mtime, "process group survived the timeout"


@posix_only
def test_command_reading_stdin_does_not_block():
    """stdin is redirected from /dev/null, so a command that reads stdin gets
    immediate EOF instead of blocking until the timeout."""
    sandbox = LocalSandbox("t")
    start = time.monotonic()
    output = sandbox.execute_command("read x; echo got", timeout=10)
    elapsed = time.monotonic() - start

    assert elapsed < 3, f"stdin read blocked, took {elapsed:.1f}s"
    assert "got" in output


@posix_only
def test_normal_command_output_exit_code_and_stderr():
    """Ordinary commands keep their existing output contract: stdout,
    appended Std Error section, and a non-zero Exit Code line."""
    sandbox = LocalSandbox("t")

    assert "hello" in sandbox.execute_command("echo hello")
    assert "Exit Code: 3" in sandbox.execute_command("exit 3")

    combined = sandbox.execute_command("echo out; echo oops >&2")
    assert "out" in combined
    assert "Std Error:" in combined
    assert "oops" in combined


def test_sandbox_config_exposes_command_timeout_default():
    cfg = SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider")
    assert cfg.bash_command_timeout == 600


def test_sandbox_config_exposes_health_check_skip_seconds_default():
    cfg = SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider")
    assert cfg.health_check_skip_seconds is None


def test_bash_tool_description_guides_backgrounding_long_lived_processes():
    """The bash tool description (seen by the model) must tell it to background
    long-lived processes like servers, so it doesn't block the turn in the
    foreground. This is the prompt-side half of the server-hang fix."""
    from deerflow.sandbox.tools import bash_tool

    description = bash_tool.description.lower()
    assert "background" in description
    assert "server" in description


def test_bash_tool_description_guides_safe_cross_platform_local_environment_probes():
    """The model-visible bash contract must recover from local path-guard failures (#4999)."""
    from deerflow.sandbox.tools import bash_tool

    description = " ".join(bash_tool.description.lower().split())
    assert "local host" in description
    assert "uname -s" in description
    assert "sw_vers" in description
    assert "only when the active sandbox policy permits it" in description
    assert "do not repeat the rejected command" in description
    assert "command-only probes" in description


@posix_only
def test_timeout_uses_saved_process_group_id_if_leader_exits(monkeypatch):
    """Timeout cleanup must target the saved group even if the leader has exited."""
    calls = []

    class ExitedLeaderProcess:
        pid = 4242

        def __init__(self):
            self.returncode = None
            self.wait_calls = 0

        def wait(self, timeout=None):
            self.wait_calls += 1
            if self.wait_calls == 1:
                # The command crossed the deadline as its leader exited; its
                # descendants may still hold the captured pipes open.
                self.returncode = 0
                raise subprocess.TimeoutExpired("fake-command", timeout)
            return self.returncode

        def kill(self):
            pytest.fail("fallback child kill should not replace process-group cleanup")

    process = ExitedLeaderProcess()
    monkeypatch.setattr(local_sandbox.subprocess, "Popen", lambda *args, **kwargs: process)
    monkeypatch.setattr(
        local_sandbox.os,
        "getpgid",
        lambda pid: pytest.fail("cleanup must not re-query an exited process leader"),
    )
    monkeypatch.setattr(
        local_sandbox.os,
        "killpg",
        lambda process_group_id, sig: calls.append((process_group_id, sig)),
    )

    _, _, returncode, timed_out = LocalSandbox._run_posix_command(["fake-command"], timeout=0.01)

    assert timed_out is True
    assert returncode == 0
    assert calls == [(process.pid, signal.SIGKILL)]


@posix_only
def test_process_group_lookup_failure_falls_back_to_direct_child_kill(monkeypatch):
    """A vanished process group is tolerated by killing/reaping the direct child."""
    calls = []

    class FakeProcess:
        pid = 12345

        def kill(self):
            calls.append(("kill-child",))

        def wait(self, timeout=None):
            calls.append(("wait", timeout))

    process = FakeProcess()

    def missing_process_group(process_group_id, sig):
        calls.append(("killpg", process_group_id, sig))
        raise ProcessLookupError

    monkeypatch.setattr(local_sandbox.os, "killpg", missing_process_group)

    # ESRCH from killpg must not escape; the direct-child fallback still runs.
    LocalSandbox._terminate_process_group(process, process.pid)

    assert calls == [
        ("killpg", process.pid, signal.SIGKILL),
        ("kill-child",),
        ("wait", 10),
    ]