deer-flow/backend/tests/test_local_sandbox_encoding.py

import builtins
import ctypes
import os
import shutil
import subprocess
import sys
from types import SimpleNamespace

import pytest
from support.shell import require_posix_sh

import deerflow.sandbox.local.local_sandbox as local_sandbox
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping, _BoundedPipeCapture


def _open(base, file, mode="r", *args, **kwargs):
    if "b" in mode:
        return base(file, mode, *args, **kwargs)
    return base(file, mode, *args, encoding=kwargs.pop("encoding", "gbk"), **kwargs)


def test_bounded_pipe_capture_decodes_non_utf8_output_with_configured_encoding():
    capture = _BoundedPipeCapture(encoding="cp1252")
    capture.append("caf\u00e9".encode("cp1252"))

    assert capture.read() == "caf\u00e9"


def test_bounded_pipe_capture_falls_back_to_locale_for_non_utf8_bytes():
    """A forced UTF-8 decode keeps the locale as a guarded fallback so output
    from native children (which still use the host code page) survives."""
    capture = _BoundedPipeCapture(encoding="utf-8", fallback_encoding="cp936")
    capture.append("\u4f60\u597d".encode("cp936"))

    assert capture.read() == "\u4f60\u597d"


def test_bounded_pipe_capture_prefers_primary_encoding_when_valid():
    capture = _BoundedPipeCapture(encoding="utf-8", fallback_encoding="cp936")
    capture.append("\u4f60\u597d".encode("utf-8"))

    assert capture.read() == "\u4f60\u597d"


def test_bounded_pipe_capture_without_fallback_replaces_non_utf8_bytes():
    capture = _BoundedPipeCapture(encoding="utf-8")
    capture.append("\u4f60\u597d".encode("cp936"))

    decoded = capture.read()
    assert "\ufffd" in decoded
    assert "\u4f60\u597d" not in decoded


def test_bounded_pipe_capture_trims_capture_cap_split_multibyte_tail():
    """The byte cap can cut a multi-byte UTF-8 char in half; strict-decoding the
    partial tail would fail the whole-buffer gate and cascade everything to the
    locale fallback, so the trailing incomplete sequence is trimmed first."""
    capture = _BoundedPipeCapture(limit_bytes=10, encoding="utf-8", fallback_encoding="cp936")
    capture.append(b"abcdefgh")
    # 2 bytes of the 6-byte "你好" fit under the cap, splitting 你 (e4 bd a0)
    # after its continuation byte — a partial sequence at the buffer end.
    capture.append("\u4f60\u597d".encode("utf-8"))

    assert capture.read() == "abcdefgh\n... [output truncated after 10 of 14 bytes; remaining output discarded] ..."


def test_bounded_pipe_capture_preserves_complete_locale_tail_for_fallback():
    """A complete buffer that ends in a byte shaped like an incomplete UTF-8
    lead must reach the locale fallback untouched: on a cp1252 host the child
    writes ``caf\xe9`` (a complete é), and trimming that byte would silently
    drop it while accepting the ASCII prefix as valid UTF-8."""
    capture = _BoundedPipeCapture(encoding="utf-8", fallback_encoding="cp1252")
    capture.append(b"caf\xe9")

    assert capture.read() == "caf\u00e9"


def test_bounded_pipe_capture_trims_partial_tail_left_by_timeout_kill():
    """A child killed mid-write can leave an incomplete multi-byte sequence at
    the end of the capture; marking the split tail authorizes the trim retry
    so the valid prefix stays decodable."""
    capture = _BoundedPipeCapture(encoding="utf-8", fallback_encoding="cp936")
    capture.append(b"abc")
    capture.append("\u4f60".encode("utf-8")[:-1])
    capture.mark_split_tail()

    assert capture.read() == "abc"


def test_bounded_pipe_capture_without_cut_signal_falls_back_on_partial_shaped_tail():
    """The same partial-shaped tail without a cut signal (no cap cut, no
    timeout kill) must NOT be trimmed away: the whole buffer goes to the
    locale fallback instead of being accepted as a truncated UTF-8 prefix.
    cp1252 is used as the fallback so every byte has a defined mapping."""
    capture = _BoundedPipeCapture(encoding="utf-8", fallback_encoding="cp1252")
    capture.append(b"abc")
    capture.append("\u4f60".encode("utf-8")[:-1])

    assert capture.read() == "abcä½"


def test_windows_command_timeout_marks_capture_for_trim(monkeypatch):
    """A real timeout kill must authorize the trim retry through
    ``_run_windows_command``: the child wrote a partial UTF-8 sequence and was
    killed mid-command, so the trimmed empty prefix survives instead of the
    whole buffer cascading to the locale fallback as U+FFFD."""
    if os.name != "nt":
        monkeypatch.setattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0, raising=False)
    partial = "\u4f60".encode("utf-8")[:-1]
    script = f"import os, time; os.write(1, bytes.fromhex('{partial.hex()}')); time.sleep(30)"

    stdout, stderr, returncode, timed_out = LocalSandbox._run_windows_command(
        [sys.executable, "-c", script],
        1,
        encoding="utf-8",
        fallback_encoding="cp936",
    )

    assert timed_out is True
    assert stdout == ""


def test_bounded_pipe_capture_applies_text_mode_newline_normalization_when_enabled():
    capture = _BoundedPipeCapture(normalize_newlines=True)
    capture.append(b"crlf\r\nbare-cr\rlf\n")

    assert capture.read() == "crlf\nbare-cr\nlf\n"


def test_bounded_pipe_capture_preserves_posix_newlines_by_default():
    capture = _BoundedPipeCapture()
    capture.append(b"crlf\r\nbare-cr\rlf\n")

    assert capture.read() == "crlf\r\nbare-cr\rlf\n"


@pytest.mark.parametrize("encoding", [None, "utf-8"])
def test_windows_pipe_capture_uses_explicit_encoding_or_locale(monkeypatch, encoding):
    """Exercise real pipes with a legacy locale, even on a POSIX test host."""
    monkeypatch.setattr(local_sandbox.locale, "getpreferredencoding", lambda _: "cp936")
    if os.name != "nt":
        monkeypatch.setattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0, raising=False)
    expected = "你好 日本語"
    payload = (expected + "\r\n").encode(encoding or "cp936")

    stdout, stderr, returncode, timed_out = LocalSandbox._run_windows_command(
        [sys.executable, "-c", f"import os; p=bytes.fromhex('{payload.hex()}'); os.write(1,p); os.write(2,p)"],
        10,
        encoding=encoding,
    )

    assert stdout == stderr == expected + "\n"
    assert returncode == 0
    assert timed_out is False


@pytest.mark.skipif(os.name != "nt", reason="Requires real Windows PowerShell")
@pytest.mark.parametrize("shell_name", ["powershell.exe", "pwsh.exe"])
@pytest.mark.parametrize("no_console", [False, True], ids=["inherited-console", "no-console"])
def test_windows_powershell_cjk_roundtrip(shell_name, no_console):
    shell = shutil.which(shell_name)
    if shell is None:
        pytest.skip(f"{shell_name} is not installed")
    probe = r"""
import sys
import deerflow.sandbox.local.local_sandbox as local_sandbox
from deerflow.sandbox.local.local_sandbox import LocalSandbox

# Keep this regression effective even on an English or UTF-8 Windows runner.
local_sandbox.locale.getencoding = lambda: "cp936"
LocalSandbox._get_shell = staticmethod(lambda: sys.argv[1])
expected = "\u4f60\u597d \u65e5\u672c\u8a9e"
command = f"Write-Output '{expected}'; [Console]::Error.WriteLine('{expected}'); exit 3"
output = LocalSandbox("encoding-probe").execute_command(command, timeout=15)
assert output == expected + "\n\nStd Error:\n" + expected + "\n\nExit Code: 3", ascii(output)
"""
    # Run the probe under Python UTF-8 mode on purpose: ``getpreferredencoding``
    # would report ``"utf-8"`` there and silently disable the locale fallback,
    # so a green run proves the fallback source is ``locale.getencoding()``.
    env = {**os.environ, "PYTHONUTF8": "1"}
    result = subprocess.run(
        [sys.executable, "-c", probe, shell],
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        env=env,
        creationflags=subprocess.CREATE_NO_WINDOW if no_console else 0,
        timeout=30,
        check=False,
    )
    assert result.returncode == 0, result.stdout + result.stderr


@pytest.mark.skipif(os.name != "nt", reason="Requires a real Windows Git Bash/MSYS shell")
def test_windows_msys_cjk_roundtrip():
    """Git Bash/MSYS always emits UTF-8, so the host code page must not decode it."""
    shell = require_posix_sh()
    if not LocalSandbox._is_msys_shell(shell):
        pytest.skip(f"{shell} is not recognised as a Git Bash/MSYS shell")
    probe = r"""
import sys
import deerflow.sandbox.local.local_sandbox as local_sandbox
from deerflow.sandbox.local.local_sandbox import LocalSandbox

# Keep this regression effective even on an English or UTF-8 Windows runner.
local_sandbox.locale.getencoding = lambda: "cp936"
LocalSandbox._get_shell = staticmethod(lambda: sys.argv[1])
expected = "\u4f60\u597d \u65e5\u672c\u8a9e"
output = LocalSandbox("encoding-probe").execute_command(f"printf '%s\n' '{expected}'", timeout=15)
assert output == expected + "\n", ascii(output)
"""
    # Run the probe under Python UTF-8 mode on purpose: ``getpreferredencoding``
    # would report ``"utf-8"`` there and silently disable the locale fallback,
    # so a green run proves the fallback source is ``locale.getencoding()``.
    env = {**os.environ, "PYTHONUTF8": "1"}
    result = subprocess.run(
        [sys.executable, "-c", probe, shell],
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        env=env,
        timeout=30,
        check=False,
    )
    assert result.returncode == 0, result.stdout + result.stderr


@pytest.mark.skipif(os.name != "nt", reason="Requires a real Windows Git Bash/MSYS shell")
def test_windows_msys_native_child_cjk_roundtrip():
    """A Windows-native child of Git Bash (python.exe) inherits the pipe and keeps
    writing the host code page; the forced UTF-8 decode must fall back to the
    locale for those bytes instead of replacing the text with U+FFFD."""
    shell = require_posix_sh()
    if not LocalSandbox._is_msys_shell(shell):
        pytest.skip(f"{shell} is not recognised as a Git Bash/MSYS shell")
    probe = r"""
import sys
import deerflow.sandbox.local.local_sandbox as local_sandbox
from deerflow.sandbox.local.local_sandbox import LocalSandbox

# Pin the fallback code page so the regression holds on non-zh-CN runners.
local_sandbox.locale.getencoding = lambda: "cp936"
LocalSandbox._get_shell = staticmethod(lambda: sys.argv[1])
expected = "\u4f60\u597d \u65e5\u672c\u8a9e"
interpreter = sys.executable.replace("\\", "/")
command = f"'{interpreter}' -c \"print('{expected}')\""
# PYTHONIOENCODING makes the native child emit CP936 even on a UTF-8 host,
# mirroring a zh-CN host's native Python writing to a redirected pipe.
output = LocalSandbox("encoding-probe").execute_command(command, timeout=30, env={"PYTHONIOENCODING": "cp936"})
assert output == expected + "\n", ascii(output)
"""
    # Run the probe under Python UTF-8 mode on purpose: ``getpreferredencoding``
    # would report ``"utf-8"`` there and silently disable the locale fallback,
    # so a green run proves the fallback source is ``locale.getencoding()``.
    env = {**os.environ, "PYTHONUTF8": "1"}
    result = subprocess.run(
        [sys.executable, "-c", probe, shell],
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        env=env,
        timeout=60,
        check=False,
    )
    assert result.returncode == 0, result.stdout + result.stderr


@pytest.mark.skipif(os.name != "nt", reason="Requires real Windows PowerShell")
@pytest.mark.parametrize("shell_name", ["powershell.exe", "pwsh.exe"])
def test_windows_powershell_native_child_cjk_roundtrip(shell_name):
    """A Windows-native child PowerShell runs (python.exe) inherits the pipe and
    keeps writing the host code page; the guarded console-UTF-8 preamble only
    covers PowerShell's own output, so the forced UTF-8 decode must fall back to
    the locale for those bytes instead of replacing the text with U+FFFD."""
    shell = shutil.which(shell_name)
    if shell is None:
        pytest.skip(f"{shell_name} is not installed")
    probe = r"""
import sys
import deerflow.sandbox.local.local_sandbox as local_sandbox
from deerflow.sandbox.local.local_sandbox import LocalSandbox

# Pin the fallback code page so the regression holds on non-zh-CN runners.
local_sandbox.locale.getencoding = lambda: "cp936"
LocalSandbox._get_shell = staticmethod(lambda: sys.argv[1])
expected = "\u4f60\u597d \u65e5\u672c\u8a9e"
interpreter = sys.executable.replace("\\", "/")
command = f"& '{interpreter}' -c \"print('{expected}')\""
# PYTHONIOENCODING makes the native child emit CP936 even on a UTF-8 host,
# mirroring a zh-CN host's native Python writing to a redirected pipe.
output = LocalSandbox("encoding-probe").execute_command(command, timeout=30, env={"PYTHONIOENCODING": "cp936"})
assert output == expected + "\n", ascii(output)
"""
    # Run the probe under Python UTF-8 mode on purpose: ``getpreferredencoding``
    # would report ``"utf-8"`` there and silently disable the locale fallback,
    # so a green run proves the fallback source is ``locale.getencoding()``.
    env = {**os.environ, "PYTHONUTF8": "1"}
    result = subprocess.run(
        [sys.executable, "-c", probe, shell],
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        env=env,
        timeout=60,
        check=False,
    )
    assert result.returncode == 0, result.stdout + result.stderr


@pytest.mark.skipif(os.name == "nt", reason="POSIX capture semantics")
def test_posix_command_capture_preserves_newlines():
    stdout, stderr, returncode, timed_out = LocalSandbox._run_posix_command(
        [sys.executable, "-c", "import os; os.write(1, b'crlf\\r\\nbare-cr\\rlf\\n')"],
        10,
    )

    assert stdout == "crlf\r\nbare-cr\rlf\n"
    assert stderr == ""
    assert returncode == 0
    assert timed_out is False


@pytest.mark.skipif(os.name != "nt", reason="Windows text-mode newline semantics")
def test_windows_command_capture_normalizes_newlines():
    stdout, stderr, returncode, timed_out = LocalSandbox._run_windows_command(
        [sys.executable, "-c", "import os; os.write(1, b'crlf\\r\\nbare-cr\\rlf\\n')"],
        10,
    )

    assert stdout == "crlf\nbare-cr\nlf\n"
    assert stderr == ""
    assert returncode == 0
    assert timed_out is False


@pytest.mark.skipif(os.name != "nt", reason="Windows text-mode encoding semantics")
@pytest.mark.parametrize(
    ("python_args", "python_utf8"),
    [([], "0"), ([], "1"), (["-X", "utf8"], "0")],
    ids=["locale-code-page", "PYTHONUTF8", "-X-utf8"],
)
def test_windows_capture_matches_subprocess_text_mode_encoding(python_args, python_utf8):
    probe = r"""
import subprocess
import sys

from deerflow.sandbox.local.local_sandbox import LocalSandbox

reference = subprocess.Popen([sys.executable, "-c", ""], stdout=subprocess.PIPE, text=True)
encoding = reference.stdout.encoding
reference.communicate()

for expected in ("caf\u00e9", "\u4f60\u597d", "\u65e5\u672c\u8a9e", "\u041f\u0440\u0438\u0432\u0435\u0442"):
    try:
        payload = expected.encode(encoding)
    except UnicodeEncodeError:
        continue
    if any(byte >= 0x80 for byte in payload):
        break
else:
    raise AssertionError(f"no non-ASCII probe text for {encoding}")

stdout, stderr, returncode, timed_out = LocalSandbox._run_windows_command(
    [sys.executable, "-c", f"import sys; sys.stdout.buffer.write(bytes.fromhex('{payload.hex()}'))"],
    10,
)
assert stdout == expected, (encoding, stdout)
assert stderr == ""
assert returncode == 0
assert timed_out is False
"""
    env = os.environ.copy()
    env["PYTHONUTF8"] = python_utf8

    result = subprocess.run(
        [sys.executable, *python_args, "-c", probe],
        capture_output=True,
        text=True,
        encoding="utf-8",
        env=env,
        check=False,
    )

    assert result.returncode == 0, result.stdout + result.stderr


def test_read_file_uses_utf8_on_windows_locale(tmp_path, monkeypatch):
    path = tmp_path / "utf8.txt"
    text = "\u201cutf8\u201d"
    path.write_text(text, encoding="utf-8")
    base = builtins.open

    monkeypatch.setattr(local_sandbox, "open", lambda file, mode="r", *args, **kwargs: _open(base, file, mode, *args, **kwargs), raising=False)

    assert LocalSandbox("t").read_file(str(path)) == text


def test_write_file_uses_utf8_on_windows_locale(tmp_path, monkeypatch):
    path = tmp_path / "utf8.txt"
    text = "emoji \U0001f600"
    base = builtins.open

    monkeypatch.setattr(local_sandbox, "open", lambda file, mode="r", *args, **kwargs: _open(base, file, mode, *args, **kwargs), raising=False)

    LocalSandbox("t").write_file(str(path), text)

    assert path.read_text(encoding="utf-8") == text


def test_get_shell_prefers_posix_shell_from_path_before_windows_fallback(monkeypatch):
    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(LocalSandbox, "_find_first_available_shell", lambda candidates: r"C:\Program Files\Git\bin\sh.exe" if candidates == ("/bin/zsh", "/bin/bash", "/bin/sh", "sh") else None)

    assert LocalSandbox._get_shell() == r"C:\Program Files\Git\bin\sh.exe"


def test_get_shell_uses_powershell_fallback_on_windows(monkeypatch):
    calls: list[tuple[str, ...]] = []

    def fake_find(candidates: tuple[str, ...]) -> str | None:
        calls.append(candidates)
        if candidates == ("/bin/zsh", "/bin/bash", "/bin/sh", "sh"):
            return None
        return r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"SystemRoot": r"C:\Windows"})
    monkeypatch.setattr(LocalSandbox, "_find_first_available_shell", fake_find)

    assert LocalSandbox._get_shell() == r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
    assert calls[1] == (
        "pwsh",
        "pwsh.exe",
        "powershell",
        "powershell.exe",
        r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe",
        "cmd.exe",
    )


def test_get_shell_uses_cmd_as_last_windows_fallback(monkeypatch):
    def fake_find(candidates: tuple[str, ...]) -> str | None:
        if candidates == ("/bin/zsh", "/bin/bash", "/bin/sh", "sh"):
            return None
        return r"C:\Windows\System32\cmd.exe"

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"SystemRoot": r"C:\Windows"})
    monkeypatch.setattr(LocalSandbox, "_find_first_available_shell", fake_find)

    assert LocalSandbox._get_shell() == r"C:\Windows\System32\cmd.exe"


def test_execute_command_uses_powershell_command_mode_on_windows(monkeypatch):
    calls: list[tuple[list[str], float, dict[str, str], str | None, str | None]] = []

    def fake_run(args, timeout, env, *, encoding=None, fallback_encoding=None):
        calls.append((args, timeout, env, encoding, fallback_encoding))
        return "ok", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Windows", "OPENAI_API_KEY": "should-not-leak"})
    monkeypatch.setattr(local_sandbox.locale, "getencoding", lambda: "cp936")
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"))
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    output = LocalSandbox("t").execute_command("Write-Output hello")

    assert output == "ok"
    # Platform secrets are scrubbed from the inherited environment even on the
    # Windows PowerShell path (#3861); benign PATH is preserved and the env is an
    # explicit scrubbed dict, no longer None.
    assert calls == [
        (
            [
                r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe",
                "-NoProfile",
                "-Command",
                "try{[Console]::InputEncoding=[System.Text.Encoding]::UTF8}catch{};try{[Console]::OutputEncoding=[System.Text.Encoding]::UTF8}catch{};$OutputEncoding=[System.Text.Encoding]::UTF8;Write-Output hello",
            ],
            600,
            {"PATH": r"C:\Windows"},
            "utf-8",
            "cp936",
        )
    ]


def test_execute_command_forces_utf8_console_for_powershell_cjk_output(monkeypatch):
    """PowerShell 5.1 defaults console output to the OEM codepage (GBK on
    zh-CN); without the UTF-8 preamble, CJK output is garbled by the UTF-8
    pipe reader even though decoding never raises (errors=replace)."""
    calls: list[tuple[list[str], float, dict[str, str], str | None, str | None]] = []

    def fake_run(args, timeout, env, *, encoding=None, fallback_encoding=None):
        calls.append((args, timeout, env, encoding, fallback_encoding))
        return "你好", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Windows"})
    monkeypatch.setattr(local_sandbox.locale, "getencoding", lambda: "cp936")
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: "pwsh"))
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    output = LocalSandbox("t").execute_command("Write-Output 你好")

    assert output == "你好"
    cmd = calls[0][0][3]
    assert cmd.startswith("try{[Console]::InputEncoding=[System.Text.Encoding]::UTF8}catch{};try{[Console]::OutputEncoding=[System.Text.Encoding]::UTF8}catch{};$OutputEncoding=[System.Text.Encoding]::UTF8;")
    assert cmd.endswith("Write-Output 你好")
    assert calls[0][3] == "utf-8"
    assert calls[0][4] == "cp936"


def test_execute_command_forces_utf8_decoding_with_locale_fallback_for_msys(monkeypatch):
    """Git Bash/MSYS writes UTF-8 to its pipes; decoding with the host code page
    (GBK on zh-CN) mangles CJK output silently because the pipe decoder replaces
    instead of raising. The native children Git Bash spawns still write the host
    code page, so the forced UTF-8 decode keeps the locale as a guarded fallback."""
    calls: list[tuple[list[str], float, dict[str, str], str | None, str | None]] = []

    def fake_run(args, timeout, env, *, encoding=None, fallback_encoding=None):
        calls.append((args, timeout, env, encoding, fallback_encoding))
        return "你好", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Program Files\Git\bin"})
    monkeypatch.setattr(local_sandbox.locale, "getencoding", lambda: "cp936")
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Program Files\Git\bin\sh.exe"))
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    output = LocalSandbox("t").execute_command("echo 你好")

    assert output == "你好"
    assert calls[0][0] == [r"C:\Program Files\Git\bin\sh.exe", "-c", "echo 你好"]
    assert calls[0][3] == "utf-8"
    # Native children of Git Bash keep writing the host code page; the locale is
    # retained as a guarded fallback instead of forcing every byte through UTF-8.
    assert calls[0][4] == "cp936"


def test_execute_command_keeps_msys_path_conversion_for_host_commands_on_windows(monkeypatch):
    calls: list[tuple[list[str], float, dict[str, str]]] = []

    def fake_run(args, timeout, env, *, encoding=None, fallback_encoding=None):
        calls.append((args, timeout, env))
        return "ok", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Program Files\Git\bin"})
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Program Files\Git\bin\sh.exe"))
    monkeypatch.setattr(LocalSandbox, "_msys_path_conversion_exclusions", lambda self: "/mnt/user-data")
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    output = LocalSandbox("t").execute_command("echo hello")

    assert output == "ok"
    assert calls == [
        (
            [r"C:\Program Files\Git\bin\sh.exe", "-c", "echo hello"],
            600,
            {
                "PATH": r"C:\Program Files\Git\bin",
                "MSYS2_ARG_CONV_EXCL": "/mnt/user-data",
            },
        )
    ]


def test_execute_command_scopes_msys_path_conversion_exclusions_on_windows(monkeypatch):
    calls: list[tuple[list[str], float, dict[str, str]]] = []

    def fake_run(args, timeout, env, *, encoding=None, fallback_encoding=None):
        calls.append((args, timeout, env))
        return "ok", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Program Files\Git\bin"})
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Program Files\Git\bin\sh.exe"))
    monkeypatch.setattr(LocalSandbox, "_msys_path_conversion_exclusions", lambda self: "/mnt/user-data")
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    output = LocalSandbox("t").execute_command("cat /mnt/user-data/workspace/input.txt")

    assert output == "ok"
    assert calls[0][2] == {
        "PATH": r"C:\Program Files\Git\bin",
        "MSYS2_ARG_CONV_EXCL": "/mnt/user-data",
    }


def test_execute_command_ignores_root_msys_mapping_for_host_commands_on_windows(monkeypatch):
    calls: list[tuple[list[str], float, dict[str, str]]] = []

    def fake_run(args, timeout, env, *, encoding=None, fallback_encoding=None):
        calls.append((args, timeout, env))
        return "ok", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Program Files\Git\bin"})
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Program Files\Git\bin\sh.exe"))
    monkeypatch.setattr(LocalSandbox, "_msys_path_conversion_exclusions", lambda self: "")
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    output = LocalSandbox("t").execute_command("echo hello")

    assert output == "ok"
    assert calls[0][2] == {"PATH": r"C:\Program Files\Git\bin"}


def test_msys_path_conversion_exclusions_omit_blanket_patterns():
    sandbox = LocalSandbox(
        "t",
        [
            PathMapping(container_path="/", local_path="C:\\"),
            PathMapping(container_path="/mnt/data;*", local_path=r"C:\data"),
            PathMapping(container_path="/mnt/user-data/", local_path=r"C:\user-data"),
            PathMapping(container_path="/mnt/user-data", local_path=r"C:\user-data"),
        ],
    )

    assert sandbox._msys_path_conversion_exclusions() == "/mnt/user-data"


def test_execute_command_does_not_set_msys_env_for_non_msys_posix_shell_on_windows(monkeypatch):
    calls: list[tuple[list[str], float, dict[str, str]]] = []

    def fake_run(args, timeout, env):
        calls.append((args, timeout, env))
        return "ok", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\tools"})
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\tools\busybox\sh.exe"))
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    output = LocalSandbox("t").execute_command("echo /mnt/skills/demo")

    assert output == "ok"
    # Non-MSYS posix shell adds no MSYS_* vars; the env is the scrubbed inherited
    # environment, not None (#3861).
    assert calls[0][2] == {"PATH": r"C:\tools"}
    assert "MSYS_NO_PATHCONV" not in calls[0][2]


def _mock_cmd_code_pages(monkeypatch, console_cp, oem_cp):
    kernel32 = SimpleNamespace(GetConsoleOutputCP=lambda: console_cp, GetOEMCP=lambda: oem_cp)
    monkeypatch.setattr(ctypes, "WinDLL", lambda *args, **kwargs: kernel32, raising=False)


def test_execute_command_uses_cmd_command_mode_on_windows(monkeypatch):
    calls: list[tuple[list[str], float, dict[str, str], dict]] = []

    def fake_run(args, timeout, env, **kwargs):
        calls.append((args, timeout, env, kwargs))
        return "ok", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Windows", "GITHUB_TOKEN": "should-not-leak"})
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Windows\System32\cmd.exe"))
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
    _mock_cmd_code_pages(monkeypatch, 850, 437)

    output = LocalSandbox("t").execute_command("echo hello")

    assert output == "ok"
    # Platform secrets are scrubbed even on the Windows cmd path (#3861); the env
    # is an explicit scrubbed dict, no longer None.
    assert calls == [
        (
            [r"C:\Windows\System32\cmd.exe", "/c", "echo hello"],
            600,
            {"PATH": r"C:\Windows"},
            {"encoding": "cp850"},
        )
    ]


def test_execute_command_cmd_branch_ignores_utf8_mode_for_console_encoding(monkeypatch):
    """Select the console page independently of Python UTF-8 mode and ANSI."""
    monkeypatch.setattr(local_sandbox.locale, "getpreferredencoding", lambda _: "utf-8")
    monkeypatch.setattr(local_sandbox.locale, "getencoding", lambda: "cp1252")
    _mock_cmd_code_pages(monkeypatch, 936, 437)

    kwargs: dict = {}

    def fake_run(args, timeout, env, **kw):
        kwargs.update(kw)
        return "ok", "", 0, False

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Windows"})
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Windows\System32\cmd.exe"))
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    LocalSandbox("t").execute_command("echo hello")

    assert kwargs["encoding"] == "cp936"


@pytest.mark.parametrize(
    "console_cp,oem_cp,pipe_encoding,probe",
    [(437, 850, "cp437", "café"), (850, 437, "cp850", "café £"), (65001, 437, "utf-8", "你好 日本語"), (0, 850, "cp850", "café £"), (0, 0, "cp1252", "café")],
    ids=["oem-437", "oem-850", "utf8-console", "no-console", "ansi-fallback"],
)
def test_execute_command_cmd_decodes_console_code_page(monkeypatch, console_cp, oem_cp, pipe_encoding, probe):
    """Decode real stdout/stderr pipes with differing console and ANSI pages."""
    _mock_cmd_code_pages(monkeypatch, console_cp, oem_cp)
    monkeypatch.setattr(local_sandbox.locale, "getencoding", lambda: "cp1252")
    monkeypatch.setattr(local_sandbox.locale, "getpreferredencoding", lambda _: "utf-8")
    if os.name != "nt":
        monkeypatch.setattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0, raising=False)
    run_windows_command = LocalSandbox._run_windows_command
    payload = (probe + "\r\n").encode(pipe_encoding)
    script = f"import os; p=bytes.fromhex('{payload.hex()}'); os.write(1,p); os.write(2,p); raise SystemExit(3)"

    def run_pipe_probe(args, timeout, env, **kwargs):
        return run_windows_command([sys.executable, "-c", script], timeout, env, **kwargs)

    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: "cmd.exe"))
    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(run_pipe_probe))

    output = LocalSandbox("t").execute_command("echo probe", timeout=10)

    assert output == f"{probe}\n\nStd Error:\n{probe}\n\nExit Code: 3"


def test_execute_command_cmd_rechecks_console_page_between_commands(monkeypatch):
    pages = iter([437, 65001])
    kernel32 = SimpleNamespace(GetConsoleOutputCP=lambda: next(pages), GetOEMCP=lambda: 437)
    monkeypatch.setattr(ctypes, "WinDLL", lambda *args, **kwargs: kernel32, raising=False)
    monkeypatch.setattr(local_sandbox.locale, "getencoding", lambda: "cp1252")
    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: "cmd.exe"))
    payloads = iter(["café".encode("cp437"), "你好".encode()])

    def fake_run(args, timeout, env, **kwargs):
        capture = _BoundedPipeCapture(**kwargs)
        capture.append(next(payloads))
        return capture.read(), "", 0, False

    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
    sandbox = LocalSandbox("t")

    assert sandbox.execute_command("echo café") == "café"
    assert sandbox.execute_command("echo 你好") == "你好"


def test_execute_command_cmd_falls_back_when_windows_api_is_unavailable(monkeypatch):
    def unavailable(*args, **kwargs):
        raise OSError("kernel32 unavailable")

    monkeypatch.setattr(ctypes, "WinDLL", unavailable, raising=False)
    monkeypatch.setattr(local_sandbox.locale, "getencoding", lambda: "cp1252")
    monkeypatch.setattr(local_sandbox.os, "name", "nt")
    monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: "cmd.exe"))
    kwargs = {}

    def fake_run(args, timeout, env, **kw):
        kwargs.update(kw)
        return "ok", "", 0, False

    monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))

    assert LocalSandbox("t").execute_command("echo hello") == "ok"
    assert kwargs["encoding"] == "cp1252"


@pytest.mark.skipif(os.name != "nt", reason="Requires real Windows cmd.exe")
@pytest.mark.parametrize("code_page", [437, 850, 936, 65001])
def test_execute_command_cmd_branch_roundtrip_with_console_code_page(code_page):
    """Use a private console so changing its page cannot affect sibling tests."""
    probe = r"""
import ctypes
import os
import sys
import deerflow.sandbox.local.local_sandbox as local_sandbox
from deerflow.sandbox.local.local_sandbox import LocalSandbox

code_page = int(sys.argv[1])
assert ctypes.windll.kernel32.SetConsoleOutputCP(code_page)
# Make ANSI and console encoding differ even on a zh-CN or UTF-8 runner.
local_sandbox.locale.getencoding = lambda: "cp1252"
shell = os.path.join(os.environ.get("SystemRoot", r"C:\Windows"), "System32", "cmd.exe")
LocalSandbox._get_shell = staticmethod(lambda: shell)
expected = "caf\u00e9" if code_page in (437, 850) else "\u4f60\u597d \u65e5\u672c\u8a9e"
output = LocalSandbox("cmd-encoding-probe").execute_command(f"echo {expected}", timeout=15)
assert output == expected + "\n", ascii(output)
"""
    startupinfo = subprocess.STARTUPINFO()
    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
    startupinfo.wShowWindow = subprocess.SW_HIDE
    result = subprocess.run(
        [sys.executable, "-c", probe, str(code_page)],
        capture_output=True,
        text=True,
        encoding="utf-8",
        errors="replace",
        env={**os.environ, "PYTHONUTF8": "1"},
        creationflags=subprocess.CREATE_NEW_CONSOLE,
        startupinfo=startupinfo,
        timeout=30,
        check=False,
    )
    assert result.returncode == 0, result.stdout + result.stderr