mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-15 00:19:14 +00:00
93f9ed3d8f
* fix(sandbox): force UTF-8 console for PowerShell so CJK output is not garbled LocalSandbox captures PowerShell output through a UTF-8 pipe reader (errors=replace), but Windows PowerShell 5.1 writes console output in the legacy OEM codepage (GBK on zh-CN Windows) unless told otherwise, so every CJK character in tool output arrives as mojibake and the decode never raises. Prepend a UTF-8 preamble ([Console]::InputEncoding/[Console]::OutputEncoding/$OutputEncoding) to the -Command payload so both directions of the console are UTF-8 before the user command runs. * fix(sandbox): pair PowerShell UTF-8 capture and guard console setup --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
424 lines
16 KiB
Python
424 lines
16 KiB
Python
import builtins
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
import deerflow.sandbox.local.local_sandbox as local_sandbox
|
|
from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping, _BoundedPipeCapture
|
|
|
|
|
|
def _open(base, file, mode="r", *args, **kwargs):
|
|
if "b" in mode:
|
|
return base(file, mode, *args, **kwargs)
|
|
return base(file, mode, *args, encoding=kwargs.pop("encoding", "gbk"), **kwargs)
|
|
|
|
|
|
def test_bounded_pipe_capture_decodes_non_utf8_output_with_configured_encoding():
|
|
capture = _BoundedPipeCapture(encoding="cp1252")
|
|
capture.append("caf\u00e9".encode("cp1252"))
|
|
|
|
assert capture.read() == "caf\u00e9"
|
|
|
|
|
|
def test_bounded_pipe_capture_applies_text_mode_newline_normalization_when_enabled():
|
|
capture = _BoundedPipeCapture(normalize_newlines=True)
|
|
capture.append(b"crlf\r\nbare-cr\rlf\n")
|
|
|
|
assert capture.read() == "crlf\nbare-cr\nlf\n"
|
|
|
|
|
|
def test_bounded_pipe_capture_preserves_posix_newlines_by_default():
|
|
capture = _BoundedPipeCapture()
|
|
capture.append(b"crlf\r\nbare-cr\rlf\n")
|
|
|
|
assert capture.read() == "crlf\r\nbare-cr\rlf\n"
|
|
|
|
|
|
@pytest.mark.parametrize("encoding", [None, "utf-8"])
|
|
def test_windows_pipe_capture_uses_explicit_encoding_or_locale(monkeypatch, encoding):
|
|
"""Exercise real pipes with a legacy locale, even on a POSIX test host."""
|
|
monkeypatch.setattr(local_sandbox.locale, "getpreferredencoding", lambda _: "cp936")
|
|
if os.name != "nt":
|
|
monkeypatch.setattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0, raising=False)
|
|
expected = "你好 日本語"
|
|
payload = (expected + "\r\n").encode(encoding or "cp936")
|
|
|
|
stdout, stderr, returncode, timed_out = LocalSandbox._run_windows_command(
|
|
[sys.executable, "-c", f"import os; p=bytes.fromhex('{payload.hex()}'); os.write(1,p); os.write(2,p)"],
|
|
10,
|
|
encoding=encoding,
|
|
)
|
|
|
|
assert stdout == stderr == expected + "\n"
|
|
assert returncode == 0
|
|
assert timed_out is False
|
|
|
|
|
|
@pytest.mark.skipif(os.name != "nt", reason="Requires real Windows PowerShell")
|
|
@pytest.mark.parametrize("shell_name", ["powershell.exe", "pwsh.exe"])
|
|
@pytest.mark.parametrize("no_console", [False, True], ids=["inherited-console", "no-console"])
|
|
def test_windows_powershell_cjk_roundtrip(shell_name, no_console):
|
|
shell = shutil.which(shell_name)
|
|
if shell is None:
|
|
pytest.skip(f"{shell_name} is not installed")
|
|
probe = r"""
|
|
import sys
|
|
import deerflow.sandbox.local.local_sandbox as local_sandbox
|
|
from deerflow.sandbox.local.local_sandbox import LocalSandbox
|
|
|
|
# Keep this regression effective even on an English or UTF-8 Windows runner.
|
|
local_sandbox.locale.getpreferredencoding = lambda _: "cp936"
|
|
LocalSandbox._get_shell = staticmethod(lambda: sys.argv[1])
|
|
expected = "\u4f60\u597d \u65e5\u672c\u8a9e"
|
|
command = f"Write-Output '{expected}'; [Console]::Error.WriteLine('{expected}'); exit 3"
|
|
output = LocalSandbox("encoding-probe").execute_command(command, timeout=15)
|
|
assert output == expected + "\n\nStd Error:\n" + expected + "\n\nExit Code: 3", ascii(output)
|
|
"""
|
|
env = {**os.environ, "PYTHONUTF8": "0"}
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", probe, shell],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
env=env,
|
|
creationflags=subprocess.CREATE_NO_WINDOW if no_console else 0,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX capture semantics")
|
|
def test_posix_command_capture_preserves_newlines():
|
|
stdout, stderr, returncode, timed_out = LocalSandbox._run_posix_command(
|
|
[sys.executable, "-c", "import os; os.write(1, b'crlf\\r\\nbare-cr\\rlf\\n')"],
|
|
10,
|
|
)
|
|
|
|
assert stdout == "crlf\r\nbare-cr\rlf\n"
|
|
assert stderr == ""
|
|
assert returncode == 0
|
|
assert timed_out is False
|
|
|
|
|
|
@pytest.mark.skipif(os.name != "nt", reason="Windows text-mode newline semantics")
|
|
def test_windows_command_capture_normalizes_newlines():
|
|
stdout, stderr, returncode, timed_out = LocalSandbox._run_windows_command(
|
|
[sys.executable, "-c", "import os; os.write(1, b'crlf\\r\\nbare-cr\\rlf\\n')"],
|
|
10,
|
|
)
|
|
|
|
assert stdout == "crlf\nbare-cr\nlf\n"
|
|
assert stderr == ""
|
|
assert returncode == 0
|
|
assert timed_out is False
|
|
|
|
|
|
@pytest.mark.skipif(os.name != "nt", reason="Windows text-mode encoding semantics")
|
|
@pytest.mark.parametrize(
|
|
("python_args", "python_utf8"),
|
|
[([], "0"), ([], "1"), (["-X", "utf8"], "0")],
|
|
ids=["locale-code-page", "PYTHONUTF8", "-X-utf8"],
|
|
)
|
|
def test_windows_capture_matches_subprocess_text_mode_encoding(python_args, python_utf8):
|
|
probe = r"""
|
|
import subprocess
|
|
import sys
|
|
|
|
from deerflow.sandbox.local.local_sandbox import LocalSandbox
|
|
|
|
reference = subprocess.Popen([sys.executable, "-c", ""], stdout=subprocess.PIPE, text=True)
|
|
encoding = reference.stdout.encoding
|
|
reference.communicate()
|
|
|
|
for expected in ("caf\u00e9", "\u4f60\u597d", "\u65e5\u672c\u8a9e", "\u041f\u0440\u0438\u0432\u0435\u0442"):
|
|
try:
|
|
payload = expected.encode(encoding)
|
|
except UnicodeEncodeError:
|
|
continue
|
|
if any(byte >= 0x80 for byte in payload):
|
|
break
|
|
else:
|
|
raise AssertionError(f"no non-ASCII probe text for {encoding}")
|
|
|
|
stdout, stderr, returncode, timed_out = LocalSandbox._run_windows_command(
|
|
[sys.executable, "-c", f"import sys; sys.stdout.buffer.write(bytes.fromhex('{payload.hex()}'))"],
|
|
10,
|
|
)
|
|
assert stdout == expected, (encoding, stdout)
|
|
assert stderr == ""
|
|
assert returncode == 0
|
|
assert timed_out is False
|
|
"""
|
|
env = os.environ.copy()
|
|
env["PYTHONUTF8"] = python_utf8
|
|
|
|
result = subprocess.run(
|
|
[sys.executable, *python_args, "-c", probe],
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
env=env,
|
|
check=False,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
|
|
def test_read_file_uses_utf8_on_windows_locale(tmp_path, monkeypatch):
|
|
path = tmp_path / "utf8.txt"
|
|
text = "\u201cutf8\u201d"
|
|
path.write_text(text, encoding="utf-8")
|
|
base = builtins.open
|
|
|
|
monkeypatch.setattr(local_sandbox, "open", lambda file, mode="r", *args, **kwargs: _open(base, file, mode, *args, **kwargs), raising=False)
|
|
|
|
assert LocalSandbox("t").read_file(str(path)) == text
|
|
|
|
|
|
def test_write_file_uses_utf8_on_windows_locale(tmp_path, monkeypatch):
|
|
path = tmp_path / "utf8.txt"
|
|
text = "emoji \U0001f600"
|
|
base = builtins.open
|
|
|
|
monkeypatch.setattr(local_sandbox, "open", lambda file, mode="r", *args, **kwargs: _open(base, file, mode, *args, **kwargs), raising=False)
|
|
|
|
LocalSandbox("t").write_file(str(path), text)
|
|
|
|
assert path.read_text(encoding="utf-8") == text
|
|
|
|
|
|
def test_get_shell_prefers_posix_shell_from_path_before_windows_fallback(monkeypatch):
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(LocalSandbox, "_find_first_available_shell", lambda candidates: r"C:\Program Files\Git\bin\sh.exe" if candidates == ("/bin/zsh", "/bin/bash", "/bin/sh", "sh") else None)
|
|
|
|
assert LocalSandbox._get_shell() == r"C:\Program Files\Git\bin\sh.exe"
|
|
|
|
|
|
def test_get_shell_uses_powershell_fallback_on_windows(monkeypatch):
|
|
calls: list[tuple[str, ...]] = []
|
|
|
|
def fake_find(candidates: tuple[str, ...]) -> str | None:
|
|
calls.append(candidates)
|
|
if candidates == ("/bin/zsh", "/bin/bash", "/bin/sh", "sh"):
|
|
return None
|
|
return r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"SystemRoot": r"C:\Windows"})
|
|
monkeypatch.setattr(LocalSandbox, "_find_first_available_shell", fake_find)
|
|
|
|
assert LocalSandbox._get_shell() == r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
|
|
assert calls[1] == (
|
|
"pwsh",
|
|
"pwsh.exe",
|
|
"powershell",
|
|
"powershell.exe",
|
|
r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe",
|
|
"cmd.exe",
|
|
)
|
|
|
|
|
|
def test_get_shell_uses_cmd_as_last_windows_fallback(monkeypatch):
|
|
def fake_find(candidates: tuple[str, ...]) -> str | None:
|
|
if candidates == ("/bin/zsh", "/bin/bash", "/bin/sh", "sh"):
|
|
return None
|
|
return r"C:\Windows\System32\cmd.exe"
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"SystemRoot": r"C:\Windows"})
|
|
monkeypatch.setattr(LocalSandbox, "_find_first_available_shell", fake_find)
|
|
|
|
assert LocalSandbox._get_shell() == r"C:\Windows\System32\cmd.exe"
|
|
|
|
|
|
def test_execute_command_uses_powershell_command_mode_on_windows(monkeypatch):
|
|
calls: list[tuple[list[str], float, dict[str, str], str | None]] = []
|
|
|
|
def fake_run(args, timeout, env, *, encoding=None):
|
|
calls.append((args, timeout, env, encoding))
|
|
return "ok", "", 0, False
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Windows", "OPENAI_API_KEY": "should-not-leak"})
|
|
monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"))
|
|
monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
|
|
|
|
output = LocalSandbox("t").execute_command("Write-Output hello")
|
|
|
|
assert output == "ok"
|
|
# Platform secrets are scrubbed from the inherited environment even on the
|
|
# Windows PowerShell path (#3861); benign PATH is preserved and the env is an
|
|
# explicit scrubbed dict, no longer None.
|
|
assert calls == [
|
|
(
|
|
[
|
|
r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe",
|
|
"-NoProfile",
|
|
"-Command",
|
|
"try{[Console]::InputEncoding=[System.Text.Encoding]::UTF8}catch{};try{[Console]::OutputEncoding=[System.Text.Encoding]::UTF8}catch{};$OutputEncoding=[System.Text.Encoding]::UTF8;Write-Output hello",
|
|
],
|
|
600,
|
|
{"PATH": r"C:\Windows"},
|
|
"utf-8",
|
|
)
|
|
]
|
|
|
|
|
|
def test_execute_command_forces_utf8_console_for_powershell_cjk_output(monkeypatch):
|
|
"""PowerShell 5.1 defaults console output to the OEM codepage (GBK on
|
|
zh-CN); without the UTF-8 preamble, CJK output is garbled by the UTF-8
|
|
pipe reader even though decoding never raises (errors=replace)."""
|
|
calls: list[tuple[list[str], float, dict[str, str], str | None]] = []
|
|
|
|
def fake_run(args, timeout, env, *, encoding=None):
|
|
calls.append((args, timeout, env, encoding))
|
|
return "你好", "", 0, False
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Windows"})
|
|
monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: "pwsh"))
|
|
monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
|
|
|
|
output = LocalSandbox("t").execute_command("Write-Output 你好")
|
|
|
|
assert output == "你好"
|
|
cmd = calls[0][0][3]
|
|
assert cmd.startswith("try{[Console]::InputEncoding=[System.Text.Encoding]::UTF8}catch{};try{[Console]::OutputEncoding=[System.Text.Encoding]::UTF8}catch{};$OutputEncoding=[System.Text.Encoding]::UTF8;")
|
|
assert cmd.endswith("Write-Output 你好")
|
|
assert calls[0][3] == "utf-8"
|
|
|
|
|
|
def test_execute_command_keeps_msys_path_conversion_for_host_commands_on_windows(monkeypatch):
|
|
calls: list[tuple[list[str], float, dict[str, str]]] = []
|
|
|
|
def fake_run(args, timeout, env):
|
|
calls.append((args, timeout, env))
|
|
return "ok", "", 0, False
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Program Files\Git\bin"})
|
|
monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Program Files\Git\bin\sh.exe"))
|
|
monkeypatch.setattr(LocalSandbox, "_msys_path_conversion_exclusions", lambda self: "/mnt/user-data")
|
|
monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
|
|
|
|
output = LocalSandbox("t").execute_command("echo hello")
|
|
|
|
assert output == "ok"
|
|
assert calls == [
|
|
(
|
|
[r"C:\Program Files\Git\bin\sh.exe", "-c", "echo hello"],
|
|
600,
|
|
{
|
|
"PATH": r"C:\Program Files\Git\bin",
|
|
"MSYS2_ARG_CONV_EXCL": "/mnt/user-data",
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_execute_command_scopes_msys_path_conversion_exclusions_on_windows(monkeypatch):
|
|
calls: list[tuple[list[str], float, dict[str, str]]] = []
|
|
|
|
def fake_run(args, timeout, env):
|
|
calls.append((args, timeout, env))
|
|
return "ok", "", 0, False
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Program Files\Git\bin"})
|
|
monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Program Files\Git\bin\sh.exe"))
|
|
monkeypatch.setattr(LocalSandbox, "_msys_path_conversion_exclusions", lambda self: "/mnt/user-data")
|
|
monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
|
|
|
|
output = LocalSandbox("t").execute_command("cat /mnt/user-data/workspace/input.txt")
|
|
|
|
assert output == "ok"
|
|
assert calls[0][2] == {
|
|
"PATH": r"C:\Program Files\Git\bin",
|
|
"MSYS2_ARG_CONV_EXCL": "/mnt/user-data",
|
|
}
|
|
|
|
|
|
def test_execute_command_ignores_root_msys_mapping_for_host_commands_on_windows(monkeypatch):
|
|
calls: list[tuple[list[str], float, dict[str, str]]] = []
|
|
|
|
def fake_run(args, timeout, env):
|
|
calls.append((args, timeout, env))
|
|
return "ok", "", 0, False
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Program Files\Git\bin"})
|
|
monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Program Files\Git\bin\sh.exe"))
|
|
monkeypatch.setattr(LocalSandbox, "_msys_path_conversion_exclusions", lambda self: "")
|
|
monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
|
|
|
|
output = LocalSandbox("t").execute_command("echo hello")
|
|
|
|
assert output == "ok"
|
|
assert calls[0][2] == {"PATH": r"C:\Program Files\Git\bin"}
|
|
|
|
|
|
def test_msys_path_conversion_exclusions_omit_blanket_patterns():
|
|
sandbox = LocalSandbox(
|
|
"t",
|
|
[
|
|
PathMapping(container_path="/", local_path="C:\\"),
|
|
PathMapping(container_path="/mnt/data;*", local_path=r"C:\data"),
|
|
PathMapping(container_path="/mnt/user-data/", local_path=r"C:\user-data"),
|
|
PathMapping(container_path="/mnt/user-data", local_path=r"C:\user-data"),
|
|
],
|
|
)
|
|
|
|
assert sandbox._msys_path_conversion_exclusions() == "/mnt/user-data"
|
|
|
|
|
|
def test_execute_command_does_not_set_msys_env_for_non_msys_posix_shell_on_windows(monkeypatch):
|
|
calls: list[tuple[list[str], float, dict[str, str]]] = []
|
|
|
|
def fake_run(args, timeout, env):
|
|
calls.append((args, timeout, env))
|
|
return "ok", "", 0, False
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\tools"})
|
|
monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\tools\busybox\sh.exe"))
|
|
monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
|
|
|
|
output = LocalSandbox("t").execute_command("echo /mnt/skills/demo")
|
|
|
|
assert output == "ok"
|
|
# Non-MSYS posix shell adds no MSYS_* vars; the env is the scrubbed inherited
|
|
# environment, not None (#3861).
|
|
assert calls[0][2] == {"PATH": r"C:\tools"}
|
|
assert "MSYS_NO_PATHCONV" not in calls[0][2]
|
|
|
|
|
|
def test_execute_command_uses_cmd_command_mode_on_windows(monkeypatch):
|
|
calls: list[tuple[list[str], float, dict[str, str]]] = []
|
|
|
|
def fake_run(args, timeout, env):
|
|
calls.append((args, timeout, env))
|
|
return "ok", "", 0, False
|
|
|
|
monkeypatch.setattr(local_sandbox.os, "name", "nt")
|
|
monkeypatch.setattr(local_sandbox.os, "environ", {"PATH": r"C:\Windows", "GITHUB_TOKEN": "should-not-leak"})
|
|
monkeypatch.setattr(LocalSandbox, "_get_shell", staticmethod(lambda: r"C:\Windows\System32\cmd.exe"))
|
|
monkeypatch.setattr(LocalSandbox, "_run_windows_command", staticmethod(fake_run))
|
|
|
|
output = LocalSandbox("t").execute_command("echo hello")
|
|
|
|
assert output == "ok"
|
|
# Platform secrets are scrubbed even on the Windows cmd path (#3861); the env
|
|
# is an explicit scrubbed dict, no longer None.
|
|
assert calls == [
|
|
(
|
|
[r"C:\Windows\System32\cmd.exe", "/c", "echo hello"],
|
|
600,
|
|
{"PATH": r"C:\Windows"},
|
|
)
|
|
]
|