diff --git a/README.md b/README.md index f48e4545f2..bf08578ac2 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,10 @@ For Google's official Gemini OpenAI-compatible endpoint, use the alone is not enough. Maintainers and AI triage tools can start with `triage.json`; the bundle includes redacted diagnostics and file manifests only, and does not include `.env`, raw conversation messages, or user file - contents. + contents. Subprocess diagnostics are captured as UTF-8, with Python helpers + emitting UTF-8 even on non-UTF-8 hosts and escaping unencodable characters. + Doctor's internal tool probes also decode UTF-8 with replacement for invalid + bytes so the remaining diagnostic output stays available. > **Advanced / manual configuration**: If you prefer to edit `config.yaml` directly, run `make config` instead to copy the full template. Optional dependency auto-detection accepts UTF-8 configuration files with or without a byte-order mark (BOM). See `config.example.yaml` for the complete reference including CLI-backed providers (Codex CLI, Claude Code OAuth), OpenRouter, Responses API, subagent runtime caps such as `subagents.max_total_per_run`, and more. diff --git a/backend/tests/test_doctor.py b/backend/tests/test_doctor.py index 4a19ac5458..5d2e9fab64 100644 --- a/backend/tests/test_doctor.py +++ b/backend/tests/test_doctor.py @@ -28,6 +28,18 @@ def _load_script(path: Path, name: str): return module +@pytest.mark.parametrize("host_encoding", ["ascii", "cp936"]) +@pytest.mark.parametrize("stream", ["stdout", "stderr"]) +def test_run_preserves_tool_diagnostics_with_invalid_utf8(monkeypatch, host_encoding, stream): + monkeypatch.setattr(doctor.subprocess, "_text_encoding", lambda: host_encoding) + output = "v22.0.0 ✓ 检查通过 ".encode() + b"\xff\n" + code = f"import sys; sys.{stream}.buffer.write({output!r})" + + result = doctor._run([sys.executable, "-c", code]) + + assert result == "v22.0.0 ✓ 检查通过 \ufffd" + + # --------------------------------------------------------------------------- # check_python # --------------------------------------------------------------------------- @@ -46,6 +58,30 @@ class TestCheckPython: class TestCheckPnpm: + @pytest.mark.parametrize("host_encoding", ["ascii", "cp936"]) + @pytest.mark.parametrize("returncode", [0, 7]) + def test_preserves_runner_diagnostics_with_invalid_utf8(self, tmp_path, monkeypatch, host_encoding, returncode): + monkeypatch.setattr(doctor.subprocess, "_text_encoding", lambda: host_encoding) + stdout = b"10.26.2\n" if returncode == 0 else "运行失败 ".encode() + b"\xff\n" + stderr = "诊断详情 ".encode() + b"\xff\n" + runner = tmp_path / "pnpm runner.py" + runner.write_text( + f"import os, sys\nassert os.getcwd() == {str(tmp_path)!r}\nassert sys.argv[1:] == ['-v']\nsys.stdout.buffer.write({stdout!r})\nsys.stderr.buffer.write({stderr!r})\nsys.exit({returncode})\n", + encoding="utf-8", + ) + monkeypatch.setattr(doctor, "PNPM_SCRIPT_PATH", runner) + monkeypatch.setattr(doctor, "FRONTEND_DIR", tmp_path) + + result = doctor.check_pnpm() + + if returncode == 0: + assert result.status == "ok" + assert result.detail == "10.26.2" + else: + assert result.status == "fail" + assert result.detail == "诊断详情 \ufffd\n运行失败 \ufffd" + assert result.fix is not None + def test_resolves_shared_runner_from_relative_script_path(self, monkeypatch): # Load the script as `scripts/doctor.py`, as a user would from the # repository root. The derived paths must not depend on that relative diff --git a/backend/tests/test_support_bundle.py b/backend/tests/test_support_bundle.py index d8a0d6fc61..8938254b6a 100644 --- a/backend/tests/test_support_bundle.py +++ b/backend/tests/test_support_bundle.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import sys import zipfile @@ -15,6 +16,69 @@ def _zip_text(zip_path, name: str) -> str: return zf.read(name).decode("utf-8") +@pytest.mark.parametrize("host_encoding", ["ascii", "cp936"]) +@pytest.mark.parametrize("returncode", [0, 7]) +def test_run_command_decodes_utf8_independently_of_host_locale(tmp_path, monkeypatch, host_encoding, returncode): + monkeypatch.setattr(support_bundle.subprocess, "_text_encoding", lambda: host_encoding) + stdout = "✓ 检查通过 Bearer fake.stdout.secret" + stderr = "✗ 检查失败 Bearer fake.stderr.secret" + code = f"import sys; sys.stdout.buffer.write({stdout.encode('utf-8')!r}); sys.stderr.buffer.write({stderr.encode('utf-8')!r}); sys.exit({returncode})" + + result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path) + + assert result == { + "ok": returncode == 0, + "returncode": returncode, + "stdout": "✓ 检查通过 Bearer ", + "stderr": "✗ 检查失败 Bearer ", + } + + +def test_run_command_gives_python_children_utf8_stdio_without_mutating_parent(tmp_path, monkeypatch): + monkeypatch.setenv("PYTHONIOENCODING", "ascii") + monkeypatch.setenv("PYTHONUTF8", "0") + monkeypatch.setenv("DEER_SUPPORT_TEST_MARKER", "inherited") + monkeypatch.setattr(support_bundle.subprocess, "_text_encoding", lambda: "ascii") + parent_env = dict(os.environ) + code = f"import os, sys; assert os.environ['DEER_SUPPORT_TEST_MARKER'] == 'inherited'; assert os.getcwd() == {str(tmp_path)!r}; sys.stdout.write({ascii('✓ 检查通过')}); sys.stderr.write({ascii('✗ 检查失败')})" + + result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path) + + assert result == {"ok": True, "returncode": 0, "stdout": "✓ 检查通过", "stderr": "✗ 检查失败"} + assert dict(os.environ) == parent_env + + +@pytest.mark.parametrize("returncode", [0, 7]) +def test_run_command_preserves_diagnostics_with_invalid_utf8(tmp_path, returncode): + code = f"import sys; sys.stdout.buffer.write(b'out \\xff Bearer fake.stdout.secret'); sys.stderr.buffer.write(b'err \\xff Bearer fake.stderr.secret'); sys.exit({returncode})" + + result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path) + + assert result == { + "ok": returncode == 0, + "returncode": returncode, + "stdout": "out \ufffd Bearer ", + "stderr": "err \ufffd Bearer ", + } + + +@pytest.mark.parametrize("returncode", [0, 7]) +def test_run_command_preserves_diagnostics_with_surrogate_characters(tmp_path, monkeypatch, returncode): + monkeypatch.setenv("PYTHONIOENCODING", "ascii:strict") + stdout = "✓ path \udcff Bearer fake.stdout.secret\ncontinued stdout" + stderr = "✗ path \udcff Bearer fake.stderr.secret\ncontinued stderr" + code = f"import sys; sys.stdout.write({ascii(stdout)}); sys.stderr.write({ascii(stderr)}); sys.exit({returncode})" + + result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path) + + assert result == { + "ok": returncode == 0, + "returncode": returncode, + "stdout": "✓ path \\udcff Bearer \ncontinued stdout", + "stderr": "✗ path \\udcff Bearer \ncontinued stderr", + } + + def test_collect_environment_routes_pnpm_through_shared_runner(tmp_path, monkeypatch): calls = [] diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 18824baf4d..e3250b3155 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -356,3 +356,12 @@ receive a JSON error and close code 1008. The support bundle's `extensions_config.json` reader accepts UTF-8 with or without a leading BOM, matching the runtime loader. Preserve redaction and avoid flagging a valid BOM-prefixed file as a syntax error in triage output. + +Support-bundle and doctor tool captures explicitly decode UTF-8 with replacement +for invalid bytes. Set `PYTHONIOENCODING=utf-8:backslashreplace` only in the copied +support-bundle child environment so Python helpers can print Unicode and escape +surrogates without aborting diagnostics or changing the parent. +Keep exit codes, timeouts and redaction intact; do not rely on the host locale. +Regressions use real local children, including ASCII/GBK capture defaults, +nonzero exits, surrogate characters and malformed output, without invoking +provider diagnostics. Doctor covers both `_run` streams and pnpm runner capture. diff --git a/scripts/doctor.py b/scripts/doctor.py index 28470bf72f..3b51416847 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -67,7 +67,7 @@ def _icon(status: Status) -> str: def _run(cmd: list[str]) -> str | None: try: - r = subprocess.run(cmd, capture_output=True, text=True, check=True) + r = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True) return (r.stdout or r.stderr).strip() except Exception: return None @@ -217,6 +217,8 @@ def check_pnpm() -> CheckResult: cwd=FRONTEND_DIR, capture_output=True, text=True, + encoding="utf-8", + errors="replace", check=False, shell=False, ) diff --git a/scripts/support_bundle.py b/scripts/support_bundle.py index 3d689e97a9..c70567c93c 100644 --- a/scripts/support_bundle.py +++ b/scripts/support_bundle.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import json +import os import platform import re import subprocess @@ -180,6 +181,12 @@ def _run_command(args: list[str], cwd: Path, timeout_s: int = 10) -> dict[str, A cwd=cwd, capture_output=True, text=True, + encoding="utf-8", + errors="replace", + # Python helpers (doctor, pnpm) must emit the encoding we capture, + # even when the host or inherited stdio setting is not UTF-8. + # Escape surrogate characters rather than aborting diagnostics. + env={**os.environ, "PYTHONIOENCODING": "utf-8:backslashreplace"}, timeout=timeout_s, check=False, )