fix(support-bundle): preserve UTF-8 subprocess diagnostics (#6391)

* fix(support-bundle): preserve UTF-8 subprocess diagnostics

* fix(diagnostics): preserve surrogate output and doctor captures

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

README.md

@@ -178,7 +178,10 @@ For Google's official Gemini OpenAI-compatible endpoint, use the
    alone is not enough. Maintainers and AI triage tools can start with
    `triage.json`; the bundle includes redacted diagnostics and file manifests
    only, and does not include `.env`, raw conversation messages, or user file
-   contents.
+   contents. Subprocess diagnostics are captured as UTF-8, with Python helpers
+   emitting UTF-8 even on non-UTF-8 hosts and escaping unencodable characters.
+   Doctor's internal tool probes also decode UTF-8 with replacement for invalid
+   bytes so the remaining diagnostic output stays available.
 
    > **Advanced / manual configuration**: If you prefer to edit `config.yaml` directly, run `make config` instead to copy the full template. Optional dependency auto-detection accepts UTF-8 configuration files with or without a byte-order mark (BOM). See `config.example.yaml` for the complete reference including CLI-backed providers (Codex CLI, Claude Code OAuth), OpenRouter, Responses API, subagent runtime caps such as `subagents.max_total_per_run`, and more.
 

backend/tests/test_doctor.py

@@ -28,6 +28,18 @@ def _load_script(path: Path, name: str):
     return module
 
 
+@pytest.mark.parametrize("host_encoding", ["ascii", "cp936"])
+@pytest.mark.parametrize("stream", ["stdout", "stderr"])
+def test_run_preserves_tool_diagnostics_with_invalid_utf8(monkeypatch, host_encoding, stream):
+    monkeypatch.setattr(doctor.subprocess, "_text_encoding", lambda: host_encoding)
+    output = "v22.0.0 ✓ 检查通过 ".encode() + b"\xff\n"
+    code = f"import sys; sys.{stream}.buffer.write({output!r})"
+
+    result = doctor._run([sys.executable, "-c", code])
+
+    assert result == "v22.0.0 ✓ 检查通过 \ufffd"
+
+
 # ---------------------------------------------------------------------------
 # check_python
 # ---------------------------------------------------------------------------
@@ -46,6 +58,30 @@ class TestCheckPython:
 
 
 class TestCheckPnpm:
+    @pytest.mark.parametrize("host_encoding", ["ascii", "cp936"])
+    @pytest.mark.parametrize("returncode", [0, 7])
+    def test_preserves_runner_diagnostics_with_invalid_utf8(self, tmp_path, monkeypatch, host_encoding, returncode):
+        monkeypatch.setattr(doctor.subprocess, "_text_encoding", lambda: host_encoding)
+        stdout = b"10.26.2\n" if returncode == 0 else "运行失败 ".encode() + b"\xff\n"
+        stderr = "诊断详情 ".encode() + b"\xff\n"
+        runner = tmp_path / "pnpm runner.py"
+        runner.write_text(
+            f"import os, sys\nassert os.getcwd() == {str(tmp_path)!r}\nassert sys.argv[1:] == ['-v']\nsys.stdout.buffer.write({stdout!r})\nsys.stderr.buffer.write({stderr!r})\nsys.exit({returncode})\n",
+            encoding="utf-8",
+        )
+        monkeypatch.setattr(doctor, "PNPM_SCRIPT_PATH", runner)
+        monkeypatch.setattr(doctor, "FRONTEND_DIR", tmp_path)
+
+        result = doctor.check_pnpm()
+
+        if returncode == 0:
+            assert result.status == "ok"
+            assert result.detail == "10.26.2"
+        else:
+            assert result.status == "fail"
+            assert result.detail == "诊断详情 \ufffd\n运行失败 \ufffd"
+            assert result.fix is not None
+
     def test_resolves_shared_runner_from_relative_script_path(self, monkeypatch):
         # Load the script as `scripts/doctor.py`, as a user would from the
         # repository root. The derived paths must not depend on that relative

backend/tests/test_support_bundle.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import os
 import sys
 import zipfile
 
@@ -15,6 +16,69 @@ def _zip_text(zip_path, name: str) -> str:
         return zf.read(name).decode("utf-8")
 
 
+@pytest.mark.parametrize("host_encoding", ["ascii", "cp936"])
+@pytest.mark.parametrize("returncode", [0, 7])
+def test_run_command_decodes_utf8_independently_of_host_locale(tmp_path, monkeypatch, host_encoding, returncode):
+    monkeypatch.setattr(support_bundle.subprocess, "_text_encoding", lambda: host_encoding)
+    stdout = "✓ 检查通过 Bearer fake.stdout.secret"
+    stderr = "✗ 检查失败 Bearer fake.stderr.secret"
+    code = f"import sys; sys.stdout.buffer.write({stdout.encode('utf-8')!r}); sys.stderr.buffer.write({stderr.encode('utf-8')!r}); sys.exit({returncode})"
+
+    result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path)
+
+    assert result == {
+        "ok": returncode == 0,
+        "returncode": returncode,
+        "stdout": "✓ 检查通过 Bearer <redacted>",
+        "stderr": "✗ 检查失败 Bearer <redacted>",
+    }
+
+
+def test_run_command_gives_python_children_utf8_stdio_without_mutating_parent(tmp_path, monkeypatch):
+    monkeypatch.setenv("PYTHONIOENCODING", "ascii")
+    monkeypatch.setenv("PYTHONUTF8", "0")
+    monkeypatch.setenv("DEER_SUPPORT_TEST_MARKER", "inherited")
+    monkeypatch.setattr(support_bundle.subprocess, "_text_encoding", lambda: "ascii")
+    parent_env = dict(os.environ)
+    code = f"import os, sys; assert os.environ['DEER_SUPPORT_TEST_MARKER'] == 'inherited'; assert os.getcwd() == {str(tmp_path)!r}; sys.stdout.write({ascii('✓ 检查通过')}); sys.stderr.write({ascii('✗ 检查失败')})"
+
+    result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path)
+
+    assert result == {"ok": True, "returncode": 0, "stdout": "✓ 检查通过", "stderr": "✗ 检查失败"}
+    assert dict(os.environ) == parent_env
+
+
+@pytest.mark.parametrize("returncode", [0, 7])
+def test_run_command_preserves_diagnostics_with_invalid_utf8(tmp_path, returncode):
+    code = f"import sys; sys.stdout.buffer.write(b'out \\xff Bearer fake.stdout.secret'); sys.stderr.buffer.write(b'err \\xff Bearer fake.stderr.secret'); sys.exit({returncode})"
+
+    result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path)
+
+    assert result == {
+        "ok": returncode == 0,
+        "returncode": returncode,
+        "stdout": "out \ufffd Bearer <redacted>",
+        "stderr": "err \ufffd Bearer <redacted>",
+    }
+
+
+@pytest.mark.parametrize("returncode", [0, 7])
+def test_run_command_preserves_diagnostics_with_surrogate_characters(tmp_path, monkeypatch, returncode):
+    monkeypatch.setenv("PYTHONIOENCODING", "ascii:strict")
+    stdout = "✓ path \udcff Bearer fake.stdout.secret\ncontinued stdout"
+    stderr = "✗ path \udcff Bearer fake.stderr.secret\ncontinued stderr"
+    code = f"import sys; sys.stdout.write({ascii(stdout)}); sys.stderr.write({ascii(stderr)}); sys.exit({returncode})"
+
+    result = support_bundle._run_command([sys.executable, "-c", code], cwd=tmp_path)
+
+    assert result == {
+        "ok": returncode == 0,
+        "returncode": returncode,
+        "stdout": "✓ path \\udcff Bearer <redacted>\ncontinued stdout",
+        "stderr": "✗ path \\udcff Bearer <redacted>\ncontinued stderr",
+    }
+
+
 def test_collect_environment_routes_pnpm_through_shared_runner(tmp_path, monkeypatch):
     calls = []
 

scripts/AGENTS.md

@@ -356,3 +356,12 @@ receive a JSON error and close code 1008.
 The support bundle's `extensions_config.json` reader accepts UTF-8 with or
 without a leading BOM, matching the runtime loader. Preserve redaction and
 avoid flagging a valid BOM-prefixed file as a syntax error in triage output.
+
+Support-bundle and doctor tool captures explicitly decode UTF-8 with replacement
+for invalid bytes. Set `PYTHONIOENCODING=utf-8:backslashreplace` only in the copied
+support-bundle child environment so Python helpers can print Unicode and escape
+surrogates without aborting diagnostics or changing the parent.
+Keep exit codes, timeouts and redaction intact; do not rely on the host locale.
+Regressions use real local children, including ASCII/GBK capture defaults,
+nonzero exits, surrogate characters and malformed output, without invoking
+provider diagnostics. Doctor covers both `_run` streams and pnpm runner capture.

scripts/doctor.py

@@ -67,7 +67,7 @@ def _icon(status: Status) -> str:
 
 def _run(cmd: list[str]) -> str | None:
     try:
-        r = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        r = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True)
         return (r.stdout or r.stderr).strip()
     except Exception:
         return None
@@ -217,6 +217,8 @@ def check_pnpm() -> CheckResult:
             cwd=FRONTEND_DIR,
             capture_output=True,
             text=True,
+            encoding="utf-8",
+            errors="replace",
             check=False,
             shell=False,
         )

scripts/support_bundle.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import platform
 import re
 import subprocess
@@ -180,6 +181,12 @@ def _run_command(args: list[str], cwd: Path, timeout_s: int = 10) -> dict[str, A
             cwd=cwd,
             capture_output=True,
             text=True,
+            encoding="utf-8",
+            errors="replace",
+            # Python helpers (doctor, pnpm) must emit the encoding we capture,
+            # even when the host or inherited stdio setting is not UTF-8.
+            # Escape surrogate characters rather than aborting diagnostics.
+            env={**os.environ, "PYTHONIOENCODING": "utf-8:backslashreplace"},
             timeout=timeout_s,
             check=False,
         )