From 041a21b6a0e7b449ce8ea4860fb21ca08472c395 Mon Sep 17 00:00:00 2001 From: sun04222 <3242054475@qq.com> Date: Sun, 25 Oct 2026 09:06:19 +0800 Subject: [PATCH] fix(skills): decode skill-review analyzer output as UTF-8 (#7432) The analyzer prints ensure_ascii=False JSON, but the parent pipes used the ambient locale codec and the child inherited the ambient stdio codec: non-ASCII skill facts crashed the review pass on cp1252 Windows and mojibake'd on cp936. Pin both sides to UTF-8. Fixes #7387 --- .../test_review_changed_public_skills_utf8.py | 101 ++++++++++++++++++ scripts/review_changed_public_skills.py | 6 ++ 2 files changed, 107 insertions(+) create mode 100644 backend/tests/test_review_changed_public_skills_utf8.py diff --git a/backend/tests/test_review_changed_public_skills_utf8.py b/backend/tests/test_review_changed_public_skills_utf8.py new file mode 100644 index 0000000000..800c5a8130 --- /dev/null +++ b/backend/tests/test_review_changed_public_skills_utf8.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +PAYLOAD = {"summary": {"blockers": 0, "errors": 0, "warnings": 0, "infos": 1}, "completeness": {}, "notes": ["判定说明 ✗ 🎉"]} + +DRIVER_SOURCE = """ +import importlib +import sys +from pathlib import Path + +scripts_dir, repo_root = sys.argv[1], Path(sys.argv[2]) +sys.path.insert(0, scripts_dir) +module = importlib.import_module("review_changed_public_skills") +facts = module.collect_review_facts(repo_root / "skills" / "public" / "demo", repo_root, sys.executable) +expected = {notes!r} +if facts is None or facts.get("notes") != expected: + print("MISMATCH:", facts) + sys.exit(1) +print("OK") +""".format(notes=PAYLOAD["notes"]) + + +@pytest.fixture +def analyzer_stub(tmp_path: Path) -> Path: + """A repo-root stand-in whose `deerflow.skills.review.cli` prints raw UTF-8 JSON.""" + stub_root = tmp_path / "repo_root" + package = stub_root / "deerflow" / "skills" / "review" + package.mkdir(parents=True) + for parent in (stub_root / "deerflow", stub_root / "deerflow" / "skills", package): + (parent / "__init__.py").write_text("", encoding="utf-8") + (package / "cli.py").write_text( + "import json\nprint(json.dumps(" + repr(PAYLOAD) + ", ensure_ascii=False))\n", + encoding="utf-8", + ) + return stub_root + + +def _load_script_module(): + scripts_dir = REPO_ROOT / "scripts" + spec = importlib.util.spec_from_file_location("review_changed_public_skills_under_test", scripts_dir / "review_changed_public_skills.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + sys.path.insert(0, str(scripts_dir)) + try: + spec.loader.exec_module(module) + finally: + sys.path.remove(str(scripts_dir)) + return module + + +def test_collect_review_facts_round_trips_non_ascii_analyzer_output(analyzer_stub: Path, tmp_path: Path): + """The analyzer emits ensure_ascii=False JSON, so both pipe directions must be UTF-8. + + The driver runs with UTF-8 mode disabled and a C locale so the ambient codec is + a non-UTF-8 one (ASCII on Linux, the ANSI code page on Windows); the assertion + fails before the UTF-8 pinning and passes after it, on every platform. + """ + driver = tmp_path / "driver.py" + driver.write_text(DRIVER_SOURCE, encoding="utf-8") + + env = { + "PATH": os.environ.get("PATH", ""), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + "PYTHONUTF8": "0", + "LC_ALL": "C", + "PYTHONCOERCECLOCALE": "0", + "PYTHONDONTWRITEBYTECODE": "1", + } + result = subprocess.run( + [sys.executable, str(driver), str(REPO_ROOT / "scripts"), str(analyzer_stub)], + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + timeout=120, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "OK" in result.stdout + + +def test_review_env_pins_analyzer_stdio_to_utf8(analyzer_stub: Path, monkeypatch): + """review_env must override any inherited PYTHONUTF8/PYTHONIOENCODING for the analyzer.""" + module = _load_script_module() + monkeypatch.setenv("PYTHONIOENCODING", "cp1252") + monkeypatch.setenv("PYTHONUTF8", "0") + env = module.review_env(analyzer_stub) + assert env["PYTHONIOENCODING"] == "utf-8" + assert env["PYTHONUTF8"] == "1" diff --git a/scripts/review_changed_public_skills.py b/scripts/review_changed_public_skills.py index 338f3a2905..d074cff49a 100644 --- a/scripts/review_changed_public_skills.py +++ b/scripts/review_changed_public_skills.py @@ -318,6 +318,7 @@ def collect_review_facts(package: Path, repo_root: Path, python_executable: str) env=review_env(repo_root), capture_output=True, text=True, + encoding="utf-8", check=False, ) if result.returncode != 0: @@ -381,6 +382,11 @@ def review_env(repo_root: Path) -> dict[str, str]: harness_path = repo_root / "backend" / "packages" / "harness" existing_pythonpath = env.get("PYTHONPATH") env["PYTHONPATH"] = str(harness_path) if not existing_pythonpath else f"{harness_path}{os.pathsep}{existing_pythonpath}" + # The analyzer prints ensure_ascii=False JSON, so its stdio must be UTF-8 + # regardless of the ambient locale, or non-ASCII skill facts crash the + # analyzer before the UTF-8-decoding parent ever sees them. + env["PYTHONIOENCODING"] = "utf-8" + env["PYTHONUTF8"] = "1" return env