From 3f3cd23965b042a93dbc2501aba345d3aabdf2ca Mon Sep 17 00:00:00 2001 From: luckiest-draw <152146179+luckiest-draw@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:03:39 +0800 Subject: [PATCH] fix(config): avoid Unicode failure after generating configuration (#7102) * fix(config): avoid Unicode failure after generating configuration * fix(setup): handle legacy stdout encodings in setup commands --------- Co-authored-by: Willem Jiang --- README.md | 2 + backend/tests/test_configure_script.py | 66 +++++++++ backend/tests/test_setup_script_encoding.py | 146 ++++++++++++++++++++ scripts/AGENTS.md | 5 + scripts/configure.py | 2 +- scripts/doctor.py | 4 + scripts/setup_wizard.py | 11 +- 7 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_configure_script.py create mode 100644 backend/tests/test_setup_script_encoding.py diff --git a/README.md b/README.md index e29450b020..2ccec1f050 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,8 @@ streams simulate chunks from a non-streaming response, while no-tool streams sta Jina can also bound concurrent HTTP attempts and FIFO waiting with the opt-in `request_admission` tool extra. One immutable budget is shared across clients and agent threads in each process; replicas have independent budgets. See [Jina request admission](backend/docs/CONFIGURATION.md#jina-request-admission). Run `make doctor` at any time to verify your setup and get actionable fix hints. + The setup wizard and health check support legacy terminal encodings, including + Windows cp936, replacing unsupported characters so both commands can finish. If you are opening a GitHub issue about a local setup or runtime problem, run `make support-bundle`. The command prints reporter next steps, writes a `*-issue-summary.md` file to paste into the issue, a `*-issue-draft.md` file diff --git a/backend/tests/test_configure_script.py b/backend/tests/test_configure_script.py new file mode 100644 index 0000000000..6972f6b52d --- /dev/null +++ b/backend/tests/test_configure_script.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def config_checkout(tmp_path: Path) -> Path: + scripts_dir = tmp_path / "scripts" + scripts_dir.mkdir() + (tmp_path / "frontend").mkdir() + shutil.copyfile(REPO_ROOT / "scripts" / "configure.py", scripts_dir / "configure.py") + for template in ("config.example.yaml", ".env.example", "frontend/.env.example"): + (tmp_path / template).write_text(f"template: {template}\n", encoding="utf-8") + return tmp_path + + +@pytest.mark.parametrize("encoding", ["ascii", "cp1252", "cp936", "utf-8"]) +def test_configure_succeeds_with_restricted_stdout_encoding(config_checkout: Path, encoding: str): + result = subprocess.run( + [sys.executable, str(config_checkout / "scripts" / "configure.py")], + env={**os.environ, "PYTHONIOENCODING": encoding}, + capture_output=True, + text=True, + encoding=encoding, + check=False, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert "Configuration files generated" in result.stdout + assert result.stderr == "" + for template, generated in ( + ("config.example.yaml", "config.yaml"), + (".env.example", ".env"), + ("frontend/.env.example", "frontend/.env"), + ): + assert (config_checkout / generated).read_bytes() == (config_checkout / template).read_bytes() + + +@pytest.mark.parametrize("config_name", ["config.yaml", "config.yml", "configure.yml"]) +def test_configure_preserves_existing_configuration(config_checkout: Path, config_name: str): + config_path = config_checkout / config_name + config_path.write_text("existing: true\n", encoding="utf-8") + + result = subprocess.run( + [sys.executable, str(config_checkout / "scripts" / "configure.py")], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + timeout=30, + ) + + assert result.returncode == 1 + assert "configuration file already exists" in result.stdout + assert config_path.read_text(encoding="utf-8") == "existing: true\n" + assert not (config_checkout / ".env").exists() + assert not (config_checkout / "frontend" / ".env").exists() diff --git a/backend/tests/test_setup_script_encoding.py b/backend/tests/test_setup_script_encoding.py new file mode 100644 index 0000000000..02a2ba6a94 --- /dev/null +++ b/backend/tests/test_setup_script_encoding.py @@ -0,0 +1,146 @@ +"""Exercise setup command output through real, strictly encoded stdout streams.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +ENCODINGS = ["ascii", "cp1252", "cp936", "utf-8"] + + +@pytest.fixture +def setup_checkout(tmp_path: Path) -> Path: + scripts = tmp_path / "scripts" + scripts.mkdir() + for name in ("doctor.py", "setup_wizard.py"): + shutil.copyfile(REPO_ROOT / "scripts" / name, scripts / name) + shutil.copytree(REPO_ROOT / "scripts" / "wizard", scripts / "wizard") + (tmp_path / ".env.example").write_text("# test environment\n", encoding="utf-8") + (tmp_path / "frontend").mkdir() + (tmp_path / "frontend" / ".env.example").write_text("# frontend environment\n", encoding="utf-8") + return tmp_path + + +def run_setup_script(checkout: Path, encoding: str, code: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + cwd=checkout, + env={**os.environ, "PYTHONPATH": str(checkout / "scripts"), "PYTHONIOENCODING": f"{encoding}:strict"}, + capture_output=True, + text=True, + encoding=encoding, + check=False, + timeout=30, + ) + + +@pytest.mark.parametrize("encoding", ENCODINGS) +@pytest.mark.parametrize("is_tty", [False, True], ids=["pipe", "tty"]) +@pytest.mark.parametrize("has_failure", [False, True], ids=["ready", "failed"]) +def test_doctor_report_survives_restricted_stdout(setup_checkout: Path, encoding: str, is_tty: bool, has_failure: bool): + result = run_setup_script( + setup_checkout, + encoding, + f""" + import sys + from unittest.mock import patch + import doctor + + sys.stdout.isatty = lambda: {is_tty!r} + checks = {{}} + for name in vars(doctor): + if name.startswith("check_"): + value = doctor.CheckResult("Example check", "ok", "Unicode detail: 检查") + if name in ("check_llm_api_key", "check_llm_auth", "check_llm_package", "check_sandbox"): + value = [value] + checks[name] = lambda *args, value=value: value + checks["check_node"] = lambda: doctor.CheckResult("Warning check", "warn") + checks["check_pnpm"] = lambda: doctor.CheckResult("Skipped check", "skip") + checks["check_nginx"] = lambda: doctor.CheckResult( + "Required check", "fail" if {has_failure!r} else "ok", fix="Run make setup" + ) + with patch.multiple(doctor, **checks), patch.object(doctor, "resolve_config_path", return_value=(None, None)): + sys.exit(doctor.main()) + """, + ) + + assert result.returncode == int(has_failure), result.stderr + assert result.stderr == "" + assert "DeerFlow Health Check" in result.stdout + assert "Skipped check" in result.stdout + assert "Run make setup" in result.stdout + assert "Status: " in result.stdout + assert ("error(s)" if has_failure else "Ready") in result.stdout + if encoding == "utf-8": + assert "✓" in result.stdout + assert "检查" in result.stdout + + +@pytest.mark.parametrize("encoding", ENCODINGS) +def test_interactive_setup_completes_with_restricted_stdout(setup_checkout: Path, encoding: str): + result = run_setup_script( + setup_checkout, + encoding, + """ + import builtins + import getpass + import sys + import setup_wizard + from wizard.providers import LLM_PROVIDERS, SEARCH_PROVIDERS, WEB_FETCH_PROVIDERS + + # Model a legacy-locale terminal while keeping this test portable. + sys.stdin.isatty = lambda: True + sys.stdout.isatty = lambda: True + choices = iter([ + str(next(i for i, provider in enumerate(LLM_PROVIDERS, 1) if provider.name == "openai")), + "", # default model + str(len(SEARCH_PROVIDERS) + 1), # skip web search + str(len(WEB_FETCH_PROVIDERS) + 1), # skip web fetch + "", # local sandbox + "", # bash disabled + "", # file writes enabled + "", # no IM channels + ]) + builtins.input = lambda prompt: next(choices) + getpass.getpass = lambda prompt: "test-api-key" + sys.exit(setup_wizard.main()) + """, + ) + + assert result.returncode == 0, result.stderr + assert result.stderr == "" + assert "Writing configuration" in result.stdout + assert "Setup complete!" in result.stdout + assert "make doctor" in result.stdout + assert (setup_checkout / "config.yaml").is_file() + assert "OPENAI_API_KEY=test-api-key" in (setup_checkout / ".env").read_text(encoding="utf-8") + assert (setup_checkout / "frontend" / ".env").read_bytes() == (setup_checkout / "frontend" / ".env.example").read_bytes() + if encoding == "utf-8": + assert "✓" in result.stdout + assert "· Writing configuration" in result.stdout + + +@pytest.mark.parametrize("encoding", ENCODINGS) +def test_setup_still_requires_an_interactive_terminal(setup_checkout: Path, encoding: str): + result = run_setup_script( + setup_checkout, + encoding, + """ + import sys + import setup_wizard + sys.exit(setup_wizard.main()) + """, + ) + + assert result.returncode == 1 + assert result.stderr == "" + assert "Non-interactive environment detected" in result.stdout + assert not (setup_checkout / "config.yaml").exists() + assert not (setup_checkout / ".env").exists() diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 6ee1f5639f..7c25b97913 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -34,6 +34,11 @@ Tests: `backend/tests/test_extension_check_script.py`. status in `make check`, independently of the host locale. Tests: `backend/tests/test_check_script.py`. +`doctor.py` and `setup_wizard.py` configure stdout with `errors="replace"` at +the start of `main`, preserving its encoding so legacy terminals and pipes can +finish reports and setup. Guard `reconfigure` for embedded/captured streams. +Offline subprocess coverage: `backend/tests/test_setup_script_encoding.py`. + ## Manual Claude OAuth Export `export_claude_code_oauth.py` validates Keychain JSON as an object containing an diff --git a/scripts/configure.py b/scripts/configure.py index 444c9969fa..ae7e25ecce 100644 --- a/scripts/configure.py +++ b/scripts/configure.py @@ -50,7 +50,7 @@ def main() -> int: ) return 1 - print("✓ Configuration files generated") + print("Configuration files generated") return 0 diff --git a/scripts/doctor.py b/scripts/doctor.py index 4883a25aa0..cb16cd98af 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -869,6 +869,10 @@ def check_env_file(project_root: Path) -> CheckResult: def main() -> int: + # Keep reporting usable on terminals and pipes with legacy encodings. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(errors="replace") + project_root = Path(__file__).resolve().parents[1] # Load .env early so key checks work diff --git a/scripts/setup_wizard.py b/scripts/setup_wizard.py index 2b86f5f6b1..077cc61fcb 100644 --- a/scripts/setup_wizard.py +++ b/scripts/setup_wizard.py @@ -19,12 +19,13 @@ def _is_interactive() -> bool: def main() -> int: + # Unsupported status symbols must not abort setup after writing config. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(errors="replace") + try: if not _is_interactive(): - print( - "Non-interactive environment detected.\n" - "Please edit config.yaml and .env directly, or run 'make setup' in a terminal." - ) + print("Non-interactive environment detected.\nPlease edit config.yaml and .env directly, or run 'make setup' in a terminal.") return 1 from wizard.ui import ( @@ -109,6 +110,7 @@ def main() -> int: env_example = project_root / ".env.example" if env_example.exists(): import shutil + shutil.copyfile(env_example, env_path) env_pairs: dict[str, str] = {} @@ -127,6 +129,7 @@ def main() -> int: frontend_env_example = project_root / "frontend" / ".env.example" if not frontend_env.exists() and frontend_env_example.exists(): import shutil + shutil.copyfile(frontend_env_example, frontend_env) print_success("frontend/.env created from example")