From e377d1b5bbede3d2a87d7f3651ee0cb2bb499168 Mon Sep 17 00:00:00 2001 From: hataa <79907651+hata33@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:04:52 +0800 Subject: [PATCH] fix(scripts): feed config-upgrade inline python on stdin, not argv (#6845) * fix(scripts): feed config-upgrade inline python on stdin, not argv On Windows setups where python/uv resolve through shim layers that reparse arguments with cmd.exe (e.g. pyenv-win), the multi-line `python -c "..."` payloads in config-upgrade.sh get mangled mid-flight (observed as `IndentationError: unexpected indent` on a stray `|| goto :error` line) and `make dev` fails before the Gateway starts. Feed both inline scripts on stdin instead, which is safe across every spawn chain, and load the checkout .env explicitly inside the resolver probe: the default load_dotenv() discovery walks up from cwd/__file__, which a stdin-fed script (`python -`) cannot satisfy. Pin both with a static regression test. * test(config): isolate upgrade fixtures from inherited dotenv selectors --------- Co-authored-by: Willem Jiang --- backend/tests/AGENTS.md | 5 +++ backend/tests/test_config_version.py | 62 +++++++++++++++++++++++++++- scripts/config-upgrade.sh | 29 ++++++++++--- 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/backend/tests/AGENTS.md b/backend/tests/AGENTS.md index 91b91ec92d..b2f6d0aef5 100644 --- a/backend/tests/AGENTS.md +++ b/backend/tests/AGENTS.md @@ -10,6 +10,11 @@ ownership and capture cleanup. Retain the real XLSX conversion coverage. Backend tests must preserve the runtime invariants they exercise without changing production execution topology. +Config-upgrade checkout fixtures clear inherited `DEER_FLOW_CONFIG_PATH`, +`DEER_FLOW_PROJECT_ROOT` and `DEER_FLOW_ENV_FILE` before applying explicit test +overrides. Exercise selector leaks through the real script with temporary +configs; assert that the simulated developer config and its backup stay untouched. + Image-context budget regressions use real PNG files through `view_image`, the state reducer, and both model-call wrappers. Check decoded payloads, cumulative base64 size and skipped file reads; keep count and byte limits independent. diff --git a/backend/tests/test_config_version.py b/backend/tests/test_config_version.py index ddfec03ce0..44d9a765c0 100644 --- a/backend/tests/test_config_version.py +++ b/backend/tests/test_config_version.py @@ -653,7 +653,7 @@ def _run_config_upgrade_in_checkout(checkout: Path, **env_overrides: str): shutil.copy2(repo_root / "scripts" / "config-upgrade.sh", checkout / "scripts" / "config-upgrade.sh") shutil.copy2(repo_root / "config.example.yaml", checkout / "config.example.yaml") - env = {key: value for key, value in os.environ.items() if key not in {"DEER_FLOW_CONFIG_PATH", "DEER_FLOW_PROJECT_ROOT"}} + env = {key: value for key, value in os.environ.items() if key not in {"DEER_FLOW_CONFIG_PATH", "DEER_FLOW_PROJECT_ROOT", "DEER_FLOW_ENV_FILE"}} env["UV_PROJECT"] = str(repo_root / "backend") env.update(env_overrides) return subprocess.run( @@ -776,3 +776,63 @@ def test_config_upgrade_creates_checkout_config_when_none_exists(tmp_path): assert "creating from example" in result.stdout assert (checkout / "config.yaml").read_bytes() == (checkout / "config.example.yaml").read_bytes() + + +def test_config_upgrade_feeds_inline_python_on_stdin(): + """The upgrade script must not pass its inline Python via a `python -c` argv payload. + + On Windows setups where python/uv resolve through shim layers that reparse + arguments with cmd.exe (e.g. pyenv-win), multi-line argv payloads get + mangled mid-flight (observed as `IndentationError: unexpected indent` on + `|| goto :error`) and `make dev` fails before the Gateway starts. Feeding + the payload on stdin is safe across every spawn chain. + """ + repo_root = Path(__file__).resolve().parents[2] + script = (repo_root / "scripts" / "config-upgrade.sh").read_text(encoding="utf-8") + assert 'python -c "' not in script + assert script.count("<<'DEERFLOW_UPGRADE_PY'") == 2 + + +@pytest.mark.skipif(SCRIPT_BASH is None, reason="repo shell-script tests need Git Bash on Windows") +def test_config_upgrade_honors_deer_flow_env_file_over_checkout_dotenv(tmp_path): + """An explicit DEER_FLOW_ENV_FILE selection must win over the checkout .env. + + The Gateway loads the selected env file and upgrades the config it points + at; this script must not pre-load the checkout .env first, or its values + would shadow the selected file's (load_dotenv uses override=False). + """ + checkout = tmp_path / "checkout" + live = tmp_path / "live" / "config.yaml" + other = tmp_path / "other" / "config.yaml" + _write_outdated_config(checkout / "config.yaml") + _write_outdated_config(live) + _write_outdated_config(other) + (checkout / ".env").write_text(f"DEER_FLOW_CONFIG_PATH={other}\n", encoding="utf-8") + stage = checkout / "stage.env" + stage.write_text(f"DEER_FLOW_CONFIG_PATH={live}\n", encoding="utf-8") + + result = _run_config_upgrade_in_checkout(checkout, DEER_FLOW_ENV_FILE=str(stage)) + assert result.returncode == 0, result.stdout + result.stderr + + assert yaml.safe_load(live.read_text(encoding="utf-8"))["config_version"] > 1 + assert yaml.safe_load(other.read_text(encoding="utf-8"))["config_version"] == 1 + + +@pytest.mark.skipif(SCRIPT_BASH is None, reason="repo shell-script tests need Git Bash on Windows") +def test_config_upgrade_checkout_helper_ignores_inherited_env_file(tmp_path, monkeypatch): + """A developer's exported selector must not redirect a fixture's upgrade.""" + checkout = tmp_path / "checkout" + _write_outdated_config(checkout / "config.yaml") + developer_config = tmp_path / "developer" / "config.yaml" + developer_text = _write_outdated_config(developer_config) + developer_env = tmp_path / "developer.env" + developer_env.write_text(f"DEER_FLOW_CONFIG_PATH={developer_config}\n", encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_ENV_FILE", str(developer_env)) + + result = _run_config_upgrade_in_checkout(checkout) + assert result.returncode == 0, result.stdout + result.stderr + + assert developer_config.read_text(encoding="utf-8") == developer_text + assert not developer_config.with_suffix(".yaml.bak").exists() + expected_version = yaml.safe_load((checkout / "config.example.yaml").read_text(encoding="utf-8"))["config_version"] + assert yaml.safe_load((checkout / "config.yaml").read_text(encoding="utf-8"))["config_version"] == expected_version diff --git a/scripts/config-upgrade.sh b/scripts/config-upgrade.sh index 81e3411e2d..d3f642c6dd 100755 --- a/scripts/config-upgrade.sh +++ b/scripts/config-upgrade.sh @@ -22,10 +22,28 @@ fi # backend/config.yaml present, `make dev` reads the checkout copy. The import # loads .env as the Gateway does; DEER_FLOW_PROJECT_ROOT then defaults to the # checkout, as in serve.sh. Prints nothing when no config exists yet. -CONFIG="$(cd "$REPO_ROOT/backend" && REPO_ROOT_WIN_PATH="$REPO_ROOT_WIN" uv run python -c " +# Pass the payload on stdin instead of `python -c`: on Windows setups where +# python/uv resolve through pyenv-win shim layers, arguments cross a cmd.exe +# reparse boundary that mangles multi-line argv (observed as a bogus +# "IndentationError: unexpected indent" on '|| goto :error'). stdin is safe. +CONFIG="$(cd "$REPO_ROOT/backend" && REPO_ROOT_WIN_PATH="$REPO_ROOT_WIN" uv run python - <<'DEERFLOW_UPGRADE_PY' import os import sys +from deerflow.env import load_selected_env_file + +# Honor an explicit DEER_FLOW_ENV_FILE selection first: values loaded here +# keep precedence over anything app_config's import-time load discovers, +# matching the Gateway. Only with no selector do we load the checkout .env +# explicitly, so the probe does not depend on dotenv's frame-based discovery +# anchoring inside the harness package. Bonus: loading before the import +# below lets DEER_FLOW_PROJECT_ROOT in the checkout .env win over the +# setdefault, as it does for the Gateway. +if not load_selected_env_file(): + from dotenv import load_dotenv + + load_dotenv(os.path.join(os.environ['REPO_ROOT_WIN_PATH'], '.env')) + from deerflow.config.app_config import AppConfig os.environ.setdefault('DEER_FLOW_PROJECT_ROOT', os.environ['REPO_ROOT_WIN_PATH']) @@ -38,7 +56,8 @@ except FileNotFoundError as exc: sys.exit(f'ERROR {exc}') except ValueError as exc: sys.exit(f'ERROR {exc}') -")" +DEERFLOW_UPGRADE_PY +)" if [ ! -f "$EXAMPLE" ]; then echo "✗ config.example.yaml not found at $EXAMPLE" @@ -61,7 +80,7 @@ else EXAMPLE_WIN="$EXAMPLE" fi -cd "$REPO_ROOT/backend" && CONFIG_WIN_PATH="$CONFIG_WIN" EXAMPLE_WIN_PATH="$EXAMPLE_WIN" uv run python -c " +cd "$REPO_ROOT/backend" && CONFIG_WIN_PATH="$CONFIG_WIN" EXAMPLE_WIN_PATH="$EXAMPLE_WIN" uv run python - <<'DEERFLOW_UPGRADE_PY' import os import sys, shutil, copy, re, secrets from pathlib import Path @@ -231,7 +250,7 @@ if migrated: added = [] def merge(target, source, path=''): - \"\"\"Recursively merge source into target, adding missing keys only.\"\"\" + """Recursively merge source into target, adding missing keys only.""" for key, value in source.items(): key_path = f'{path}.{key}' if path else key if key not in target: @@ -265,4 +284,4 @@ if not migrated and not added: print() print(f'OK config.yaml upgraded to version {example_version}.') print(' Please review the changes and set any new required values.') -" +DEERFLOW_UPGRADE_PY