From fb553747d1d6fa9fd0d5f55729895cfe2e8d15f7 Mon Sep 17 00:00:00 2001 From: Hyeonsang Cho Date: Tue, 27 Oct 2026 23:19:33 +0900 Subject: [PATCH] fix(scripts): read the checkout .env as data in serve.sh (#7606) * fix(scripts): read the checkout .env as data in serve.sh serve.sh sourced the root .env under set -a, so bash ran it as a script: $$ became the shell's PID, backticks and $(...) ran commands, and an unquoted value with a space ran its second word as a command (exit 127 under set -e, stopping make dev). The exports also beat the Gateway's own load_dotenv(override=False), so e.g. a $$ in AUTH_JWT_SECRET changed the signing secret on every launch. Parse .env with a stdlib-only scripts/dotenv_exports.py on the host Python and eval its shell-quoted export lines. It tokenizes like python-dotenv, so values match what the Gateway reads, and keeps the launcher's precedence over inherited variables and its $NAME / ${NAME:-x} / leading-~ expansion that support_bundle.py mirrors. $$ is now one token in both. Unparseable lines and invalid names are skipped with warnings that never echo values; a non-UTF-8 file stops the launcher. Tests run serve.sh's real .env block: no execution, parity with python-dotenv, precedence, warnings, the non-UTF-8 stop and a stdlib-only run; the support-bundle launcher cases now exercise the parser. * docs(changelog): reference #7606 for serve.sh reading .env as data --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 11 ++ CHANGELOG_zh.md | 8 + README.md | 6 + backend/tests/AGENTS.md | 10 + backend/tests/test_serve_config_presence.py | 2 +- backend/tests/test_serve_dotenv.py | 146 +++++++++++++++ .../test_serve_gateway_startup_timeout.py | 2 +- backend/tests/test_support_bundle.py | 13 +- scripts/dotenv_exports.py | 173 ++++++++++++++++++ scripts/serve.sh | 25 ++- scripts/support_bundle.py | 9 +- 11 files changed, 389 insertions(+), 16 deletions(-) create mode 100644 backend/tests/test_serve_dotenv.py create mode 100644 scripts/dotenv_exports.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d1ad3499b2..ba13c6219e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1402,6 +1402,16 @@ As of 2026-10-21, the milestone includes **829 merged pull requests**. ### Fixed +- **scripts:** `make dev` and `make start` no longer run the checkout `.env` + as a shell script. `serve.sh` sourced it, so `$$` became the shell's PID, + backticks and `$(...)` ran commands, and an unquoted value with a space ran + its second word as a command and stopped the launcher with exit 127. Those + values also beat the Gateway's own `.env` read, so a `$$` in + `AUTH_JWT_SECRET` changed the secret on every launch. The launcher now + parses `.env` like python-dotenv, keeps its precedence over shell exports and + its `$NAME`, `${NAME:-default}` and leading `~` expansion, and leaves `$$`, + backticks and `$(...)` as written. ([#7606]) + - **channels:** IM replies that present files no longer resolve those files on the event loop. For every artifact, both reply paths (`runs.wait` and streaming) resolved the virtual path (`realpath`), resolved the outputs @@ -13203,3 +13213,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7591]: https://github.com/bytedance/deer-flow/pull/7591 [#7593]: https://github.com/bytedance/deer-flow/pull/7593 [#7594]: https://github.com/bytedance/deer-flow/pull/7594 +[#7606]: https://github.com/bytedance/deer-flow/pull/7606 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 69084adcbb..a310aac91f 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -1116,6 +1116,13 @@ ### 修复 +- **脚本：** `make dev` 和 `make start` 不再把检出目录的 `.env` 当作 shell 脚本执行。此前 + `serve.sh` 用 `source` 读取它，`$$` 会变成 shell 的 PID，反引号和 `$(...)` 会执行命令，未加引号 + 且含空格的值会把第二个词当作命令运行，启动脚本以退出码 127 中止。这些值还会覆盖 Gateway + 自己读取的 `.env`，例如 `AUTH_JWT_SECRET` 中的 `$$` 会让密钥每次启动都变化。现在启动脚本按 + python-dotenv 的规则解析 `.env`，保留其优先于 shell 导出变量的行为以及 `$NAME`、 + `${NAME:-default}` 和开头 `~` 的展开，`$$`、反引号和 `$(...)` 则保持原样。([#7606]) + - **渠道：** 展示文件的 IM 回复不再在事件循环上解析这些文件。此前两条回复路径（`runs.wait` 和流式）都会对每个产物内联解析虚拟路径（`realpath`）、解析输出根目录并用 `stat` 检查文件， 在网络存储上会让其他渠道和请求停顿。现在这些工作在工作线程中进行。([#7594]) @@ -10529,3 +10536,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7591]: https://github.com/bytedance/deer-flow/pull/7591 [#7593]: https://github.com/bytedance/deer-flow/pull/7593 [#7594]: https://github.com/bytedance/deer-flow/pull/7594 +[#7606]: https://github.com/bytedance/deer-flow/pull/7606 diff --git a/README.md b/README.md index 5c810882c4..bab1317d11 100644 --- a/README.md +++ b/README.md @@ -473,6 +473,12 @@ fences, including unclosed fences, in complete and streamed replies. TAVILY_API_KEY=your-tavily-api-key ``` + `make dev` and `make start` read `.env` as data, never as a shell script, so a + value means what the Gateway reads: `$$`, backticks and `$(...)` stay as + written, and an unquoted value may contain spaces. `$NAME`, `${NAME}`, + `${NAME:-default}` and an unquoted leading `~` still expand, and + single-quoted values stay literal. + For an explicit backend dotenv file, export `DEER_FLOW_ENV_FILE` before startup, diff --git a/backend/tests/AGENTS.md b/backend/tests/AGENTS.md index 1b0726395e..393b5a1cec 100644 --- a/backend/tests/AGENTS.md +++ b/backend/tests/AGENTS.md @@ -491,6 +491,16 @@ smoke checks. Keep config content and secret stability checks portable; isolate POSIX owner-only permission assertions so Windows skips only those checks. Nginx path expectations must include quoted-string escaping for native paths. +## Launcher dotenv parsing + +`serve.sh` never sources root `.env`: stdlib-only `scripts/dotenv_exports.py` +(host Python, before `uv sync`) tokenizes it like python-dotenv, keeps `.env` +over shell exports and the `$NAME`/`${NAME:-x}`/leading-`~` expansion that +`support_bundle.py` mirrors (keep both expanders in step), and prints +shell-quoted exports for `eval`. `test_serve_dotenv.py` and the launcher-block +cases in `test_support_bundle.py` run the real block; harnesses that copy +`serve.sh` with a `.env` must copy the parser too. + ## Channel artifact turn sequencing `test_channels.py` models consecutive artifact-producing turns by awaiting an diff --git a/backend/tests/test_serve_config_presence.py b/backend/tests/test_serve_config_presence.py index 20e299ac1d..e7a72bc124 100644 --- a/backend/tests/test_serve_config_presence.py +++ b/backend/tests/test_serve_config_presence.py @@ -39,7 +39,7 @@ def test_serve_config_presence_reaches_dependency_sync(tmp_path: Path, mode: str scripts = checkout / "scripts" scripts.mkdir(parents=True) (checkout / "backend").mkdir() - for name in ("serve.sh", "detect_uv_extras.py"): + for name in ("serve.sh", "dotenv_exports.py", "detect_uv_extras.py"): shutil.copy2(REPO_ROOT / "scripts" / name, scripts / name) (scripts / "nginx-local-conf.sh").write_text("printf 'test-nginx.conf\\n'\n", encoding="utf-8") (scripts / "cleanup-containers.sh").write_text("exit 0\n", encoding="utf-8") diff --git a/backend/tests/test_serve_dotenv.py b/backend/tests/test_serve_dotenv.py new file mode 100644 index 0000000000..c9f38b5f1b --- /dev/null +++ b/backend/tests/test_serve_dotenv.py @@ -0,0 +1,146 @@ +"""serve.sh reads the checkout .env as data, never as a shell script. + +It used to ``source`` the file under ``set -a``: ``$$`` became the shell's PID, +backticks and ``$(...)`` ran commands, and an unquoted value with a space ran +its second word as a command (exit 127 under ``set -e``, so ``make dev`` died). +The exported values then beat the Gateway's own python-dotenv read, which never +overrides an existing variable. + +These tests run serve.sh's actual ``.env`` block (and the ``_pick_python`` it +calls) in bash against a scratch checkout, then read back what it exported. +The launcher's expansion rules and precedence are pinned against +``support_bundle.py`` in ``test_support_bundle.py``. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +from dotenv import dotenv_values +from support.shell import require_script_bash + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = REPO_ROOT / "scripts" + +_DUMP_ENVIRONMENT = "\"$TEST_PYTHON\" -c 'import json, os, sys; sys.stdout.write(json.dumps(dict(os.environ)))'" + + +def _dotenv_block() -> str: + serve = (SCRIPTS / "serve.sh").read_text(encoding="utf-8") + pick_python = serve[serve.index("_pick_python() {") :].split("\n}\n", 1)[0] + "\n}\n" + block = serve.split("# ── Load .env", 1)[1].split("\n\n", 1)[1].split("# ── Argument parsing", 1)[0] + return pick_python + block + + +def _run_dotenv_block(project: Path, dotenv: bytes, **environment: str) -> subprocess.CompletedProcess[str]: + (project / "scripts").mkdir(parents=True, exist_ok=True) + shutil.copy2(SCRIPTS / "dotenv_exports.py", project / "scripts" / "dotenv_exports.py") + (project / ".env").write_bytes(dotenv) + env = {key: value for key, value in os.environ.items() if key not in {"BASH_ENV", "ENV"}} + env.update(environment, REPO_ROOT=str(project), TEST_PYTHON=Path(sys.executable).as_posix()) + return subprocess.run( + [require_script_bash(), "-c", 'set -e\ncd "$REPO_ROOT"\n' + _dotenv_block() + _DUMP_ENVIRONMENT], + cwd=project, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + + +def _exported(project: Path, dotenv: bytes, **environment: str) -> dict[str, str]: + result = _run_dotenv_block(project, dotenv, **environment) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def test_dotenv_values_are_data_not_shell_code(tmp_path: Path) -> None: + dotenv = 'PASSWORD=pa$$word\nQUOTED_PASSWORD="pa$$word"\nBACKTICKS=`touch backtick-ran`kept\nSUBSTITUTION=$(touch substitution-ran)\nSPACED=hello world\nCHAINED=first; touch chained-ran\nAFTER=still-loaded\n' + + exported = _exported(tmp_path, dotenv.encode("utf-8")) + + assert exported["PASSWORD"] == "pa$$word" + assert exported["QUOTED_PASSWORD"] == "pa$$word" + assert exported["BACKTICKS"] == "`touch backtick-ran`kept" + assert exported["SUBSTITUTION"] == "$(touch substitution-ran)" + assert exported["SPACED"] == "hello world" + assert exported["CHAINED"] == "first; touch chained-ran" + assert exported["AFTER"] == "still-loaded" + assert sorted(path.name for path in tmp_path.glob("*-ran")) == [] + + +def test_literal_values_match_what_the_gateway_reads(tmp_path: Path) -> None: + """Without references or tildes, every value equals python-dotenv's reading.""" + dotenv = ( + "FIRST=after-bom\r\n" + "# a comment line\r\n" + "export EXPORTED=prefixed\r\n" + "INLINE=value # trailing comment\r\n" + "HASH=no#comment\r\n" + "SPACED_EQUALS = padded\r\n" + "EMPTY=\r\n" + "SINGLE='single \\'quoted\\' # not a comment'\r\n" + 'DOUBLE="tab\\there \\"quoted\\""\r\n' + 'MULTILINE="line one\r\nline two"\r\n' + "'QUOTED_KEY'=quoted key\r\n" + "UNICODE=café ✓\r\n" + "BARE_NAME\r\n" + ) + path = tmp_path / "gateway.env" + path.write_text(dotenv, encoding="utf-8", newline="") + gateway = {key: value for key, value in dotenv_values(path, encoding="utf-8-sig").items() if value is not None} + + exported = _exported(tmp_path, dotenv.encode("utf-8")) + + assert {key: exported.get(key) for key in gateway} == gateway + assert "BARE_NAME" not in exported + + +def test_dotenv_still_overrides_inherited_variables(tmp_path: Path) -> None: + exported = _exported(tmp_path, b"FROM_SHELL=from-dotenv\nEMPTIED=\n", FROM_SHELL="from-shell", EMPTIED="from-shell") + + assert exported["FROM_SHELL"] == "from-dotenv" + assert exported["EMPTIED"] == "" + + +def test_unparseable_statements_and_names_are_skipped_without_echoing_values(tmp_path: Path) -> None: + dotenv = b'BEFORE=kept\nNOT-A-NAME=secret-one\nBROKEN="unterminated secret-two\nAFTER=kept\n' + + result = _run_dotenv_block(tmp_path, dotenv) + + assert result.returncode == 0, result.stderr + exported = json.loads(result.stdout) + assert exported["BEFORE"] == "kept" + assert exported["AFTER"] == "kept" + assert "dotenv_exports: skipping 'NOT-A-NAME' at line 2: not a shell variable name" in result.stderr + assert "dotenv_exports: skipping a statement that does not parse at line 3" in result.stderr + assert "secret-one" not in result.stderr + assert "secret-two" not in result.stderr + + +def test_non_utf8_dotenv_stops_the_launcher_before_exporting(tmp_path: Path) -> None: + result = _run_dotenv_block(tmp_path, b"FIRST=kept\nLATIN1=caf\xe9\n") + + assert result.returncode != 0 + assert "is not valid UTF-8" in result.stderr + assert result.stdout == "" + + +@pytest.mark.parametrize("source", ["`touch {marker}`", "$(touch {marker})"]) +def test_parser_runs_on_the_standard_library_and_never_executes(tmp_path: Path, source: str) -> None: + """serve.sh runs it with the host Python, before ``uv sync`` installs anything.""" + marker = tmp_path / "ran" + dotenv = tmp_path / ".env" + dotenv.write_text(f"VALUE={source.format(marker=marker.as_posix())}\n", encoding="utf-8") + + result = subprocess.run([sys.executable, "-S", str(SCRIPTS / "dotenv_exports.py"), str(dotenv)], capture_output=True, check=True) + + assert result.stdout.decode("utf-8") == f"export VALUE='{source.format(marker=marker.as_posix())}'\n" + assert not marker.exists() diff --git a/backend/tests/test_serve_gateway_startup_timeout.py b/backend/tests/test_serve_gateway_startup_timeout.py index a69dcd12c5..b615ce13f4 100644 --- a/backend/tests/test_serve_gateway_startup_timeout.py +++ b/backend/tests/test_serve_gateway_startup_timeout.py @@ -61,7 +61,7 @@ def _run_launcher(tmp_path: Path, *, timeout: str | None, ready_at: int = 145, a scripts = checkout / "scripts" scripts.mkdir(parents=True) (checkout / "backend").mkdir() - for name in ("serve.sh", "wait-for-port.sh", "detect_uv_extras.py"): + for name in ("serve.sh", "dotenv_exports.py", "wait-for-port.sh", "detect_uv_extras.py"): shutil.copy2(REPO_ROOT / "scripts" / name, scripts / name) (scripts / "nginx-local-conf.sh").write_text("printf 'test-nginx.conf\\n'\n", encoding="utf-8", newline="\n") (scripts / "config-upgrade.sh").write_text("exit 0\n", encoding="utf-8", newline="\n") diff --git a/backend/tests/test_support_bundle.py b/backend/tests/test_support_bundle.py index a107b009c1..d354ffb83f 100644 --- a/backend/tests/test_support_bundle.py +++ b/backend/tests/test_support_bundle.py @@ -4,6 +4,7 @@ from __future__ import annotations import json import os +import shutil import subprocess import sys import zipfile @@ -996,8 +997,13 @@ def test_thread_summary_honors_project_root_override(tmp_path, monkeypatch, sour def _local_launcher_home(project): """Execute the launcher's actual dotenv/default blocks without starting services.""" bash = require_script_bash() - serve = (Path(support_bundle.__file__).parent / "serve.sh").read_text(encoding="utf-8") - dotenv_block = serve.split("# ── Load .env", 1)[1].split("\n\n", 1)[1].split("_pick_python()", 1)[0] + scripts = Path(support_bundle.__file__).parent + serve = (scripts / "serve.sh").read_text(encoding="utf-8") + pick_python = serve[serve.index("_pick_python() {") :].split("\n}\n", 1)[0] + "\n}\n" + dotenv_block = serve.split("# ── Load .env", 1)[1].split("\n\n", 1)[1].split("# ── Argument parsing", 1)[0] + # The block hands .env to the parser beside it in the checkout's scripts/. + (project / "scripts").mkdir(exist_ok=True) + shutil.copy2(scripts / "dotenv_exports.py", project / "scripts" / "dotenv_exports.py") defaults = serve[serve.index("# Runtime path defaults.") :].split("# Extra flags", 1)[0] result = subprocess.run( [ @@ -1006,7 +1012,7 @@ def _local_launcher_home(project): # Git Bash answers with POSIX paths, and its mount table (for example Windows TEMP # at /tmp) is not reproducible from Python, so convert inside the shell. Passing the # value through argv instead would let MSYS rewrite shell syntax inside it. - 'set -e\ncd "$REPO_ROOT"\n' + dotenv_block + defaults + '\nif command -v cygpath >/dev/null 2>&1; then cygpath -w "$DEER_FLOW_HOME"; else printf "%s" "$DEER_FLOW_HOME"; fi', + 'set -e\ncd "$REPO_ROOT"\n' + pick_python + dotenv_block + defaults + '\nif command -v cygpath >/dev/null 2>&1; then cygpath -w "$DEER_FLOW_HOME"; else printf "%s" "$DEER_FLOW_HOME"; fi', ], cwd=project, env={**os.environ, "REPO_ROOT": str(project)}, @@ -1122,6 +1128,7 @@ def test_missing_dotenv_keeps_support_bundle_available(tmp_path, monkeypatch, so ("", "'$PWD/backend/.deer-flow'"), ("", "'${PWD}/backend/.deer-flow'"), ("", r'"\$PWD/backend/.deer-flow"'), + ("", '"$PWD/backend/$$runtime-data"'), ('RUNTIME_DATA="$PWD/runtime-data"\n', '"$RUNTIME_DATA"'), ('RUNTIME_DATA="$PWD/old"\nRUNTIME_DATA="$PWD/new"\n', '"${RUNTIME_DATA}"'), ('RUNTIME_DATA="$PWD/first"\nRUNTIME_DATA="$RUNTIME_DATA/second"\n', '"$RUNTIME_DATA"'), diff --git a/scripts/dotenv_exports.py b/scripts/dotenv_exports.py new file mode 100644 index 0000000000..172faf1679 --- /dev/null +++ b/scripts/dotenv_exports.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Print the checkout `.env` as shell `export` lines for `serve.sh`. + +`serve.sh` used to `source` the file, so bash ran it as a script: `$$` became +the shell's PID, backticks and `$(...)` ran commands, and an unquoted value +with a space ran its second word as a command (exit 127 under `set -e`). The +Gateway reads the same file with python-dotenv but never overrides a variable +that is already exported, so the shell's reading won. + +This reads the file as data instead and never executes any of it: + +- Statements are tokenized like python-dotenv (comments, `export` prefixes, + single/double quotes and their escapes, multi-line quoted values, inline + comments after unquoted values), so a value means what the Gateway reads. +- The launcher's own semantics stay, as `support_bundle.py` mirrors them: + `.env` assignments override inherited variables, even with an empty value; + `$NAME`, `${NAME}` and `${NAME:-literal}` expand in unquoted and + double-quoted values from the environment, the checkout `PWD` and earlier + assignments; `\\$` is a literal dollar and `$$` stays as written; an + unquoted leading `~` expands; single-quoted values stay literal. + +Each assignment is printed as `export NAME=` for `eval`. +Statements that do not parse, and keys that are not shell variable names, are +skipped with a warning on stderr naming only the line or key, never a value. +An unreadable or non-UTF-8 file exits 1 before anything is printed. + +Intentionally implemented with the standard library only: `serve.sh` runs this +with the host Python before `uv sync` has populated the venv. +""" + +from __future__ import annotations + +import codecs +import os +import re +import shlex +import sys +from collections.abc import Mapping +from pathlib import Path + +# Statement grammar of python-dotenv's parser (`dotenv/parser.py`). +_LEADING_SPACE = re.compile(r"\s*") +_EXPORT = re.compile(r"(?:export[^\S\r\n]+)?") +_SINGLE_QUOTED_KEY = re.compile(r"'([^']+)'") +_UNQUOTED_KEY = re.compile(r"([^=\#\s]+)") +_SPACE = re.compile(r"[^\S\r\n]*") +_EQUAL_SIGN = re.compile(r"=[^\S\r\n]*") +_SINGLE_QUOTED_VALUE = re.compile(r"'((?:\\'|[^'])*)'") +_DOUBLE_QUOTED_VALUE = re.compile(r'"((?:\\"|[^"])*)"') +_UNQUOTED_VALUE = re.compile(r"[^\r\n]*") +_COMMENT = re.compile(r"(?:[^\S\r\n]*#[^\r\n]*)?") +_END_OF_LINE = re.compile(r"[^\S\r\n]*(?:\r\n|\n|\r|$)") +_REST_OF_LINE = re.compile(r"[^\r\n]*(?:\r\n|\n|\r)?") +_SINGLE_QUOTE_ESCAPES = re.compile(r"\\[\\']") +_DOUBLE_QUOTE_ESCAPES = re.compile(r"\\[\\'\"abfnrtv]") + +# The launcher's variable references, as support_bundle.py resolves them. `$$` +# is one token, as in the shell, so its second `$` never starts a reference. +_REFERENCE = re.compile(r"\\\$|\$\$|\$(?:\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}|([A-Za-z_][A-Za-z0-9_]*))") +_SHELL_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") + + +class _Unparsed(Exception): + pass + + +def _match(pattern: re.Pattern[str], text: str, position: int) -> re.Match[str]: + match = pattern.match(text, position) + if match is None: + raise _Unparsed + return match + + +def _decode_escapes(pattern: re.Pattern[str], value: str) -> str: + return pattern.sub(lambda match: codecs.decode(match[0], "unicode-escape"), value) + + +def _expand_references(value: str, environment: Mapping[str, str]) -> str: + """Expand simple shell references in one pass, without executing shell syntax.""" + + def replace(match: re.Match[str]) -> str: + if match[0] == "\\$": + return "$" + if match[0] == "$$": + return "$$" + resolved = environment.get(match[1] or match[3], "") + return match[2] if match[2] is not None and not resolved else resolved + + return _REFERENCE.sub(replace, value) + + +def parse_dotenv(text: str, environment: Mapping[str, str]) -> list[tuple[str, str]]: + """Return the `(name, value)` assignments in `text`, in file order.""" + scope = dict(environment) + assignments: list[tuple[str, str]] = [] + position = 0 + while position < len(text): + position = _LEADING_SPACE.match(text, position).end() + if position >= len(text): + break + line = text.count("\n", 0, position) + 1 + try: + position = _EXPORT.match(text, position).end() + key = None + if text.startswith("'", position): + match = _match(_SINGLE_QUOTED_KEY, text, position) + key, position = match[1], match.end() + elif not text.startswith("#", position): + match = _match(_UNQUOTED_KEY, text, position) + key, position = match[1], match.end() + position = _SPACE.match(text, position).end() + quote = "" + value = None + if text.startswith("=", position): + position = _EQUAL_SIGN.match(text, position).end() + quote = text[position : position + 1] + if quote == "'": + match = _match(_SINGLE_QUOTED_VALUE, text, position) + value = _decode_escapes(_SINGLE_QUOTE_ESCAPES, match[1]) + elif quote == '"': + match = _match(_DOUBLE_QUOTED_VALUE, text, position) + value = _decode_escapes(_DOUBLE_QUOTE_ESCAPES, match[1]) + else: + match = _UNQUOTED_VALUE.match(text, position) + value = re.sub(r"\s+#.*", "", match[0]).rstrip() + position = match.end() + position = _COMMENT.match(text, position).end() + position = _match(_END_OF_LINE, text, position).end() + except _Unparsed: + print(f"dotenv_exports: skipping a statement that does not parse at line {line}", file=sys.stderr) + position = _REST_OF_LINE.match(text, position).end() + continue + # A comment, or a bare name without `=`, assigns nothing. + if key is None or value is None: + continue + if not _SHELL_NAME.match(key): + print(f"dotenv_exports: skipping {key!r} at line {line}: not a shell variable name", file=sys.stderr) + continue + if quote != "'": + if quote != '"': + value = os.path.expanduser(value) + value = _expand_references(value, scope) + scope[key] = value + assignments.append((key, value)) + return assignments + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + if len(args) != 1: + print("usage: dotenv_exports.py ", file=sys.stderr) + return 2 + path = Path(args[0]) + try: + # utf-8-sig drops a leading BOM, which would otherwise join the first key. + text = path.read_text(encoding="utf-8-sig") + except UnicodeDecodeError: + print(f"dotenv_exports: {path} is not valid UTF-8", file=sys.stderr) + return 1 + except OSError as exc: + print(f"dotenv_exports: cannot read {path}: {exc.strerror or exc}", file=sys.stderr) + return 1 + environment = dict(os.environ) + # serve.sh runs this from the checkout, so `$PWD` resolves against it. + environment.setdefault("PWD", os.getcwd()) + exports = "".join(f"export {name}={shlex.quote(value)}\n" for name, value in parse_dotenv(text, environment)) + # Bytes with "\n" endings: Windows text mode would add a "\r" to every value. + sys.stdout.buffer.write(exports.encode("utf-8")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/serve.sh b/scripts/serve.sh index 60550516f5..277065cf80 100755 --- a/scripts/serve.sh +++ b/scripts/serve.sh @@ -31,14 +31,6 @@ set -e REPO_ROOT="$(builtin cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd -P)" cd "$REPO_ROOT" -# ── Load .env ──────────────────────────────────────────────────────────────── - -if [ -f "$REPO_ROOT/.env" ]; then - set -a - source "$REPO_ROOT/.env" - set +a -fi - _pick_python() { local candidate for candidate in python3 python py; do @@ -58,6 +50,23 @@ _pick_python() { return 1 } +# ── Load .env ──────────────────────────────────────────────────────────────── + +# .env is data, not a script: sourcing it turned `$$` into this shell's PID, ran +# backticks and `$(...)`, and ran the second word of an unquoted value with a +# space as a command. dotenv_exports.py parses it like python-dotenv, keeping the +# launcher's `$NAME` / `${NAME:-default}` / leading `~` expansion and its +# precedence over inherited variables, and prints shell-quoted exports. +if [ -f "$REPO_ROOT/.env" ]; then + if ! DOTENV_PYTHON="$(_pick_python)"; then + echo "✗ Python 3 is required to read .env." >&2 + exit 1 + fi + DOTENV_EXPORTS="$("$DOTENV_PYTHON" "$REPO_ROOT/scripts/dotenv_exports.py" "$REPO_ROOT/.env")" || exit 1 + eval "$DOTENV_EXPORTS" + unset DOTENV_PYTHON DOTENV_EXPORTS +fi + # ── Argument parsing ───────────────────────────────────────────────────────── DEV_MODE=true diff --git a/scripts/support_bundle.py b/scripts/support_bundle.py index 5e3bdeadd3..b2d3c65680 100644 --- a/scripts/support_bundle.py +++ b/scripts/support_bundle.py @@ -267,13 +267,16 @@ def _expand_launcher_variables(value: str, environment: dict[str, str]) -> str: """Expand simple shell references in one pass, without executing shell syntax.""" def replace(match: re.Match[str]) -> str: - if match[0].startswith("\\"): - return match[0][1:] + if match[0] == "\\$": + return "$" + if match[0] == "$$": + # One token, as in the shell: its second `$` never starts a reference. + return "$$" name = match[1] or match[3] resolved = environment.get(name, "") return match[2] if match[2] is not None and not resolved else resolved - return re.sub(r"\\?\$(?:\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}|([A-Za-z_][A-Za-z0-9_]*))", replace, value) + return re.sub(r"\\\$|\$\$|\$(?:\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}|([A-Za-z_][A-Za-z0-9_]*))", replace, value) def _thread_data_roots(project_root: Path) -> list[tuple[Path, str]]: