diff --git a/.env.example b/.env.example index 9c0868019e..dd57cd9d88 100644 --- a/.env.example +++ b/.env.example @@ -78,8 +78,9 @@ SOFYA_API_KEY=your-sofya-api-key # GATEWAY_ENABLE_DOCS=false # Shared internal Gateway auth token for multi-worker deployments. -# `make up` generates and persists this automatically; set it manually only -# when you run Gateway workers outside the bundled deploy script. +# `make up` honors a value set here (as it does for BETTER_AUTH_SECRET) and +# otherwise generates one and persists it under DEER_FLOW_HOME; set it +# manually when Gateway workers also run outside the bundled deploy script. # DEER_FLOW_INTERNAL_AUTH_TOKEN=your-shared-internal-token # Provisioner API key for sandbox authentication (required when using provisioner/K8s sandbox mode). diff --git a/CHANGELOG.md b/CHANGELOG.md index 4be7a38d87..af91831449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -331,6 +331,21 @@ This release closes that milestone with **181 merged pull requests**. `/proc/net/if_inet6` is absent since #2027, and the Helm chart mirrors it; the production compose file was the one launcher still without the guard. It now uses the same launcher, and starts nginx with `exec` so it is PID 1. ([#5900]) +- **deploy:** `make up` / `scripts/deploy.sh` now honors `BETTER_AUTH_SECRET` + and `DEER_FLOW_INTERNAL_AUTH_TOKEN` written to the repo-root `.env`. The + script only checked the shell before reloading a persisted secret or + generating a new one and exporting it, and Compose interpolation lets shell + variables outrank `--env-file`, so the value the deployment docs tell + operators to put in `.env` was silently replaced: sessions were signed with + a secret the operator never chose, and Gateway workers running outside the + stack with the configured token got `401`. A `.env`-provided secret is now + left for Compose to read itself (shell → `.env` → persisted file → + generated). Whether `.env` provides one is decided by Compose itself — the + script renders a stub project with `${KEY}` through `docker compose config` + and reads the value back — so `KEY: VALUE` lines and `${VAR}` interpolation + count the way Compose counts them on every Compose v2 client. A value that + resolves empty, like an exported-but-empty shell variable, still triggers + generation because Compose would otherwise pass the empty value through. ([#5928]) - **skills:** `skill_manage(action="remove_file")` and `write_file` now work on binary support files, and reject directories cleanly. A `.skill` archive may carry `assets/logo.png` (the installer only rejects *executable* binaries), @@ -6140,4 +6155,5 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#5884]: https://github.com/bytedance/deer-flow/pull/5884 [#5893]: https://github.com/bytedance/deer-flow/pull/5893 [#5900]: https://github.com/bytedance/deer-flow/pull/5900 +[#5928]: https://github.com/bytedance/deer-flow/pull/5928 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 90ccf51165..fd5b1887ab 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -334,6 +334,16 @@ 会在 `/proc/net/if_inet6` 不存在时去掉 IPv6 监听,Helm chart 也做了同样处理;生产用 compose 文件是唯一还没有这层保护的启动器。现在它使用相同的启动脚本,并以 `exec` 启动 nginx 使其成为 PID 1。([#5900]) +- **部署:** `make up` / `scripts/deploy.sh` 现在会采用写在仓库根目录 `.env` 中的 + `BETTER_AUTH_SECRET` 与 `DEER_FLOW_INTERNAL_AUTH_TOKEN`。此前脚本只检查 shell 环境,随后 + 就重新加载已持久化的密钥或生成新密钥并 export;而 Compose 插值时 shell 变量优先于 + `--env-file`,于是部署文档让运维写进 `.env` 的值被悄悄替换:会话用的是运维从未选择的密钥, + 在栈外运行、持有配置 token 的 Gateway worker 则收到 `401`。现在 `.env` 提供的密钥交由 + Compose 自行读取(优先级 shell → `.env` → 持久化文件 → 新生成)。`.env` 是否提供了值由 + Compose 自己决定:脚本用 `docker compose config` 渲染一个只含 `${KEY}` 的桩项目并读回结果, + 因此 `KEY: VALUE` 写法和值内的 `${VAR}` 插值在所有 Compose v2 客户端上都按 Compose 的规则 + 计算;解析结果为空的值(例如已 export 但为空的 shell 变量)仍会触发生成,因为 Compose 否则 + 会把空值直接传下去。([#5928]) - **技能:** `skill_manage(action="remove_file")` 与 `write_file` 现在可以处理二进制支持文件, 并会干净地拒绝目录。`.skill` 压缩包可以包含 `assets/logo.png`(安装器只拒绝*可执行*二进制), 但这两个操作在改动文件之前会先把原有内容按 UTF-8 文本读出(仅用于历史记录),于是二进制 @@ -5221,3 +5231,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写,核心包含子 [#5884]: https://github.com/bytedance/deer-flow/pull/5884 [#5893]: https://github.com/bytedance/deer-flow/pull/5893 [#5900]: https://github.com/bytedance/deer-flow/pull/5900 +[#5928]: https://github.com/bytedance/deer-flow/pull/5928 diff --git a/backend/tests/test_deploy_dotenv_secrets.py b/backend/tests/test_deploy_dotenv_secrets.py new file mode 100644 index 0000000000..a0b4c6186a --- /dev/null +++ b/backend/tests/test_deploy_dotenv_secrets.py @@ -0,0 +1,382 @@ +"""deploy.sh must not shadow secrets the operator wrote to the repo-root .env. + +Compose interpolates ``${BETTER_AUTH_SECRET}`` / ``${DEER_FLOW_INTERNAL_AUTH_TOKEN}`` +from the shell environment first and ``--env-file`` second. deploy.sh only ever +looked at the shell before generating (or reloading a persisted) secret and +exporting it, so a value in ``.env`` -- the surface every deployment doc points +at -- was silently replaced by the auto-generated one. + +Whether ``.env`` provides a value is Compose's call, not a ``KEY=VALUE`` grep: +Compose accepts ``KEY: VALUE`` lines and interpolates ``${VAR}`` inside values. +The script therefore renders a stub project whose one environment entry is +``${KEY}`` through ``docker compose config`` and reads the value back -- a +probe every Compose v2 client can answer. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +from pathlib import Path + +import pytest +from support.shell import find_script_bash + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# Every test here shells out to bash; on Windows that must be Git Bash -- +# the WSL launcher and Store alias stubs cannot run the repo scripts. +BASH = find_script_bash() +pytestmark = pytest.mark.skipif(BASH is None, reason="repo shell-script tests need Git Bash on Windows") + +SECRETS = ("BETTER_AUTH_SECRET", "DEER_FLOW_INTERNAL_AUTH_TOKEN") +PERSISTED_FILE = { + "BETTER_AUTH_SECRET": ".better-auth-secret", + "DEER_FLOW_INTERNAL_AUTH_TOKEN": ".internal-auth-token", +} +GENERATED = re.compile(r"set:[A-Za-z0-9_\-]{32,}") + +# The fake docker answers `compose ... config` the way Compose renders the +# script's stub project: it reads the `${KEY}` reference off stdin, looks the +# key up in a canned resolved environment (what real Compose would compute for +# the .env under test) and prints the environment entry, `""` when the key is +# empty or missing -- or fails with FAKE_COMPOSE_CONFIG_RC. Every other +# invocation stands in for `compose build` and records, per secret, whether +# the variable reached its environment and with which value: "set:" is +# what Compose would take from the shell, "" means Compose falls through to +# --env-file. +_FAKE_DOCKER = """#!/usr/bin/env sh +case " $* " in + *" config "*) + for arg in "$@"; do printf "%s\\n" "$arg"; done > "$CAPTURE_CONFIG_ARGS" + cat > "$CAPTURE_CONFIG_STDIN" + if [ -n "${REAL_DOCKER:-}" ]; then exec "$REAL_DOCKER" "$@" < "$CAPTURE_CONFIG_STDIN"; fi + if [ "${FAKE_COMPOSE_CONFIG_RC:-0}" != 0 ]; then echo "fake compose: cannot load project" >&2; exit "$FAKE_COMPOSE_CONFIG_RC"; fi + key="$(sed -n 's/.*\\${\\([A-Za-z_][A-Za-z0-9_]*\\)}.*/\\1/p' "$CAPTURE_CONFIG_STDIN" | head -n 1)" + value="" + [ -z "${FAKE_COMPOSE_ENVIRONMENT:-}" ] || value="$(sed -n "s/^${key}=//p" "$FAKE_COMPOSE_ENVIRONMENT" | head -n 1)" + [ -n "$value" ] || value='""' + printf 'name: probe\\nservices:\\n probe:\\n environment:\\n DEER_FLOW_PROBE_VALUE: %s\\n image: scratch\\n' "$value" + exit 0 + ;; +esac +{ + printf 'BETTER_AUTH_SECRET=%s\\n' "${BETTER_AUTH_SECRET+set:}${BETTER_AUTH_SECRET:-}" + printf 'DEER_FLOW_INTERNAL_AUTH_TOKEN=%s\\n' "${DEER_FLOW_INTERNAL_AUTH_TOKEN+set:}${DEER_FLOW_INTERNAL_AUTH_TOKEN:-}" +} > "$CAPTURE_SECRETS" +for arg in "$@"; do printf "%s\\n" "$arg"; done > "$CAPTURE_DOCKER_ARGS" +exit 0 +""" + + +def _worktree(tmp_path: Path) -> Path: + worktree = tmp_path / "repo" + shutil.copytree(REPO_ROOT / "scripts", worktree / "scripts") + shutil.copytree(REPO_ROOT / "docker", worktree / "docker") + (worktree / "backend").mkdir() + (worktree / "config.yaml").write_text("database:\n backend: sqlite\n", encoding="utf-8") + (worktree / "extensions_config.json").write_text('{"mcpServers":{},"skills":{}}\n', encoding="utf-8") + return worktree + + +def _run_deploy_build( + tmp_path: Path, + worktree: Path, + *, + compose_environment: dict[str, str] | None = None, + compose_config_rc: int = 0, + real_docker: str | None = None, + shell_env: dict[str, str] | None = None, + check: bool = True, +): + """Run ``deploy.sh build`` against the fake docker and return what it observed.""" + capture_secrets = tmp_path / "secrets.txt" + capture_args = tmp_path / "docker_args.txt" + capture_config_args = tmp_path / "config_args.txt" + capture_config_stdin = tmp_path / "config_stdin.yaml" + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + docker = bin_dir / "docker" + docker.write_text(_FAKE_DOCKER, encoding="utf-8") + docker.chmod(0o755) + + env = os.environ.copy() + for key in (*SECRETS, "UV_EXTRAS", "REAL_DOCKER", "FAKE_COMPOSE_ENVIRONMENT", "FAKE_COMPOSE_CONFIG_RC"): + env.pop(key, None) + env["DEER_FLOW_HOME"] = str(tmp_path / "deer-flow-home") + env["CAPTURE_SECRETS"] = str(capture_secrets) + env["CAPTURE_DOCKER_ARGS"] = str(capture_args) + env["CAPTURE_CONFIG_ARGS"] = str(capture_config_args) + env["CAPTURE_CONFIG_STDIN"] = str(capture_config_stdin) + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + if compose_environment is not None: + canned = tmp_path / "compose_environment.txt" + canned.write_text("".join(f"{k}={v}\n" for k, v in compose_environment.items()), encoding="utf-8") + env["FAKE_COMPOSE_ENVIRONMENT"] = str(canned) + if compose_config_rc: + env["FAKE_COMPOSE_CONFIG_RC"] = str(compose_config_rc) + if real_docker: + env["REAL_DOCKER"] = real_docker + env.update(shell_env or {}) + + result = subprocess.run( + [BASH, str(worktree / "scripts" / "deploy.sh"), "build"], + cwd=worktree, + env=env, + check=check, + text=True, + capture_output=True, + ) + observed = {} + if capture_secrets.exists(): + observed = dict(line.split("=", 1) for line in capture_secrets.read_text(encoding="utf-8").splitlines()) + args = capture_args.read_text(encoding="utf-8").splitlines() if capture_args.exists() else [] + config_args = capture_config_args.read_text(encoding="utf-8").splitlines() if capture_config_args.exists() else [] + config_stdin = capture_config_stdin.read_text(encoding="utf-8") if capture_config_stdin.exists() else "" + return result, observed, args, config_args, config_stdin, Path(env["DEER_FLOW_HOME"]) + + +def _other(key: str) -> str: + return next(secret for secret in SECRETS if secret != key) + + +# ── The script asks Compose, and trusts its answer ────────────────────────── + + +def test_deploy_asks_compose_to_interpolate_the_secret_like_the_real_project(tmp_path): + """The probe renders ``${KEY}`` with the same --env-file and project directory as the real command.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("BETTER_AUTH_SECRET=from-dotenv\n", encoding="utf-8") + + _, _, args, config_args, config_stdin, _ = _run_deploy_build(tmp_path, worktree, compose_environment={"BETTER_AUTH_SECRET": "from-dotenv", "DEER_FLOW_INTERNAL_AUTH_TOKEN": "x"}) + + assert config_args[:1] == ["compose"] + assert "config" in config_args + assert "--env-file" in config_args + assert config_args[config_args.index("--env-file") + 1] == args[args.index("--env-file") + 1] + # The probe resolves the default .env from the same project directory as + # the real command (the compose file's directory), not from the cwd. + assert "--project-directory" in config_args + probe_dir = config_args[config_args.index("--project-directory") + 1] + assert Path(probe_dir).resolve() == Path(args[args.index("-f") + 1]).resolve().parent + # The stub project on stdin is what gets interpolated: it must reference + # the secret and nothing from the real compose file. + assert config_args[config_args.index("-f") + 1] == "-" + assert "${DEER_FLOW_INTERNAL_AUTH_TOKEN}" in config_stdin + assert "docker-compose.yaml" not in config_stdin + + +@pytest.mark.parametrize("key", SECRETS) +@pytest.mark.parametrize( + "dotenv_line", + ["{key}=from-dotenv", "{key}: from-dotenv", '{key}="from-${{OTHER}}"'], + ids=["equals", "colon", "quoted-interpolated"], +) +def test_deploy_leaves_a_compose_resolved_secret_for_compose_instead_of_generating_one(tmp_path, key, dotenv_line): + """Whatever the dotenv spelling, a value Compose resolves is left to Compose.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("OTHER=dotenv\n" + dotenv_line.format(key=key) + "\n", encoding="utf-8") + + result, observed, args, _, _, home = _run_deploy_build(tmp_path, worktree, compose_environment={key: "from-dotenv"}) + + # Compose reads the dotenv itself; an exported copy would outrank it. + assert "--env-file" in args + assert observed[key] == "", f"{key} exported into the compose environment: {observed[key]!r}" + assert not (home / PERSISTED_FILE[key]).exists(), "a persisted secret was generated despite the .env value" + assert f"{key} loaded from" in result.stdout + assert ".env" in result.stdout + + +@pytest.mark.parametrize("key", SECRETS) +def test_deploy_generates_when_compose_resolves_the_dotenv_value_to_empty(tmp_path, key): + """``KEY=${UNSET}`` looks set to a grep but is empty to Compose: generate.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text(f"{key}=${{UNSET_DEPLOY_SECRET}}\n", encoding="utf-8") + + _, observed, _, _, _, home = _run_deploy_build(tmp_path, worktree, compose_environment={key: "", _other(key): "x"}) + + assert GENERATED.fullmatch(observed[key]), observed[key] + assert (home / PERSISTED_FILE[key]).exists() + + +@pytest.mark.parametrize("key", SECRETS) +def test_deploy_prefers_dotenv_secret_over_the_persisted_generated_one(tmp_path, key): + """An operator-written .env value wins over the file an earlier run generated.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text(f"{key}=from-dotenv\n", encoding="utf-8") + home = tmp_path / "deer-flow-home" + home.mkdir() + (home / PERSISTED_FILE[key]).write_text("from-persisted-file\n", encoding="utf-8") + + _, observed, _, _, _, _ = _run_deploy_build(tmp_path, worktree, compose_environment={key: "from-dotenv"}) + + assert observed[key] == "", f"the persisted secret shadowed the .env value: {observed[key]!r}" + + +@pytest.mark.parametrize("key", SECRETS) +def test_deploy_still_generates_and_persists_a_secret_when_dotenv_has_none(tmp_path, key): + """Without an operator value the script keeps its generate-once contract.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text(f"{_other(key)}=x\nPORT=2026\n", encoding="utf-8") + + _, observed, _, _, _, home = _run_deploy_build(tmp_path, worktree, compose_environment={_other(key): "x", "PORT": "2026"}) + + assert GENERATED.fullmatch(observed[key]), observed[key] + generated = observed[key].removeprefix("set:") + persisted = home / PERSISTED_FILE[key] + assert persisted.read_text(encoding="utf-8").strip() == generated + assert (persisted.stat().st_mode & 0o777) == 0o600 or os.name == "nt" + + +@pytest.mark.parametrize("key", SECRETS) +def test_deploy_keeps_shell_export_ahead_of_dotenv(tmp_path, key): + """An exported shell value keeps compose precedence: it wins over .env untouched.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text(f"{key}=from-dotenv\n", encoding="utf-8") + + _, observed, _, _, _, home = _run_deploy_build(tmp_path, worktree, compose_environment={key: "from-shell"}, shell_env={key: "from-shell"}) + + assert observed[key] == "set:from-shell" + assert not (home / PERSISTED_FILE[key]).exists() + + +@pytest.mark.parametrize("key", SECRETS) +def test_deploy_treats_an_empty_shell_export_as_missing_not_as_dotenv_provided(tmp_path, key): + """Compose lets an exported-but-empty shell variable outrank .env. + + Compose renders that as ``""``; leaving it alone would hand the stack an + empty secret, so the script must still generate one (and export it). + """ + worktree = _worktree(tmp_path) + (worktree / ".env").write_text(f"{key}=from-dotenv\n", encoding="utf-8") + + _, observed, _, _, _, home = _run_deploy_build(tmp_path, worktree, compose_environment={key: ""}, shell_env={key: ""}) + + assert GENERATED.fullmatch(observed[key]), observed[key] + assert (home / PERSISTED_FILE[key]).exists() + + +def test_deploy_stops_when_compose_cannot_interpolate_instead_of_guessing(tmp_path): + """A failing probe must not silently fall through to a generated, shadowing secret.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("BETTER_AUTH_SECRET=from-dotenv\n", encoding="utf-8") + + result, observed, args, config_args, _, home = _run_deploy_build(tmp_path, worktree, compose_config_rc=15, check=False) + + assert config_args, "the script must have asked Compose" + assert result.returncode != 0 + assert "could not resolve BETTER_AUTH_SECRET" in result.stderr + assert "fake compose: cannot load project" in result.stderr + assert not args, "no build was attempted" + assert not observed + assert not (home / PERSISTED_FILE["BETTER_AUTH_SECRET"]).exists() + + +# ── Against real Compose clients, when installed ──────────────────────────── + + +def _compose_clients() -> list[tuple[str, str]]: + """The `docker` CLI plus any standalone binaries named in DEER_FLOW_TEST_COMPOSE_BINARIES.""" + clients: list[tuple[str, str]] = [] + docker = shutil.which("docker") + if docker and _renders_stub_project([docker, "compose"]): + clients.append(("docker", docker)) + for binary in filter(None, os.environ.get("DEER_FLOW_TEST_COMPOSE_BINARIES", "").split(os.pathsep)): + if _renders_stub_project([binary]): + clients.append((Path(binary).name, binary)) + return clients + + +def _renders_stub_project(command: list[str]) -> bool: + try: + probe = subprocess.run( + [*command, "-f", "-", "config"], + input="services:\n probe:\n image: scratch\n", + capture_output=True, + text=True, + timeout=60, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return probe.returncode == 0 + + +COMPOSE_CLIENTS = _compose_clients() +real_compose = pytest.mark.parametrize( + "client", + [pytest.param(path, id=name) for name, path in COMPOSE_CLIENTS] or [pytest.param(None, id="none", marks=pytest.mark.skip(reason="no docker compose client installed"))], +) + + +@pytest.fixture +def real_docker(client: str, tmp_path: Path) -> str: + """A `docker`-shaped entry point for the client: the CLI itself, or a shim that drops the `compose` word for a standalone binary.""" + if Path(client).name == "docker": + return client + shim = tmp_path / "compose-shim" / "docker" + shim.parent.mkdir() + shim.write_text(f'#!/usr/bin/env sh\n[ "$1" = compose ] && shift\nexec "{client}" "$@"\n', encoding="utf-8") + shim.chmod(0o755) + return str(shim) + + +def _resolved_by(real_docker: str, dotenv: Path, key: str) -> str: + rendered = subprocess.run( + [real_docker, "compose", "--env-file", str(dotenv), "-f", "-", "config"], + input=f"services:\n probe:\n image: scratch\n environment:\n DEER_FLOW_PROBE_VALUE: ${{{key}}}\n", + capture_output=True, + text=True, + check=True, + ).stdout + return re.search(r"^\s*DEER_FLOW_PROBE_VALUE: (.*)$", rendered, re.M).group(1) + + +@real_compose +@pytest.mark.parametrize("key", SECRETS) +@pytest.mark.parametrize( + ("dotenv_line", "expected"), + [ + ("{key}=from-dotenv", "from-dotenv"), + ("{key}: from-colon", "from-colon"), + ('{key}="from-${{OTHER}}"', "from-dotenv"), + ("{key}=${{UNSET_DEPLOY_SECRET:-defaulted}}", "defaulted"), + ], + ids=["equals", "colon", "quoted-interpolated", "default-expansion"], +) +def test_real_compose_dotenv_forms_are_left_for_compose(tmp_path, real_docker, key, dotenv_line, expected): + """Every spelling Compose accepts counts as provided, and Compose sees the operator's value.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("OTHER=dotenv\n" + dotenv_line.format(key=key) + "\n", encoding="utf-8") + + _, observed, _, _, _, home = _run_deploy_build(tmp_path, worktree, real_docker=real_docker) + + assert observed[key] == "", f"{key} exported into the compose environment: {observed[key]!r}" + assert not (home / PERSISTED_FILE[key]).exists() + assert _resolved_by(real_docker, worktree / ".env", key) == expected + + +@real_compose +@pytest.mark.parametrize("key", SECRETS) +def test_real_compose_unset_interpolation_in_dotenv_still_gets_a_generated_secret(tmp_path, real_docker, key): + """``KEY=${UNSET}`` is empty to Compose, so the script must generate.""" + worktree = _worktree(tmp_path) + (worktree / ".env").write_text(f"{key}=${{UNSET_DEPLOY_SECRET}}\n", encoding="utf-8") + + _, observed, _, _, _, home = _run_deploy_build(tmp_path, worktree, real_docker=real_docker) + + assert GENERATED.fullmatch(observed[key]), observed[key] + assert (home / PERSISTED_FILE[key]).exists() + + +@real_compose +@pytest.mark.parametrize("key", SECRETS) +def test_real_compose_empty_shell_export_still_gets_a_generated_secret(tmp_path, real_docker, key): + worktree = _worktree(tmp_path) + (worktree / ".env").write_text(f"{key}=from-dotenv\n", encoding="utf-8") + + _, observed, _, _, _, _ = _run_deploy_build(tmp_path, worktree, real_docker=real_docker, shell_env={key: ""}) + + assert GENERATED.fullmatch(observed[key]), observed[key] diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 122447bb4b..9d31ea1425 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -19,6 +19,27 @@ synchronized environment with `uv run --no-sync`. Production Compose probes Gateway `/health`, and `deploy.sh` waits for all services before reporting success; failures print Compose status and recent Gateway logs. +`deploy.sh` never sources the repo-root `.env`; Compose reads it via +`--env-file`, and shell exports outrank that file during interpolation (an +exported-but-empty variable still wins). So `BETTER_AUTH_SECRET` and +`DEER_FLOW_INTERNAL_AUTH_TOKEN` resolve shell → `.env` → persisted file under +`DEER_FLOW_HOME` → freshly generated, and a `.env`-provided value is left +unexported so Compose parses it itself. Whether `.env` provides one is +Compose's answer, not a `KEY=VALUE` grep: Compose also accepts `KEY: VALUE` +lines and interpolates `${VAR}` inside values, so the script renders a stub +project whose only environment entry is `${KEY}` through +`docker compose config` (same `--env-file`, stub on stdin, project directory +`docker/`) and reads the value back; `""` means empty or unset and falls +through to the persisted/generated secret. This works on every Compose v2 +(the README floor is 2.24; `config --environment` would need 2.28), and a +failing probe stops the script rather than guessing. `read_dotenv_value` +stays for the end-of-run summary only. Do not export a value the script read +from `.env`: that shadows Compose's own dotenv parsing and re-creates the bug +where `make up` replaced the operator's secret with a generated one. +`backend/tests/test_deploy_dotenv_secrets.py` pins the order and the probe; +its real-Compose cases run against the installed `docker` CLI and against any +standalone binaries listed in `DEER_FLOW_TEST_COMPOSE_BINARIES`. + Root `make install` runs pre-commit through uv, so uv's tool bin directory need not be on `PATH`. diff --git a/scripts/deploy.sh b/scripts/deploy.sh index dbc168b4c7..44c2ece5a2 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -44,11 +44,11 @@ cd "$REPO_ROOT" ENV_FILE="$REPO_ROOT/.env" DOCKER_DIR="$REPO_ROOT/docker" +COMPOSE_ENV_FILE_ARGS=() if [ -f "$ENV_FILE" ]; then - COMPOSE_CMD=(docker compose --env-file "$ENV_FILE" -p deer-flow -f "$DOCKER_DIR/docker-compose.yaml") -else - COMPOSE_CMD=(docker compose -p deer-flow -f "$DOCKER_DIR/docker-compose.yaml") + COMPOSE_ENV_FILE_ARGS=(--env-file "$ENV_FILE") fi +COMPOSE_CMD=(docker compose "${COMPOSE_ENV_FILE_ARGS[@]}" -p deer-flow -f "$DOCKER_DIR/docker-compose.yaml") load_uv_extras_from_dotenv() { local line="" @@ -167,12 +167,50 @@ else fi +# Compose interpolates ${BETTER_AUTH_SECRET} and ${DEER_FLOW_INTERNAL_AUTH_TOKEN} +# from the shell environment first and --env-file second. A secret the operator +# wrote to $ENV_FILE therefore only reaches the stack while this script does not +# export a competing value: generating or reloading one here would silently +# replace it. Leave a dotenv-provided secret unexported so compose parses it +# from $ENV_FILE itself, and only fall back to the persisted/generated one +# when neither the shell nor $ENV_FILE provides it. +# +# "Provided" has to mean what Compose will see, not what a KEY=VALUE grep +# finds: Compose also accepts `KEY: VALUE` lines and interpolates `${VAR}` +# inside values, so `TOKEN: abc` is set and `TOKEN=${UNSET}` is empty. Ask +# Compose itself: render a stub project whose only environment entry is +# `${KEY}`, with the same --env-file and project directory as the real +# command, and read the interpolated value back. This works on every Compose +# v2 (unlike `config --environment`, which needs 2.28), needs no daemon, and +# cannot fail on the real compose file before the secrets are decided. An +# empty or unset variable renders as `""`. +compose_interpolated_value() { + local key="$1" + local rendered="" + if ! rendered="$(printf 'services:\n probe:\n image: scratch\n environment:\n DEER_FLOW_PROBE_VALUE: ${%s}\n' "$key" \ + | docker compose "${COMPOSE_ENV_FILE_ARGS[@]}" --project-directory "$DOCKER_DIR" -f - config 2>&1)"; then + echo -e "${RED}✗ docker compose could not resolve $key from the environment and $ENV_FILE:${NC}" >&2 + printf '%s\n' "$rendered" >&2 + exit 1 + fi + printf '%s\n' "$rendered" | sed -n 's/^[[:space:]]*DEER_FLOW_PROBE_VALUE: //p' | head -n 1 +} + +dotenv_provides_secret() { + local value="" + # The probe runs in a subshell, so its exit 1 must be re-raised here. + value="$(compose_interpolated_value "$1")" || exit 1 + [ -n "$value" ] && [ "$value" != '""' ] +} + # ── BETTER_AUTH_SECRET ─────────────────────────────────────────────────────── # Required by Next.js in production. Generated once and persisted so auth # sessions survive container restarts. _secret_file="$DEER_FLOW_HOME/.better-auth-secret" -if [ -z "$BETTER_AUTH_SECRET" ]; then +if [ -z "$BETTER_AUTH_SECRET" ] && dotenv_provides_secret BETTER_AUTH_SECRET; then + echo -e "${GREEN}✓ BETTER_AUTH_SECRET loaded from $ENV_FILE${NC}" +elif [ -z "$BETTER_AUTH_SECRET" ]; then if [ -f "$_secret_file" ]; then export BETTER_AUTH_SECRET BETTER_AUTH_SECRET="$(cat "$_secret_file")" @@ -204,7 +242,9 @@ fi # APIs even when the request is handled by a different Uvicorn worker. _internal_auth_token_file="$DEER_FLOW_HOME/.internal-auth-token" -if [ "$CMD" != "down" ] && [ -z "$DEER_FLOW_INTERNAL_AUTH_TOKEN" ]; then +if [ "$CMD" != "down" ] && [ -z "$DEER_FLOW_INTERNAL_AUTH_TOKEN" ] && dotenv_provides_secret DEER_FLOW_INTERNAL_AUTH_TOKEN; then + echo -e "${GREEN}✓ DEER_FLOW_INTERNAL_AUTH_TOKEN loaded from $ENV_FILE${NC}" +elif [ "$CMD" != "down" ] && [ -z "$DEER_FLOW_INTERNAL_AUTH_TOKEN" ]; then if [ -f "$_internal_auth_token_file" ]; then export DEER_FLOW_INTERNAL_AUTH_TOKEN DEER_FLOW_INTERNAL_AUTH_TOKEN="$(cat "$_internal_auth_token_file")"