From 4f948c7b8d02840fc9fbe12f21c52ffbc901212e Mon Sep 17 00:00:00 2001 From: ningkexin96 Date: Wed, 21 Oct 2026 15:58:30 +0800 Subject: [PATCH] feat(deploy): select production Compose env files (#6522) * feat(deploy): select production Compose env files Use an operator-selected dotenv file consistently for interpolation, container environment, secret probes and build extras. Preserve default selection and shutdown after a profile is removed. Co-Authored-By: Codex * fix(deploy): address production env selection review Clarify Docker launcher entry points and render real Compose through the launcher to verify both service environments. Co-Authored-By: Codex * test(deploy): cover unreadable env selectors Reject unreadable explicit selectors before Docker is invoked, using POSIX permission checks and restoring the fixture mode in finally. Co-Authored-By: Codex * test(deploy): scope dotenv assertions to selected secrets --------- Co-authored-by: Codex --- README.md | 6 + backend/docs/CONFIGURATION.md | 41 ++++ .../tests/test_compose_default_bind_host.py | 2 + backend/tests/test_deploy_env_file.py | 221 ++++++++++++++++++ docker/docker-compose.yaml | 6 +- scripts/AGENTS.md | 11 + scripts/deploy.sh | 16 +- 7 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_deploy_env_file.py diff --git a/README.md b/README.md index 9a50312e3d..e29450b020 100644 --- a/README.md +++ b/README.md @@ -571,6 +571,12 @@ recovery command and retry. The check honors secret overrides from the shell or `.env`, accepts readable read-only secret files, and leaves `make down` available without reading or generating secrets. +Production Docker commands can select a root dotenv profile with +`DEER_FLOW_COMPOSE_ENV_FILE=/srv/deer-flow/stage.env make up`. The same file +feeds Compose interpolation, Gateway/provisioner variables, and deployment +secret checks. YAML and runtime-state locations remain separate selectors; +see [production env files](backend/docs/CONFIGURATION.md#production-compose-env-files). + For persistent deployments, configure `database.backend` as `sqlite` or `postgres`. The selected backend is shared by the LangGraph checkpointer, LangGraph Store, and DeerFlow application data. The deprecated `checkpointer` diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 40804e4abd..de249c7fcb 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -1925,6 +1925,47 @@ For containers, explicitly pass the selector and mount the selected file at a container-visible path. Database, runtime-home, storage and tenant isolation must be configured separately; selecting a dotenv file does not provide them. + +## Production Compose env files + +Set `DEER_FLOW_COMPOSE_ENV_FILE` in the launcher environment to select a +production dotenv file for `make up`, `make down`, and direct +`./scripts/deploy.sh build|start|down` calls: + +```bash +DEER_FLOW_COMPOSE_ENV_FILE=/srv/deer-flow/stage.env make up +``` + +Relative paths are resolved from the checkout root. The production launcher +passes the same file to Compose's `--env-file`, the Gateway and provisioner +`env_file` entries, the secret-resolution probe, and optional build-extra +discovery. It never sources the file as shell code. An explicit missing, +directory, or unreadable selector fails before Docker is invoked for startup +or build. `make down` can still stop the stack if the selected file was removed. +An unset or empty selector preserves the optional repo-root `.env` default. + +For direct Compose calls, use an absolute path and provide both selectors: +`--env-file` controls interpolation; service `env_file` controls container +variables. + +```bash +DEER_FLOW_COMPOSE_ENV_FILE=/srv/deer-flow/stage.env \ + docker compose --env-file /srv/deer-flow/stage.env \ + -f docker/docker-compose.yaml config +``` + +Existing shell variables retain Compose's precedence. Export +`DEER_FLOW_CONFIG_PATH`, `DEER_FLOW_EXTENSIONS_CONFIG_PATH`, and +`DEER_FLOW_HOME` independently when choosing different configuration and state +locations. This is a production Docker file-selection option. `make start` uses +the local +`serve.sh --prod` launcher, and `make docker-*` uses Docker development; those +commands keep their repo-root `.env` handling. +`frontend/.env`, project/container names, database separation, and tenant +isolation are not changed. Stop an existing stack before switching its profile. +Keep credential-bearing profiles outside the checkout or in local Git exclusions. +`DEER_FLOW_ENV_FILE` remains the separate backend Python dotenv selector. + ## Configuration Location The configuration file should be placed in the **project root directory** (`deer-flow/config.yaml`). Set `DEER_FLOW_PROJECT_ROOT` when the process may start from another working directory, or set `DEER_FLOW_CONFIG_PATH` to point at a specific file. diff --git a/backend/tests/test_compose_default_bind_host.py b/backend/tests/test_compose_default_bind_host.py index 94a0ab360f..6ef5d8d463 100644 --- a/backend/tests/test_compose_default_bind_host.py +++ b/backend/tests/test_compose_default_bind_host.py @@ -142,6 +142,8 @@ def test_compose_env_files_are_optional(variant: str): """ compose = yaml.safe_load(COMPOSE_PATHS[variant].read_text(encoding="utf-8")) for service_name, path in EXPECTED_ENV_FILES.items(): + if variant == "prod" and service_name in {"gateway", "provisioner"}: + path = "${DEER_FLOW_COMPOSE_ENV_FILE:-../.env}" entries = compose["services"][service_name]["env_file"] assert entries == [{"path": path, "required": False}], f"{variant} {service_name} env_file must be optional; got: {entries!r}" diff --git a/backend/tests/test_deploy_env_file.py b/backend/tests/test_deploy_env_file.py new file mode 100644 index 0000000000..6fd3600e6b --- /dev/null +++ b/backend/tests/test_deploy_env_file.py @@ -0,0 +1,221 @@ +"""Production Compose dotenv selection is explicit and shared by all consumers.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess + +import pytest +import test_deploy_dotenv_secrets as deploy_secret_tests +from test_deploy_dotenv_secrets import BASH, _run_deploy_build, _worktree +from test_deploy_home_writability import _run_deploy +from test_deploy_home_writability import deploy_fixture as deploy_fixture + +pytestmark = pytest.mark.skipif(BASH is None, reason="repo shell-script tests need Git Bash on Windows") + + +@pytest.mark.parametrize("selector_kind", ["relative", "absolute", "spaces"]) +def test_deploy_selects_one_env_file_for_build_and_secret_probe(tmp_path, selector_kind): + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("UV_EXTRAS=redis\n", encoding="utf-8") + selected = worktree / ("profiles/stage.env" if selector_kind != "spaces" else "profiles with spaces/stage env") + selected.parent.mkdir() + selected.write_text("UV_EXTRAS=discord\n", encoding="utf-8") + selector = str(selected) if selector_kind == "absolute" else str(selected.relative_to(worktree)) + + _, _, args, config_args, _, _ = _run_deploy_build( + tmp_path, + worktree, + shell_env={"DEER_FLOW_COMPOSE_ENV_FILE": selector}, + compose_environment={"BETTER_AUTH_SECRET": "test-auth", "DEER_FLOW_INTERNAL_AUTH_TOKEN": "test-internal"}, + ) + + assert "--env-file" in args + env_arg = args[args.index("--env-file") + 1] + assert config_args[config_args.index("--env-file") + 1] == env_arg + probe = subprocess.run( + [BASH, "-c", 'test -f "$1" && cmp -s "$1" "$2"', "--", env_arg, str(selected)], + capture_output=True, + check=False, + timeout=10, + ) + assert probe.returncode == 0, "Compose and its secret probe must read the selected file" + + +@pytest.mark.parametrize("invalid_kind", ["missing", "directory", "unreadable"]) +def test_deploy_rejects_invalid_explicit_env_file_before_docker(tmp_path, invalid_kind): + worktree = _worktree(tmp_path) + selected = worktree / "stage.env" + if invalid_kind == "directory": + selected.mkdir() + elif invalid_kind == "unreadable": + if os.name != "posix": + pytest.skip("file read permissions require a POSIX filesystem") + if os.geteuid() == 0: + pytest.skip("root can read files despite removed read permissions") + selected.write_text("UV_EXTRAS=discord\n", encoding="utf-8") + + original_mode = selected.stat().st_mode if invalid_kind == "unreadable" else None + try: + if original_mode is not None: + selected.chmod(0o000) + if os.access(selected, os.R_OK): + pytest.skip("filesystem does not enforce file read permissions") + + result, _, args, config_args, _, _ = _run_deploy_build( + tmp_path, + worktree, + shell_env={"DEER_FLOW_COMPOSE_ENV_FILE": str(selected)}, + compose_environment={"BETTER_AUTH_SECRET": "test-auth", "DEER_FLOW_INTERNAL_AUTH_TOKEN": "test-internal"}, + check=False, + ) + finally: + if original_mode is not None: + selected.chmod(original_mode) + + assert result.returncode != 0 + assert "DEER_FLOW_COMPOSE_ENV_FILE" in result.stderr + assert not args + assert not config_args + + +def test_deploy_reads_uv_extras_from_selected_env_file(tmp_path): + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("UV_EXTRAS=redis\n", encoding="utf-8") + selected = worktree / "stage.env" + selected.write_text("UV_EXTRAS=discord\n", encoding="utf-8") + capture = tmp_path / "selected-extras.txt" + docker = tmp_path / "bin" / "docker" + docker.parent.mkdir() + docker.write_text( + '#!/usr/bin/env sh\nprintf "%s" "${UV_EXTRAS:-}" > "$CAPTURE_EXTRAS"\nexit 0\n', + encoding="utf-8", + ) + docker.chmod(0o755) + env = os.environ.copy() + env.pop("UV_EXTRAS", None) + env.update( + PATH=f"{docker.parent}{os.pathsep}{env['PATH']}", + DEER_FLOW_COMPOSE_ENV_FILE=str(selected), + DEER_FLOW_HOME=str(tmp_path / "home"), + BETTER_AUTH_SECRET="test-auth", + DEER_FLOW_INTERNAL_AUTH_TOKEN="test-internal", + CAPTURE_EXTRAS=str(capture), + ) + + result = subprocess.run( + [BASH, str(worktree / "scripts/deploy.sh"), "build"], + cwd=worktree, + env=env, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert capture.read_text(encoding="utf-8") == "discord" + + +@pytest.mark.parametrize("use_selector", [False, True, ""], ids=["default", "selected", "empty"]) +def test_compose_uses_selected_env_for_gateway_and_provisioner(tmp_path, use_selector): + docker = shutil.which("docker") + if not docker: + pytest.skip("real Docker Compose CLI required to render the production project") + version = subprocess.run([docker, "compose", "version"], capture_output=True, check=False, timeout=15) + if version.returncode: + pytest.skip("Docker Compose plugin unavailable") + + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("DEERFLOW_ENV_SELECTION_TEST=default\n", encoding="utf-8") + selected = worktree / "stage.env" + selected.write_text("DEERFLOW_ENV_SELECTION_TEST=stage\n", encoding="utf-8") + env = os.environ.copy() + env.pop("DEER_FLOW_COMPOSE_ENV_FILE", None) + if use_selector == "": + env["DEER_FLOW_COMPOSE_ENV_FILE"] = "" + elif use_selector: + env["DEER_FLOW_COMPOSE_ENV_FILE"] = str(selected) + env.update( + DEER_FLOW_HOME=str(tmp_path / "home"), + DEER_FLOW_CONFIG_PATH=str(worktree / "config.yaml"), + DEER_FLOW_EXTENSIONS_CONFIG_PATH=str(worktree / "extensions_config.json"), + BETTER_AUTH_SECRET="test-auth", + DEER_FLOW_INTERNAL_AUTH_TOKEN="test-internal", + ) + result = subprocess.run( + [docker, "compose", "--env-file", str(selected if use_selector else worktree / ".env"), "-f", str(worktree / "docker/docker-compose.yaml"), "config", "--format", "json"], + cwd=worktree, + env=env, + text=True, + capture_output=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stderr + config = json.loads(result.stdout) + for service in ("gateway", "provisioner"): + assert config["services"][service]["environment"]["DEERFLOW_ENV_SELECTION_TEST"] == ("stage" if use_selector else "default") + + +def test_deploy_down_still_works_when_selected_env_file_was_removed(deploy_fixture): + worktree, env, home, capture = deploy_fixture + env["DEER_FLOW_COMPOSE_ENV_FILE"] = str(worktree / "profiles/missing.env") + env.pop("BETTER_AUTH_SECRET") + env.pop("DEER_FLOW_INTERNAL_AUTH_TOKEN") + + result = _run_deploy(deploy_fixture, "down") + + assert result.returncode == 0, result.stderr + assert capture.read_text(encoding="utf-8").splitlines()[-1] == "down" + assert not list(home.iterdir()) + + +def test_deploy_env_file_reaches_containers_through_real_compose(tmp_path, monkeypatch): + docker = shutil.which("docker") + if not docker: + pytest.skip("real Docker Compose CLI required for the native launcher probe") + version = subprocess.run([docker, "compose", "version"], capture_output=True, check=False, timeout=15) + if version.returncode: + pytest.skip("Docker Compose plugin unavailable") + + worktree = _worktree(tmp_path) + (worktree / ".env").write_text("NOT_A_SECRET=default\n", encoding="utf-8") + selected = worktree / "profiles with spaces" / "stage env" + selected.parent.mkdir() + selected.write_text("BETTER_AUTH_SECRET=test-auth\nDEER_FLOW_INTERNAL_AUTH_TOKEN=test-internal\nDEERFLOW_ENV_SELECTION_TEST=stage\n", encoding="utf-8") + + rendered = tmp_path / "rendered-project.json" + stub = deploy_secret_tests._FAKE_DOCKER.replace("#!/usr/bin/env sh", "#!/usr/bin/env bash", 1) + prefix, suffix = stub.rsplit("exit 0\n", 1) + stub = ( + prefix + + """args=() +for arg in "$@"; do + [ "$arg" != "build" ] || break + args+=("$arg") +done +"$REAL_DOCKER" "${args[@]}" config --format json > "$CAPTURE_RENDERED_PROJECT" +exit 0 +""" + + suffix + ) + monkeypatch.setattr(deploy_secret_tests, "_FAKE_DOCKER", stub) + + _, observed, _, _, _, home = _run_deploy_build( + tmp_path, + worktree, + shell_env={"DEER_FLOW_COMPOSE_ENV_FILE": str(selected), "CAPTURE_RENDERED_PROJECT": str(rendered)}, + real_docker=docker, + ) + project = json.loads(rendered.read_text(encoding="utf-8")) + for service in ("gateway", "provisioner"): + assert project["services"][service]["environment"]["DEERFLOW_ENV_SELECTION_TEST"] == "stage" + + assert observed["BETTER_AUTH_SECRET"] == "" + assert observed["DEER_FLOW_INTERNAL_AUTH_TOKEN"] == "" + assert project["services"]["frontend"]["environment"]["BETTER_AUTH_SECRET"] == "test-auth" + assert project["services"]["gateway"]["environment"]["DEER_FLOW_INTERNAL_AUTH_TOKEN"] == "test-internal" + for key in ("BETTER_AUTH_SECRET", "DEER_FLOW_INTERNAL_AUTH_TOKEN"): + assert not (home / deploy_secret_tests.PERSISTED_FILE[key]).exists(), f"Selected dotenv {key} must not be replaced with a generated secret" diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index b5ee1a56a0..a474bed90d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -188,12 +188,12 @@ services: # request arrives through it, so direct mode would only bring back the # shared lockout. Set another list in .env to change whom to trust. - AUTH_TRUSTED_PROXIES=${AUTH_TRUSTED_PROXIES:-nginx} - # Proxy values (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY) are inherited from ../.env via env_file. + # Proxy values (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY) are inherited from the selected root dotenv file. # Only NO_PROXY is declared here so internal service hostnames are always exempt from the proxy. - NO_PROXY=${NO_PROXY:-}${NO_PROXY:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,openviking,host.docker.internal - no_proxy=${no_proxy:-}${no_proxy:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,openviking,host.docker.internal env_file: - - path: ../.env + - path: ${DEER_FLOW_COMPOSE_ENV_FILE:-../.env} required: false extra_hosts: - "host.docker.internal:host-gateway" @@ -243,7 +243,7 @@ services: - NODE_HOST=host.docker.internal - K8S_API_SERVER=https://host.docker.internal:26443 env_file: - - path: ../.env + - path: ${DEER_FLOW_COMPOSE_ENV_FILE:-../.env} required: false extra_hosts: - "host.docker.internal:host-gateway" diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index f182cd42f8..6ee1f5639f 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -469,3 +469,14 @@ Keep exit codes, timeouts and redaction intact; do not rely on the host locale. Regressions use real local children, including ASCII/GBK capture defaults, nonzero exits, surrogate characters and malformed output, without invoking provider diagnostics. Doctor covers both `_run` streams and pnpm runner capture. + +## Production dotenv selection + +`deploy.sh` resolves `DEER_FLOW_COMPOSE_ENV_FILE` from the checkout before +creating state or probing secrets. Keep Compose interpolation, Gateway and +provisioner env files, secret probes, and UV_EXTRAS discovery on that one path. +Normalize host paths for Git Bash; quote path arguments rather than sourcing +dotenv files. Unset/empty selection preserves the optional root .env default; +explicit invalid files reject build/start before Docker. Stopping remains +possible after the selected file is removed. Tests: `test_deploy_env_file.py` +and the existing deployment/Compose suites. diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 322a815046..f5f7072947 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -42,7 +42,19 @@ esac REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" -ENV_FILE="$REPO_ROOT/.env" +ENV_FILE="${DEER_FLOW_COMPOSE_ENV_FILE:-$REPO_ROOT/.env}" +# Docker Compose's interpolation file and service env_file must select the same +# host file. Relative selectors are rooted at this checkout, not docker/. +if command -v cygpath >/dev/null 2>&1; then + ENV_FILE="$(cygpath -au -- "$ENV_FILE")" +elif [[ "$ENV_FILE" != /* ]]; then + ENV_FILE="$REPO_ROOT/$ENV_FILE" +fi +if [ -n "${DEER_FLOW_COMPOSE_ENV_FILE:-}" ] && [ "$CMD" != "down" ] && { [ ! -f "$ENV_FILE" ] || [ ! -r "$ENV_FILE" ]; }; then + echo "DEER_FLOW_COMPOSE_ENV_FILE must name a readable file: $ENV_FILE" >&2 + exit 1 +fi +export DEER_FLOW_COMPOSE_ENV_FILE="$ENV_FILE" DOCKER_DIR="$REPO_ROOT/docker" COMPOSE_ENV_FILE_ARGS=() if [ -f "$ENV_FILE" ]; then @@ -531,7 +543,7 @@ echo " API: /api/langgraph/* → Gateway" echo "" if [ "$RESOLVED_BIND_HOST" = "127.0.0.1" ] || [ "$RESOLVED_BIND_HOST" = "::1" ] || [ "$RESOLVED_BIND_HOST" = "localhost" ]; then echo " 🔒 Bound to ${RESOLVED_BIND_HOST} — reachable from this machine only." - echo " To expose it, set BIND_HOST in .env, put TLS/auth in front, and" + echo " To expose it, set BIND_HOST in the selected dotenv file, put TLS/auth in front, and" echo " create the admin account before the host becomes reachable." else echo " ⚠️ Bound to ${RESOLVED_BIND_HOST} — reachable from the network."