From 9eb01b371a478d9be477476eb19bc8381433b2ba Mon Sep 17 00:00:00 2001 From: creed <115765389+97Three@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:11:48 +0800 Subject: [PATCH] feat(config): select an explicit backend dotenv file (#6227) Signed-off-by: 97three <2212371308@qq.com> --- README.md | 17 ++ backend/app/gateway/auth/config.py | 8 +- backend/debug.py | 5 +- backend/docs/CONFIGURATION.md | 32 ++++ .../harness/deerflow/config/AGENTS.md | 6 +- .../harness/deerflow/config/app_config.py | 4 +- backend/packages/harness/deerflow/env.py | 34 ++++ backend/tests/test_env_file.py | 148 ++++++++++++++++++ 8 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 backend/packages/harness/deerflow/env.py create mode 100644 backend/tests/test_env_file.py diff --git a/README.md b/README.md index 01aea9f8e3..94eb8bdf2c 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,23 @@ For Google's official Gemini OpenAI-compatible endpoint, use the +For an explicit backend dotenv file, export `DEER_FLOW_ENV_FILE` before startup, +alongside `DEER_FLOW_CONFIG_PATH` if needed. For example, from `backend/`: + +```bash +DEER_FLOW_ENV_FILE=/srv/deer-flow/stage.env DEER_FLOW_CONFIG_PATH=/srv/deer-flow/stage.yaml make gateway +``` + +Relative dotenv paths use the backend process working directory; absolute paths +work regardless of that directory. Existing process variables win. An unset +selector preserves default dotenv discovery; a specified empty, missing, +non-file or unreadable path fails startup. Explicit selection also fails when +`PYTHON_DOTENV_DISABLED` disables dotenv loading. Restart after changing the file. +This selects backend dotenv input only: shell launchers, Docker Compose and the +frontend retain their own environment loading. Values they already export win. +It does not select `ENV` profiles or isolate databases, storage or tenants. +See [backend dotenv selection](backend/docs/CONFIGURATION.md#backend-dotenv-selection). + ### Running the Application #### Deployment Sizing diff --git a/backend/app/gateway/auth/config.py b/backend/app/gateway/auth/config.py index 27c1984f1a..ddd002df54 100644 --- a/backend/app/gateway/auth/config.py +++ b/backend/app/gateway/auth/config.py @@ -64,7 +64,10 @@ def get_auth_config() -> AuthConfig: if _auth_config is None: from dotenv import load_dotenv - load_dotenv() + from deerflow.env import load_selected_env_file + + if not load_selected_env_file(): + load_dotenv() jwt_secret = os.environ.get("AUTH_JWT_SECRET") if not jwt_secret: jwt_secret = _load_or_create_secret() @@ -72,7 +75,8 @@ def get_auth_config() -> AuthConfig: logger.warning( "⚠ AUTH_JWT_SECRET is not set — using an auto-generated secret " "persisted to .jwt_secret. Sessions will survive restarts. " - "For production, add AUTH_JWT_SECRET to your .env file: " + "For production, add AUTH_JWT_SECRET to your dotenv file " + "(the DEER_FLOW_ENV_FILE target, or .env by default): " 'python -c "import secrets; print(secrets.token_urlsafe(32))"' ) _auth_config = AuthConfig(jwt_secret=jwt_secret) diff --git a/backend/debug.py b/backend/debug.py index 97e12ec66d..5e1ad77c26 100644 --- a/backend/debug.py +++ b/backend/debug.py @@ -20,6 +20,8 @@ import logging from dotenv import load_dotenv +from deerflow.env import load_selected_env_file + try: from prompt_toolkit import PromptSession from prompt_toolkit.history import InMemoryHistory @@ -28,7 +30,8 @@ try: except ImportError: _HAS_PROMPT_TOOLKIT = False -load_dotenv() +if not load_selected_env_file(): + load_dotenv() _LOG_FMT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" _LOG_DATEFMT = "%Y-%m-%d %H:%M:%S" diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index c1dd4eedda..ae1fa4ad6c 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -1314,6 +1314,38 @@ models: - `DEER_FLOW_SKILLS_PATH` - Skills directory when `skills.path` is omitted - `GATEWAY_ENABLE_DOCS` - Set to `false` to disable Swagger UI (`/docs`), ReDoc (`/redoc`), and OpenAPI schema (`/openapi.json`) endpoints (default: `true`) +## Backend dotenv selection + +Set `DEER_FLOW_ENV_FILE` in the backend process environment **before startup** +to load one explicit UTF-8 dotenv file instead of default dotenv discovery. +Use an absolute path for launches from unrelated directories. Relative paths +are resolved against the backend process working directory, not the YAML file, +project root or this documentation's directory. No automatic `ENV` profile +naming or config-relative dotenv lookup is added. + +Existing process variables, including empty values, take precedence. An unset +selector keeps the existing default lookup; a set but empty selector, missing +file, directory or unreadable file raises an actionable startup error without +printing file contents. An empty **file** is valid and loads no defaults. +Explicit selection also raises when `PYTHON_DOTENV_DISABLED` is `1`, `true`, +`t`, `yes` or `y` (case-insensitive), even for an empty file. Unset either option +to resolve the conflict. Without a selector, python-dotenv's normal disable +behavior is unchanged. Restart after changing the selector or file contents. + +`DEER_FLOW_CONFIG_PATH` continues to select YAML independently. For example, +from `backend/`: + +```bash +DEER_FLOW_ENV_FILE=/srv/deer-flow/stage.env DEER_FLOW_CONFIG_PATH=/srv/deer-flow/stage.yaml make gateway +``` + +This option covers backend Python startup (including auth and `debug.py`). It +does not change shell launcher, Docker Compose or frontend dotenv handling; +values already injected by those layers remain process variables and win. +For containers, explicitly pass the selector and mount the selected file at a +container-visible path. Database, runtime-home, storage and tenant isolation +must be configured separately; selecting a dotenv file does not provide them. + ## Configuration Location The configuration file should be placed in the **project root directory** (`deer-flow/config.yaml`). Set `DEER_FLOW_PROJECT_ROOT` when the process may start from another working directory, or set `DEER_FLOW_CONFIG_PATH` to point at a specific file. diff --git a/backend/packages/harness/deerflow/config/AGENTS.md b/backend/packages/harness/deerflow/config/AGENTS.md index 16c026a140..33cf7855df 100644 --- a/backend/packages/harness/deerflow/config/AGENTS.md +++ b/backend/packages/harness/deerflow/config/AGENTS.md @@ -26,7 +26,11 @@ raise, and API create/update validation remains strict. **Main Configuration** (`config.yaml`): -Setup: Copy `config.example.yaml` to `config.yaml` in the **project root** directory. +Setup: Copy root `config.example.yaml` to `config.yaml`. Startup callers use +`deerflow.env.load_selected_env_file()` for optional `DEER_FLOW_ENV_FILE` (cwd-relative, +strict readable file, process env wins). Reject a disabling `PYTHON_DOTENV_DISABLED` +when selection is explicit; otherwise retain the original `load_dotenv()`. +Keep this helper free of config imports so auth/debug cannot preload defaults. **Config Versioning**: `config.example.yaml` has a `config_version` field. On startup, `AppConfig.from_file()` compares user version vs example version and emits a warning if outdated. Missing `config_version` = version 0. Run `make config-upgrade` to auto-merge missing fields. When changing the config schema, bump `config_version` in `config.example.yaml`. diff --git a/backend/packages/harness/deerflow/config/app_config.py b/backend/packages/harness/deerflow/config/app_config.py index e4fedf5f66..1c7d136c43 100644 --- a/backend/packages/harness/deerflow/config/app_config.py +++ b/backend/packages/harness/deerflow/config/app_config.py @@ -61,9 +61,11 @@ from deerflow.config.tool_progress_config import ToolProgressConfig from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict from deerflow.config.typesafe_config import TypeSafeConfig, load_typesafe_config_from_dict from deerflow.config.verification_config import VerificationConfig +from deerflow.env import load_selected_env_file from deerflow.extensions.loader import ExtensionSpec -load_dotenv() +if not load_selected_env_file(): + load_dotenv() logger = logging.getLogger(__name__) diff --git a/backend/packages/harness/deerflow/env.py b/backend/packages/harness/deerflow/env.py new file mode 100644 index 0000000000..5cd54739a2 --- /dev/null +++ b/backend/packages/harness/deerflow/env.py @@ -0,0 +1,34 @@ +"""Explicit backend dotenv selection, independent of configuration imports.""" + +import os +from io import StringIO +from pathlib import Path + +from dotenv import load_dotenv + + +def load_selected_env_file() -> bool: + """Load DEER_FLOW_ENV_FILE; return False only when the option is unset. + + Relative paths use the process working directory. Callers retain their + original load_dotenv() call for default discovery when this returns False. + Existing process variables (including empty values) always take precedence. + """ + selected = os.environ.get("DEER_FLOW_ENV_FILE") + if selected is None: + return False + # Match python-dotenv's switch without silently ignoring explicit selection. + if os.environ.get("PYTHON_DOTENV_DISABLED", "").casefold() in {"1", "true", "t", "yes", "y"}: + raise ValueError("DEER_FLOW_ENV_FILE cannot be used while PYTHON_DOTENV_DISABLED disables dotenv loading. Unset PYTHON_DOTENV_DISABLED or DEER_FLOW_ENV_FILE.") + + try: + path = Path(selected) + if not selected or not path.is_file(): + raise ValueError("Expected a regular file") + # Read fully before parsing so decode failures cannot partially load env. + contents = path.read_text(encoding="utf-8") + except (OSError, ValueError): + raise ValueError("DEER_FLOW_ENV_FILE must point to a readable UTF-8 regular file. Check the path and permissions; relative paths use the backend process working directory.") from None + + load_dotenv(stream=StringIO(contents), override=False) + return True diff --git a/backend/tests/test_env_file.py b/backend/tests/test_env_file.py new file mode 100644 index 0000000000..dfb333c5d9 --- /dev/null +++ b/backend/tests/test_env_file.py @@ -0,0 +1,148 @@ +"""Exercise startup in fresh processes without reading checkout dotenv files.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +BACKEND = Path(__file__).resolve().parents[1] +ENTRYPOINTS = { + "config": "import deerflow.config.app_config", + "auth": "from app.gateway.auth.config import get_auth_config; get_auth_config(); import deerflow.config.app_config", + "debug": f"import runpy; runpy.run_path({str(BACKEND / 'debug.py')!r}); import deerflow.config.app_config", +} + + +def run_startup(tmp_path, entrypoint="config", selector=None, extra_env=None, prelude=""): + # Only propagate interpreter necessities, never ambient credentials/config. + env = {key: os.environ[key] for key in ("PATH", "SYSTEMROOT") if key in os.environ} + env.update(PYTHONPATH=os.pathsep.join((str(BACKEND), str(BACKEND / "packages/harness"))), AUTH_JWT_SECRET="test-only") + if selector is not None: + env["DEER_FLOW_ENV_FILE"] = str(selector) + env.update(extra_env or {}) + default = tmp_path / ".env" + default.write_text("ENV_FILE_TEST_VALUE=default\nENV_FILE_TEST_DEFAULT_ONLY=unselected\n", encoding="utf-8") + cwd = tmp_path / "unrelated" + cwd.mkdir(exist_ok=True) + # Control discovery only; parsing, precedence and startup imports are real. + # This prevents default discovery from reading any developer secrets. + script = f""" +import json, os +import dotenv.main +dotenv.main.find_dotenv = lambda *args, **kwargs: {str(default)!r} +{prelude} +{ENTRYPOINTS[entrypoint]} +print(json.dumps([os.getenv('ENV_FILE_TEST_VALUE'), os.getenv('ENV_FILE_TEST_DEFAULT_ONLY')])) +""" + return subprocess.run([sys.executable, "-c", script], cwd=cwd, env=env, capture_output=True, text=True, timeout=30) + + +@pytest.mark.parametrize("entrypoint", ENTRYPOINTS) +@pytest.mark.parametrize("relative", [False, True]) +def test_explicit_selection_from_unrelated_cwd(tmp_path, entrypoint, relative): + selected = tmp_path / "stage.env" + selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8") + result = run_startup(tmp_path, entrypoint, "../stage.env" if relative else selected) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == ["selected", None] + + +@pytest.mark.parametrize("value", ["exported", ""]) +def test_process_environment_wins(tmp_path, value): + selected = tmp_path / "stage.env" + selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8") + result = run_startup(tmp_path, selector=selected, extra_env={"ENV_FILE_TEST_VALUE": value}) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == [value, None] + + +@pytest.mark.parametrize("entrypoint", ENTRYPOINTS) +def test_unset_preserves_default_loading(tmp_path, entrypoint): + result = run_startup(tmp_path, entrypoint) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == ["default", "unselected"] + + +@pytest.mark.parametrize("entrypoint", ENTRYPOINTS) +@pytest.mark.parametrize("kind", ["missing", "directory", "empty", "unreadable", "invalid_encoding"]) +def test_invalid_explicit_file_fails_without_fallback(tmp_path, entrypoint, kind): + selected = tmp_path / "invalid.env" + prelude = "" + if kind == "directory": + selected.mkdir() + elif kind == "unreadable": + selected.write_text("ENV_FILE_TEST_VALUE=do-not-print-this\n", encoding="utf-8") + # Deterministic even under root and on Windows: emulate OS denial on open. + prelude = f""" +import sys +def deny_open(event, args): + if event == 'open' and str(args[0]) == {str(selected)!r}: + raise PermissionError('do-not-print-this') +sys.addaudithook(deny_open) +""" + elif kind == "invalid_encoding": + selected.write_bytes(b"ENV_FILE_TEST_VALUE=do-not-print-this\xff") + result = run_startup(tmp_path, entrypoint, "" if kind == "empty" else selected, prelude=prelude) + assert result.returncode != 0 + assert "DEER_FLOW_ENV_FILE" in result.stderr + assert "readable UTF-8 regular file" in result.stderr + assert "do-not-print-this" not in result.stderr + assert "unselected" not in result.stdout + + +def test_empty_selected_file_does_not_load_defaults(tmp_path): + selected = tmp_path / "empty.env" + selected.touch() + result = run_startup(tmp_path, selector=selected) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == [None, None] + + +@pytest.mark.parametrize("entrypoint", ENTRYPOINTS) +@pytest.mark.parametrize("disabled", ["1", "TRUE", "t", "YeS", "Y"]) +def test_explicit_selection_rejects_disabled_dotenv(tmp_path, entrypoint, disabled): + selected = tmp_path / "stage.env" + selected.write_text("ENV_FILE_TEST_VALUE=do-not-print-this\n", encoding="utf-8") + result = run_startup(tmp_path, entrypoint, selected, {"PYTHON_DOTENV_DISABLED": disabled}) + assert result.returncode != 0 + assert "DEER_FLOW_ENV_FILE" in result.stderr + assert "PYTHON_DOTENV_DISABLED" in result.stderr + assert "do-not-print-this" not in result.stderr + assert "unselected" not in result.stdout + + +@pytest.mark.parametrize("disabled", ["0", "false", "no", "", " true "]) +def test_non_disabling_values_allow_explicit_selection(tmp_path, disabled): + selected = tmp_path / "stage.env" + selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8") + result = run_startup(tmp_path, selector=selected, extra_env={"PYTHON_DOTENV_DISABLED": disabled}) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == ["selected", None] + + +@pytest.mark.parametrize("entrypoint", ENTRYPOINTS) +def test_disabled_dotenv_without_selector_keeps_default_behavior(tmp_path, entrypoint): + result = run_startup(tmp_path, entrypoint, extra_env={"PYTHON_DOTENV_DISABLED": "1"}) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == [None, None] + + +def test_empty_selected_file_still_rejects_disabled_dotenv(tmp_path): + selected = tmp_path / "empty.env" + selected.touch() + result = run_startup(tmp_path, selector=selected, extra_env={"PYTHON_DOTENV_DISABLED": "true"}) + assert result.returncode != 0 + assert "PYTHON_DOTENV_DISABLED" in result.stderr + + +def test_auth_secret_warning_mentions_selected_env_file(tmp_path): + selected = tmp_path / "stage.env" + selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8") + prelude = "import app.gateway.auth.config as auth_config; auth_config._load_or_create_secret = lambda: 'test-generated-secret'" + result = run_startup(tmp_path, "auth", selected, {"AUTH_JWT_SECRET": ""}, prelude) + assert result.returncode == 0, result.stderr + assert "DEER_FLOW_ENV_FILE" in result.stderr + assert ".env by default" in result.stderr