feat(config): select an explicit backend dotenv file (#6227)

Signed-off-by: 97three <2212371308@qq.com>

README.md

@@ -344,6 +344,23 @@ For Google's official Gemini OpenAI-compatible endpoint, use the
 
    </details>
 
+For an explicit backend dotenv file, export `DEER_FLOW_ENV_FILE` before startup,
+alongside `DEER_FLOW_CONFIG_PATH` if needed. For example, from `backend/`:
+
+```bash
+DEER_FLOW_ENV_FILE=/srv/deer-flow/stage.env DEER_FLOW_CONFIG_PATH=/srv/deer-flow/stage.yaml make gateway
+```
+
+Relative dotenv paths use the backend process working directory; absolute paths
+work regardless of that directory. Existing process variables win. An unset
+selector preserves default dotenv discovery; a specified empty, missing,
+non-file or unreadable path fails startup. Explicit selection also fails when
+`PYTHON_DOTENV_DISABLED` disables dotenv loading. Restart after changing the file.
+This selects backend dotenv input only: shell launchers, Docker Compose and the
+frontend retain their own environment loading. Values they already export win.
+It does not select `ENV` profiles or isolate databases, storage or tenants.
+See [backend dotenv selection](backend/docs/CONFIGURATION.md#backend-dotenv-selection).
+
 ### Running the Application
 
 #### Deployment Sizing

backend/app/gateway/auth/config.py

@@ -64,7 +64,10 @@ def get_auth_config() -> AuthConfig:
     if _auth_config is None:
         from dotenv import load_dotenv
 
-        load_dotenv()
+        from deerflow.env import load_selected_env_file
+
+        if not load_selected_env_file():
+            load_dotenv()
         jwt_secret = os.environ.get("AUTH_JWT_SECRET")
         if not jwt_secret:
             jwt_secret = _load_or_create_secret()
@@ -72,7 +75,8 @@ def get_auth_config() -> AuthConfig:
             logger.warning(
                 "⚠ AUTH_JWT_SECRET is not set — using an auto-generated secret "
                 "persisted to .jwt_secret. Sessions will survive restarts. "
-                "For production, add AUTH_JWT_SECRET to your .env file: "
+                "For production, add AUTH_JWT_SECRET to your dotenv file "
+                "(the DEER_FLOW_ENV_FILE target, or .env by default): "
                 'python -c "import secrets; print(secrets.token_urlsafe(32))"'
             )
         _auth_config = AuthConfig(jwt_secret=jwt_secret)

backend/debug.py

@@ -20,6 +20,8 @@ import logging
 
 from dotenv import load_dotenv
 
+from deerflow.env import load_selected_env_file
+
 try:
     from prompt_toolkit import PromptSession
     from prompt_toolkit.history import InMemoryHistory
@@ -28,7 +30,8 @@ try:
 except ImportError:
     _HAS_PROMPT_TOOLKIT = False
 
-load_dotenv()
+if not load_selected_env_file():
+    load_dotenv()
 
 _LOG_FMT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 _LOG_DATEFMT = "%Y-%m-%d %H:%M:%S"

backend/docs/CONFIGURATION.md

@@ -1314,6 +1314,38 @@ models:
 - `DEER_FLOW_SKILLS_PATH` - Skills directory when `skills.path` is omitted
 - `GATEWAY_ENABLE_DOCS` - Set to `false` to disable Swagger UI (`/docs`), ReDoc (`/redoc`), and OpenAPI schema (`/openapi.json`) endpoints (default: `true`)
 
+## Backend dotenv selection
+
+Set `DEER_FLOW_ENV_FILE` in the backend process environment **before startup**
+to load one explicit UTF-8 dotenv file instead of default dotenv discovery.
+Use an absolute path for launches from unrelated directories. Relative paths
+are resolved against the backend process working directory, not the YAML file,
+project root or this documentation's directory. No automatic `ENV` profile
+naming or config-relative dotenv lookup is added.
+
+Existing process variables, including empty values, take precedence. An unset
+selector keeps the existing default lookup; a set but empty selector, missing
+file, directory or unreadable file raises an actionable startup error without
+printing file contents. An empty **file** is valid and loads no defaults.
+Explicit selection also raises when `PYTHON_DOTENV_DISABLED` is `1`, `true`,
+`t`, `yes` or `y` (case-insensitive), even for an empty file. Unset either option
+to resolve the conflict. Without a selector, python-dotenv's normal disable
+behavior is unchanged. Restart after changing the selector or file contents.
+
+`DEER_FLOW_CONFIG_PATH` continues to select YAML independently. For example,
+from `backend/`:
+
+```bash
+DEER_FLOW_ENV_FILE=/srv/deer-flow/stage.env DEER_FLOW_CONFIG_PATH=/srv/deer-flow/stage.yaml make gateway
+```
+
+This option covers backend Python startup (including auth and `debug.py`). It
+does not change shell launcher, Docker Compose or frontend dotenv handling;
+values already injected by those layers remain process variables and win.
+For containers, explicitly pass the selector and mount the selected file at a
+container-visible path. Database, runtime-home, storage and tenant isolation
+must be configured separately; selecting a dotenv file does not provide them.
+
 ## Configuration Location
 
 The configuration file should be placed in the **project root directory** (`deer-flow/config.yaml`). Set `DEER_FLOW_PROJECT_ROOT` when the process may start from another working directory, or set `DEER_FLOW_CONFIG_PATH` to point at a specific file.

backend/packages/harness/deerflow/config/AGENTS.md

@@ -26,7 +26,11 @@ raise, and API create/update validation remains strict.
 
 **Main Configuration** (`config.yaml`):
 
-Setup: Copy `config.example.yaml` to `config.yaml` in the **project root** directory.
+Setup: Copy root `config.example.yaml` to `config.yaml`. Startup callers use
+`deerflow.env.load_selected_env_file()` for optional `DEER_FLOW_ENV_FILE` (cwd-relative,
+strict readable file, process env wins). Reject a disabling `PYTHON_DOTENV_DISABLED`
+when selection is explicit; otherwise retain the original `load_dotenv()`.
+Keep this helper free of config imports so auth/debug cannot preload defaults.
 
 **Config Versioning**: `config.example.yaml` has a `config_version` field. On startup, `AppConfig.from_file()` compares user version vs example version and emits a warning if outdated. Missing `config_version` = version 0. Run `make config-upgrade` to auto-merge missing fields. When changing the config schema, bump `config_version` in `config.example.yaml`.
 

backend/packages/harness/deerflow/config/app_config.py

@@ -61,9 +61,11 @@ from deerflow.config.tool_progress_config import ToolProgressConfig
 from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
 from deerflow.config.typesafe_config import TypeSafeConfig, load_typesafe_config_from_dict
 from deerflow.config.verification_config import VerificationConfig
+from deerflow.env import load_selected_env_file
 from deerflow.extensions.loader import ExtensionSpec
 
-load_dotenv()
+if not load_selected_env_file():
+    load_dotenv()
 
 logger = logging.getLogger(__name__)
 

backend/packages/harness/deerflow/env.py

@@ -0,0 +1,34 @@
+"""Explicit backend dotenv selection, independent of configuration imports."""
+
+import os
+from io import StringIO
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+
+def load_selected_env_file() -> bool:
+    """Load DEER_FLOW_ENV_FILE; return False only when the option is unset.
+
+    Relative paths use the process working directory. Callers retain their
+    original load_dotenv() call for default discovery when this returns False.
+    Existing process variables (including empty values) always take precedence.
+    """
+    selected = os.environ.get("DEER_FLOW_ENV_FILE")
+    if selected is None:
+        return False
+    # Match python-dotenv's switch without silently ignoring explicit selection.
+    if os.environ.get("PYTHON_DOTENV_DISABLED", "").casefold() in {"1", "true", "t", "yes", "y"}:
+        raise ValueError("DEER_FLOW_ENV_FILE cannot be used while PYTHON_DOTENV_DISABLED disables dotenv loading. Unset PYTHON_DOTENV_DISABLED or DEER_FLOW_ENV_FILE.")
+
+    try:
+        path = Path(selected)
+        if not selected or not path.is_file():
+            raise ValueError("Expected a regular file")
+        # Read fully before parsing so decode failures cannot partially load env.
+        contents = path.read_text(encoding="utf-8")
+    except (OSError, ValueError):
+        raise ValueError("DEER_FLOW_ENV_FILE must point to a readable UTF-8 regular file. Check the path and permissions; relative paths use the backend process working directory.") from None
+
+    load_dotenv(stream=StringIO(contents), override=False)
+    return True

backend/tests/test_env_file.py

@@ -0,0 +1,148 @@
+"""Exercise startup in fresh processes without reading checkout dotenv files."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+BACKEND = Path(__file__).resolve().parents[1]
+ENTRYPOINTS = {
+    "config": "import deerflow.config.app_config",
+    "auth": "from app.gateway.auth.config import get_auth_config; get_auth_config(); import deerflow.config.app_config",
+    "debug": f"import runpy; runpy.run_path({str(BACKEND / 'debug.py')!r}); import deerflow.config.app_config",
+}
+
+
+def run_startup(tmp_path, entrypoint="config", selector=None, extra_env=None, prelude=""):
+    # Only propagate interpreter necessities, never ambient credentials/config.
+    env = {key: os.environ[key] for key in ("PATH", "SYSTEMROOT") if key in os.environ}
+    env.update(PYTHONPATH=os.pathsep.join((str(BACKEND), str(BACKEND / "packages/harness"))), AUTH_JWT_SECRET="test-only")
+    if selector is not None:
+        env["DEER_FLOW_ENV_FILE"] = str(selector)
+    env.update(extra_env or {})
+    default = tmp_path / ".env"
+    default.write_text("ENV_FILE_TEST_VALUE=default\nENV_FILE_TEST_DEFAULT_ONLY=unselected\n", encoding="utf-8")
+    cwd = tmp_path / "unrelated"
+    cwd.mkdir(exist_ok=True)
+    # Control discovery only; parsing, precedence and startup imports are real.
+    # This prevents default discovery from reading any developer secrets.
+    script = f"""
+import json, os
+import dotenv.main
+dotenv.main.find_dotenv = lambda *args, **kwargs: {str(default)!r}
+{prelude}
+{ENTRYPOINTS[entrypoint]}
+print(json.dumps([os.getenv('ENV_FILE_TEST_VALUE'), os.getenv('ENV_FILE_TEST_DEFAULT_ONLY')]))
+"""
+    return subprocess.run([sys.executable, "-c", script], cwd=cwd, env=env, capture_output=True, text=True, timeout=30)
+
+
+@pytest.mark.parametrize("entrypoint", ENTRYPOINTS)
+@pytest.mark.parametrize("relative", [False, True])
+def test_explicit_selection_from_unrelated_cwd(tmp_path, entrypoint, relative):
+    selected = tmp_path / "stage.env"
+    selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8")
+    result = run_startup(tmp_path, entrypoint, "../stage.env" if relative else selected)
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout) == ["selected", None]
+
+
+@pytest.mark.parametrize("value", ["exported", ""])
+def test_process_environment_wins(tmp_path, value):
+    selected = tmp_path / "stage.env"
+    selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8")
+    result = run_startup(tmp_path, selector=selected, extra_env={"ENV_FILE_TEST_VALUE": value})
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout) == [value, None]
+
+
+@pytest.mark.parametrize("entrypoint", ENTRYPOINTS)
+def test_unset_preserves_default_loading(tmp_path, entrypoint):
+    result = run_startup(tmp_path, entrypoint)
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout) == ["default", "unselected"]
+
+
+@pytest.mark.parametrize("entrypoint", ENTRYPOINTS)
+@pytest.mark.parametrize("kind", ["missing", "directory", "empty", "unreadable", "invalid_encoding"])
+def test_invalid_explicit_file_fails_without_fallback(tmp_path, entrypoint, kind):
+    selected = tmp_path / "invalid.env"
+    prelude = ""
+    if kind == "directory":
+        selected.mkdir()
+    elif kind == "unreadable":
+        selected.write_text("ENV_FILE_TEST_VALUE=do-not-print-this\n", encoding="utf-8")
+        # Deterministic even under root and on Windows: emulate OS denial on open.
+        prelude = f"""
+import sys
+def deny_open(event, args):
+    if event == 'open' and str(args[0]) == {str(selected)!r}:
+        raise PermissionError('do-not-print-this')
+sys.addaudithook(deny_open)
+"""
+    elif kind == "invalid_encoding":
+        selected.write_bytes(b"ENV_FILE_TEST_VALUE=do-not-print-this\xff")
+    result = run_startup(tmp_path, entrypoint, "" if kind == "empty" else selected, prelude=prelude)
+    assert result.returncode != 0
+    assert "DEER_FLOW_ENV_FILE" in result.stderr
+    assert "readable UTF-8 regular file" in result.stderr
+    assert "do-not-print-this" not in result.stderr
+    assert "unselected" not in result.stdout
+
+
+def test_empty_selected_file_does_not_load_defaults(tmp_path):
+    selected = tmp_path / "empty.env"
+    selected.touch()
+    result = run_startup(tmp_path, selector=selected)
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout) == [None, None]
+
+
+@pytest.mark.parametrize("entrypoint", ENTRYPOINTS)
+@pytest.mark.parametrize("disabled", ["1", "TRUE", "t", "YeS", "Y"])
+def test_explicit_selection_rejects_disabled_dotenv(tmp_path, entrypoint, disabled):
+    selected = tmp_path / "stage.env"
+    selected.write_text("ENV_FILE_TEST_VALUE=do-not-print-this\n", encoding="utf-8")
+    result = run_startup(tmp_path, entrypoint, selected, {"PYTHON_DOTENV_DISABLED": disabled})
+    assert result.returncode != 0
+    assert "DEER_FLOW_ENV_FILE" in result.stderr
+    assert "PYTHON_DOTENV_DISABLED" in result.stderr
+    assert "do-not-print-this" not in result.stderr
+    assert "unselected" not in result.stdout
+
+
+@pytest.mark.parametrize("disabled", ["0", "false", "no", "", " true "])
+def test_non_disabling_values_allow_explicit_selection(tmp_path, disabled):
+    selected = tmp_path / "stage.env"
+    selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8")
+    result = run_startup(tmp_path, selector=selected, extra_env={"PYTHON_DOTENV_DISABLED": disabled})
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout) == ["selected", None]
+
+
+@pytest.mark.parametrize("entrypoint", ENTRYPOINTS)
+def test_disabled_dotenv_without_selector_keeps_default_behavior(tmp_path, entrypoint):
+    result = run_startup(tmp_path, entrypoint, extra_env={"PYTHON_DOTENV_DISABLED": "1"})
+    assert result.returncode == 0, result.stderr
+    assert json.loads(result.stdout) == [None, None]
+
+
+def test_empty_selected_file_still_rejects_disabled_dotenv(tmp_path):
+    selected = tmp_path / "empty.env"
+    selected.touch()
+    result = run_startup(tmp_path, selector=selected, extra_env={"PYTHON_DOTENV_DISABLED": "true"})
+    assert result.returncode != 0
+    assert "PYTHON_DOTENV_DISABLED" in result.stderr
+
+
+def test_auth_secret_warning_mentions_selected_env_file(tmp_path):
+    selected = tmp_path / "stage.env"
+    selected.write_text("ENV_FILE_TEST_VALUE=selected\n", encoding="utf-8")
+    prelude = "import app.gateway.auth.config as auth_config; auth_config._load_or_create_secret = lambda: 'test-generated-secret'"
+    result = run_startup(tmp_path, "auth", selected, {"AUTH_JWT_SECRET": ""}, prelude)
+    assert result.returncode == 0, result.stderr
+    assert "DEER_FLOW_ENV_FILE" in result.stderr
+    assert ".env by default" in result.stderr