From 4e90f5d771501007c6b8c1cc8d0605d1dc983547 Mon Sep 17 00:00:00 2001 From: Hyeonsang Cho Date: Sun, 8 Nov 2026 08:56:27 +0900 Subject: [PATCH] fix(docker): keep local config out of the image build context (#7958) * fix(docker): keep local config out of the image build context backend/Dockerfile copies the backend tree wholesale (COPY backend ./backend), and frontend/Dockerfile does the same for frontend/. .dockerignore excluded .env files and runtime data, but not the local configuration: config.yaml, the config.yaml.bak that make config-upgrade writes beside it, extensions_config.json and legacy mcp_config.json. The config resolvers still read these from the legacy backend/ location, so on a checkout that keeps them there every locally built gateway image (dev and prod) carried the model API keys and MCP credentials in its layers, under the file Compose mounts over at runtime. frontend/.env.local, the documented place for BETTER_AUTH_SECRET in local development, was copied into the frontend image the same way, where next build and next start also read it. Exclude those files at the root and in backend/, and .env*.local at any depth. None has tracked content; the build's file list is otherwise unchanged. * docs(changelog): reference #7958 for the Docker build-context fix * fix(docker): keep local HTTPS keys and snapshot credentials out of images frontend/.gitignore also ignores *.pem, the mkcert key pair for local HTTPS, and COPY frontend ./frontend shipped it into the dev and prod frontend images. Exclude frontend/*.pem. The trailing !backend/extensions/sources/** re-inclusion wins over every earlier rule, so a hand-edited extension snapshot could still carry .env files or keys into the gateway image. Exclude, after the re-inclusion, exactly the files the extension manager already refuses to snapshot; a test pins that set to the manager's own _SENSITIVE_FILENAMES and _SENSITIVE_SUFFIXES. The test's matcher now translates patterns as Docker does (** across segments, * within one, character classes, parent-directory matches), so it can model the mid-path ** rules; its predictions agree with BuildKit on every test path. The README and changelog now say that a locally built frontend image no longer inlines NEXT_PUBLIC_* values from frontend/.env.local. --------- Co-authored-by: Willem Jiang --- .dockerignore | 31 +++++ CHANGELOG.md | 15 +++ CHANGELOG_zh.md | 11 ++ README.md | 13 ++ ...test_dockerignore_excludes_runtime_data.py | 120 +++++++++++++++--- 5 files changed, 173 insertions(+), 17 deletions(-) diff --git a/.dockerignore b/.dockerignore index c058d7d2a2..a438684ac3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -73,6 +73,23 @@ backend/.coverage # root that the build client cannot read. **/.deer-flow/ backend/sandbox/ + +# Local configuration holds model API keys, MCP credentials, frontend secrets and +# local HTTPS keys (frontend/*.pem, the frontend .gitignore's mkcert pair). +# Compose and Helm supply it at runtime. The config resolvers still read the legacy +# backend/ copies, and `COPY backend ./backend` / `COPY frontend ./frontend` would +# otherwise bake them into every image layer built from this checkout. +config.yaml +config.yaml.bak +extensions_config.json +mcp_config.json +backend/config.yaml +backend/config.yaml.bak +backend/extensions_config.json +backend/mcp_config.json +**/.env*.local +frontend/*.pem + *.md !README.md !frontend/README.md @@ -84,3 +101,17 @@ backend/sandbox/ !backend/extensions/ !backend/extensions/sources/ !backend/extensions/sources/** + +# The re-inclusion above wins over every earlier rule, so the files the extension +# manager refuses to snapshot (deerflow/extensions/manager.py) are excluded again +# here. A snapshot the manager wrote never contains them; one edited by hand +# must not carry credentials into the image either. +backend/extensions/sources/**/.env +backend/extensions/sources/**/.env.* +backend/extensions/sources/**/.npmrc +backend/extensions/sources/**/.pypirc +backend/extensions/sources/**/credentials.json +backend/extensions/sources/**/*.key +backend/extensions/sources/**/*.pem +backend/extensions/sources/**/*.p12 +backend/extensions/sources/**/*.pfx diff --git a/CHANGELOG.md b/CHANGELOG.md index d59e01e144..d986ac29ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,20 @@ This section accumulates work toward the **2.3.0** milestone ### Fixed +- **docker:** Images built from a checkout no longer contain its local + configuration. `.dockerignore` did not exclude `config.yaml`, + `config.yaml.bak`, `extensions_config.json` or `mcp_config.json`, so a + checkout that keeps them in the legacy `backend/` location baked its model + API keys and MCP credentials into every gateway image layer through + `COPY backend ./backend`. `frontend/.env.local` reached the frontend image the + same way. They are now excluded at the root and in `backend/`, along with + `.env*.local` at any depth, `frontend/*.pem`, and the credential files the + extension manager refuses inside extension snapshots. A locally built + frontend image no longer inlines `NEXT_PUBLIC_*` values from + `frontend/.env.local`; the Docker stack's single nginx origin needs none of + them. Published CI images were not affected; rebuild local images that were + pushed or shared. ([#7958]) + - **agents:** Unfinished todos no longer trigger additional model calls or tool execution after a loop-detection, run token-budget, provider-safety, or model output-length stop. Todo state is preserved so users can continue in a @@ -16071,3 +16085,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7892]: https://github.com/bytedance/deer-flow/pull/7892 [#7907]: https://github.com/bytedance/deer-flow/pull/7907 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 +[#7958]: https://github.com/bytedance/deer-flow/pull/7958 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index e4ce499844..fdb21f2db0 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -23,6 +23,16 @@ ### 修复 +- **docker：** 从本地 checkout 构建的镜像不再包含本地配置。`.dockerignore` + 此前没有排除 `config.yaml`、`config.yaml.bak`、`extensions_config.json` 和 + `mcp_config.json`，因此配置放在旧位置 `backend/` 时，`COPY backend ./backend` + 会把模型 API 密钥和 MCP 凭据写进每一层 gateway 镜像；`frontend/.env.local` + 也以同样方式进入前端镜像。现在根目录和 `backend/` 下的这些文件、任意层级的 + `.env*.local`、`frontend/*.pem`，以及扩展管理器拒绝写入扩展快照的凭据文件都会 + 被排除。本地构建的前端镜像不再内联 `frontend/.env.local` 中的 `NEXT_PUBLIC_*` + 值；Docker 栈通过同一个 nginx 源提供前端和 Gateway，不需要这些变量。CI 发布的 + 镜像不受影响；已推送或分享过的本地镜像请重新构建。([#7958]) + - **智能体：** 循环检测、运行 token 预算、模型服务安全限制或输出长度限制触发停止后， 未完成的 Todo 不再触发额外的模型调用或工具执行。Todo 状态仍会保留， 用户可在后续运行中继续处理。([#7953]) @@ -12785,3 +12795,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7831]: https://github.com/bytedance/deer-flow/pull/7831 [#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 +[#7958]: https://github.com/bytedance/deer-flow/pull/7958 diff --git a/README.md b/README.md index 9de2c22070..0d230173c0 100644 --- a/README.md +++ b/README.md @@ -711,6 +711,19 @@ exits non-zero and prints the container status plus recent Gateway logs. The production image starts from its already-built environment and never resolves or installs Python dependencies at container startup. +Images built from a checkout never contain its local configuration: +`.dockerignore` keeps `config.yaml`, `config.yaml.bak`, `extensions_config.json` +and `mcp_config.json` (at the root or in the legacy `backend/` location), `.env` +and `.env*.local` files, local HTTPS keys (`frontend/*.pem`), and the credential +files the extension manager refuses inside `backend/extensions/sources/` out of +the build context. Compose mounts the config and passes the environment at +runtime, so pushing or sharing an image does not publish those credentials. +A locally built frontend image therefore no longer picks up `NEXT_PUBLIC_*` +values from `frontend/.env.local`: `next build` inlines them into the client +bundle, Compose's environment reaches only `next start`, and `frontend/Dockerfile` +takes no build arguments for them. The Docker stack serves the frontend and +Gateway from one nginx origin and needs none of them. + If `make up` reports an unwritable runtime home or an unreadable persisted secret after `make docker-start`, run the printed `sudo chown -R : ''` recovery command and retry. The check honors secret overrides from the shell or diff --git a/backend/tests/test_dockerignore_excludes_runtime_data.py b/backend/tests/test_dockerignore_excludes_runtime_data.py index 3da6189686..dd0b1924dd 100644 --- a/backend/tests/test_dockerignore_excludes_runtime_data.py +++ b/backend/tests/test_dockerignore_excludes_runtime_data.py @@ -11,6 +11,17 @@ a build or local deployment: per-user agent definitions and uploads, and ``.jwt_secret``. - ``backend/sandbox`` is the local sandbox provider's workspace root, created by ``backend/Makefile`` and written by agent runs. +- ``config.yaml`` (and the ``config.yaml.bak`` that ``make config-upgrade`` writes + beside it), ``extensions_config.json`` and legacy ``mcp_config.json`` hold model + API keys and MCP credentials. The resolvers still read them from the legacy + ``backend/`` location, where ``COPY backend ./backend`` would ship them. +- ``.env*.local`` files hold local overrides; ``frontend/.env.local`` is the + documented place for ``BETTER_AUTH_SECRET``, and ``COPY frontend ./frontend`` + ships it into the image that ``next build`` and ``next start`` read it from. + ``frontend/*.pem`` is the mkcert key pair for local HTTPS. +- Extension snapshots under ``backend/extensions/sources/`` are re-included + wholesale, so the files the extension manager refuses to snapshot are excluded + again after that re-inclusion; a hand-edited snapshot cannot carry them in. Leaving them in the context has two consequences. Anyone who builds an image on a host that has run DeerFlow bakes that state — including the JWT secret and the user @@ -26,11 +37,13 @@ None of these paths has tracked content, so excluding them costs the build nothi from __future__ import annotations -from fnmatch import fnmatchcase +import re from pathlib import Path, PurePosixPath import pytest +from deerflow.extensions import manager as extension_manager + REPO_ROOT = Path(__file__).resolve().parents[2] DOCKERIGNORE = REPO_ROOT / ".dockerignore" @@ -44,6 +57,22 @@ HOST_LOCAL_PATHS = [ "backend/.deer-flow/.jwt_secret", "backend/.deer-flow/users/some-user/agents/my-agent/config.yaml", "backend/sandbox/some-thread/scratch.py", + "config.yaml", + "config.yaml.bak", + "extensions_config.json", + "mcp_config.json", + "backend/config.yaml", + "backend/config.yaml.bak", + "backend/extensions_config.json", + "backend/mcp_config.json", + "frontend/.env.local", + "frontend/.env.production.local", + "backend/.env.local", + "frontend/localhost.pem", + "frontend/localhost-key.pem", + "backend/extensions/sources/demo/.env", + "backend/extensions/sources/demo/.env.local", + "backend/extensions/sources/demo/src/demo/certs/client.pem", ] # Paths the build genuinely needs; the exclusions must not swallow them. @@ -53,32 +82,60 @@ BUILD_INPUT_PATHS = [ "backend/pyproject.toml", "backend/app/gateway/app.py", "backend/packages/harness/deerflow/config/extensions_config.py", + "backend/packages/harness/deerflow/config/app_config.py", + "frontend/src/env.js", + "backend/extensions/sources/demo/pyproject.toml", + "backend/extensions/sources/demo/README.md", + "backend/extensions/sources/demo/src/demo/__init__.py", + "backend/extensions/sources/demo/src/demo/_native.so", ] +# A directory inside an extension snapshot; refused files must stay out at any depth. +SNAPSHOT_ROOT = "backend/extensions/sources/demo/src/demo" + def _ignore_patterns() -> list[str]: lines = DOCKERIGNORE.read_text(encoding="utf-8").splitlines() return [line.strip() for line in lines if line.strip() and not line.lstrip().startswith("#")] -def _pattern_matches(pattern: str, path: str) -> bool: - """Whether one non-negated pattern matches *path*. +def _pattern_regex(pattern: str) -> re.Pattern[str]: + """Translate one pattern the way Docker's matcher does. - This intentionally models the pattern shapes used by this repository rather - than reimplementing Docker's full matcher: root-relative names/globs, - directory prefixes, trailing ``/**``, and leading ``**/name`` patterns. + ``**`` spans any number of path segments, ``*`` and ``?`` stay within one + segment, ``[...]`` is a character class, and every pattern is anchored at + the build-context root. """ - pattern = pattern.rstrip("/") - if pattern.startswith("**/"): - name = pattern[3:] - return name in PurePosixPath(path).parts - if pattern.endswith("/**"): - pattern = pattern[:-3].rstrip("/") - if "/" not in pattern: - root_name = PurePosixPath(path).parts[0] - return fnmatchcase(root_name, pattern) - prefix = f"{pattern}/" - return path == pattern or path.startswith(prefix) + out: list[str] = [] + index = 0 + while index < len(pattern): + if pattern.startswith("**/", index): + out.append("(?:.*/)?") + index += 3 + elif pattern.startswith("**", index): + out.append(".*") + index += 2 + elif pattern[index] == "*": + out.append("[^/]*") + index += 1 + elif pattern[index] == "?": + out.append("[^/]") + index += 1 + elif pattern[index] == "[" and (end := pattern.find("]", index)) != -1: + body = pattern[index + 1 : end] + out.append("[" + ("^" + body[1:] if body.startswith(("!", "^")) else body) + "]") + index = end + 1 + else: + out.append(re.escape(pattern[index])) + index += 1 + return re.compile("".join(out) + r"\Z") + + +def _pattern_matches(pattern: str, path: str) -> bool: + """Whether one non-negated pattern matches *path* or one of its parent directories.""" + regex = _pattern_regex(pattern.rstrip("/")) + parts = PurePosixPath(path).parts + return any(regex.match("/".join(parts[:depth])) for depth in range(1, len(parts) + 1)) def _is_excluded(patterns: list[str], path: str) -> bool: @@ -103,6 +160,26 @@ def test_exclusion_uses_last_matching_pattern(patterns: list[str], expected: boo assert _is_excluded(patterns, "backend/runtime.json") is expected +@pytest.mark.parametrize( + ("pattern", "path", "expected"), + [ + ("config.yaml", "config.yaml", True), + ("config.yaml", "backend/config.yaml", False), + ("frontend/*.pem", "frontend/localhost.pem", True), + ("frontend/*.pem", "frontend/certs/localhost.pem", False), + ("**/.env*.local", "frontend/.env.local", True), + ("**/.env*.local", ".env.example", False), + ("backend/extensions/sources/**/.env", "backend/extensions/sources/demo/.env", True), + ("backend/extensions/sources/**/.env", "backend/extensions/sources/demo/a/b/.env", True), + ("**/.deer-flow/", "backend/.deer-flow/data/deerflow.db", True), + ("*.py[cod]", "module.pyc", True), + ("*.py[cod]", "backend/module.pyc", False), + ], +) +def test_pattern_matcher_models_docker(pattern: str, path: str, expected: bool) -> None: + assert _pattern_matches(pattern, path) is expected + + @pytest.mark.parametrize("local_path", HOST_LOCAL_PATHS) def test_host_local_data_is_excluded_from_build_context(local_path: str) -> None: patterns = _ignore_patterns() @@ -113,3 +190,12 @@ def test_host_local_data_is_excluded_from_build_context(local_path: str) -> None def test_build_inputs_are_still_included(build_input: str) -> None: patterns = _ignore_patterns() assert not _is_excluded(patterns, build_input), f"{build_input} is needed by the build but is excluded" + + +@pytest.mark.parametrize( + "name", + [".env", ".env.local", ".env.production", *sorted(extension_manager._SENSITIVE_FILENAMES), *(f"secret{suffix}" for suffix in sorted(extension_manager._SENSITIVE_SUFFIXES))], +) +def test_files_the_extension_manager_refuses_stay_out_of_snapshots(name: str) -> None: + patterns = _ignore_patterns() + assert _is_excluded(patterns, f"{SNAPSHOT_ROOT}/{name}"), f"extension snapshots are re-included wholesale; exclude {name} after the re-inclusion"