diff --git a/.dockerignore b/.dockerignore index c058d7d2a2..a438684ac3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -73,6 +73,23 @@ backend/.coverage # root that the build client cannot read. **/.deer-flow/ backend/sandbox/ + +# Local configuration holds model API keys, MCP credentials, frontend secrets and +# local HTTPS keys (frontend/*.pem, the frontend .gitignore's mkcert pair). +# Compose and Helm supply it at runtime. The config resolvers still read the legacy +# backend/ copies, and `COPY backend ./backend` / `COPY frontend ./frontend` would +# otherwise bake them into every image layer built from this checkout. +config.yaml +config.yaml.bak +extensions_config.json +mcp_config.json +backend/config.yaml +backend/config.yaml.bak +backend/extensions_config.json +backend/mcp_config.json +**/.env*.local +frontend/*.pem + *.md !README.md !frontend/README.md @@ -84,3 +101,17 @@ backend/sandbox/ !backend/extensions/ !backend/extensions/sources/ !backend/extensions/sources/** + +# The re-inclusion above wins over every earlier rule, so the files the extension +# manager refuses to snapshot (deerflow/extensions/manager.py) are excluded again +# here. A snapshot the manager wrote never contains them; one edited by hand +# must not carry credentials into the image either. +backend/extensions/sources/**/.env +backend/extensions/sources/**/.env.* +backend/extensions/sources/**/.npmrc +backend/extensions/sources/**/.pypirc +backend/extensions/sources/**/credentials.json +backend/extensions/sources/**/*.key +backend/extensions/sources/**/*.pem +backend/extensions/sources/**/*.p12 +backend/extensions/sources/**/*.pfx diff --git a/CHANGELOG.md b/CHANGELOG.md index d59e01e144..d986ac29ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,20 @@ This section accumulates work toward the **2.3.0** milestone ### Fixed +- **docker:** Images built from a checkout no longer contain its local + configuration. `.dockerignore` did not exclude `config.yaml`, + `config.yaml.bak`, `extensions_config.json` or `mcp_config.json`, so a + checkout that keeps them in the legacy `backend/` location baked its model + API keys and MCP credentials into every gateway image layer through + `COPY backend ./backend`. `frontend/.env.local` reached the frontend image the + same way. They are now excluded at the root and in `backend/`, along with + `.env*.local` at any depth, `frontend/*.pem`, and the credential files the + extension manager refuses inside extension snapshots. A locally built + frontend image no longer inlines `NEXT_PUBLIC_*` values from + `frontend/.env.local`; the Docker stack's single nginx origin needs none of + them. Published CI images were not affected; rebuild local images that were + pushed or shared. ([#7958]) + - **agents:** Unfinished todos no longer trigger additional model calls or tool execution after a loop-detection, run token-budget, provider-safety, or model output-length stop. Todo state is preserved so users can continue in a @@ -16071,3 +16085,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7892]: https://github.com/bytedance/deer-flow/pull/7892 [#7907]: https://github.com/bytedance/deer-flow/pull/7907 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 +[#7958]: https://github.com/bytedance/deer-flow/pull/7958 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index e4ce499844..fdb21f2db0 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -23,6 +23,16 @@ ### 修复 +- **docker:** 从本地 checkout 构建的镜像不再包含本地配置。`.dockerignore` + 此前没有排除 `config.yaml`、`config.yaml.bak`、`extensions_config.json` 和 + `mcp_config.json`,因此配置放在旧位置 `backend/` 时,`COPY backend ./backend` + 会把模型 API 密钥和 MCP 凭据写进每一层 gateway 镜像;`frontend/.env.local` + 也以同样方式进入前端镜像。现在根目录和 `backend/` 下的这些文件、任意层级的 + `.env*.local`、`frontend/*.pem`,以及扩展管理器拒绝写入扩展快照的凭据文件都会 + 被排除。本地构建的前端镜像不再内联 `frontend/.env.local` 中的 `NEXT_PUBLIC_*` + 值;Docker 栈通过同一个 nginx 源提供前端和 Gateway,不需要这些变量。CI 发布的 + 镜像不受影响;已推送或分享过的本地镜像请重新构建。([#7958]) + - **智能体:** 循环检测、运行 token 预算、模型服务安全限制或输出长度限制触发停止后, 未完成的 Todo 不再触发额外的模型调用或工具执行。Todo 状态仍会保留, 用户可在后续运行中继续处理。([#7953]) @@ -12785,3 +12795,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写,核心包含子 [#7831]: https://github.com/bytedance/deer-flow/pull/7831 [#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 +[#7958]: https://github.com/bytedance/deer-flow/pull/7958 diff --git a/README.md b/README.md index 9de2c22070..0d230173c0 100644 --- a/README.md +++ b/README.md @@ -711,6 +711,19 @@ exits non-zero and prints the container status plus recent Gateway logs. The production image starts from its already-built environment and never resolves or installs Python dependencies at container startup. +Images built from a checkout never contain its local configuration: +`.dockerignore` keeps `config.yaml`, `config.yaml.bak`, `extensions_config.json` +and `mcp_config.json` (at the root or in the legacy `backend/` location), `.env` +and `.env*.local` files, local HTTPS keys (`frontend/*.pem`), and the credential +files the extension manager refuses inside `backend/extensions/sources/` out of +the build context. Compose mounts the config and passes the environment at +runtime, so pushing or sharing an image does not publish those credentials. +A locally built frontend image therefore no longer picks up `NEXT_PUBLIC_*` +values from `frontend/.env.local`: `next build` inlines them into the client +bundle, Compose's environment reaches only `next start`, and `frontend/Dockerfile` +takes no build arguments for them. The Docker stack serves the frontend and +Gateway from one nginx origin and needs none of them. + If `make up` reports an unwritable runtime home or an unreadable persisted secret after `make docker-start`, run the printed `sudo chown -R : ''` recovery command and retry. The check honors secret overrides from the shell or diff --git a/backend/tests/test_dockerignore_excludes_runtime_data.py b/backend/tests/test_dockerignore_excludes_runtime_data.py index 3da6189686..dd0b1924dd 100644 --- a/backend/tests/test_dockerignore_excludes_runtime_data.py +++ b/backend/tests/test_dockerignore_excludes_runtime_data.py @@ -11,6 +11,17 @@ a build or local deployment: per-user agent definitions and uploads, and ``.jwt_secret``. - ``backend/sandbox`` is the local sandbox provider's workspace root, created by ``backend/Makefile`` and written by agent runs. +- ``config.yaml`` (and the ``config.yaml.bak`` that ``make config-upgrade`` writes + beside it), ``extensions_config.json`` and legacy ``mcp_config.json`` hold model + API keys and MCP credentials. The resolvers still read them from the legacy + ``backend/`` location, where ``COPY backend ./backend`` would ship them. +- ``.env*.local`` files hold local overrides; ``frontend/.env.local`` is the + documented place for ``BETTER_AUTH_SECRET``, and ``COPY frontend ./frontend`` + ships it into the image that ``next build`` and ``next start`` read it from. + ``frontend/*.pem`` is the mkcert key pair for local HTTPS. +- Extension snapshots under ``backend/extensions/sources/`` are re-included + wholesale, so the files the extension manager refuses to snapshot are excluded + again after that re-inclusion; a hand-edited snapshot cannot carry them in. Leaving them in the context has two consequences. Anyone who builds an image on a host that has run DeerFlow bakes that state — including the JWT secret and the user @@ -26,11 +37,13 @@ None of these paths has tracked content, so excluding them costs the build nothi from __future__ import annotations -from fnmatch import fnmatchcase +import re from pathlib import Path, PurePosixPath import pytest +from deerflow.extensions import manager as extension_manager + REPO_ROOT = Path(__file__).resolve().parents[2] DOCKERIGNORE = REPO_ROOT / ".dockerignore" @@ -44,6 +57,22 @@ HOST_LOCAL_PATHS = [ "backend/.deer-flow/.jwt_secret", "backend/.deer-flow/users/some-user/agents/my-agent/config.yaml", "backend/sandbox/some-thread/scratch.py", + "config.yaml", + "config.yaml.bak", + "extensions_config.json", + "mcp_config.json", + "backend/config.yaml", + "backend/config.yaml.bak", + "backend/extensions_config.json", + "backend/mcp_config.json", + "frontend/.env.local", + "frontend/.env.production.local", + "backend/.env.local", + "frontend/localhost.pem", + "frontend/localhost-key.pem", + "backend/extensions/sources/demo/.env", + "backend/extensions/sources/demo/.env.local", + "backend/extensions/sources/demo/src/demo/certs/client.pem", ] # Paths the build genuinely needs; the exclusions must not swallow them. @@ -53,32 +82,60 @@ BUILD_INPUT_PATHS = [ "backend/pyproject.toml", "backend/app/gateway/app.py", "backend/packages/harness/deerflow/config/extensions_config.py", + "backend/packages/harness/deerflow/config/app_config.py", + "frontend/src/env.js", + "backend/extensions/sources/demo/pyproject.toml", + "backend/extensions/sources/demo/README.md", + "backend/extensions/sources/demo/src/demo/__init__.py", + "backend/extensions/sources/demo/src/demo/_native.so", ] +# A directory inside an extension snapshot; refused files must stay out at any depth. +SNAPSHOT_ROOT = "backend/extensions/sources/demo/src/demo" + def _ignore_patterns() -> list[str]: lines = DOCKERIGNORE.read_text(encoding="utf-8").splitlines() return [line.strip() for line in lines if line.strip() and not line.lstrip().startswith("#")] -def _pattern_matches(pattern: str, path: str) -> bool: - """Whether one non-negated pattern matches *path*. +def _pattern_regex(pattern: str) -> re.Pattern[str]: + """Translate one pattern the way Docker's matcher does. - This intentionally models the pattern shapes used by this repository rather - than reimplementing Docker's full matcher: root-relative names/globs, - directory prefixes, trailing ``/**``, and leading ``**/name`` patterns. + ``**`` spans any number of path segments, ``*`` and ``?`` stay within one + segment, ``[...]`` is a character class, and every pattern is anchored at + the build-context root. """ - pattern = pattern.rstrip("/") - if pattern.startswith("**/"): - name = pattern[3:] - return name in PurePosixPath(path).parts - if pattern.endswith("/**"): - pattern = pattern[:-3].rstrip("/") - if "/" not in pattern: - root_name = PurePosixPath(path).parts[0] - return fnmatchcase(root_name, pattern) - prefix = f"{pattern}/" - return path == pattern or path.startswith(prefix) + out: list[str] = [] + index = 0 + while index < len(pattern): + if pattern.startswith("**/", index): + out.append("(?:.*/)?") + index += 3 + elif pattern.startswith("**", index): + out.append(".*") + index += 2 + elif pattern[index] == "*": + out.append("[^/]*") + index += 1 + elif pattern[index] == "?": + out.append("[^/]") + index += 1 + elif pattern[index] == "[" and (end := pattern.find("]", index)) != -1: + body = pattern[index + 1 : end] + out.append("[" + ("^" + body[1:] if body.startswith(("!", "^")) else body) + "]") + index = end + 1 + else: + out.append(re.escape(pattern[index])) + index += 1 + return re.compile("".join(out) + r"\Z") + + +def _pattern_matches(pattern: str, path: str) -> bool: + """Whether one non-negated pattern matches *path* or one of its parent directories.""" + regex = _pattern_regex(pattern.rstrip("/")) + parts = PurePosixPath(path).parts + return any(regex.match("/".join(parts[:depth])) for depth in range(1, len(parts) + 1)) def _is_excluded(patterns: list[str], path: str) -> bool: @@ -103,6 +160,26 @@ def test_exclusion_uses_last_matching_pattern(patterns: list[str], expected: boo assert _is_excluded(patterns, "backend/runtime.json") is expected +@pytest.mark.parametrize( + ("pattern", "path", "expected"), + [ + ("config.yaml", "config.yaml", True), + ("config.yaml", "backend/config.yaml", False), + ("frontend/*.pem", "frontend/localhost.pem", True), + ("frontend/*.pem", "frontend/certs/localhost.pem", False), + ("**/.env*.local", "frontend/.env.local", True), + ("**/.env*.local", ".env.example", False), + ("backend/extensions/sources/**/.env", "backend/extensions/sources/demo/.env", True), + ("backend/extensions/sources/**/.env", "backend/extensions/sources/demo/a/b/.env", True), + ("**/.deer-flow/", "backend/.deer-flow/data/deerflow.db", True), + ("*.py[cod]", "module.pyc", True), + ("*.py[cod]", "backend/module.pyc", False), + ], +) +def test_pattern_matcher_models_docker(pattern: str, path: str, expected: bool) -> None: + assert _pattern_matches(pattern, path) is expected + + @pytest.mark.parametrize("local_path", HOST_LOCAL_PATHS) def test_host_local_data_is_excluded_from_build_context(local_path: str) -> None: patterns = _ignore_patterns() @@ -113,3 +190,12 @@ def test_host_local_data_is_excluded_from_build_context(local_path: str) -> None def test_build_inputs_are_still_included(build_input: str) -> None: patterns = _ignore_patterns() assert not _is_excluded(patterns, build_input), f"{build_input} is needed by the build but is excluded" + + +@pytest.mark.parametrize( + "name", + [".env", ".env.local", ".env.production", *sorted(extension_manager._SENSITIVE_FILENAMES), *(f"secret{suffix}" for suffix in sorted(extension_manager._SENSITIVE_SUFFIXES))], +) +def test_files_the_extension_manager_refuses_stay_out_of_snapshots(name: str) -> None: + patterns = _ignore_patterns() + assert _is_excluded(patterns, f"{SNAPSHOT_ROOT}/{name}"), f"extension snapshots are re-included wholesale; exclude {name} after the re-inclusion"