diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b10cb006..5a0f5c0436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 This section accumulates work toward the **2.3.0** milestone ([2.3.0](https://github.com/bytedance/deer-flow/milestone/4)). +### Fixed + +- **memory:** `GET /api/memory/config` and `/api/memory/status` now mask + credential values in `backend_config` (`api_key`, tokens, passwords, + `Authorization` headers, at any depth) as `***`. They returned the config as + loaded, with `$VAR` references already resolved, so a DeerMem extraction + model key or a Honcho `api_key` reached every caller with `memory:read` in + plaintext, which with authorization disabled (the default) is every + signed-in user. Non-secret knobs, `api_key_env` names and unset values are + unchanged. ([#7665]) + ## [2.2.0] — 2026-10-29 This section accumulates work toward the **2.2.0** milestone @@ -15915,3 +15926,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7643]: https://github.com/bytedance/deer-flow/pull/7643 [#7651]: https://github.com/bytedance/deer-flow/pull/7651 [#7659]: https://github.com/bytedance/deer-flow/pull/7659 +[#7665]: https://github.com/bytedance/deer-flow/pull/7665 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 85717a9887..7f6d1fc9b1 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -12,6 +12,14 @@ 本节累积面向 **2.3.0** 里程碑（[2.3.0](https://github.com/bytedance/deer-flow/milestone/4)）的工作。 截至 2026-10-28，该里程碑已合并 **1068 个拉取请求**。 +### 修复 + +- **记忆：** `GET /api/memory/config` 与 `/api/memory/status` 现在会把 `backend_config` + 中的凭据值（`api_key`、token、密码、`Authorization` 请求头，任意层级）遮蔽为 `***`。 + 此前它们按加载结果原样返回配置，而 `$VAR` 引用此时已被解析，因此 DeerMem 提取模型的密钥 + 或 Honcho 的 `api_key` 会以明文返回给所有拥有 `memory:read` 的调用方；在默认关闭授权时， + 这就是所有已登录用户。非机密配置项、`api_key_env` 变量名和未设置的值保持不变。([#7665]) + ## [2.2.0] — 2026-10-29 本节累积面向 **2.2.0** 里程碑（[2.2.0](https://github.com/bytedance/deer-flow/milestone/3)）的工作。 @@ -12686,3 +12694,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7643]: https://github.com/bytedance/deer-flow/pull/7643 [#7651]: https://github.com/bytedance/deer-flow/pull/7651 [#7659]: https://github.com/bytedance/deer-flow/pull/7659 +[#7665]: https://github.com/bytedance/deer-flow/pull/7665 diff --git a/backend/README.md b/backend/README.md index 263d7aa768..0d725809d3 100644 --- a/backend/README.md +++ b/backend/README.md @@ -156,8 +156,8 @@ to a boundary (SQLite) or causing the search to fail (PostgreSQL). | `POST /api/skills/install` | Install skill from `.skill` archive | | `GET /api/memory` | Retrieve memory data | | `POST /api/memory/reload` | Force memory reload | -| `GET /api/memory/config` | Memory configuration | -| `GET /api/memory/status` | Combined config + data | +| `GET /api/memory/config` | Memory configuration; credential values in `backend_config` are masked | +| `GET /api/memory/status` | Combined config (masked the same way) + data | | `GET /api/threads/{id}/runs/{run_id}/events` | Debug/audit events for one run; filter `event_types=context:memory` for effective memory identity | | `POST /api/threads/{id}/uploads` | Upload files (optional PDF/PPT/Excel/Word-to-Markdown conversion, rejects directory paths, auto-renames duplicate filenames in one request) | | `GET /api/threads/{id}/uploads/list` | List uploaded files | diff --git a/backend/README_zh.md b/backend/README_zh.md index 7335255eca..687775f704 100644 --- a/backend/README_zh.md +++ b/backend/README_zh.md @@ -126,8 +126,8 @@ FastAPI 应用程序，为前端集成提供 REST 接口： | `POST /api/skills/install` | 从 `.skill` 归档文件安装技能 | | `GET /api/memory` | 获取记忆数据 | | `POST /api/memory/reload` | 强制重新加载记忆 | -| `GET /api/memory/config` | 获取记忆配置 | -| `GET /api/memory/status` | 获取组合后的配置和数据状态 | +| `GET /api/memory/config` | 获取记忆配置；`backend_config` 中的凭据值会被遮蔽 | +| `GET /api/memory/status` | 获取组合后的配置（同样遮蔽凭据）和数据状态 | | `GET /api/threads/{id}/runs/{run_id}/events` | 获取某次运行的调试或审计事件；可使用 `event_types=context:memory` 筛选实际记忆标识 | | `POST /api/threads/{id}/uploads` | 上传文件（可选地将 PDF、PPT、Excel、Word 转换为 Markdown；拒绝目录路径；自动重命名单次请求中的重复文件名） | | `GET /api/threads/{id}/uploads/list` | 列出已上传的文件 | diff --git a/backend/app/gateway/routers/mcp.py b/backend/app/gateway/routers/mcp.py index 4ffc0cb336..94506f0e0e 100644 --- a/backend/app/gateway/routers/mcp.py +++ b/backend/app/gateway/routers/mcp.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_valida from app.gateway.deps import require_admin_user from app.gateway.mcp_oauth_tokens import forget_unconfigured_mcp_oauth_tokens from app.gateway.persistent_writes import run_drained_write +from app.gateway.secret_masking import MASKED_VALUE, mask_secret_values, normalize_config_key from deerflow.config.extensions_config import ( ExtensionsConfig, McpRoutingConfig, @@ -552,29 +553,19 @@ class McpCacheResetResponse(BaseModel): message: str = Field(description="Human-readable reset status") -_MASKED_VALUE = "***" +_MASKED_VALUE = MASKED_VALUE _SENSITIVE_EXTRA_KEY_RE = re.compile( r"(^|_)(api_key|apikey|access_key|private_key|client_secret|secret|token|password|passwd|credential|credentials|authorization|bearer)(_|$)", re.IGNORECASE, ) -def _normalize_config_key(key: str) -> str: - with_boundaries = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", key) - with_boundaries = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", with_boundaries) - return re.sub(r"[^a-z0-9]+", "_", with_boundaries.lower()).strip("_") - - def _is_sensitive_extra_key(key: str) -> bool: - return bool(_SENSITIVE_EXTRA_KEY_RE.search(_normalize_config_key(key))) + return bool(_SENSITIVE_EXTRA_KEY_RE.search(normalize_config_key(key))) def _mask_sensitive_extra_value(value: Any) -> Any: - if isinstance(value, dict): - return {key: _MASKED_VALUE if _is_sensitive_extra_key(str(key)) else _mask_sensitive_extra_value(nested) for key, nested in value.items()} - if isinstance(value, list): - return [_mask_sensitive_extra_value(item) for item in value] - return value + return mask_secret_values(value, _is_sensitive_extra_key) def _contains_masked_sensitive_extra_value(key: str, value: Any) -> bool: diff --git a/backend/app/gateway/routers/memory.py b/backend/app/gateway/routers/memory.py index cb42d98fa2..80f3ead0b5 100644 --- a/backend/app/gateway/routers/memory.py +++ b/backend/app/gateway/routers/memory.py @@ -1,6 +1,7 @@ """Memory API router for retrieving and managing global memory data.""" import asyncio +import re from typing import Any, Literal from fastapi import APIRouter, HTTPException, Request @@ -9,6 +10,7 @@ from pydantic import BaseModel, Field, field_validator from app.gateway.authz import require_permission from app.gateway.internal_auth import get_trusted_internal_owner_user_id from app.gateway.persistent_writes import run_drained_write +from app.gateway.secret_masking import mask_secret_values, normalize_config_key from deerflow.agents.memory import MemoryConflictError, MemoryCorruptionError, MemoryManager, get_memory_manager from deerflow.config.agents_config import AGENT_NAME_PATTERN from deerflow.config.memory_config import get_memory_config @@ -234,6 +236,18 @@ class FactPatchRequest(BaseModel): confidence: float | None = Field(default=None, ge=0.0, le=1.0, description="Confidence score (0-1)") +# Anchored at the end of the key: DeerMem knobs such as ``token_counting`` and +# ``guaranteed_token_budget`` are not secrets, and ``api_key_env`` names an +# environment variable rather than holding the key. This is a denylist: plural +# ``tokens`` stays out because ``max_injection_tokens`` is a knob, and bare +# ``key``, ``auth`` or ``session`` are too generic to mask. +_SECRET_CONFIG_KEY_RE = re.compile(r"(^|_)(api_?keys?|access_keys?|secret_keys?|private_keys?|auth_keys?|secrets?|token|passwords?|passwds?|credentials?|authorization|bearer|cookies?)$") + + +def _is_secret_config_key(key: str) -> bool: + return bool(_SECRET_CONFIG_KEY_RE.search(normalize_config_key(key))) + + class MemoryConfigResponse(BaseModel): """Response model for memory configuration.""" @@ -242,7 +256,13 @@ class MemoryConfigResponse(BaseModel): injection_enabled: bool = Field(..., description="Whether memory is injected into the system prompt (call-site gate).") shutdown_flush_timeout_seconds: float = Field(..., description="Hard budget (s) to drain pending memory updates on Gateway graceful shutdown; must fit inside the pod's K8s terminationGracePeriodSeconds.") manager_class: str = Field(..., description="Active memory backend selector (backend name or dotted path).") - backend_config: dict = Field(..., description="Backend-private config (self-interpreted by the backend).") + backend_config: dict = Field(..., description="Backend-private config (self-interpreted by the backend); credential values such as api_key are masked as ***.") + + @field_validator("backend_config") + @classmethod + def _mask_backend_secrets(cls, value: dict) -> dict: + """Config loading already resolved ``$VAR`` references, so keys would be plaintext.""" + return mask_secret_values(value, _is_secret_config_key, keep_unset=True) class MemoryStatusResponse(BaseModel): diff --git a/backend/app/gateway/secret_masking.py b/backend/app/gateway/secret_masking.py new file mode 100644 index 0000000000..494711ec1e --- /dev/null +++ b/backend/app/gateway/secret_masking.py @@ -0,0 +1,30 @@ +"""Shared masking for credential values that Gateway routes echo back from config.""" + +import re +from collections.abc import Callable +from typing import Any + +MASKED_VALUE = "***" + + +def normalize_config_key(key: str) -> str: + """Snake-case a config key so ``apiKey``, ``API-Key`` and ``api_key`` compare equal.""" + with_boundaries = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", key) + with_boundaries = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", with_boundaries) + return re.sub(r"[^a-z0-9]+", "_", with_boundaries.lower()).strip("_") + + +def mask_secret_values(value: Any, is_secret_key: Callable[[str], bool], *, keep_unset: bool = False) -> Any: + """Return a copy of ``value`` with every value under a secret-named key masked, at any depth. + + A secret-named key masks its whole value, containers included: a + ``credentials`` mapping can hold keys under arbitrary names, so masking only + its secret-named leaves could pass one through. With ``keep_unset``, a + ``None`` or ``""`` value stays as it is, so a response still shows that no + secret is configured. + """ + if isinstance(value, dict): + return {key: MASKED_VALUE if is_secret_key(str(key)) and not (keep_unset and nested in (None, "")) else mask_secret_values(nested, is_secret_key, keep_unset=keep_unset) for key, nested in value.items()} + if isinstance(value, list): + return [mask_secret_values(item, is_secret_key, keep_unset=keep_unset) for item in value] + return value diff --git a/backend/tests/test_gateway_secret_masking.py b/backend/tests/test_gateway_secret_masking.py new file mode 100644 index 0000000000..49c483a537 --- /dev/null +++ b/backend/tests/test_gateway_secret_masking.py @@ -0,0 +1,31 @@ +"""Shared credential masking used by the MCP and memory config routes.""" + +import pytest + +from app.gateway.secret_masking import MASKED_VALUE, mask_secret_values, normalize_config_key + + +@pytest.mark.parametrize( + ("key", "normalized"), + [("api_key", "api_key"), ("apiKey", "api_key"), ("APIKey", "api_key"), ("API-Key", "api_key"), ("accessToken", "access_token"), ("X-Api-Key", "x_api_key")], +) +def test_normalize_config_key_snake_cases_spellings(key: str, normalized: str) -> None: + assert normalize_config_key(key) == normalized + + +def _is_secret(key: str) -> bool: + return key in {"api_key", "credentials"} + + +def test_mask_secret_values_masks_at_any_depth_without_mutating() -> None: + value = {"api_key": "sk-top", "nested": {"api_key": "sk-nested", "model": "m"}, "items": [{"api_key": "sk-list"}, "plain"], "credentials": {"client_id": "id", "client_secret": "s"}} + + masked = mask_secret_values(value, _is_secret) + + assert masked == {"api_key": MASKED_VALUE, "nested": {"api_key": MASKED_VALUE, "model": "m"}, "items": [{"api_key": MASKED_VALUE}, "plain"], "credentials": MASKED_VALUE} + assert value["nested"]["api_key"] == "sk-nested" + + +@pytest.mark.parametrize(("keep_unset", "expected"), [(False, {"api_key": MASKED_VALUE, "credentials": MASKED_VALUE}), (True, {"api_key": None, "credentials": ""})]) +def test_mask_secret_values_keeps_unset_values_only_when_asked(keep_unset: bool, expected: dict) -> None: + assert mask_secret_values({"api_key": None, "credentials": ""}, _is_secret, keep_unset=keep_unset) == expected diff --git a/backend/tests/test_memory_router.py b/backend/tests/test_memory_router.py index ab5c63c058..f1a6333b55 100644 --- a/backend/tests/test_memory_router.py +++ b/backend/tests/test_memory_router.py @@ -4,12 +4,15 @@ import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch +import pytest from _router_auth_helpers import call_unwrapped, make_authed_test_app from fastapi.testclient import TestClient from app.gateway.routers import memory from deerflow.agents.memory import MemoryConflictError, MemoryCorruptionError from deerflow.agents.memory.backends.deermem.deer_mem import DeerMem +from deerflow.config.app_config import AppConfig +from deerflow.config.memory_config import MemoryConfig def _sample_memory(facts: list[dict] | None = None) -> dict: @@ -678,6 +681,79 @@ def test_update_memory_fact_route_returns_specific_error_for_invalid_confidence( assert response.json()["detail"] == "Invalid confidence value; must be between 0 and 1." +# ── config / status ──────────────────────────────────────────────────────── + + +def _get_config_route(route: str, config: MemoryConfig): + app = make_authed_test_app() + app.include_router(memory.router) + manager = MagicMock() + manager.get_memory.return_value = _sample_memory() + with ( + patch("app.gateway.routers.memory.get_memory_config", return_value=config), + patch("app.gateway.routers.memory.get_memory_manager", return_value=manager), + TestClient(app) as client, + ): + response = client.get(route) + assert response.status_code == 200 + return response + + +@pytest.mark.parametrize(("route", "config_path"), [("/api/memory/config", ()), ("/api/memory/status", ("config",))]) +def test_memory_config_routes_mask_resolved_backend_secrets(monkeypatch, route: str, config_path: tuple[str, ...]) -> None: + """Loading resolves ``$VAR`` references, so backend keys reached these routes in plaintext.""" + monkeypatch.setenv("TEST_MEMORY_LLM_KEY", "sk-memory-llm-secret") + monkeypatch.setenv("TEST_HONCHO_KEY", "hk-honcho-secret") + raw = { + "memory": { + "enabled": True, + "manager_class": "honcho", + "backend_config": { + "base_url": "https://honcho.example", + "api_key": "$TEST_HONCHO_KEY", + "model": {"provider": "openai", "model": "gpt-4o-mini", "api_key": "$TEST_MEMORY_LLM_KEY", "default_headers": {"Authorization": "Bearer header-secret"}}, + "peers": [{"name": "deerflow", "accessToken": "list-token-secret"}], + "api_keys": {"prod": "plural-secret"}, + "credentials": {"client_id": "deerflow-client", "client_secret": "container-secret"}, + }, + } + } + config = MemoryConfig(**AppConfig.resolve_env_variables(raw)["memory"]) + + response = _get_config_route(route, config) + + for secret in ("sk-memory-llm-secret", "hk-honcho-secret", "header-secret", "list-token-secret", "plural-secret", "container-secret"): + assert secret not in response.text + body = response.json() + for key in config_path: + body = body[key] + assert body["backend_config"] == { + "base_url": "https://honcho.example", + "api_key": "***", + "model": {"provider": "openai", "model": "gpt-4o-mini", "api_key": "***", "default_headers": {"Authorization": "***"}}, + "peers": [{"name": "deerflow", "accessToken": "***"}], + # A secret-named container is masked whole: its keys can be arbitrary names. + "api_keys": "***", + "credentials": "***", + } + assert config.backend_config["model"]["api_key"] == "sk-memory-llm-secret", "the backend itself still gets the real key" + + +def test_memory_config_keeps_non_secret_and_unset_backend_values() -> None: + backend_config = { + "api_key_env": "OPENVIKING_API_KEY", + "token_counting": "tiktoken", + "max_injection_tokens": 2000, + "guaranteed_token_budget": 500, + "model": {"model": "gpt-4o-mini", "api_key": None, "base_url": ""}, + "password": "", + } + + response = _get_config_route("/api/memory/config", MemoryConfig(enabled=True, backend_config=backend_config)) + + assert response.json()["backend_config"] == backend_config + + # ── bound-owner (internal caller) ──────────────────────────────────────────