From b4b2a68ee5a57e6aba228a65d3db008c06bec59b Mon Sep 17 00:00:00 2001 From: Hyeonsang Cho Date: Fri, 30 Oct 2026 15:53:33 +0900 Subject: [PATCH] fix(memory): mask backend secrets in the memory config responses (#7665) * fix(memory): mask backend secrets in the memory config responses GET /api/memory/config and /api/memory/status returned memory.backend_config as loaded. Config loading has already resolved $VAR references by then, so a DeerMem extraction model api_key or a Honcho api_key reached every caller with memory:read in plaintext, which with authorization disabled (the default) is every signed-in user. MemoryConfigResponse now masks credential-named values (api_key, *_token, password, authorization, ...) at any depth as "***". The key is matched on its end, so knobs such as token_counting and guaranteed_token_budget and the api_key_env variable name stay readable, and unset values stay null or empty. The loaded config the backend uses is not changed. * docs(changelog): reference #7665 for the memory config secret masking fix * refactor(gateway): share config secret masking between MCP and memory The memory config masking forked the MCP router's key normalization and recursive masker. Both now use app/gateway/secret_masking.py, and each router keeps its own key rule: MCP matches credential terms anywhere in the key, memory matches them at the end and keeps unset values. MCP masking is unchanged. The memory rule now also covers plural and cookie keys (api_keys, secrets, passwords, cookies). Plural tokens stays out because max_injection_tokens is a DeerMem knob. A secret-named container is still masked whole, since its keys can be arbitrary names; the helper docstring and a test now say so. * docs(changelog): move the #7665 entry to Unreleased --- CHANGELOG.md | 12 ++++ CHANGELOG_zh.md | 9 +++ backend/README.md | 4 +- backend/README_zh.md | 4 +- backend/app/gateway/routers/mcp.py | 17 ++--- backend/app/gateway/routers/memory.py | 22 +++++- backend/app/gateway/secret_masking.py | 30 ++++++++ backend/tests/test_gateway_secret_masking.py | 31 ++++++++ backend/tests/test_memory_router.py | 76 ++++++++++++++++++++ 9 files changed, 187 insertions(+), 18 deletions(-) create mode 100644 backend/app/gateway/secret_masking.py create mode 100644 backend/tests/test_gateway_secret_masking.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b10cb006..5a0f5c0436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 This section accumulates work toward the **2.3.0** milestone ([2.3.0](https://github.com/bytedance/deer-flow/milestone/4)). +### Fixed + +- **memory:** `GET /api/memory/config` and `/api/memory/status` now mask + credential values in `backend_config` (`api_key`, tokens, passwords, + `Authorization` headers, at any depth) as `***`. They returned the config as + loaded, with `$VAR` references already resolved, so a DeerMem extraction + model key or a Honcho `api_key` reached every caller with `memory:read` in + plaintext, which with authorization disabled (the default) is every + signed-in user. Non-secret knobs, `api_key_env` names and unset values are + unchanged. ([#7665]) + ## [2.2.0] — 2026-10-29 This section accumulates work toward the **2.2.0** milestone @@ -15915,3 +15926,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7643]: https://github.com/bytedance/deer-flow/pull/7643 [#7651]: https://github.com/bytedance/deer-flow/pull/7651 [#7659]: https://github.com/bytedance/deer-flow/pull/7659 +[#7665]: https://github.com/bytedance/deer-flow/pull/7665 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 85717a9887..7f6d1fc9b1 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -12,6 +12,14 @@ 本节累积面向 **2.3.0** 里程碑（[2.3.0](https://github.com/bytedance/deer-flow/milestone/4)）的工作。 截至 2026-10-28，该里程碑已合并 **1068 个拉取请求**。 +### 修复 + +- **记忆：** `GET /api/memory/config` 与 `/api/memory/status` 现在会把 `backend_config` + 中的凭据值（`api_key`、token、密码、`Authorization` 请求头，任意层级）遮蔽为 `***`。 + 此前它们按加载结果原样返回配置，而 `$VAR` 引用此时已被解析，因此 DeerMem 提取模型的密钥 + 或 Honcho 的 `api_key` 会以明文返回给所有拥有 `memory:read` 的调用方；在默认关闭授权时， + 这就是所有已登录用户。非机密配置项、`api_key_env` 变量名和未设置的值保持不变。([#7665]) + ## [2.2.0] — 2026-10-29 本节累积面向 **2.2.0** 里程碑（[2.2.0](https://github.com/bytedance/deer-flow/milestone/3)）的工作。 @@ -12686,3 +12694,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7643]: https://github.com/bytedance/deer-flow/pull/7643 [#7651]: https://github.com/bytedance/deer-flow/pull/7651 [#7659]: https://github.com/bytedance/deer-flow/pull/7659 +[#7665]: https://github.com/bytedance/deer-flow/pull/7665 diff --git a/backend/README.md b/backend/README.md index 263d7aa768..0d725809d3 100644 --- a/backend/README.md +++ b/backend/README.md @@ -156,8 +156,8 @@ to a boundary (SQLite) or causing the search to fail (PostgreSQL). | `POST /api/skills/install` | Install skill from `.skill` archive | | `GET /api/memory` | Retrieve memory data | | `POST /api/memory/reload` | Force memory reload | -| `GET /api/memory/config` | Memory configuration | -| `GET /api/memory/status` | Combined config + data | +| `GET /api/memory/config` | Memory configuration; credential values in `backend_config` are masked | +| `GET /api/memory/status` | Combined config (masked the same way) + data | | `GET /api/threads/{id}/runs/{run_id}/events` | Debug/audit events for one run; filter `event_types=context:memory` for effective memory identity | | `POST /api/threads/{id}/uploads` | Upload files (optional PDF/PPT/Excel/Word-to-Markdown conversion, rejects directory paths, auto-renames duplicate filenames in one request) | | `GET /api/threads/{id}/uploads/list` | List uploaded files | diff --git a/backend/README_zh.md b/backend/README_zh.md index 7335255eca..687775f704 100644 --- a/backend/README_zh.md +++ b/backend/README_zh.md @@ -126,8 +126,8 @@ FastAPI 应用程序，为前端集成提供 REST 接口： | `POST /api/skills/install` | 从 `.skill` 归档文件安装技能 | | `GET /api/memory` | 获取记忆数据 | | `POST /api/memory/reload` | 强制重新加载记忆 | -| `GET /api/memory/config` | 获取记忆配置 | -| `GET /api/memory/status` | 获取组合后的配置和数据状态 | +| `GET /api/memory/config` | 获取记忆配置；`backend_config` 中的凭据值会被遮蔽 | +| `GET /api/memory/status` | 获取组合后的配置（同样遮蔽凭据）和数据状态 | | `GET /api/threads/{id}/runs/{run_id}/events` | 获取某次运行的调试或审计事件；可使用 `event_types=context:memory` 筛选实际记忆标识 | | `POST /api/threads/{id}/uploads` | 上传文件（可选地将 PDF、PPT、Excel、Word 转换为 Markdown；拒绝目录路径；自动重命名单次请求中的重复文件名） | | `GET /api/threads/{id}/uploads/list` | 列出已上传的文件 | diff --git a/backend/app/gateway/routers/mcp.py b/backend/app/gateway/routers/mcp.py index 4ffc0cb336..94506f0e0e 100644 --- a/backend/app/gateway/routers/mcp.py +++ b/backend/app/gateway/routers/mcp.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_valida from app.gateway.deps import require_admin_user from app.gateway.mcp_oauth_tokens import forget_unconfigured_mcp_oauth_tokens from app.gateway.persistent_writes import run_drained_write +from app.gateway.secret_masking import MASKED_VALUE, mask_secret_values, normalize_config_key from deerflow.config.extensions_config import ( ExtensionsConfig, McpRoutingConfig, @@ -552,29 +553,19 @@ class McpCacheResetResponse(BaseModel): message: str = Field(description="Human-readable reset status") -_MASKED_VALUE = "***" +_MASKED_VALUE = MASKED_VALUE _SENSITIVE_EXTRA_KEY_RE = re.compile( r"(^|_)(api_key|apikey|access_key|private_key|client_secret|secret|token|password|passwd|credential|credentials|authorization|bearer)(_|$)", re.IGNORECASE, ) -def _normalize_config_key(key: str) -> str: - with_boundaries = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", key) - with_boundaries = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", with_boundaries) - return re.sub(r"[^a-z0-9]+", "_", with_boundaries.lower()).strip("_") - - def _is_sensitive_extra_key(key: str) -> bool: - return bool(_SENSITIVE_EXTRA_KEY_RE.search(_normalize_config_key(key))) + return bool(_SENSITIVE_EXTRA_KEY_RE.search(normalize_config_key(key))) def _mask_sensitive_extra_value(value: Any) -> Any: - if isinstance(value, dict): - return {key: _MASKED_VALUE if _is_sensitive_extra_key(str(key)) else _mask_sensitive_extra_value(nested) for key, nested in value.items()} - if isinstance(value, list): - return [_mask_sensitive_extra_value(item) for item in value] - return value + return mask_secret_values(value, _is_sensitive_extra_key) def _contains_masked_sensitive_extra_value(key: str, value: Any) -> bool: diff --git a/backend/app/gateway/routers/memory.py b/backend/app/gateway/routers/memory.py index cb42d98fa2..80f3ead0b5 100644 --- a/backend/app/gateway/routers/memory.py +++ b/backend/app/gateway/routers/memory.py @@ -1,6 +1,7 @@ """Memory API router for retrieving and managing global memory data.""" import asyncio +import re from typing import Any, Literal from fastapi import APIRouter, HTTPException, Request @@ -9,6 +10,7 @@ from pydantic import BaseModel, Field, field_validator from app.gateway.authz import require_permission from app.gateway.internal_auth import get_trusted_internal_owner_user_id from app.gateway.persistent_writes import run_drained_write +from app.gateway.secret_masking import mask_secret_values, normalize_config_key from deerflow.agents.memory import MemoryConflictError, MemoryCorruptionError, MemoryManager, get_memory_manager from deerflow.config.agents_config import AGENT_NAME_PATTERN from deerflow.config.memory_config import get_memory_config @@ -234,6 +236,18 @@ class FactPatchRequest(BaseModel): confidence: float | None = Field(default=None, ge=0.0, le=1.0, description="Confidence score (0-1)") +# Anchored at the end of the key: DeerMem knobs such as ``token_counting`` and +# ``guaranteed_token_budget`` are not secrets, and ``api_key_env`` names an +# environment variable rather than holding the key. This is a denylist: plural +# ``tokens`` stays out because ``max_injection_tokens`` is a knob, and bare +# ``key``, ``auth`` or ``session`` are too generic to mask. +_SECRET_CONFIG_KEY_RE = re.compile(r"(^|_)(api_?keys?|access_keys?|secret_keys?|private_keys?|auth_keys?|secrets?|token|passwords?|passwds?|credentials?|authorization|bearer|cookies?)$") + + +def _is_secret_config_key(key: str) -> bool: + return bool(_SECRET_CONFIG_KEY_RE.search(normalize_config_key(key))) + + class MemoryConfigResponse(BaseModel): """Response model for memory configuration.""" @@ -242,7 +256,13 @@ class MemoryConfigResponse(BaseModel): injection_enabled: bool = Field(..., description="Whether memory is injected into the system prompt (call-site gate).") shutdown_flush_timeout_seconds: float = Field(..., description="Hard budget (s) to drain pending memory updates on Gateway graceful shutdown; must fit inside the pod's K8s terminationGracePeriodSeconds.") manager_class: str = Field(..., description="Active memory backend selector (backend name or dotted path).") - backend_config: dict = Field(..., description="Backend-private config (self-interpreted by the backend).") + backend_config: dict = Field(..., description="Backend-private config (self-interpreted by the backend); credential values such as api_key are masked as ***.") + + @field_validator("backend_config") + @classmethod + def _mask_backend_secrets(cls, value: dict) -> dict: + """Config loading already resolved ``$VAR`` references, so keys would be plaintext.""" + return mask_secret_values(value, _is_secret_config_key, keep_unset=True) class MemoryStatusResponse(BaseModel): diff --git a/backend/app/gateway/secret_masking.py b/backend/app/gateway/secret_masking.py new file mode 100644 index 0000000000..494711ec1e --- /dev/null +++ b/backend/app/gateway/secret_masking.py @@ -0,0 +1,30 @@ +"""Shared masking for credential values that Gateway routes echo back from config.""" + +import re +from collections.abc import Callable +from typing import Any + +MASKED_VALUE = "***" + + +def normalize_config_key(key: str) -> str: + """Snake-case a config key so ``apiKey``, ``API-Key`` and ``api_key`` compare equal.""" + with_boundaries = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", key) + with_boundaries = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", with_boundaries) + return re.sub(r"[^a-z0-9]+", "_", with_boundaries.lower()).strip("_") + + +def mask_secret_values(value: Any, is_secret_key: Callable[[str], bool], *, keep_unset: bool = False) -> Any: + """Return a copy of ``value`` with every value under a secret-named key masked, at any depth. + + A secret-named key masks its whole value, containers included: a + ``credentials`` mapping can hold keys under arbitrary names, so masking only + its secret-named leaves could pass one through. With ``keep_unset``, a + ``None`` or ``""`` value stays as it is, so a response still shows that no + secret is configured. + """ + if isinstance(value, dict): + return {key: MASKED_VALUE if is_secret_key(str(key)) and not (keep_unset and nested in (None, "")) else mask_secret_values(nested, is_secret_key, keep_unset=keep_unset) for key, nested in value.items()} + if isinstance(value, list): + return [mask_secret_values(item, is_secret_key, keep_unset=keep_unset) for item in value] + return value diff --git a/backend/tests/test_gateway_secret_masking.py b/backend/tests/test_gateway_secret_masking.py new file mode 100644 index 0000000000..49c483a537 --- /dev/null +++ b/backend/tests/test_gateway_secret_masking.py @@ -0,0 +1,31 @@ +"""Shared credential masking used by the MCP and memory config routes.""" + +import pytest + +from app.gateway.secret_masking import MASKED_VALUE, mask_secret_values, normalize_config_key + + +@pytest.mark.parametrize( + ("key", "normalized"), + [("api_key", "api_key"), ("apiKey", "api_key"), ("APIKey", "api_key"), ("API-Key", "api_key"), ("accessToken", "access_token"), ("X-Api-Key", "x_api_key")], +) +def test_normalize_config_key_snake_cases_spellings(key: str, normalized: str) -> None: + assert normalize_config_key(key) == normalized + + +def _is_secret(key: str) -> bool: + return key in {"api_key", "credentials"} + + +def test_mask_secret_values_masks_at_any_depth_without_mutating() -> None: + value = {"api_key": "sk-top", "nested": {"api_key": "sk-nested", "model": "m"}, "items": [{"api_key": "sk-list"}, "plain"], "credentials": {"client_id": "id", "client_secret": "s"}} + + masked = mask_secret_values(value, _is_secret) + + assert masked == {"api_key": MASKED_VALUE, "nested": {"api_key": MASKED_VALUE, "model": "m"}, "items": [{"api_key": MASKED_VALUE}, "plain"], "credentials": MASKED_VALUE} + assert value["nested"]["api_key"] == "sk-nested" + + +@pytest.mark.parametrize(("keep_unset", "expected"), [(False, {"api_key": MASKED_VALUE, "credentials": MASKED_VALUE}), (True, {"api_key": None, "credentials": ""})]) +def test_mask_secret_values_keeps_unset_values_only_when_asked(keep_unset: bool, expected: dict) -> None: + assert mask_secret_values({"api_key": None, "credentials": ""}, _is_secret, keep_unset=keep_unset) == expected diff --git a/backend/tests/test_memory_router.py b/backend/tests/test_memory_router.py index ab5c63c058..f1a6333b55 100644 --- a/backend/tests/test_memory_router.py +++ b/backend/tests/test_memory_router.py @@ -4,12 +4,15 @@ import threading from types import SimpleNamespace from unittest.mock import MagicMock, patch +import pytest from _router_auth_helpers import call_unwrapped, make_authed_test_app from fastapi.testclient import TestClient from app.gateway.routers import memory from deerflow.agents.memory import MemoryConflictError, MemoryCorruptionError from deerflow.agents.memory.backends.deermem.deer_mem import DeerMem +from deerflow.config.app_config import AppConfig +from deerflow.config.memory_config import MemoryConfig def _sample_memory(facts: list[dict] | None = None) -> dict: @@ -678,6 +681,79 @@ def test_update_memory_fact_route_returns_specific_error_for_invalid_confidence( assert response.json()["detail"] == "Invalid confidence value; must be between 0 and 1." +# ── config / status ──────────────────────────────────────────────────────── + + +def _get_config_route(route: str, config: MemoryConfig): + app = make_authed_test_app() + app.include_router(memory.router) + manager = MagicMock() + manager.get_memory.return_value = _sample_memory() + with ( + patch("app.gateway.routers.memory.get_memory_config", return_value=config), + patch("app.gateway.routers.memory.get_memory_manager", return_value=manager), + TestClient(app) as client, + ): + response = client.get(route) + assert response.status_code == 200 + return response + + +@pytest.mark.parametrize(("route", "config_path"), [("/api/memory/config", ()), ("/api/memory/status", ("config",))]) +def test_memory_config_routes_mask_resolved_backend_secrets(monkeypatch, route: str, config_path: tuple[str, ...]) -> None: + """Loading resolves ``$VAR`` references, so backend keys reached these routes in plaintext.""" + monkeypatch.setenv("TEST_MEMORY_LLM_KEY", "sk-memory-llm-secret") + monkeypatch.setenv("TEST_HONCHO_KEY", "hk-honcho-secret") + raw = { + "memory": { + "enabled": True, + "manager_class": "honcho", + "backend_config": { + "base_url": "https://honcho.example", + "api_key": "$TEST_HONCHO_KEY", + "model": {"provider": "openai", "model": "gpt-4o-mini", "api_key": "$TEST_MEMORY_LLM_KEY", "default_headers": {"Authorization": "Bearer header-secret"}}, + "peers": [{"name": "deerflow", "accessToken": "list-token-secret"}], + "api_keys": {"prod": "plural-secret"}, + "credentials": {"client_id": "deerflow-client", "client_secret": "container-secret"}, + }, + } + } + config = MemoryConfig(**AppConfig.resolve_env_variables(raw)["memory"]) + + response = _get_config_route(route, config) + + for secret in ("sk-memory-llm-secret", "hk-honcho-secret", "header-secret", "list-token-secret", "plural-secret", "container-secret"): + assert secret not in response.text + body = response.json() + for key in config_path: + body = body[key] + assert body["backend_config"] == { + "base_url": "https://honcho.example", + "api_key": "***", + "model": {"provider": "openai", "model": "gpt-4o-mini", "api_key": "***", "default_headers": {"Authorization": "***"}}, + "peers": [{"name": "deerflow", "accessToken": "***"}], + # A secret-named container is masked whole: its keys can be arbitrary names. + "api_keys": "***", + "credentials": "***", + } + assert config.backend_config["model"]["api_key"] == "sk-memory-llm-secret", "the backend itself still gets the real key" + + +def test_memory_config_keeps_non_secret_and_unset_backend_values() -> None: + backend_config = { + "api_key_env": "OPENVIKING_API_KEY", + "token_counting": "tiktoken", + "max_injection_tokens": 2000, + "guaranteed_token_budget": 500, + "model": {"model": "gpt-4o-mini", "api_key": None, "base_url": ""}, + "password": "", + } + + response = _get_config_route("/api/memory/config", MemoryConfig(enabled=True, backend_config=backend_config)) + + assert response.json()["backend_config"] == backend_config + + # ── bound-owner (internal caller) ──────────────────────────────────────────