diff --git a/CHANGELOG.md b/CHANGELOG.md index ee9fa16edc..24459d5f69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1397,6 +1397,13 @@ As of 2026-10-21, the milestone includes **829 merged pull requests**. ### Fixed +- **authz:** Authenticated Gateway requests no longer read `config.yaml` and + `extensions_config.json` on the event loop. Resolving route permissions + checked both files (a `stat` and a full sha256 of each) for every request, + and reparsed `config.yaml` there after an edit, stalling other requests and + streams meanwhile. The read now runs in a worker, as the plugin + authorization path already did. ([#7557]) + - **memory:** Automatic context compaction no longer blocks the event loop while it saves the messages it is about to summarize to memory. With the mem0, Honcho or OpenViking backend, that save is a synchronous HTTP request, @@ -13107,3 +13114,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7507]: https://github.com/bytedance/deer-flow/pull/7507 [#7521]: https://github.com/bytedance/deer-flow/pull/7521 [#7530]: https://github.com/bytedance/deer-flow/pull/7530 +[#7557]: https://github.com/bytedance/deer-flow/pull/7557 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index d25a6d64d5..f667c2b842 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -1112,6 +1112,11 @@ ### 修复 +- **鉴权：** 已认证的 Gateway 请求不再在事件循环上读取 `config.yaml` 和 + `extensions_config.json`。解析路由权限时，每个请求都会检查这两个文件（各做一次 + `stat` 和完整的 sha256），编辑后还会在循环上重新解析 `config.yaml`，期间其他请求和 + 流式输出都会停顿。现在这次读取与插件鉴权路径一样在工作线程中执行。([#7557]) + - **记忆：** 自动上下文压缩在把即将被总结的消息保存到记忆时，不再阻塞事件循环。 使用 mem0、Honcho 或 OpenViking 后端时，这次保存是同步 HTTP 请求，Gateway 在记忆 服务器响应之前无法处理其他请求和流式输出。现在它与每轮结束后的记忆写入一样在工作 @@ -10463,3 +10468,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7507]: https://github.com/bytedance/deer-flow/pull/7507 [#7521]: https://github.com/bytedance/deer-flow/pull/7521 [#7530]: https://github.com/bytedance/deer-flow/pull/7530 +[#7557]: https://github.com/bytedance/deer-flow/pull/7557 diff --git a/backend/app/gateway/authz.py b/backend/app/gateway/authz.py index 0275556e5e..47a4f1eef4 100644 --- a/backend/app/gateway/authz.py +++ b/backend/app/gateway/authz.py @@ -189,6 +189,9 @@ def _get_route_authorization_config() -> AuthorizationConfig: Falls back to a disabled config when AppConfig is not available (e.g. test environments without a config.yaml), preserving legacy all-permissions behavior. + ``get_app_config()`` stats and hashes ``config.yaml`` and + ``extensions_config.json`` on every call, so async request paths use + :func:`_route_authorization_config_async`. """ from deerflow.config.app_config import get_app_config @@ -198,6 +201,11 @@ def _get_route_authorization_config() -> AuthorizationConfig: return AuthorizationConfig() +async def _route_authorization_config_async() -> AuthorizationConfig: + """Off-loop :func:`_get_route_authorization_config` for async request paths.""" + return await asyncio.to_thread(_get_route_authorization_config) + + # --- Provider cache (W1/F1) --- # Keyed by config object identity (id) so the expensive model_dump() signature # is only recomputed when get_app_config() returns a new object (hot-reload). @@ -274,8 +282,10 @@ async def resolve_route_permissions(user: User, *, is_internal: bool) -> list[st When enabled, every registered ``resource:action`` permission is evaluated independently so a provider failure affects only the route being checked. Provider instances are cached per config signature (hot-reload safe). + The config is read in a worker: ``AuthMiddleware`` calls this for every + authenticated request. """ - config = _get_route_authorization_config() + config = await _route_authorization_config_async() if config.enabled is not True: return list(_ALL_PERMISSIONS) diff --git a/backend/packages/harness/deerflow/config/AGENTS.md b/backend/packages/harness/deerflow/config/AGENTS.md index 9db9320a84..27578d820e 100644 --- a/backend/packages/harness/deerflow/config/AGENTS.md +++ b/backend/packages/harness/deerflow/config/AGENTS.md @@ -68,6 +68,9 @@ working-directory changes cannot redirect it. Failed explicit loads preserve the prior pin; reset, injection and argument-less reload clear it. Ambient reads never pin a path. Runtime-scoped overrides bypass the singleton lock; `peek_loaded_app_config()` also returns the existing snapshot without locking. +Each read stats and hashes `config.yaml` and `extensions_config.json`, so async +request paths call it through `asyncio.to_thread` (Gateway route and plugin +authorization in `app/gateway/authz.py`). **Config Hot-Reload Boundary**: Gateway dependencies route through `get_app_config()` on every request, so per-run fields like `models[*].max_tokens`, `summarization.*`, `title.*`, `memory.*`, `subagents.*`, `verification.*`, `tools[*]`, and the agent system prompt pick up `config.yaml` edits on the next message. `AppConfig` is intentionally **not** cached on `app.state` — `lifespan()` keeps a local `startup_config` variable for one-shot bootstrap work and passes it to `langgraph_runtime(app, startup_config)`. diff --git a/backend/tests/blocking_io/test_auth_middleware_route_permissions.py b/backend/tests/blocking_io/test_auth_middleware_route_permissions.py new file mode 100644 index 0000000000..7191053505 --- /dev/null +++ b/backend/tests/blocking_io/test_auth_middleware_route_permissions.py @@ -0,0 +1,118 @@ +"""Regression anchor: route permission resolution must read the config off the loop. + +``AuthMiddleware`` resolves the caller's route permissions for every +authenticated request. That reads the hot-reloaded authorization config through +``get_app_config()``, which stats and hashes ``config.yaml`` and +``extensions_config.json`` on every call and reparses them after an edit, so +calling it inline put that file IO on the event loop for every request. + +The anchor drives the real middleware against real config files, with +authorization both disabled (the default) and enabled through the built-in RBAC +provider. The RBAC case returns a permission subset that only the file on disk +declares, so the read cannot be skipped without the assertion failing. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import httpx +import pytest +from blockbuster import BlockingError +from fastapi import FastAPI, Request + +import app.gateway.authz as authz +from app.gateway.auth_middleware import AuthMiddleware +from app.gateway.authz import Permissions +from app.gateway.internal_auth import create_internal_auth_headers +from deerflow.config.app_config import reset_app_config +from deerflow.config.extensions_config import reset_extensions_config + +pytestmark = pytest.mark.asyncio + +_SANDBOX = "sandbox:\n use: deerflow.sandbox.local:LocalSandboxProvider\n" +_DISABLED = _SANDBOX + "authorization:\n enabled: false\n" +_RBAC = ( + _SANDBOX + + """authorization: + enabled: true + fail_closed: true + default_role: user + provider: + use: deerflow.authz.rbac:RbacAuthorizationProvider + config: + roles: + user: + routes: + allow: ["threads:read", "runs:read"] +""" +) + + +@pytest.fixture +def write_configs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + config_path = tmp_path / "config.yaml" + extensions_path = tmp_path / "extensions_config.json" + monkeypatch.setenv("DEER_FLOW_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path)) + # Start from an empty route provider cache and restore it afterwards. + monkeypatch.setattr(authz, "_route_provider_cache", {}) + monkeypatch.setattr(authz, "_route_provider_config_id", None) + monkeypatch.setattr(authz, "_route_provider_config_sig", None) + + def write(config_yaml: str) -> None: + config_path.write_text(config_yaml, encoding="utf-8") + extensions_path.write_text(json.dumps({"mcpServers": {}, "skills": {}}), encoding="utf-8") + + reset_app_config() + reset_extensions_config() + yield write + reset_app_config() + reset_extensions_config() + + +def _app() -> FastAPI: + app = FastAPI() + app.add_middleware(AuthMiddleware) + + @app.get("/api/probe") + async def probe(request: Request) -> list[str]: + return list(request.state.auth.permissions) + + return app + + +async def _permissions(client: httpx.AsyncClient) -> list[str]: + response = await client.get("/api/probe", headers=create_internal_auth_headers()) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.parametrize( + ("config_yaml", "expected"), + [ + pytest.param(_DISABLED, list(authz._ALL_PERMISSIONS), id="disabled"), + pytest.param(_RBAC, [Permissions.THREADS_READ, Permissions.RUNS_READ], id="rbac"), + ], +) +async def test_auth_middleware_reads_route_authorization_config_off_the_loop(write_configs, config_yaml: str, expected: list[str]) -> None: + await asyncio.to_thread(write_configs, config_yaml) + + transport = httpx.ASGITransport(app=_app()) + async with httpx.AsyncClient(transport=transport, base_url="http://gateway.test") as client: + # The first request loads the config; the second only revalidates it, + # which is the per-request cost. + assert await _permissions(client) == expected + assert await _permissions(client) == expected + + +async def test_blocking_probe_inline_route_config_read_actually_trips_the_gate(write_configs) -> None: + """Meta-check: revalidating a loaded config inline on the loop is caught by the gate.""" + await asyncio.to_thread(write_configs, _RBAC) + loaded = await asyncio.to_thread(authz._get_route_authorization_config) + assert loaded.enabled is True + + with pytest.raises(BlockingError): + authz._get_route_authorization_config() diff --git a/backend/tests/test_authorization_route_permissions.py b/backend/tests/test_authorization_route_permissions.py index be8691ac4b..98d63a817d 100644 --- a/backend/tests/test_authorization_route_permissions.py +++ b/backend/tests/test_authorization_route_permissions.py @@ -1,5 +1,6 @@ """Route-level authorization tests for the Gateway permission decorators.""" +import threading from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -19,7 +20,9 @@ from app.gateway.authz import ( from app.gateway.routers import runs, scheduled_tasks from deerflow.authz.provider import AuthzDecision, AuthzReason from deerflow.authz.rbac import RbacAuthorizationProvider +from deerflow.config.app_config import AppConfig, pop_current_app_config, push_current_app_config from deerflow.config.authorization_config import AuthorizationConfig, AuthorizationProviderConfig +from deerflow.config.sandbox_config import SandboxConfig class _RecordingProvider: @@ -251,6 +254,44 @@ async def test_route_permissions_use_builtin_rbac_route_policy(monkeypatch): assert permissions == [Permissions.THREADS_READ, Permissions.RUNS_READ] +@pytest.mark.asyncio +async def test_route_permissions_read_config_in_a_worker_with_the_caller_context(monkeypatch): + """The config read runs off the loop thread and still sees a runtime config override.""" + import app.gateway.authz as authz_module + + monkeypatch.setattr(authz_module, "_route_provider_cache", {}) + monkeypatch.setattr(authz_module, "_route_provider_config_id", None) + monkeypatch.setattr(authz_module, "_route_provider_config_sig", None) + read_threads: list[int] = [] + read_config = authz_module._get_route_authorization_config + + def recording_read(): + read_threads.append(threading.get_ident()) + return read_config() + + monkeypatch.setattr(authz_module, "_get_route_authorization_config", recording_read) + override = AppConfig( + sandbox=SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider"), + authorization=AuthorizationConfig( + enabled=True, + default_role="user", + provider=AuthorizationProviderConfig( + use="deerflow.authz.rbac:RbacAuthorizationProvider", + config={"roles": {"user": {"routes": {"allow": [Permissions.THREADS_READ]}}}}, + ), + ), + ) + + push_current_app_config(override) + try: + permissions = await resolve_route_permissions(_user(), is_internal=False) + finally: + pop_current_app_config() + + assert read_threads and threading.get_ident() not in read_threads + assert permissions == [Permissions.THREADS_READ] + + @pytest.mark.asyncio async def test_authenticate_uses_route_permission_resolution(monkeypatch): user = User(email="route-authz@example.com", password_hash="hash")