From 6c94ecc6cbf1fa05564dbcd21fa977fe81a633a8 Mon Sep 17 00:00:00 2001 From: Hyeonsang Cho Date: Sun, 8 Nov 2026 09:44:56 +0900 Subject: [PATCH] fix(community): strip provider API keys before they reach a header (#7981) * fix(community): strip provider API keys before they reach a header GroundRoute (from GROUNDROUTE_API_KEY), InfoQuest, Tavily, Exa, Jina, Firecrawl, fastCRW and Crawl4AI put the configured or environment key into an HTTP header as read. A key read from a file keeps its trailing newline (a Kubernetes Secret created with --from-file and mounted through envFrom), and the transport refuses it with an error that quotes the whole header: h11's "Illegal header value b'Bearer \n'" or requests' "... in header value: '\n'". The providers returned that message as the tool result, so the key reached the model and the checkpointed conversation, and GroundRoute, InfoQuest, Jina and Crawl4AI also wrote it to the error log. Resolve each key through provider_credentials.resolve_api_key, as the other bundled providers already strip theirs. Tavily, Exa and Firecrawl now read their environment fallback themselves instead of leaving it to the SDK, which would read the raw value again. A blank configured key now falls back to the environment, as it already does for GroundRoute and the stripping providers. * docs(changelog): reference #7981 for the provider API key fix * fix(community): keep fastCRW from sending the Firecrawl key fastCRW builds the Firecrawl SDK client, and the SDK reads FIRECRAWL_API_KEY whenever api_key is None. Once a blank configured key resolved to None, a fastCRW config with api_key: "" and no CRW_API_KEY sent the Firecrawl key to the fastCRW endpoint. With no api_key configured at all, main already did the same. Pass "" when fastCRW resolves no key of its own; the SDK falls back only on None and sends no Authorization header for an empty key. A loopback test pins the empty, blank and absent cases with FIRECRAWL_API_KEY set. * fix(community): keep a blank environment key away from the SDKs Tavily, Exa and Firecrawl resolved their environment fallback themselves, but passed None when it was blank, and each SDK then re-read the same variable unstripped. Tavily and Firecrawl now pass "", which the SDKs treat as a missing key; Exa sends "" as is, so it fails before any request instead. The fastCRW comment records that only the Firecrawl SDK's v2 client honours "", and the fastCRW regression test now covers web_fetch as well as web_search, so a move to the v1 client would fail it. --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 15 + CHANGELOG_zh.md | 12 + .../deerflow/community/crawl4ai/tools.py | 3 +- .../harness/deerflow/community/exa/tools.py | 8 +- .../deerflow/community/fastcrw/tools.py | 11 +- .../deerflow/community/firecrawl/tools.py | 8 +- .../deerflow/community/groundroute/AGENTS.md | 3 +- .../deerflow/community/groundroute/tools.py | 7 +- .../community/infoquest/infoquest_client.py | 8 +- .../deerflow/community/jina_ai/jina_client.py | 7 +- .../community/provider_credentials.py | 8 +- .../deerflow/community/tavily/AGENTS.md | 5 +- .../deerflow/community/tavily/tools.py | 10 +- backend/tests/test_firecrawl_tools.py | 2 +- .../tests/test_provider_api_key_whitespace.py | 260 ++++++++++++++++++ backend/tests/test_tavily_tools.py | 6 +- 16 files changed, 347 insertions(+), 26 deletions(-) create mode 100644 backend/tests/test_provider_api_key_whitespace.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d986ac29ac..f2a477ba00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,20 @@ This section accumulates work toward the **2.3.0** milestone ### Fixed +- **community:** A search or fetch API key that ends in a newline no longer + leaks into the conversation. GroundRoute (`GROUNDROUTE_API_KEY`), InfoQuest, + Tavily, Exa, Jina, Firecrawl, fastCRW and Crawl4AI put the key into an HTTP + header as read, so a key from a file (a Kubernetes Secret created with + `--from-file`) made the transport fail with `Illegal header value b'Bearer + \n'`. That message, key included, became the tool result the model and + the conversation history kept, and four of them also logged it. Keys now go + through `provider_credentials.resolve_api_key` like the other providers', and + a blank configured key falls back to the environment for Tavily, Exa, + Firecrawl and fastCRW. fastCRW without a key of its own no longer lets the + Firecrawl SDK send `FIRECRAWL_API_KEY` to the fastCRW endpoint, and a blank + `TAVILY_API_KEY`, `EXA_API_KEY` or `FIRECRAWL_API_KEY` now counts as missing + instead of reaching the SDK as is. ([#7981]) + - **docker:** Images built from a checkout no longer contain its local configuration. `.dockerignore` did not exclude `config.yaml`, `config.yaml.bak`, `extensions_config.json` or `mcp_config.json`, so a @@ -16086,3 +16100,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7907]: https://github.com/bytedance/deer-flow/pull/7907 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 [#7958]: https://github.com/bytedance/deer-flow/pull/7958 +[#7981]: https://github.com/bytedance/deer-flow/pull/7981 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index fdb21f2db0..ea09422c7a 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -23,6 +23,17 @@ ### 修复 +- **社区工具：** 以换行结尾的搜索或抓取 API 密钥不再泄露到对话中。GroundRoute + （`GROUNDROUTE_API_KEY`）、InfoQuest、Tavily、Exa、Jina、Firecrawl、fastCRW 和 + Crawl4AI 此前把读取到的密钥原样写入 HTTP 请求头；从文件读取的密钥（例如用 + `--from-file` 创建的 Kubernetes Secret）会让传输层报出 `Illegal header value + b'Bearer \n'`，这条包含完整密钥的消息成为模型和对话历史保存的工具结果， + 其中四个提供方还会把它写入日志。现在这些密钥与其他提供方一样经过 + `provider_credentials.resolve_api_key` 处理；Tavily、Exa、Firecrawl 和 fastCRW + 的配置密钥为空时会回退到环境变量。fastCRW 没有自己的密钥时，不再让 Firecrawl SDK + 把 `FIRECRAWL_API_KEY` 发送到 fastCRW 端点；`TAVILY_API_KEY`、`EXA_API_KEY` 或 + `FIRECRAWL_API_KEY` 只含空白时按未设置处理，不再原样交给 SDK。([#7981]) + - **docker：** 从本地 checkout 构建的镜像不再包含本地配置。`.dockerignore` 此前没有排除 `config.yaml`、`config.yaml.bak`、`extensions_config.json` 和 `mcp_config.json`，因此配置放在旧位置 `backend/` 时，`COPY backend ./backend` @@ -12796,3 +12807,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 [#7958]: https://github.com/bytedance/deer-flow/pull/7958 +[#7981]: https://github.com/bytedance/deer-flow/pull/7981 diff --git a/backend/packages/harness/deerflow/community/crawl4ai/tools.py b/backend/packages/harness/deerflow/community/crawl4ai/tools.py index 416bc8ad4d..e718d29d6c 100644 --- a/backend/packages/harness/deerflow/community/crawl4ai/tools.py +++ b/backend/packages/harness/deerflow/community/crawl4ai/tools.py @@ -3,6 +3,7 @@ import logging from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config @@ -82,7 +83,7 @@ def _build_client(cfg: dict | None) -> Crawl4AiClient: timeout_s: float = float(DEFAULT_TIMEOUT_S) if cfg is not None: base_url = cfg.get("base_url", base_url) - token = cfg.get("token", token) + token = resolve_api_key(cfg.get("token")) or "" timeout_s = _coerce_timeout(cfg.get("timeout"), DEFAULT_TIMEOUT_S) return Crawl4AiClient(base_url=base_url, token=token, timeout_s=timeout_s) diff --git a/backend/packages/harness/deerflow/community/exa/tools.py b/backend/packages/harness/deerflow/community/exa/tools.py index 9ac4f0899b..72b3f7375f 100644 --- a/backend/packages/harness/deerflow/community/exa/tools.py +++ b/backend/packages/harness/deerflow/community/exa/tools.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse from exa_py import Exa from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import safe_public_result_url, validate_delegated_backend_url from deerflow.config import get_app_config @@ -82,7 +83,12 @@ def _get_exa_client(tool_name: str = "web_search") -> Exa: def _build_exa_client(extra: dict, base_url: str | None) -> Exa: - kwargs = {"api_key": extra.get("api_key")} + api_key = resolve_api_key(extra.get("api_key")) or resolve_api_key(os.getenv("EXA_API_KEY")) + # The SDK re-reads EXA_API_KEY unstripped when api_key is None and sends "" as + # is, so a missing or blank key fails here, before any request. + if api_key is None: + raise ValueError("Exa API key must be set in the tool config or EXA_API_KEY") + kwargs = {"api_key": api_key} if base_url: kwargs["base_url"] = base_url return Exa(**kwargs) diff --git a/backend/packages/harness/deerflow/community/fastcrw/tools.py b/backend/packages/harness/deerflow/community/fastcrw/tools.py index 68130a37de..af5a9ca362 100644 --- a/backend/packages/harness/deerflow/community/fastcrw/tools.py +++ b/backend/packages/harness/deerflow/community/fastcrw/tools.py @@ -5,6 +5,7 @@ import os from firecrawl import FirecrawlApp from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.search_max_results import DEFAULT_MAX_RESULTS, coerce_max_results from deerflow.community.url_safety import safe_public_result_url, validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config @@ -33,14 +34,18 @@ def _get_fastcrw_client( """ if cfg is None: cfg = _get_tool_config_extra(tool_name) - api_key = cfg.get("api_key") + api_key = resolve_api_key(cfg.get("api_key")) if base_url is None: base_url = cfg.get("base_url") if api_key is None: - api_key = os.getenv("CRW_API_KEY") + api_key = resolve_api_key(os.getenv("CRW_API_KEY")) if base_url is None: base_url = os.getenv("CRW_API_URL", DEFAULT_BASE_URL) - return FirecrawlApp(api_key=api_key, api_url=base_url) # type: ignore[arg-type] + # The Firecrawl SDK reads FIRECRAWL_API_KEY when api_key is None; "" stops it + # sending another provider's key to the fastCRW endpoint. Only the SDK's v2 + # client, which serves search and scrape, honours "": its v1 client still reads + # FIRECRAWL_API_KEY, so never call ``client.v1`` here. + return FirecrawlApp(api_key=api_key or "", api_url=base_url) # type: ignore[arg-type] def _resolve_fastcrw_base_url(cfg: dict | None) -> str: diff --git a/backend/packages/harness/deerflow/community/firecrawl/tools.py b/backend/packages/harness/deerflow/community/firecrawl/tools.py index 11a1879edf..5b6a8ab8d9 100644 --- a/backend/packages/harness/deerflow/community/firecrawl/tools.py +++ b/backend/packages/harness/deerflow/community/firecrawl/tools.py @@ -2,10 +2,12 @@ import asyncio import inspect import json import logging +import os from firecrawl import AsyncFirecrawlApp from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.search_max_results import DEFAULT_MAX_RESULTS, coerce_max_results from deerflow.community.url_safety import safe_public_result_url, validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config @@ -83,10 +85,12 @@ def _get_firecrawl_client( """ if cfg is None: cfg = _get_tool_config_extra(tool_name) - api_key = cfg.get("api_key") + api_key = resolve_api_key(cfg.get("api_key")) or resolve_api_key(os.getenv("FIRECRAWL_API_KEY")) if api_url is None: api_url = cfg.get("base_url") - kwargs = {"api_key": api_key} + # The SDK re-reads FIRECRAWL_API_KEY unstripped when api_key is None; "" makes + # a blank value act as a missing key. + kwargs = {"api_key": api_key or ""} if api_url: kwargs["api_url"] = api_url return AsyncFirecrawlApp(**kwargs) # type: ignore[arg-type] diff --git a/backend/packages/harness/deerflow/community/groundroute/AGENTS.md b/backend/packages/harness/deerflow/community/groundroute/AGENTS.md index ae74289dc9..94b45c0bc6 100644 --- a/backend/packages/harness/deerflow/community/groundroute/AGENTS.md +++ b/backend/packages/harness/deerflow/community/groundroute/AGENTS.md @@ -25,7 +25,8 @@ would satisfy the thread assertion; the anchor records which client factory the provider reached for. Both tools resolve the key and `max_results` through `_resolve_tool_settings`, one -config read per call. Do not add a second seam around that read: a wrapper with no +config read per call. The config key and `GROUNDROUTE_API_KEY` both pass through +`provider_credentials.resolve_api_key`. Do not add a second seam around that read: a wrapper with no production caller can drift from the real call path. `web_search` returns the structured `{"error": ..., "query": ...}` response; `web_fetch` diff --git a/backend/packages/harness/deerflow/community/groundroute/tools.py b/backend/packages/harness/deerflow/community/groundroute/tools.py index b3522634fe..6100dc99f1 100644 --- a/backend/packages/harness/deerflow/community/groundroute/tools.py +++ b/backend/packages/harness/deerflow/community/groundroute/tools.py @@ -24,6 +24,7 @@ import os import httpx from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import safe_public_result_url from deerflow.config import get_app_config @@ -56,11 +57,9 @@ def _resolve_tool_settings(tool_name: str) -> tuple[str | None, object]: if tool_config is not None: extra = tool_config.model_extra or {} max_results = extra.get("max_results") - candidate = extra.get("api_key") - if isinstance(candidate, str) and candidate.strip(): - api_key = candidate.strip() + api_key = resolve_api_key(extra.get("api_key")) if api_key is None: - api_key = os.getenv("GROUNDROUTE_API_KEY") + api_key = resolve_api_key(os.getenv("GROUNDROUTE_API_KEY")) return api_key, max_results diff --git a/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py b/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py index d7f68e3450..9700d619b9 100644 --- a/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py +++ b/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py @@ -12,6 +12,7 @@ from typing import Any import httpx +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import safe_public_result_url as _safe_public_url logger = logging.getLogger(__name__) @@ -43,7 +44,7 @@ class InfoQuestClient: self.search_time_range = search_time_range self.image_search_time_range = image_search_time_range self.image_size = image_size - self.api_key_set = bool(os.getenv("INFOQUEST_API_KEY")) + self.api_key_set = resolve_api_key(os.getenv("INFOQUEST_API_KEY")) is not None if logger.isEnabledFor(logging.DEBUG): config_details = ( f"\n📋 Configuration Details:\n" @@ -147,8 +148,9 @@ class InfoQuestClient: } # Add API key if available - if os.getenv("INFOQUEST_API_KEY"): - headers["Authorization"] = f"Bearer {os.getenv('INFOQUEST_API_KEY')}" + api_key = resolve_api_key(os.getenv("INFOQUEST_API_KEY")) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" logger.debug("API key added to request headers") else: logger.warning("InfoQuest API key is not set. Provide your own key for authentication.") diff --git a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py index c4fefc85ab..039dad7434 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py +++ b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py @@ -11,6 +11,8 @@ from email.utils import parsedate_to_datetime import httpx +from deerflow.community.provider_credentials import resolve_api_key + from .request_admission import JinaAdmissionError, get_admission logger = logging.getLogger(__name__) @@ -162,8 +164,9 @@ class JinaClient: "X-Return-Format": return_format, "X-Timeout": str(timeout), } - if os.getenv("JINA_API_KEY"): - headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}" + api_key = resolve_api_key(os.getenv("JINA_API_KEY")) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" elif not _api_key_warned: _api_key_warned = True logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.") diff --git a/backend/packages/harness/deerflow/community/provider_credentials.py b/backend/packages/harness/deerflow/community/provider_credentials.py index 0533a8fe23..3c74e6da3b 100644 --- a/backend/packages/harness/deerflow/community/provider_credentials.py +++ b/backend/packages/harness/deerflow/community/provider_credentials.py @@ -7,6 +7,12 @@ key is scrubbed out of text before it reaches a log line or a tool error, and why a configured base URL must not carry userinfo. Two owners for one rule means a fix to one provider can silently leave the other one leaking. +Search and fetch providers that read a key from config or the environment +resolve it through ``resolve_api_key`` too. A key read from a file keeps its +trailing newline, and putting that raw value into a header makes the HTTP +transport refuse it with an error that quotes the whole header, key included +(``tests/test_provider_api_key_whitespace.py``). + The marker and the message are module constants because the strings are part of the contract: provider tests and operators' log grep patterns assert on them. """ @@ -20,7 +26,7 @@ URL_USERINFO_ERROR = "base_url must not contain username or password information def resolve_api_key(raw: str | SecretStr | None) -> str | None: - """Return the configured key with surrounding whitespace removed. + """Return the configured or environment key with surrounding whitespace removed. Blank or absent reads as ``None``: a missing key is not an error, it is an unconfigured provider. Unwrapping happens only here, so no provider can end diff --git a/backend/packages/harness/deerflow/community/tavily/AGENTS.md b/backend/packages/harness/deerflow/community/tavily/AGENTS.md index c6c043db32..182ad8a251 100644 --- a/backend/packages/harness/deerflow/community/tavily/AGENTS.md +++ b/backend/packages/harness/deerflow/community/tavily/AGENTS.md @@ -1,8 +1,9 @@ # Tavily tools The client helper selects credentials by tool name: search defaults to -`web_search`, fetch passes `web_fetch`, and an omitted key uses the SDK's -`TAVILY_API_KEY` fallback. Keep credential regressions in `backend/tests/test_tavily_tools.py` +`web_search`, fetch passes `web_fetch`, and an omitted key falls back to +`TAVILY_API_KEY`. Both go through `provider_credentials.resolve_api_key`; if neither +resolves, the helper passes `""` so the SDK never re-reads the raw variable. Keep credential regressions in `backend/tests/test_tavily_tools.py` on the real helper and SDK constructor, mocking only search/extract calls. Each tool owns its per-call async client and awaits `close()` in `finally`, diff --git a/backend/packages/harness/deerflow/community/tavily/tools.py b/backend/packages/harness/deerflow/community/tavily/tools.py index 8237f0d414..35d32094d8 100644 --- a/backend/packages/harness/deerflow/community/tavily/tools.py +++ b/backend/packages/harness/deerflow/community/tavily/tools.py @@ -1,10 +1,12 @@ import asyncio import json import logging +import os from langchain.tools import tool from tavily import AsyncTavilyClient +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.search_time_range import SearchTimeRange from deerflow.community.url_safety import safe_public_result_url from deerflow.config import get_app_config @@ -22,8 +24,12 @@ def _get_tavily_client(tool_name: str = "web_search", config: ToolConfig | None config = get_app_config().get_tool_config(tool_name) api_key = None if config is not None and "api_key" in config.model_extra: - api_key = config.model_extra.get("api_key") - return AsyncTavilyClient(api_key=api_key) + api_key = resolve_api_key(config.model_extra.get("api_key")) + if api_key is None: + api_key = resolve_api_key(os.getenv("TAVILY_API_KEY")) + # The SDK re-reads TAVILY_API_KEY unstripped when api_key is None; "" makes a + # blank value fail as a missing key instead. + return AsyncTavilyClient(api_key=api_key or "") def _response_objects(data: object, container: str) -> list[dict] | None: diff --git a/backend/tests/test_firecrawl_tools.py b/backend/tests/test_firecrawl_tools.py index abbedc213d..93c70fc2e7 100644 --- a/backend/tests/test_firecrawl_tools.py +++ b/backend/tests/test_firecrawl_tools.py @@ -143,7 +143,7 @@ class TestFirecrawlBaseUrl: result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) assert result == "# Fetched Page\n\nFetched markdown" - mock_firecrawl_cls.assert_called_once_with(api_key=None, api_url="http://192.168.0.47:3002") + mock_firecrawl_cls.assert_called_once_with(api_key="", api_url="http://192.168.0.47:3002") @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") diff --git a/backend/tests/test_provider_api_key_whitespace.py b/backend/tests/test_provider_api_key_whitespace.py new file mode 100644 index 0000000000..44d1bd7679 --- /dev/null +++ b/backend/tests/test_provider_api_key_whitespace.py @@ -0,0 +1,260 @@ +"""Search and fetch providers send a configured key without surrounding whitespace. + +A key read from a file keeps its trailing newline: a Kubernetes Secret created +with ``kubectl create secret --from-file`` and mounted through ``envFrom`` is +the usual source. These providers put the raw value into an HTTP header, which +the transport refuses, and the refusal quotes the whole header: +``Illegal header value b'Bearer \\n'`` from h11, or ``... in header value: +'\\n'`` from requests. The providers returned that message as the tool +result, so the key reached the model, the checkpointed conversation and, for +some of them, the error log. + +Every provider here now resolves its key through +``provider_credentials.resolve_api_key``, as the other bundled providers already +strip theirs. The tests run each one over a real loopback HTTP server, so the +transport's own header validation is in the loop: the request must arrive with +the stripped key, and nothing the provider returns or logs may contain the key. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from types import SimpleNamespace + +import httpx +import pytest + +KEY = "sk-provider-test-4f7c2a91" +RAW_KEY = f"{KEY}\r\n" + + +class _Recorder: + def __init__(self) -> None: + self.headers: list[dict[str, str]] = [] + + +@pytest.fixture +def backend(): + """A loopback HTTP server that records request headers and answers ``{}``.""" + recorder = _Recorder() + + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + self.rfile.read(int(self.headers.get("content-length") or 0)) + recorder.headers.append({name.lower(): value for name, value in self.headers.items()}) + body = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = _answer + + def log_message(self, *args) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield SimpleNamespace(url=f"http://127.0.0.1:{server.server_port}", recorder=recorder) + finally: + server.shutdown() + server.server_close() + thread.join() + + +class _LoopbackTransport(httpx.AsyncHTTPTransport): + """Send every request to the loopback server through the real httpcore/h11 stack.""" + + def __init__(self, base_url: str) -> None: + super().__init__() + self._base = httpx.URL(base_url) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + request.url = request.url.copy_with(scheme=self._base.scheme, host=self._base.host, port=self._base.port) + return await super().handle_async_request(request) + + +@pytest.fixture +def loopback_httpx(backend, monkeypatch): + """Route clients built with a fixed provider host (``r.jina.ai``, InfoQuest, Tavily) to the loopback server.""" + original = httpx.AsyncClient + + def client(*args, **kwargs): + kwargs.pop("transport", None) + return original(*args, transport=_LoopbackTransport(backend.url), **kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + return backend + + +def _app_config(extra: dict | None): + return SimpleNamespace(get_tool_config=lambda name: SimpleNamespace(model_extra=dict(extra)) if extra is not None else None) + + +async def _outcome(call: Callable[[], Awaitable[object]]) -> str: + """The provider's result, or the text ToolErrorHandlingMiddleware would build from its exception.""" + try: + return str(await call()) + except Exception as exc: # noqa: BLE001 - the middleware turns any tool exception into a ToolMessage + return f"{type(exc).__name__}: {exc}" + + +async def _groundroute(source: str, backend, monkeypatch) -> str: + from deerflow.community.groundroute import tools + + monkeypatch.setattr(tools, "_GROUNDROUTE_ENDPOINT", f"{backend.url}/v1/search") + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"api_key": RAW_KEY} if source == "config" else {})) + if source == "env": + monkeypatch.setenv("GROUNDROUTE_API_KEY", RAW_KEY) + return await _outcome(lambda: tools.web_search_tool.ainvoke({"query": "q"})) + + +async def _infoquest(source: str, backend, monkeypatch) -> str: + from deerflow.community.infoquest.infoquest_client import InfoQuestClient + + monkeypatch.setenv("INFOQUEST_API_KEY", RAW_KEY) + return await _outcome(lambda: InfoQuestClient().web_search("q")) + + +async def _tavily(source: str, backend, monkeypatch) -> str: + from deerflow.community.tavily import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"api_key": RAW_KEY} if source == "config" else {})) + if source == "env": + monkeypatch.setenv("TAVILY_API_KEY", RAW_KEY) + return await _outcome(lambda: tools.web_search_tool.ainvoke({"query": "q"})) + + +async def _exa(source: str, backend, monkeypatch) -> str: + from deerflow.community.exa import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, **({"api_key": RAW_KEY} if source == "config" else {})})) + if source == "env": + monkeypatch.setenv("EXA_API_KEY", RAW_KEY) + return await _outcome(lambda: asyncio.to_thread(tools.web_search_tool.invoke, {"query": "q"})) + + +async def _jina(source: str, backend, monkeypatch) -> str: + from deerflow.community.jina_ai.jina_client import JinaClient + + monkeypatch.setenv("JINA_API_KEY", RAW_KEY) + return await _outcome(lambda: JinaClient().crawl("https://example.com")) + + +async def _firecrawl(source: str, backend, monkeypatch) -> str: + from deerflow.community.firecrawl import tools + + extra = {"base_url": backend.url, "network_isolation_confirmed": True, **({"api_key": RAW_KEY} if source == "config" else {})} + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config(extra)) + if source == "env": + monkeypatch.setenv("FIRECRAWL_API_KEY", RAW_KEY) + return await _outcome(lambda: tools.web_search_tool.ainvoke({"query": "q"})) + + +async def _fastcrw(source: str, backend, monkeypatch) -> str: + from deerflow.community.fastcrw import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, **({"api_key": RAW_KEY} if source == "config" else {})})) + if source == "env": + monkeypatch.setenv("CRW_API_KEY", RAW_KEY) + return await _outcome(lambda: asyncio.to_thread(tools.web_search_tool.invoke, {"query": "q"})) + + +async def _crawl4ai(source: str, backend, monkeypatch) -> str: + from deerflow.community.crawl4ai import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, "network_isolation_confirmed": True, "token": RAW_KEY})) + # The fetch target is screened by DNS; it never leaves the loopback server here. + monkeypatch.setattr(tools, "validate_public_http_url", lambda url, **kwargs: None) + return await _outcome(lambda: tools.web_fetch_tool.ainvoke({"url": "https://example.com"})) + + +# provider, key source, call, header that carries the key, its expected value +CASES = [ + ("groundroute", "config", _groundroute, "authorization", f"Bearer {KEY}"), + ("groundroute", "env", _groundroute, "authorization", f"Bearer {KEY}"), + ("infoquest", "env", _infoquest, "authorization", f"Bearer {KEY}"), + ("tavily", "config", _tavily, "authorization", f"Bearer {KEY}"), + ("tavily", "env", _tavily, "authorization", f"Bearer {KEY}"), + ("exa", "config", _exa, "x-api-key", KEY), + ("exa", "env", _exa, "x-api-key", KEY), + ("jina", "env", _jina, "authorization", f"Bearer {KEY}"), + ("firecrawl", "config", _firecrawl, "authorization", f"Bearer {KEY}"), + ("firecrawl", "env", _firecrawl, "authorization", f"Bearer {KEY}"), + ("fastcrw", "config", _fastcrw, "authorization", f"Bearer {KEY}"), + ("fastcrw", "env", _fastcrw, "authorization", f"Bearer {KEY}"), + ("crawl4ai", "config", _crawl4ai, "authorization", f"Bearer {KEY}"), +] + + +@pytest.mark.anyio +@pytest.mark.parametrize(("provider", "source", "call", "header", "expected"), CASES, ids=[f"{case[0]}-{case[1]}" for case in CASES]) +async def test_key_with_trailing_newline_is_sent_stripped_and_never_echoed(provider, source, call, header, expected, loopback_httpx, monkeypatch, caplog): + backend = loopback_httpx + for name in ("GROUNDROUTE_API_KEY", "INFOQUEST_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "EXA_BASE_URL", "JINA_API_KEY", "FIRECRAWL_API_KEY", "CRW_API_KEY", "CRW_API_URL"): + monkeypatch.delenv(name, raising=False) + caplog.set_level(logging.DEBUG) + + result = await call(source, backend, monkeypatch) + + sent = [request.get(header) for request in backend.recorder.headers] + assert sent and set(sent) == {expected}, f"{provider} never sent its key cleanly; it returned {result!r}" + assert KEY not in result + assert KEY not in caplog.text + + +@pytest.mark.anyio +@pytest.mark.parametrize("tool_name", ["web_search", "web_fetch"]) +@pytest.mark.parametrize("configured", [{"api_key": ""}, {"api_key": " \n"}, {}], ids=["empty", "blank", "absent"]) +async def test_fastcrw_without_its_own_key_never_sends_the_firecrawl_key(configured, tool_name, backend, monkeypatch, caplog): + """fastCRW builds the Firecrawl SDK client, which reads FIRECRAWL_API_KEY when it gets no key. + + Only the SDK's v2 client honours the ``""`` sentinel; running both tools pins + that search and scrape still reach it rather than the v1 client. + """ + from deerflow.community.fastcrw import tools + + firecrawl_key = "fc-other-provider-key-93b1" + monkeypatch.delenv("CRW_API_KEY", raising=False) + monkeypatch.delenv("CRW_API_URL", raising=False) + monkeypatch.setenv("FIRECRAWL_API_KEY", firecrawl_key) + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, "network_isolation_confirmed": True, **configured})) + monkeypatch.setattr(tools, "validate_public_http_url", lambda url, **kwargs: None) + caplog.set_level(logging.DEBUG) + + if tool_name == "web_search": + result = await _outcome(lambda: asyncio.to_thread(tools.web_search_tool.invoke, {"query": "q"})) + else: + result = await _outcome(lambda: asyncio.to_thread(tools.web_fetch_tool.invoke, {"url": "https://example.com"})) + + assert backend.recorder.headers, f"fastCRW never reached its endpoint; it returned {result!r}" + assert all("authorization" not in request for request in backend.recorder.headers) + assert firecrawl_key not in result + assert firecrawl_key not in caplog.text + + +BLANK_ENV_CASES = [ + ("tavily", "TAVILY_API_KEY", _tavily), + ("exa", "EXA_API_KEY", _exa), + ("firecrawl", "FIRECRAWL_API_KEY", _firecrawl), +] + + +@pytest.mark.anyio +@pytest.mark.parametrize(("provider", "env_name", "call"), BLANK_ENV_CASES, ids=[case[0] for case in BLANK_ENV_CASES]) +async def test_blank_environment_key_is_never_handed_back_to_the_sdk(provider, env_name, call, loopback_httpx, monkeypatch): + """These SDKs re-read their variable unstripped when given no key; a blank one must act as missing.""" + backend = loopback_httpx + monkeypatch.setenv(env_name, " \r\n") + + result = await call("none", backend, monkeypatch) + + assert "header value" not in result, f"{provider} handed the raw variable back to its SDK: {result!r}" + assert all(request.get("authorization", "").strip() == "" and "x-api-key" not in request for request in backend.recorder.headers) diff --git a/backend/tests/test_tavily_tools.py b/backend/tests/test_tavily_tools.py index e5235f52a3..9566dfca46 100644 --- a/backend/tests/test_tavily_tools.py +++ b/backend/tests/test_tavily_tools.py @@ -68,8 +68,8 @@ async def test_web_fetch_uses_own_credentials(monkeypatch, search_provider, fetc mock_config.return_value.get_tool_config.side_effect = configs.get await web_fetch_tool.ainvoke({"url": "https://example.com/report"}) - # web_fetch 用自己的 api_key 构造 client；配置缺省时传 None，交由 SDK 读取环境变量 - mock_client_cls.assert_called_once_with(api_key=fetch_key) + # web_fetch 用自己的 api_key 构造 client；配置缺省时回退到去除首尾空白后的 TAVILY_API_KEY + mock_client_cls.assert_called_once_with(api_key=fetch_key or "env-key") mock_client_cls.return_value.extract.assert_called_once_with(["https://example.com/report"]) @@ -90,7 +90,7 @@ async def test_web_search_preserves_own_credentials(monkeypatch, search_key) -> await web_search_tool.ainvoke({"query": "documentation"}) # web_search 用自己的 api_key（search_key）构造 client，而不是 web_fetch 的 fetch-key - mock_client_cls.assert_called_once_with(api_key=search_key) + mock_client_cls.assert_called_once_with(api_key=search_key or "env-key") mock_client_cls.return_value.search.assert_called_once_with("documentation", max_results=5)