diff --git a/CHANGELOG.md b/CHANGELOG.md index d986ac29ac..f2a477ba00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,20 @@ This section accumulates work toward the **2.3.0** milestone ### Fixed +- **community:** A search or fetch API key that ends in a newline no longer + leaks into the conversation. GroundRoute (`GROUNDROUTE_API_KEY`), InfoQuest, + Tavily, Exa, Jina, Firecrawl, fastCRW and Crawl4AI put the key into an HTTP + header as read, so a key from a file (a Kubernetes Secret created with + `--from-file`) made the transport fail with `Illegal header value b'Bearer + \n'`. That message, key included, became the tool result the model and + the conversation history kept, and four of them also logged it. Keys now go + through `provider_credentials.resolve_api_key` like the other providers', and + a blank configured key falls back to the environment for Tavily, Exa, + Firecrawl and fastCRW. fastCRW without a key of its own no longer lets the + Firecrawl SDK send `FIRECRAWL_API_KEY` to the fastCRW endpoint, and a blank + `TAVILY_API_KEY`, `EXA_API_KEY` or `FIRECRAWL_API_KEY` now counts as missing + instead of reaching the SDK as is. ([#7981]) + - **docker:** Images built from a checkout no longer contain its local configuration. `.dockerignore` did not exclude `config.yaml`, `config.yaml.bak`, `extensions_config.json` or `mcp_config.json`, so a @@ -16086,3 +16100,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7907]: https://github.com/bytedance/deer-flow/pull/7907 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 [#7958]: https://github.com/bytedance/deer-flow/pull/7958 +[#7981]: https://github.com/bytedance/deer-flow/pull/7981 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index fdb21f2db0..ea09422c7a 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -23,6 +23,17 @@ ### 修复 +- **社区工具：** 以换行结尾的搜索或抓取 API 密钥不再泄露到对话中。GroundRoute + （`GROUNDROUTE_API_KEY`）、InfoQuest、Tavily、Exa、Jina、Firecrawl、fastCRW 和 + Crawl4AI 此前把读取到的密钥原样写入 HTTP 请求头；从文件读取的密钥（例如用 + `--from-file` 创建的 Kubernetes Secret）会让传输层报出 `Illegal header value + b'Bearer \n'`，这条包含完整密钥的消息成为模型和对话历史保存的工具结果， + 其中四个提供方还会把它写入日志。现在这些密钥与其他提供方一样经过 + `provider_credentials.resolve_api_key` 处理；Tavily、Exa、Firecrawl 和 fastCRW + 的配置密钥为空时会回退到环境变量。fastCRW 没有自己的密钥时，不再让 Firecrawl SDK + 把 `FIRECRAWL_API_KEY` 发送到 fastCRW 端点；`TAVILY_API_KEY`、`EXA_API_KEY` 或 + `FIRECRAWL_API_KEY` 只含空白时按未设置处理，不再原样交给 SDK。([#7981]) + - **docker：** 从本地 checkout 构建的镜像不再包含本地配置。`.dockerignore` 此前没有排除 `config.yaml`、`config.yaml.bak`、`extensions_config.json` 和 `mcp_config.json`，因此配置放在旧位置 `backend/` 时，`COPY backend ./backend` @@ -12796,3 +12807,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7782]: https://github.com/bytedance/deer-flow/pull/7782 [#7953]: https://github.com/bytedance/deer-flow/issues/7953 [#7958]: https://github.com/bytedance/deer-flow/pull/7958 +[#7981]: https://github.com/bytedance/deer-flow/pull/7981 diff --git a/backend/packages/harness/deerflow/community/crawl4ai/tools.py b/backend/packages/harness/deerflow/community/crawl4ai/tools.py index 416bc8ad4d..e718d29d6c 100644 --- a/backend/packages/harness/deerflow/community/crawl4ai/tools.py +++ b/backend/packages/harness/deerflow/community/crawl4ai/tools.py @@ -3,6 +3,7 @@ import logging from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config @@ -82,7 +83,7 @@ def _build_client(cfg: dict | None) -> Crawl4AiClient: timeout_s: float = float(DEFAULT_TIMEOUT_S) if cfg is not None: base_url = cfg.get("base_url", base_url) - token = cfg.get("token", token) + token = resolve_api_key(cfg.get("token")) or "" timeout_s = _coerce_timeout(cfg.get("timeout"), DEFAULT_TIMEOUT_S) return Crawl4AiClient(base_url=base_url, token=token, timeout_s=timeout_s) diff --git a/backend/packages/harness/deerflow/community/exa/tools.py b/backend/packages/harness/deerflow/community/exa/tools.py index 9ac4f0899b..72b3f7375f 100644 --- a/backend/packages/harness/deerflow/community/exa/tools.py +++ b/backend/packages/harness/deerflow/community/exa/tools.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse from exa_py import Exa from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import safe_public_result_url, validate_delegated_backend_url from deerflow.config import get_app_config @@ -82,7 +83,12 @@ def _get_exa_client(tool_name: str = "web_search") -> Exa: def _build_exa_client(extra: dict, base_url: str | None) -> Exa: - kwargs = {"api_key": extra.get("api_key")} + api_key = resolve_api_key(extra.get("api_key")) or resolve_api_key(os.getenv("EXA_API_KEY")) + # The SDK re-reads EXA_API_KEY unstripped when api_key is None and sends "" as + # is, so a missing or blank key fails here, before any request. + if api_key is None: + raise ValueError("Exa API key must be set in the tool config or EXA_API_KEY") + kwargs = {"api_key": api_key} if base_url: kwargs["base_url"] = base_url return Exa(**kwargs) diff --git a/backend/packages/harness/deerflow/community/fastcrw/tools.py b/backend/packages/harness/deerflow/community/fastcrw/tools.py index 68130a37de..af5a9ca362 100644 --- a/backend/packages/harness/deerflow/community/fastcrw/tools.py +++ b/backend/packages/harness/deerflow/community/fastcrw/tools.py @@ -5,6 +5,7 @@ import os from firecrawl import FirecrawlApp from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.search_max_results import DEFAULT_MAX_RESULTS, coerce_max_results from deerflow.community.url_safety import safe_public_result_url, validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config @@ -33,14 +34,18 @@ def _get_fastcrw_client( """ if cfg is None: cfg = _get_tool_config_extra(tool_name) - api_key = cfg.get("api_key") + api_key = resolve_api_key(cfg.get("api_key")) if base_url is None: base_url = cfg.get("base_url") if api_key is None: - api_key = os.getenv("CRW_API_KEY") + api_key = resolve_api_key(os.getenv("CRW_API_KEY")) if base_url is None: base_url = os.getenv("CRW_API_URL", DEFAULT_BASE_URL) - return FirecrawlApp(api_key=api_key, api_url=base_url) # type: ignore[arg-type] + # The Firecrawl SDK reads FIRECRAWL_API_KEY when api_key is None; "" stops it + # sending another provider's key to the fastCRW endpoint. Only the SDK's v2 + # client, which serves search and scrape, honours "": its v1 client still reads + # FIRECRAWL_API_KEY, so never call ``client.v1`` here. + return FirecrawlApp(api_key=api_key or "", api_url=base_url) # type: ignore[arg-type] def _resolve_fastcrw_base_url(cfg: dict | None) -> str: diff --git a/backend/packages/harness/deerflow/community/firecrawl/tools.py b/backend/packages/harness/deerflow/community/firecrawl/tools.py index 11a1879edf..5b6a8ab8d9 100644 --- a/backend/packages/harness/deerflow/community/firecrawl/tools.py +++ b/backend/packages/harness/deerflow/community/firecrawl/tools.py @@ -2,10 +2,12 @@ import asyncio import inspect import json import logging +import os from firecrawl import AsyncFirecrawlApp from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.search_max_results import DEFAULT_MAX_RESULTS, coerce_max_results from deerflow.community.url_safety import safe_public_result_url, validate_delegated_backend_url, validate_public_http_url from deerflow.config import get_app_config @@ -83,10 +85,12 @@ def _get_firecrawl_client( """ if cfg is None: cfg = _get_tool_config_extra(tool_name) - api_key = cfg.get("api_key") + api_key = resolve_api_key(cfg.get("api_key")) or resolve_api_key(os.getenv("FIRECRAWL_API_KEY")) if api_url is None: api_url = cfg.get("base_url") - kwargs = {"api_key": api_key} + # The SDK re-reads FIRECRAWL_API_KEY unstripped when api_key is None; "" makes + # a blank value act as a missing key. + kwargs = {"api_key": api_key or ""} if api_url: kwargs["api_url"] = api_url return AsyncFirecrawlApp(**kwargs) # type: ignore[arg-type] diff --git a/backend/packages/harness/deerflow/community/groundroute/AGENTS.md b/backend/packages/harness/deerflow/community/groundroute/AGENTS.md index ae74289dc9..94b45c0bc6 100644 --- a/backend/packages/harness/deerflow/community/groundroute/AGENTS.md +++ b/backend/packages/harness/deerflow/community/groundroute/AGENTS.md @@ -25,7 +25,8 @@ would satisfy the thread assertion; the anchor records which client factory the provider reached for. Both tools resolve the key and `max_results` through `_resolve_tool_settings`, one -config read per call. Do not add a second seam around that read: a wrapper with no +config read per call. The config key and `GROUNDROUTE_API_KEY` both pass through +`provider_credentials.resolve_api_key`. Do not add a second seam around that read: a wrapper with no production caller can drift from the real call path. `web_search` returns the structured `{"error": ..., "query": ...}` response; `web_fetch` diff --git a/backend/packages/harness/deerflow/community/groundroute/tools.py b/backend/packages/harness/deerflow/community/groundroute/tools.py index b3522634fe..6100dc99f1 100644 --- a/backend/packages/harness/deerflow/community/groundroute/tools.py +++ b/backend/packages/harness/deerflow/community/groundroute/tools.py @@ -24,6 +24,7 @@ import os import httpx from langchain.tools import tool +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import safe_public_result_url from deerflow.config import get_app_config @@ -56,11 +57,9 @@ def _resolve_tool_settings(tool_name: str) -> tuple[str | None, object]: if tool_config is not None: extra = tool_config.model_extra or {} max_results = extra.get("max_results") - candidate = extra.get("api_key") - if isinstance(candidate, str) and candidate.strip(): - api_key = candidate.strip() + api_key = resolve_api_key(extra.get("api_key")) if api_key is None: - api_key = os.getenv("GROUNDROUTE_API_KEY") + api_key = resolve_api_key(os.getenv("GROUNDROUTE_API_KEY")) return api_key, max_results diff --git a/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py b/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py index d7f68e3450..9700d619b9 100644 --- a/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py +++ b/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py @@ -12,6 +12,7 @@ from typing import Any import httpx +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.url_safety import safe_public_result_url as _safe_public_url logger = logging.getLogger(__name__) @@ -43,7 +44,7 @@ class InfoQuestClient: self.search_time_range = search_time_range self.image_search_time_range = image_search_time_range self.image_size = image_size - self.api_key_set = bool(os.getenv("INFOQUEST_API_KEY")) + self.api_key_set = resolve_api_key(os.getenv("INFOQUEST_API_KEY")) is not None if logger.isEnabledFor(logging.DEBUG): config_details = ( f"\n📋 Configuration Details:\n" @@ -147,8 +148,9 @@ class InfoQuestClient: } # Add API key if available - if os.getenv("INFOQUEST_API_KEY"): - headers["Authorization"] = f"Bearer {os.getenv('INFOQUEST_API_KEY')}" + api_key = resolve_api_key(os.getenv("INFOQUEST_API_KEY")) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" logger.debug("API key added to request headers") else: logger.warning("InfoQuest API key is not set. Provide your own key for authentication.") diff --git a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py index c4fefc85ab..039dad7434 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py +++ b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py @@ -11,6 +11,8 @@ from email.utils import parsedate_to_datetime import httpx +from deerflow.community.provider_credentials import resolve_api_key + from .request_admission import JinaAdmissionError, get_admission logger = logging.getLogger(__name__) @@ -162,8 +164,9 @@ class JinaClient: "X-Return-Format": return_format, "X-Timeout": str(timeout), } - if os.getenv("JINA_API_KEY"): - headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}" + api_key = resolve_api_key(os.getenv("JINA_API_KEY")) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" elif not _api_key_warned: _api_key_warned = True logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.") diff --git a/backend/packages/harness/deerflow/community/provider_credentials.py b/backend/packages/harness/deerflow/community/provider_credentials.py index 0533a8fe23..3c74e6da3b 100644 --- a/backend/packages/harness/deerflow/community/provider_credentials.py +++ b/backend/packages/harness/deerflow/community/provider_credentials.py @@ -7,6 +7,12 @@ key is scrubbed out of text before it reaches a log line or a tool error, and why a configured base URL must not carry userinfo. Two owners for one rule means a fix to one provider can silently leave the other one leaking. +Search and fetch providers that read a key from config or the environment +resolve it through ``resolve_api_key`` too. A key read from a file keeps its +trailing newline, and putting that raw value into a header makes the HTTP +transport refuse it with an error that quotes the whole header, key included +(``tests/test_provider_api_key_whitespace.py``). + The marker and the message are module constants because the strings are part of the contract: provider tests and operators' log grep patterns assert on them. """ @@ -20,7 +26,7 @@ URL_USERINFO_ERROR = "base_url must not contain username or password information def resolve_api_key(raw: str | SecretStr | None) -> str | None: - """Return the configured key with surrounding whitespace removed. + """Return the configured or environment key with surrounding whitespace removed. Blank or absent reads as ``None``: a missing key is not an error, it is an unconfigured provider. Unwrapping happens only here, so no provider can end diff --git a/backend/packages/harness/deerflow/community/tavily/AGENTS.md b/backend/packages/harness/deerflow/community/tavily/AGENTS.md index c6c043db32..182ad8a251 100644 --- a/backend/packages/harness/deerflow/community/tavily/AGENTS.md +++ b/backend/packages/harness/deerflow/community/tavily/AGENTS.md @@ -1,8 +1,9 @@ # Tavily tools The client helper selects credentials by tool name: search defaults to -`web_search`, fetch passes `web_fetch`, and an omitted key uses the SDK's -`TAVILY_API_KEY` fallback. Keep credential regressions in `backend/tests/test_tavily_tools.py` +`web_search`, fetch passes `web_fetch`, and an omitted key falls back to +`TAVILY_API_KEY`. Both go through `provider_credentials.resolve_api_key`; if neither +resolves, the helper passes `""` so the SDK never re-reads the raw variable. Keep credential regressions in `backend/tests/test_tavily_tools.py` on the real helper and SDK constructor, mocking only search/extract calls. Each tool owns its per-call async client and awaits `close()` in `finally`, diff --git a/backend/packages/harness/deerflow/community/tavily/tools.py b/backend/packages/harness/deerflow/community/tavily/tools.py index 8237f0d414..35d32094d8 100644 --- a/backend/packages/harness/deerflow/community/tavily/tools.py +++ b/backend/packages/harness/deerflow/community/tavily/tools.py @@ -1,10 +1,12 @@ import asyncio import json import logging +import os from langchain.tools import tool from tavily import AsyncTavilyClient +from deerflow.community.provider_credentials import resolve_api_key from deerflow.community.search_time_range import SearchTimeRange from deerflow.community.url_safety import safe_public_result_url from deerflow.config import get_app_config @@ -22,8 +24,12 @@ def _get_tavily_client(tool_name: str = "web_search", config: ToolConfig | None config = get_app_config().get_tool_config(tool_name) api_key = None if config is not None and "api_key" in config.model_extra: - api_key = config.model_extra.get("api_key") - return AsyncTavilyClient(api_key=api_key) + api_key = resolve_api_key(config.model_extra.get("api_key")) + if api_key is None: + api_key = resolve_api_key(os.getenv("TAVILY_API_KEY")) + # The SDK re-reads TAVILY_API_KEY unstripped when api_key is None; "" makes a + # blank value fail as a missing key instead. + return AsyncTavilyClient(api_key=api_key or "") def _response_objects(data: object, container: str) -> list[dict] | None: diff --git a/backend/tests/test_firecrawl_tools.py b/backend/tests/test_firecrawl_tools.py index abbedc213d..93c70fc2e7 100644 --- a/backend/tests/test_firecrawl_tools.py +++ b/backend/tests/test_firecrawl_tools.py @@ -143,7 +143,7 @@ class TestFirecrawlBaseUrl: result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) assert result == "# Fetched Page\n\nFetched markdown" - mock_firecrawl_cls.assert_called_once_with(api_key=None, api_url="http://192.168.0.47:3002") + mock_firecrawl_cls.assert_called_once_with(api_key="", api_url="http://192.168.0.47:3002") @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") diff --git a/backend/tests/test_provider_api_key_whitespace.py b/backend/tests/test_provider_api_key_whitespace.py new file mode 100644 index 0000000000..44d1bd7679 --- /dev/null +++ b/backend/tests/test_provider_api_key_whitespace.py @@ -0,0 +1,260 @@ +"""Search and fetch providers send a configured key without surrounding whitespace. + +A key read from a file keeps its trailing newline: a Kubernetes Secret created +with ``kubectl create secret --from-file`` and mounted through ``envFrom`` is +the usual source. These providers put the raw value into an HTTP header, which +the transport refuses, and the refusal quotes the whole header: +``Illegal header value b'Bearer \\n'`` from h11, or ``... in header value: +'\\n'`` from requests. The providers returned that message as the tool +result, so the key reached the model, the checkpointed conversation and, for +some of them, the error log. + +Every provider here now resolves its key through +``provider_credentials.resolve_api_key``, as the other bundled providers already +strip theirs. The tests run each one over a real loopback HTTP server, so the +transport's own header validation is in the loop: the request must arrive with +the stripped key, and nothing the provider returns or logs may contain the key. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +from types import SimpleNamespace + +import httpx +import pytest + +KEY = "sk-provider-test-4f7c2a91" +RAW_KEY = f"{KEY}\r\n" + + +class _Recorder: + def __init__(self) -> None: + self.headers: list[dict[str, str]] = [] + + +@pytest.fixture +def backend(): + """A loopback HTTP server that records request headers and answers ``{}``.""" + recorder = _Recorder() + + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + self.rfile.read(int(self.headers.get("content-length") or 0)) + recorder.headers.append({name.lower(): value for name, value in self.headers.items()}) + body = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = _answer + + def log_message(self, *args) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield SimpleNamespace(url=f"http://127.0.0.1:{server.server_port}", recorder=recorder) + finally: + server.shutdown() + server.server_close() + thread.join() + + +class _LoopbackTransport(httpx.AsyncHTTPTransport): + """Send every request to the loopback server through the real httpcore/h11 stack.""" + + def __init__(self, base_url: str) -> None: + super().__init__() + self._base = httpx.URL(base_url) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + request.url = request.url.copy_with(scheme=self._base.scheme, host=self._base.host, port=self._base.port) + return await super().handle_async_request(request) + + +@pytest.fixture +def loopback_httpx(backend, monkeypatch): + """Route clients built with a fixed provider host (``r.jina.ai``, InfoQuest, Tavily) to the loopback server.""" + original = httpx.AsyncClient + + def client(*args, **kwargs): + kwargs.pop("transport", None) + return original(*args, transport=_LoopbackTransport(backend.url), **kwargs) + + monkeypatch.setattr(httpx, "AsyncClient", client) + return backend + + +def _app_config(extra: dict | None): + return SimpleNamespace(get_tool_config=lambda name: SimpleNamespace(model_extra=dict(extra)) if extra is not None else None) + + +async def _outcome(call: Callable[[], Awaitable[object]]) -> str: + """The provider's result, or the text ToolErrorHandlingMiddleware would build from its exception.""" + try: + return str(await call()) + except Exception as exc: # noqa: BLE001 - the middleware turns any tool exception into a ToolMessage + return f"{type(exc).__name__}: {exc}" + + +async def _groundroute(source: str, backend, monkeypatch) -> str: + from deerflow.community.groundroute import tools + + monkeypatch.setattr(tools, "_GROUNDROUTE_ENDPOINT", f"{backend.url}/v1/search") + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"api_key": RAW_KEY} if source == "config" else {})) + if source == "env": + monkeypatch.setenv("GROUNDROUTE_API_KEY", RAW_KEY) + return await _outcome(lambda: tools.web_search_tool.ainvoke({"query": "q"})) + + +async def _infoquest(source: str, backend, monkeypatch) -> str: + from deerflow.community.infoquest.infoquest_client import InfoQuestClient + + monkeypatch.setenv("INFOQUEST_API_KEY", RAW_KEY) + return await _outcome(lambda: InfoQuestClient().web_search("q")) + + +async def _tavily(source: str, backend, monkeypatch) -> str: + from deerflow.community.tavily import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"api_key": RAW_KEY} if source == "config" else {})) + if source == "env": + monkeypatch.setenv("TAVILY_API_KEY", RAW_KEY) + return await _outcome(lambda: tools.web_search_tool.ainvoke({"query": "q"})) + + +async def _exa(source: str, backend, monkeypatch) -> str: + from deerflow.community.exa import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, **({"api_key": RAW_KEY} if source == "config" else {})})) + if source == "env": + monkeypatch.setenv("EXA_API_KEY", RAW_KEY) + return await _outcome(lambda: asyncio.to_thread(tools.web_search_tool.invoke, {"query": "q"})) + + +async def _jina(source: str, backend, monkeypatch) -> str: + from deerflow.community.jina_ai.jina_client import JinaClient + + monkeypatch.setenv("JINA_API_KEY", RAW_KEY) + return await _outcome(lambda: JinaClient().crawl("https://example.com")) + + +async def _firecrawl(source: str, backend, monkeypatch) -> str: + from deerflow.community.firecrawl import tools + + extra = {"base_url": backend.url, "network_isolation_confirmed": True, **({"api_key": RAW_KEY} if source == "config" else {})} + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config(extra)) + if source == "env": + monkeypatch.setenv("FIRECRAWL_API_KEY", RAW_KEY) + return await _outcome(lambda: tools.web_search_tool.ainvoke({"query": "q"})) + + +async def _fastcrw(source: str, backend, monkeypatch) -> str: + from deerflow.community.fastcrw import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, **({"api_key": RAW_KEY} if source == "config" else {})})) + if source == "env": + monkeypatch.setenv("CRW_API_KEY", RAW_KEY) + return await _outcome(lambda: asyncio.to_thread(tools.web_search_tool.invoke, {"query": "q"})) + + +async def _crawl4ai(source: str, backend, monkeypatch) -> str: + from deerflow.community.crawl4ai import tools + + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, "network_isolation_confirmed": True, "token": RAW_KEY})) + # The fetch target is screened by DNS; it never leaves the loopback server here. + monkeypatch.setattr(tools, "validate_public_http_url", lambda url, **kwargs: None) + return await _outcome(lambda: tools.web_fetch_tool.ainvoke({"url": "https://example.com"})) + + +# provider, key source, call, header that carries the key, its expected value +CASES = [ + ("groundroute", "config", _groundroute, "authorization", f"Bearer {KEY}"), + ("groundroute", "env", _groundroute, "authorization", f"Bearer {KEY}"), + ("infoquest", "env", _infoquest, "authorization", f"Bearer {KEY}"), + ("tavily", "config", _tavily, "authorization", f"Bearer {KEY}"), + ("tavily", "env", _tavily, "authorization", f"Bearer {KEY}"), + ("exa", "config", _exa, "x-api-key", KEY), + ("exa", "env", _exa, "x-api-key", KEY), + ("jina", "env", _jina, "authorization", f"Bearer {KEY}"), + ("firecrawl", "config", _firecrawl, "authorization", f"Bearer {KEY}"), + ("firecrawl", "env", _firecrawl, "authorization", f"Bearer {KEY}"), + ("fastcrw", "config", _fastcrw, "authorization", f"Bearer {KEY}"), + ("fastcrw", "env", _fastcrw, "authorization", f"Bearer {KEY}"), + ("crawl4ai", "config", _crawl4ai, "authorization", f"Bearer {KEY}"), +] + + +@pytest.mark.anyio +@pytest.mark.parametrize(("provider", "source", "call", "header", "expected"), CASES, ids=[f"{case[0]}-{case[1]}" for case in CASES]) +async def test_key_with_trailing_newline_is_sent_stripped_and_never_echoed(provider, source, call, header, expected, loopback_httpx, monkeypatch, caplog): + backend = loopback_httpx + for name in ("GROUNDROUTE_API_KEY", "INFOQUEST_API_KEY", "TAVILY_API_KEY", "EXA_API_KEY", "EXA_BASE_URL", "JINA_API_KEY", "FIRECRAWL_API_KEY", "CRW_API_KEY", "CRW_API_URL"): + monkeypatch.delenv(name, raising=False) + caplog.set_level(logging.DEBUG) + + result = await call(source, backend, monkeypatch) + + sent = [request.get(header) for request in backend.recorder.headers] + assert sent and set(sent) == {expected}, f"{provider} never sent its key cleanly; it returned {result!r}" + assert KEY not in result + assert KEY not in caplog.text + + +@pytest.mark.anyio +@pytest.mark.parametrize("tool_name", ["web_search", "web_fetch"]) +@pytest.mark.parametrize("configured", [{"api_key": ""}, {"api_key": " \n"}, {}], ids=["empty", "blank", "absent"]) +async def test_fastcrw_without_its_own_key_never_sends_the_firecrawl_key(configured, tool_name, backend, monkeypatch, caplog): + """fastCRW builds the Firecrawl SDK client, which reads FIRECRAWL_API_KEY when it gets no key. + + Only the SDK's v2 client honours the ``""`` sentinel; running both tools pins + that search and scrape still reach it rather than the v1 client. + """ + from deerflow.community.fastcrw import tools + + firecrawl_key = "fc-other-provider-key-93b1" + monkeypatch.delenv("CRW_API_KEY", raising=False) + monkeypatch.delenv("CRW_API_URL", raising=False) + monkeypatch.setenv("FIRECRAWL_API_KEY", firecrawl_key) + monkeypatch.setattr(tools, "get_app_config", lambda: _app_config({"base_url": backend.url, "network_isolation_confirmed": True, **configured})) + monkeypatch.setattr(tools, "validate_public_http_url", lambda url, **kwargs: None) + caplog.set_level(logging.DEBUG) + + if tool_name == "web_search": + result = await _outcome(lambda: asyncio.to_thread(tools.web_search_tool.invoke, {"query": "q"})) + else: + result = await _outcome(lambda: asyncio.to_thread(tools.web_fetch_tool.invoke, {"url": "https://example.com"})) + + assert backend.recorder.headers, f"fastCRW never reached its endpoint; it returned {result!r}" + assert all("authorization" not in request for request in backend.recorder.headers) + assert firecrawl_key not in result + assert firecrawl_key not in caplog.text + + +BLANK_ENV_CASES = [ + ("tavily", "TAVILY_API_KEY", _tavily), + ("exa", "EXA_API_KEY", _exa), + ("firecrawl", "FIRECRAWL_API_KEY", _firecrawl), +] + + +@pytest.mark.anyio +@pytest.mark.parametrize(("provider", "env_name", "call"), BLANK_ENV_CASES, ids=[case[0] for case in BLANK_ENV_CASES]) +async def test_blank_environment_key_is_never_handed_back_to_the_sdk(provider, env_name, call, loopback_httpx, monkeypatch): + """These SDKs re-read their variable unstripped when given no key; a blank one must act as missing.""" + backend = loopback_httpx + monkeypatch.setenv(env_name, " \r\n") + + result = await call("none", backend, monkeypatch) + + assert "header value" not in result, f"{provider} handed the raw variable back to its SDK: {result!r}" + assert all(request.get("authorization", "").strip() == "" and "x-api-key" not in request for request in backend.recorder.headers) diff --git a/backend/tests/test_tavily_tools.py b/backend/tests/test_tavily_tools.py index e5235f52a3..9566dfca46 100644 --- a/backend/tests/test_tavily_tools.py +++ b/backend/tests/test_tavily_tools.py @@ -68,8 +68,8 @@ async def test_web_fetch_uses_own_credentials(monkeypatch, search_provider, fetc mock_config.return_value.get_tool_config.side_effect = configs.get await web_fetch_tool.ainvoke({"url": "https://example.com/report"}) - # web_fetch 用自己的 api_key 构造 client；配置缺省时传 None，交由 SDK 读取环境变量 - mock_client_cls.assert_called_once_with(api_key=fetch_key) + # web_fetch 用自己的 api_key 构造 client；配置缺省时回退到去除首尾空白后的 TAVILY_API_KEY + mock_client_cls.assert_called_once_with(api_key=fetch_key or "env-key") mock_client_cls.return_value.extract.assert_called_once_with(["https://example.com/report"]) @@ -90,7 +90,7 @@ async def test_web_search_preserves_own_credentials(monkeypatch, search_key) -> await web_search_tool.ainvoke({"query": "documentation"}) # web_search 用自己的 api_key（search_key）构造 client，而不是 web_fetch 的 fetch-key - mock_client_cls.assert_called_once_with(api_key=search_key) + mock_client_cls.assert_called_once_with(api_key=search_key or "env-key") mock_client_cls.return_value.search.assert_called_once_with("documentation", max_results=5)