From fb0ed9c960767d4116fb732f4c1aa8929c223ea7 Mon Sep 17 00:00:00 2001 From: Hyeonsang Cho Date: Thu, 8 Oct 2026 22:14:26 +0900 Subject: [PATCH] fix(browserless): send the web_fetch token as a query parameter (#6484) * fix(browserless): send the web_fetch token as a query parameter fetch_html_with_status put the configured token in the /content JSON body. Browserless reads the token only from the token query parameter or the Authorization header, and checks it before reading the body, so with BROWSERLESS_TOKEN set every web_fetch failed with HTTP 401; against an instance started without TOKEN the unknown body key failed schema validation with HTTP 400. capture_screenshot already sent ?token=, so web_capture worked with the same token. Both endpoints now take the token from BrowserlessClient._auth_params(). The test that pinned the body token is rewritten, and a stand-in for the Browserless request gate (query/header auth, no extra body keys) drives both endpoints through the real httpx request path. * docs(changelog): reference #6484 --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 10 ++++ CHANGELOG_zh.md | 5 ++ backend/docs/CONFIGURATION.md | 2 +- .../browserless/browserless_client.py | 16 ++++-- .../packages/harness/deerflow/tools/AGENTS.md | 2 +- backend/tests/test_browserless_client.py | 55 ++++++++++++++++++- 6 files changed, 80 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c5cb0f7a7..fadf328a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -905,6 +905,15 @@ This release closes that milestone with **439 merged pull requests**. goal." and `get_goal`/`set_goal`/`clear_goal` raised. The goal helpers now fall back to the synchronous methods for those savers. The web UI was not affected. ([#6448]) +- **community:** The Browserless `web_fetch` provider now authenticates. It sent + the configured token inside the `/content` JSON body, but Browserless reads + the token only from the `token` query parameter or the `Authorization` + header, and checks it before reading the body. With `BROWSERLESS_TOKEN` set + as the configuration guide describes, every `web_fetch` failed with + `Browserless HTTP 401` (and with HTTP 400 against an instance started without + `TOKEN`, whose body schema rejects the unknown key) while `web_capture` + worked with the same token. Both tools now send the token as a query + parameter. ([#6484]) - **mcp:** An MCP server with `task_toolsets` that is unreachable or times out during tool discovery no longer removes every MCP tool. Discovery skipped the failed server with an empty tool list, the task-toolset check then reported @@ -9227,5 +9236,6 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#6448]: https://github.com/bytedance/deer-flow/pull/6448 [#6450]: https://github.com/bytedance/deer-flow/pull/6450 [#6481]: https://github.com/bytedance/deer-flow/pull/6481 +[#6484]: https://github.com/bytedance/deer-flow/pull/6484 [#6495]: https://github.com/bytedance/deer-flow/pull/6495 [#6506]: https://github.com/bytedance/deer-flow/pull/6506 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 08cd6095c0..66a513855d 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -724,6 +724,10 @@ 同步检查点保存器定义了异步方法，但调用时会抛出 `NotImplementedError`，而目标读写只要异步方法存在就会调用它，因此 `/goal` 只会显示 "Could not set goal."，`get_goal`/`set_goal`/`clear_goal` 则直接抛出异常。现在目标读写在这类保存器上 会改用同步方法。Web UI 不受影响。([#6448]) +- **community：** Browserless 的 `web_fetch` 提供方现在可以通过认证。此前它把配置的令牌放在 `/content` 请求的 JSON 正文里， + 而 Browserless 只从 `token` 查询参数或 `Authorization` 请求头读取令牌，并在读取正文之前完成校验。按照配置指南设置 + `BROWSERLESS_TOKEN` 后，每次 `web_fetch` 都返回 `Browserless HTTP 401`（对未设置 `TOKEN` 启动的实例则因正文模式拒绝 + 未知字段而返回 HTTP 400），而使用同一令牌的 `web_capture` 正常工作。现在两个工具都以查询参数发送令牌。([#6484]) - **mcp：** 配置了 `task_toolsets` 的 MCP 服务器在工具发现时无法连接或超时，不再导致所有 MCP 工具丢失。此前发现阶段以空 工具列表跳过失败的服务器，随后任务工具集检查把它的 submit、status、cancel 工具报告为缺失，抛出的错误连同所有正常服务器 的工具一起丢弃；由于缓存从未发布，每次构建智能体都会重新发现全部服务器，重启 stdio 服务器并重新请求 OAuth 令牌。现在发现 @@ -7531,4 +7535,5 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#6448]: https://github.com/bytedance/deer-flow/pull/6448 [#6450]: https://github.com/bytedance/deer-flow/pull/6450 [#6481]: https://github.com/bytedance/deer-flow/pull/6481 +[#6484]: https://github.com/bytedance/deer-flow/pull/6484 [#6506]: https://github.com/bytedance/deer-flow/pull/6506 diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 3f8815d88f..bdd83703bc 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -1464,7 +1464,7 @@ models: - `GROUNDROUTE_API_KEY` - GroundRoute meta-search API key for `web_search` and `web_fetch` (routes across Serper, Brave, Exa, Tavily, Firecrawl, Perplexity with gain-share pricing) - `SOFYA_API_KEY` - [Sofya](https://sofya.co) key for `web_search` and `web_fetch` - `UNBROWSE_API_KEY` - [Unbrowse](https://unbrowse.ai) key for `web_fetch` -- `BROWSERLESS_TOKEN` - Browserless Cloud token for `web_capture` (optional for self-hosted Browserless) +- `BROWSERLESS_TOKEN` - Browserless token for `web_fetch` (Browserless provider) and `web_capture`, sent as the `token` query parameter (required by Browserless Cloud and by a self-hosted instance started with `TOKEN`) - `DEER_FLOW_PROJECT_ROOT` - Project root for relative runtime paths - `DEER_FLOW_CONFIG_PATH` - Custom config file path - `DEER_FLOW_EXTENSIONS_CONFIG_PATH` - Custom extensions config file path diff --git a/backend/packages/harness/deerflow/community/browserless/browserless_client.py b/backend/packages/harness/deerflow/community/browserless/browserless_client.py index c9ce0a4798..75c339f250 100644 --- a/backend/packages/harness/deerflow/community/browserless/browserless_client.py +++ b/backend/packages/harness/deerflow/community/browserless/browserless_client.py @@ -38,6 +38,15 @@ class BrowserlessClient: self.token = token self.timeout_s = timeout_s + def _auth_params(self) -> dict[str, str] | None: + """Query parameters that authenticate a request to Browserless. + + Browserless reads the token from the ``token`` query parameter or the + ``Authorization`` header only, and its request-body schemas reject + unknown keys, so the token must never be sent in the JSON body. + """ + return {"token": self.token} if self.token else None + async def fetch_html( self, url: str, @@ -117,8 +126,6 @@ class BrowserlessClient: "url": url, } - if self.token: - payload["token"] = self.token if wait_for_event: payload["waitForEvent"] = wait_for_event if wait_for_timeout_ms > 0: @@ -139,6 +146,7 @@ class BrowserlessClient: resp = await client.post( f"{self.base_url}/content", json=payload, + params=self._auth_params(), headers={ "Content-Type": "application/json", "Cache-Control": "no-cache", @@ -222,15 +230,13 @@ class BrowserlessClient: if best_attempt: payload["bestAttempt"] = True - params = {"token": self.token} if self.token else None - logger.debug(f"Capturing URL screenshot via Browserless: {url}") try: async with httpx.AsyncClient(timeout=self.timeout_s) as client: resp = await client.post( f"{self.base_url}/screenshot", json=payload, - params=params, + params=self._auth_params(), headers={ "Content-Type": "application/json", "Cache-Control": "no-cache", diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index 47d3d4d01a..35525d3e5d 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -70,7 +70,7 @@ Durable MCP task-management tools are added only while the process-local task su The shared SSRF guard (`community/url_safety.py`) resolves hostnames with blocking `socket.getaddrinfo`, so async tools, Gateway routes, and the Playwright request guard (`BrowserSession._install_request_guard`, which runs on the shared browser loop) call `validate_public_http_url` (or a wrapper such as `validate_browser_url`) through `asyncio.to_thread`. The guard checks at validation time only, so a launched browser sends every TCP connection (HTTP, HTTPS, WebSocket) through its per-session loopback SOCKS5 proxy (`browser_automation/egress.py`, `bypass: <-loopback>`): Chromium hands it every hostname, and it resolves once via `resolve_browser_egress` → `url_safety.resolve_public_addresses` and connects to exactly those addresses, closing the DNS-rebinding window (`mcp/personal_network.py` pins the same way). Every `acquire_session`/`get_session` caller must pass `egress_resolver`, because a thread's first caller fixes its launch options. WebRTC UDP does not traverse a SOCKS proxy and is not pinned (out of scope, so this is not a complete egress guarantee). Delegated fetch services (crawl4ai, Browserless, fastcrw) resolve on their own side and cannot be pinned; CDP-attached Chrome has neither guard. The strict blocking-IO gate carries a `socket.getaddrinfo` rule for this; `tests/blocking_io/test_web_tool_url_validation.py` drives each web tool entry point, the request guard, and the egress proxy; `tests/test_browser_egress.py` pins the proxy protocol, the session/manager/router wiring, and (with Playwright installed) a real-Chromium rebinding case. -Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`, `unbrowse`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. +Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`, `unbrowse`); see each subpackage for specifics. Browserless authenticates only from the `token` query parameter or `Authorization` header and rejects unknown body keys, so `BrowserlessClient._auth_params()` carries the token for every endpoint; never add it to the JSON body. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. E2B output sync records remote file versions and actual host file metadata in a thread-local manifest. The manifest binds to the remote sandbox ID. A complete output listing removes entries for deleted files. This avoids repeat downloads when the host filesystem rounds modification times. A single release-time sync pass is bounded by aggregate ceilings (`_MAX_SYNC_TOTAL_BYTES`, `_MAX_SYNC_FILES`, `_SYNC_DEADLINE_SECONDS`) on top of the per-file `_MAX_DOWNLOAD_SIZE` cap, so a pathological outputs tree cannot make release download unboundedly; a truncated pass logs what it dropped and leaves the manifest un-pruned (only entries observed in that pass are reconciled), so files it never reached are retried on the next release rather than being forgotten. diff --git a/backend/tests/test_browserless_client.py b/backend/tests/test_browserless_client.py index de179c35a2..e72aae3dfe 100644 --- a/backend/tests/test_browserless_client.py +++ b/backend/tests/test_browserless_client.py @@ -1,9 +1,11 @@ """Tests for Browserless community tools.""" import ipaddress +import json from types import SimpleNamespace from unittest.mock import MagicMock, patch +import httpx import pytest from deerflow.community.browserless import tools @@ -159,7 +161,7 @@ class TestBrowserlessClient: assert "timed out" in result.lower() or "timeout" in result.lower() async def test_fetch_html_with_token(self): - """fetch_html includes token in payload when set.""" + """fetch_html sends the token as a query parameter, never in the body.""" with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls: mock_ctx = MagicMock() mock_cls.return_value.__aenter__.return_value = mock_ctx @@ -173,8 +175,9 @@ class TestBrowserlessClient: client = BrowserlessClient(base_url="http://browserless:3000", token="my-token") await client.fetch_html("https://example.com") - payload = mock_ctx.post.call_args.kwargs["json"] - assert payload["token"] == "my-token" + call = mock_ctx.post.call_args + assert call.kwargs["params"] == {"token": "my-token"} + assert "token" not in call.kwargs["json"] async def test_fetch_html_with_wait_for_selector(self): """fetch_html sends waitForSelector when selector is set.""" @@ -306,6 +309,52 @@ class TestBrowserlessClient: assert result == "Error: Browserless returned empty screenshot response" +def _browserless_transport(server_token: str | None) -> httpx.MockTransport: + """Stand-in for a Browserless server's request gate. + + Browserless authenticates from the ``token`` query parameter or an + ``Authorization`` header only, before it reads the body, and its body + schemas reject unknown keys such as ``token``. + """ + + def handler(request: httpx.Request) -> httpx.Response: + if server_token is not None: + bearer = request.headers.get("Authorization", "").removeprefix("Bearer ") + if server_token not in (request.url.params.get("token"), bearer): + return httpx.Response(401, text="Bad or missing authentication.") + if "token" in json.loads(request.content): + return httpx.Response(400, text="POST Body validation failed: must NOT have additional properties") + if request.url.path == "/screenshot": + return httpx.Response(200, content=b"\x89PNG\r\n\x1a\n", headers={"Content-Type": "image/png"}) + return httpx.Response(200, text="OK") + + return httpx.MockTransport(handler) + + +async def _fetch(client: BrowserlessClient): + return await client.fetch_html_with_status("https://example.com") + + +async def _capture(client: BrowserlessClient): + return await client.capture_screenshot("https://example.com") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call", [_fetch, _capture], ids=["fetch", "capture"]) +@pytest.mark.parametrize("server_token", ["secret-token", None], ids=["server-requires-token", "server-without-token"]) +async def test_configured_token_passes_browserless_request_gate(call, server_token) -> None: + transport = _browserless_transport(server_token) + real_client = httpx.AsyncClient + + def client_with_transport(**kwargs): + return real_client(transport=transport, **kwargs) + + with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient", side_effect=client_with_transport): + result = await call(BrowserlessClient(base_url="http://browserless:3000", token="secret-token")) + + assert not isinstance(result, str), result + + @pytest.mark.asyncio class TestBrowserlessTools: """Tests for the Browserless tool functions."""