From f889ce7517ab59a6caedf2d188bea4ccabf190cd Mon Sep 17 00:00:00 2001 From: lekt8 Date: Mon, 28 Sep 2026 15:08:49 +0800 Subject: [PATCH] feat(community): add Unbrowse web fetch provider (#5981) * feat(community): add Unbrowse web fetch provider Add a community web_fetch provider backed by Unbrowse (https://unbrowse.ai), a hosted service that returns a page as markdown, over plain HTTP when that is enough and through a cloud browser when the page needs JavaScript. The tool sends one JSON-RPC tools/call POST (unbrowse.scrape) to Unbrowse's MCP endpoint, so it needs no extra Python package and no MCP client (uses httpx, already a dependency). Unbrowse is not a general web search engine, so only web_fetch is added. Changes: - backend/packages/harness/deerflow/community/unbrowse/{__init__,tools}.py web_fetch_tool with the same output shape as the Sofya, Exa and Firecrawl fetch tools (# plus markdown, 4KB cap). API key is read from the config.yaml api_key field or the UNBROWSE_API_KEY env var. Optional render: auto|never|always; unknown values log a warning and fall back to auto. JSON-RPC errors, isError tool results, HTTP errors and malformed payloads come back as "Error: ..." strings. - backend/tests/test_unbrowse_tools.py: key resolution, render handling, request shape, structuredContent vs text payloads, error paths. - scripts/doctor.py, scripts/wizard/providers.py: register the key and the web_fetch provider; backend/tests/test_doctor.py covers both doctor paths. - config.example.yaml, .env.example, backend/docs/CONFIGURATION.md, tools/AGENTS.md, EN/ZH harness/tools.mdx: document the provider. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * docs(changelog): add Unbrowse web_fetch provider entry Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com> --- .env.example | 3 + CHANGELOG.md | 5 + CHANGELOG_zh.md | 5 + backend/docs/CONFIGURATION.md | 3 +- .../deerflow/community/unbrowse/__init__.py | 3 + .../deerflow/community/unbrowse/tools.py | 174 ++++++++++ .../packages/harness/deerflow/tools/AGENTS.md | 3 +- backend/tests/test_doctor.py | 16 + backend/tests/test_unbrowse_tools.py | 323 ++++++++++++++++++ config.example.yaml | 12 + frontend/src/content/en/harness/tools.mdx | 11 +- frontend/src/content/zh/harness/tools.mdx | 11 +- scripts/doctor.py | 1 + scripts/wizard/providers.py | 8 + 14 files changed, 574 insertions(+), 4 deletions(-) create mode 100644 backend/packages/harness/deerflow/community/unbrowse/__init__.py create mode 100644 backend/packages/harness/deerflow/community/unbrowse/tools.py create mode 100644 backend/tests/test_unbrowse_tools.py diff --git a/.env.example b/.env.example index dd57cd9d88..93c179442d 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,9 @@ INFOQUEST_API_KEY=your-infoquest-api-key # Sofya API Key (web search and fetch) - https://sofya.co SOFYA_API_KEY=your-sofya-api-key +# Unbrowse API Key (web fetch) - https://unbrowse.ai/app +UNBROWSE_API_KEY=your-unbrowse-api-key + # Browser CORS allowlist for split-origin or port-forwarded deployments (comma-separated exact origins). # Leave unset when using the unified nginx endpoint, e.g. http://localhost:2026. # GATEWAY_CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000 diff --git a/CHANGELOG.md b/CHANGELOG.md index a4ee9bf2ad..5661dadec9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,6 +140,10 @@ This release closes that milestone with **181 merged pull requests**. #### Models & integrations +- **community:** New Unbrowse `web_fetch` provider - returns a page as + markdown over plain HTTP, or through Unbrowse's hosted cloud browser for + JavaScript-heavy pages (`render: auto|never|always`). One JSON-RPC POST per + fetch, no new dependencies. ([#5981]) - **models:** Administrators can manage shared models from Settings → Models without editing server configuration. New administrator-only `GET/PUT /api/managed-models` and `POST /api/managed-models/test` endpoints @@ -6189,4 +6193,5 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#5935]: https://github.com/bytedance/deer-flow/pull/5935 [#5945]: https://github.com/bytedance/deer-flow/pull/5945 [#5964]: https://github.com/bytedance/deer-flow/pull/5964 +[#5981]: https://github.com/bytedance/deer-flow/pull/5981 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 742c94462d..edf2cb7d8f 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -141,6 +141,10 @@ #### 模型与集成 +- **社区工具：** 新增 Unbrowse `web_fetch` provider——以 Markdown 形式返回页面 + 内容：普通 HTTP 足够时直接抓取，需要 JavaScript 的页面则在 Unbrowse 托管的 + 云端浏览器中渲染（`render: auto|never|always`）。每次抓取仅一次 JSON-RPC + POST，不引入新依赖。([#5981]) - **模型：** 管理员现在可以从“设置 → 模型”管理共享模型， 无需编辑服务器配置。新的仅管理员端点 `GET/PUT /api/managed-models` 与 @@ -5253,3 +5257,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#5935]: https://github.com/bytedance/deer-flow/pull/5935 [#5945]: https://github.com/bytedance/deer-flow/pull/5945 [#5964]: https://github.com/bytedance/deer-flow/pull/5964 +[#5981]: https://github.com/bytedance/deer-flow/pull/5981 diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 802acee570..13f332fe20 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -576,7 +576,7 @@ and `time_range`; the filters do not apply to `web_fetch` or other search provid **Built-in Tools**: - `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Serply, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute, Sofya) -- `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless, Sofya) +- `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless, Sofya, Unbrowse) - `web_capture` - Capture rendered webpage screenshots as artifacts (Browserless) - `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper, Brave) - `ls` - List directory contents @@ -1207,6 +1207,7 @@ models: - `SERPLY_API_KEY` - [Serply](https://serply.io) key for `web_search` (Google Search, plus Google News and Google Scholar via `vertical`) - `GROUNDROUTE_API_KEY` - GroundRoute meta-search API key for `web_search` and `web_fetch` (routes across Serper, Brave, Exa, Tavily, Firecrawl, Perplexity with gain-share pricing) - `SOFYA_API_KEY` - [Sofya](https://sofya.co) key for `web_search` and `web_fetch` +- `UNBROWSE_API_KEY` - [Unbrowse](https://unbrowse.ai) key for `web_fetch` - `BROWSERLESS_TOKEN` - Browserless Cloud token for `web_capture` (optional for self-hosted Browserless) - `DEER_FLOW_PROJECT_ROOT` - Project root for relative runtime paths - `DEER_FLOW_CONFIG_PATH` - Custom config file path diff --git a/backend/packages/harness/deerflow/community/unbrowse/__init__.py b/backend/packages/harness/deerflow/community/unbrowse/__init__.py new file mode 100644 index 0000000000..a28ef03a27 --- /dev/null +++ b/backend/packages/harness/deerflow/community/unbrowse/__init__.py @@ -0,0 +1,3 @@ +from .tools import web_fetch_tool + +__all__ = ["web_fetch_tool"] diff --git a/backend/packages/harness/deerflow/community/unbrowse/tools.py b/backend/packages/harness/deerflow/community/unbrowse/tools.py new file mode 100644 index 0000000000..dc497f4035 --- /dev/null +++ b/backend/packages/harness/deerflow/community/unbrowse/tools.py @@ -0,0 +1,174 @@ +""" +Web fetch tool powered by Unbrowse. + +Unbrowse is a hosted service that turns websites into APIs for agents. This +provider calls its ``unbrowse.scrape`` tool, which returns a page as markdown: +over plain HTTP when that is enough, or through a cloud browser when the page +needs JavaScript to render. Calls go to Unbrowse's MCP endpoint as a single +JSON-RPC ``tools/call`` POST. An API key is required. Get one at +https://unbrowse.ai/app. +""" + +import json +import logging +import os + +import httpx +from langchain.tools import tool + +from deerflow.config import get_app_config + +logger = logging.getLogger(__name__) + +_UNBROWSE_MCP_URL = "https://unbrowse.ai/api/mcp" +_UNBROWSE_SCRAPE_TOOL = "unbrowse.scrape" +_UNBROWSE_TIMEOUT = 90 +_UNBROWSE_FETCH_MAX_CHARS = 4096 +_DEFAULT_RENDER = "auto" +_RENDER_MODES = ("auto", "never", "always") +_api_key_warned: set[str] = set() + + +def _get_api_key(tool_name: str) -> str | None: + config = get_app_config().get_tool_config(tool_name) + if config is not None: + api_key = config.model_extra.get("api_key") + if isinstance(api_key, str) and api_key.strip(): + return api_key.strip() + env_key = os.getenv("UNBROWSE_API_KEY") + if isinstance(env_key, str) and env_key.strip(): + return env_key.strip() + return None + + +def _resolve_render(value: object) -> str: + """Return a supported render mode, falling back to the default with a warning.""" + if value is None: + return _DEFAULT_RENDER + render = str(value).strip().lower() + if render in _RENDER_MODES: + return render + logger.warning("Ignoring unsupported Unbrowse render %r; using %r (supported: %s)", value, _DEFAULT_RENDER, ", ".join(_RENDER_MODES)) + return _DEFAULT_RENDER + + +def _missing_key_message(tool_name: str) -> str: + if tool_name not in _api_key_warned: + _api_key_warned.add(tool_name) + logger.warning("Unbrowse API key is not set for '%s'. Set UNBROWSE_API_KEY in your environment or provide api_key in config.yaml. Get a key at https://unbrowse.ai/app", tool_name) + return "UNBROWSE_API_KEY is not configured" + + +def _tool_result_text(result: dict) -> str: + """Join the text blocks of an MCP tool result.""" + content = result.get("content") + if not isinstance(content, list): + return "" + return "\n".join(block["text"] for block in content if isinstance(block, dict) and isinstance(block.get("text"), str)) + + +def _call_unbrowse_tool(name: str, api_key: str, arguments: dict) -> tuple[dict | None, str | None]: + """Call an Unbrowse MCP tool with one JSON-RPC ``tools/call`` request. + + Returns a ``(data, error)`` tuple: on success ``data`` is the tool's JSON + payload and ``error`` is ``None``; on failure ``data`` is ``None`` and + ``error`` is a message the caller can hand back to the model. + """ + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + } + request = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": name, "arguments": arguments}, + } + + try: + with httpx.Client(timeout=_UNBROWSE_TIMEOUT) as client: + response = client.post(_UNBROWSE_MCP_URL, headers=headers, json=request) + response.raise_for_status() + data = response.json() + except httpx.HTTPStatusError as e: + logger.error("Unbrowse API returned HTTP %s: %s", e.response.status_code, (e.response.text or "")[:500]) + return None, f"Unbrowse API error: HTTP {e.response.status_code}" + except Exception as e: + logger.error("Unbrowse request failed: %s: %s", type(e).__name__, str(e)[:500]) + return None, str(e)[:500] + + if not isinstance(data, dict): + logger.error("Unbrowse returned an unexpected payload type: %s", type(data).__name__) + return None, "Unbrowse returned an unexpected response format" + + rpc_error = data.get("error") + if rpc_error is not None: + message = rpc_error.get("message") if isinstance(rpc_error, dict) else None + message = str(message or rpc_error).strip()[:500] + logger.error("Unbrowse %s failed: %s", name, message) + return None, f"Unbrowse error: {message}" + + result = data.get("result") + if not isinstance(result, dict): + logger.error("Unbrowse returned an unexpected 'result' payload type: %s", type(result).__name__) + return None, "Unbrowse returned an unexpected response format" + + text = _tool_result_text(result) + if result.get("isError"): + message = text.strip()[:500] or "tool call failed" + logger.error("Unbrowse %s returned a tool error: %s", name, message) + return None, f"Unbrowse error: {message}" + + structured = result.get("structuredContent") + if isinstance(structured, dict): + return structured, None + + try: + payload = json.loads(text) + except (TypeError, ValueError): + logger.error("Unbrowse %s returned non-JSON content", name) + return None, "Unbrowse returned an unexpected response format" + if not isinstance(payload, dict): + logger.error("Unbrowse %s returned an unexpected content type: %s", name, type(payload).__name__) + return None, "Unbrowse returned an unexpected response format" + return payload, None + + +def _clip(value: object, limit: int) -> str: + """Coerce a result field to text and truncate it. A limit of 0 means no truncation.""" + text = value if isinstance(value, str) else str(value) + return text if limit <= 0 else text[:limit] + + +@tool("web_fetch", parse_docstring=True) +def web_fetch_tool(url: str) -> str: + """Fetch the contents of a web page at a given URL. + Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools. + This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls. + Do NOT add www. to URLs that do NOT have them. + URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL. + + Args: + url: The URL to fetch the contents of. + """ + api_key = _get_api_key("web_fetch") + if not api_key: + return f"Error: {_missing_key_message('web_fetch')}" + + config = get_app_config().get_tool_config("web_fetch") + config_extra = (config.model_extra or {}) if config is not None else {} + render = _resolve_render(config_extra.get("render")) + + arguments = {"url": url, "formats": ["markdown"], "render": render} + data, error = _call_unbrowse_tool(_UNBROWSE_SCRAPE_TOOL, api_key, arguments) + if error is not None: + return f"Error: {error}" + + content = _clip(data.get("markdown") or "", _UNBROWSE_FETCH_MAX_CHARS) + if not content: + return "Error: No content found" + + metadata = data.get("metadata") + title = (metadata.get("title") if isinstance(metadata, dict) else None) or "Untitled" + return f"# {title}\n\n{content}" diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index 23d4af91d6..a106683ec7 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -49,13 +49,14 @@ Durable MCP task-management tools are added only while the process-local task su - `tavily/` - Web search (5 results default) and web fetch (4KB limit) - `sofya/` - Web search (5 results default, per-result content capped at 2000 chars) and web fetch (4KB limit) - `jina_ai/` - Web fetch via Jina reader API with readability extraction +- `unbrowse/` - Web fetch (4KB limit) via one JSON-RPC `tools/call` to Unbrowse's hosted MCP endpoint; `render: auto|never|always` picks plain HTTP or its cloud browser - `firecrawl/` - Web scraping via Firecrawl API - `image_search/` - Image search via DuckDuckGo - `aio_sandbox/` - Docker-based isolation (`AioSandboxProvider`) - `browser_automation/` - Agentic browser control (stateful `navigate → observe → click/type` loop) via Playwright, distinct from the read-only `web_fetch`/`web_capture` tools. Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close` (config `group: browser`). A process-local `BrowserSessionManager` owns one private, loop-affine Playwright event-loop thread (same pattern as the BoxLite provider) so a per-thread browser session survives across turns regardless of the caller's loop (Gateway / TUI / test). Each action returns a fresh page snapshot whose interactive elements are addressed by a stable numeric `[ref]` index (stamped as `data-df-ref` during snapshot), so the model acts on what it just observed instead of holding stale handles or guessing selectors. URLs are SSRF-screened via the shared `validate_public_http_url` (opt-out `allow_private_addresses` only for intentional internal targets). CDP attachment cannot install the request guard on an existing Chrome context, so `cdp_url` fails closed unless the operator explicitly sets `allow_unguarded_cdp: true` for a trusted local browser. Browser REST/Live access also requires an exact non-NULL thread owner, rather than the general legacy shared-thread policy, because retained pages may contain authenticated state. Session admission is a hard `max_sessions` cap: pinned Live/operation sessions are never evicted, and a new thread is rejected when no unpinned session can be closed; one Live viewer owns a session at a time. Optional dependency: `cd backend && uv sync --extra browser && uv run playwright install chromium`; `scripts/detect_uv_extras.py` preserves the extra when `config.yaml` enables `browser_navigate`, and Gateway startup fails fast if configured browser control cannot import Playwright. Tests: `tests/test_browser_automation.py` (mocked tools + a real-Chromium integration test guarded by `importorskip`); `tests/manual_browser_live_check.py` is a manual DeepSeek-driven end-to-end check (not collected by pytest). Live UI input dispatch is kept independent from JPEG capture: non-move actions start a rate-limited background refresh loop, so pointer, wheel, or keyboard input stays responsive while continuous gestures still produce frames throughout the interaction. -Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. +Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`, `unbrowse`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. E2B output sync records remote file versions and actual host file metadata in a thread-local manifest. The manifest binds to the remote sandbox ID. A complete output listing removes entries for deleted files. This avoids repeat downloads when the host filesystem rounds modification times. A single release-time sync pass is bounded by aggregate ceilings (`_MAX_SYNC_TOTAL_BYTES`, `_MAX_SYNC_FILES`, `_SYNC_DEADLINE_SECONDS`) on top of the per-file `_MAX_DOWNLOAD_SIZE` cap, so a pathological outputs tree cannot make release download unboundedly; a truncated pass logs what it dropped and leaves the manifest un-pruned (only entries observed in that pass are reconciled), so files it never reached are retried on the next release rather than being forgotten. diff --git a/backend/tests/test_doctor.py b/backend/tests/test_doctor.py index c0cd9a59bc..0f7039c3dc 100644 --- a/backend/tests/test_doctor.py +++ b/backend/tests/test_doctor.py @@ -633,6 +633,22 @@ class TestCheckWebFetch: assert result.status == "warn" assert "SOFYA_API_KEY" in (result.fix or "") + def test_unbrowse_with_key_ok(self, tmp_path, monkeypatch): + monkeypatch.setenv("UNBROWSE_API_KEY", "test-key") + cfg = tmp_path / "config.yaml" + cfg.write_text("config_version: 5\ntools:\n - name: web_fetch\n use: deerflow.community.unbrowse.tools:web_fetch_tool\n") + result = doctor.check_web_fetch(cfg) + assert result.status == "ok" + assert "unbrowse" in result.detail + + def test_unbrowse_without_key_warns(self, tmp_path, monkeypatch): + monkeypatch.delenv("UNBROWSE_API_KEY", raising=False) + cfg = tmp_path / "config.yaml" + cfg.write_text("config_version: 5\ntools:\n - name: web_fetch\n use: deerflow.community.unbrowse.tools:web_fetch_tool\n") + result = doctor.check_web_fetch(cfg) + assert result.status == "warn" + assert "UNBROWSE_API_KEY" in (result.fix or "") + def test_no_fetch_tool_warns(self, tmp_path): cfg = tmp_path / "config.yaml" cfg.write_text("config_version: 5\ntools: []\n") diff --git a/backend/tests/test_unbrowse_tools.py b/backend/tests/test_unbrowse_tools.py new file mode 100644 index 0000000000..518bbe5a90 --- /dev/null +++ b/backend/tests/test_unbrowse_tools.py @@ -0,0 +1,323 @@ +"""Unit tests for the Unbrowse community web fetch tool.""" + +import json +import logging +from unittest.mock import MagicMock, patch + +import httpx +import pytest + + +@pytest.fixture(autouse=True) +def reset_api_key_warned(): + """Reset the module-level warning flag before each test.""" + import deerflow.community.unbrowse.tools as unbrowse_mod + + unbrowse_mod._api_key_warned = set() + yield + unbrowse_mod._api_key_warned = set() + + +@pytest.fixture +def mock_config_with_key(): + with patch("deerflow.community.unbrowse.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": "test-unbrowse-key"} + mock.return_value.get_tool_config.return_value = tool_config + yield mock + + +@pytest.fixture +def mock_config_no_key(): + with patch("deerflow.community.unbrowse.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {} + mock.return_value.get_tool_config.return_value = tool_config + yield mock + + +def _make_response(payload: object) -> MagicMock: + mock_resp = MagicMock() + mock_resp.json.return_value = payload + mock_resp.raise_for_status = MagicMock() + return mock_resp + + +def _make_scrape_response(scrape: dict, *, structured: bool = False) -> MagicMock: + """Build a JSON-RPC tools/call response the way Unbrowse's MCP endpoint returns it.""" + result: dict = {"content": [{"type": "text", "text": json.dumps(scrape)}], "isError": False} + if structured: + result["structuredContent"] = scrape + return _make_response({"jsonrpc": "2.0", "id": 1, "result": result}) + + +def _scrape(markdown: object = "# Example Domain\n\nBody", title: object = "Example Domain") -> dict: + return { + "url": "https://example.com", + "finalUrl": "https://example.com/", + "via": "http", + "metadata": {"title": title, "sourceURL": "https://example.com/", "statusCode": 200}, + "markdown": markdown, + } + + +class TestGetApiKey: + def test_returns_config_key_when_present(self): + with patch("deerflow.community.unbrowse.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": "from-config"} + mock.return_value.get_tool_config.return_value = tool_config + + from deerflow.community.unbrowse.tools import _get_api_key + + assert _get_api_key("web_fetch") == "from-config" + + def test_falls_back_to_env_when_config_key_whitespace(self): + with patch("deerflow.community.unbrowse.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": " "} + mock.return_value.get_tool_config.return_value = tool_config + with patch.dict("os.environ", {"UNBROWSE_API_KEY": "env-key"}): + from deerflow.community.unbrowse.tools import _get_api_key + + assert _get_api_key("web_fetch") == "env-key" + + def test_uses_env_when_tool_is_not_configured(self): + with patch("deerflow.community.unbrowse.tools.get_app_config") as mock: + mock.return_value.get_tool_config.return_value = None + with patch.dict("os.environ", {"UNBROWSE_API_KEY": "env-only"}): + from deerflow.community.unbrowse.tools import _get_api_key + + assert _get_api_key("web_fetch") == "env-only" + + def test_returns_none_when_no_key_anywhere(self): + with patch("deerflow.community.unbrowse.tools.get_app_config") as mock: + mock.return_value.get_tool_config.return_value = None + with patch.dict("os.environ", {}, clear=True): + from deerflow.community.unbrowse.tools import _get_api_key + + assert _get_api_key("web_fetch") is None + + def test_returns_none_when_env_key_whitespace(self): + with patch("deerflow.community.unbrowse.tools.get_app_config") as mock: + mock.return_value.get_tool_config.return_value = None + with patch.dict("os.environ", {"UNBROWSE_API_KEY": " "}, clear=True): + from deerflow.community.unbrowse.tools import _get_api_key + + assert _get_api_key("web_fetch") is None + + +class TestResolveRender: + def test_defaults_to_auto(self): + from deerflow.community.unbrowse.tools import _resolve_render + + assert _resolve_render(None) == "auto" + + def test_normalizes_supported_modes(self): + from deerflow.community.unbrowse.tools import _resolve_render + + assert _resolve_render(" Never ") == "never" + assert _resolve_render("always") == "always" + + def test_unsupported_mode_falls_back_with_warning(self, caplog): + from deerflow.community.unbrowse.tools import _resolve_render + + with caplog.at_level(logging.WARNING, logger="deerflow.community.unbrowse.tools"): + assert _resolve_render("headless") == "auto" + + assert "headless" in caplog.text + + +class TestMissingKeyMessage: + def test_warns_once_per_tool_name(self, caplog): + from deerflow.community.unbrowse.tools import _missing_key_message + + with caplog.at_level(logging.WARNING, logger="deerflow.community.unbrowse.tools"): + assert _missing_key_message("web_fetch") == "UNBROWSE_API_KEY is not configured" + assert _missing_key_message("web_fetch") == "UNBROWSE_API_KEY is not configured" + + assert sum("Unbrowse API key is not set" in r.message for r in caplog.records) == 1 + + +class TestWebFetchTool: + def test_returns_title_and_markdown(self, mock_config_with_key): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_post = mock_client_cls.return_value.__enter__.return_value.post + mock_post.return_value = _make_scrape_response(_scrape()) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# Example Domain\n\n# Example Domain\n\nBody" + + def test_sends_one_json_rpc_tools_call(self, mock_config_with_key): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_post = mock_client_cls.return_value.__enter__.return_value.post + mock_post.return_value = _make_scrape_response(_scrape()) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + web_fetch_tool.invoke({"url": "https://example.com"}) + + assert mock_post.call_count == 1 + assert mock_post.call_args.args[0] == "https://unbrowse.ai/api/mcp" + assert mock_post.call_args.kwargs["headers"]["Authorization"] == "Bearer test-unbrowse-key" + assert mock_post.call_args.kwargs["json"] == { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "unbrowse.scrape", + "arguments": {"url": "https://example.com", "formats": ["markdown"], "render": "auto"}, + }, + } + + def test_render_can_be_set_from_config(self, mock_config_with_key): + mock_config_with_key.return_value.get_tool_config.return_value.model_extra = {"api_key": "k", "render": "never"} + + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_post = mock_client_cls.return_value.__enter__.return_value.post + mock_post.return_value = _make_scrape_response(_scrape()) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + web_fetch_tool.invoke({"url": "https://example.com"}) + + assert mock_post.call_args.kwargs["json"]["params"]["arguments"]["render"] == "never" + + def test_prefers_structured_content(self, mock_config_with_key): + response = _make_scrape_response(_scrape(markdown="from text"), structured=True) + response.json.return_value["result"]["structuredContent"] = _scrape(markdown="from structured") + + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = response + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# Example Domain\n\nfrom structured" + + def test_truncates_long_content(self, mock_config_with_key): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = _make_scrape_response(_scrape(markdown="x" * 9000, title="Long")) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert len(result) == len("# Long\n\n") + 4096 + + def test_non_string_markdown_does_not_raise(self, mock_config_with_key): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = _make_scrape_response(_scrape(markdown=12345, title="Numeric")) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# Numeric\n\n12345" + + def test_falls_back_to_untitled(self, mock_config_with_key): + scrape = _scrape(markdown="Body") + scrape["metadata"] = None + + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = _make_scrape_response(scrape) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# Untitled\n\nBody" + + @pytest.mark.parametrize("markdown", ["", None]) + def test_empty_markdown_returns_error(self, mock_config_with_key, markdown): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = _make_scrape_response(_scrape(markdown=markdown)) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "Error: No content found" + + def test_json_rpc_error_returns_its_message(self, mock_config_with_key): + payload = {"jsonrpc": "2.0", "id": 1, "error": {"code": -32000, "message": "page.goto: net::ERR_NAME_NOT_RESOLVED", "data": None}} + + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = _make_response(payload) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://nope.invalid"}) + + assert result == "Error: Unbrowse error: page.goto: net::ERR_NAME_NOT_RESOLVED" + + def test_tool_error_result_returns_its_text(self, mock_config_with_key): + payload = {"jsonrpc": "2.0", "id": 1, "result": {"content": [{"type": "text", "text": "render failed"}], "isError": True}} + + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = _make_response(payload) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "Error: Unbrowse error: render failed" + + @pytest.mark.parametrize( + "payload", + [ + ["not", "a", "dict"], + {"jsonrpc": "2.0", "id": 1, "result": "oops"}, + {"jsonrpc": "2.0", "id": 1, "result": {"content": [{"type": "text", "text": "not json"}]}}, + {"jsonrpc": "2.0", "id": 1, "result": {"content": [{"type": "text", "text": "[1, 2]"}]}}, + ], + ids=["non-dict-body", "non-dict-result", "non-json-text", "non-object-json"], + ) + def test_malformed_response_returns_error(self, mock_config_with_key, payload): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = _make_response(payload) + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "Error: Unbrowse returned an unexpected response format" + + def test_http_error_returns_error_string(self, mock_config_with_key): + request = httpx.Request("POST", "https://unbrowse.ai/api/mcp") + response = httpx.Response(401, text='{"error":"invalid_token"}', request=request) + mock_resp = MagicMock() + mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError("error", request=request, response=response) + + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.return_value = mock_resp + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "Error: Unbrowse API error: HTTP 401" + + def test_network_error_returns_error_string(self, mock_config_with_key): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + mock_client_cls.return_value.__enter__.return_value.post.side_effect = httpx.ConnectError("boom") + + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "Error: boom" + + def test_missing_key_returns_error_string(self, mock_config_no_key): + with patch.dict("os.environ", {}, clear=True): + with patch("deerflow.community.unbrowse.tools.httpx.Client") as mock_client_cls: + from deerflow.community.unbrowse.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "Error: UNBROWSE_API_KEY is not configured" + mock_client_cls.assert_not_called() diff --git a/config.example.yaml b/config.example.yaml index 8c892d4779..7618feff17 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1216,6 +1216,18 @@ tools: # use: deerflow.community.sofya.tools:web_fetch_tool # # api_key: $SOFYA_API_KEY + # Web fetch tool (uses Unbrowse, requires UNBROWSE_API_KEY) + # Returns the page as markdown. Fetches over plain HTTP when that is enough and + # falls back to a hosted cloud browser for pages that need JavaScript. + # Get a key at https://unbrowse.ai/app. + # NOTE: Only one web_fetch provider can be active at a time. + # Comment out the Jina AI web_fetch entry above before enabling this one. + # - name: web_fetch + # group: web + # use: deerflow.community.unbrowse.tools:web_fetch_tool + # # render: auto # auto | never (plain HTTP only) | always (cloud browser) + # # api_key: $UNBROWSE_API_KEY + # Image search tool (uses DuckDuckGo) # Use this to find reference images before image generation - name: image_search diff --git a/frontend/src/content/en/harness/tools.mdx b/frontend/src/content/en/harness/tools.mdx index 67305e54d0..8e7506af8b 100644 --- a/frontend/src/content/en/harness/tools.mdx +++ b/frontend/src/content/en/harness/tools.mdx @@ -210,7 +210,7 @@ Results include the content of each page, not only a snippet. Requires a [Sofya] ### Web fetch (page content extraction) -<Tabs items={["Jina AI (default)", "Browserless", "Exa", "InfoQuest", "Firecrawl", "GroundRoute", "Sofya"]}> +<Tabs items={["Jina AI (default)", "Browserless", "Exa", "InfoQuest", "Firecrawl", "GroundRoute", "Sofya", "Unbrowse"]}> <Tabs.Tab> ```yaml tools: @@ -267,6 +267,15 @@ tools: ``` Returns the page as markdown. Requires a [Sofya](https://sofya.co) API key. </Tabs.Tab> + <Tabs.Tab> +```yaml +tools: + - use: deerflow.community.unbrowse.tools:web_fetch_tool + api_key: $UNBROWSE_API_KEY + # render: auto # auto | never (plain HTTP only) | always (cloud browser) +``` +Returns the page as markdown, fetching over plain HTTP when that is enough and rendering JavaScript-heavy pages in a hosted browser. Requires an [Unbrowse](https://unbrowse.ai) API key. + </Tabs.Tab> </Tabs> ### Web capture (rendered screenshots) diff --git a/frontend/src/content/zh/harness/tools.mdx b/frontend/src/content/zh/harness/tools.mdx index 610bd7612a..ea22fe1bfb 100644 --- a/frontend/src/content/zh/harness/tools.mdx +++ b/frontend/src/content/zh/harness/tools.mdx @@ -204,7 +204,7 @@ tools: ### 网页内容抓取 -<Tabs items={["Jina AI（默认）", "Browserless", "Exa", "InfoQuest", "Firecrawl", "GroundRoute", "Sofya"]}> +<Tabs items={["Jina AI（默认）", "Browserless", "Exa", "InfoQuest", "Firecrawl", "GroundRoute", "Sofya", "Unbrowse"]}> <Tabs.Tab> ```yaml tools: @@ -260,6 +260,15 @@ tools: ``` 以 Markdown 形式返回页面内容。需要 [Sofya](https://sofya.co) API Key。 </Tabs.Tab> + <Tabs.Tab> +```yaml +tools: + - use: deerflow.community.unbrowse.tools:web_fetch_tool + api_key: $UNBROWSE_API_KEY + # render: auto # auto | never（仅普通 HTTP）| always（云端浏览器） +``` +以 Markdown 形式返回页面内容：普通 HTTP 足够时直接抓取，需要 JavaScript 的页面则在托管浏览器中渲染。需要 [Unbrowse](https://unbrowse.ai) API Key。 + </Tabs.Tab> </Tabs> ### 网页截图 diff --git a/scripts/doctor.py b/scripts/doctor.py index fc08691b1e..a18641fb0d 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -552,6 +552,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes "firecrawl": "FIRECRAWL_API_KEY", "fastcrw": "CRW_API_KEY", "sofya": "SOFYA_API_KEY", + "unbrowse": "UNBROWSE_API_KEY", }, "image_search": { "brave": "BRAVE_SEARCH_API_KEY", diff --git a/scripts/wizard/providers.py b/scripts/wizard/providers.py index fd0abc75db..f75d090f2f 100644 --- a/scripts/wizard/providers.py +++ b/scripts/wizard/providers.py @@ -735,6 +735,14 @@ WEB_FETCH_PROVIDERS: list[WebProvider] = [ env_var="SOFYA_API_KEY", tool_name="web_fetch", ), + WebProvider( + name="unbrowse", + display_name="Unbrowse", + description="Page as markdown, cloud browser for JavaScript pages, API key required", + use="deerflow.community.unbrowse.tools:web_fetch_tool", + env_var="UNBROWSE_API_KEY", + tool_name="web_fetch", + ), WebProvider( name="crawl4ai", display_name="Crawl4AI",