From 4dbfe37ff3003d1701d823e5549ce3938374bbab Mon Sep 17 00:00:00 2001 From: Serply <59339358+googio@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:30:30 -0400 Subject: [PATCH] feat(community): add Serply web search tool (#5023) Add deerflow.community.serply.tools:web_search_tool, a Google SERP provider for the web_search slot that also covers Google News and Google Scholar through an optional `vertical` config option. Reads the key from api_key in config.yaml or SERPLY_API_KEY, clamps max_results to Serply's 1-100 range, and returns the same structured JSON errors as the Serper and Brave tools. Register the provider in config.example.yaml, scripts/doctor.py, scripts/wizard/providers.py, .env.example, backend/docs/CONFIGURATION.md, the en/zh tools.mdx provider tabs, and tools/AGENTS.md. Tests mock httpx. --- .env.example | 3 + backend/docs/CONFIGURATION.md | 3 +- .../deerflow/community/serply/__init__.py | 3 + .../deerflow/community/serply/tools.py | 211 ++++++++++++ .../packages/harness/deerflow/tools/AGENTS.md | 2 +- backend/tests/test_doctor.py | 16 + backend/tests/test_serply_tools.py | 318 ++++++++++++++++++ config.example.yaml | 12 + frontend/src/content/en/harness/tools.mdx | 14 +- frontend/src/content/zh/harness/tools.mdx | 14 +- scripts/doctor.py | 1 + scripts/wizard/providers.py | 8 + 12 files changed, 601 insertions(+), 4 deletions(-) create mode 100644 backend/packages/harness/deerflow/community/serply/__init__.py create mode 100644 backend/packages/harness/deerflow/community/serply/tools.py create mode 100644 backend/tests/test_serply_tools.py diff --git a/.env.example b/.env.example index d1ed25e28..88a7b2f46 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,9 @@ # Serper API Key (Google Search) - https://serper.dev SERPER_API_KEY=your-serper-api-key +# Serply API Key (Google Search, News and Scholar) - https://serply.io +SERPLY_API_KEY=your-serply-api-key + # TAVILY API Key TAVILY_API_KEY=your-tavily-api-key diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index bc8374eef..644a0a4f0 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -367,7 +367,7 @@ tools: ``` **Built-in Tools**: -- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute) +- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Serply, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute) - `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless) - `web_capture` - Capture rendered webpage screenshots as artifacts (Browserless) - `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper, Brave) @@ -809,6 +809,7 @@ models: - `TAVILY_API_KEY` - Tavily search API key - `BRAVE_SEARCH_API_KEY` - Brave Search API key for `web_search` and `image_search` - `SERPER_API_KEY` - Serper (Google Search/Images API) key for `web_search` and `image_search` +- `SERPLY_API_KEY` - [Serply](https://serply.io) key for `web_search` (Google Search, plus Google News and Google Scholar via `vertical`) - `GROUNDROUTE_API_KEY` - GroundRoute meta-search API key for `web_search` and `web_fetch` (routes across Serper, Brave, Exa, Tavily, Firecrawl, Perplexity with gain-share pricing) - `BROWSERLESS_TOKEN` - Browserless Cloud token for `web_capture` (optional for self-hosted Browserless) - `DEER_FLOW_PROJECT_ROOT` - Project root for relative runtime paths diff --git a/backend/packages/harness/deerflow/community/serply/__init__.py b/backend/packages/harness/deerflow/community/serply/__init__.py new file mode 100644 index 000000000..876167859 --- /dev/null +++ b/backend/packages/harness/deerflow/community/serply/__init__.py @@ -0,0 +1,3 @@ +from .tools import web_search_tool + +__all__ = ["web_search_tool"] diff --git a/backend/packages/harness/deerflow/community/serply/tools.py b/backend/packages/harness/deerflow/community/serply/tools.py new file mode 100644 index 000000000..7d476d959 --- /dev/null +++ b/backend/packages/harness/deerflow/community/serply/tools.py @@ -0,0 +1,211 @@ +""" +Web search tool powered by the Serply API. + +Serply returns live Google results as JSON. One API key covers the regular +web SERP plus the Google News and Google Scholar verticals, so a research run +can be pointed at recent coverage or at papers by switching ``vertical`` in +config.yaml. An API key is required. Sign up at https://serply.io and see +https://serply.io/docs for the endpoint reference. +""" + +import html +import json +import logging +import os +import re + +import httpx +from langchain.tools import tool + +from deerflow.config import get_app_config + +logger = logging.getLogger(__name__) + +_SERPLY_BASE_URL = "https://api.serply.io/v1" +_DEFAULT_MAX_RESULTS = 5 +# Serply accepts ``num`` values from 1 to 100 per request. +_SERPLY_MAX_RESULTS = 100 +_DEFAULT_VERTICAL = "search" +# vertical -> (URL path segment, response key that holds the result rows) +_VERTICALS: dict[str, tuple[str, str]] = { + "search": ("search", "results"), + "news": ("news", "entries"), + "scholar": ("scholar", "articles"), +} +# Optional request parameters that are passed through from config.yaml as-is. +_PASSTHROUGH_PARAMS = ("gl", "hl") +_TAG_RE = re.compile(r"<[^>]+>") +_api_key_warned: set[str] = set() + + +def _get_api_key(tool_name: str = "web_search") -> str | None: + config = get_app_config().get_tool_config(tool_name) + if config is not None: + api_key = (config.model_extra or {}).get("api_key") + if isinstance(api_key, str) and api_key.strip(): + return api_key.strip() + env_key = os.getenv("SERPLY_API_KEY") + if isinstance(env_key, str) and env_key.strip(): + return env_key.strip() + return None + + +def _coerce_max_results( + value: object, + *, + default: int = _DEFAULT_MAX_RESULTS, + max_allowed: int = _SERPLY_MAX_RESULTS, +) -> int: + try: + coerced = int(value) + except (TypeError, ValueError): + logger.warning("Invalid Serply max_results=%r; using default %s", value, default) + coerced = default + if coerced < 1: + logger.warning("Invalid Serply max_results=%r; using default %s", value, default) + coerced = default + return min(coerced, max_allowed) + + +def _coerce_vertical(value: object) -> str: + if value is None: + return _DEFAULT_VERTICAL + if isinstance(value, str) and value.strip().lower() in _VERTICALS: + return value.strip().lower() + logger.warning("Invalid Serply vertical=%r; using %r (one of %s)", value, _DEFAULT_VERTICAL, sorted(_VERTICALS)) + return _DEFAULT_VERTICAL + + +def _clean_query(query: str, *, max_length: int = 500) -> str: + query = query.strip() + if len(query) > max_length: + query = query[:max_length] + return query + + +def _clean_text(value: object) -> str: + """Flatten a Serply text field to plain text (news summaries arrive as HTML).""" + if not isinstance(value, str): + return "" + return html.unescape(_TAG_RE.sub("", value)).strip() + + +def _missing_key_error(query: str, tool_name: str) -> str: + if tool_name not in _api_key_warned: + _api_key_warned.add(tool_name) + logger.warning( + "Serply API key is not set for '%s'. Set SERPLY_API_KEY in your environment or provide api_key in config.yaml. Sign up at https://serply.io", + tool_name, + ) + return json.dumps({"error": "SERPLY_API_KEY is not configured", "query": query}, ensure_ascii=False) + + +def _unexpected_format_error(query: str) -> str: + return json.dumps({"error": "Serply returned an unexpected response format", "query": query}, ensure_ascii=False) + + +def _serply_get(path: str, api_key: str, query: str, params: dict[str, object]) -> tuple[dict | None, str | None]: + """Send a GET request to a Serply endpoint. + + Returns a ``(data, error_json)`` tuple: on success ``data`` is the parsed + JSON response and ``error_json`` is ``None``; on failure ``data`` is ``None`` + and ``error_json`` is a serialized structured error ready to return. + """ + headers = { + "X-Api-Key": api_key, + "Accept": "application/json", + "User-Agent": "deerflow", + } + try: + with httpx.Client(timeout=30) as client: + response = client.get(f"{_SERPLY_BASE_URL}/{path}/", headers=headers, params=params) + response.raise_for_status() + data = response.json() + if not isinstance(data, dict): + logger.error("Serply returned an unexpected payload type: %s", type(data).__name__) + return None, _unexpected_format_error(query) + return data, None + except httpx.HTTPStatusError as e: + resp_text = (e.response.text or "")[:500] + logger.error("Serply API returned HTTP %s: %s", e.response.status_code, resp_text) + return None, json.dumps({"error": f"Serply API error: HTTP {e.response.status_code}", "query": query}, ensure_ascii=False) + except Exception as e: + logger.error("Serply request failed: %s: %s", type(e).__name__, str(e)[:500]) + return None, json.dumps({"error": str(e)[:500], "query": query}, ensure_ascii=False) + + +def _normalize_row(vertical: str, row: dict) -> dict: + """Map one Serply row onto the common ``title``/``url``/``content`` shape. + + News and Scholar rows carry a few extra fields worth surfacing to the model. + """ + result = {"title": row.get("title", ""), "url": row.get("link", "")} + if vertical == "news": + result["content"] = _clean_text(row.get("summary")) + result["published"] = row.get("published", "") + source = row.get("source") if isinstance(row.get("source"), dict) else {} + result["source"] = source.get("title", "") + elif vertical == "scholar": + result["content"] = row.get("description", "") + author = row.get("author") if isinstance(row.get("author"), dict) else {} + authors = author.get("authors") if isinstance(author.get("authors"), list) else [] + result["authors"] = [a.get("name", "") for a in authors if isinstance(a, dict)] + extras = row.get("extras") if isinstance(row.get("extras"), dict) else {} + citations = extras.get("citations") if isinstance(extras.get("citations"), dict) else {} + result["cited_by"] = citations.get("count", 0) + doc = row.get("doc") if isinstance(row.get("doc"), dict) else {} + result["pdf_url"] = doc.get("link", "") + else: + result["content"] = row.get("description", "") + return result + + +@tool("web_search", parse_docstring=True) +def web_search_tool(query: str, max_results: int = 5) -> str: + """Search the web for information using Google Search via Serply. + + Args: + query: Search keywords describing what you want to find. Be specific for better results. + max_results: Maximum number of search results to return. Default is 5, capped at 100. + """ + config = get_app_config().get_tool_config("web_search") + extra = (config.model_extra or {}) if config is not None else {} + if "max_results" in extra: + max_results = extra["max_results"] + max_results = _coerce_max_results(max_results) + vertical = _coerce_vertical(extra.get("vertical")) + query = _clean_query(query) + + api_key = _get_api_key("web_search") + if not api_key: + return _missing_key_error(query, "web_search") + + path, rows_key = _VERTICALS[vertical] + params: dict[str, object] = {"q": query, "num": max_results} + for key in _PASSTHROUGH_PARAMS: + if key in extra: + params[key] = extra[key] + + data, error_json = _serply_get(path, api_key, query, params) + if error_json is not None: + return error_json + + rows = data.get(rows_key) + if rows is None: + rows = [] + if not isinstance(rows, list): + logger.error("Serply returned unexpected '%s' payload type: %s", rows_key, type(rows).__name__) + return _unexpected_format_error(query) + rows = [row for row in rows if isinstance(row, dict)] + if not rows: + return json.dumps({"error": "No results found", "query": query}, ensure_ascii=False) + + # The news feed ignores ``num`` server-side, so the cap is also applied here. + normalized_results = [_normalize_row(vertical, row) for row in rows[:max_results]] + + output = { + "query": query, + "total_results": len(normalized_results), + "results": normalized_results, + } + return json.dumps(output, indent=2, ensure_ascii=False) diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index 47549a811..bbbc242be 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -28,7 +28,7 @@ Durable MCP task-management tools are added only while the process-local task su - `browser_automation/` - Agentic browser control (stateful `navigate → observe → click/type` loop) via Playwright, distinct from the read-only `web_fetch`/`web_capture` tools. Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close` (config `group: browser`). A process-local `BrowserSessionManager` owns one private, loop-affine Playwright event-loop thread (same pattern as the BoxLite provider) so a per-thread browser session survives across turns regardless of the caller's loop (Gateway / TUI / test). Each action returns a fresh page snapshot whose interactive elements are addressed by a stable numeric `[ref]` index (stamped as `data-df-ref` during snapshot), so the model acts on what it just observed instead of holding stale handles or guessing selectors. URLs are SSRF-screened via the shared `validate_public_http_url` (opt-out `allow_private_addresses` only for intentional internal targets). CDP attachment cannot install the request guard on an existing Chrome context, so `cdp_url` fails closed unless the operator explicitly sets `allow_unguarded_cdp: true` for a trusted local browser. Browser REST/Live access also requires an exact non-NULL thread owner, rather than the general legacy shared-thread policy, because retained pages may contain authenticated state. Session admission is a hard `max_sessions` cap: pinned Live/operation sessions are never evicted, and a new thread is rejected when no unpinned session can be closed; one Live viewer owns a session at a time. Optional dependency: `cd backend && uv sync --extra browser && uv run playwright install chromium`; `scripts/detect_uv_extras.py` preserves the extra when `config.yaml` enables `browser_navigate`, and Gateway startup fails fast if configured browser control cannot import Playwright. Tests: `tests/test_browser_automation.py` (mocked tools + a real-Chromium integration test guarded by `importorskip`); `tests/manual_browser_live_check.py` is a manual DeepSeek-driven end-to-end check (not collected by pytest). Live UI input dispatch is kept independent from JPEG capture: non-move actions start a rate-limited background refresh loop, so pointer, wheel, or keyboard input stays responsive while continuous gestures still produce frames throughout the interaction. -Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `tencent_wsa`, `tenki`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. +Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `tencent_wsa`, `tenki`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. E2B output sync records remote file versions and actual host file metadata in a thread-local manifest. The manifest binds to the remote sandbox ID. A complete output listing removes entries for deleted files. This avoids repeat downloads when the host filesystem rounds modification times. A single release-time sync pass is bounded by aggregate ceilings (`_MAX_SYNC_TOTAL_BYTES`, `_MAX_SYNC_FILES`, `_SYNC_DEADLINE_SECONDS`) on top of the per-file `_MAX_DOWNLOAD_SIZE` cap, so a pathological outputs tree cannot make release download unboundedly; a truncated pass logs what it dropped and leaves the manifest un-pruned (only entries observed in that pass are reconciled), so files it never reached are retried on the next release rather than being forgotten. diff --git a/backend/tests/test_doctor.py b/backend/tests/test_doctor.py index 88523bae1..d92b145e5 100644 --- a/backend/tests/test_doctor.py +++ b/backend/tests/test_doctor.py @@ -367,6 +367,22 @@ class TestCheckWebSearch: assert "tencent_wsa configured but TENCENTCLOUD_WSA_APIKEY not set" in result.detail assert "TENCENTCLOUD_WSA_APIKEY" in (result.fix or "") + def test_serply_with_key_ok(self, tmp_path, monkeypatch): + monkeypatch.setenv("SERPLY_API_KEY", "test-key") + cfg = tmp_path / "config.yaml" + cfg.write_text("config_version: 5\ntools:\n - name: web_search\n use: deerflow.community.serply.tools:web_search_tool\n") + result = doctor.check_web_search(cfg) + assert result.status == "ok" + assert "serply" in result.detail + + def test_serply_without_key_warns(self, tmp_path, monkeypatch): + monkeypatch.delenv("SERPLY_API_KEY", raising=False) + cfg = tmp_path / "config.yaml" + cfg.write_text("config_version: 5\ntools:\n - name: web_search\n use: deerflow.community.serply.tools:web_search_tool\n") + result = doctor.check_web_search(cfg) + assert result.status == "warn" + assert "SERPLY_API_KEY" in (result.fix or "") + def test_no_search_tool_warns(self, tmp_path): cfg = tmp_path / "config.yaml" cfg.write_text("config_version: 5\ntools: []\n") diff --git a/backend/tests/test_serply_tools.py b/backend/tests/test_serply_tools.py new file mode 100644 index 000000000..89fed878e --- /dev/null +++ b/backend/tests/test_serply_tools.py @@ -0,0 +1,318 @@ +"""Unit tests for the Serply community web search tool.""" + +import json +import logging +from unittest.mock import MagicMock, patch + +import httpx +import pytest + + +@pytest.fixture(autouse=True) +def reset_api_key_warned(): + """Reset the module-level warning flag before each test.""" + import deerflow.community.serply.tools as serply_mod + + serply_mod._api_key_warned = set() + yield + serply_mod._api_key_warned = set() + + +def _patch_config(extra: dict | None): + """Patch get_app_config so web_search resolves to a tool config with ``extra``.""" + patcher = patch("deerflow.community.serply.tools.get_app_config") + mock = patcher.start() + if extra is None: + mock.return_value.get_tool_config.return_value = None + else: + tool_config = MagicMock() + tool_config.model_extra = extra + mock.return_value.get_tool_config.return_value = tool_config + return patcher, mock + + +@pytest.fixture +def mock_config_with_key(): + patcher, mock = _patch_config({"api_key": "test-serply-key", "max_results": 5}) + yield mock + patcher.stop() + + +@pytest.fixture +def mock_config_no_key(): + patcher, mock = _patch_config({}) + yield mock + patcher.stop() + + +def _make_response(payload: object) -> MagicMock: + mock_resp = MagicMock() + mock_resp.json.return_value = payload + mock_resp.raise_for_status = MagicMock() + return mock_resp + + +def _search_rows(n: int) -> list[dict]: + return [{"title": f"Result {i}", "link": f"https://example.com/{i}", "description": f"Snippet {i}", "position": i} for i in range(1, n + 1)] + + +def _run(query: str = "test query", max_results: int = 5) -> dict: + from deerflow.community.serply.tools import web_search_tool + + return json.loads(web_search_tool.invoke({"query": query, "max_results": max_results})) + + +class TestGetApiKey: + def test_returns_config_key_when_present(self): + patcher, _ = _patch_config({"api_key": "from-config"}) + try: + from deerflow.community.serply.tools import _get_api_key + + assert _get_api_key("web_search") == "from-config" + finally: + patcher.stop() + + def test_falls_back_to_env_when_config_key_blank(self): + patcher, _ = _patch_config({"api_key": " "}) + try: + with patch.dict("os.environ", {"SERPLY_API_KEY": "env-key"}): + from deerflow.community.serply.tools import _get_api_key + + assert _get_api_key("web_search") == "env-key" + finally: + patcher.stop() + + def test_falls_back_to_env_when_no_config(self): + patcher, _ = _patch_config(None) + try: + with patch.dict("os.environ", {"SERPLY_API_KEY": "env-only"}): + from deerflow.community.serply.tools import _get_api_key + + assert _get_api_key("web_search") == "env-only" + finally: + patcher.stop() + + def test_returns_none_when_no_key_anywhere(self): + patcher, _ = _patch_config(None) + try: + with patch.dict("os.environ", {}, clear=True): + from deerflow.community.serply.tools import _get_api_key + + assert _get_api_key("web_search") is None + finally: + patcher.stop() + + +class TestCoerceMaxResults: + def test_returns_value_when_valid(self): + from deerflow.community.serply.tools import _coerce_max_results + + assert _coerce_max_results(3) == 3 + assert _coerce_max_results("7") == 7 + + def test_caps_at_serply_maximum(self): + from deerflow.community.serply.tools import _coerce_max_results + + assert _coerce_max_results(999) == 100 + + def test_invalid_values_fall_back_to_default(self): + from deerflow.community.serply.tools import _coerce_max_results + + assert _coerce_max_results("oops") == 5 + assert _coerce_max_results(None) == 5 + assert _coerce_max_results(0) == 5 + assert _coerce_max_results(-3) == 5 + + +class TestCoerceVertical: + def test_accepts_known_verticals(self): + from deerflow.community.serply.tools import _coerce_vertical + + assert _coerce_vertical(None) == "search" + assert _coerce_vertical("news") == "news" + assert _coerce_vertical(" Scholar ") == "scholar" + + def test_unknown_vertical_falls_back_to_search(self, caplog): + from deerflow.community.serply.tools import _coerce_vertical + + with caplog.at_level(logging.WARNING): + assert _coerce_vertical("images") == "search" + assert "Invalid Serply vertical" in caplog.text + + +class TestWebSearchTool: + def test_basic_search_returns_normalized_results(self, mock_config_with_key): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": _search_rows(2)}) + result = _run("test query") + + assert result["query"] == "test query" + assert result["total_results"] == 2 + assert result["results"][0] == {"title": "Result 1", "url": "https://example.com/1", "content": "Snippet 1"} + + def test_sends_correct_headers_and_params(self, mock_config_with_key): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_get = mock_client.return_value.__enter__.return_value.get + mock_get.return_value = _make_response({"results": _search_rows(1)}) + _run(" padded query ") + + args, kwargs = mock_get.call_args + assert args[0] == "https://api.serply.io/v1/search/" + assert kwargs["headers"]["X-Api-Key"] == "test-serply-key" + assert kwargs["headers"]["Accept"] == "application/json" + assert kwargs["params"] == {"q": "padded query", "num": 5} + + def test_config_max_results_overrides_parameter(self): + patcher, _ = _patch_config({"api_key": "k", "max_results": 2}) + try: + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_get = mock_client.return_value.__enter__.return_value.get + mock_get.return_value = _make_response({"results": _search_rows(5)}) + result = _run("q", max_results=5) + finally: + patcher.stop() + + assert mock_get.call_args.kwargs["params"]["num"] == 2 + assert result["total_results"] == 2 + + def test_passes_through_locale_params_from_config(self): + patcher, _ = _patch_config({"api_key": "k", "gl": "fr", "hl": "fr", "ignored": "x"}) + try: + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_get = mock_client.return_value.__enter__.return_value.get + mock_get.return_value = _make_response({"results": _search_rows(1)}) + _run("q") + finally: + patcher.stop() + + params = mock_get.call_args.kwargs["params"] + assert params["gl"] == "fr" + assert params["hl"] == "fr" + assert "ignored" not in params + + def test_news_vertical_uses_news_endpoint_and_trims_client_side(self): + entries = [ + { + "title": f"Story {i}", + "link": f"https://news.example.com/{i}", + "summary": "Genuine attention & chatbots", + "published": "Mon, 01 Jun 2026 08:00:00 GMT", + "source": {"title": "Example Times"}, + } + for i in range(10) + ] + patcher, _ = _patch_config({"api_key": "k", "max_results": 3, "vertical": "news"}) + try: + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_get = mock_client.return_value.__enter__.return_value.get + mock_get.return_value = _make_response({"entries": entries}) + result = _run("q") + finally: + patcher.stop() + + assert mock_get.call_args.args[0] == "https://api.serply.io/v1/news/" + assert result["total_results"] == 3 + first = result["results"][0] + assert first["content"] == "Genuine\xa0attention & chatbots" + assert first["published"] == "Mon, 01 Jun 2026 08:00:00 GMT" + assert first["source"] == "Example Times" + + def test_scholar_vertical_maps_authors_and_citations(self): + articles = [ + { + "title": "Attention Is All You Need", + "link": "https://arxiv.org/abs/1706.03762", + "description": "The dominant sequence transduction models...", + "author": { + "names": "A Vaswani, N Shazeer - NeurIPS, 2017", + "authors": [{"name": "A Vaswani", "link": "https://openalex.org/A1"}, {"name": "N Shazeer", "link": "https://openalex.org/A2"}], + }, + "extras": {"citations": {"count": 120000}}, + "doc": {"link": "https://arxiv.org/pdf/1706.03762", "type": "PDF"}, + }, + {"title": "No metadata", "link": "https://example.org/paper"}, + ] + patcher, _ = _patch_config({"api_key": "k", "vertical": "scholar"}) + try: + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_get = mock_client.return_value.__enter__.return_value.get + mock_get.return_value = _make_response({"articles": articles}) + result = _run("transformers") + finally: + patcher.stop() + + assert mock_get.call_args.args[0] == "https://api.serply.io/v1/scholar/" + assert result["results"][0]["authors"] == ["A Vaswani", "N Shazeer"] + assert result["results"][0]["cited_by"] == 120000 + assert result["results"][0]["pdf_url"] == "https://arxiv.org/pdf/1706.03762" + assert result["results"][1] == {"title": "No metadata", "url": "https://example.org/paper", "content": "", "authors": [], "cited_by": 0, "pdf_url": ""} + + def test_empty_results_returns_error_json(self, mock_config_with_key): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": []}) + assert _run("nothing") == {"error": "No results found", "query": "nothing"} + + def test_missing_results_key_is_treated_as_no_results(self, mock_config_with_key): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": None}) + assert _run("nothing")["error"] == "No results found" + + def test_unexpected_payload_shape_returns_error_json(self, mock_config_with_key): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": "not-a-list"}) + assert "unexpected response format" in _run("q")["error"] + + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_client.return_value.__enter__.return_value.get.return_value = _make_response(["not", "a", "dict"]) + assert "unexpected response format" in _run("q")["error"] + + def test_missing_api_key_returns_error_json_and_warns_once(self, mock_config_no_key, caplog): + with patch.dict("os.environ", {}, clear=True), caplog.at_level(logging.WARNING): + first = _run("q") + second = _run("q") + + assert first == {"error": "SERPLY_API_KEY is not configured", "query": "q"} + assert second == first + warnings = [r for r in caplog.records if r.levelno == logging.WARNING] + assert len(warnings) == 1 + assert "SERPLY_API_KEY" in warnings[0].getMessage() + + def test_http_error_returns_structured_error(self, mock_config_with_key): + mock_resp = MagicMock() + mock_resp.status_code = 403 + mock_resp.text = "Forbidden" + mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError("403", request=MagicMock(), response=mock_resp) + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_client.return_value.__enter__.return_value.get.return_value = mock_resp + result = _run("q") + + assert result == {"error": "Serply API error: HTTP 403", "query": "q"} + + def test_network_exception_returns_error_json(self, mock_config_with_key): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_client.return_value.__enter__.return_value.get.side_effect = httpx.ConnectError("boom") + result = _run("q") + + assert result == {"error": "boom", "query": "q"} + + def test_long_query_is_truncated(self, mock_config_with_key): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_get = mock_client.return_value.__enter__.return_value.get + mock_get.return_value = _make_response({"results": _search_rows(1)}) + _run("x" * 600) + + assert len(mock_get.call_args.kwargs["params"]["q"]) == 500 + + def test_uses_env_key_when_config_absent(self): + patcher, _ = _patch_config(None) + try: + with patch.dict("os.environ", {"SERPLY_API_KEY": "env-key"}): + with patch("deerflow.community.serply.tools.httpx.Client") as mock_client: + mock_get = mock_client.return_value.__enter__.return_value.get + mock_get.return_value = _make_response({"results": _search_rows(1)}) + result = _run("q") + finally: + patcher.stop() + + assert result["total_results"] == 1 + assert mock_get.call_args.kwargs["headers"]["X-Api-Key"] == "env-key" diff --git a/config.example.yaml b/config.example.yaml index 4377f3619..0d50d0cd7 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -730,6 +730,18 @@ tools: # max_results: 5 # capped at 10 by the Serper provider # # api_key: $SERPER_API_KEY # Optional explicit env-var reference + # Web search tool (uses Serply - Google Search API, requires SERPLY_API_KEY) + # Serply returns live Google results and covers Google News and Google Scholar + # with the same key. Sign up at https://serply.io (docs: https://serply.io/docs) + # - name: web_search + # group: web + # use: deerflow.community.serply.tools:web_search_tool + # max_results: 5 # Serply accepts 1-100 per request + # # vertical: search # search (default), news, or scholar + # # gl: us # Optional country code for the results + # # hl: en # Optional interface language + # # api_key: $SERPLY_API_KEY # Optional if the env var is set + # Web search tool (uses Brave Search API, requires BRAVE_SEARCH_API_KEY) # Brave Search returns results from an independent index. Sign up at # https://brave.com/search/api/ to get a key. Unlike the DuckDuckGo diff --git a/frontend/src/content/en/harness/tools.mdx b/frontend/src/content/en/harness/tools.mdx index 5fc6dc1d8..c3294c784 100644 --- a/frontend/src/content/en/harness/tools.mdx +++ b/frontend/src/content/en/harness/tools.mdx @@ -113,7 +113,7 @@ Community tools connect the agent to external services. They are configured in ` ### Web search - + ```yaml tools: @@ -140,6 +140,18 @@ tools: ``` Results from Brave's independent index via the official [Brave Search API](https://brave.com/search/api/). Requires an API key. `max_results` is capped at 20. +No extra dependency required. + + + +```yaml +tools: + - use: deerflow.community.serply.tools:web_search_tool + api_key: $SERPLY_API_KEY + # vertical: news # or scholar; defaults to regular Google Search +``` +Live Google results as JSON. Requires a [Serply](https://serply.io) API key. Set `vertical: news` or `vertical: scholar` to search Google News or Google Scholar with the same key; `gl` and `hl` set the result country and language. See the [Serply docs](https://serply.io/docs) for details. + No extra dependency required. diff --git a/frontend/src/content/zh/harness/tools.mdx b/frontend/src/content/zh/harness/tools.mdx index ba544febd..a701c10d4 100644 --- a/frontend/src/content/zh/harness/tools.mdx +++ b/frontend/src/content/zh/harness/tools.mdx @@ -110,7 +110,7 @@ tools: ### 网络搜索 - + ```yaml tools: @@ -137,6 +137,18 @@ tools: ``` 通过官方 [Brave Search API](https://brave.com/search/api/) 返回 Brave 独立索引的结果。需要 API Key，`max_results` 上限为 20。 +无需额外依赖。 + + + +```yaml +tools: + - use: deerflow.community.serply.tools:web_search_tool + api_key: $SERPLY_API_KEY + # vertical: news # 或 scholar；默认为普通 Google 搜索 +``` +以 JSON 形式返回实时 Google 搜索结果。需要 [Serply](https://serply.io) API Key。设置 `vertical: news` 或 `vertical: scholar` 可用同一个 Key 搜索 Google News 或 Google Scholar；`gl` 和 `hl` 用于设置结果的国家和语言。详见 [Serply 文档](https://serply.io/docs)。 + 无需额外依赖。 diff --git a/scripts/doctor.py b/scripts/doctor.py index b77485c50..3304b6f09 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -496,6 +496,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes "fastcrw": "CRW_API_KEY", "brave": "BRAVE_SEARCH_API_KEY", "serper": "SERPER_API_KEY", + "serply": "SERPLY_API_KEY", "tencent_wsa": "TENCENTCLOUD_WSA_APIKEY", }, "web_fetch": { diff --git a/scripts/wizard/providers.py b/scripts/wizard/providers.py index bb60fdf92..f655b2bb9 100644 --- a/scripts/wizard/providers.py +++ b/scripts/wizard/providers.py @@ -606,6 +606,14 @@ SEARCH_PROVIDERS: list[SearchProvider] = [ env_var="BRAVE_SEARCH_API_KEY", extra_config={"max_results": 5}, ), + SearchProvider( + name="serply", + display_name="Serply", + description="Google Search, News and Scholar results, API key required", + use="deerflow.community.serply.tools:web_search_tool", + env_var="SERPLY_API_KEY", + extra_config={"max_results": 5}, + ), SearchProvider( name="groundroute", display_name="GroundRoute",