mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-09 13:39:26 +00:00
feat(community): add Serply web search tool (#5023)
Add deerflow.community.serply.tools:web_search_tool, a Google SERP provider for the web_search slot that also covers Google News and Google Scholar through an optional `vertical` config option. Reads the key from api_key in config.yaml or SERPLY_API_KEY, clamps max_results to Serply's 1-100 range, and returns the same structured JSON errors as the Serper and Brave tools. Register the provider in config.example.yaml, scripts/doctor.py, scripts/wizard/providers.py, .env.example, backend/docs/CONFIGURATION.md, the en/zh tools.mdx provider tabs, and tools/AGENTS.md. Tests mock httpx.
This commit is contained in:
Serply
GitHub
parent
e09b2d48df
commit
4dbfe37ff3
@ -1,6 +1,9 @@
|
||||
# Serper API Key (Google Search) - https://serper.dev
|
||||
SERPER_API_KEY=your-serper-api-key
|
||||
|
||||
# Serply API Key (Google Search, News and Scholar) - https://serply.io
|
||||
SERPLY_API_KEY=your-serply-api-key
|
||||
|
||||
# TAVILY API Key
|
||||
TAVILY_API_KEY=your-tavily-api-key
|
||||
|
||||
|
||||
@ -367,7 +367,7 @@ tools:
|
||||
```
|
||||
|
||||
**Built-in Tools**:
|
||||
- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute)
|
||||
- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Serply, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute)
|
||||
- `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless)
|
||||
- `web_capture` - Capture rendered webpage screenshots as artifacts (Browserless)
|
||||
- `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper, Brave)
|
||||
@ -809,6 +809,7 @@ models:
|
||||
- `TAVILY_API_KEY` - Tavily search API key
|
||||
- `BRAVE_SEARCH_API_KEY` - Brave Search API key for `web_search` and `image_search`
|
||||
- `SERPER_API_KEY` - Serper (Google Search/Images API) key for `web_search` and `image_search`
|
||||
- `SERPLY_API_KEY` - [Serply](https://serply.io) key for `web_search` (Google Search, plus Google News and Google Scholar via `vertical`)
|
||||
- `GROUNDROUTE_API_KEY` - GroundRoute meta-search API key for `web_search` and `web_fetch` (routes across Serper, Brave, Exa, Tavily, Firecrawl, Perplexity with gain-share pricing)
|
||||
- `BROWSERLESS_TOKEN` - Browserless Cloud token for `web_capture` (optional for self-hosted Browserless)
|
||||
- `DEER_FLOW_PROJECT_ROOT` - Project root for relative runtime paths
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
from .tools import web_search_tool
|
||||
|
||||
__all__ = ["web_search_tool"]
|
||||
211
backend/packages/harness/deerflow/community/serply/tools.py
Normal file
211
backend/packages/harness/deerflow/community/serply/tools.py
Normal file
@ -0,0 +1,211 @@
|
||||
"""
|
||||
Web search tool powered by the Serply API.
|
||||
|
||||
Serply returns live Google results as JSON. One API key covers the regular
|
||||
web SERP plus the Google News and Google Scholar verticals, so a research run
|
||||
can be pointed at recent coverage or at papers by switching ``vertical`` in
|
||||
config.yaml. An API key is required. Sign up at https://serply.io and see
|
||||
https://serply.io/docs for the endpoint reference.
|
||||
"""
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
import httpx
|
||||
from langchain.tools import tool
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SERPLY_BASE_URL = "https://api.serply.io/v1"
|
||||
_DEFAULT_MAX_RESULTS = 5
|
||||
# Serply accepts ``num`` values from 1 to 100 per request.
|
||||
_SERPLY_MAX_RESULTS = 100
|
||||
_DEFAULT_VERTICAL = "search"
|
||||
# vertical -> (URL path segment, response key that holds the result rows)
|
||||
_VERTICALS: dict[str, tuple[str, str]] = {
|
||||
"search": ("search", "results"),
|
||||
"news": ("news", "entries"),
|
||||
"scholar": ("scholar", "articles"),
|
||||
}
|
||||
# Optional request parameters that are passed through from config.yaml as-is.
|
||||
_PASSTHROUGH_PARAMS = ("gl", "hl")
|
||||
_TAG_RE = re.compile(r"<[^>]+>")
|
||||
_api_key_warned: set[str] = set()
|
||||
|
||||
|
||||
def _get_api_key(tool_name: str = "web_search") -> str | None:
|
||||
config = get_app_config().get_tool_config(tool_name)
|
||||
if config is not None:
|
||||
api_key = (config.model_extra or {}).get("api_key")
|
||||
if isinstance(api_key, str) and api_key.strip():
|
||||
return api_key.strip()
|
||||
env_key = os.getenv("SERPLY_API_KEY")
|
||||
if isinstance(env_key, str) and env_key.strip():
|
||||
return env_key.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_max_results(
|
||||
value: object,
|
||||
*,
|
||||
default: int = _DEFAULT_MAX_RESULTS,
|
||||
max_allowed: int = _SERPLY_MAX_RESULTS,
|
||||
) -> int:
|
||||
try:
|
||||
coerced = int(value)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Invalid Serply max_results=%r; using default %s", value, default)
|
||||
coerced = default
|
||||
if coerced < 1:
|
||||
logger.warning("Invalid Serply max_results=%r; using default %s", value, default)
|
||||
coerced = default
|
||||
return min(coerced, max_allowed)
|
||||
|
||||
|
||||
def _coerce_vertical(value: object) -> str:
|
||||
if value is None:
|
||||
return _DEFAULT_VERTICAL
|
||||
if isinstance(value, str) and value.strip().lower() in _VERTICALS:
|
||||
return value.strip().lower()
|
||||
logger.warning("Invalid Serply vertical=%r; using %r (one of %s)", value, _DEFAULT_VERTICAL, sorted(_VERTICALS))
|
||||
return _DEFAULT_VERTICAL
|
||||
|
||||
|
||||
def _clean_query(query: str, *, max_length: int = 500) -> str:
|
||||
query = query.strip()
|
||||
if len(query) > max_length:
|
||||
query = query[:max_length]
|
||||
return query
|
||||
|
||||
|
||||
def _clean_text(value: object) -> str:
|
||||
"""Flatten a Serply text field to plain text (news summaries arrive as HTML)."""
|
||||
if not isinstance(value, str):
|
||||
return ""
|
||||
return html.unescape(_TAG_RE.sub("", value)).strip()
|
||||
|
||||
|
||||
def _missing_key_error(query: str, tool_name: str) -> str:
|
||||
if tool_name not in _api_key_warned:
|
||||
_api_key_warned.add(tool_name)
|
||||
logger.warning(
|
||||
"Serply API key is not set for '%s'. Set SERPLY_API_KEY in your environment or provide api_key in config.yaml. Sign up at https://serply.io",
|
||||
tool_name,
|
||||
)
|
||||
return json.dumps({"error": "SERPLY_API_KEY is not configured", "query": query}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _unexpected_format_error(query: str) -> str:
|
||||
return json.dumps({"error": "Serply returned an unexpected response format", "query": query}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _serply_get(path: str, api_key: str, query: str, params: dict[str, object]) -> tuple[dict | None, str | None]:
|
||||
"""Send a GET request to a Serply endpoint.
|
||||
|
||||
Returns a ``(data, error_json)`` tuple: on success ``data`` is the parsed
|
||||
JSON response and ``error_json`` is ``None``; on failure ``data`` is ``None``
|
||||
and ``error_json`` is a serialized structured error ready to return.
|
||||
"""
|
||||
headers = {
|
||||
"X-Api-Key": api_key,
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "deerflow",
|
||||
}
|
||||
try:
|
||||
with httpx.Client(timeout=30) as client:
|
||||
response = client.get(f"{_SERPLY_BASE_URL}/{path}/", headers=headers, params=params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if not isinstance(data, dict):
|
||||
logger.error("Serply returned an unexpected payload type: %s", type(data).__name__)
|
||||
return None, _unexpected_format_error(query)
|
||||
return data, None
|
||||
except httpx.HTTPStatusError as e:
|
||||
resp_text = (e.response.text or "")[:500]
|
||||
logger.error("Serply API returned HTTP %s: %s", e.response.status_code, resp_text)
|
||||
return None, json.dumps({"error": f"Serply API error: HTTP {e.response.status_code}", "query": query}, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
logger.error("Serply request failed: %s: %s", type(e).__name__, str(e)[:500])
|
||||
return None, json.dumps({"error": str(e)[:500], "query": query}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _normalize_row(vertical: str, row: dict) -> dict:
|
||||
"""Map one Serply row onto the common ``title``/``url``/``content`` shape.
|
||||
|
||||
News and Scholar rows carry a few extra fields worth surfacing to the model.
|
||||
"""
|
||||
result = {"title": row.get("title", ""), "url": row.get("link", "")}
|
||||
if vertical == "news":
|
||||
result["content"] = _clean_text(row.get("summary"))
|
||||
result["published"] = row.get("published", "")
|
||||
source = row.get("source") if isinstance(row.get("source"), dict) else {}
|
||||
result["source"] = source.get("title", "")
|
||||
elif vertical == "scholar":
|
||||
result["content"] = row.get("description", "")
|
||||
author = row.get("author") if isinstance(row.get("author"), dict) else {}
|
||||
authors = author.get("authors") if isinstance(author.get("authors"), list) else []
|
||||
result["authors"] = [a.get("name", "") for a in authors if isinstance(a, dict)]
|
||||
extras = row.get("extras") if isinstance(row.get("extras"), dict) else {}
|
||||
citations = extras.get("citations") if isinstance(extras.get("citations"), dict) else {}
|
||||
result["cited_by"] = citations.get("count", 0)
|
||||
doc = row.get("doc") if isinstance(row.get("doc"), dict) else {}
|
||||
result["pdf_url"] = doc.get("link", "")
|
||||
else:
|
||||
result["content"] = row.get("description", "")
|
||||
return result
|
||||
|
||||
|
||||
@tool("web_search", parse_docstring=True)
|
||||
def web_search_tool(query: str, max_results: int = 5) -> str:
|
||||
"""Search the web for information using Google Search via Serply.
|
||||
|
||||
Args:
|
||||
query: Search keywords describing what you want to find. Be specific for better results.
|
||||
max_results: Maximum number of search results to return. Default is 5, capped at 100.
|
||||
"""
|
||||
config = get_app_config().get_tool_config("web_search")
|
||||
extra = (config.model_extra or {}) if config is not None else {}
|
||||
if "max_results" in extra:
|
||||
max_results = extra["max_results"]
|
||||
max_results = _coerce_max_results(max_results)
|
||||
vertical = _coerce_vertical(extra.get("vertical"))
|
||||
query = _clean_query(query)
|
||||
|
||||
api_key = _get_api_key("web_search")
|
||||
if not api_key:
|
||||
return _missing_key_error(query, "web_search")
|
||||
|
||||
path, rows_key = _VERTICALS[vertical]
|
||||
params: dict[str, object] = {"q": query, "num": max_results}
|
||||
for key in _PASSTHROUGH_PARAMS:
|
||||
if key in extra:
|
||||
params[key] = extra[key]
|
||||
|
||||
data, error_json = _serply_get(path, api_key, query, params)
|
||||
if error_json is not None:
|
||||
return error_json
|
||||
|
||||
rows = data.get(rows_key)
|
||||
if rows is None:
|
||||
rows = []
|
||||
if not isinstance(rows, list):
|
||||
logger.error("Serply returned unexpected '%s' payload type: %s", rows_key, type(rows).__name__)
|
||||
return _unexpected_format_error(query)
|
||||
rows = [row for row in rows if isinstance(row, dict)]
|
||||
if not rows:
|
||||
return json.dumps({"error": "No results found", "query": query}, ensure_ascii=False)
|
||||
|
||||
# The news feed ignores ``num`` server-side, so the cap is also applied here.
|
||||
normalized_results = [_normalize_row(vertical, row) for row in rows[:max_results]]
|
||||
|
||||
output = {
|
||||
"query": query,
|
||||
"total_results": len(normalized_results),
|
||||
"results": normalized_results,
|
||||
}
|
||||
return json.dumps(output, indent=2, ensure_ascii=False)
|
||||
@ -28,7 +28,7 @@ Durable MCP task-management tools are added only while the process-local task su
|
||||
- `browser_automation/` - Agentic browser control (stateful `navigate → observe → click/type` loop) via Playwright, distinct from the read-only `web_fetch`/`web_capture` tools. Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close` (config `group: browser`). A process-local `BrowserSessionManager` owns one private, loop-affine Playwright event-loop thread (same pattern as the BoxLite provider) so a per-thread browser session survives across turns regardless of the caller's loop (Gateway / TUI / test). Each action returns a fresh page snapshot whose interactive elements are addressed by a stable numeric `[ref]` index (stamped as `data-df-ref` during snapshot), so the model acts on what it just observed instead of holding stale handles or guessing selectors. URLs are SSRF-screened via the shared `validate_public_http_url` (opt-out `allow_private_addresses` only for intentional internal targets). CDP attachment cannot install the request guard on an existing Chrome context, so `cdp_url` fails closed unless the operator explicitly sets `allow_unguarded_cdp: true` for a trusted local browser. Browser REST/Live access also requires an exact non-NULL thread owner, rather than the general legacy shared-thread policy, because retained pages may contain authenticated state. Session admission is a hard `max_sessions` cap: pinned Live/operation sessions are never evicted, and a new thread is rejected when no unpinned session can be closed; one Live viewer owns a session at a time. Optional dependency: `cd backend && uv sync --extra browser && uv run playwright install chromium`; `scripts/detect_uv_extras.py` preserves the extra when `config.yaml` enables `browser_navigate`, and Gateway startup fails fast if configured browser control cannot import Playwright. Tests: `tests/test_browser_automation.py` (mocked tools + a real-Chromium integration test guarded by `importorskip`); `tests/manual_browser_live_check.py` is a manual DeepSeek-driven end-to-end check (not collected by pytest).
|
||||
Live UI input dispatch is kept independent from JPEG capture: non-move actions start a rate-limited background refresh loop, so pointer, wheel, or keyboard input stays responsive while continuous gestures still produce frames throughout the interaction.
|
||||
|
||||
Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `tencent_wsa`, `tenki`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional.
|
||||
Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `tencent_wsa`, `tenki`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional.
|
||||
|
||||
E2B output sync records remote file versions and actual host file metadata in a thread-local manifest. The manifest binds to the remote sandbox ID. A complete output listing removes entries for deleted files. This avoids repeat downloads when the host filesystem rounds modification times. A single release-time sync pass is bounded by aggregate ceilings (`_MAX_SYNC_TOTAL_BYTES`, `_MAX_SYNC_FILES`, `_SYNC_DEADLINE_SECONDS`) on top of the per-file `_MAX_DOWNLOAD_SIZE` cap, so a pathological outputs tree cannot make release download unboundedly; a truncated pass logs what it dropped and leaves the manifest un-pruned (only entries observed in that pass are reconciled), so files it never reached are retried on the next release rather than being forgotten.
|
||||
|
||||
|
||||
@ -367,6 +367,22 @@ class TestCheckWebSearch:
|
||||
assert "tencent_wsa configured but TENCENTCLOUD_WSA_APIKEY not set" in result.detail
|
||||
assert "TENCENTCLOUD_WSA_APIKEY" in (result.fix or "")
|
||||
|
||||
def test_serply_with_key_ok(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SERPLY_API_KEY", "test-key")
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text("config_version: 5\ntools:\n - name: web_search\n use: deerflow.community.serply.tools:web_search_tool\n")
|
||||
result = doctor.check_web_search(cfg)
|
||||
assert result.status == "ok"
|
||||
assert "serply" in result.detail
|
||||
|
||||
def test_serply_without_key_warns(self, tmp_path, monkeypatch):
|
||||
monkeypatch.delenv("SERPLY_API_KEY", raising=False)
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text("config_version: 5\ntools:\n - name: web_search\n use: deerflow.community.serply.tools:web_search_tool\n")
|
||||
result = doctor.check_web_search(cfg)
|
||||
assert result.status == "warn"
|
||||
assert "SERPLY_API_KEY" in (result.fix or "")
|
||||
|
||||
def test_no_search_tool_warns(self, tmp_path):
|
||||
cfg = tmp_path / "config.yaml"
|
||||
cfg.write_text("config_version: 5\ntools: []\n")
|
||||
|
||||
318
backend/tests/test_serply_tools.py
Normal file
318
backend/tests/test_serply_tools.py
Normal file
@ -0,0 +1,318 @@
|
||||
"""Unit tests for the Serply community web search tool."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_api_key_warned():
|
||||
"""Reset the module-level warning flag before each test."""
|
||||
import deerflow.community.serply.tools as serply_mod
|
||||
|
||||
serply_mod._api_key_warned = set()
|
||||
yield
|
||||
serply_mod._api_key_warned = set()
|
||||
|
||||
|
||||
def _patch_config(extra: dict | None):
|
||||
"""Patch get_app_config so web_search resolves to a tool config with ``extra``."""
|
||||
patcher = patch("deerflow.community.serply.tools.get_app_config")
|
||||
mock = patcher.start()
|
||||
if extra is None:
|
||||
mock.return_value.get_tool_config.return_value = None
|
||||
else:
|
||||
tool_config = MagicMock()
|
||||
tool_config.model_extra = extra
|
||||
mock.return_value.get_tool_config.return_value = tool_config
|
||||
return patcher, mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config_with_key():
|
||||
patcher, mock = _patch_config({"api_key": "test-serply-key", "max_results": 5})
|
||||
yield mock
|
||||
patcher.stop()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config_no_key():
|
||||
patcher, mock = _patch_config({})
|
||||
yield mock
|
||||
patcher.stop()
|
||||
|
||||
|
||||
def _make_response(payload: object) -> MagicMock:
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.json.return_value = payload
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
return mock_resp
|
||||
|
||||
|
||||
def _search_rows(n: int) -> list[dict]:
|
||||
return [{"title": f"Result {i}", "link": f"https://example.com/{i}", "description": f"Snippet {i}", "position": i} for i in range(1, n + 1)]
|
||||
|
||||
|
||||
def _run(query: str = "test query", max_results: int = 5) -> dict:
|
||||
from deerflow.community.serply.tools import web_search_tool
|
||||
|
||||
return json.loads(web_search_tool.invoke({"query": query, "max_results": max_results}))
|
||||
|
||||
|
||||
class TestGetApiKey:
|
||||
def test_returns_config_key_when_present(self):
|
||||
patcher, _ = _patch_config({"api_key": "from-config"})
|
||||
try:
|
||||
from deerflow.community.serply.tools import _get_api_key
|
||||
|
||||
assert _get_api_key("web_search") == "from-config"
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
def test_falls_back_to_env_when_config_key_blank(self):
|
||||
patcher, _ = _patch_config({"api_key": " "})
|
||||
try:
|
||||
with patch.dict("os.environ", {"SERPLY_API_KEY": "env-key"}):
|
||||
from deerflow.community.serply.tools import _get_api_key
|
||||
|
||||
assert _get_api_key("web_search") == "env-key"
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
def test_falls_back_to_env_when_no_config(self):
|
||||
patcher, _ = _patch_config(None)
|
||||
try:
|
||||
with patch.dict("os.environ", {"SERPLY_API_KEY": "env-only"}):
|
||||
from deerflow.community.serply.tools import _get_api_key
|
||||
|
||||
assert _get_api_key("web_search") == "env-only"
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
def test_returns_none_when_no_key_anywhere(self):
|
||||
patcher, _ = _patch_config(None)
|
||||
try:
|
||||
with patch.dict("os.environ", {}, clear=True):
|
||||
from deerflow.community.serply.tools import _get_api_key
|
||||
|
||||
assert _get_api_key("web_search") is None
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
|
||||
class TestCoerceMaxResults:
|
||||
def test_returns_value_when_valid(self):
|
||||
from deerflow.community.serply.tools import _coerce_max_results
|
||||
|
||||
assert _coerce_max_results(3) == 3
|
||||
assert _coerce_max_results("7") == 7
|
||||
|
||||
def test_caps_at_serply_maximum(self):
|
||||
from deerflow.community.serply.tools import _coerce_max_results
|
||||
|
||||
assert _coerce_max_results(999) == 100
|
||||
|
||||
def test_invalid_values_fall_back_to_default(self):
|
||||
from deerflow.community.serply.tools import _coerce_max_results
|
||||
|
||||
assert _coerce_max_results("oops") == 5
|
||||
assert _coerce_max_results(None) == 5
|
||||
assert _coerce_max_results(0) == 5
|
||||
assert _coerce_max_results(-3) == 5
|
||||
|
||||
|
||||
class TestCoerceVertical:
|
||||
def test_accepts_known_verticals(self):
|
||||
from deerflow.community.serply.tools import _coerce_vertical
|
||||
|
||||
assert _coerce_vertical(None) == "search"
|
||||
assert _coerce_vertical("news") == "news"
|
||||
assert _coerce_vertical(" Scholar ") == "scholar"
|
||||
|
||||
def test_unknown_vertical_falls_back_to_search(self, caplog):
|
||||
from deerflow.community.serply.tools import _coerce_vertical
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
assert _coerce_vertical("images") == "search"
|
||||
assert "Invalid Serply vertical" in caplog.text
|
||||
|
||||
|
||||
class TestWebSearchTool:
|
||||
def test_basic_search_returns_normalized_results(self, mock_config_with_key):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": _search_rows(2)})
|
||||
result = _run("test query")
|
||||
|
||||
assert result["query"] == "test query"
|
||||
assert result["total_results"] == 2
|
||||
assert result["results"][0] == {"title": "Result 1", "url": "https://example.com/1", "content": "Snippet 1"}
|
||||
|
||||
def test_sends_correct_headers_and_params(self, mock_config_with_key):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_get = mock_client.return_value.__enter__.return_value.get
|
||||
mock_get.return_value = _make_response({"results": _search_rows(1)})
|
||||
_run(" padded query ")
|
||||
|
||||
args, kwargs = mock_get.call_args
|
||||
assert args[0] == "https://api.serply.io/v1/search/"
|
||||
assert kwargs["headers"]["X-Api-Key"] == "test-serply-key"
|
||||
assert kwargs["headers"]["Accept"] == "application/json"
|
||||
assert kwargs["params"] == {"q": "padded query", "num": 5}
|
||||
|
||||
def test_config_max_results_overrides_parameter(self):
|
||||
patcher, _ = _patch_config({"api_key": "k", "max_results": 2})
|
||||
try:
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_get = mock_client.return_value.__enter__.return_value.get
|
||||
mock_get.return_value = _make_response({"results": _search_rows(5)})
|
||||
result = _run("q", max_results=5)
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert mock_get.call_args.kwargs["params"]["num"] == 2
|
||||
assert result["total_results"] == 2
|
||||
|
||||
def test_passes_through_locale_params_from_config(self):
|
||||
patcher, _ = _patch_config({"api_key": "k", "gl": "fr", "hl": "fr", "ignored": "x"})
|
||||
try:
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_get = mock_client.return_value.__enter__.return_value.get
|
||||
mock_get.return_value = _make_response({"results": _search_rows(1)})
|
||||
_run("q")
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
params = mock_get.call_args.kwargs["params"]
|
||||
assert params["gl"] == "fr"
|
||||
assert params["hl"] == "fr"
|
||||
assert "ignored" not in params
|
||||
|
||||
def test_news_vertical_uses_news_endpoint_and_trims_client_side(self):
|
||||
entries = [
|
||||
{
|
||||
"title": f"Story {i}",
|
||||
"link": f"https://news.example.com/{i}",
|
||||
"summary": "<a href='x'>Genuine attention</a> & chatbots",
|
||||
"published": "Mon, 01 Jun 2026 08:00:00 GMT",
|
||||
"source": {"title": "Example Times"},
|
||||
}
|
||||
for i in range(10)
|
||||
]
|
||||
patcher, _ = _patch_config({"api_key": "k", "max_results": 3, "vertical": "news"})
|
||||
try:
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_get = mock_client.return_value.__enter__.return_value.get
|
||||
mock_get.return_value = _make_response({"entries": entries})
|
||||
result = _run("q")
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert mock_get.call_args.args[0] == "https://api.serply.io/v1/news/"
|
||||
assert result["total_results"] == 3
|
||||
first = result["results"][0]
|
||||
assert first["content"] == "Genuine\xa0attention & chatbots"
|
||||
assert first["published"] == "Mon, 01 Jun 2026 08:00:00 GMT"
|
||||
assert first["source"] == "Example Times"
|
||||
|
||||
def test_scholar_vertical_maps_authors_and_citations(self):
|
||||
articles = [
|
||||
{
|
||||
"title": "Attention Is All You Need",
|
||||
"link": "https://arxiv.org/abs/1706.03762",
|
||||
"description": "The dominant sequence transduction models...",
|
||||
"author": {
|
||||
"names": "A Vaswani, N Shazeer - NeurIPS, 2017",
|
||||
"authors": [{"name": "A Vaswani", "link": "https://openalex.org/A1"}, {"name": "N Shazeer", "link": "https://openalex.org/A2"}],
|
||||
},
|
||||
"extras": {"citations": {"count": 120000}},
|
||||
"doc": {"link": "https://arxiv.org/pdf/1706.03762", "type": "PDF"},
|
||||
},
|
||||
{"title": "No metadata", "link": "https://example.org/paper"},
|
||||
]
|
||||
patcher, _ = _patch_config({"api_key": "k", "vertical": "scholar"})
|
||||
try:
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_get = mock_client.return_value.__enter__.return_value.get
|
||||
mock_get.return_value = _make_response({"articles": articles})
|
||||
result = _run("transformers")
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert mock_get.call_args.args[0] == "https://api.serply.io/v1/scholar/"
|
||||
assert result["results"][0]["authors"] == ["A Vaswani", "N Shazeer"]
|
||||
assert result["results"][0]["cited_by"] == 120000
|
||||
assert result["results"][0]["pdf_url"] == "https://arxiv.org/pdf/1706.03762"
|
||||
assert result["results"][1] == {"title": "No metadata", "url": "https://example.org/paper", "content": "", "authors": [], "cited_by": 0, "pdf_url": ""}
|
||||
|
||||
def test_empty_results_returns_error_json(self, mock_config_with_key):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": []})
|
||||
assert _run("nothing") == {"error": "No results found", "query": "nothing"}
|
||||
|
||||
def test_missing_results_key_is_treated_as_no_results(self, mock_config_with_key):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": None})
|
||||
assert _run("nothing")["error"] == "No results found"
|
||||
|
||||
def test_unexpected_payload_shape_returns_error_json(self, mock_config_with_key):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_client.return_value.__enter__.return_value.get.return_value = _make_response({"results": "not-a-list"})
|
||||
assert "unexpected response format" in _run("q")["error"]
|
||||
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_client.return_value.__enter__.return_value.get.return_value = _make_response(["not", "a", "dict"])
|
||||
assert "unexpected response format" in _run("q")["error"]
|
||||
|
||||
def test_missing_api_key_returns_error_json_and_warns_once(self, mock_config_no_key, caplog):
|
||||
with patch.dict("os.environ", {}, clear=True), caplog.at_level(logging.WARNING):
|
||||
first = _run("q")
|
||||
second = _run("q")
|
||||
|
||||
assert first == {"error": "SERPLY_API_KEY is not configured", "query": "q"}
|
||||
assert second == first
|
||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert len(warnings) == 1
|
||||
assert "SERPLY_API_KEY" in warnings[0].getMessage()
|
||||
|
||||
def test_http_error_returns_structured_error(self, mock_config_with_key):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 403
|
||||
mock_resp.text = "Forbidden"
|
||||
mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError("403", request=MagicMock(), response=mock_resp)
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_client.return_value.__enter__.return_value.get.return_value = mock_resp
|
||||
result = _run("q")
|
||||
|
||||
assert result == {"error": "Serply API error: HTTP 403", "query": "q"}
|
||||
|
||||
def test_network_exception_returns_error_json(self, mock_config_with_key):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_client.return_value.__enter__.return_value.get.side_effect = httpx.ConnectError("boom")
|
||||
result = _run("q")
|
||||
|
||||
assert result == {"error": "boom", "query": "q"}
|
||||
|
||||
def test_long_query_is_truncated(self, mock_config_with_key):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_get = mock_client.return_value.__enter__.return_value.get
|
||||
mock_get.return_value = _make_response({"results": _search_rows(1)})
|
||||
_run("x" * 600)
|
||||
|
||||
assert len(mock_get.call_args.kwargs["params"]["q"]) == 500
|
||||
|
||||
def test_uses_env_key_when_config_absent(self):
|
||||
patcher, _ = _patch_config(None)
|
||||
try:
|
||||
with patch.dict("os.environ", {"SERPLY_API_KEY": "env-key"}):
|
||||
with patch("deerflow.community.serply.tools.httpx.Client") as mock_client:
|
||||
mock_get = mock_client.return_value.__enter__.return_value.get
|
||||
mock_get.return_value = _make_response({"results": _search_rows(1)})
|
||||
result = _run("q")
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert result["total_results"] == 1
|
||||
assert mock_get.call_args.kwargs["headers"]["X-Api-Key"] == "env-key"
|
||||
@ -730,6 +730,18 @@ tools:
|
||||
# max_results: 5 # capped at 10 by the Serper provider
|
||||
# # api_key: $SERPER_API_KEY # Optional explicit env-var reference
|
||||
|
||||
# Web search tool (uses Serply - Google Search API, requires SERPLY_API_KEY)
|
||||
# Serply returns live Google results and covers Google News and Google Scholar
|
||||
# with the same key. Sign up at https://serply.io (docs: https://serply.io/docs)
|
||||
# - name: web_search
|
||||
# group: web
|
||||
# use: deerflow.community.serply.tools:web_search_tool
|
||||
# max_results: 5 # Serply accepts 1-100 per request
|
||||
# # vertical: search # search (default), news, or scholar
|
||||
# # gl: us # Optional country code for the results
|
||||
# # hl: en # Optional interface language
|
||||
# # api_key: $SERPLY_API_KEY # Optional if the env var is set
|
||||
|
||||
# Web search tool (uses Brave Search API, requires BRAVE_SEARCH_API_KEY)
|
||||
# Brave Search returns results from an independent index. Sign up at
|
||||
# https://brave.com/search/api/ to get a key. Unlike the DuckDuckGo
|
||||
|
||||
@ -113,7 +113,7 @@ Community tools connect the agent to external services. They are configured in `
|
||||
|
||||
### Web search
|
||||
|
||||
<Tabs items={["DuckDuckGo (default)", "Tavily", "Brave", "Exa", "InfoQuest", "Firecrawl", "GroundRoute"]}>
|
||||
<Tabs items={["DuckDuckGo (default)", "Tavily", "Brave", "Serply", "Exa", "InfoQuest", "Firecrawl", "GroundRoute"]}>
|
||||
<Tabs.Tab>
|
||||
```yaml
|
||||
tools:
|
||||
@ -140,6 +140,18 @@ tools:
|
||||
```
|
||||
Results from Brave's independent index via the official [Brave Search API](https://brave.com/search/api/). Requires an API key. `max_results` is capped at 20.
|
||||
|
||||
No extra dependency required.
|
||||
|
||||
</Tabs.Tab>
|
||||
<Tabs.Tab>
|
||||
```yaml
|
||||
tools:
|
||||
- use: deerflow.community.serply.tools:web_search_tool
|
||||
api_key: $SERPLY_API_KEY
|
||||
# vertical: news # or scholar; defaults to regular Google Search
|
||||
```
|
||||
Live Google results as JSON. Requires a [Serply](https://serply.io) API key. Set `vertical: news` or `vertical: scholar` to search Google News or Google Scholar with the same key; `gl` and `hl` set the result country and language. See the [Serply docs](https://serply.io/docs) for details.
|
||||
|
||||
No extra dependency required.
|
||||
|
||||
</Tabs.Tab>
|
||||
|
||||
@ -110,7 +110,7 @@ tools:
|
||||
|
||||
### 网络搜索
|
||||
|
||||
<Tabs items={["DuckDuckGo（默认）", "Tavily", "Brave", "Exa", "InfoQuest", "Firecrawl", "GroundRoute"]}>
|
||||
<Tabs items={["DuckDuckGo（默认）", "Tavily", "Brave", "Serply", "Exa", "InfoQuest", "Firecrawl", "GroundRoute"]}>
|
||||
<Tabs.Tab>
|
||||
```yaml
|
||||
tools:
|
||||
@ -137,6 +137,18 @@ tools:
|
||||
```
|
||||
通过官方 [Brave Search API](https://brave.com/search/api/) 返回 Brave 独立索引的结果。需要 API Key，`max_results` 上限为 20。
|
||||
|
||||
无需额外依赖。
|
||||
|
||||
</Tabs.Tab>
|
||||
<Tabs.Tab>
|
||||
```yaml
|
||||
tools:
|
||||
- use: deerflow.community.serply.tools:web_search_tool
|
||||
api_key: $SERPLY_API_KEY
|
||||
# vertical: news # 或 scholar；默认为普通 Google 搜索
|
||||
```
|
||||
以 JSON 形式返回实时 Google 搜索结果。需要 [Serply](https://serply.io) API Key。设置 `vertical: news` 或 `vertical: scholar` 可用同一个 Key 搜索 Google News 或 Google Scholar；`gl` 和 `hl` 用于设置结果的国家和语言。详见 [Serply 文档](https://serply.io/docs)。
|
||||
|
||||
无需额外依赖。
|
||||
|
||||
</Tabs.Tab>
|
||||
|
||||
@ -496,6 +496,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes
|
||||
"fastcrw": "CRW_API_KEY",
|
||||
"brave": "BRAVE_SEARCH_API_KEY",
|
||||
"serper": "SERPER_API_KEY",
|
||||
"serply": "SERPLY_API_KEY",
|
||||
"tencent_wsa": "TENCENTCLOUD_WSA_APIKEY",
|
||||
},
|
||||
"web_fetch": {
|
||||
|
||||
@ -606,6 +606,14 @@ SEARCH_PROVIDERS: list[SearchProvider] = [
|
||||
env_var="BRAVE_SEARCH_API_KEY",
|
||||
extra_config={"max_results": 5},
|
||||
),
|
||||
SearchProvider(
|
||||
name="serply",
|
||||
display_name="Serply",
|
||||
description="Google Search, News and Scholar results, API key required",
|
||||
use="deerflow.community.serply.tools:web_search_tool",
|
||||
env_var="SERPLY_API_KEY",
|
||||
extra_config={"max_results": 5},
|
||||
),
|
||||
SearchProvider(
|
||||
name="groundroute",
|
||||
display_name="GroundRoute",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user