mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-11 14:38:38 +00:00
feat(search): add Tencent Cloud WSA provider (#5057)
* feat(search): add Tencent Cloud WSA provider * docs: restore README to upstream * docs: remove README changes from WSA provider PR * fix(doctor): validate Tencent WSA API key
This commit is contained in:
parent
9600414002
commit
846c716523
@ -367,7 +367,7 @@ tools:
|
|||||||
```
|
```
|
||||||
|
|
||||||
**Built-in Tools**:
|
**Built-in Tools**:
|
||||||
- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute)
|
- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute)
|
||||||
- `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless)
|
- `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless)
|
||||||
- `web_capture` - Capture rendered webpage screenshots as artifacts (Browserless)
|
- `web_capture` - Capture rendered webpage screenshots as artifacts (Browserless)
|
||||||
- `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper, Brave)
|
- `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper, Brave)
|
||||||
|
|||||||
@ -0,0 +1 @@
|
|||||||
|
"""Tencent Cloud Web Search API community provider."""
|
||||||
260
backend/packages/harness/deerflow/community/tencent_wsa/tools.py
Normal file
260
backend/packages/harness/deerflow/community/tencent_wsa/tools.py
Normal file
@ -0,0 +1,260 @@
|
|||||||
|
"""Web search tool powered by Tencent Cloud Web Search API (WSA)."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from langchain.tools import tool
|
||||||
|
|
||||||
|
from deerflow.config import get_app_config
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_SEARCH_ENDPOINT = "https://api.wsa.cloud.tencent.com/SearchPro"
|
||||||
|
_API_KEY_ENV = "TENCENTCLOUD_WSA_APIKEY"
|
||||||
|
_DEFAULT_MAX_RESULTS = 5
|
||||||
|
_DEFAULT_API_RESULT_COUNT = 10
|
||||||
|
_MAX_RESULTS = 50
|
||||||
|
_REQUEST_TIMEOUT_S = 30.0
|
||||||
|
_api_key_warned: set[str] = set()
|
||||||
|
|
||||||
|
|
||||||
|
def _get_tool_extras(tool_name: str) -> Mapping[str, Any]:
|
||||||
|
config = get_app_config().get_tool_config(tool_name)
|
||||||
|
if config is None or config.model_extra is None:
|
||||||
|
return {}
|
||||||
|
return config.model_extra
|
||||||
|
|
||||||
|
|
||||||
|
def _get_api_key(tool_name: str = "web_search", *, extras: Mapping[str, Any] | None = None) -> str | None:
|
||||||
|
api_key = (extras if extras is not None else _get_tool_extras(tool_name)).get("api_key")
|
||||||
|
if isinstance(api_key, str) and api_key.strip():
|
||||||
|
return api_key.strip()
|
||||||
|
|
||||||
|
env_key = os.getenv(_API_KEY_ENV)
|
||||||
|
if isinstance(env_key, str) and env_key.strip():
|
||||||
|
return env_key.strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_max_results(value: object, *, default: int = _DEFAULT_MAX_RESULTS) -> int:
|
||||||
|
if isinstance(value, int) and not isinstance(value, bool):
|
||||||
|
result = value
|
||||||
|
elif isinstance(value, str) and value.strip().isdigit():
|
||||||
|
result = int(value.strip())
|
||||||
|
else:
|
||||||
|
logger.warning("Invalid Tencent Cloud WSA max_results=%r; using default %s", value, default)
|
||||||
|
return default
|
||||||
|
if result <= 0:
|
||||||
|
logger.warning("Invalid Tencent Cloud WSA max_results=%r; using default %s", value, default)
|
||||||
|
return default
|
||||||
|
if result > _MAX_RESULTS:
|
||||||
|
logger.warning("Tencent Cloud WSA max_results=%s exceeds maximum %s; clamping", result, _MAX_RESULTS)
|
||||||
|
return _MAX_RESULTS
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _get_mode(*, extras: Mapping[str, Any] | None = None) -> int | None:
|
||||||
|
"""Return an explicitly configured WSA result mode, if valid.
|
||||||
|
|
||||||
|
Tencent Cloud defaults to natural web results when ``Mode`` is omitted. Keep
|
||||||
|
that default so the provider does not request VR or mixed results implicitly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
extras = extras if extras is not None else _get_tool_extras("web_search")
|
||||||
|
if "mode" not in extras:
|
||||||
|
return None
|
||||||
|
mode = extras["mode"]
|
||||||
|
if not isinstance(mode, int) or isinstance(mode, bool):
|
||||||
|
logger.warning("Invalid Tencent Cloud WSA mode=%r; omitting Mode", extras["mode"])
|
||||||
|
return None
|
||||||
|
if mode not in {0, 1, 2}:
|
||||||
|
logger.warning("Tencent Cloud WSA mode=%r is outside 0, 1, 2; omitting Mode", mode)
|
||||||
|
return None
|
||||||
|
return mode
|
||||||
|
|
||||||
|
|
||||||
|
def _request_count(max_results: int) -> int | None:
|
||||||
|
"""Return Tencent Cloud's supported Cnt value when one is needed.
|
||||||
|
|
||||||
|
The API's default response size is 10. Cnt is available only on Tencent
|
||||||
|
Cloud plans that support it, so omit it for requests that fit in the
|
||||||
|
default response and request the smallest supported batch otherwise.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if max_results <= _DEFAULT_API_RESULT_COUNT:
|
||||||
|
return None
|
||||||
|
return ((max_results + _DEFAULT_API_RESULT_COUNT - 1) // _DEFAULT_API_RESULT_COUNT) * _DEFAULT_API_RESULT_COUNT
|
||||||
|
|
||||||
|
|
||||||
|
def _error(message: str, query: str, *, request_id: str | None = None) -> str:
|
||||||
|
result: dict[str, str] = {"error": message, "query": query}
|
||||||
|
if request_id:
|
||||||
|
result["request_id"] = request_id
|
||||||
|
return json.dumps(result, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _request_id(response: Mapping[str, Any]) -> str | None:
|
||||||
|
value = response.get("RequestId")
|
||||||
|
return value if isinstance(value, str) and value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _missing_key_error(query: str, tool_name: str) -> str:
|
||||||
|
if tool_name not in _api_key_warned:
|
||||||
|
_api_key_warned.add(tool_name)
|
||||||
|
logger.warning(
|
||||||
|
"Tencent Cloud WSA API key is not set for '%s'. Set %s in the environment or provide api_key in config.yaml.",
|
||||||
|
tool_name,
|
||||||
|
_API_KEY_ENV,
|
||||||
|
)
|
||||||
|
return _error(f"{_API_KEY_ENV} is not configured", query)
|
||||||
|
|
||||||
|
|
||||||
|
def _search(api_key: str, payload: dict[str, object], query: str) -> tuple[dict[str, Any] | None, str | None]:
|
||||||
|
try:
|
||||||
|
with httpx.Client(timeout=_REQUEST_TIMEOUT_S) as client:
|
||||||
|
response = client.post(
|
||||||
|
_SEARCH_ENDPOINT,
|
||||||
|
headers={
|
||||||
|
"Authorization": f"Bearer {api_key}",
|
||||||
|
"Content-Type": "application/json; charset=utf-8",
|
||||||
|
},
|
||||||
|
json=payload,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json()
|
||||||
|
except httpx.HTTPStatusError as exc:
|
||||||
|
logger.error("Tencent Cloud WSA API returned HTTP %s", exc.response.status_code)
|
||||||
|
return None, _error(f"Tencent Cloud WSA API error: HTTP {exc.response.status_code}", query)
|
||||||
|
except httpx.HTTPError as exc:
|
||||||
|
logger.error("Tencent Cloud WSA request failed: %s", exc)
|
||||||
|
return None, _error("Tencent Cloud WSA request failed", query)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
logger.error("Tencent Cloud WSA returned an invalid JSON response")
|
||||||
|
return None, _error("Tencent Cloud WSA returned an invalid JSON response", query)
|
||||||
|
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
logger.error("Tencent Cloud WSA returned an unexpected payload type: %s", type(data).__name__)
|
||||||
|
return None, _error("Tencent Cloud WSA returned an unexpected response format", query)
|
||||||
|
return data, None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_response(data: dict[str, Any], query: str) -> tuple[dict[str, Any] | None, str | None, str | None]:
|
||||||
|
response = data.get("Response")
|
||||||
|
if not isinstance(response, dict):
|
||||||
|
logger.error("Tencent Cloud WSA response did not contain a Response object")
|
||||||
|
return None, None, _error("Tencent Cloud WSA returned an unexpected response format", query)
|
||||||
|
|
||||||
|
request_id = _request_id(response)
|
||||||
|
api_error = response.get("Error")
|
||||||
|
if isinstance(api_error, dict):
|
||||||
|
code = api_error.get("Code")
|
||||||
|
code = code if isinstance(code, str) and code else "UnknownError"
|
||||||
|
logger.error("Tencent Cloud WSA API returned error code %s (request_id=%s)", code, request_id)
|
||||||
|
return None, request_id, _error(f"Tencent Cloud WSA API error: {code}", query, request_id=request_id)
|
||||||
|
return response, request_id, None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_results(response: dict[str, Any], *, max_results: int) -> list[dict[str, Any]] | None:
|
||||||
|
pages = response.get("Pages")
|
||||||
|
if pages is None:
|
||||||
|
return []
|
||||||
|
if not isinstance(pages, list):
|
||||||
|
logger.error("Tencent Cloud WSA returned non-list Pages value")
|
||||||
|
return None
|
||||||
|
|
||||||
|
results: list[dict[str, Any]] = []
|
||||||
|
for page in pages:
|
||||||
|
if isinstance(page, str):
|
||||||
|
try:
|
||||||
|
page_data = json.loads(page)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
logger.warning("Skipping malformed Tencent Cloud WSA page entry")
|
||||||
|
continue
|
||||||
|
elif isinstance(page, dict):
|
||||||
|
# The documented schema is a JSON-string array. Accepting an object
|
||||||
|
# too keeps the provider forward-compatible with a harmless API
|
||||||
|
# representation change.
|
||||||
|
page_data = page
|
||||||
|
else:
|
||||||
|
logger.warning("Skipping Tencent Cloud WSA page entry of type %s", type(page).__name__)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not isinstance(page_data, dict):
|
||||||
|
logger.warning("Skipping Tencent Cloud WSA page entry that is not an object")
|
||||||
|
continue
|
||||||
|
|
||||||
|
title = page_data.get("title")
|
||||||
|
url = page_data.get("url")
|
||||||
|
content = page_data.get("content") or page_data.get("passage") or ""
|
||||||
|
result = {
|
||||||
|
"title": title if isinstance(title, str) else "",
|
||||||
|
"url": url if isinstance(url, str) else "",
|
||||||
|
"snippet": content if isinstance(content, str) else "",
|
||||||
|
}
|
||||||
|
for field in ("date", "site", "score"):
|
||||||
|
value = page_data.get(field)
|
||||||
|
if isinstance(value, (str, int, float)) and not isinstance(value, bool):
|
||||||
|
result[field] = value
|
||||||
|
results.append(result)
|
||||||
|
if len(results) >= max_results:
|
||||||
|
break
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
@tool("web_search", parse_docstring=True)
|
||||||
|
def web_search_tool(query: str, max_results: int = _DEFAULT_MAX_RESULTS) -> str:
|
||||||
|
"""Search the web using Tencent Cloud Web Search API.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: Search keywords describing what you want to find.
|
||||||
|
max_results: Maximum number of search results to return. Default is 5.
|
||||||
|
"""
|
||||||
|
|
||||||
|
extras = _get_tool_extras("web_search")
|
||||||
|
if "max_results" in extras:
|
||||||
|
max_results = extras["max_results"]
|
||||||
|
max_results = _coerce_max_results(max_results)
|
||||||
|
query = query.strip()
|
||||||
|
if not query:
|
||||||
|
return _error("Search query must not be empty", query)
|
||||||
|
|
||||||
|
api_key = _get_api_key("web_search", extras=extras)
|
||||||
|
if not api_key:
|
||||||
|
return _missing_key_error(query, "web_search")
|
||||||
|
|
||||||
|
payload: dict[str, object] = {"Query": query}
|
||||||
|
mode = _get_mode(extras=extras)
|
||||||
|
if mode is not None:
|
||||||
|
payload["Mode"] = mode
|
||||||
|
request_count = _request_count(max_results)
|
||||||
|
if request_count is not None:
|
||||||
|
payload["Cnt"] = request_count
|
||||||
|
|
||||||
|
data, error_json = _search(api_key, payload, query)
|
||||||
|
if error_json is not None:
|
||||||
|
return error_json
|
||||||
|
assert data is not None
|
||||||
|
|
||||||
|
response, request_id, error_json = _get_response(data, query)
|
||||||
|
if error_json is not None:
|
||||||
|
return error_json
|
||||||
|
assert response is not None
|
||||||
|
|
||||||
|
results = _parse_results(response, max_results=max_results)
|
||||||
|
if results is None:
|
||||||
|
return _error("Tencent Cloud WSA returned an unexpected response format", query, request_id=request_id)
|
||||||
|
if not results:
|
||||||
|
return _error("No results found", query, request_id=request_id)
|
||||||
|
|
||||||
|
output: dict[str, object] = {
|
||||||
|
"query": query,
|
||||||
|
"total_results": len(results),
|
||||||
|
"results": results,
|
||||||
|
}
|
||||||
|
if request_id:
|
||||||
|
output["request_id"] = request_id
|
||||||
|
return json.dumps(output, indent=2, ensure_ascii=False)
|
||||||
@ -28,7 +28,7 @@ Durable MCP task-management tools are added only while the process-local task su
|
|||||||
- `browser_automation/` - Agentic browser control (stateful `navigate → observe → click/type` loop) via Playwright, distinct from the read-only `web_fetch`/`web_capture` tools. Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close` (config `group: browser`). A process-local `BrowserSessionManager` owns one private, loop-affine Playwright event-loop thread (same pattern as the BoxLite provider) so a per-thread browser session survives across turns regardless of the caller's loop (Gateway / TUI / test). Each action returns a fresh page snapshot whose interactive elements are addressed by a stable numeric `[ref]` index (stamped as `data-df-ref` during snapshot), so the model acts on what it just observed instead of holding stale handles or guessing selectors. URLs are SSRF-screened via the shared `validate_public_http_url` (opt-out `allow_private_addresses` only for intentional internal targets). CDP attachment cannot install the request guard on an existing Chrome context, so `cdp_url` fails closed unless the operator explicitly sets `allow_unguarded_cdp: true` for a trusted local browser. Browser REST/Live access also requires an exact non-NULL thread owner, rather than the general legacy shared-thread policy, because retained pages may contain authenticated state. Session admission is a hard `max_sessions` cap: pinned Live/operation sessions are never evicted, and a new thread is rejected when no unpinned session can be closed; one Live viewer owns a session at a time. Optional dependency: `cd backend && uv sync --extra browser && uv run playwright install chromium`; `scripts/detect_uv_extras.py` preserves the extra when `config.yaml` enables `browser_navigate`, and Gateway startup fails fast if configured browser control cannot import Playwright. Tests: `tests/test_browser_automation.py` (mocked tools + a real-Chromium integration test guarded by `importorskip`); `tests/manual_browser_live_check.py` is a manual DeepSeek-driven end-to-end check (not collected by pytest).
|
- `browser_automation/` - Agentic browser control (stateful `navigate → observe → click/type` loop) via Playwright, distinct from the read-only `web_fetch`/`web_capture` tools. Tools: `browser_navigate`, `browser_snapshot`, `browser_click`, `browser_type`, `browser_get_text`, `browser_back`, `browser_screenshot`, `browser_close` (config `group: browser`). A process-local `BrowserSessionManager` owns one private, loop-affine Playwright event-loop thread (same pattern as the BoxLite provider) so a per-thread browser session survives across turns regardless of the caller's loop (Gateway / TUI / test). Each action returns a fresh page snapshot whose interactive elements are addressed by a stable numeric `[ref]` index (stamped as `data-df-ref` during snapshot), so the model acts on what it just observed instead of holding stale handles or guessing selectors. URLs are SSRF-screened via the shared `validate_public_http_url` (opt-out `allow_private_addresses` only for intentional internal targets). CDP attachment cannot install the request guard on an existing Chrome context, so `cdp_url` fails closed unless the operator explicitly sets `allow_unguarded_cdp: true` for a trusted local browser. Browser REST/Live access also requires an exact non-NULL thread owner, rather than the general legacy shared-thread policy, because retained pages may contain authenticated state. Session admission is a hard `max_sessions` cap: pinned Live/operation sessions are never evicted, and a new thread is rejected when no unpinned session can be closed; one Live viewer owns a session at a time. Optional dependency: `cd backend && uv sync --extra browser && uv run playwright install chromium`; `scripts/detect_uv_extras.py` preserves the extra when `config.yaml` enables `browser_navigate`, and Gateway startup fails fast if configured browser control cannot import Playwright. Tests: `tests/test_browser_automation.py` (mocked tools + a real-Chromium integration test guarded by `importorskip`); `tests/manual_browser_live_check.py` is a manual DeepSeek-driven end-to-end check (not collected by pytest).
|
||||||
Live UI input dispatch is kept independent from JPEG capture: non-move actions start a rate-limited background refresh loop, so pointer, wheel, or keyboard input stays responsive while continuous gestures still produce frames throughout the interaction.
|
Live UI input dispatch is kept independent from JPEG capture: non-move actions start a rate-limited background refresh loop, so pointer, wheel, or keyboard input stays responsive while continuous gestures still produce frames throughout the interaction.
|
||||||
|
|
||||||
Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `tenki`); see each subpackage for specifics. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional.
|
Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `tencent_wsa`, `tenki`); see each subpackage for specifics. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional.
|
||||||
|
|
||||||
E2B output sync records remote file versions and actual host file metadata in a thread-local manifest. The manifest binds to the remote sandbox ID. A complete output listing removes entries for deleted files. This avoids repeat downloads when the host filesystem rounds modification times. A single release-time sync pass is bounded by aggregate ceilings (`_MAX_SYNC_TOTAL_BYTES`, `_MAX_SYNC_FILES`, `_SYNC_DEADLINE_SECONDS`) on top of the per-file `_MAX_DOWNLOAD_SIZE` cap, so a pathological outputs tree cannot make release download unboundedly; a truncated pass logs what it dropped and leaves the manifest un-pruned (only entries observed in that pass are reconciled), so files it never reached are retried on the next release rather than being forgotten.
|
E2B output sync records remote file versions and actual host file metadata in a thread-local manifest. The manifest binds to the remote sandbox ID. A complete output listing removes entries for deleted files. This avoids repeat downloads when the host filesystem rounds modification times. A single release-time sync pass is bounded by aggregate ceilings (`_MAX_SYNC_TOTAL_BYTES`, `_MAX_SYNC_FILES`, `_SYNC_DEADLINE_SECONDS`) on top of the per-file `_MAX_DOWNLOAD_SIZE` cap, so a pathological outputs tree cannot make release download unboundedly; a truncated pass logs what it dropped and leaves the manifest un-pruned (only entries observed in that pass are reconciled), so files it never reached are retried on the next release rather than being forgotten.
|
||||||
|
|
||||||
|
|||||||
@ -356,6 +356,17 @@ class TestCheckWebSearch:
|
|||||||
assert result.status == "warn"
|
assert result.status == "warn"
|
||||||
assert "SERPER_API_KEY" in (result.fix or "")
|
assert "SERPER_API_KEY" in (result.fix or "")
|
||||||
|
|
||||||
|
def test_tencent_wsa_without_key_warns(self, tmp_path, monkeypatch):
|
||||||
|
monkeypatch.delenv("TENCENTCLOUD_WSA_APIKEY", raising=False)
|
||||||
|
cfg = tmp_path / "config.yaml"
|
||||||
|
cfg.write_text("config_version: 5\ntools:\n - name: web_search\n use: deerflow.community.tencent_wsa.tools:web_search_tool\n")
|
||||||
|
|
||||||
|
result = doctor.check_web_search(cfg)
|
||||||
|
|
||||||
|
assert result.status == "warn"
|
||||||
|
assert "tencent_wsa configured but TENCENTCLOUD_WSA_APIKEY not set" in result.detail
|
||||||
|
assert "TENCENTCLOUD_WSA_APIKEY" in (result.fix or "")
|
||||||
|
|
||||||
def test_no_search_tool_warns(self, tmp_path):
|
def test_no_search_tool_warns(self, tmp_path):
|
||||||
cfg = tmp_path / "config.yaml"
|
cfg = tmp_path / "config.yaml"
|
||||||
cfg.write_text("config_version: 5\ntools: []\n")
|
cfg.write_text("config_version: 5\ntools: []\n")
|
||||||
|
|||||||
285
backend/tests/test_tencent_wsa_tools.py
Normal file
285
backend/tests/test_tencent_wsa_tools.py
Normal file
@ -0,0 +1,285 @@
|
|||||||
|
"""Unit tests for the Tencent Cloud Web Search API community provider."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def reset_api_key_warned():
|
||||||
|
import deerflow.community.tencent_wsa.tools as wsa
|
||||||
|
|
||||||
|
wsa._api_key_warned = set()
|
||||||
|
yield
|
||||||
|
wsa._api_key_warned = set()
|
||||||
|
|
||||||
|
|
||||||
|
def _tool_config(extras: dict | None) -> MagicMock:
|
||||||
|
config = MagicMock()
|
||||||
|
config.model_extra = extras
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def _response(pages: list[object] | None = None, **extra: object) -> dict:
|
||||||
|
response: dict[str, object] = {"RequestId": "request-123"}
|
||||||
|
if pages is not None:
|
||||||
|
response["Pages"] = pages
|
||||||
|
response.update(extra)
|
||||||
|
return {"Response": response}
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_http_client(response: MagicMock):
|
||||||
|
client = MagicMock()
|
||||||
|
client.post.return_value = response
|
||||||
|
context_manager = MagicMock()
|
||||||
|
context_manager.__enter__.return_value = client
|
||||||
|
context_manager.__exit__.return_value = False
|
||||||
|
return client, context_manager
|
||||||
|
|
||||||
|
|
||||||
|
class TestTencentWsaApiKey:
|
||||||
|
def test_config_key_takes_precedence_over_environment(self, monkeypatch):
|
||||||
|
monkeypatch.setenv("TENCENTCLOUD_WSA_APIKEY", "environment-key")
|
||||||
|
with patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config:
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "config-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import _get_api_key
|
||||||
|
|
||||||
|
assert _get_api_key() == "config-key"
|
||||||
|
|
||||||
|
def test_environment_key_is_used_as_fallback(self, monkeypatch):
|
||||||
|
monkeypatch.setenv("TENCENTCLOUD_WSA_APIKEY", "environment-key")
|
||||||
|
with patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config:
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": " "})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import _get_api_key
|
||||||
|
|
||||||
|
assert _get_api_key() == "environment-key"
|
||||||
|
|
||||||
|
def test_missing_key_returns_a_structured_error(self, monkeypatch):
|
||||||
|
monkeypatch.delenv("TENCENTCLOUD_WSA_APIKEY", raising=False)
|
||||||
|
with patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config:
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": "腾讯云"}))
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"error": "TENCENTCLOUD_WSA_APIKEY is not configured",
|
||||||
|
"query": "腾讯云",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestTencentWsaSearch:
|
||||||
|
def test_search_normalizes_documented_pages_and_honors_config(self):
|
||||||
|
page_one = json.dumps(
|
||||||
|
{
|
||||||
|
"title": "第一条",
|
||||||
|
"url": "https://example.com/one",
|
||||||
|
"passage": "摘要一",
|
||||||
|
"date": "2026-08-27",
|
||||||
|
"site": "示例站点",
|
||||||
|
"score": 0.9,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
page_two = json.dumps(
|
||||||
|
{
|
||||||
|
"title": "第二条",
|
||||||
|
"url": "https://example.com/two",
|
||||||
|
"content": "动态摘要二",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
http_response = MagicMock()
|
||||||
|
http_response.json.return_value = _response([page_one, page_two])
|
||||||
|
client, context_manager = _mock_http_client(http_response)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client", return_value=context_manager),
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key", "max_results": 1, "mode": 2})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": " 腾讯云搜索 ", "max_results": 99}))
|
||||||
|
|
||||||
|
client.post.assert_called_once_with(
|
||||||
|
"https://api.wsa.cloud.tencent.com/SearchPro",
|
||||||
|
headers={
|
||||||
|
"Authorization": "Bearer test-key",
|
||||||
|
"Content-Type": "application/json; charset=utf-8",
|
||||||
|
},
|
||||||
|
json={"Query": "腾讯云搜索", "Mode": 2},
|
||||||
|
)
|
||||||
|
assert get_config.call_count == 1
|
||||||
|
assert result == {
|
||||||
|
"query": "腾讯云搜索",
|
||||||
|
"total_results": 1,
|
||||||
|
"request_id": "request-123",
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"title": "第一条",
|
||||||
|
"url": "https://example.com/one",
|
||||||
|
"snippet": "摘要一",
|
||||||
|
"date": "2026-08-27",
|
||||||
|
"site": "示例站点",
|
||||||
|
"score": 0.9,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_search_omits_mode_when_not_configured(self):
|
||||||
|
http_response = MagicMock()
|
||||||
|
http_response.json.return_value = _response([json.dumps({"title": "结果", "passage": "摘要"})])
|
||||||
|
client, context_manager = _mock_http_client(http_response)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client", return_value=context_manager),
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
web_search_tool.run({"query": "腾讯云"})
|
||||||
|
|
||||||
|
assert client.post.call_args.kwargs["json"] == {"Query": "腾讯云"}
|
||||||
|
|
||||||
|
def test_search_requests_supported_cnt_for_more_than_default_results(self):
|
||||||
|
pages = [json.dumps({"title": f"结果 {index}", "passage": "摘要"}) for index in range(20)]
|
||||||
|
http_response = MagicMock()
|
||||||
|
http_response.json.return_value = _response(pages)
|
||||||
|
client, context_manager = _mock_http_client(http_response)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client", return_value=context_manager),
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": "腾讯云", "max_results": 20}))
|
||||||
|
|
||||||
|
assert client.post.call_args.kwargs["json"] == {"Query": "腾讯云", "Cnt": 20}
|
||||||
|
assert result["total_results"] == 20
|
||||||
|
|
||||||
|
def test_empty_query_does_not_call_paid_api(self):
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client") as client,
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": " "}))
|
||||||
|
|
||||||
|
assert result == {"error": "Search query must not be empty", "query": ""}
|
||||||
|
client.assert_not_called()
|
||||||
|
|
||||||
|
def test_search_skips_malformed_pages_without_losing_valid_results(self):
|
||||||
|
http_response = MagicMock()
|
||||||
|
http_response.json.return_value = _response(["not-json", 42, json.dumps({"title": "有效结果", "url": "https://example.com", "passage": "摘要"})])
|
||||||
|
_, context_manager = _mock_http_client(http_response)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client", return_value=context_manager),
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": "腾讯云"}))
|
||||||
|
|
||||||
|
assert result["total_results"] == 1
|
||||||
|
assert result["results"][0]["title"] == "有效结果"
|
||||||
|
|
||||||
|
def test_response_error_is_reported_even_with_http_200(self):
|
||||||
|
http_response = MagicMock()
|
||||||
|
http_response.json.return_value = _response(Error={"Code": "RequestLimitExceeded", "Message": "do not expose this"})
|
||||||
|
_, context_manager = _mock_http_client(http_response)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client", return_value=context_manager),
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": "腾讯云"}))
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"error": "Tencent Cloud WSA API error: RequestLimitExceeded",
|
||||||
|
"query": "腾讯云",
|
||||||
|
"request_id": "request-123",
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_http_error_does_not_expose_upstream_body(self):
|
||||||
|
response = MagicMock()
|
||||||
|
response.status_code = 503
|
||||||
|
response.text = "sensitive upstream diagnostic"
|
||||||
|
http_response = MagicMock()
|
||||||
|
http_response.raise_for_status.side_effect = httpx.HTTPStatusError("unavailable", request=MagicMock(), response=response)
|
||||||
|
client, context_manager = _mock_http_client(http_response)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client", return_value=context_manager),
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": "腾讯云"}))
|
||||||
|
|
||||||
|
assert result == {"error": "Tencent Cloud WSA API error: HTTP 503", "query": "腾讯云"}
|
||||||
|
assert "sensitive" not in json.dumps(result)
|
||||||
|
client.post.assert_called_once()
|
||||||
|
|
||||||
|
def test_non_list_pages_returns_unexpected_format_error(self):
|
||||||
|
http_response = MagicMock()
|
||||||
|
http_response.json.return_value = _response("not-a-list")
|
||||||
|
_, context_manager = _mock_http_client(http_response)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config,
|
||||||
|
patch("deerflow.community.tencent_wsa.tools.httpx.Client", return_value=context_manager),
|
||||||
|
):
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config({"api_key": "test-key"})
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import web_search_tool
|
||||||
|
|
||||||
|
result = json.loads(web_search_tool.run({"query": "腾讯云"}))
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"error": "Tencent Cloud WSA returned an unexpected response format",
|
||||||
|
"query": "腾讯云",
|
||||||
|
"request_id": "request-123",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestTencentWsaConfiguration:
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("value", "expected"),
|
||||||
|
[(3, 3), ("7", 7), (True, 5), (2.9, 5), (0, 5), (-1, 5), (500, 50), ("bad", 5)],
|
||||||
|
)
|
||||||
|
def test_coerce_max_results(self, value, expected):
|
||||||
|
from deerflow.community.tencent_wsa.tools import _coerce_max_results
|
||||||
|
|
||||||
|
assert _coerce_max_results(value) == expected
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("value", (None, "2", True, 2.0, "bad", -1, 3))
|
||||||
|
def test_invalid_mode_is_omitted(self, value):
|
||||||
|
with patch("deerflow.community.tencent_wsa.tools.get_app_config") as get_config:
|
||||||
|
extras = {} if value is None else {"mode": value}
|
||||||
|
get_config.return_value.get_tool_config.return_value = _tool_config(extras)
|
||||||
|
|
||||||
|
from deerflow.community.tencent_wsa.tools import _get_mode
|
||||||
|
|
||||||
|
assert _get_mode() is None
|
||||||
@ -754,6 +754,18 @@ tools:
|
|||||||
# # Used to limit the scope of search results, only returns content within the specified time range. Set to -1 to disable time filtering
|
# # Used to limit the scope of search results, only returns content within the specified time range. Set to -1 to disable time filtering
|
||||||
# search_time_range: 10
|
# search_time_range: 10
|
||||||
|
|
||||||
|
# Web search tool (uses Tencent Cloud Web Search API, requires a service API key)
|
||||||
|
# Create a service API key in the Tencent Cloud WSA console, then set
|
||||||
|
# TENCENTCLOUD_WSA_APIKEY in the Gateway environment. Do not use Tencent Cloud
|
||||||
|
# SecretId/SecretKey for this provider.
|
||||||
|
# - name: web_search
|
||||||
|
# group: web
|
||||||
|
# use: deerflow.community.tencent_wsa.tools:web_search_tool
|
||||||
|
# max_results: 5 # 1-50; values above 10 request Tencent's Cnt option
|
||||||
|
# # (requires a Tencent plan that supports Cnt)
|
||||||
|
# # mode: 0 # Optional: 0=web, 1=VR, 2=mixed
|
||||||
|
# # api_key: $TENCENTCLOUD_WSA_APIKEY # Optional explicit env-var reference
|
||||||
|
|
||||||
# Web search tool (uses Exa, requires EXA_API_KEY)
|
# Web search tool (uses Exa, requires EXA_API_KEY)
|
||||||
# - name: web_search
|
# - name: web_search
|
||||||
# group: web
|
# group: web
|
||||||
|
|||||||
@ -496,6 +496,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes
|
|||||||
"fastcrw": "CRW_API_KEY",
|
"fastcrw": "CRW_API_KEY",
|
||||||
"brave": "BRAVE_SEARCH_API_KEY",
|
"brave": "BRAVE_SEARCH_API_KEY",
|
||||||
"serper": "SERPER_API_KEY",
|
"serper": "SERPER_API_KEY",
|
||||||
|
"tencent_wsa": "TENCENTCLOUD_WSA_APIKEY",
|
||||||
},
|
},
|
||||||
"web_fetch": {
|
"web_fetch": {
|
||||||
"infoquest": "INFOQUEST_API_KEY",
|
"infoquest": "INFOQUEST_API_KEY",
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user