diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index ef79ca613a..aafba34b00 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -979,7 +979,7 @@ offline mocked tests verify request composition and local filtering, not provide retrieval behavior. No paid API calls are needed for the regression suite. **Built-in Tools**: -- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Serper, Serply, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute, Sofya) +- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Serper, Serply, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute, Sofya, You.com) - `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless, Sofya, Unbrowse) - `web_capture` - Capture rendered webpage screenshots as artifacts (Browserless) - `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper, Brave) @@ -1654,6 +1654,7 @@ models: - `GROUNDROUTE_API_KEY` - GroundRoute meta-search API key for `web_search` and `web_fetch` (routes across Serper, Brave, Exa, Tavily, Firecrawl, Perplexity with gain-share pricing) - `SOFYA_API_KEY` - [Sofya](https://sofya.co) key for `web_search` and `web_fetch` - `UNBROWSE_API_KEY` - [Unbrowse](https://unbrowse.ai) key for `web_fetch` +- `YDC_API_KEY` - You.com Search API key for `web_search` (optional: without it the provider uses You.com's keyless free-tier endpoint) - `BROWSERLESS_TOKEN` - Browserless token for `web_fetch` (Browserless provider) and `web_capture`, sent as the `token` query parameter (required by Browserless Cloud and by a self-hosted instance started with `TOKEN`) - `DEER_FLOW_PROJECT_ROOT` - Project root for relative runtime paths - `DEER_FLOW_CONFIG_PATH` - Custom config file path diff --git a/backend/packages/harness/deerflow/community/search_max_results.py b/backend/packages/harness/deerflow/community/search_max_results.py index 023a499c54..7a7de221a2 100644 --- a/backend/packages/harness/deerflow/community/search_max_results.py +++ b/backend/packages/harness/deerflow/community/search_max_results.py @@ -7,6 +7,9 @@ the #5852 bar; folding the identical copies into one function is what lets the remaining providers be brought onto the same bar one at a time without re-deciding in each file what "invalid" means. +You.com also uses this shared coercion, then applies its 100-result cap to +the normalized count before sending the request. + Other providers retain local normalization, including Exa's generic integer helper and SearXNG's inline validation; Tavily passes configured values through. ``tests/test_search_max_results_single_owner.py`` tracks the known helper and diff --git a/backend/packages/harness/deerflow/community/youcom/__init__.py b/backend/packages/harness/deerflow/community/youcom/__init__.py new file mode 100644 index 0000000000..8761678597 --- /dev/null +++ b/backend/packages/harness/deerflow/community/youcom/__init__.py @@ -0,0 +1,3 @@ +from .tools import web_search_tool + +__all__ = ["web_search_tool"] diff --git a/backend/packages/harness/deerflow/community/youcom/tools.py b/backend/packages/harness/deerflow/community/youcom/tools.py new file mode 100644 index 0000000000..305a26eba4 --- /dev/null +++ b/backend/packages/harness/deerflow/community/youcom/tools.py @@ -0,0 +1,189 @@ +"""You.com community web search tool. + +You.com (https://you.com) runs its own web index and exposes it as an +LLM-oriented Search API: each hit carries the page title, URL, and a set of +extracted snippets rather than a single meta description, so a research loop +gets usable context without a follow-up fetch. + +The provider talks to two endpoints and picks between them by whether a key is +configured, so a research run works out of the box and scales up when the +operator adds a key: + +* **No key** — ``/v1/agents/search``, no auth header, free tier (currently 100 + searches/day). This is what a fresh ``make setup`` install gets. +* **``YDC_API_KEY`` set** (env var or the tool's ``api_key`` config) — + ``/v1/search`` with an ``X-API-Key`` header: higher limits and the full + Search API surface. + +Endpoint and headers are always chosen together: the keyless endpoint rejects an +``X-API-Key`` header, so a stale key must never be sent there. + +Both endpoints return the same body — ``{"results": {"web": [...], "news": +[...]}}`` — so one normalizer covers both. ``web_search`` returns a JSON list of +``{title, url, snippet}``. + +This module is self-contained (httpx only, no You.com SDK and no new +dependency). Requests to You.com carry a ``User-Agent`` identifying DeerFlow so +You.com can attribute integration traffic; it is sent only to You.com hosts. +""" + +import json +import logging +import os +from functools import cache +from importlib.metadata import PackageNotFoundError, version + +import httpx +from langchain.tools import tool + +from deerflow.community.search_max_results import coerce_max_results +from deerflow.config import get_app_config + +logger = logging.getLogger(__name__) + +# Keyed Search API: full surface, higher limits, needs YDC_API_KEY. +_YOUCOM_SEARCH_ENDPOINT = "https://api.you.com/v1/search" +# Keyless endpoint: no auth header, free tier. Sending a key here is rejected. +_YOUCOM_KEYLESS_ENDPOINT = "https://api.you.com/v1/agents/search" +_DEFAULT_MAX_RESULTS = 5 +# You.com clamps `count` to 1-100 server-side; clamp here too to mirror it. +_MAX_RESULTS_CAP = 100 +_TIMEOUT_S = 30.0 +# Actionable hints for the failures an operator can actually fix. Any other +# status just reports its code. +_STATUS_HINTS = { + 401: "invalid or expired YDC_API_KEY", + 402: "You.com credit balance depleted", + 429: "rate limit reached — set YDC_API_KEY for higher limits", +} + + +@cache +def _user_agent() -> str: + """Identify DeerFlow to You.com. Only ever sent to You.com hosts.""" + try: + harness_version = version("deerflow-harness") + except PackageNotFoundError: # pragma: no cover - source checkouts only + harness_version = "unknown" + return f"deerflow-harness/{harness_version} youdotcom-integration/bytedance-deer-flow" + + +def _get_api_key(tool_name: str = "web_search") -> str | None: + """Resolve the You.com key from the tool's config block, then the env var. + + Returns None when neither is set, which is not an error here: the caller + falls back to the keyless endpoint. + """ + config = get_app_config().get_tool_config(tool_name) + if config is not None: + api_key = (config.model_extra or {}).get("api_key") + if isinstance(api_key, str) and api_key.strip(): + return api_key.strip() + env_key = os.getenv("YDC_API_KEY") + if isinstance(env_key, str) and env_key.strip(): + return env_key.strip() + return None + + +def _count_for_request(value: object) -> int: + """Coerce ``max_results`` via the shared owner, then mirror You.com's clamp. + + ``deerflow.community.search_max_results.coerce_max_results`` owns what + "invalid" means (booleans, non-integral floats and ``OverflowError`` all + fall back to the default); the 1-100 cap stays here because clamping + policy is deliberately per-provider. + """ + return min( + coerce_max_results(value, provider="You.com", logger=logger, default=_DEFAULT_MAX_RESULTS), + _MAX_RESULTS_CAP, + ) + + +def _search(api_key: str | None, query: str, count: int) -> dict: + """GET the keyed endpoint when a key is configured, else the keyless one.""" + headers = {"Accept": "application/json", "User-Agent": _user_agent()} + if api_key: + endpoint = _YOUCOM_SEARCH_ENDPOINT + headers["X-API-Key"] = api_key + else: + endpoint = _YOUCOM_KEYLESS_ENDPOINT + + with httpx.Client(timeout=_TIMEOUT_S) as client: + response = client.get(endpoint, params={"query": query, "count": count}, headers=headers) + response.raise_for_status() + return response.json() + + +def _snippet(result: dict) -> str: + """Prefer the extracted snippets; fall back to the meta description. + + Web hits carry `snippets`; news hits only carry `description`. + """ + snippets = result.get("snippets") + if isinstance(snippets, list): + joined = "\n".join(s for s in snippets if isinstance(s, str) and s.strip()) + if joined: + return joined + description = result.get("description") + return description if isinstance(description, str) else "" + + +@tool("web_search", parse_docstring=True) +def web_search_tool(query: str, max_results: int | None = None) -> str: + """Search the web for information using You.com. + + Each result carries extracted snippets from the page, not just a meta + description, so the content is usable without fetching the page. + + Args: + query: Search keywords describing what you want to find. Be specific for better results. + max_results: Maximum number of search results to return. If omitted, uses the configured value (default 5). Clamped to 1-100. + """ + # Honor the caller-supplied max_results; fall back to config only when omitted. + if max_results is None: + config = get_app_config().get_tool_config("web_search") + if config is not None: + max_results = (config.model_extra or {}).get("max_results") + count = _DEFAULT_MAX_RESULTS if max_results is None else _count_for_request(max_results) + + api_key = _get_api_key("web_search") + + try: + data = _search(api_key, query, count) + except httpx.HTTPStatusError as e: + status = e.response.status_code + logger.error("You.com API returned HTTP %s: %s", status, e.response.text) + hint = _STATUS_HINTS.get(status) + message = f"You.com API error: HTTP {status}" + if hint: + message = f"{message} ({hint})" + return json.dumps({"error": message, "query": query}, ensure_ascii=False) + except Exception as e: + logger.error("You.com search failed: %s: %s", type(e).__name__, e) + return json.dumps({"error": str(e), "query": query}, ensure_ascii=False) + + if not isinstance(data, dict): + logger.error("You.com returned an unexpected payload type: %s", type(data).__name__) + return json.dumps( + {"error": "You.com returned an unexpected response format", "query": query}, + ensure_ascii=False, + ) + + sections = data.get("results") + if not isinstance(sections, dict): + sections = {} + # `count` applies per section, so web + news together can exceed it. Web + # results lead; the merged list is trimmed back to what the caller asked for. + results = [r for section in ("web", "news") for r in (sections.get(section) or []) if isinstance(r, dict)] + if not results: + return json.dumps({"error": "No results found", "query": query}, ensure_ascii=False) + + normalized_results = [ + { + "title": r.get("title", ""), + "url": r.get("url", ""), + "snippet": _snippet(r), + } + for r in results[:count] + ] + return json.dumps(normalized_results, indent=2, ensure_ascii=False) diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index 5358ac55f1..feff4bdb12 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -82,7 +82,7 @@ Durable MCP task-management tools are added only while the process-local task su The shared SSRF guard (`community/url_safety.py`) resolves hostnames with blocking `socket.getaddrinfo`, so async tools, Gateway routes, and the Playwright request guard (`BrowserSession._install_request_guard`, which runs on the shared browser loop) call `validate_public_http_url` (or a wrapper such as `validate_browser_url`) through `asyncio.to_thread`. The guard checks at validation time only, so a launched browser sends every TCP connection (HTTP, HTTPS, WebSocket) through its per-session loopback SOCKS5 proxy (`browser_automation/egress.py`, `bypass: <-loopback>`): Chromium hands it every hostname, and it resolves once via `resolve_browser_egress` → `url_safety.resolve_public_addresses` and connects to exactly those addresses, closing the DNS-rebinding window (`mcp/personal_network.py` pins the same way). Every `acquire_session`/`get_session` caller must pass `egress_resolver`, because a thread's first caller fixes its launch options. WebRTC UDP does not traverse a SOCKS proxy and is not pinned (out of scope, so this is not a complete egress guarantee). Delegated fetch services (Browserless, crawl4ai, fastcrw, firecrawl) resolve on their own side and cannot be pinned, so each `web_fetch`/`web_capture` entry point instead screens its own backend base URL with `validate_delegated_backend_url` and fails closed on a self-hosted backend unless the operator confirms the backend's egress is isolated via `network_isolation_confirmed`; CDP-attached Chrome has neither guard. The strict blocking-IO gate carries a `socket.getaddrinfo` rule for this; `tests/blocking_io/test_web_tool_url_validation.py` drives each web tool entry point, the request guard, and the egress proxy; `tests/test_browser_egress.py` pins the proxy protocol, the session/manager/router wiring, and (with Playwright installed) a real-Chromium rebinding case. -Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`, `unbrowse`); see each subpackage for specifics. Browserless authenticates only from the `token` query parameter or `Authorization` header and rejects unknown body keys, so `BrowserlessClient._auth_params()` carries the token for every endpoint; never add it to the JSON body. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. +Additional providers also live here (`boxlite`, `brave`, `browserless`, `crawl4ai`, `ddg_search`, `e2b_sandbox`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`, `serply`, `sofya`, `tencent_wsa`, `tenki`, `unbrowse`, `youcom`); see each subpackage for specifics. Browserless authenticates only from the `token` query parameter or `Authorization` header and rejects unknown body keys, so `BrowserlessClient._auth_params()` carries the token for every endpoint; never add it to the JSON body. `tencent_wsa` uses Tencent Cloud Web Search's service API key endpoint (`TENCENTCLOUD_WSA_APIKEY`), not Tencent Cloud SecretId/SecretKey signing. Its `max_results` is capped at 50; requests above 10 use Tencent's optional `Cnt` parameter, which needs a Tencent Cloud plan that supports it. E2B bootstrap is required. If it fails, the provider kills and closes the unusable remote sandbox. New sandbox creation raises an error. Warm-pool reclaim and remote discovery discard the sandbox and continue acquisition. E2B mounts remain optional. E2B output sync records remote file versions and actual host file metadata in a thread-local manifest. The manifest binds to the remote sandbox ID. A complete output listing removes entries for deleted files. This avoids repeat downloads when the host filesystem rounds modification times. A single release-time sync pass is bounded by aggregate ceilings (`_MAX_SYNC_TOTAL_BYTES`, `_MAX_SYNC_FILES`, `_SYNC_DEADLINE_SECONDS`) on top of the per-file `_MAX_DOWNLOAD_SIZE` cap, so a pathological outputs tree cannot make release download unboundedly; a truncated pass logs what it dropped and leaves the manifest un-pruned (only entries observed in that pass are reconciled), so files it never reached are retried on the next release rather than being forgotten. diff --git a/backend/tests/AGENTS.md b/backend/tests/AGENTS.md index 176f8e7537..f25ddbfa6b 100644 --- a/backend/tests/AGENTS.md +++ b/backend/tests/AGENTS.md @@ -13,6 +13,9 @@ inline integer assignments. Include Exa and SearXNG in the deferred census; pin new-provider copies and docstring false positives with source fixtures. This pattern gate does not establish arbitrary provider semantics or change deferred providers' validation policy. +Include You.com in the shared-owner provider checks; it applies its 100-result +cap after shared normalization. Pin invalid-value defaults and positive count +bounds through the outbound request in `test_youcom_tools.py`. Mixed-version resume admission in `test_thread_run_idempotency.py` uses a real shared SQL repository and an old-column projection. The frozen pre-6499 helper diff --git a/backend/tests/test_search_max_results_single_owner.py b/backend/tests/test_search_max_results_single_owner.py index ff83dfb125..fae8c4043a 100644 --- a/backend/tests/test_search_max_results_single_owner.py +++ b/backend/tests/test_search_max_results_single_owner.py @@ -5,8 +5,8 @@ floats and ``OverflowError``. Those audits had to be read file by file because each provider owned its own copy of the answer. -``deerflow.community.search_max_results`` owns coercion for the four migrated -providers whose copies were byte-identical. These tests pin three things the +``deerflow.community.search_max_results`` owns coercion for the migrated +providers. These tests pin three things the refactor must not silently give back: 1. the shared function's behaviour, value by value (the bar itself); @@ -31,6 +31,7 @@ from deerflow.community.fastcrw import tools as fastcrw_tools from deerflow.community.firecrawl import tools as firecrawl_tools from deerflow.community.image_search import tools as image_search_tools from deerflow.community.search_max_results import DEFAULT_MAX_RESULTS, coerce_max_results +from deerflow.community.youcom import tools as youcom_tools COMMUNITY_ROOT = Path(ddg_search_tools.__file__).resolve().parent.parent @@ -43,6 +44,7 @@ SHARED_OWNER_PROVIDERS = { "image_search": ("DDG image search", image_search_tools), "fastcrw": ("fastCRW", fastcrw_tools), "firecrawl": ("Firecrawl", firecrawl_tools), + "youcom": ("You.com", youcom_tools), } # Providers that still normalize locally, including Exa's generic helper and diff --git a/backend/tests/test_youcom_tools.py b/backend/tests/test_youcom_tools.py new file mode 100644 index 0000000000..bfddeed9dd --- /dev/null +++ b/backend/tests/test_youcom_tools.py @@ -0,0 +1,390 @@ +"""Unit tests for the You.com community web search tool.""" + +import json +import logging +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +KEYED_ENDPOINT = "https://api.you.com/v1/search" +KEYLESS_ENDPOINT = "https://api.you.com/v1/agents/search" + + +@pytest.fixture +def mock_config_with_key(): + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": "test-ydc-key", "max_results": 5} + mock.return_value.get_tool_config.return_value = tool_config + yield mock + + +@pytest.fixture +def mock_config_no_key(): + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {} + mock.return_value.get_tool_config.return_value = tool_config + yield mock + + +def _make_response(payload: object) -> MagicMock: + mock_resp = MagicMock() + mock_resp.json.return_value = payload + mock_resp.raise_for_status = MagicMock() + return mock_resp + + +def _patch_get(mock_resp: MagicMock): + """Patch httpx.Client so the context-managed .get returns mock_resp.""" + patcher = patch("deerflow.community.youcom.tools.httpx.Client") + mock_client_cls = patcher.start() + mock_client_cls.return_value.__enter__.return_value.get.return_value = mock_resp + return patcher, mock_client_cls + + +def _get_call(mock_client_cls): + return mock_client_cls.return_value.__enter__.return_value.get.call_args + + +def _web(**overrides) -> dict: + result = {"url": "https://ex.com/a", "title": "A", "description": "d1", "snippets": ["s1", "s2"]} + result.update(overrides) + return result + + +class TestGetApiKey: + def test_returns_config_key_when_present(self): + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": "from-config"} + mock.return_value.get_tool_config.return_value = tool_config + + from deerflow.community.youcom.tools import _get_api_key + + assert _get_api_key("web_search") == "from-config" + + def test_falls_back_to_env_when_config_key_blank(self): + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": " "} + mock.return_value.get_tool_config.return_value = tool_config + with patch.dict("os.environ", {"YDC_API_KEY": "env-key"}, clear=True): + from deerflow.community.youcom.tools import _get_api_key + + assert _get_api_key("web_search") == "env-key" + + def test_returns_none_when_no_key_anywhere(self): + """No key is not an error: it selects the keyless endpoint.""" + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + mock.return_value.get_tool_config.return_value = None + with patch.dict("os.environ", {}, clear=True): + from deerflow.community.youcom.tools import _get_api_key + + assert _get_api_key("web_search") is None + + +class TestEndpointSelection: + def test_configured_key_uses_keyed_endpoint_with_header(self, mock_config_with_key): + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test"}) + call = _get_call(mock_client_cls) + finally: + patcher.stop() + + assert call.args[0] == KEYED_ENDPOINT + assert call.kwargs["headers"]["X-API-Key"] == "test-ydc-key" + assert call.kwargs["params"] == {"query": "test", "count": 5} + + def test_no_key_uses_keyless_endpoint_without_auth_header(self, mock_config_no_key): + """The keyless endpoint rejects an auth header, so none may be sent.""" + with patch.dict("os.environ", {}, clear=True): + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test"}) + call = _get_call(mock_client_cls) + finally: + patcher.stop() + + assert call.args[0] == KEYLESS_ENDPOINT + assert "X-API-Key" not in call.kwargs["headers"] + + def test_env_key_uses_keyed_endpoint(self, mock_config_no_key): + with patch.dict("os.environ", {"YDC_API_KEY": "env-key"}, clear=True): + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test"}) + call = _get_call(mock_client_cls) + finally: + patcher.stop() + + assert call.args[0] == KEYED_ENDPOINT + assert call.kwargs["headers"]["X-API-Key"] == "env-key" + + @pytest.mark.parametrize("with_key", [True, False]) + def test_identifies_deerflow_in_user_agent(self, with_key, mock_config_no_key): + """Both endpoints identify DeerFlow so You.com can attribute the traffic.""" + env = {"YDC_API_KEY": "k"} if with_key else {} + with patch.dict("os.environ", env, clear=True): + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test"}) + call = _get_call(mock_client_cls) + finally: + patcher.stop() + + user_agent = call.kwargs["headers"]["User-Agent"] + assert user_agent.startswith("deerflow-harness/") + assert "youdotcom-integration/bytedance-deer-flow" in user_agent + + +class TestWebSearchTool: + def test_returns_normalized_list_with_joined_snippets(self, mock_config_with_key): + payload = { + "results": { + "web": [ + _web(url="https://ex.com/a", title="A", snippets=["s1", "s2"]), + _web(url="https://ex.com/b", title="B", snippets=["s3"]), + ] + } + } + patcher, _ = _patch_get(_make_response(payload)) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "vector databases"})) + finally: + patcher.stop() + + assert parsed == [ + {"title": "A", "url": "https://ex.com/a", "snippet": "s1\ns2"}, + {"title": "B", "url": "https://ex.com/b", "snippet": "s3"}, + ] + + def test_falls_back_to_description_when_no_snippets(self, mock_config_with_key): + """News hits carry only a description; web hits with empty snippets do too.""" + payload = { + "results": { + "web": [_web(snippets=[], description="web-desc")], + "news": [{"url": "https://n.com", "title": "N", "description": "news-desc"}], + } + } + patcher, _ = _patch_get(_make_response(payload)) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "news"})) + finally: + patcher.stop() + + assert [r["snippet"] for r in parsed] == ["web-desc", "news-desc"] + + def test_web_and_news_merged_and_trimmed_to_count(self, mock_config_with_key): + """`count` applies per section, so the merged list can overshoot it.""" + payload = { + "results": { + "web": [_web(url=f"https://w{i}.com", title=f"W{i}") for i in range(5)], + "news": [{"url": f"https://n{i}.com", "title": f"N{i}", "description": "d"} for i in range(5)], + } + } + patcher, _ = _patch_get(_make_response(payload)) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "test"})) + finally: + patcher.stop() + + assert len(parsed) == 5 + assert [r["title"] for r in parsed] == ["W0", "W1", "W2", "W3", "W4"] + + def test_agent_max_results_is_honored_over_config(self, mock_config_with_key): + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test", "max_results": 20}) + params = _get_call(mock_client_cls).kwargs["params"] + finally: + patcher.stop() + + assert params["count"] == 20 + + def test_config_max_results_used_when_caller_omits(self, mock_config_with_key): + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test"}) + params = _get_call(mock_client_cls).kwargs["params"] + finally: + patcher.stop() + + assert params["count"] == 5 + + @pytest.mark.parametrize(("max_results", "expected"), [(0, 5), (-2, 5), (1, 1), (100, 100), ("500", 100)]) + def test_max_results_clamped_to_provider_bounds(self, max_results, expected): + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": "k", "max_results": max_results} + mock.return_value.get_tool_config.return_value = tool_config + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test"}) + params = _get_call(mock_client_cls).kwargs["params"] + finally: + patcher.stop() + + assert params["count"] == expected + + def test_invalid_max_results_falls_back_to_default_with_warning(self, caplog): + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": "k", "max_results": "not-a-number"} + mock.return_value.get_tool_config.return_value = tool_config + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + with caplog.at_level(logging.WARNING, logger="deerflow.community.youcom.tools"): + web_search_tool.invoke({"query": "test"}) + params = _get_call(mock_client_cls).kwargs["params"] + finally: + patcher.stop() + + assert params["count"] == 5 + assert any("Invalid You.com max_results" in record.message for record in caplog.records) + + def test_empty_results_returns_error_json(self, mock_config_with_key): + patcher, _ = _patch_get(_make_response({"results": {"web": []}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "no results"})) + finally: + patcher.stop() + + assert parsed["error"] == "No results found" + assert parsed["query"] == "no results" + + def test_missing_results_section_returns_error_json(self, mock_config_with_key): + patcher, _ = _patch_get(_make_response({"metadata": {"latency": 0.1}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "test"})) + finally: + patcher.stop() + + assert parsed["error"] == "No results found" + + def test_non_dict_payload_returns_format_error(self, mock_config_with_key): + patcher, _ = _patch_get(_make_response(["not", "a", "dict"])) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "test"})) + finally: + patcher.stop() + + assert "unexpected response format" in parsed["error"] + + def test_partial_fields_default_to_empty_string(self, mock_config_with_key): + patcher, _ = _patch_get(_make_response({"results": {"web": [{}]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "test"})) + finally: + patcher.stop() + + assert parsed[0] == {"title": "", "url": "", "snippet": ""} + + @pytest.mark.parametrize( + ("status", "hint"), + [ + (401, "invalid or expired YDC_API_KEY"), + (402, "You.com credit balance depleted"), + (429, "rate limit reached — set YDC_API_KEY for higher limits"), + (500, None), + ], + ) + def test_http_error_returns_structured_error(self, status, hint, mock_config_with_key): + mock_resp = MagicMock() + mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError( + str(status), + request=MagicMock(), + response=MagicMock(status_code=status, text="boom"), + ) + patcher, _ = _patch_get(mock_resp) + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "test"})) + finally: + patcher.stop() + + assert str(status) in parsed["error"] + if hint: + assert hint in parsed["error"] + + def test_network_exception_returns_error_json(self, mock_config_with_key): + patcher, mock_client_cls = _patch_get(MagicMock()) + mock_client_cls.return_value.__enter__.return_value.get.side_effect = Exception("timeout") + try: + from deerflow.community.youcom.tools import web_search_tool + + parsed = json.loads(web_search_tool.invoke({"query": "test"})) + finally: + patcher.stop() + + assert parsed["error"] == "timeout" + + +@pytest.mark.parametrize( + "raw", + [float("inf"), True, 3.5], + ids=["yaml-inf", "bool", "non-integral-float"], +) +def test_invalid_max_results_shapes_fall_back_to_default(raw): + """Invalid max_results shapes fall back to the default, never crash or truncate. + + These pin the shared-owner bar for this provider: a YAML ``.inf`` makes + ``int()`` raise ``OverflowError``, a YAML ``true`` is a bool (never 1), + and ``3.5`` must not silently truncate to 3. The shared + ``coerce_max_results`` owns the rejection profile; this provider only + adds the 1-100 cap on top. + """ + with patch("deerflow.community.youcom.tools.get_app_config") as mock: + tool_config = MagicMock() + tool_config.model_extra = {"api_key": "***", "max_results": raw} + mock.return_value.get_tool_config.return_value = tool_config + patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}})) + try: + from deerflow.community.youcom.tools import web_search_tool + + web_search_tool.invoke({"query": "test"}) + params = _get_call(mock_client_cls).kwargs["params"] + finally: + patcher.stop() + + assert params["count"] == 5 + + +def test_package_exports_web_search_tool(): + from deerflow.community.youcom import web_search_tool + from deerflow.community.youcom.tools import web_search_tool as direct_web_search_tool + + assert web_search_tool is direct_web_search_tool diff --git a/config.example.yaml b/config.example.yaml index 547b2770e1..288cf2d932 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1054,6 +1054,18 @@ tools: # contents_max_characters: 2000 # per-result content cap; 0 disables it # # api_key: $SOFYA_API_KEY # Optional explicit env-var reference + # Web search tool (uses You.com; YDC_API_KEY optional) + # You.com runs its own web index and returns extracted page snippets rather + # than a single meta description, so results are usable without a follow-up + # fetch. With no key it uses the keyless endpoint (free tier, currently 100 + # searches/day); set YDC_API_KEY and it switches to the full Search API. + # Get a key at https://you.com/platform + # - name: web_search + # group: web + # use: deerflow.community.youcom.tools:web_search_tool + # max_results: 5 # Clamped to 1-100 by You.com + # # api_key: $YDC_API_KEY # Omit to use the keyless free tier + # Web fetch tool (uses Browserless - headless Chrome, self-hosted or cloud) # Browserless renders pages with a real headless Chrome, ideal for JavaScript-heavy # sites and SPAs. Deploy your own: https://github.com/browserless/browserless diff --git a/frontend/src/content/en/harness/tools.mdx b/frontend/src/content/en/harness/tools.mdx index 391a329fa9..2589287be7 100644 --- a/frontend/src/content/en/harness/tools.mdx +++ b/frontend/src/content/en/harness/tools.mdx @@ -113,7 +113,7 @@ Community tools connect the agent to external services. They are configured in ` ### Web search - + ```yaml tools: @@ -206,6 +206,22 @@ tools: ``` Results include the content of each page, not only a snippet. Requires a [Sofya](https://sofya.co) API key. `max_results` is capped at 20, and each result's content is capped at `contents_max_characters` (default 2000, set 0 for no cap). Also supports `web_fetch`. + +```yaml +tools: + - use: deerflow.community.youcom.tools:web_search_tool + # api_key: $YDC_API_KEY # optional; omit to use the keyless free tier +``` +Results from You.com's own index, each carrying extracted page snippets rather than a +single meta description. Without a key it uses the keyless endpoint (free tier, currently +100 searches/day); set `YDC_API_KEY` and it switches to the full Search API with higher +limits. `max_results` is capped at 100. + +Optional API key from [You.com](https://you.com/platform). + +No extra dependency required. + + ### Web fetch (page content extraction) diff --git a/frontend/src/content/zh/harness/tools.mdx b/frontend/src/content/zh/harness/tools.mdx index 1ab5ba6dcc..ec7cfc66c1 100644 --- a/frontend/src/content/zh/harness/tools.mdx +++ b/frontend/src/content/zh/harness/tools.mdx @@ -110,7 +110,7 @@ tools: ### 网络搜索 - + ```yaml tools: @@ -200,6 +200,19 @@ tools: ``` 搜索结果包含每个页面的正文内容，而不仅是摘要。需要 [Sofya](https://sofya.co) API Key，`max_results` 上限为 20，每条结果的正文长度上限由 `contents_max_characters` 控制（默认 2000，设为 0 则不限制）。同时支持 `web_fetch`。 + +```yaml +tools: + - use: deerflow.community.youcom.tools:web_search_tool + # api_key: $YDC_API_KEY # 可选；不填则使用免密钥的免费额度 +``` +返回 You.com 自有索引的结果，每条结果附带从页面提取的片段，而不只是一段元描述。未配置 Key 时使用免密钥端点（免费额度，目前为每天 100 次搜索）；配置 `YDC_API_KEY` 后自动切换到额度更高的完整 Search API。`max_results` 上限为 100。 + +API Key 可选，可在 [You.com](https://you.com/platform) 获取。 + +无需额外依赖。 + + ### 网页内容抓取 diff --git a/scripts/doctor.py b/scripts/doctor.py index 1d4a276871..0cbbbeb5f1 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -589,7 +589,10 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes ) free_providers = { - "web_search": {"ddg_search": "DuckDuckGo (no key needed)"}, + "web_search": { + "ddg_search": "DuckDuckGo (no key needed)", + "youcom": "You.com (works with no key; YDC_API_KEY enables the full Search API)", + }, "web_fetch": {"jina_ai": "Jina AI Reader (no key needed)", "crawl4ai": "Crawl4AI (self-hosted, no key needed)"}, "image_search": {"deerflow.community.image_search.tools": "DuckDuckGo Images (no key needed)"}, } diff --git a/scripts/wizard/providers.py b/scripts/wizard/providers.py index ea9e3450d9..a2ba692ea3 100644 --- a/scripts/wizard/providers.py +++ b/scripts/wizard/providers.py @@ -699,6 +699,17 @@ SEARCH_PROVIDERS: list[SearchProvider] = [ env_var="WEBZ_API_KEY", extra_config={"max_results": 5}, ), + SearchProvider( + name="youcom", + display_name="You.com", + description="Own index with extracted page snippets, free tier needs no key", + use="deerflow.community.youcom.tools:web_search_tool", + # No env_var: the provider falls back to You.com's keyless endpoint, so the + # wizard should not demand a key. Operators who have one set YDC_API_KEY in + # .env (or api_key in config.yaml) and the provider switches endpoints. + env_var=None, + extra_config={"max_results": 5}, + ), ] WEB_FETCH_PROVIDERS: list[WebProvider] = [