mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
feat(community): migrate You.com tools to httpx.AsyncClient (#7186)
* feat(community): migrate You.com tools to async HTTP The You.com provider sent its search request through a sync httpx.Client. A sync LangChain tool invoked through ainvoke runs on the default executor, so the request held one of that pool's threads for the whole network round-trip, and several providers queried in parallel queued behind each other. Make the tool a coroutine over httpx.AsyncClient, resolve the API key and the configured max_results in a single config read, and move both disk reads off the event loop: the tool config, and the User-Agent, which reads the installed package metadata the first time it is needed. * refactor(community): drop the dead You.com API key wrapper The async migration resolved the key and max_results through _resolve_tool_settings, which left _get_api_key with no production caller: only TestGetApiKey still exercised it. Keeping a second seam around the same config read lets it drift from the real call path, so drop the wrapper and point the tests at _resolve_tool_settings. Also keep the hot path off the executor: _user_agent is cached, so only the first call reads the installed package metadata. Later calls are in-memory hits and now skip the to_thread round-trip. Finally, record which httpx client factory the provider reaches for in the runtime anchor. MockTransport runs its handler on the calling thread without real blocking I/O, so a provider that stayed async but sent the request through a sync client would satisfy the thread assertion; the recorded factory is what catches that partial regression. * test(community): align You.com URL safety stub with async tool The URL safety matrix still patched the removed _get_api_key seam and invoked the You.com tool synchronously. Mock the awaited search with AsyncMock and call the tool through ainvoke so URL screening remains covered after the async migration. --------- Co-authored-by: treasuresyq <309743290+treasuresyq@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
parent
3e9a417ea6
commit
6caa5addf3
@ -25,8 +25,21 @@ Both endpoints return the same body — ``{"results": {"web": [...], "news":
|
||||
This module is self-contained (httpx only, no You.com SDK and no new
|
||||
dependency). Requests to You.com carry a ``User-Agent`` identifying DeerFlow so
|
||||
You.com can attribute integration traffic; it is sent only to You.com hosts.
|
||||
|
||||
The tool is a coroutine and sends its request through ``httpx.AsyncClient``. A
|
||||
sync client would run the whole tool on the default executor and hold one of
|
||||
that pool's threads for the network round-trip, so several providers queried in
|
||||
parallel queued behind each other. Two things in this module read from disk,
|
||||
and both are resolved off the event loop: the tool config (which stats the
|
||||
config file) always runs on an executor thread, and the ``User-Agent`` (which
|
||||
reads the installed package metadata) does so on its first, cold call only.
|
||||
Later calls hit the in-memory cache and are read directly, so the hot path
|
||||
does not pay an executor round-trip for a string it already has.
|
||||
``backend/tests/blocking_io/test_youcom_search_async.py`` pins the result — the
|
||||
request is awaited on the loop thread, and neither disk read runs there.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@ -69,21 +82,42 @@ def _user_agent() -> str:
|
||||
return f"deerflow-harness/{harness_version} youdotcom-integration/bytedance-deer-flow"
|
||||
|
||||
|
||||
def _get_api_key(tool_name: str = "web_search") -> str | None:
|
||||
"""Resolve the You.com key from the tool's config block, then the env var.
|
||||
def _resolve_tool_settings(tool_name: str) -> tuple[str | None, object]:
|
||||
"""Resolve the API key and the configured `max_results` in one config read.
|
||||
|
||||
Returns None when neither is set, which is not an error here: the caller
|
||||
falls back to the keyless endpoint.
|
||||
Config resolution stats the config file, so callers must run this off the
|
||||
agent event loop. Reading the key and the result count from a single
|
||||
resolution keeps the tool from touching the config file twice per call.
|
||||
|
||||
The key is None when neither the config nor the environment sets one,
|
||||
which is not an error here: the caller falls back to the keyless endpoint.
|
||||
"""
|
||||
config = get_app_config().get_tool_config(tool_name)
|
||||
max_results = None
|
||||
api_key = None
|
||||
if config is not None:
|
||||
api_key = (config.model_extra or {}).get("api_key")
|
||||
if isinstance(api_key, str) and api_key.strip():
|
||||
return api_key.strip()
|
||||
env_key = os.getenv("YDC_API_KEY")
|
||||
if isinstance(env_key, str) and env_key.strip():
|
||||
return env_key.strip()
|
||||
return None
|
||||
extra = config.model_extra or {}
|
||||
max_results = extra.get("max_results")
|
||||
candidate = extra.get("api_key")
|
||||
if isinstance(candidate, str) and candidate.strip():
|
||||
api_key = candidate.strip()
|
||||
if api_key is None:
|
||||
env_key = os.getenv("YDC_API_KEY")
|
||||
if isinstance(env_key, str) and env_key.strip():
|
||||
api_key = env_key.strip()
|
||||
return api_key, max_results
|
||||
|
||||
|
||||
async def _resolve_user_agent() -> str:
|
||||
"""Return the DeerFlow User-Agent, keeping the cold metadata read off the loop.
|
||||
|
||||
The first call reads the installed package metadata from disk, so it runs
|
||||
on an executor thread. Every later call is an in-memory cache hit and is
|
||||
read directly, so the hot path costs no executor round-trip.
|
||||
"""
|
||||
if _user_agent.cache_info().currsize:
|
||||
return _user_agent()
|
||||
return await asyncio.to_thread(_user_agent)
|
||||
|
||||
|
||||
def _count_for_request(value: object) -> int:
|
||||
@ -100,17 +134,17 @@ def _count_for_request(value: object) -> int:
|
||||
)
|
||||
|
||||
|
||||
def _search(api_key: str | None, query: str, count: int) -> dict:
|
||||
async def _search(api_key: str | None, query: str, count: int) -> dict:
|
||||
"""GET the keyed endpoint when a key is configured, else the keyless one."""
|
||||
headers = {"Accept": "application/json", "User-Agent": _user_agent()}
|
||||
headers = {"Accept": "application/json", "User-Agent": await _resolve_user_agent()}
|
||||
if api_key:
|
||||
endpoint = _YOUCOM_SEARCH_ENDPOINT
|
||||
headers["X-API-Key"] = api_key
|
||||
else:
|
||||
endpoint = _YOUCOM_KEYLESS_ENDPOINT
|
||||
|
||||
with httpx.Client(timeout=_TIMEOUT_S) as client:
|
||||
response = client.get(endpoint, params={"query": query, "count": count}, headers=headers)
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT_S) as client:
|
||||
response = await client.get(endpoint, params={"query": query, "count": count}, headers=headers)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
@ -130,7 +164,7 @@ def _snippet(result: dict) -> str:
|
||||
|
||||
|
||||
@tool("web_search", parse_docstring=True)
|
||||
def web_search_tool(query: str, max_results: int | None = None) -> str:
|
||||
async def web_search_tool(query: str, max_results: int | None = None) -> str:
|
||||
"""Search the web for information using You.com.
|
||||
|
||||
Each result carries extracted snippets from the page, not just a meta
|
||||
@ -140,17 +174,15 @@ def web_search_tool(query: str, max_results: int | None = None) -> str:
|
||||
query: Search keywords describing what you want to find. Be specific for better results.
|
||||
max_results: Maximum number of search results to return. If omitted, uses the configured value (default 5). Clamped to 1-100.
|
||||
"""
|
||||
# Config resolution can read files; keep it off the event loop.
|
||||
api_key, configured_max_results = await asyncio.to_thread(_resolve_tool_settings, "web_search")
|
||||
# Honor the caller-supplied max_results; fall back to config only when omitted.
|
||||
if max_results is None:
|
||||
config = get_app_config().get_tool_config("web_search")
|
||||
if config is not None:
|
||||
max_results = (config.model_extra or {}).get("max_results")
|
||||
max_results = configured_max_results
|
||||
count = _DEFAULT_MAX_RESULTS if max_results is None else _count_for_request(max_results)
|
||||
|
||||
api_key = _get_api_key("web_search")
|
||||
|
||||
try:
|
||||
data = _search(api_key, query, count)
|
||||
data = await _search(api_key, query, count)
|
||||
except httpx.HTTPStatusError as e:
|
||||
status = e.response.status_code
|
||||
logger.error("You.com API returned HTTP %s: %s", status, e.response.text)
|
||||
|
||||
159
backend/tests/blocking_io/test_youcom_search_async.py
Normal file
159
backend/tests/blocking_io/test_youcom_search_async.py
Normal file
@ -0,0 +1,159 @@
|
||||
"""The You.com community tool must not occupy an executor thread or block the loop.
|
||||
|
||||
The You.com provider resolved its config and sent its search request through
|
||||
a synchronous ``httpx.Client``. A sync LangChain tool invoked through
|
||||
``ainvoke`` runs on the default executor, so the request held one of that
|
||||
pool's threads for the whole network round-trip: with several providers
|
||||
queried in parallel the default pool ran out of threads and the tool calls
|
||||
queued behind each other.
|
||||
|
||||
The probes below assert the whole fix:
|
||||
|
||||
* the request is awaited on the event loop thread instead of an executor
|
||||
thread, so no pool thread is held for the round-trip;
|
||||
* the config lookup, which reads the config file, still runs off the loop.
|
||||
The tool body is now a coroutine that executes on the loop, so a file read
|
||||
left in it would block the loop; the strict blocking-I/O gate fails if the
|
||||
lookup regresses onto the loop.
|
||||
* the request goes out through the async client. A `MockTransport` runs its
|
||||
handler on the calling thread and performs no real blocking I/O, so a
|
||||
provider that stayed async but sent the request through a sync
|
||||
`httpx.Client` would satisfy the thread assertion above; the recorded
|
||||
client factory is what catches that partial regression.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from deerflow.community.youcom import tools as youcom_tools
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
def _patch_transport(monkeypatch, request_threads: list[int], clients: list[str]) -> None:
|
||||
"""Answer every request locally and record the thread that served it.
|
||||
|
||||
The provider's module holds the shared ``httpx`` module, so the reference
|
||||
itself is replaced with a namespace exposing both client factories. That
|
||||
keeps the test hermetic — no request leaves the machine — whichever
|
||||
factory the provider uses. Each factory records its name in ``clients``
|
||||
so a test can assert which one the provider reached for.
|
||||
"""
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
request_threads.append(threading.get_ident())
|
||||
return httpx.Response(200, json={"results": {"web": []}})
|
||||
|
||||
transport = httpx.MockTransport(respond)
|
||||
|
||||
def async_client(**kwargs):
|
||||
clients.append("AsyncClient")
|
||||
return httpx.AsyncClient(transport=transport, **kwargs)
|
||||
|
||||
def sync_client(**kwargs):
|
||||
clients.append("Client")
|
||||
return httpx.Client(transport=transport, **kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
youcom_tools,
|
||||
"httpx",
|
||||
SimpleNamespace(
|
||||
AsyncClient=async_client,
|
||||
Client=sync_client,
|
||||
HTTPStatusError=httpx.HTTPStatusError,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _patch_config(monkeypatch, load_config=None) -> None:
|
||||
def config():
|
||||
return SimpleNamespace(get_tool_config=lambda _name: SimpleNamespace(model_extra={"api_key": "synthetic-key"}))
|
||||
|
||||
monkeypatch.setattr(youcom_tools, "get_app_config", load_config or config)
|
||||
|
||||
|
||||
async def test_youcom_search_request_runs_on_the_event_loop(monkeypatch) -> None:
|
||||
"""The request is awaited on the loop, not dispatched to an executor thread."""
|
||||
request_threads: list[int] = []
|
||||
clients: list[str] = []
|
||||
loop_thread = threading.get_ident()
|
||||
_patch_transport(monkeypatch, request_threads, clients)
|
||||
_patch_config(monkeypatch)
|
||||
|
||||
result = await youcom_tools.web_search_tool.ainvoke({"query": "vector databases"})
|
||||
|
||||
assert '"error": "No results found"' in result
|
||||
assert request_threads, "the request must reach the transport"
|
||||
assert all(thread == loop_thread for thread in request_threads)
|
||||
assert clients == ["AsyncClient"], "the request must go out through the async client"
|
||||
|
||||
|
||||
async def test_youcom_config_lookup_does_not_block_loop(monkeypatch) -> None:
|
||||
"""The tool body runs on the loop, so the config file read must not."""
|
||||
config_threads: list[int] = []
|
||||
loop_thread = threading.get_ident()
|
||||
|
||||
def load_config():
|
||||
# Keep a real file read in the lookup so the strict blocking-I/O gate
|
||||
# fails if the tool resolves config on the event loop.
|
||||
Path(__file__).read_text(encoding="utf-8")
|
||||
config_threads.append(threading.get_ident())
|
||||
return SimpleNamespace(get_tool_config=lambda _name: SimpleNamespace(model_extra={"api_key": "synthetic-key"}))
|
||||
|
||||
request_threads: list[int] = []
|
||||
clients: list[str] = []
|
||||
_patch_transport(monkeypatch, request_threads, clients)
|
||||
_patch_config(monkeypatch, load_config)
|
||||
|
||||
result = await youcom_tools.web_search_tool.ainvoke({"query": "vector databases"})
|
||||
|
||||
assert '"error": "No results found"' in result
|
||||
assert config_threads, "config lookup must run"
|
||||
assert all(thread != loop_thread for thread in config_threads)
|
||||
assert clients == ["AsyncClient"], "the request must go out through the async client"
|
||||
|
||||
|
||||
async def test_youcom_user_agent_reads_metadata_off_the_loop_once(monkeypatch) -> None:
|
||||
"""The cold User-Agent read goes to an executor; the warm hit stays in memory.
|
||||
|
||||
``_user_agent`` is cached, so only the first call reads the installed
|
||||
package metadata. That read must stay off the event loop, and every later
|
||||
call is a pure memory hit that must not pay an executor round-trip.
|
||||
"""
|
||||
loop_thread = threading.get_ident()
|
||||
reads: list[int] = []
|
||||
real_version = youcom_tools.version
|
||||
|
||||
def read_version(name: str) -> str:
|
||||
reads.append(threading.get_ident())
|
||||
return real_version(name)
|
||||
|
||||
monkeypatch.setattr(youcom_tools, "version", read_version)
|
||||
youcom_tools._user_agent.cache_clear()
|
||||
try:
|
||||
cold = await youcom_tools._resolve_user_agent()
|
||||
assert reads, "the cold call must read the installed package metadata"
|
||||
assert all(thread != loop_thread for thread in reads), "the metadata read must not run on the event loop"
|
||||
|
||||
dispatched: list[object] = []
|
||||
real_to_thread = asyncio.to_thread
|
||||
|
||||
async def spy_to_thread(func, /, *args, **kwargs):
|
||||
dispatched.append(func)
|
||||
return await real_to_thread(func, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(asyncio, "to_thread", spy_to_thread)
|
||||
reads.clear()
|
||||
|
||||
warm = await youcom_tools._resolve_user_agent()
|
||||
|
||||
assert warm == cold
|
||||
assert not reads, "the warm call must not read the metadata again"
|
||||
assert not dispatched, "the warm call must not pay an executor round-trip"
|
||||
finally:
|
||||
youcom_tools._user_agent.cache_clear()
|
||||
@ -80,7 +80,7 @@ def _stub_exa(module, url):
|
||||
|
||||
def _stub_youcom(module, url):
|
||||
data = {"results": {"web": [{"title": "t", "url": url, "description": "c"}]}}
|
||||
return [patch.object(module, "_get_api_key", return_value="k"), patch.object(module, "_search", return_value=data)], _sync(module.web_search_tool)
|
||||
return [patch.object(module, "_search", new=AsyncMock(return_value=data))], _async(module.web_search_tool)
|
||||
|
||||
|
||||
def _stub_tencent_wsa(module, url):
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
import json
|
||||
import logging
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@ -10,6 +10,8 @@ import pytest
|
||||
KEYED_ENDPOINT = "https://api.you.com/v1/search"
|
||||
KEYLESS_ENDPOINT = "https://api.you.com/v1/agents/search"
|
||||
|
||||
pytestmark = pytest.mark.anyio
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config_with_key():
|
||||
@ -37,15 +39,15 @@ def _make_response(payload: object) -> MagicMock:
|
||||
|
||||
|
||||
def _patch_get(mock_resp: MagicMock):
|
||||
"""Patch httpx.Client so the context-managed .get returns mock_resp."""
|
||||
patcher = patch("deerflow.community.youcom.tools.httpx.Client")
|
||||
"""Patch httpx.AsyncClient so the context-managed .get returns mock_resp."""
|
||||
patcher = patch("deerflow.community.youcom.tools.httpx.AsyncClient")
|
||||
mock_client_cls = patcher.start()
|
||||
mock_client_cls.return_value.__enter__.return_value.get.return_value = mock_resp
|
||||
mock_client_cls.return_value.__aenter__.return_value.get = AsyncMock(return_value=mock_resp)
|
||||
return patcher, mock_client_cls
|
||||
|
||||
|
||||
def _get_call(mock_client_cls):
|
||||
return mock_client_cls.return_value.__enter__.return_value.get.call_args
|
||||
return mock_client_cls.return_value.__aenter__.return_value.get.call_args
|
||||
|
||||
|
||||
def _web(**overrides) -> dict:
|
||||
@ -54,16 +56,16 @@ def _web(**overrides) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
class TestGetApiKey:
|
||||
class TestResolveToolSettings:
|
||||
def test_returns_config_key_when_present(self):
|
||||
with patch("deerflow.community.youcom.tools.get_app_config") as mock:
|
||||
tool_config = MagicMock()
|
||||
tool_config.model_extra = {"api_key": "from-config"}
|
||||
mock.return_value.get_tool_config.return_value = tool_config
|
||||
|
||||
from deerflow.community.youcom.tools import _get_api_key
|
||||
from deerflow.community.youcom.tools import _resolve_tool_settings
|
||||
|
||||
assert _get_api_key("web_search") == "from-config"
|
||||
assert _resolve_tool_settings("web_search")[0] == "from-config"
|
||||
|
||||
def test_falls_back_to_env_when_config_key_blank(self):
|
||||
with patch("deerflow.community.youcom.tools.get_app_config") as mock:
|
||||
@ -71,27 +73,27 @@ class TestGetApiKey:
|
||||
tool_config.model_extra = {"api_key": " "}
|
||||
mock.return_value.get_tool_config.return_value = tool_config
|
||||
with patch.dict("os.environ", {"YDC_API_KEY": "env-key"}, clear=True):
|
||||
from deerflow.community.youcom.tools import _get_api_key
|
||||
from deerflow.community.youcom.tools import _resolve_tool_settings
|
||||
|
||||
assert _get_api_key("web_search") == "env-key"
|
||||
assert _resolve_tool_settings("web_search")[0] == "env-key"
|
||||
|
||||
def test_returns_none_when_no_key_anywhere(self):
|
||||
"""No key is not an error: it selects the keyless endpoint."""
|
||||
with patch("deerflow.community.youcom.tools.get_app_config") as mock:
|
||||
mock.return_value.get_tool_config.return_value = None
|
||||
with patch.dict("os.environ", {}, clear=True):
|
||||
from deerflow.community.youcom.tools import _get_api_key
|
||||
from deerflow.community.youcom.tools import _resolve_tool_settings
|
||||
|
||||
assert _get_api_key("web_search") is None
|
||||
assert _resolve_tool_settings("web_search")[0] is None
|
||||
|
||||
|
||||
class TestEndpointSelection:
|
||||
def test_configured_key_uses_keyed_endpoint_with_header(self, mock_config_with_key):
|
||||
async def test_configured_key_uses_keyed_endpoint_with_header(self, mock_config_with_key):
|
||||
patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
call = _get_call(mock_client_cls)
|
||||
finally:
|
||||
patcher.stop()
|
||||
@ -100,14 +102,14 @@ class TestEndpointSelection:
|
||||
assert call.kwargs["headers"]["X-API-Key"] == "test-ydc-key"
|
||||
assert call.kwargs["params"] == {"query": "test", "count": 5}
|
||||
|
||||
def test_no_key_uses_keyless_endpoint_without_auth_header(self, mock_config_no_key):
|
||||
async def test_no_key_uses_keyless_endpoint_without_auth_header(self, mock_config_no_key):
|
||||
"""The keyless endpoint rejects an auth header, so none may be sent."""
|
||||
with patch.dict("os.environ", {}, clear=True):
|
||||
patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
call = _get_call(mock_client_cls)
|
||||
finally:
|
||||
patcher.stop()
|
||||
@ -115,13 +117,13 @@ class TestEndpointSelection:
|
||||
assert call.args[0] == KEYLESS_ENDPOINT
|
||||
assert "X-API-Key" not in call.kwargs["headers"]
|
||||
|
||||
def test_env_key_uses_keyed_endpoint(self, mock_config_no_key):
|
||||
async def test_env_key_uses_keyed_endpoint(self, mock_config_no_key):
|
||||
with patch.dict("os.environ", {"YDC_API_KEY": "env-key"}, clear=True):
|
||||
patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
call = _get_call(mock_client_cls)
|
||||
finally:
|
||||
patcher.stop()
|
||||
@ -130,7 +132,7 @@ class TestEndpointSelection:
|
||||
assert call.kwargs["headers"]["X-API-Key"] == "env-key"
|
||||
|
||||
@pytest.mark.parametrize("with_key", [True, False])
|
||||
def test_identifies_deerflow_in_user_agent(self, with_key, mock_config_no_key):
|
||||
async def test_identifies_deerflow_in_user_agent(self, with_key, mock_config_no_key):
|
||||
"""Both endpoints identify DeerFlow so You.com can attribute the traffic."""
|
||||
env = {"YDC_API_KEY": "k"} if with_key else {}
|
||||
with patch.dict("os.environ", env, clear=True):
|
||||
@ -138,7 +140,7 @@ class TestEndpointSelection:
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
call = _get_call(mock_client_cls)
|
||||
finally:
|
||||
patcher.stop()
|
||||
@ -149,7 +151,7 @@ class TestEndpointSelection:
|
||||
|
||||
|
||||
class TestWebSearchTool:
|
||||
def test_returns_normalized_list_with_joined_snippets(self, mock_config_with_key):
|
||||
async def test_returns_normalized_list_with_joined_snippets(self, mock_config_with_key):
|
||||
payload = {
|
||||
"results": {
|
||||
"web": [
|
||||
@ -162,7 +164,7 @@ class TestWebSearchTool:
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "vector databases"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "vector databases"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
@ -171,7 +173,7 @@ class TestWebSearchTool:
|
||||
{"title": "B", "url": "https://ex.com/b", "snippet": "s3"},
|
||||
]
|
||||
|
||||
def test_falls_back_to_description_when_no_snippets(self, mock_config_with_key):
|
||||
async def test_falls_back_to_description_when_no_snippets(self, mock_config_with_key):
|
||||
"""News hits carry only a description; web hits with empty snippets do too."""
|
||||
payload = {
|
||||
"results": {
|
||||
@ -183,13 +185,13 @@ class TestWebSearchTool:
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "news"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "news"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert [r["snippet"] for r in parsed] == ["web-desc", "news-desc"]
|
||||
|
||||
def test_web_and_news_merged_and_trimmed_to_count(self, mock_config_with_key):
|
||||
async def test_web_and_news_merged_and_trimmed_to_count(self, mock_config_with_key):
|
||||
"""`count` applies per section, so the merged list can overshoot it."""
|
||||
payload = {
|
||||
"results": {
|
||||
@ -201,31 +203,31 @@ class TestWebSearchTool:
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "test"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "test"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert len(parsed) == 5
|
||||
assert [r["title"] for r in parsed] == ["W0", "W1", "W2", "W3", "W4"]
|
||||
|
||||
def test_agent_max_results_is_honored_over_config(self, mock_config_with_key):
|
||||
async def test_agent_max_results_is_honored_over_config(self, mock_config_with_key):
|
||||
patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test", "max_results": 20})
|
||||
await web_search_tool.ainvoke({"query": "test", "max_results": 20})
|
||||
params = _get_call(mock_client_cls).kwargs["params"]
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert params["count"] == 20
|
||||
|
||||
def test_config_max_results_used_when_caller_omits(self, mock_config_with_key):
|
||||
async def test_config_max_results_used_when_caller_omits(self, mock_config_with_key):
|
||||
patcher, mock_client_cls = _patch_get(_make_response({"results": {"web": [_web()]}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
params = _get_call(mock_client_cls).kwargs["params"]
|
||||
finally:
|
||||
patcher.stop()
|
||||
@ -233,7 +235,7 @@ class TestWebSearchTool:
|
||||
assert params["count"] == 5
|
||||
|
||||
@pytest.mark.parametrize(("max_results", "expected"), [(0, 5), (-2, 5), (1, 1), (100, 100), ("500", 100)])
|
||||
def test_max_results_clamped_to_provider_bounds(self, max_results, expected):
|
||||
async def test_max_results_clamped_to_provider_bounds(self, max_results, expected):
|
||||
with patch("deerflow.community.youcom.tools.get_app_config") as mock:
|
||||
tool_config = MagicMock()
|
||||
tool_config.model_extra = {"api_key": "k", "max_results": max_results}
|
||||
@ -242,14 +244,14 @@ class TestWebSearchTool:
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
params = _get_call(mock_client_cls).kwargs["params"]
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert params["count"] == expected
|
||||
|
||||
def test_invalid_max_results_falls_back_to_default_with_warning(self, caplog):
|
||||
async def test_invalid_max_results_falls_back_to_default_with_warning(self, caplog):
|
||||
with patch("deerflow.community.youcom.tools.get_app_config") as mock:
|
||||
tool_config = MagicMock()
|
||||
tool_config.model_extra = {"api_key": "k", "max_results": "not-a-number"}
|
||||
@ -259,7 +261,7 @@ class TestWebSearchTool:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="deerflow.community.youcom.tools"):
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
params = _get_call(mock_client_cls).kwargs["params"]
|
||||
finally:
|
||||
patcher.stop()
|
||||
@ -267,46 +269,46 @@ class TestWebSearchTool:
|
||||
assert params["count"] == 5
|
||||
assert any("Invalid You.com max_results" in record.message for record in caplog.records)
|
||||
|
||||
def test_empty_results_returns_error_json(self, mock_config_with_key):
|
||||
async def test_empty_results_returns_error_json(self, mock_config_with_key):
|
||||
patcher, _ = _patch_get(_make_response({"results": {"web": []}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "no results"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "no results"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert parsed["error"] == "No results found"
|
||||
assert parsed["query"] == "no results"
|
||||
|
||||
def test_missing_results_section_returns_error_json(self, mock_config_with_key):
|
||||
async def test_missing_results_section_returns_error_json(self, mock_config_with_key):
|
||||
patcher, _ = _patch_get(_make_response({"metadata": {"latency": 0.1}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "test"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "test"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert parsed["error"] == "No results found"
|
||||
|
||||
def test_non_dict_payload_returns_format_error(self, mock_config_with_key):
|
||||
async def test_non_dict_payload_returns_format_error(self, mock_config_with_key):
|
||||
patcher, _ = _patch_get(_make_response(["not", "a", "dict"]))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "test"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "test"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
assert "unexpected response format" in parsed["error"]
|
||||
|
||||
def test_partial_fields_default_to_empty_string(self, mock_config_with_key):
|
||||
async def test_partial_fields_default_to_empty_string(self, mock_config_with_key):
|
||||
patcher, _ = _patch_get(_make_response({"results": {"web": [{}]}}))
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "test"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "test"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
@ -321,7 +323,7 @@ class TestWebSearchTool:
|
||||
(500, None),
|
||||
],
|
||||
)
|
||||
def test_http_error_returns_structured_error(self, status, hint, mock_config_with_key):
|
||||
async def test_http_error_returns_structured_error(self, status, hint, mock_config_with_key):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError(
|
||||
str(status),
|
||||
@ -332,7 +334,7 @@ class TestWebSearchTool:
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "test"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "test"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
@ -340,13 +342,13 @@ class TestWebSearchTool:
|
||||
if hint:
|
||||
assert hint in parsed["error"]
|
||||
|
||||
def test_network_exception_returns_error_json(self, mock_config_with_key):
|
||||
async def test_network_exception_returns_error_json(self, mock_config_with_key):
|
||||
patcher, mock_client_cls = _patch_get(MagicMock())
|
||||
mock_client_cls.return_value.__enter__.return_value.get.side_effect = Exception("timeout")
|
||||
mock_client_cls.return_value.__aenter__.return_value.get.side_effect = Exception("timeout")
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
parsed = json.loads(web_search_tool.invoke({"query": "test"}))
|
||||
parsed = json.loads(await web_search_tool.ainvoke({"query": "test"}))
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
@ -358,7 +360,7 @@ class TestWebSearchTool:
|
||||
[float("inf"), True, 3.5],
|
||||
ids=["yaml-inf", "bool", "non-integral-float"],
|
||||
)
|
||||
def test_invalid_max_results_shapes_fall_back_to_default(raw):
|
||||
async def test_invalid_max_results_shapes_fall_back_to_default(raw):
|
||||
"""Invalid max_results shapes fall back to the default, never crash or truncate.
|
||||
|
||||
These pin the shared-owner bar for this provider: a YAML ``.inf`` makes
|
||||
@ -375,7 +377,7 @@ def test_invalid_max_results_shapes_fall_back_to_default(raw):
|
||||
try:
|
||||
from deerflow.community.youcom.tools import web_search_tool
|
||||
|
||||
web_search_tool.invoke({"query": "test"})
|
||||
await web_search_tool.ainvoke({"query": "test"})
|
||||
params = _get_call(mock_client_cls).kwargs["params"]
|
||||
finally:
|
||||
patcher.stop()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user