From 4c4a4e985807c086fa4d194e4514db2291373790 Mon Sep 17 00:00:00 2001 From: Cordis798 <2378285593@qq.com> Date: Wed, 4 Nov 2026 20:54:43 +0800 Subject: [PATCH] fix(community): initialize search TLS clients off the event loop (#7889) Co-authored-by: Cordis798 <284918518+Cordis798@users.noreply.github.com> Co-authored-by: Willem Jiang --- README.md | 2 + backend/AGENTS.md | 4 +- .../deerflow/community/groundroute/tools.py | 2 +- .../harness/deerflow/community/webz/tools.py | 2 +- .../deerflow/community/youcom/tools.py | 2 +- .../blocking_io/test_search_client_ssl.py | 71 +++++++++++++++++++ 6 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 backend/tests/blocking_io/test_search_client_ssl.py diff --git a/README.md b/README.md index 0c9e15e187..4f02abbc1f 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,8 @@ fences, including unclosed fences, in complete and streamed replies. Brave web search preserves valid entries in mixed result lists. Malformed response containers or lists containing no usable entries return a structured format error; missing, null, or empty results keep the existing "No results found" response. Format errors also log the malformed container's path and type, or the absence of usable result objects, without including search queries, credentials, or payload values. + You.com, GroundRoute and Webz initialize HTTPX clients in a worker thread so synchronous TLS trust loading does not block the agent event loop; requests and session cleanup remain asynchronous. + GroundRoute web search and fetch contain malformed HTTP-success payloads the same way: a non-object payload, a `results` container that is not a list, or a non-empty list with no object entries returns the provider's format error, while non-object entries in an otherwise valid list are skipped in order. Missing, null, or empty results keep the existing "No results found" response. DuckDuckGo web search distinguishes execution failures from empty results: rate limits, timeouts, blocked requests, client-setup failures, SDK errors, and a missing `ddgs` dependency return a structured `DuckDuckGo search failed: ...` error instead of "No results found", while a genuinely empty result set keeps the existing "No results found" response. The structured error text lets the harness classify an outage as transient or a configuration problem instead of a query that matched nothing. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 2c2529b432..8328788123 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -136,8 +136,8 @@ Read the nearest scoped `AGENTS.md` for subsystem ownership. Read-only knowledge: [contract](docs/KNOWLEDGE.md) covers independent retrieval tools, scope intersection and extension-owned providers. -RAGFlow/LightRAG clients offload HTTPX construction (synchronous TLS trust loading) -with `asyncio.to_thread`; keep request execution and session cleanup on the loop. +HTTPX clients for RAGFlow, LightRAG, You.com, GroundRoute and Webz use +`asyncio.to_thread` for synchronous TLS setup; requests and cleanup stay on-loop. ### Harness / App Split diff --git a/backend/packages/harness/deerflow/community/groundroute/tools.py b/backend/packages/harness/deerflow/community/groundroute/tools.py index eb67af2800..b3522634fe 100644 --- a/backend/packages/harness/deerflow/community/groundroute/tools.py +++ b/backend/packages/harness/deerflow/community/groundroute/tools.py @@ -112,7 +112,7 @@ def _response_results(data: object) -> list[dict] | None: async def _post_search(api_key: str, body: dict) -> dict: - async with httpx.AsyncClient(timeout=_TIMEOUT_S) as client: + async with await asyncio.to_thread(httpx.AsyncClient, timeout=_TIMEOUT_S) as client: response = await client.post( _GROUNDROUTE_ENDPOINT, json=body, diff --git a/backend/packages/harness/deerflow/community/webz/tools.py b/backend/packages/harness/deerflow/community/webz/tools.py index bbef1c00c5..5ec44aa621 100644 --- a/backend/packages/harness/deerflow/community/webz/tools.py +++ b/backend/packages/harness/deerflow/community/webz/tools.py @@ -135,7 +135,7 @@ async def web_search_tool( if filters: body["filters"] = filters try: - async with httpx.AsyncClient(timeout=30.0, follow_redirects=False) as client: + async with await asyncio.to_thread(httpx.AsyncClient, timeout=30.0, follow_redirects=False) as client: response = await client.post( "https://api.webz.io/api/news/context", headers={"Authorization": f"Bearer {key}", "Accept": "application/json"}, diff --git a/backend/packages/harness/deerflow/community/youcom/tools.py b/backend/packages/harness/deerflow/community/youcom/tools.py index 9dc3743019..c9bc3c55b6 100644 --- a/backend/packages/harness/deerflow/community/youcom/tools.py +++ b/backend/packages/harness/deerflow/community/youcom/tools.py @@ -143,7 +143,7 @@ async def _search(api_key: str | None, query: str, count: int) -> dict: else: endpoint = _YOUCOM_KEYLESS_ENDPOINT - async with httpx.AsyncClient(timeout=_TIMEOUT_S) as client: + async with await asyncio.to_thread(httpx.AsyncClient, timeout=_TIMEOUT_S) as client: response = await client.get(endpoint, params={"query": query, "count": count}, headers=headers) response.raise_for_status() return response.json() diff --git a/backend/tests/blocking_io/test_search_client_ssl.py b/backend/tests/blocking_io/test_search_client_ssl.py new file mode 100644 index 0000000000..7e5e763ee0 --- /dev/null +++ b/backend/tests/blocking_io/test_search_client_ssl.py @@ -0,0 +1,71 @@ +"""Exercise real search-tool TLS setup while replacing only network responses.""" + +import json +import ssl +import threading +from types import SimpleNamespace + +import httpx +import pytest + +from deerflow.community.groundroute import tools as groundroute +from deerflow.community.webz import tools as webz +from deerflow.community.youcom import tools as youcom + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ["youcom-keyed", "youcom-keyless", "groundroute-search", "groundroute-fetch", "webz"]) +async def test_search_tool_tls_setup_runs_off_loop(provider, monkeypatch): + loop_thread = threading.get_ident() + original_context = ssl.create_default_context + contexts = [] + clients = [] + + def create_context(*args, **kwargs): + assert threading.get_ident() != loop_thread, "TLS trust loading blocks the event loop" + context = original_context(*args, **kwargs) + contexts.append(context) + return context + + async def send(self, request, **kwargs): + assert threading.get_ident() == loop_thread + clients.append(self) + if provider.startswith("youcom"): + assert request.url.host == "api.you.com" + assert request.url.params["count"] == "5" + if provider == "youcom-keyed": + assert request.headers["X-API-Key"] == "synthetic-key" + assert request.url.path == "/v1/search" + else: + assert "X-API-Key" not in request.headers + assert request.url.path == "/v1/agents/search" + payload = {"results": {"web": [{"url": "https://example.com", "title": "Synthetic", "snippets": ["Synthetic text"]}]}} + elif provider.startswith("groundroute"): + assert request.url.host == "api.groundroute.ai" + assert request.headers["Authorization"] == "Bearer synthetic-key" + body = json.loads(request.content) + assert body["max_results"] == (1 if provider.endswith("fetch") else 5) + payload = {"results": [{"url": "https://example.com", "title": "Synthetic", "snippet": "Synthetic text", "content": "Synthetic text"}]} + else: + assert request.url.host == "api.webz.io" + assert request.headers["Authorization"] == "Bearer synthetic-key" + assert json.loads(request.content) == {"query": "synthetic query", "k": 5} + payload = {"results": [{"article": {"url": "https://example.com", "title": "Synthetic", "summary": "Synthetic text"}}]} + return httpx.Response(200, request=request, json=payload) + + def config(): + key = "" if provider == "youcom-keyless" else "synthetic-key" + return SimpleNamespace(get_tool_config=lambda _: SimpleNamespace(model_extra={"api_key": key, "max_results": 5})) + + for module in (youcom, groundroute, webz): + monkeypatch.setattr(module, "get_app_config", config) + monkeypatch.delenv("YDC_API_KEY", raising=False) + monkeypatch.setattr(ssl, "create_default_context", create_context) + monkeypatch.setattr(httpx.AsyncClient, "send", send) + module = youcom if provider.startswith("youcom") else groundroute if provider.startswith("groundroute") else webz + tool = module.web_fetch_tool if provider.endswith("fetch") else module.web_search_tool + result = await tool.ainvoke({"url": "https://example.com"} if provider.endswith("fetch") else {"query": "synthetic query"}) + assert "Synthetic text" in result + assert contexts + assert len(clients) == 1 + assert clients[0].is_closed