github/deer-flow
1
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-10-26 13:16:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(community): migrate infoquest client to httpx.AsyncClient (#5782)

Browse Source 
* feat(community): migrate infoquest client to httpx.AsyncClient

Replace blocking sync requests.post calls with httpx.AsyncClient in the
InfoQuest community tool, as tracked in backend/docs/TODO.md planned
features. fetch/web_search/image_search and their raw-results helpers are
now async, keeping the 30s transport timeout and external behavior
(return values, error strings) unchanged.

The langchain tool wrappers in infoquest/tools.py become async functions;
web_fetch offloads readability extraction to a thread (same pattern as
the already-migrated jina_ai tool). Tests are updated to mock
httpx.AsyncClient instead of requests.post.

* test(community): use AsyncMock for infoquest fetch in relative-links test

The infoquest web_fetch_tool now awaits client.fetch, so the test stub
must return an awaitable instead of a sync lambda.

* fix(community): follow InfoQuest API redirects like the sync client did

Address the review note on #5782: requests.post follows redirects by
default while httpx.AsyncClient defaults to follow_redirects=False, so
a 3xx answer from the reader/search endpoints would have surfaced as
'Error: fetch API returned status 30x' (or fallen into the generic
error path after raise_for_status) instead of the final content.

All three AsyncClient call sites now pass follow_redirects=True. Adds
regression tests asserting each client is constructed with
follow_redirects=True, and documents the parity in the test contract.

* test(community): fix InfoQuest mock endpoint typo
This commit is contained in:
yc2bgr8 2026-09-25 20:58:17 +08:00 committed by GitHub
parent 65ed502e54
commit 3d921afa5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 269 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
backend/packages/harness/deerflow/community/infoquest/infoquest_client.py
View File

@ -9,11 +9,11 @@ import logging
import os
from typing import Any
import requests
import httpx
logger = logging.getLogger(__name__)
# Requests has no default timeout. This bounds local connect/read inactivity;
# httpx defaults to a 5s timeout. This bounds local connect/read inactivity;
# fetch_timeout and fetch_navigation_timeout configure the remote crawl only.
_REQUEST_TIMEOUT_SECONDS = 30
@ -46,7 +46,7 @@ class InfoQuestClient:
logger.debug(config_details)
logger.debug("\n" + "*" * 70 + "\n")
def fetch(self, url: str, return_format: str = "html") -> str:
async def fetch(self, url: str, return_format: str = "html") -> str:
if logger.isEnabledFor(logging.DEBUG):
url_truncated = url[:50] + "..." if len(url) > 50 else url
logger.debug(
@ -56,7 +56,7 @@ class InfoQuestClient:
f"has_timeout_filter={self.fetch_timeout > 0} | timeout_filter={self.fetch_timeout} | "
f"has_fetch_time_filter={self.fetch_time > 0} | fetch_time_filter={self.fetch_time} | "
f"has_navigation_timeout_filter={self.fetch_navigation_timeout > 0} | navi_timeout_filter={self.fetch_navigation_timeout} | "
f"request_type=sync"
f"request_type=async"
)
# Prepare headers
@ -67,7 +67,8 @@ class InfoQuestClient:
logger.debug("Sending crawl request to InfoQuest API")
try:
response = requests.post("https://reader.infoquest.bytepluses.com", headers=headers, json=data, timeout=_REQUEST_TIMEOUT_SECONDS)
async with httpx.AsyncClient(follow_redirects=True) as client:
response = await client.post("https://reader.infoquest.bytepluses.com", headers=headers, json=data, timeout=_REQUEST_TIMEOUT_SECONDS)
# Check if status code is not 200
if response.status_code != 200:
@ -152,13 +153,13 @@ class InfoQuestClient:
return data
def web_search_raw_results(
async def web_search_raw_results(
self,
query: str,
site: str,
output_format: str = "JSON",
) -> dict:
"""Get results from the InfoQuest Web-Search API synchronously."""
"""Get results from the InfoQuest Web-Search API asynchronously."""
headers = self._prepare_headers()
params = {"format": output_format, "query": query}
@ -168,7 +169,8 @@ class InfoQuestClient:
if site != "":
params["site"] = site
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params, timeout=_REQUEST_TIMEOUT_SECONDS)
async with httpx.AsyncClient(follow_redirects=True) as client:
response = await client.post("https://search.infoquest.bytepluses.com", headers=headers, json=params, timeout=_REQUEST_TIMEOUT_SECONDS)
response.raise_for_status()
# Print partial response for debugging
@ -235,7 +237,7 @@ class InfoQuestClient:
return clean_results
def web_search(
async def web_search(
self,
query: str,
site: str = "",
@ -249,12 +251,12 @@ class InfoQuestClient:
f"query_truncated={query_truncated} | "
f"has_time_filter={self.search_time_range > 0} | time_filter={self.search_time_range} | "
f"has_site_filter={bool(site)} | site={site} | "
f"request_type=sync"
f"request_type=async"
)
try:
logger.debug("InfoQuest Web-Search - Executing search with parameters")
raw_results = self.web_search_raw_results(
raw_results = await self.web_search_raw_results(
query,
site,
output_format,
@ -268,7 +270,7 @@ class InfoQuestClient:
result_json = json.dumps(cleaned_results, indent=2, ensure_ascii=False)
logger.debug(f"InfoQuest Web-Search - Search tool execution completed | mode=synchronous | results_count={len(cleaned_results)}")
logger.debug(f"InfoQuest Web-Search - Search tool execution completed | mode=asynchronous | results_count={len(cleaned_results)}")
return result_json
elif "content" in raw_results:
@ -282,7 +284,7 @@ class InfoQuestClient:
return json.dumps(raw_results, indent=2, ensure_ascii=False)
except Exception as e:
error_message = f"InfoQuest Web-Search - Search tool execution failed | mode=synchronous | error={str(e)}"
error_message = f"InfoQuest Web-Search - Search tool execution failed | mode=asynchronous | error={str(e)}"
logger.error(error_message)
return f"Error: {error_message}"
@ -316,13 +318,13 @@ class InfoQuestClient:
return clean_results
def image_search_raw_results(
async def image_search_raw_results(
self,
query: str,
site: str = "",
output_format: str = "JSON",
) -> dict:
"""Get image search results from the InfoQuest Web-Search API synchronously."""
"""Get image search results from the InfoQuest Web-Search API asynchronously."""
headers = self._prepare_headers()
params = {"format": output_format, "query": query, "search_type": "Images"}
@ -343,7 +345,8 @@ class InfoQuestClient:
elif self.image_size:
logger.warning(f"image_size {self.image_size} is not valid, must be 'l', 'm', or 'i'")
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params, timeout=_REQUEST_TIMEOUT_SECONDS)
async with httpx.AsyncClient(follow_redirects=True) as client:
response = await client.post("https://search.infoquest.bytepluses.com", headers=headers, json=params, timeout=_REQUEST_TIMEOUT_SECONDS)
response.raise_for_status()
# Print partial response for debugging
@ -354,7 +357,7 @@ class InfoQuestClient:
return response_json
def image_search(
async def image_search(
self,
query: str,
site: str = "",
@ -369,12 +372,12 @@ class InfoQuestClient:
f"has_site_filter={bool(site)} | site={site} | "
f"image_search_time_range={self.image_search_time_range if self.image_search_time_range >= 1 and self.image_search_time_range <= 365 else 'default'} | "
f"image_size={self.image_size} |"
f"request_type=sync"
f"request_type=async"
)
try:
logger.info("InfoQuest Image Search - Executing search with parameters")
raw_results = self.image_search_raw_results(
raw_results = await self.image_search_raw_results(
query,
site,
output_format,
@ -389,7 +392,7 @@ class InfoQuestClient:
result_json = json.dumps(cleaned_results, indent=2, ensure_ascii=False)
logger.debug(f"InfoQuest Image Search - Image search tool execution completed | mode=synchronous | results_count={len(cleaned_results)}")
logger.debug(f"InfoQuest Image Search - Image search tool execution completed | mode=asynchronous | results_count={len(cleaned_results)}")
return result_json
elif "content" in raw_results:
@ -403,6 +406,6 @@ class InfoQuestClient:
return json.dumps(raw_results, indent=2, ensure_ascii=False)
except Exception as e:
error_message = f"InfoQuest Image Search - Image search tool execution failed | mode=synchronous | error={str(e)}"
error_message = f"InfoQuest Image Search - Image search tool execution failed | mode=asynchronous | error={str(e)}"
logger.error(error_message)
return f"Error: {error_message}"

16
backend/packages/harness/deerflow/community/infoquest/tools.py
View File

@ -1,3 +1,5 @@
import asyncio
from langchain.tools import tool
from deerflow.config import get_app_config
@ -44,7 +46,7 @@ def _get_infoquest_client() -> InfoQuestClient:
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
async def web_search_tool(query: str) -> str:
"""Search the web.
Args:
@ -52,11 +54,11 @@ def web_search_tool(query: str) -> str:
"""
client = _get_infoquest_client()
return client.web_search(query)
return await client.web_search(query)
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
async def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
@ -67,15 +69,15 @@ def web_fetch_tool(url: str) -> str:
url: The URL to fetch the contents of.
"""
client = _get_infoquest_client()
result = client.fetch(url)
result = await client.fetch(url)
if result.startswith("Error: "):
return result
article = readability_extractor.extract_article(result, url=url)
article = await asyncio.to_thread(readability_extractor.extract_article, result, url=url)
return article.to_markdown()[:4096]
@tool("image_search", parse_docstring=True)
def image_search_tool(query: str) -> str:
async def image_search_tool(query: str) -> str:
"""Search for images online. Use this tool BEFORE image generation to find reference images for characters, portraits, objects, scenes, or any content requiring visual accuracy.
**When to use:**
@ -90,4 +92,4 @@ def image_search_tool(query: str) -> str:
query: The query to search for images.
"""
client = _get_infoquest_client()
return client.image_search(query)
return await client.image_search(query)

298
backend/tests/test_infoquest_client.py
View File

@ -1,12 +1,23 @@
"""Tests for InfoQuest client and tools."""
import json
from unittest.mock import MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from deerflow.community.infoquest import tools
from deerflow.community.infoquest.infoquest_client import InfoQuestClient
def _mock_async_client(response: httpx.Response) -> MagicMock:
client = MagicMock()
client.post = AsyncMock(return_value=response)
client.__aenter__ = AsyncMock(return_value=client)
client.__aexit__ = AsyncMock(return_value=None)
return client
class TestInfoQuestClient:
def test_infoquest_client_initialization(self):
"""Test InfoQuestClient initialization with different parameters."""
@ -24,77 +35,80 @@ class TestInfoQuestClient:
assert client.fetch_navigation_timeout == 60
assert client.search_time_range == 24
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_fetch_success(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_fetch_success(self, mock_async_client_cls):
"""Test successful fetch operation."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = json.dumps({"reader_result": "<html><body>Test content</body></html>"})
mock_post.return_value = mock_response
mock_response = httpx.Response(200, text=json.dumps({"reader_result": "<html><body>Test content</body></html>"}), request=httpx.Request("POST", "https://reader.infoquest.bytepluses.com"))
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient()
result = client.fetch("https://example.com")
result = await client.fetch("https://example.com")
assert result == "<html><body>Test content</body></html>"
mock_post.assert_called_once()
args, kwargs = mock_post.call_args
mock_async_client_cls.return_value.post.assert_called_once()
args, kwargs = mock_async_client_cls.return_value.post.call_args
assert args[0] == "https://reader.infoquest.bytepluses.com"
assert kwargs["json"]["url"] == "https://example.com"
assert kwargs["json"]["format"] == "HTML"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_fetch_non_200_status(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_fetch_non_200_status(self, mock_async_client_cls):
"""Test fetch operation with non-200 status code."""
mock_response = MagicMock()
mock_response.status_code = 404
mock_response.text = "Not Found"
mock_post.return_value = mock_response
mock_response = httpx.Response(404, text="Not Found", request=httpx.Request("POST", "https://reader.infoquest.bytepluses.com"))
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient()
result = client.fetch("https://example.com")
result = await client.fetch("https://example.com")
assert result == "Error: fetch API returned status 404: Not Found"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_fetch_empty_response(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_fetch_empty_response(self, mock_async_client_cls):
"""Test fetch operation with empty response."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.text = ""
mock_post.return_value = mock_response
mock_response = httpx.Response(200, text="", request=httpx.Request("POST", "https://reader.infoquest.bytepluses.com"))
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient()
result = client.fetch("https://example.com")
result = await client.fetch("https://example.com")
assert result == "Error: no result found"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_web_search_raw_results_success(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_web_search_raw_results_success(self, mock_async_client_cls):
"""Test successful web_search_raw_results operation."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"search_result": {"results": [{"content": {"results": {"organic": [{"title": "Test Result", "desc": "Test description", "url": "https://example.com"}]}}}], "images_results": []}}
mock_post.return_value = mock_response
mock_response = httpx.Response(
200,
json={"search_result": {"results": [{"content": {"results": {"organic": [{"title": "Test Result", "desc": "Test description", "url": "https://example.com"}]}}}], "images_results": []}},
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
)
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient()
result = client.web_search_raw_results("test query", "")
result = await client.web_search_raw_results("test query", "")
assert "search_result" in result
mock_post.assert_called_once()
args, kwargs = mock_post.call_args
mock_async_client_cls.return_value.post.assert_called_once()
args, kwargs = mock_async_client_cls.return_value.post.call_args
assert args[0] == "https://search.infoquest.bytepluses.com"
assert kwargs["json"]["query"] == "test query"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_web_search_success(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_web_search_success(self, mock_async_client_cls):
"""Test successful web_search operation."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"search_result": {"results": [{"content": {"results": {"organic": [{"title": "Test Result", "desc": "Test description", "url": "https://example.com"}]}}}], "images_results": []}}
mock_post.return_value = mock_response
mock_response = httpx.Response(
200,
json={"search_result": {"results": [{"content": {"results": {"organic": [{"title": "Test Result", "desc": "Test description", "url": "https://example.com"}]}}}], "images_results": []}},
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
)
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient()
result = client.web_search("test query")
result = await client.web_search("test query")
# Check if result is a valid JSON string with expected content
result_data = json.loads(result)
@ -124,30 +138,32 @@ class TestInfoQuestClient:
assert cleaned[1]["title"] == "Test News"
@patch("deerflow.community.infoquest.tools._get_infoquest_client")
def test_web_search_tool(self, mock_get_client):
@pytest.mark.anyio
async def test_web_search_tool(self, mock_get_client):
"""Test web_search_tool function."""
mock_client = MagicMock()
mock_client.web_search.return_value = json.dumps([])
mock_client.web_search = AsyncMock(return_value=json.dumps([]))
mock_get_client.return_value = mock_client
result = tools.web_search_tool.run("test query")
result = await tools.web_search_tool.ainvoke("test query")
assert result == json.dumps([])
mock_get_client.assert_called_once()
mock_client.web_search.assert_called_once_with("test query")
mock_client.web_search.assert_awaited_once_with("test query")
@patch("deerflow.community.infoquest.tools._get_infoquest_client")
def test_web_fetch_tool(self, mock_get_client):
@pytest.mark.anyio
async def test_web_fetch_tool(self, mock_get_client):
"""Test web_fetch_tool function."""
mock_client = MagicMock()
mock_client.fetch.return_value = "<html><body>Test content</body></html>"
mock_client.fetch = AsyncMock(return_value="<html><body>Test content</body></html>")
mock_get_client.return_value = mock_client
result = tools.web_fetch_tool.run("https://example.com")
result = await tools.web_fetch_tool.ainvoke("https://example.com")
assert result == "# Untitled\n\nTest content"
mock_get_client.assert_called_once()
mock_client.fetch.assert_called_once_with("https://example.com")
mock_client.fetch.assert_awaited_once_with("https://example.com")
@patch("deerflow.community.infoquest.tools.get_app_config")
def test_get_infoquest_client(self, mock_get_app_config):
@ -170,13 +186,15 @@ class TestInfoQuestClient:
assert client.image_search_time_range == 7
assert client.image_size == "l"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_web_search_api_error(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_web_search_api_error(self, mock_async_client_cls):
"""Test web_search operation with API error."""
mock_post.side_effect = Exception("Connection error")
mock_async_client_cls.return_value = _mock_async_client(httpx.Response(200, text="ok", request=httpx.Request("POST", "https://search.infoquest.bytepluses.com")))
mock_async_client_cls.return_value.post.side_effect = Exception("Connection error")
client = InfoQuestClient()
result = client.web_search("test query")
result = await client.web_search("test query")
assert "Error" in result
@ -205,75 +223,85 @@ class TestInfoQuestClient:
class TestImageSearch:
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_image_search_raw_results_success(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_image_search_raw_results_success(self, mock_async_client_cls):
"""Test successful image_search_raw_results operation."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg", "title": "Test Image", "url": "https://example.com/page1"}]}}}]}}
mock_post.return_value = mock_response
mock_response = httpx.Response(
200,
json={"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg", "title": "Test Image", "url": "https://example.com/page1"}]}}}]}},
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
)
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient()
result = client.image_search_raw_results("test query")
result = await client.image_search_raw_results("test query")
assert "search_result" in result
mock_post.assert_called_once()
args, kwargs = mock_post.call_args
mock_async_client_cls.return_value.post.assert_called_once()
args, kwargs = mock_async_client_cls.return_value.post.call_args
assert args[0] == "https://search.infoquest.bytepluses.com"
assert kwargs["json"]["query"] == "test query"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_image_search_raw_results_with_parameters(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_image_search_raw_results_with_parameters(self, mock_async_client_cls):
"""Test image_search_raw_results with all parameters."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg"}]}}}]}}
mock_post.return_value = mock_response
mock_response = httpx.Response(
200,
json={"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg"}]}}}]}},
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
)
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient(image_search_time_range=30, image_size="l")
client.image_search_raw_results(query="cat", site="unsplash.com", output_format="JSON")
await client.image_search_raw_results(query="cat", site="unsplash.com", output_format="JSON")
mock_post.assert_called_once()
args, kwargs = mock_post.call_args
mock_async_client_cls.return_value.post.assert_called_once()
args, kwargs = mock_async_client_cls.return_value.post.call_args
assert kwargs["json"]["query"] == "cat"
assert kwargs["json"]["time_range"] == 30
assert kwargs["json"]["site"] == "unsplash.com"
assert kwargs["json"]["image_size"] == "l"
assert kwargs["json"]["format"] == "JSON"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_image_search_raw_results_invalid_time_range(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_image_search_raw_results_invalid_time_range(self, mock_async_client_cls):
"""Test image_search_raw_results with invalid time_range parameter."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"search_result": {"results": [{"content": {"results": {"images_results": []}}}]}}
mock_post.return_value = mock_response
mock_response = httpx.Response(
200,
json={"search_result": {"results": [{"content": {"results": {"images_results": []}}}]}},
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
)
mock_async_client_cls.return_value = _mock_async_client(mock_response)
# Create client with invalid time_range (should be ignored)
client = InfoQuestClient(image_search_time_range=400, image_size="x")
client.image_search_raw_results(
await client.image_search_raw_results(
query="test",
site="",
)
mock_post.assert_called_once()
args, kwargs = mock_post.call_args
mock_async_client_cls.return_value.post.assert_called_once()
args, kwargs = mock_async_client_cls.return_value.post.call_args
assert kwargs["json"]["query"] == "test"
assert "time_range" not in kwargs["json"]
assert "image_size" not in kwargs["json"]
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_image_search_success(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_image_search_success(self, mock_async_client_cls):
"""Test successful image_search operation."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg", "title": "Test Image", "url": "https://example.com/page1"}]}}}]}}
mock_post.return_value = mock_response
mock_response = httpx.Response(
200,
json={"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg", "title": "Test Image", "url": "https://example.com/page1"}]}}}]}},
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
)
mock_async_client_cls.return_value = _mock_async_client(mock_response)
client = InfoQuestClient()
result = client.image_search("cat")
result = await client.image_search("cat")
# Check if result is a valid JSON string with expected content
result_data = json.loads(result)
@ -284,65 +312,111 @@ class TestImageSearch:
assert result_data[0]["title"] == "Test Image"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_image_search_with_all_parameters(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_image_search_with_all_parameters(self, mock_async_client_cls):
"""Test image_search with all optional parameters."""
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg"}]}}}]}}
mock_post.return_value = mock_response
mock_response = httpx.Response(
200,
json={"search_result": {"results": [{"content": {"results": {"images_results": [{"original": "https://example.com/image1.jpg"}]}}}]}},
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
)
mock_async_client_cls.return_value = _mock_async_client(mock_response)
# Create client with image search parameters
client = InfoQuestClient(image_search_time_range=7, image_size="m")
client.image_search(query="dog", site="flickr.com", output_format="JSON")
await client.image_search(query="dog", site="flickr.com", output_format="JSON")
mock_post.assert_called_once()
args, kwargs = mock_post.call_args
mock_async_client_cls.return_value.post.assert_called_once()
args, kwargs = mock_async_client_cls.return_value.post.call_args
assert kwargs["json"]["query"] == "dog"
assert kwargs["json"]["time_range"] == 7
assert kwargs["json"]["site"] == "flickr.com"
assert kwargs["json"]["image_size"] == "m"
@patch("deerflow.community.infoquest.infoquest_client.requests.post")
def test_image_search_api_error(self, mock_post):
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_image_search_api_error(self, mock_async_client_cls):
"""Test image_search operation with API error."""
mock_post.side_effect = Exception("Connection error")
mock_async_client_cls.return_value = _mock_async_client(httpx.Response(200, text="ok", request=httpx.Request("POST", "https://search.infoquest.bytepluses.com")))
mock_async_client_cls.return_value.post.side_effect = Exception("Connection error")
client = InfoQuestClient()
result = client.image_search("test query")
result = await client.image_search("test query")
assert "Error" in result
@patch("deerflow.community.infoquest.tools._get_infoquest_client")
def test_image_search_tool(self, mock_get_client):
@pytest.mark.anyio
async def test_image_search_tool(self, mock_get_client):
"""Test image_search_tool function."""
mock_client = MagicMock()
mock_client.image_search.return_value = json.dumps([{"image_url": "https://example.com/image1.jpg"}])
mock_client.image_search = AsyncMock(return_value=json.dumps([{"image_url": "https://example.com/image1.jpg"}]))
mock_get_client.return_value = mock_client
result = tools.image_search_tool.run({"query": "test query"})
result = await tools.image_search_tool.ainvoke({"query": "test query"})
# Check if result is a valid JSON string
result_data = json.loads(result)
assert len(result_data) == 1
assert result_data[0]["image_url"] == "https://example.com/image1.jpg"
mock_get_client.assert_called_once()
mock_client.image_search.assert_called_once_with("test query")
# In /Users/bytedance/python/deer-flowv2/deer-flow/backend/tests/test_infoquest_client.py
mock_client.image_search.assert_awaited_once_with("test query")
@patch("deerflow.community.infoquest.tools._get_infoquest_client")
def test_image_search_tool_with_parameters(self, mock_get_client):
@pytest.mark.anyio
async def test_image_search_tool_with_parameters(self, mock_get_client):
"""Test image_search_tool function with all parameters (extra parameters will be ignored)."""
mock_client = MagicMock()
mock_client.image_search.return_value = json.dumps([{"image_url": "https://example.com/image1.jpg"}])
mock_client.image_search = AsyncMock(return_value=json.dumps([{"image_url": "https://example.com/image1.jpg"}]))
mock_get_client.return_value = mock_client
# Pass all parameters as a dictionary (extra parameters will be ignored)
tools.image_search_tool.run({"query": "sunset", "time_range": 30, "site": "unsplash.com", "image_size": "l"})
await tools.image_search_tool.ainvoke({"query": "sunset", "time_range": 30, "site": "unsplash.com", "image_size": "l"})
mock_get_client.assert_called_once()
# image_search_tool only passes query to client.image_search
# site parameter is empty string by default
mock_client.image_search.assert_called_once_with("sunset")
mock_client.image_search.assert_awaited_once_with("sunset")
class TestRedirectParity:
"""The sync `requests.post` client followed redirects by default; the
httpx migration must keep that behavior or a 3xx answer from the
reader/search endpoints surfaces as an error instead of the content.
Review note on #5782: httpx.AsyncClient defaults to
follow_redirects=False, unlike requests.
"""
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_fetch_client_follows_redirects(self, mock_async_client_cls):
mock_response = httpx.Response(200, text=json.dumps({"reader_result": "<html>ok</html>"}), request=httpx.Request("POST", "https://reader.infoquest.bytepluses.com"))
mock_async_client_cls.return_value = _mock_async_client(mock_response)
await InfoQuestClient().fetch("https://example.com")
mock_async_client_cls.assert_called_once_with(follow_redirects=True)
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_web_search_client_follows_redirects(self, mock_async_client_cls):
payload = {"type": "search", "results": []}
mock_response = httpx.Response(200, text=json.dumps(payload), request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"))
mock_async_client_cls.return_value = _mock_async_client(mock_response)
await InfoQuestClient().web_search_raw_results("query", site="")
mock_async_client_cls.assert_called_once_with(follow_redirects=True)
@patch("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient")
@pytest.mark.anyio
async def test_image_search_client_follows_redirects(self, mock_async_client_cls):
payload = {"type": "Images", "results": []}
mock_response = httpx.Response(200, text=json.dumps(payload), request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"))
mock_async_client_cls.return_value = _mock_async_client(mock_response)
await InfoQuestClient().image_search_raw_results("query")
mock_async_client_cls.assert_called_once_with(follow_redirects=True)

68
backend/tests/test_infoquest_http_timeout.py
View File

@ -1,35 +1,65 @@
"""InfoQuest's remote crawl timeout must not leave local HTTP waits unbounded."""
from unittest.mock import Mock
import json
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import requests
from deerflow.community.infoquest.infoquest_client import InfoQuestClient
_SEARCH_PAYLOAD = {"search_result": {"results": []}}
@pytest.mark.parametrize("operation", ["fetch", "web_search", "image_search"])
def test_infoquest_sets_transport_timeout(monkeypatch, operation):
response = Mock(status_code=200, text='{"reader_result":"<p>Content</p>"}')
response.json.return_value = {"search_result": {"results": []}}
post = Mock(return_value=response)
monkeypatch.setattr(requests, "post", post)
def _mock_async_client_cls() -> MagicMock:
client = MagicMock()
client.__aenter__.return_value = client
client.__aexit__.return_value = None
client.post = AsyncMock()
cls = MagicMock()
cls.return_value = client
return cls
def _install_response(monkeypatch, response: httpx.Response):
mock_async_client_cls = _mock_async_client_cls()
mock_async_client_cls.return_value.post.return_value = response
monkeypatch.setattr("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient", mock_async_client_cls)
monkeypatch.setenv("INFOQUEST_API_KEY", "test-placeholder")
client = InfoQuestClient(fetch_timeout=10, fetch_navigation_timeout=20)
result = getattr(client, operation)("https://example.com" if operation == "fetch" else "query")
assert result == ("<p>Content</p>" if operation == "fetch" else "[]")
post.assert_called_once()
assert post.call_args.kwargs["timeout"] == 30
return mock_async_client_cls
@pytest.mark.anyio
@pytest.mark.parametrize("operation", ["fetch", "web_search", "image_search"])
async def test_infoquest_sets_transport_timeout(monkeypatch, operation):
post_url = "https://reader.infoquest.bytepluses.com" if operation == "fetch" else "https://search.infoquest.bytepluses.com"
if operation == "fetch":
assert post.call_args.kwargs["json"]["timeout"] == 10
assert post.call_args.kwargs["json"]["navi_timeout"] == 20
response = httpx.Response(200, text=json.dumps({"reader_result": "<p>Content</p>"}), request=httpx.Request("POST", post_url))
else:
response = httpx.Response(200, json=_SEARCH_PAYLOAD, request=httpx.Request("POST", post_url))
mock_async_client_cls = _install_response(monkeypatch, response)
client = InfoQuestClient(fetch_timeout=10, fetch_navigation_timeout=20)
result = await getattr(client, operation)("https://example.com" if operation == "fetch" else "query")
assert result == ("<p>Content</p>" if operation == "fetch" else "[]")
mock_async_client_cls.return_value.post.assert_awaited_once()
assert mock_async_client_cls.return_value.post.call_args.kwargs["timeout"] == 30
if operation == "fetch":
assert mock_async_client_cls.return_value.post.call_args.kwargs["json"]["timeout"] == 10
assert mock_async_client_cls.return_value.post.call_args.kwargs["json"]["navi_timeout"] == 20
@pytest.mark.anyio
@pytest.mark.parametrize("operation", ["fetch", "web_search", "image_search"])
@pytest.mark.parametrize("error_type", [requests.ConnectTimeout, requests.ReadTimeout])
def test_infoquest_transport_timeout_returns_existing_error(monkeypatch, operation, error_type):
monkeypatch.setattr(requests, "post", Mock(side_effect=error_type("synthetic timeout")))
@pytest.mark.parametrize("error_type", [httpx.ConnectTimeout, httpx.ReadTimeout])
async def test_infoquest_transport_timeout_returns_existing_error(monkeypatch, operation, error_type):
mock_async_client_cls = _mock_async_client_cls()
mock_async_client_cls.return_value.post.side_effect = error_type("synthetic timeout")
monkeypatch.setattr("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient", mock_async_client_cls)
monkeypatch.setenv("INFOQUEST_API_KEY", "test-placeholder")
result = getattr(InfoQuestClient(), operation)("https://example.com" if operation == "fetch" else "query")
result = await getattr(InfoQuestClient(), operation)("https://example.com" if operation == "fetch" else "query")
assert result.startswith("Error:")
assert "synthetic timeout" in result

2
backend/tests/test_web_fetch_relative_links.py
View File

@ -27,7 +27,7 @@ async def test_web_fetch_resolves_relative_links_through_real_extraction(monkeyp
if provider == "jina_ai":
monkeypatch.setattr(module.JinaClient, "crawl", AsyncMock(return_value=html))
elif provider == "infoquest":
monkeypatch.setattr(module, "_get_infoquest_client", lambda: SimpleNamespace(fetch=lambda url: html))
monkeypatch.setattr(module, "_get_infoquest_client", lambda: SimpleNamespace(fetch=AsyncMock(return_value=html)))
else:
client = SimpleNamespace(fetch_html_with_status=AsyncMock(return_value=BrowserlessFetchResult(html, "200", "OK")))
monkeypatch.setattr(module, "_get_browserless_client", lambda name: client)