mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-09 13:39:26 +00:00
fix(infoquest): bound HTTP connect and read waits (#5315)
Apply an explicit 30-second connect/read inactivity timeout to InfoQuest reader, web-search and image-search calls. Fixes #5314 Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com>
This commit is contained in:
parent
06c827903a
commit
0d4925305a
@ -35,6 +35,10 @@ The landing-page case studies open as allowlisted, read-only showcases without r
|
||||
|
||||
## InfoQuest
|
||||
|
||||
InfoQuest reader, web search, and image search use a 30-second HTTP connect/read
|
||||
inactivity timeout. The crawl `timeout` and `navigation_timeout` settings remain
|
||||
separate server-side options; they do not control the local HTTP timeout.
|
||||
|
||||
DeerFlow has newly integrated the intelligent search and crawling toolset independently developed by BytePlus--[InfoQuest (supports free online experience)](https://docs.byteplus.com/en/docs/InfoQuest/What_is_Info_Quest)
|
||||
|
||||
<a href="https://docs.byteplus.com/en/docs/InfoQuest/What_is_Info_Quest" target="_blank">
|
||||
|
||||
@ -256,9 +256,8 @@ Direct pytest collection or execution of `tests/test_client_live.py` remains
|
||||
skipped unless `DEER_FLOW_RUN_LIVE_TESTS=1` is set. Do not add that opt-in to
|
||||
default CI workflows.
|
||||
|
||||
Jina request-failure logging tests set a dummy API key so the separate once-per-process
|
||||
missing-key warning cannot make assertions depend on test order or shard placement.
|
||||
Missing-key behavior has its own tests in `tests/test_jina_client.py`.
|
||||
Jina logging tests isolate missing-key warnings with dummy keys (`tests/test_jina_client.py`).
|
||||
InfoQuest HTTP calls share a 30s connect/read inactivity timeout, separate from remote crawl timeouts; see `tests/test_infoquest_http_timeout.py`.
|
||||
|
||||
### Running the Full Application
|
||||
|
||||
|
||||
@ -13,6 +13,10 @@ import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Requests has no default timeout. This bounds local connect/read inactivity;
|
||||
# fetch_timeout and fetch_navigation_timeout configure the remote crawl only.
|
||||
_REQUEST_TIMEOUT_SECONDS = 30
|
||||
|
||||
|
||||
class InfoQuestClient:
|
||||
"""Client for interacting with the InfoQuest web search and fetch API."""
|
||||
@ -63,7 +67,7 @@ class InfoQuestClient:
|
||||
|
||||
logger.debug("Sending crawl request to InfoQuest API")
|
||||
try:
|
||||
response = requests.post("https://reader.infoquest.bytepluses.com", headers=headers, json=data)
|
||||
response = requests.post("https://reader.infoquest.bytepluses.com", headers=headers, json=data, timeout=_REQUEST_TIMEOUT_SECONDS)
|
||||
|
||||
# Check if status code is not 200
|
||||
if response.status_code != 200:
|
||||
@ -164,7 +168,7 @@ class InfoQuestClient:
|
||||
if site != "":
|
||||
params["site"] = site
|
||||
|
||||
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params)
|
||||
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params, timeout=_REQUEST_TIMEOUT_SECONDS)
|
||||
response.raise_for_status()
|
||||
|
||||
# Print partial response for debugging
|
||||
@ -339,7 +343,7 @@ class InfoQuestClient:
|
||||
elif self.image_size:
|
||||
logger.warning(f"image_size {self.image_size} is not valid, must be 'l', 'm', or 'i'")
|
||||
|
||||
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params)
|
||||
response = requests.post("https://search.infoquest.bytepluses.com", headers=headers, json=params, timeout=_REQUEST_TIMEOUT_SECONDS)
|
||||
response.raise_for_status()
|
||||
|
||||
# Print partial response for debugging
|
||||
|
||||
35
backend/tests/test_infoquest_http_timeout.py
Normal file
35
backend/tests/test_infoquest_http_timeout.py
Normal file
@ -0,0 +1,35 @@
|
||||
"""InfoQuest's remote crawl timeout must not leave local HTTP waits unbounded."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from deerflow.community.infoquest.infoquest_client import InfoQuestClient
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["fetch", "web_search", "image_search"])
|
||||
def test_infoquest_sets_transport_timeout(monkeypatch, operation):
|
||||
response = Mock(status_code=200, text='{"reader_result":"<p>Content</p>"}')
|
||||
response.json.return_value = {"search_result": {"results": []}}
|
||||
post = Mock(return_value=response)
|
||||
monkeypatch.setattr(requests, "post", post)
|
||||
monkeypatch.setenv("INFOQUEST_API_KEY", "test-placeholder")
|
||||
client = InfoQuestClient(fetch_timeout=10, fetch_navigation_timeout=20)
|
||||
result = getattr(client, operation)("https://example.com" if operation == "fetch" else "query")
|
||||
assert result == ("<p>Content</p>" if operation == "fetch" else "[]")
|
||||
post.assert_called_once()
|
||||
assert post.call_args.kwargs["timeout"] == 30
|
||||
if operation == "fetch":
|
||||
assert post.call_args.kwargs["json"]["timeout"] == 10
|
||||
assert post.call_args.kwargs["json"]["navi_timeout"] == 20
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["fetch", "web_search", "image_search"])
|
||||
@pytest.mark.parametrize("error_type", [requests.ConnectTimeout, requests.ReadTimeout])
|
||||
def test_infoquest_transport_timeout_returns_existing_error(monkeypatch, operation, error_type):
|
||||
monkeypatch.setattr(requests, "post", Mock(side_effect=error_type("synthetic timeout")))
|
||||
monkeypatch.setenv("INFOQUEST_API_KEY", "test-placeholder")
|
||||
result = getattr(InfoQuestClient(), operation)("https://example.com" if operation == "fetch" else "query")
|
||||
assert result.startswith("Error:")
|
||||
assert "synthetic timeout" in result
|
||||
Loading…
x
Reference in New Issue
Block a user