mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
fix(community): coerce InfoQuest crawl timeouts before comparing them (#5883)
config.example.yaml documents web_fetch timeout/fetch_time/navigation_timeout and the two search_time_range options as seconds, and a $VAR reference resolves verbatim from the environment, so any of them can reach InfoQuestClient as a string. The client compares each with 0, which raises TypeError for a string, None or a mapping and takes the whole web_fetch/web_search/image_search call down. Reuse the provider-local coercion the sibling tools already have: accept an int, integral float or integer-form string, otherwise warn and keep the documented -1 (no filter).
This commit is contained in:
parent
dfad2dcc74
commit
e747dff159
@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from langchain.tools import tool
|
||||
|
||||
@ -7,30 +8,67 @@ from deerflow.utils.readability import ReadabilityExtractor
|
||||
|
||||
from .infoquest_client import InfoQuestClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
readability_extractor = ReadabilityExtractor()
|
||||
|
||||
# config.example.yaml documents -1 as "no time filtering" for every option below.
|
||||
_NO_TIME_FILTER = -1
|
||||
|
||||
|
||||
def _coerce_seconds(value: object, option: str) -> int:
|
||||
"""Normalize one of the documented crawl timeouts before handing it to the client.
|
||||
|
||||
``$VAR`` references in config.yaml resolve verbatim from the environment, so a
|
||||
configured ``timeout: $FETCH_TIMEOUT`` arrives as ``"10"``. The client compares each
|
||||
of these options with ``0``, which raises for a string, ``None`` or a mapping and
|
||||
takes the whole ``web_fetch``/``web_search``/``image_search`` call down with it.
|
||||
Only an integer, an integral float or an integer-form string is accepted: plain
|
||||
``int()`` would silently truncate ``3.5`` and turn ``true`` into 1. Anything else
|
||||
warns and keeps the documented default instead.
|
||||
"""
|
||||
seconds: int | None
|
||||
if isinstance(value, bool):
|
||||
seconds = None
|
||||
elif isinstance(value, int):
|
||||
seconds = value
|
||||
elif isinstance(value, float):
|
||||
seconds = int(value) if value.is_integer() else None
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
seconds = int(value.strip())
|
||||
except ValueError:
|
||||
seconds = None
|
||||
else:
|
||||
seconds = None
|
||||
|
||||
if seconds is None:
|
||||
logger.warning("Invalid InfoQuest %s=%r; using default %s", option, value, _NO_TIME_FILTER)
|
||||
return _NO_TIME_FILTER
|
||||
return seconds
|
||||
|
||||
|
||||
def _get_infoquest_client() -> InfoQuestClient:
|
||||
search_config = get_app_config().get_tool_config("web_search")
|
||||
search_time_range = -1
|
||||
search_time_range = _NO_TIME_FILTER
|
||||
if search_config is not None and "search_time_range" in search_config.model_extra:
|
||||
search_time_range = search_config.model_extra.get("search_time_range")
|
||||
search_time_range = _coerce_seconds(search_config.model_extra.get("search_time_range"), "search_time_range")
|
||||
|
||||
fetch_config = get_app_config().get_tool_config("web_fetch")
|
||||
fetch_time = -1
|
||||
fetch_time = _NO_TIME_FILTER
|
||||
if fetch_config is not None and "fetch_time" in fetch_config.model_extra:
|
||||
fetch_time = fetch_config.model_extra.get("fetch_time")
|
||||
fetch_timeout = -1
|
||||
fetch_time = _coerce_seconds(fetch_config.model_extra.get("fetch_time"), "fetch_time")
|
||||
fetch_timeout = _NO_TIME_FILTER
|
||||
if fetch_config is not None and "timeout" in fetch_config.model_extra:
|
||||
fetch_timeout = fetch_config.model_extra.get("timeout")
|
||||
navigation_timeout = -1
|
||||
fetch_timeout = _coerce_seconds(fetch_config.model_extra.get("timeout"), "timeout")
|
||||
navigation_timeout = _NO_TIME_FILTER
|
||||
if fetch_config is not None and "navigation_timeout" in fetch_config.model_extra:
|
||||
navigation_timeout = fetch_config.model_extra.get("navigation_timeout")
|
||||
navigation_timeout = _coerce_seconds(fetch_config.model_extra.get("navigation_timeout"), "navigation_timeout")
|
||||
|
||||
image_search_config = get_app_config().get_tool_config("image_search")
|
||||
image_search_time_range = -1
|
||||
image_search_time_range = _NO_TIME_FILTER
|
||||
if image_search_config is not None and "image_search_time_range" in image_search_config.model_extra:
|
||||
image_search_time_range = image_search_config.model_extra.get("image_search_time_range")
|
||||
image_search_time_range = _coerce_seconds(image_search_config.model_extra.get("image_search_time_range"), "image_search_time_range")
|
||||
image_size = "i"
|
||||
if image_search_config is not None and "image_size" in image_search_config.model_extra:
|
||||
image_size = image_search_config.model_extra.get("image_size")
|
||||
|
||||
162
backend/tests/test_infoquest_config_timeout_coercion.py
Normal file
162
backend/tests/test_infoquest_config_timeout_coercion.py
Normal file
@ -0,0 +1,162 @@
|
||||
"""InfoQuest reads its crawl timeouts straight out of ``config.yaml``.
|
||||
|
||||
``config.example.yaml`` documents ``timeout``, ``fetch_time`` and ``navigation_timeout``
|
||||
as seconds (with -1 to disable), and a ``$VAR`` reference in the config is substituted
|
||||
verbatim from the environment, so the value the tool receives can be a string. The
|
||||
client then compares those values with ``0``, which raises for anything but a number and
|
||||
takes the whole ``web_fetch`` / ``web_search`` call down with it.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from deerflow.community.infoquest import tools
|
||||
|
||||
_URL = "https://example.com"
|
||||
_FETCH_PAYLOAD = {"reader_result": "<p>Content</p>"}
|
||||
_SEARCH_PAYLOAD = {"search_result": {"results": []}}
|
||||
|
||||
|
||||
def _app_config(web_search=None, web_fetch=None, image_search=None):
|
||||
"""One get_app_config() double whose tool sections are read in order.
|
||||
|
||||
The tool calls get_app_config() once per section, so the same double must be
|
||||
returned every time or its read queue restarts and the wrong section is served.
|
||||
"""
|
||||
config = MagicMock()
|
||||
config.get_tool_config.side_effect = [
|
||||
MagicMock(model_extra=web_search or {}),
|
||||
MagicMock(model_extra=web_fetch or {}),
|
||||
MagicMock(model_extra=image_search or {}),
|
||||
]
|
||||
return config
|
||||
|
||||
|
||||
def _mock_transport(monkeypatch, payload):
|
||||
client = MagicMock()
|
||||
client.__aenter__.return_value = client
|
||||
client.__aexit__.return_value = None
|
||||
client.post = AsyncMock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json=payload,
|
||||
request=httpx.Request("POST", "https://search.infoquest.bytepluses.com"),
|
||||
)
|
||||
)
|
||||
cls = MagicMock()
|
||||
cls.return_value = client
|
||||
monkeypatch.setattr("deerflow.community.infoquest.infoquest_client.httpx.AsyncClient", cls)
|
||||
monkeypatch.setenv("INFOQUEST_API_KEY", "test-placeholder")
|
||||
return client
|
||||
|
||||
|
||||
def _client_from_config(monkeypatch, **extras):
|
||||
config = _app_config(**extras)
|
||||
monkeypatch.setattr(tools, "get_app_config", lambda: config)
|
||||
return tools._get_infoquest_client()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
("10", 10),
|
||||
(" 10 ", 10),
|
||||
("-1", -1),
|
||||
(10, 10),
|
||||
(10.0, 10),
|
||||
("", -1),
|
||||
("off", -1),
|
||||
(2.5, -1),
|
||||
(True, -1),
|
||||
(None, -1),
|
||||
({"seconds": 10}, -1),
|
||||
],
|
||||
)
|
||||
def test_timeouts_reach_the_client_as_integers(monkeypatch, configured, expected):
|
||||
client = _client_from_config(monkeypatch, web_fetch={"timeout": configured})
|
||||
|
||||
assert client.fetch_timeout == expected
|
||||
assert isinstance(client.fetch_timeout, int)
|
||||
|
||||
|
||||
def test_configured_numbers_keep_their_meaning(monkeypatch):
|
||||
"""The shapes that already work must not change."""
|
||||
client = _client_from_config(
|
||||
monkeypatch,
|
||||
web_search={"search_time_range": 24},
|
||||
web_fetch={"timeout": 10, "fetch_time": 5, "navigation_timeout": 30},
|
||||
image_search={"image_search_time_range": 7, "image_size": "l"},
|
||||
)
|
||||
|
||||
assert (client.search_time_range, client.image_search_time_range) == (24, 7)
|
||||
assert (client.fetch_timeout, client.fetch_time, client.fetch_navigation_timeout) == (10, 5, 30)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "expected"),
|
||||
[
|
||||
("10", 10),
|
||||
("", -1),
|
||||
("off", -1),
|
||||
(None, -1),
|
||||
],
|
||||
)
|
||||
def test_string_timeouts_apply_to_every_configured_option(monkeypatch, configured, expected):
|
||||
client = _client_from_config(
|
||||
monkeypatch,
|
||||
web_search={"search_time_range": configured},
|
||||
web_fetch={"timeout": configured, "fetch_time": configured, "navigation_timeout": configured},
|
||||
image_search={"image_search_time_range": configured},
|
||||
)
|
||||
|
||||
assert client.search_time_range == expected
|
||||
assert client.fetch_timeout == expected
|
||||
assert client.fetch_time == expected
|
||||
assert client.fetch_navigation_timeout == expected
|
||||
assert client.image_search_time_range == expected
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_env_string_timeout_reaches_the_crawl_request_as_a_number(monkeypatch):
|
||||
"""``timeout: $FETCH_TIMEOUT`` is documented, and an outbound crawl must carry 10."""
|
||||
client = _client_from_config(
|
||||
monkeypatch,
|
||||
web_fetch={"timeout": "10", "navigation_timeout": "30", "fetch_time": "5"},
|
||||
)
|
||||
transport = _mock_transport(monkeypatch, _FETCH_PAYLOAD)
|
||||
|
||||
assert await client.fetch(_URL) == "<p>Content</p>"
|
||||
|
||||
sent = transport.post.call_args.kwargs["json"]
|
||||
assert sent["timeout"] == 10
|
||||
assert sent["navi_timeout"] == 30
|
||||
assert sent["fetch_time"] == 5
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_env_string_time_range_reaches_the_search_request_as_a_number(monkeypatch):
|
||||
client = _client_from_config(monkeypatch, web_search={"search_time_range": "24"})
|
||||
transport = _mock_transport(monkeypatch, _SEARCH_PAYLOAD)
|
||||
|
||||
assert await client.web_search("query") == "[]"
|
||||
|
||||
assert transport.post.call_args.kwargs["json"]["time_range"] == 24
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_an_unusable_timeout_disables_the_filter_instead_of_failing_the_tool(monkeypatch, caplog):
|
||||
client = _client_from_config(
|
||||
monkeypatch,
|
||||
web_fetch={"timeout": "", "navigation_timeout": "off"},
|
||||
)
|
||||
transport = _mock_transport(monkeypatch, _FETCH_PAYLOAD)
|
||||
|
||||
assert await client.fetch(_URL) == "<p>Content</p>"
|
||||
|
||||
sent = transport.post.call_args.kwargs["json"]
|
||||
assert "timeout" not in sent
|
||||
assert "navi_timeout" not in sent
|
||||
assert "fetch_time" not in sent
|
||||
assert "Invalid InfoQuest" in caplog.text
|
||||
Loading…
x
Reference in New Issue
Block a user