github/deer-flow
1
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-09-25 14:06:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(browserless): read web_fetch wait timeouts like web_capture (#5702)

Browse Source 
web_fetch_tool parsed `wait_for_timeout_ms` with a bare int(), so any value
that is not already an int raised ValueError and aborted the whole tool:
a quoted number in YAML, or a typo like `2s`, turned every fetch into
"Error: invalid literal for int() with base 10". web_capture_tool reads the
same documented key through the tolerant `_as_int` helper two functions
above, so the two tools disagreed about the same config key.

While aligning them, web_fetch_tool now also reads
`wait_for_selector_timeout_ms` instead of hardcoding 5000, so a
`wait_for_selector` on a slow page can be given more than five seconds --
web_capture_tool already honours the key and the client has always
accepted it.

Co-authored-by: RXQ6 <RXQ6@users.noreply.github.com>
This commit is contained in:
XuRuuuy 2026-09-22 19:47:46 +08:00 committed by GitHub
parent 53352287a7
commit 6102179c67
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 67 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/packages/harness/deerflow/community/browserless/tools.py
View File

@ -253,9 +253,9 @@ async def web_fetch_tool(url: str) -> str:
reject_request_pattern: list[str] | None = None
wait_for_event = cfg.get("wait_for_event", wait_for_event)
raw_wait = cfg.get("wait_for_timeout_ms", wait_for_timeout_ms)
wait_for_timeout_ms = int(raw_wait) if not isinstance(raw_wait, int) else raw_wait
wait_for_timeout_ms = _as_int(cfg.get("wait_for_timeout_ms"), wait_for_timeout_ms)
wait_for_selector = cfg.get("wait_for_selector", wait_for_selector)
wait_for_selector_timeout_ms = _as_int(cfg.get("wait_for_selector_timeout_ms"), wait_for_selector_timeout_ms)
client = _get_browserless_client("web_fetch")
result = await client.fetch_html_with_status(

64
backend/tests/test_browserless_client.py
View File

@ -482,6 +482,70 @@ class TestBrowserlessTools:
assert "Error:" not in result
assert "warning:" not in result
@patch("deerflow.community.browserless.tools._get_browserless_client")
async def test_web_fetch_tool_reads_wait_timeouts_like_web_capture(self, mock_get_client):
"""web_fetch_tool resolves both wait timeouts with the tolerant int coercion.
web_capture_tool reads ``wait_for_timeout_ms`` and
``wait_for_selector_timeout_ms`` through ``_as_int``, so a loosely typed
value falls back to the default instead of raising. web_fetch_tool used a
bare ``int()`` for the first key and never read the second, so a typo'd
value made every fetch return "invalid literal for int()" and a slow
selector could not be given more than the hardcoded 5000 ms.
"""
mock_client = MagicMock()
mock_client.fetch_html_with_status = AsyncMock(
return_value=BrowserlessFetchResult(
html="<html><body><article><h1>Title</h1></article></body></html>",
target_status_code="200",
target_status="OK",
)
)
mock_get_client.return_value = mock_client
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
mock_cfg.return_value = {
"wait_for_timeout_ms": "2500", # quoted number: parsed, not rejected
"wait_for_selector": "article",
"wait_for_selector_timeout_ms": 9000,
}
with patch(
"deerflow.community.browserless.tools._resolve_host_addresses",
return_value=[ipaddress.ip_address("93.184.216.34")],
):
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
assert "Error:" not in result
kwargs = mock_client.fetch_html_with_status.call_args.kwargs
assert kwargs["wait_for_timeout_ms"] == 2500
assert kwargs["wait_for_selector_timeout_ms"] == 9000
@patch("deerflow.community.browserless.tools._get_browserless_client")
async def test_web_fetch_tool_bad_wait_timeout_falls_back_to_default(self, mock_get_client):
"""A non-numeric wait timeout must not fail the whole fetch."""
mock_client = MagicMock()
mock_client.fetch_html_with_status = AsyncMock(
return_value=BrowserlessFetchResult(
html="<html><body><article><h1>Title</h1></article></body></html>",
target_status_code="200",
target_status="OK",
)
)
mock_get_client.return_value = mock_client
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
mock_cfg.return_value = {"wait_for_timeout_ms": "2s"}
with patch(
"deerflow.community.browserless.tools._resolve_host_addresses",
return_value=[ipaddress.ip_address("93.184.216.34")],
):
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
assert "Error:" not in result
kwargs = mock_client.fetch_html_with_status.call_args.kwargs
assert kwargs["wait_for_timeout_ms"] == 0
assert kwargs["wait_for_selector_timeout_ms"] == 5000
async def test_web_fetch_and_web_capture_tools_agree_on_target_error_warning(self, tmp_path):
"""web_fetch_tool and web_capture_tool surface the identical warning for identical target-error headers.

1
config.example.yaml
View File

@ -991,6 +991,7 @@ tools:
# # wait_for_event: "networkidle" # Wait for a page event before returning (e.g. "load", "networkidle")
# # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds
# # wait_for_selector: "article" # CSS selector to wait for before returning
# # wait_for_selector_timeout_ms: 5000 # How long to wait for that selector, in milliseconds
# Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no third-party API key)
# Crawl4AI returns server-cleaned "fit" markdown directly (no readability step needed),