mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-27 05:30:39 +00:00
457aa3a492
* test(blocking-io): catch socket.getaddrinfo on the event loop Blockbuster wraps socket.socket methods but not the module-level resolver, so a synchronous hostname lookup on the loop passes the strict gate. The SSRF guard in deerflow.community.url_safety resolves with socket.getaddrinfo from async web tools, and the anchors added next stay green against that unfixed code without this rule. Add socket.getaddrinfo to _PROJECT_BLOCKING_RULES, scoped to app.* and deerflow.* callers. loop.getaddrinfo and asyncio.to_thread resolve in a worker thread and are not flagged. The full make test-blocking-io suite stays green with the rule active (187 passed). * fix(community): resolve SSRF-guarded hostnames off the event loop validate_public_http_url resolves hostnames with blocking socket.getaddrinfo. The crawl4ai and Browserless web_fetch, web_capture, browser_navigate, the Gateway browser navigate route, and the Live stream's navigate input and seed all called it directly from async code, so a slow DNS answer for a model- or user-chosen URL stalled the Gateway event loop for the whole lookup. Run the guard through asyncio.to_thread at each of those call sites, the same way personal_network already offloads it. The screening result is unchanged. fastcrw's web_fetch is a sync tool and personal MCP validation already runs in a worker, so neither needed a change. * docs(changelog): reference #6140 in the SSRF guard event-loop entry * fix(browser): run the Playwright request guard off the shared browser loop BrowserSession._install_request_guard screens every redirect hop, iframe and subresource by calling the SSRF guard inside the async route callback. The guard resolves hostnames with blocking socket.getaddrinfo, and that callback runs on the process-wide Playwright loop, which also pumps Live frames and input for every browser session, so one slow lookup stalled all of them. Run the guard through asyncio.to_thread there as well, and pin it with a strict-gate anchor that drives the real route handler. Also note in the url_safety docstring that the screen is check-time only: a caller that re-resolves at connect time is not pinned to the vetted IPs. --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
117 lines
4.6 KiB
Python
117 lines
4.6 KiB
Python
"""Web tools must resolve SSRF-screened hostnames off the agent event loop.
|
|
|
|
``http://127.1/`` is not an ``ipaddress`` literal, so the URL guard hands it to
|
|
the real ``socket.getaddrinfo``, which expands it to 127.0.0.1 without any DNS
|
|
traffic. The strict gate's ``socket.getaddrinfo`` rule fails each test if the
|
|
tool resolves on the loop instead of in a worker thread.
|
|
"""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from deerflow.community.browser_automation import tools as browser_tools
|
|
from deerflow.community.browserless import tools as browserless_tools
|
|
from deerflow.community.crawl4ai import tools as crawl4ai_tools
|
|
|
|
pytestmark = pytest.mark.asyncio
|
|
|
|
_UNRESOLVED_LOOPBACK_URL = "http://127.1/"
|
|
|
|
|
|
async def test_crawl4ai_web_fetch_resolves_off_loop() -> None:
|
|
with (
|
|
patch.object(crawl4ai_tools, "_get_tool_config", return_value={}),
|
|
patch.object(crawl4ai_tools, "_build_client") as build_client,
|
|
):
|
|
result = await crawl4ai_tools.web_fetch_tool.ainvoke({"url": _UNRESOLVED_LOOPBACK_URL})
|
|
|
|
assert result == "Error: Refusing to fetch a private, loopback, or metadata address"
|
|
build_client.assert_not_called()
|
|
|
|
|
|
async def test_browserless_web_fetch_resolves_off_loop() -> None:
|
|
with (
|
|
patch.object(browserless_tools, "_get_tool_config", return_value={}),
|
|
patch.object(browserless_tools, "_get_browserless_client") as get_client,
|
|
):
|
|
result = await browserless_tools.web_fetch_tool.ainvoke({"url": _UNRESOLVED_LOOPBACK_URL})
|
|
|
|
assert result == "Error: Refusing to fetch a private, loopback, or metadata address"
|
|
get_client.assert_not_called()
|
|
|
|
|
|
async def test_browserless_web_capture_resolves_off_loop() -> None:
|
|
with (
|
|
patch.object(browserless_tools, "_get_tool_config", return_value={}),
|
|
patch.object(browserless_tools, "_get_browserless_client") as get_client,
|
|
):
|
|
result = await browserless_tools.web_capture_tool.coroutine(
|
|
runtime=SimpleNamespace(context={}, state={}),
|
|
url=_UNRESOLVED_LOOPBACK_URL,
|
|
tool_call_id="call-1",
|
|
)
|
|
|
|
assert result.update["messages"][0].content == "Error: Refusing to capture a private, loopback, or metadata address"
|
|
get_client.assert_not_called()
|
|
|
|
|
|
async def test_browser_navigate_tool_resolves_off_loop() -> None:
|
|
manager = MagicMock()
|
|
with (
|
|
patch.object(browser_tools, "_get_tool_config", return_value={}),
|
|
patch.object(browser_tools, "get_browser_session_manager", return_value=manager),
|
|
):
|
|
result = await browser_tools.browser_navigate_tool.coroutine(
|
|
runtime=SimpleNamespace(context={"thread_id": "thread-1"}, state={}),
|
|
url=_UNRESOLVED_LOOPBACK_URL,
|
|
tool_call_id="call-1",
|
|
)
|
|
|
|
assert result.update["messages"][0].content == "Error: Refusing to browse a private, loopback, or metadata address"
|
|
manager.acquire_session.assert_not_called()
|
|
|
|
|
|
async def test_browser_navigate_and_capture_resolves_off_loop(tmp_path) -> None:
|
|
manager = MagicMock()
|
|
with (
|
|
patch.object(browser_tools, "_get_tool_config", return_value={}),
|
|
patch.object(browser_tools, "get_browser_session_manager", return_value=manager),
|
|
pytest.raises(ValueError, match="Refusing to browse a private, loopback, or metadata address"),
|
|
):
|
|
await browser_tools.navigate_and_capture(thread_id="thread-1", url=_UNRESOLVED_LOOPBACK_URL, outputs_path=tmp_path)
|
|
|
|
manager.acquire_session.assert_not_called()
|
|
|
|
|
|
async def test_browser_request_guard_resolves_off_loop() -> None:
|
|
# The context route guard screens every redirect hop and subresource on the
|
|
# shared Playwright loop, which also pumps Live frames for every session.
|
|
from deerflow.community.browser_automation.session import BrowserSession
|
|
|
|
captured = {}
|
|
|
|
class _FakeContext:
|
|
async def route(self, _pattern, handler):
|
|
captured["handler"] = handler
|
|
|
|
class _FakeRoute:
|
|
request = SimpleNamespace(url=_UNRESOLVED_LOOPBACK_URL)
|
|
aborted_with = None
|
|
|
|
async def abort(self, error_code):
|
|
self.aborted_with = error_code
|
|
|
|
async def continue_(self):
|
|
raise AssertionError("a loopback request must not continue")
|
|
|
|
session = BrowserSession(MagicMock(), headless=True, timeout_ms=1000, viewport={"width": 1000, "height": 500}, url_guard=browser_tools.validate_browser_url)
|
|
session._context = _FakeContext()
|
|
route = _FakeRoute()
|
|
with patch.object(browser_tools, "_get_tool_config", return_value={}):
|
|
await session._install_request_guard()
|
|
await captured["handler"](route)
|
|
|
|
assert route.aborted_with == "blockedbyclient"
|