mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 21:26:39 +00:00
3b8141aa43
* Fail closed on delegated fetch backends to close SSRF gap * fix(community): resolve delegated-fetch credentials from one config snapshot Address review feedback on the fail-closed SSRF guard: - firecrawl/fastcrw client construction no longer re-reads get_app_config(), so the API key and endpoint come from the same revision the endpoint screen validated (prevents cross-revision credential misrouting). - fix the browserless test double to accept the new two-argument _get_browserless_client signature. - document the check-only (DNS rebinding) caveat on validate_delegated_backend_url. - add a CHANGELOG upgrade note for the default self-hosted backend break. * fix(doctor): consolidate delegated fetch backend checks and deployment guidance --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
1091 lines
51 KiB
Python
1091 lines
51 KiB
Python
"""Tests for Browserless community tools."""
|
|
|
|
import ipaddress
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from deerflow.community.browserless import tools
|
|
from deerflow.community.browserless.browserless_client import BrowserlessClient, BrowserlessFetchResult, BrowserlessScreenshotResult
|
|
|
|
|
|
class AsyncMock(MagicMock):
|
|
"""Mock that supports async call."""
|
|
|
|
async def __call__(self, *args, **kwargs):
|
|
return super().__call__(*args, **kwargs)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestBrowserlessClient:
|
|
"""Tests for the BrowserlessClient class."""
|
|
|
|
async def test_fetch_html_success(self):
|
|
"""fetch_html returns the rendered HTML as a plain string on success.
|
|
|
|
Regression guard for the fetch_html() public string contract:
|
|
BrowserlessClient is re-exported from deerflow.community.browserless.__all__,
|
|
so harness consumers that call string methods, compare the result, or pass
|
|
it directly to a parser must keep getting a str back. Status-aware callers
|
|
(e.g. web_fetch_tool) use fetch_html_with_status() instead - see
|
|
test_fetch_html_with_status_surfaces_target_status_headers below.
|
|
"""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html><body>Page content</body></html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com")
|
|
|
|
assert isinstance(result, str)
|
|
assert result == "<html><body>Page content</body></html>"
|
|
call_kwargs = mock_ctx.post.call_args.kwargs
|
|
assert call_kwargs["json"]["url"] == "https://example.com"
|
|
assert "waitUntil" not in call_kwargs["json"]
|
|
assert "gotoTimeout" not in call_kwargs["json"]
|
|
assert "bestAttempt" not in call_kwargs["json"]
|
|
|
|
async def test_fetch_html_returns_plain_string_even_when_target_page_errored(self):
|
|
"""fetch_html stays a plain string even when the target page itself errored.
|
|
|
|
Browserless returns HTTP 200 for the render request itself even when the
|
|
target page responded with a 404 (or an anti-bot block page). Before this
|
|
fix, a successful fetch_html() call started returning a BrowserlessFetchResult
|
|
object instead of a string in exactly this case, breaking existing callers
|
|
(.lower(), string concatenation, parsers expecting str) even though their
|
|
fetch technically succeeded. fetch_html() must keep unwrapping to the plain
|
|
HTML string regardless of the target status; only fetch_html_with_status()
|
|
exposes the richer result.
|
|
"""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html><body>Access Denied</body></html>"
|
|
mock_resp.headers = {
|
|
"X-Response-Code": "404",
|
|
"X-Response-Status": "Not Found",
|
|
}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com/blocked")
|
|
|
|
assert isinstance(result, str)
|
|
assert result == "<html><body>Access Denied</body></html>"
|
|
|
|
async def test_fetch_html_with_status_surfaces_target_status_headers(self):
|
|
"""fetch_html_with_status carries the target page's real status headers on the result.
|
|
|
|
Browserless returns HTTP 200 for the render request itself even when the
|
|
target page responded with a 404 (or an anti-bot block page), so status-aware
|
|
callers need the X-Response-Code/X-Response-Status headers to tell the two
|
|
apart. This richer result is opt-in via fetch_html_with_status(); plain
|
|
fetch_html() stays a str (see test_fetch_html_success above).
|
|
"""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html><body>Access Denied</body></html>"
|
|
mock_resp.headers = {
|
|
"X-Response-Code": "404",
|
|
"X-Response-Status": "Not Found",
|
|
}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html_with_status("https://example.com/blocked")
|
|
|
|
assert isinstance(result, BrowserlessFetchResult)
|
|
assert result.html == "<html><body>Access Denied</body></html>"
|
|
assert result.target_status_code == "404"
|
|
assert result.target_status == "Not Found"
|
|
|
|
async def test_fetch_html_empty_response(self):
|
|
"""fetch_html returns error for empty response."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = " "
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com")
|
|
assert result == "Error: Browserless returned empty response"
|
|
|
|
async def test_fetch_html_http_error(self):
|
|
"""fetch_html returns error for non-200 status."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 500
|
|
mock_resp.text = "Internal error"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.fetch_html("https://example.com")
|
|
assert "Error: Browserless HTTP 500" in result
|
|
|
|
async def test_fetch_html_timeout(self):
|
|
"""fetch_html returns timeout error."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
import httpx
|
|
|
|
mock_ctx.post = AsyncMock(side_effect=httpx.TimeoutException("Timed out"))
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000", timeout_s=10)
|
|
result = await client.fetch_html("https://example.com")
|
|
assert "timed out" in result.lower() or "timeout" in result.lower()
|
|
|
|
async def test_fetch_html_with_token(self):
|
|
"""fetch_html sends the token as a query parameter, never in the body."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html>OK</html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000", token="my-token")
|
|
await client.fetch_html("https://example.com")
|
|
|
|
call = mock_ctx.post.call_args
|
|
assert call.kwargs["params"] == {"token": "my-token"}
|
|
assert "token" not in call.kwargs["json"]
|
|
|
|
async def test_fetch_html_with_wait_for_selector(self):
|
|
"""fetch_html sends waitForSelector when selector is set."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html>OK</html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
await client.fetch_html("https://example.com", wait_for_selector="article")
|
|
|
|
payload = mock_ctx.post.call_args.kwargs["json"]
|
|
assert payload["waitForSelector"]["selector"] == "article"
|
|
|
|
async def test_fetch_html_with_reject_params(self):
|
|
"""fetch_html sends reject params when set."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.text = "<html>OK</html>"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
await client.fetch_html(
|
|
"https://example.com",
|
|
reject_resource_types=["image"],
|
|
reject_request_pattern=[r"\.css$"],
|
|
)
|
|
|
|
payload = mock_ctx.post.call_args.kwargs["json"]
|
|
assert payload["rejectResourceTypes"] == ["image"]
|
|
assert payload["rejectRequestPattern"] == [r"\.css$"]
|
|
|
|
async def test_capture_screenshot_success(self):
|
|
"""capture_screenshot posts to /screenshot and returns image bytes."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.content = b"\x89PNG\r\n\x1a\nimage"
|
|
mock_resp.text = "<binary>"
|
|
mock_resp.headers = {
|
|
"Content-Type": "image/png",
|
|
"X-Response-Code": "200",
|
|
"X-Response-Status": "OK",
|
|
"X-Response-URL": "https://example.com/final",
|
|
}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000", token="secret-token")
|
|
result = await client.capture_screenshot(
|
|
"https://example.com",
|
|
full_page=True,
|
|
output_format="png",
|
|
viewport={"width": 1280, "height": 720},
|
|
wait_for_selector="main",
|
|
wait_for_selector_timeout_ms=3000,
|
|
wait_for_timeout_ms=500,
|
|
best_attempt=True,
|
|
)
|
|
|
|
assert isinstance(result, BrowserlessScreenshotResult)
|
|
assert result.content == b"\x89PNG\r\n\x1a\nimage"
|
|
assert result.content_type == "image/png"
|
|
assert result.target_status_code == "200"
|
|
assert result.target_status == "OK"
|
|
assert result.final_url == "https://example.com/final"
|
|
|
|
call = mock_ctx.post.call_args
|
|
assert call.args == ("http://browserless:3000/screenshot",)
|
|
payload = call.kwargs["json"]
|
|
assert payload["url"] == "https://example.com"
|
|
assert payload["options"] == {
|
|
"fullPage": True,
|
|
"type": "png",
|
|
}
|
|
assert call.kwargs["params"] == {"token": "secret-token"}
|
|
assert payload["viewport"] == {"width": 1280, "height": 720}
|
|
assert payload["waitForSelector"] == {"selector": "main", "timeout": 3000}
|
|
assert payload["waitForTimeout"] == 500
|
|
assert payload["bestAttempt"] is True
|
|
|
|
async def test_capture_screenshot_http_error(self):
|
|
"""capture_screenshot returns a bounded error on non-200 responses."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 500
|
|
mock_resp.text = "Internal browserless error"
|
|
mock_resp.headers = {}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.capture_screenshot("https://example.com")
|
|
|
|
assert isinstance(result, str)
|
|
assert "Error: Browserless HTTP 500" in result
|
|
assert "Internal browserless error" in result
|
|
|
|
async def test_capture_screenshot_empty_response(self):
|
|
"""capture_screenshot returns a clear error for empty binary content."""
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient") as mock_cls:
|
|
mock_ctx = MagicMock()
|
|
mock_cls.return_value.__aenter__.return_value = mock_ctx
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.status_code = 200
|
|
mock_resp.content = b""
|
|
mock_resp.text = ""
|
|
mock_resp.headers = {"Content-Type": "image/png"}
|
|
mock_ctx.post = AsyncMock(return_value=mock_resp)
|
|
|
|
client = BrowserlessClient(base_url="http://browserless:3000")
|
|
result = await client.capture_screenshot("https://example.com")
|
|
|
|
assert result == "Error: Browserless returned empty screenshot response"
|
|
|
|
|
|
def _browserless_transport(server_token: str | None) -> httpx.MockTransport:
|
|
"""Stand-in for a Browserless server's request gate.
|
|
|
|
Browserless authenticates from the ``token`` query parameter or an
|
|
``Authorization`` header only, before it reads the body, and its body
|
|
schemas reject unknown keys such as ``token``.
|
|
"""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if server_token is not None:
|
|
bearer = request.headers.get("Authorization", "").removeprefix("Bearer ")
|
|
if server_token not in (request.url.params.get("token"), bearer):
|
|
return httpx.Response(401, text="Bad or missing authentication.")
|
|
if "token" in json.loads(request.content):
|
|
return httpx.Response(400, text="POST Body validation failed: must NOT have additional properties")
|
|
if request.url.path == "/screenshot":
|
|
return httpx.Response(200, content=b"\x89PNG\r\n\x1a\n", headers={"Content-Type": "image/png"})
|
|
return httpx.Response(200, text="<html><body>OK</body></html>")
|
|
|
|
return httpx.MockTransport(handler)
|
|
|
|
|
|
async def _fetch(client: BrowserlessClient):
|
|
return await client.fetch_html_with_status("https://example.com")
|
|
|
|
|
|
async def _capture(client: BrowserlessClient):
|
|
return await client.capture_screenshot("https://example.com")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("call", [_fetch, _capture], ids=["fetch", "capture"])
|
|
@pytest.mark.parametrize("server_token", ["secret-token", None], ids=["server-requires-token", "server-without-token"])
|
|
async def test_configured_token_passes_browserless_request_gate(call, server_token) -> None:
|
|
transport = _browserless_transport(server_token)
|
|
real_client = httpx.AsyncClient
|
|
|
|
def client_with_transport(**kwargs):
|
|
return real_client(transport=transport, **kwargs)
|
|
|
|
with patch("deerflow.community.browserless.browserless_client.httpx.AsyncClient", side_effect=client_with_transport):
|
|
result = await call(BrowserlessClient(base_url="http://browserless:3000", token="secret-token"))
|
|
|
|
assert not isinstance(result, str), result
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestBrowserlessTools:
|
|
"""Tests for the Browserless tool functions."""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _resolve_example_com(self):
|
|
"""Keep tool tests offline while preserving URL-safety validation."""
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
yield
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _backend_isolation_confirmed(self):
|
|
"""Assume the delegated backend's egress is isolated for these tests.
|
|
|
|
These tests exercise fetch/capture behavior, not backend isolation; the
|
|
fail-closed backend guard is covered in TestBrowserlessBackendIsolation.
|
|
"""
|
|
with patch("deerflow.community.browserless.tools._validate_backend_base_url", return_value=None):
|
|
yield
|
|
|
|
async def test_get_browserless_client_uses_env_token_fallback(self):
|
|
"""Browserless tools use BROWSERLESS_TOKEN when config omits token."""
|
|
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
|
|
mock_cfg.return_value = {"base_url": "https://production-sfo.browserless.io"}
|
|
with patch.dict("os.environ", {"BROWSERLESS_TOKEN": "env-token"}, clear=True):
|
|
client = tools._get_browserless_client("web_capture")
|
|
|
|
assert client.token == "env-token"
|
|
|
|
def _client_with_config(self, cfg: dict):
|
|
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
|
|
mock_cfg.return_value = cfg
|
|
with patch.dict("os.environ", {}, clear=True):
|
|
return tools._get_browserless_client("web_capture")
|
|
|
|
async def test_timeout_s_config_key_is_honored(self):
|
|
"""The documented `timeout_s` key keeps working (back-compat)."""
|
|
assert self._client_with_config({"timeout_s": 45}).timeout_s == 45.0
|
|
|
|
async def test_timeout_config_key_is_honored(self):
|
|
"""`timeout` is accepted too: it is what crawl4ai and jina_ai read.
|
|
|
|
Copying that spelling across providers previously produced a silently
|
|
ignored key and the 30s default.
|
|
"""
|
|
assert self._client_with_config({"timeout": 45}).timeout_s == 45.0
|
|
|
|
async def test_timeout_s_wins_when_both_keys_are_present(self):
|
|
"""`timeout_s` is this provider's documented key, so it takes precedence."""
|
|
assert self._client_with_config({"timeout_s": 45, "timeout": 10}).timeout_s == 45.0
|
|
|
|
async def test_invalid_timeout_falls_back_to_default(self):
|
|
"""A non-numeric timeout must not crash tool construction."""
|
|
assert self._client_with_config({"timeout_s": "not-a-number"}).timeout_s == 30.0
|
|
|
|
async def test_boolean_timeout_falls_back_to_default(self):
|
|
"""YAML `timeout_s: off` parses as False; 0.0 would time out every request."""
|
|
assert self._client_with_config({"timeout_s": False}).timeout_s == 30.0
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_success(self, mock_get_client):
|
|
"""web_fetch_tool successfully fetches and extracts content."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Title</h1><p>Content</p></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
|
|
|
|
assert "Error:" not in result
|
|
assert "warning:" not in result
|
|
|
|
async def _fetch_kwargs_with_config(self, cfg: dict) -> dict:
|
|
"""Invoke web_fetch_tool against ``cfg`` and return the kwargs it sent."""
|
|
with (
|
|
patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client,
|
|
patch("deerflow.community.browserless.tools._get_tool_config", return_value=cfg),
|
|
):
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><p>ok</p></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
return mock_client.fetch_html_with_status.call_args.kwargs
|
|
|
|
async def test_web_fetch_tool_reads_reject_lists_from_config(self):
|
|
"""`reject_resource_types` / `reject_request_pattern` reach the client.
|
|
|
|
web_fetch_tool declared both and passed them to the client, but only ever
|
|
assigned ``None``: the client supports them (and is tested for it) while
|
|
nothing in the tool ever read them from config, so the provider's
|
|
resource blocking was unreachable.
|
|
"""
|
|
kwargs = await self._fetch_kwargs_with_config(
|
|
{
|
|
"reject_resource_types": ["image", "media"],
|
|
"reject_request_pattern": [r"\.css$"],
|
|
}
|
|
)
|
|
|
|
assert kwargs["reject_resource_types"] == ["image", "media"]
|
|
assert kwargs["reject_request_pattern"] == [r"\.css$"]
|
|
|
|
async def test_web_fetch_tool_accepts_comma_separated_reject_lists(self):
|
|
"""A comma-separated string is accepted too, since YAML makes it natural."""
|
|
kwargs = await self._fetch_kwargs_with_config({"reject_resource_types": "image, media"})
|
|
|
|
assert kwargs["reject_resource_types"] == ["image", "media"]
|
|
|
|
async def test_web_fetch_tool_omits_reject_params_when_unset(self):
|
|
"""An unset key stays ``None`` so the parameter is left out of the payload."""
|
|
kwargs = await self._fetch_kwargs_with_config({})
|
|
|
|
assert kwargs["reject_resource_types"] is None
|
|
assert kwargs["reject_request_pattern"] is None
|
|
|
|
async def test_web_fetch_tool_ignores_unusable_reject_values(self):
|
|
"""A scalar or empty value yields ``None`` instead of an unusable payload."""
|
|
kwargs = await self._fetch_kwargs_with_config({"reject_resource_types": 5, "reject_request_pattern": []})
|
|
|
|
assert kwargs["reject_resource_types"] is None
|
|
assert kwargs["reject_request_pattern"] is None
|
|
|
|
async def test_web_fetch_tool_ignores_non_string_reject_list_items(self):
|
|
"""Non-string list items stay out of the Browserless payload."""
|
|
kwargs = await self._fetch_kwargs_with_config(
|
|
{
|
|
"reject_resource_types": ["image", 5, "", " media "],
|
|
"reject_request_pattern": [False],
|
|
}
|
|
)
|
|
|
|
assert kwargs["reject_resource_types"] == ["image", "media"]
|
|
assert kwargs["reject_request_pattern"] is None
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_error(self, mock_get_client):
|
|
"""web_fetch_tool returns error when fetch fails."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(return_value="Error: Browserless returned empty response")
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert result.startswith("Error:")
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_exception(self, mock_get_client):
|
|
"""web_fetch_tool returns error when client raises exception."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(side_effect=Exception("Unexpected error"))
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert result.startswith("Error:")
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_rejects_metadata_ip(self, mock_get_client):
|
|
"""web_fetch_tool blocks the cloud-metadata link-local endpoint."""
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("http://169.254.169.254/latest/meta-data/")
|
|
|
|
assert "private, loopback, or metadata" in result
|
|
mock_get_client.assert_not_called()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_rejects_dns_resolving_to_private(self, mock_get_client):
|
|
"""web_fetch_tool blocks hostnames that resolve to internal IPs."""
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("10.0.0.5")],
|
|
):
|
|
result = await tools.web_fetch_tool.ainvoke("https://internal.example.com/")
|
|
|
|
assert "private, loopback, or metadata" in result
|
|
mock_get_client.assert_not_called()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_allows_private_when_opted_in(self, mock_get_client):
|
|
"""web_fetch_tool allows internal targets only when explicitly configured."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><p>internal</p></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value={"allow_private_addresses": True}):
|
|
result = await tools.web_fetch_tool.ainvoke("http://10.0.0.5/dashboard")
|
|
|
|
assert "Error:" not in result
|
|
mock_client.fetch_html_with_status.assert_called_once()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_warns_on_target_error_status(self, mock_get_client):
|
|
"""web_fetch_tool surfaces a warning when the fetched page itself errored.
|
|
|
|
Mirrors test_web_capture_tool_warns_on_target_error_status below:
|
|
Browserless returns HTTP 200 for the render request even when the target
|
|
page is a 404 or an anti-bot block page, so fetch_html_with_status's
|
|
target-status headers must produce the same visible warning
|
|
web_capture_tool already gives via _target_status_warning.
|
|
"""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Not Found</h1><p>The page does not exist.</p></article></body></html>",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/missing")
|
|
|
|
assert "Error:" not in result
|
|
assert "warning: target page responded 404 Not Found" in result
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_no_warning_for_normal_target_status(self, mock_get_client):
|
|
"""web_fetch_tool does not warn when the target page responded normally.
|
|
|
|
Regression guard: a legitimate 200-target-page fetch must be unaffected
|
|
by the target-status warning added for error/blocked pages.
|
|
"""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Title</h1><p>Content</p></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
|
|
|
|
assert "Error:" not in result
|
|
assert "warning:" not in result
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_reads_wait_timeouts_like_web_capture(self, mock_get_client):
|
|
"""web_fetch_tool resolves both wait timeouts with the tolerant int coercion.
|
|
|
|
web_capture_tool reads ``wait_for_timeout_ms`` and
|
|
``wait_for_selector_timeout_ms`` through ``_as_int``, so a loosely typed
|
|
value falls back to the default instead of raising. web_fetch_tool used a
|
|
bare ``int()`` for the first key and never read the second, so a typo'd
|
|
value made every fetch return "invalid literal for int()" and a slow
|
|
selector could not be given more than the hardcoded 5000 ms.
|
|
"""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Title</h1></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
|
|
mock_cfg.return_value = {
|
|
"wait_for_timeout_ms": "2500", # quoted number: parsed, not rejected
|
|
"wait_for_selector": "article",
|
|
"wait_for_selector_timeout_ms": 9000,
|
|
}
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
|
|
|
|
assert "Error:" not in result
|
|
kwargs = mock_client.fetch_html_with_status.call_args.kwargs
|
|
assert kwargs["wait_for_timeout_ms"] == 2500
|
|
assert kwargs["wait_for_selector_timeout_ms"] == 9000
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_bad_wait_timeout_falls_back_to_default(self, mock_get_client):
|
|
"""A non-numeric wait timeout must not fail the whole fetch."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Title</h1></article></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
|
|
mock_cfg.return_value = {"wait_for_timeout_ms": "2s"}
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com/article")
|
|
|
|
assert "Error:" not in result
|
|
kwargs = mock_client.fetch_html_with_status.call_args.kwargs
|
|
assert kwargs["wait_for_timeout_ms"] == 0
|
|
assert kwargs["wait_for_selector_timeout_ms"] == 5000
|
|
|
|
async def test_web_fetch_and_web_capture_tools_agree_on_target_error_warning(self, tmp_path):
|
|
"""web_fetch_tool and web_capture_tool surface the identical warning for identical target-error headers.
|
|
|
|
Both tools sit on top of the same Browserless target-status headers
|
|
(X-Response-Code / X-Response-Status). Before this fix, only
|
|
web_capture_tool (via capture_screenshot + _target_status_warning)
|
|
surfaced them; web_fetch_tool (via fetch_html_with_status) silently
|
|
returned the blocked page's raw content with no indication anything was
|
|
wrong. This drives both real tool functions with the same headers and
|
|
asserts they now agree.
|
|
"""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
fetch_client = MagicMock()
|
|
fetch_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><article><h1>Blocked</h1><p>Access denied.</p></article></body></html>",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
)
|
|
)
|
|
|
|
capture_client = MagicMock()
|
|
capture_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
final_url="https://example.com/missing",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=fetch_client):
|
|
fetch_result = await tools.web_fetch_tool.ainvoke("https://example.com/missing")
|
|
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=capture_client):
|
|
capture_command = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/missing",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
capture_message = capture_command.update["messages"][0].content
|
|
|
|
assert "warning: target page responded 404 Not Found" in fetch_result
|
|
assert "warning: target page responded 404 Not Found" in capture_message
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_capture_tool_writes_artifact(self, mock_get_client, tmp_path):
|
|
"""web_capture_tool writes screenshots into thread outputs and presents the artifact."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
final_url="https://example.com/final",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
|
|
mock_cfg.side_effect = lambda name: {
|
|
"web_capture": {
|
|
"full_page": False,
|
|
"output_format": "png",
|
|
"viewport_width": 1024,
|
|
"viewport_height": 768,
|
|
"wait_for_selector": "main",
|
|
"wait_for_selector_timeout_ms": 4000,
|
|
"wait_for_timeout_ms": 250,
|
|
"best_attempt": True,
|
|
}
|
|
}.get(name)
|
|
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/dashboard",
|
|
tool_call_id="tool-1",
|
|
filename="Dashboard Capture.png",
|
|
)
|
|
|
|
artifact_path = result.update["artifacts"][0]
|
|
assert artifact_path == "/mnt/user-data/outputs/Dashboard_Capture.png"
|
|
written = outputs_dir / "Dashboard_Capture.png"
|
|
assert written.read_bytes() == b"\x89PNG\r\n\x1a\nimage"
|
|
assert result.update["messages"][0].content == "Captured screenshot: /mnt/user-data/outputs/Dashboard_Capture.png"
|
|
mock_cfg.assert_any_call("web_capture")
|
|
mock_client.capture_screenshot.assert_called_once_with(
|
|
url="https://example.com/dashboard",
|
|
full_page=False,
|
|
output_format="png",
|
|
quality=None,
|
|
viewport={"width": 1024, "height": 768},
|
|
wait_for_selector="main",
|
|
wait_for_selector_timeout_ms=4000,
|
|
wait_for_timeout_ms=250,
|
|
best_attempt=True,
|
|
)
|
|
|
|
async def test_web_capture_tool_rejects_non_http_url(self, tmp_path):
|
|
"""web_capture_tool only accepts explicit http(s) URLs."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="file:///etc/passwd",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "Only http:// and https:// URLs are supported" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
assert list(outputs_dir.iterdir()) == []
|
|
|
|
async def test_web_capture_tool_rejects_loopback_url(self, tmp_path):
|
|
"""web_capture_tool blocks loopback/localhost targets to prevent SSRF."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="http://localhost:8080/admin",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "private or loopback" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
assert list(outputs_dir.iterdir()) == []
|
|
|
|
async def test_web_capture_tool_rejects_metadata_ip(self, tmp_path):
|
|
"""web_capture_tool blocks the cloud-metadata link-local endpoint."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="http://169.254.169.254/latest/meta-data/",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "private, loopback, or metadata" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
|
|
async def test_web_capture_tool_rejects_dns_resolving_to_private(self, tmp_path):
|
|
"""web_capture_tool blocks a public hostname that resolves to an internal IP."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("10.0.0.5")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client:
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://internal.example.com/",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "private, loopback, or metadata" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
|
|
async def test_web_capture_tool_allows_private_when_opted_in(self, tmp_path):
|
|
"""web_capture_tool honors allow_private_addresses for internal targets."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
final_url="http://10.0.0.5/final",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value={"allow_private_addresses": True}):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=mock_client):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="http://10.0.0.5/dashboard",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert result.update["artifacts"][0].startswith("/mnt/user-data/outputs/")
|
|
assert (outputs_dir / result.update["artifacts"][0].split("/")[-1]).read_bytes() == b"\x89PNG\r\n\x1a\nimage"
|
|
mock_client.capture_screenshot.assert_called_once()
|
|
|
|
async def test_web_capture_tool_warns_on_target_error_status(self, tmp_path):
|
|
"""web_capture_tool surfaces a warning when the captured page itself errored."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nimage",
|
|
content_type="image/png",
|
|
target_status_code="404",
|
|
target_status="Not Found",
|
|
final_url="https://example.com/missing",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=mock_client):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/missing",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
message = result.update["messages"][0].content
|
|
assert "warning: target page responded 404 Not Found" in message
|
|
assert result.update["artifacts"]
|
|
|
|
async def test_web_capture_tool_dedupes_existing_filename(self, tmp_path):
|
|
"""web_capture_tool appends a suffix instead of overwriting an existing capture."""
|
|
outputs_dir = tmp_path / "outputs"
|
|
outputs_dir.mkdir()
|
|
(outputs_dir / "report.png").write_bytes(b"existing")
|
|
runtime = SimpleNamespace(state={"thread_data": {"outputs_path": str(outputs_dir)}})
|
|
|
|
mock_client = MagicMock()
|
|
mock_client.capture_screenshot = AsyncMock(
|
|
return_value=BrowserlessScreenshotResult(
|
|
content=b"\x89PNG\r\n\x1a\nnew",
|
|
content_type="image/png",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
final_url="https://example.com/",
|
|
)
|
|
)
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
with patch("deerflow.community.browserless.tools._get_browserless_client", return_value=mock_client):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com/",
|
|
tool_call_id="tool-1",
|
|
filename="report.png",
|
|
)
|
|
|
|
assert result.update["artifacts"] == ["/mnt/user-data/outputs/report-1.png"]
|
|
assert (outputs_dir / "report.png").read_bytes() == b"existing"
|
|
assert (outputs_dir / "report-1.png").read_bytes() == b"\x89PNG\r\n\x1a\nnew"
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_capture_tool_missing_outputs_path_returns_error(self, mock_get_client):
|
|
"""web_capture_tool requires ThreadDataMiddleware outputs_path."""
|
|
runtime = SimpleNamespace(state={"thread_data": {}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "Thread outputs path is not available" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestBrowserlessBackendIsolation:
|
|
"""Fail-closed delegated-backend SSRF guard (issue #5970)."""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _resolve_host_addresses(self):
|
|
"""Resolve hostnames to a public address to keep target validation offline."""
|
|
with patch(
|
|
"deerflow.community.browserless.tools._resolve_host_addresses",
|
|
return_value=[ipaddress.ip_address("93.184.216.34")],
|
|
):
|
|
yield
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_fails_closed_on_default_backend(self, mock_get_client):
|
|
"""A default (localhost) backend fails closed without isolation confirmation."""
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert "network_isolation_confirmed" in result
|
|
mock_get_client.assert_not_called()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_fails_closed_even_with_allow_private(self, mock_get_client):
|
|
"""allow_private_addresses governs the target, not the backend's egress."""
|
|
with patch(
|
|
"deerflow.community.browserless.tools._get_tool_config",
|
|
return_value={"allow_private_addresses": True},
|
|
):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert "network_isolation_confirmed" in result
|
|
mock_get_client.assert_not_called()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_allows_self_hosted_backend_when_confirmed(self, mock_get_client):
|
|
"""network_isolation_confirmed opts into a self-hosted backend."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><p>ok</p></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch(
|
|
"deerflow.community.browserless.tools._get_tool_config",
|
|
return_value={"network_isolation_confirmed": True},
|
|
):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert "Error:" not in result
|
|
mock_client.fetch_html_with_status.assert_called_once()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_fetch_tool_allows_public_backend(self, mock_get_client):
|
|
"""A public backend (Browserless Cloud) needs no isolation confirmation."""
|
|
mock_client = MagicMock()
|
|
mock_client.fetch_html_with_status = AsyncMock(
|
|
return_value=BrowserlessFetchResult(
|
|
html="<html><body><p>ok</p></body></html>",
|
|
target_status_code="200",
|
|
target_status="OK",
|
|
)
|
|
)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
with patch(
|
|
"deerflow.community.browserless.tools._get_tool_config",
|
|
return_value={"base_url": "https://production-sfo.browserless.io"},
|
|
):
|
|
result = await tools.web_fetch_tool.ainvoke("https://example.com")
|
|
|
|
assert "Error:" not in result
|
|
mock_client.fetch_html_with_status.assert_called_once()
|
|
|
|
@patch("deerflow.community.browserless.tools._get_browserless_client")
|
|
async def test_web_capture_tool_fails_closed_on_default_backend(self, mock_get_client):
|
|
"""web_capture_tool applies the same fail-closed backend guard."""
|
|
runtime = SimpleNamespace(state={"thread_data": {}})
|
|
|
|
with patch("deerflow.community.browserless.tools._get_tool_config", return_value=None):
|
|
result = await tools.web_capture_tool.coroutine(
|
|
runtime=runtime,
|
|
url="https://example.com",
|
|
tool_call_id="tool-1",
|
|
)
|
|
|
|
assert "network_isolation_confirmed" in result.update["messages"][0].content
|
|
assert "artifacts" not in result.update
|
|
mock_get_client.assert_not_called()
|