diff --git a/backend/packages/harness/deerflow/community/browserless/tools.py b/backend/packages/harness/deerflow/community/browserless/tools.py index ec1c629171..79e4d82e1d 100644 --- a/backend/packages/harness/deerflow/community/browserless/tools.py +++ b/backend/packages/harness/deerflow/community/browserless/tools.py @@ -114,6 +114,26 @@ def _as_int(value: object, default: int) -> int: return default +def _as_str_list(value: object) -> list[str] | None: + """Coerce a config value into a non-empty list of strings, or ``None``. + + Accepts a list of strings and a comma-separated string, since YAML makes + both spellings natural. Non-string sequence items are ignored. Unsupported + top-level values, an empty list, or a blank string yield ``None`` so the + parameter stays out of the payload instead of reaching Browserless as an + unusable value. + """ + if isinstance(value, str): + parts = value.split(",") + elif isinstance(value, (list, tuple)): + parts = [part for part in value if isinstance(part, str)] + else: + return None + + items = [part.strip() for part in parts if part.strip()] + return items or None + + def _as_optional_quality(value: object, output_format: str) -> int | None: if output_format not in {"jpeg", "webp"}: return None @@ -249,8 +269,8 @@ async def web_fetch_tool(url: str) -> str: wait_for_timeout_ms = 0 wait_for_selector = "" wait_for_selector_timeout_ms = 5000 - reject_resource_types: list[str] | None = None - reject_request_pattern: list[str] | None = None + reject_resource_types = _as_str_list(cfg.get("reject_resource_types")) + reject_request_pattern = _as_str_list(cfg.get("reject_request_pattern")) wait_for_event = cfg.get("wait_for_event", wait_for_event) wait_for_timeout_ms = _as_int(cfg.get("wait_for_timeout_ms"), wait_for_timeout_ms) diff --git a/backend/tests/test_browserless_client.py b/backend/tests/test_browserless_client.py index 3c63263e00..de179c35a2 100644 --- a/backend/tests/test_browserless_client.py +++ b/backend/tests/test_browserless_client.py @@ -310,6 +310,15 @@ class TestBrowserlessClient: class TestBrowserlessTools: """Tests for the Browserless tool functions.""" + @pytest.fixture(autouse=True) + def _resolve_example_com(self): + """Keep tool tests offline while preserving URL-safety validation.""" + with patch( + "deerflow.community.browserless.tools._resolve_host_addresses", + return_value=[ipaddress.ip_address("93.184.216.34")], + ): + yield + async def test_get_browserless_client_uses_env_token_fallback(self): """Browserless tools use BROWSERLESS_TOKEN when config omits token.""" with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg: @@ -368,6 +377,74 @@ class TestBrowserlessTools: assert "Error:" not in result assert "warning:" not in result + async def _fetch_kwargs_with_config(self, cfg: dict) -> dict: + """Invoke web_fetch_tool against ``cfg`` and return the kwargs it sent.""" + with ( + patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client, + patch("deerflow.community.browserless.tools._get_tool_config", return_value=cfg), + ): + mock_client = MagicMock() + mock_client.fetch_html_with_status = AsyncMock( + return_value=BrowserlessFetchResult( + html="

ok

", + target_status_code="200", + target_status="OK", + ) + ) + mock_get_client.return_value = mock_client + await tools.web_fetch_tool.ainvoke("https://example.com") + return mock_client.fetch_html_with_status.call_args.kwargs + + async def test_web_fetch_tool_reads_reject_lists_from_config(self): + """`reject_resource_types` / `reject_request_pattern` reach the client. + + web_fetch_tool declared both and passed them to the client, but only ever + assigned ``None``: the client supports them (and is tested for it) while + nothing in the tool ever read them from config, so the provider's + resource blocking was unreachable. + """ + kwargs = await self._fetch_kwargs_with_config( + { + "reject_resource_types": ["image", "media"], + "reject_request_pattern": [r"\.css$"], + } + ) + + assert kwargs["reject_resource_types"] == ["image", "media"] + assert kwargs["reject_request_pattern"] == [r"\.css$"] + + async def test_web_fetch_tool_accepts_comma_separated_reject_lists(self): + """A comma-separated string is accepted too, since YAML makes it natural.""" + kwargs = await self._fetch_kwargs_with_config({"reject_resource_types": "image, media"}) + + assert kwargs["reject_resource_types"] == ["image", "media"] + + async def test_web_fetch_tool_omits_reject_params_when_unset(self): + """An unset key stays ``None`` so the parameter is left out of the payload.""" + kwargs = await self._fetch_kwargs_with_config({}) + + assert kwargs["reject_resource_types"] is None + assert kwargs["reject_request_pattern"] is None + + async def test_web_fetch_tool_ignores_unusable_reject_values(self): + """A scalar or empty value yields ``None`` instead of an unusable payload.""" + kwargs = await self._fetch_kwargs_with_config({"reject_resource_types": 5, "reject_request_pattern": []}) + + assert kwargs["reject_resource_types"] is None + assert kwargs["reject_request_pattern"] is None + + async def test_web_fetch_tool_ignores_non_string_reject_list_items(self): + """Non-string list items stay out of the Browserless payload.""" + kwargs = await self._fetch_kwargs_with_config( + { + "reject_resource_types": ["image", 5, "", " media "], + "reject_request_pattern": [False], + } + ) + + assert kwargs["reject_resource_types"] == ["image", "media"] + assert kwargs["reject_request_pattern"] is None + @patch("deerflow.community.browserless.tools._get_browserless_client") async def test_web_fetch_tool_error(self, mock_get_client): """web_fetch_tool returns error when fetch fails.""" diff --git a/config.example.yaml b/config.example.yaml index 3bc5888a3f..a9238740a3 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1041,6 +1041,11 @@ tools: # # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds # # wait_for_selector: "article" # CSS selector to wait for before returning # # wait_for_selector_timeout_ms: 5000 # How long to wait for that selector, in milliseconds + # # reject_resource_types: ["image", "media", "font"] # Skip these resource types + # # reject_request_pattern: ["\\.css$", "\\.png$"] # Skip URLs matching these regexes + # # # Both accept a YAML list or a + # # # comma-separated string. Regexes containing + # # # commas must use the YAML list form. # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no third-party API key) # Crawl4AI returns server-cleaned "fit" markdown directly (no readability step needed),