fix(browserless): read reject_resource_types / reject_request_pattern from config (#5719)

* fix(browserless): read reject_resource_types / reject_request_pattern from config

web_fetch_tool declared both parameters, initialised them to None, and passed
them to the client -- but never read them from tool config, so they were always
None on the wire. BrowserlessClient implements them (and is tested for it at
`test_fetch_html_with_reject_params`), so the provider's resource blocking was
unreachable without patching the source. Skipping images/media/fonts is the
main lever for render latency and bandwidth against a self-hosted Browserless,
and it could not be turned on from config.yaml.

Read both keys through a new `_as_str_list` helper that accepts a YAML list or
a comma-separated string and yields None for anything unusable, so an unparseable
value omits the parameter instead of reaching the server. Both keys are now
documented on the Browserless web_fetch entry in config.example.yaml.

Default behaviour is unchanged: an unset key is still None, so the payload is
identical to before for every existing config.

Adds four regression tests covering the list form, the comma-separated form,
the unset case, and unusable values.

* fix(browserless): validate reject list entries

---------

Co-authored-by: RXQ6 <RXQ6@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

backend/packages/harness/deerflow/community/browserless/tools.py

@@ -114,6 +114,26 @@ def _as_int(value: object, default: int) -> int:
     return default
 
 
+def _as_str_list(value: object) -> list[str] | None:
+    """Coerce a config value into a non-empty list of strings, or ``None``.
+
+    Accepts a list of strings and a comma-separated string, since YAML makes
+    both spellings natural. Non-string sequence items are ignored. Unsupported
+    top-level values, an empty list, or a blank string yield ``None`` so the
+    parameter stays out of the payload instead of reaching Browserless as an
+    unusable value.
+    """
+    if isinstance(value, str):
+        parts = value.split(",")
+    elif isinstance(value, (list, tuple)):
+        parts = [part for part in value if isinstance(part, str)]
+    else:
+        return None
+
+    items = [part.strip() for part in parts if part.strip()]
+    return items or None
+
+
 def _as_optional_quality(value: object, output_format: str) -> int | None:
     if output_format not in {"jpeg", "webp"}:
         return None
@@ -249,8 +269,8 @@ async def web_fetch_tool(url: str) -> str:
         wait_for_timeout_ms = 0
         wait_for_selector = ""
         wait_for_selector_timeout_ms = 5000
-        reject_resource_types: list[str] | None = None
-        reject_request_pattern: list[str] | None = None
+        reject_resource_types = _as_str_list(cfg.get("reject_resource_types"))
+        reject_request_pattern = _as_str_list(cfg.get("reject_request_pattern"))
 
         wait_for_event = cfg.get("wait_for_event", wait_for_event)
         wait_for_timeout_ms = _as_int(cfg.get("wait_for_timeout_ms"), wait_for_timeout_ms)

backend/tests/test_browserless_client.py

@@ -310,6 +310,15 @@ class TestBrowserlessClient:
 class TestBrowserlessTools:
     """Tests for the Browserless tool functions."""
 
+    @pytest.fixture(autouse=True)
+    def _resolve_example_com(self):
+        """Keep tool tests offline while preserving URL-safety validation."""
+        with patch(
+            "deerflow.community.browserless.tools._resolve_host_addresses",
+            return_value=[ipaddress.ip_address("93.184.216.34")],
+        ):
+            yield
+
     async def test_get_browserless_client_uses_env_token_fallback(self):
         """Browserless tools use BROWSERLESS_TOKEN when config omits token."""
         with patch("deerflow.community.browserless.tools._get_tool_config") as mock_cfg:
@@ -368,6 +377,74 @@ class TestBrowserlessTools:
         assert "Error:" not in result
         assert "warning:" not in result
 
+    async def _fetch_kwargs_with_config(self, cfg: dict) -> dict:
+        """Invoke web_fetch_tool against ``cfg`` and return the kwargs it sent."""
+        with (
+            patch("deerflow.community.browserless.tools._get_browserless_client") as mock_get_client,
+            patch("deerflow.community.browserless.tools._get_tool_config", return_value=cfg),
+        ):
+            mock_client = MagicMock()
+            mock_client.fetch_html_with_status = AsyncMock(
+                return_value=BrowserlessFetchResult(
+                    html="<html><body><p>ok</p></body></html>",
+                    target_status_code="200",
+                    target_status="OK",
+                )
+            )
+            mock_get_client.return_value = mock_client
+            await tools.web_fetch_tool.ainvoke("https://example.com")
+            return mock_client.fetch_html_with_status.call_args.kwargs
+
+    async def test_web_fetch_tool_reads_reject_lists_from_config(self):
+        """`reject_resource_types` / `reject_request_pattern` reach the client.
+
+        web_fetch_tool declared both and passed them to the client, but only ever
+        assigned ``None``: the client supports them (and is tested for it) while
+        nothing in the tool ever read them from config, so the provider's
+        resource blocking was unreachable.
+        """
+        kwargs = await self._fetch_kwargs_with_config(
+            {
+                "reject_resource_types": ["image", "media"],
+                "reject_request_pattern": [r"\.css$"],
+            }
+        )
+
+        assert kwargs["reject_resource_types"] == ["image", "media"]
+        assert kwargs["reject_request_pattern"] == [r"\.css$"]
+
+    async def test_web_fetch_tool_accepts_comma_separated_reject_lists(self):
+        """A comma-separated string is accepted too, since YAML makes it natural."""
+        kwargs = await self._fetch_kwargs_with_config({"reject_resource_types": "image, media"})
+
+        assert kwargs["reject_resource_types"] == ["image", "media"]
+
+    async def test_web_fetch_tool_omits_reject_params_when_unset(self):
+        """An unset key stays ``None`` so the parameter is left out of the payload."""
+        kwargs = await self._fetch_kwargs_with_config({})
+
+        assert kwargs["reject_resource_types"] is None
+        assert kwargs["reject_request_pattern"] is None
+
+    async def test_web_fetch_tool_ignores_unusable_reject_values(self):
+        """A scalar or empty value yields ``None`` instead of an unusable payload."""
+        kwargs = await self._fetch_kwargs_with_config({"reject_resource_types": 5, "reject_request_pattern": []})
+
+        assert kwargs["reject_resource_types"] is None
+        assert kwargs["reject_request_pattern"] is None
+
+    async def test_web_fetch_tool_ignores_non_string_reject_list_items(self):
+        """Non-string list items stay out of the Browserless payload."""
+        kwargs = await self._fetch_kwargs_with_config(
+            {
+                "reject_resource_types": ["image", 5, "", " media "],
+                "reject_request_pattern": [False],
+            }
+        )
+
+        assert kwargs["reject_resource_types"] == ["image", "media"]
+        assert kwargs["reject_request_pattern"] is None
+
     @patch("deerflow.community.browserless.tools._get_browserless_client")
     async def test_web_fetch_tool_error(self, mock_get_client):
         """web_fetch_tool returns error when fetch fails."""

config.example.yaml

@@ -1041,6 +1041,11 @@ tools:
   #   # wait_for_timeout_ms: 2000        # Extra wait after page load in milliseconds
   #   # wait_for_selector: "article"     # CSS selector to wait for before returning
   #   # wait_for_selector_timeout_ms: 5000  # How long to wait for that selector, in milliseconds
+  #   # reject_resource_types: ["image", "media", "font"]  # Skip these resource types
+  #   # reject_request_pattern: ["\\.css$", "\\.png$"]     # Skip URLs matching these regexes
+  #   #                                    # Both accept a YAML list or a
+  #   #                                    # comma-separated string. Regexes containing
+  #   #                                    # commas must use the YAML list form.
 
   # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no third-party API key)
   # Crawl4AI returns server-cleaned "fit" markdown directly (no readability step needed),