fix(searxng): walk pageno so max_results above one page is honored (#5705)

* fix(searxng): walk pageno so max_results above one page is honored

The SearXNG search API has no `limit` parameter: `/search` answers with one
page of results (the instance's `results_per_page`, 10 by default) and
ignores a limit it is handed. The client sent `limit=max_results` anyway and
hardcoded `pageno=1`, so any configured `max_results` larger than a page was
silently truncated to whatever the first page held -- an instance could not
tell a truncated response from a complete one.

Collect results by walking `pageno` until `max_results` is reached, a page
adds nothing new, or a page comes back empty. The walk is capped at
`_MAX_PAGES` so an unexpectedly large `max_results` cannot fan out into
unbounded requests. The unsupported `limit` parameter is no longer sent.

Adds four regression tests covering cross-page collection, the single-request
fast path, dedup-driven early stop, and the absence of `limit`.

* Remove redundant condition for limit check

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: RXQ6 <RXQ6@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

backend/packages/harness/deerflow/community/searxng/searxng_client.py

@@ -11,6 +11,13 @@ logger = logging.getLogger(__name__)
 class SearxngClient:
     """Client for SearXNG meta search engine API."""
 
+    # SearXNG's search API has no ``limit`` parameter: ``/search`` answers with
+    # one page of results (the instance's ``results_per_page``, 10 by default)
+    # and ignores a limit it is handed. A ``max_results`` larger than a page
+    # therefore has to be collected by walking ``pageno``. Cap the walk so an
+    # unexpectedly large ``max_results`` cannot fan out into unbounded requests.
+    _MAX_PAGES = 5
+
     def __init__(self, base_url: str) -> None:
         self.base_url = base_url.rstrip("/")
 
@@ -25,27 +32,67 @@ class SearxngClient:
 
         Args:
             query: The search query.
-            max_results: Maximum number of results to return.
+            max_results: Maximum number of results to return. SearXNG returns
+                one page per request, so a value larger than a page is collected
+                across ``pageno`` pages. A falsy value means "no cap".
             categories: Search categories to use.
             time_range: Optional relative publication/update window.
 
         Returns:
             List of search result dictionaries.
         """
+        limit = max_results if isinstance(max_results, int) and max_results > 0 else None
+
+        collected: list[dict[str, Any]] = []
+        seen: set[str] = set()
+        for pageno in range(1, self._MAX_PAGES + 1):
+            rows = await self._search_page(query, pageno=pageno, categories=categories, time_range=time_range)
+            if not rows:
+                break
+
+            added = 0
+            for row in rows:
+                key = str(row.get("url") or row.get("title") or "")
+                if key and key in seen:
+                    continue
+                seen.add(key)
+                collected.append(row)
+                added += 1
+                if limit is not None and len(collected) >= limit:
+                    return collected
+
+            # A page that adds nothing new means the instance is repeating
+            # itself or the query is exhausted -- ask for no more.
+            if added == 0:
+                break
+
+        return collected
+
+    async def _search_page(
+        self,
+        query: str,
+        pageno: int,
+        categories: list[str] | None,
+        time_range: SearchTimeRange | None,
+    ) -> list[dict[str, Any]]:
+        """Fetch a single page of results.
+
+        ``limit`` is deliberately not sent: it is not part of the SearXNG search
+        API, so an instance ignores it and the caller cannot tell a truncated
+        response from a complete one.
+        """
         params: dict[str, Any] = {
             "q": query,
             "format": "json",
             "language": "auto",
-            "pageno": 1,
+            "pageno": pageno,
         }
-        if max_results:
-            params["limit"] = max_results
         if categories:
             params["categories"] = ",".join(categories)
         if time_range is not None:
             params["time_range"] = time_range
 
-        logger.debug(f"Searching SearXNG at {self.base_url} with query: {query}")
+        logger.debug(f"Searching SearXNG at {self.base_url} with query: {query} (page {pageno})")
         try:
             async with httpx.AsyncClient(timeout=30) as client:
                 resp = await client.get(
@@ -58,8 +105,7 @@ class SearxngClient:
                 )
                 resp.raise_for_status()
                 data = resp.json()
-                results = data.get("results", [])
+                return data.get("results") or []
-                return results[:max_results] if max_results else results
         except httpx.HTTPStatusError as e:
             logger.error(f"SearXNG search returned error status: {e}")
             raise

backend/tests/test_searxng_client.py

@@ -1,6 +1,7 @@
 """Tests for SearXNG community tools."""
 
 import json
+from contextlib import contextmanager
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -16,6 +17,28 @@ class AsyncMock(MagicMock):
         return super().__call__(*args, **kwargs)
 
 
+@contextmanager
+def _searxng_pages(pages: dict[int, list[dict]]):
+    """Patch httpx so request page N answers with ``pages[N]``.
+
+    A page that was not given answers with an empty result set, which is what a
+    real instance does once the query is exhausted.
+    """
+    with patch("deerflow.community.searxng.searxng_client.httpx.AsyncClient") as mock_cls:
+        mock_ctx = MagicMock()
+        mock_cls.return_value.__aenter__.return_value = mock_ctx
+
+        def _get(url, params=None, headers=None):
+            mock_resp = MagicMock()
+            mock_resp.status_code = 200
+            mock_resp.json.return_value = {"results": pages.get(params["pageno"], [])}
+            mock_resp.raise_for_status.return_value = None
+            return mock_resp
+
+        mock_ctx.get = AsyncMock(side_effect=_get)
+        yield mock_ctx
+
+
 @pytest.mark.asyncio
 class TestSearxngClient:
     """Tests for the SearxngClient class."""
@@ -144,6 +167,56 @@ class TestSearxngClient:
             params = mock_ctx.get.call_args.kwargs["params"]
             assert "time_range" not in params
 
+    async def test_search_collects_results_across_pages(self):
+        """A max_results larger than one page is collected by walking pageno.
+
+        SearXNG's search API has no `limit` parameter -- /search answers with one
+        page (the instance's results_per_page, 10 by default) and ignores a limit
+        it is handed -- so a configured max_results above a page used to be
+        silently capped at whatever the first page held.
+        """
+        pages = {
+            1: [{"title": f"page1-{i}", "url": f"https://example.com/1/{i}", "content": "c"} for i in range(3)],
+            2: [{"title": f"page2-{i}", "url": f"https://example.com/2/{i}", "content": "c"} for i in range(3)],
+        }
+
+        with _searxng_pages(pages) as mock_ctx:
+            client = SearxngClient(base_url="http://searxng:8080")
+            result = await client.search("test query", max_results=5)
+
+        assert len(result) == 5
+        assert [call.kwargs["params"]["pageno"] for call in mock_ctx.get.call_args_list] == [1, 2]
+
+    async def test_search_makes_one_request_when_a_page_is_enough(self):
+        """The default max_results still costs exactly one request."""
+        rows = [{"title": f"r{i}", "url": f"https://example.com/{i}", "content": "c"} for i in range(10)]
+
+        with _searxng_pages({1: rows}) as mock_ctx:
+            client = SearxngClient(base_url="http://searxng:8080")
+            result = await client.search("test query", max_results=5)
+
+        assert len(result) == 5
+        assert mock_ctx.get.call_count == 1
+
+    async def test_search_stops_when_a_page_adds_nothing_new(self):
+        """A repeating page ends the walk instead of duplicating results."""
+        repeated = [{"title": f"r{i}", "url": f"https://example.com/{i}", "content": "c"} for i in range(2)]
+
+        with _searxng_pages({1: repeated, 2: repeated, 3: repeated}) as mock_ctx:
+            client = SearxngClient(base_url="http://searxng:8080")
+            result = await client.search("test query", max_results=50)
+
+        assert len(result) == 2
+        assert mock_ctx.get.call_count == 2
+
+    async def test_search_never_sends_the_unsupported_limit_parameter(self):
+        """`limit` is not part of the SearXNG search API, so it is not sent."""
+        with _searxng_pages({1: [{"title": "t", "url": "https://example.com/1", "content": "c"}]}) as mock_ctx:
+            client = SearxngClient(base_url="http://searxng:8080")
+            await client.search("test query", max_results=20)
+
+        assert "limit" not in mock_ctx.get.call_args_list[0].kwargs["params"]
+
 
 @pytest.mark.asyncio
 class TestSearxngTools: