diff --git a/backend/packages/harness/deerflow/community/searxng/searxng_client.py b/backend/packages/harness/deerflow/community/searxng/searxng_client.py index 24cac2d06..ca59e43ea 100644 --- a/backend/packages/harness/deerflow/community/searxng/searxng_client.py +++ b/backend/packages/harness/deerflow/community/searxng/searxng_client.py @@ -11,6 +11,13 @@ logger = logging.getLogger(__name__) class SearxngClient: """Client for SearXNG meta search engine API.""" + # SearXNG's search API has no ``limit`` parameter: ``/search`` answers with + # one page of results (the instance's ``results_per_page``, 10 by default) + # and ignores a limit it is handed. A ``max_results`` larger than a page + # therefore has to be collected by walking ``pageno``. Cap the walk so an + # unexpectedly large ``max_results`` cannot fan out into unbounded requests. + _MAX_PAGES = 5 + def __init__(self, base_url: str) -> None: self.base_url = base_url.rstrip("/") @@ -25,27 +32,67 @@ class SearxngClient: Args: query: The search query. - max_results: Maximum number of results to return. + max_results: Maximum number of results to return. SearXNG returns + one page per request, so a value larger than a page is collected + across ``pageno`` pages. A falsy value means "no cap". categories: Search categories to use. time_range: Optional relative publication/update window. Returns: List of search result dictionaries. """ + limit = max_results if isinstance(max_results, int) and max_results > 0 else None + + collected: list[dict[str, Any]] = [] + seen: set[str] = set() + for pageno in range(1, self._MAX_PAGES + 1): + rows = await self._search_page(query, pageno=pageno, categories=categories, time_range=time_range) + if not rows: + break + + added = 0 + for row in rows: + key = str(row.get("url") or row.get("title") or "") + if key and key in seen: + continue + seen.add(key) + collected.append(row) + added += 1 + if limit is not None and len(collected) >= limit: + return collected + + # A page that adds nothing new means the instance is repeating + # itself or the query is exhausted -- ask for no more. + if added == 0: + break + + return collected + + async def _search_page( + self, + query: str, + pageno: int, + categories: list[str] | None, + time_range: SearchTimeRange | None, + ) -> list[dict[str, Any]]: + """Fetch a single page of results. + + ``limit`` is deliberately not sent: it is not part of the SearXNG search + API, so an instance ignores it and the caller cannot tell a truncated + response from a complete one. + """ params: dict[str, Any] = { "q": query, "format": "json", "language": "auto", - "pageno": 1, + "pageno": pageno, } - if max_results: - params["limit"] = max_results if categories: params["categories"] = ",".join(categories) if time_range is not None: params["time_range"] = time_range - logger.debug(f"Searching SearXNG at {self.base_url} with query: {query}") + logger.debug(f"Searching SearXNG at {self.base_url} with query: {query} (page {pageno})") try: async with httpx.AsyncClient(timeout=30) as client: resp = await client.get( @@ -58,8 +105,7 @@ class SearxngClient: ) resp.raise_for_status() data = resp.json() - results = data.get("results", []) - return results[:max_results] if max_results else results + return data.get("results") or [] except httpx.HTTPStatusError as e: logger.error(f"SearXNG search returned error status: {e}") raise diff --git a/backend/tests/test_searxng_client.py b/backend/tests/test_searxng_client.py index f61fd63b8..366b103f9 100644 --- a/backend/tests/test_searxng_client.py +++ b/backend/tests/test_searxng_client.py @@ -1,6 +1,7 @@ """Tests for SearXNG community tools.""" import json +from contextlib import contextmanager from unittest.mock import MagicMock, patch import pytest @@ -16,6 +17,28 @@ class AsyncMock(MagicMock): return super().__call__(*args, **kwargs) +@contextmanager +def _searxng_pages(pages: dict[int, list[dict]]): + """Patch httpx so request page N answers with ``pages[N]``. + + A page that was not given answers with an empty result set, which is what a + real instance does once the query is exhausted. + """ + with patch("deerflow.community.searxng.searxng_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + def _get(url, params=None, headers=None): + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"results": pages.get(params["pageno"], [])} + mock_resp.raise_for_status.return_value = None + return mock_resp + + mock_ctx.get = AsyncMock(side_effect=_get) + yield mock_ctx + + @pytest.mark.asyncio class TestSearxngClient: """Tests for the SearxngClient class.""" @@ -144,6 +167,56 @@ class TestSearxngClient: params = mock_ctx.get.call_args.kwargs["params"] assert "time_range" not in params + async def test_search_collects_results_across_pages(self): + """A max_results larger than one page is collected by walking pageno. + + SearXNG's search API has no `limit` parameter -- /search answers with one + page (the instance's results_per_page, 10 by default) and ignores a limit + it is handed -- so a configured max_results above a page used to be + silently capped at whatever the first page held. + """ + pages = { + 1: [{"title": f"page1-{i}", "url": f"https://example.com/1/{i}", "content": "c"} for i in range(3)], + 2: [{"title": f"page2-{i}", "url": f"https://example.com/2/{i}", "content": "c"} for i in range(3)], + } + + with _searxng_pages(pages) as mock_ctx: + client = SearxngClient(base_url="http://searxng:8080") + result = await client.search("test query", max_results=5) + + assert len(result) == 5 + assert [call.kwargs["params"]["pageno"] for call in mock_ctx.get.call_args_list] == [1, 2] + + async def test_search_makes_one_request_when_a_page_is_enough(self): + """The default max_results still costs exactly one request.""" + rows = [{"title": f"r{i}", "url": f"https://example.com/{i}", "content": "c"} for i in range(10)] + + with _searxng_pages({1: rows}) as mock_ctx: + client = SearxngClient(base_url="http://searxng:8080") + result = await client.search("test query", max_results=5) + + assert len(result) == 5 + assert mock_ctx.get.call_count == 1 + + async def test_search_stops_when_a_page_adds_nothing_new(self): + """A repeating page ends the walk instead of duplicating results.""" + repeated = [{"title": f"r{i}", "url": f"https://example.com/{i}", "content": "c"} for i in range(2)] + + with _searxng_pages({1: repeated, 2: repeated, 3: repeated}) as mock_ctx: + client = SearxngClient(base_url="http://searxng:8080") + result = await client.search("test query", max_results=50) + + assert len(result) == 2 + assert mock_ctx.get.call_count == 2 + + async def test_search_never_sends_the_unsupported_limit_parameter(self): + """`limit` is not part of the SearXNG search API, so it is not sent.""" + with _searxng_pages({1: [{"title": "t", "url": "https://example.com/1", "content": "c"}]}) as mock_ctx: + client = SearxngClient(base_url="http://searxng:8080") + await client.search("test query", max_results=20) + + assert "limit" not in mock_ctx.get.call_args_list[0].kwargs["params"] + @pytest.mark.asyncio class TestSearxngTools: