github/deer-flow
1
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-09-25 14:06:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(searxng): walk pageno so max_results above one page is honored (#5705)

Browse Source 
* fix(searxng): walk pageno so max_results above one page is honored

The SearXNG search API has no `limit` parameter: `/search` answers with one
page of results (the instance's `results_per_page`, 10 by default) and
ignores a limit it is handed. The client sent `limit=max_results` anyway and
hardcoded `pageno=1`, so any configured `max_results` larger than a page was
silently truncated to whatever the first page held -- an instance could not
tell a truncated response from a complete one.

Collect results by walking `pageno` until `max_results` is reached, a page
adds nothing new, or a page comes back empty. The walk is capped at
`_MAX_PAGES` so an unexpectedly large `max_results` cannot fan out into
unbounded requests. The unsupported `limit` parameter is no longer sent.

Adds four regression tests covering cross-page collection, the single-request
fast path, dedup-driven early stop, and the absence of `limit`.

* Remove redundant condition for limit check

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: RXQ6 <RXQ6@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
XuRuuuy 2026-09-22 21:18:49 +08:00 committed by GitHub
parent 43c32ade00
commit 0758794cfc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 126 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
backend/packages/harness/deerflow/community/searxng/searxng_client.py
View File

@ -11,6 +11,13 @@ logger = logging.getLogger(__name__)
class SearxngClient:
"""Client for SearXNG meta search engine API."""
# SearXNG's search API has no ``limit`` parameter: ``/search`` answers with
# one page of results (the instance's ``results_per_page``, 10 by default)
# and ignores a limit it is handed. A ``max_results`` larger than a page
# therefore has to be collected by walking ``pageno``. Cap the walk so an
# unexpectedly large ``max_results`` cannot fan out into unbounded requests.
_MAX_PAGES = 5
def __init__(self, base_url: str) -> None:
self.base_url = base_url.rstrip("/")
@ -25,27 +32,67 @@ class SearxngClient:
Args:
query: The search query.
max_results: Maximum number of results to return.
max_results: Maximum number of results to return. SearXNG returns
one page per request, so a value larger than a page is collected
across ``pageno`` pages. A falsy value means "no cap".
categories: Search categories to use.
time_range: Optional relative publication/update window.
Returns:
List of search result dictionaries.
"""
limit = max_results if isinstance(max_results, int) and max_results > 0 else None
collected: list[dict[str, Any]] = []
seen: set[str] = set()
for pageno in range(1, self._MAX_PAGES + 1):
rows = await self._search_page(query, pageno=pageno, categories=categories, time_range=time_range)
if not rows:
break
added = 0
for row in rows:
key = str(row.get("url") or row.get("title") or "")
if key and key in seen:
continue
seen.add(key)
collected.append(row)
added += 1
if limit is not None and len(collected) >= limit:
return collected
# A page that adds nothing new means the instance is repeating
# itself or the query is exhausted -- ask for no more.
if added == 0:
break
return collected
async def _search_page(
self,
query: str,
pageno: int,
categories: list[str] | None,
time_range: SearchTimeRange | None,
) -> list[dict[str, Any]]:
"""Fetch a single page of results.
``limit`` is deliberately not sent: it is not part of the SearXNG search
API, so an instance ignores it and the caller cannot tell a truncated
response from a complete one.
"""
params: dict[str, Any] = {
"q": query,
"format": "json",
"language": "auto",
"pageno": 1,
"pageno": pageno,
}
if max_results:
params["limit"] = max_results
if categories:
params["categories"] = ",".join(categories)
if time_range is not None:
params["time_range"] = time_range
logger.debug(f"Searching SearXNG at {self.base_url} with query: {query}")
logger.debug(f"Searching SearXNG at {self.base_url} with query: {query} (page {pageno})")
try:
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.get(
@ -58,8 +105,7 @@ class SearxngClient:
)
resp.raise_for_status()
data = resp.json()
results = data.get("results", [])
return results[:max_results] if max_results else results
return data.get("results") or []
except httpx.HTTPStatusError as e:
logger.error(f"SearXNG search returned error status: {e}")
raise

73
backend/tests/test_searxng_client.py
View File

@ -1,6 +1,7 @@
"""Tests for SearXNG community tools."""
import json
from contextlib import contextmanager
from unittest.mock import MagicMock, patch
import pytest
@ -16,6 +17,28 @@ class AsyncMock(MagicMock):
return super().__call__(*args, **kwargs)
@contextmanager
def _searxng_pages(pages: dict[int, list[dict]]):
"""Patch httpx so request page N answers with ``pages[N]``.
A page that was not given answers with an empty result set, which is what a
real instance does once the query is exhausted.
"""
with patch("deerflow.community.searxng.searxng_client.httpx.AsyncClient") as mock_cls:
mock_ctx = MagicMock()
mock_cls.return_value.__aenter__.return_value = mock_ctx
def _get(url, params=None, headers=None):
mock_resp = MagicMock()
mock_resp.status_code = 200
mock_resp.json.return_value = {"results": pages.get(params["pageno"], [])}
mock_resp.raise_for_status.return_value = None
return mock_resp
mock_ctx.get = AsyncMock(side_effect=_get)
yield mock_ctx
@pytest.mark.asyncio
class TestSearxngClient:
"""Tests for the SearxngClient class."""
@ -144,6 +167,56 @@ class TestSearxngClient:
params = mock_ctx.get.call_args.kwargs["params"]
assert "time_range" not in params
async def test_search_collects_results_across_pages(self):
"""A max_results larger than one page is collected by walking pageno.
SearXNG's search API has no `limit` parameter -- /search answers with one
page (the instance's results_per_page, 10 by default) and ignores a limit
it is handed -- so a configured max_results above a page used to be
silently capped at whatever the first page held.
"""
pages = {
1: [{"title": f"page1-{i}", "url": f"https://example.com/1/{i}", "content": "c"} for i in range(3)],
2: [{"title": f"page2-{i}", "url": f"https://example.com/2/{i}", "content": "c"} for i in range(3)],
}
with _searxng_pages(pages) as mock_ctx:
client = SearxngClient(base_url="http://searxng:8080")
result = await client.search("test query", max_results=5)
assert len(result) == 5
assert [call.kwargs["params"]["pageno"] for call in mock_ctx.get.call_args_list] == [1, 2]
async def test_search_makes_one_request_when_a_page_is_enough(self):
"""The default max_results still costs exactly one request."""
rows = [{"title": f"r{i}", "url": f"https://example.com/{i}", "content": "c"} for i in range(10)]
with _searxng_pages({1: rows}) as mock_ctx:
client = SearxngClient(base_url="http://searxng:8080")
result = await client.search("test query", max_results=5)
assert len(result) == 5
assert mock_ctx.get.call_count == 1
async def test_search_stops_when_a_page_adds_nothing_new(self):
"""A repeating page ends the walk instead of duplicating results."""
repeated = [{"title": f"r{i}", "url": f"https://example.com/{i}", "content": "c"} for i in range(2)]
with _searxng_pages({1: repeated, 2: repeated, 3: repeated}) as mock_ctx:
client = SearxngClient(base_url="http://searxng:8080")
result = await client.search("test query", max_results=50)
assert len(result) == 2
assert mock_ctx.get.call_count == 2
async def test_search_never_sends_the_unsupported_limit_parameter(self):
"""`limit` is not part of the SearXNG search API, so it is not sent."""
with _searxng_pages({1: [{"title": "t", "url": "https://example.com/1", "content": "c"}]}) as mock_ctx:
client = SearxngClient(base_url="http://searxng:8080")
await client.search("test query", max_results=20)
assert "limit" not in mock_ctx.get.call_args_list[0].kwargs["params"]
@pytest.mark.asyncio
class TestSearxngTools: