feat(search): add native recency filters (#5099)

* feat(search): add native recency filters

* fix(search): enforce recency across backends

* docs(search): record recency provider contract
This commit is contained in:
Ryker_Feng 2026-08-30 21:25:05 +08:00 committed by GitHub
parent 8eda71fd97
commit 8c8c5ac246
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 328 additions and 15 deletions

View File

@ -940,7 +940,7 @@ cd backend
uv run python -m deerflow.skills.review.cli ../skills/public/data-analysis --format text --fail-on error --fail-on-incomplete
```
Tools follow the same philosophy. DeerFlow comes with a core toolset — web search, web fetch, rendered web capture, file operations, bash execution — and supports custom tools via MCP servers and Python functions. Swap anything. Add anything.
Tools follow the same philosophy. DeerFlow comes with a core toolset — web search, web fetch, rendered web capture, file operations, bash execution — and supports custom tools via MCP servers and Python functions. The bundled DDG, Brave, Tavily, and SearXNG search providers accept an optional `time_range` of `day`, `week`, `month`, or `year`; omitting it preserves existing search behavior. For DDG recency searches, DeerFlow excludes DDGS backends that ignore time limits. Swap anything. Add anything.
Advanced deployments can enable pluggable authorization with `authorization.enabled` in `config.yaml`. A configured `AuthorizationProvider` filters denied tools before they reach the model or deferred-tool catalog, then the same provider is checked again before every business-tool execution through the existing guardrail middleware. Gateway `threads:*` and `runs:*` route permissions are derived from the same provider, while existing owner checks and admin-only management gates remain in force. Every HTTP route that starts or enables a future Agent run requires `runs:create`: this includes the stateless `POST /api/runs/stream` and `POST /api/runs/wait` endpoints plus scheduled-task create, update, resume, and manual-trigger mutations. Scheduled-task mutations retain their existing `threads:write` requirement, and the stateless routes separately enforce ownership when the optional thread ID is supplied in the request body. A generated `tool_search` may bypass the second tool check only when it fronts the current build's already-filtered deferred catalog. Model access follows the same provider: the Gateway `models` list is filtered per principal, `model:use` is enforced on model detail requests and again when the runtime resolves the agent's model, and a denied default model falls back to the first remaining candidate that also passes `model:use`. The built-in RBAC provider supports per-role `tools`, `routes`, `models`, `skills`, and `sandbox` allow/deny policies and validates that `default_role` names a configured role; authorization is disabled by default. See `config.example.yaml` and the [authorization RFC](docs/plans/2026-07-10-pluggable-authorization-rfc.md).

View File

@ -285,6 +285,15 @@ When using `make dev` from root, the frontend automatically connects through ngi
## Key Features
### Web Search Recency
DDG, Brave, Tavily, and SearXNG `web_search` share optional
`time_range=day|week|month|year`; omission preserves request shape. DDG maps to
`d|w|m|y`, Brave to `pd|pw|pm|py`, and Tavily/SearXNG pass values unchanged.
For recency, DDGS 9.14.1 uses only enabled Brave, DuckDuckGo, and Yahoo engines
that honor `timelimit`: `auto`/`all` resolves to this set, incompatible configured
engines are removed, and an empty set falls back to it. Re-check on DDGS upgrades.
### File Upload
Multi-file upload with automatic document conversion:

View File

@ -19,6 +19,7 @@ from urllib.parse import urlparse
import httpx
from langchain.tools import tool
from deerflow.community.search_time_range import BRAVE_FRESHNESS_BY_TIME_RANGE, SearchTimeRange
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
@ -230,12 +231,13 @@ def _brave_get(
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str, max_results: int = 5) -> str:
def web_search_tool(query: str, max_results: int = 5, time_range: SearchTimeRange | None = None) -> str:
"""Search the web for information using Brave Search.
Args:
query: Search keywords describing what you want to find. Be specific for better results.
max_results: Maximum number of search results to return. Default is 5.
time_range: Optional relative publication/update window. Use only when the request requires recent results.
"""
config = get_app_config().get_tool_config("web_search")
if config is not None and "max_results" in (config.model_extra or {}):
@ -249,6 +251,8 @@ def web_search_tool(query: str, max_results: int = 5) -> str:
return _missing_key_error(query, "web_search")
params = {"q": query, "count": count, "text_decorations": False}
if time_range is not None:
params["freshness"] = BRAVE_FRESHNESS_BY_TIME_RANGE[time_range]
data, error_json = _brave_get(_BRAVE_WEB_ENDPOINT, api_key, query, params, service_name="Brave Search")
if error_json is not None:

View File

@ -7,6 +7,7 @@ import logging
from langchain.tools import tool
from deerflow.community.search_time_range import DDGS_TIMELIMIT_BY_TIME_RANGE, SearchTimeRange
from deerflow.config import get_app_config
logger = logging.getLogger(__name__)
@ -17,6 +18,10 @@ DEFAULT_SAFESEARCH = "moderate"
DEFAULT_WIKIPEDIA_REGION = "us-en"
WIKIPEDIA_BACKENDS = {"auto", "all", "wikipedia"}
# ddgs 9.14.1: enabled text engines whose implementations honor ``timelimit``.
# Google and Bing also implement it but are disabled upstream in this release.
TIME_RANGE_CAPABLE_BACKENDS = ("brave", "duckduckgo", "yahoo")
DEFAULT_TIME_RANGE_BACKEND = ",".join(TIME_RANGE_CAPABLE_BACKENDS)
WIKIPEDIA_LANGUAGE_ALIASES = {
"jp": "ja",
"kr": "ko",
@ -37,6 +42,20 @@ def _normalize_setting(value: str | None, default: str) -> str:
return str(value).strip() if value else default
def _resolve_time_range_backend(backend: str | list[str] | tuple[str, ...] | None) -> str:
"""Exclude DDGS text backends that ignore the native time limit."""
normalized_backend = _normalize_backend(backend)
configured_backends = [part.strip().lower() for part in normalized_backend.split(",") if part.strip()]
if any(part in {"auto", "all"} for part in configured_backends):
return DEFAULT_TIME_RANGE_BACKEND
supported_backends = [part for part in configured_backends if part in TIME_RANGE_CAPABLE_BACKENDS]
excluded_backends = [part for part in configured_backends if part not in TIME_RANGE_CAPABLE_BACKENDS]
if excluded_backends:
logger.warning("Ignoring DDGS backends without time-range support: %s", ", ".join(excluded_backends))
return ",".join(supported_backends) or DEFAULT_TIME_RANGE_BACKEND
def _backend_includes_wikipedia(backend: str | list[str] | tuple[str, ...] | None) -> bool:
backend = _normalize_backend(backend)
return any(part.strip().lower() in WIKIPEDIA_BACKENDS for part in backend.split(","))
@ -90,6 +109,7 @@ def _search_text(
region: str | None = DEFAULT_REGION,
safesearch: str | None = DEFAULT_SAFESEARCH,
backend: str | list[str] | tuple[str, ...] | None = DEFAULT_BACKEND,
time_range: SearchTimeRange | None = None,
) -> list[dict]:
"""
Execute text search using DuckDuckGo.
@ -100,6 +120,7 @@ def _search_text(
region: Search region
safesearch: Safe search level
backend: DDGS backend(s), e.g. "auto", "duckduckgo", or "duckduckgo,brave"
time_range: Optional relative publication/update window
Returns:
List of search results
@ -113,16 +134,18 @@ def _search_text(
ddgs = DDGS(timeout=30)
try:
backend = _normalize_backend(backend)
backend = _resolve_time_range_backend(backend) if time_range is not None else _normalize_backend(backend)
safesearch = _normalize_setting(safesearch, DEFAULT_SAFESEARCH)
effective_region = _resolve_ddgs_region(query, region, backend)
results = ddgs.text(
query,
region=effective_region,
safesearch=safesearch,
max_results=max_results,
backend=backend,
)
search_kwargs: dict[str, object] = {
"region": effective_region,
"safesearch": safesearch,
"max_results": max_results,
"backend": backend,
}
if time_range is not None:
search_kwargs["timelimit"] = DDGS_TIMELIMIT_BY_TIME_RANGE[time_range]
results = ddgs.text(query, **search_kwargs)
return list(results) if results else []
except Exception as e:
@ -134,12 +157,14 @@ def _search_text(
def web_search_tool(
query: str,
max_results: int = 5,
time_range: SearchTimeRange | None = None,
) -> str:
"""Search the web for information. Use this tool to find current information, news, articles, and facts from the internet.
Args:
query: Search keywords describing what you want to find. Be specific for better results.
max_results: Maximum number of results to return. Default is 5.
time_range: Optional relative publication/update window. Use only when the request requires recent results.
"""
config = get_app_config().get_tool_config("web_search")
region = DEFAULT_REGION
@ -159,6 +184,7 @@ def web_search_tool(
region=region,
safesearch=safesearch,
backend=backend,
time_range=time_range,
)
if not results:

View File

@ -0,0 +1,19 @@
"""Shared relative time-range contract for supported bundled web-search providers."""
from typing import Literal
type SearchTimeRange = Literal["day", "week", "month", "year"]
DDGS_TIMELIMIT_BY_TIME_RANGE: dict[SearchTimeRange, str] = {
"day": "d",
"week": "w",
"month": "m",
"year": "y",
}
BRAVE_FRESHNESS_BY_TIME_RANGE: dict[SearchTimeRange, str] = {
"day": "pd",
"week": "pw",
"month": "pm",
"year": "py",
}

View File

@ -3,6 +3,8 @@ from typing import Any
import httpx
from deerflow.community.search_time_range import SearchTimeRange
logger = logging.getLogger(__name__)
@ -17,6 +19,7 @@ class SearxngClient:
query: str,
max_results: int = 5,
categories: list[str] | None = None,
time_range: SearchTimeRange | None = None,
) -> list[dict[str, Any]]:
"""Search the web using SearXNG.
@ -24,6 +27,7 @@ class SearxngClient:
query: The search query.
max_results: Maximum number of results to return.
categories: Search categories to use.
time_range: Optional relative publication/update window.
Returns:
List of search result dictionaries.
@ -38,6 +42,8 @@ class SearxngClient:
params["limit"] = max_results
if categories:
params["categories"] = ",".join(categories)
if time_range is not None:
params["time_range"] = time_range
logger.debug(f"Searching SearXNG at {self.base_url} with query: {query}")
try:

View File

@ -3,6 +3,7 @@ import logging
from langchain.tools import tool
from deerflow.community.search_time_range import SearchTimeRange
from deerflow.config import get_app_config
from .searxng_client import SearxngClient
@ -28,11 +29,12 @@ def _get_searxng_client() -> SearxngClient:
@tool("web_search", parse_docstring=True)
async def web_search_tool(query: str) -> str:
async def web_search_tool(query: str, time_range: SearchTimeRange | None = None) -> str:
"""Search the web using SearXNG.
Args:
query: The query to search for.
time_range: Optional relative publication/update window. Use only when the request requires recent results.
"""
try:
cfg = _get_tool_config("web_search")
@ -42,7 +44,10 @@ async def web_search_tool(query: str) -> str:
max_results = int(raw) if not isinstance(raw, int) else raw
client = _get_searxng_client()
results = await client.search(query, max_results=max_results)
search_kwargs: dict[str, object] = {"max_results": max_results}
if time_range is not None:
search_kwargs["time_range"] = time_range
results = await client.search(query, **search_kwargs)
normalized = [
{

View File

@ -3,6 +3,7 @@ import json
from langchain.tools import tool
from tavily import TavilyClient
from deerflow.community.search_time_range import SearchTimeRange
from deerflow.config import get_app_config
@ -15,11 +16,12 @@ def _get_tavily_client() -> TavilyClient:
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
def web_search_tool(query: str, time_range: SearchTimeRange | None = None) -> str:
"""Search the web.
Args:
query: The query to search for.
time_range: Optional relative publication/update window. Use only when the request requires recent results.
"""
config = get_app_config().get_tool_config("web_search")
max_results = 5
@ -27,7 +29,10 @@ def web_search_tool(query: str) -> str:
max_results = config.model_extra.get("max_results")
client = _get_tavily_client()
res = client.search(query, max_results=max_results)
search_kwargs: dict[str, object] = {"max_results": max_results}
if time_range is not None:
search_kwargs["time_range"] = time_range
res = client.search(query, **search_kwargs)
normalized_results = [
{
"title": result["title"],

View File

@ -354,6 +354,47 @@ class TestWebSearchTool:
assert params["q"] == "hello world"
assert params["count"] == 5
@pytest.mark.parametrize(
("time_range", "expected_freshness"),
[
("day", "pd"),
("week", "pw"),
("month", "pm"),
("year", "py"),
],
)
def test_maps_time_range_to_freshness(self, mock_config_with_key, time_range: str, expected_freshness: str):
results = [{"title": "T", "url": "https://x.com", "description": "D"}]
mock_resp = _make_brave_response(results)
with patch("deerflow.community.brave.tools.httpx.Client") as mock_client_cls:
mock_get = mock_client_cls.return_value.__enter__.return_value.get
mock_get.return_value = mock_resp
from deerflow.community.brave.tools import web_search_tool
web_search_tool.invoke({"query": "latest releases", "time_range": time_range})
params = mock_get.call_args.kwargs["params"]
assert params["freshness"] == expected_freshness
def test_omits_freshness_without_time_range(self, mock_config_with_key):
results = [{"title": "T", "url": "https://x.com", "description": "D"}]
mock_resp = _make_brave_response(results)
with patch("deerflow.community.brave.tools.httpx.Client") as mock_client_cls:
mock_get = mock_client_cls.return_value.__enter__.return_value.get
mock_get.return_value = mock_resp
from deerflow.community.brave.tools import web_search_tool
web_search_tool.invoke({"query": "stable documentation"})
params = mock_get.call_args.kwargs["params"]
assert "freshness" not in params
def test_long_query_is_truncated_to_brave_limit(self, mock_config_with_key):
results = [{"title": "T", "url": "https://x.com", "description": "D"}]
mock_resp = _make_brave_response(results)

View File

@ -5,6 +5,8 @@ import sys
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from deerflow.community.ddg_search import tools
@ -46,6 +48,86 @@ def test_search_text_passes_wikipedia_safe_region_to_ddgs(monkeypatch) -> None:
assert calls["timeout"] == 30
assert calls["region"] == "cn-zh"
assert calls["backend"] == "auto"
assert "timelimit" not in calls
@pytest.mark.parametrize(
("time_range", "expected_timelimit"),
[
("day", "d"),
("week", "w"),
("month", "m"),
("year", "y"),
],
)
def test_search_text_maps_time_range_to_ddgs_timelimit(monkeypatch, time_range: str, expected_timelimit: str) -> None:
calls = {}
class FakeDDGS:
def __init__(self, timeout: int) -> None:
calls["timeout"] = timeout
def text(self, query: str, **kwargs):
calls["query"] = query
calls.update(kwargs)
return [{"title": "Result", "href": "https://example.com", "body": "Snippet"}]
monkeypatch.setitem(sys.modules, "ddgs", SimpleNamespace(DDGS=FakeDDGS))
results = tools._search_text("latest release", backend="duckduckgo", time_range=time_range)
assert results == [{"title": "Result", "href": "https://example.com", "body": "Snippet"}]
assert calls["timelimit"] == expected_timelimit
def test_search_text_time_range_replaces_auto_with_filter_capable_backends(monkeypatch) -> None:
calls = {}
class FakeDDGS:
def __init__(self, timeout: int) -> None:
calls["timeout"] = timeout
def text(self, query: str, **kwargs):
calls.update(kwargs)
return []
monkeypatch.setitem(sys.modules, "ddgs", SimpleNamespace(DDGS=FakeDDGS))
tools._search_text("latest release", backend="auto", time_range="week")
assert calls["backend"] == "brave,duckduckgo,yahoo"
assert calls["region"] == "wt-wt"
assert calls["timelimit"] == "w"
@pytest.mark.parametrize(
("configured_backend", "expected_backend"),
[
("wikipedia,duckduckgo,yandex", "duckduckgo"),
("wikipedia", "brave,duckduckgo,yahoo"),
("all", "brave,duckduckgo,yahoo"),
],
)
def test_search_text_time_range_excludes_explicit_filter_agnostic_backends(
monkeypatch,
configured_backend: str,
expected_backend: str,
) -> None:
calls = {}
class FakeDDGS:
def __init__(self, timeout: int) -> None:
calls["timeout"] = timeout
def text(self, query: str, **kwargs):
calls.update(kwargs)
return []
monkeypatch.setitem(sys.modules, "ddgs", SimpleNamespace(DDGS=FakeDDGS))
tools._search_text("latest release", backend=configured_backend, time_range="day")
assert calls["backend"] == expected_backend
def test_web_search_tool_reads_ddgs_options_from_config() -> None:
@ -62,7 +144,7 @@ def test_web_search_tool_reads_ddgs_options_from_config() -> None:
with patch("deerflow.community.ddg_search.tools._search_text") as mock_search:
mock_search.return_value = [{"title": "Result", "href": "https://example.com", "body": "Snippet"}]
result = tools.web_search_tool.invoke({"query": "latest news", "max_results": 8})
result = tools.web_search_tool.invoke({"query": "latest news", "max_results": 8, "time_range": "week"})
parsed = json.loads(result)
assert parsed["total_results"] == 1
@ -72,4 +154,5 @@ def test_web_search_tool_reads_ddgs_options_from_config() -> None:
region="us-en",
safesearch="off",
backend="auto",
time_range="week",
)

View File

@ -110,6 +110,40 @@ class TestSearxngClient:
call_kwargs = mock_ctx.get.call_args.kwargs
assert call_kwargs["params"]["categories"] == "news,science"
async def test_search_with_time_range(self):
"""Search passes a native relative time range."""
with patch("deerflow.community.searxng.searxng_client.httpx.AsyncClient") as mock_cls:
mock_ctx = MagicMock()
mock_cls.return_value.__aenter__.return_value = mock_ctx
mock_resp = MagicMock()
mock_resp.json.return_value = {"results": []}
mock_resp.raise_for_status.return_value = None
mock_ctx.get = AsyncMock(return_value=mock_resp)
client = SearxngClient(base_url="http://searxng:8080")
await client.search("latest release", time_range="month")
params = mock_ctx.get.call_args.kwargs["params"]
assert params["time_range"] == "month"
async def test_search_without_time_range_omits_parameter(self):
"""The default request shape remains unchanged."""
with patch("deerflow.community.searxng.searxng_client.httpx.AsyncClient") as mock_cls:
mock_ctx = MagicMock()
mock_cls.return_value.__aenter__.return_value = mock_ctx
mock_resp = MagicMock()
mock_resp.json.return_value = {"results": []}
mock_resp.raise_for_status.return_value = None
mock_ctx.get = AsyncMock(return_value=mock_resp)
client = SearxngClient(base_url="http://searxng:8080")
await client.search("stable documentation")
params = mock_ctx.get.call_args.kwargs["params"]
assert "time_range" not in params
@pytest.mark.asyncio
class TestSearxngTools:
@ -161,3 +195,15 @@ class TestSearxngTools:
mock_client.search.assert_called_once()
call_kwargs = mock_client.search.call_args.kwargs
assert call_kwargs["max_results"] == 3
@patch("deerflow.community.searxng.tools._get_searxng_client")
async def test_web_search_tool_forwards_time_range(self, mock_get_client):
"""web_search_tool forwards the requested relative time range."""
mock_client = MagicMock()
mock_client.search = AsyncMock(return_value=[])
mock_get_client.return_value = mock_client
with patch("deerflow.community.searxng.tools._get_tool_config", return_value=None):
await tools.web_search_tool.ainvoke({"query": "latest release", "time_range": "week"})
mock_client.search.assert_called_once_with("latest release", max_results=5, time_range="week")

View File

@ -0,0 +1,43 @@
"""Unit tests for the Tavily community web search tool."""
import json
from unittest.mock import MagicMock, patch
from deerflow.community.tavily.tools import web_search_tool
def _tavily_response() -> dict:
return {
"results": [
{
"title": "Release notes",
"url": "https://example.com/releases",
"content": "A recent release.",
}
]
}
def test_web_search_forwards_time_range_to_tavily() -> None:
client = MagicMock()
client.search.return_value = _tavily_response()
with patch("deerflow.community.tavily.tools.get_app_config") as mock_config:
mock_config.return_value.get_tool_config.return_value = None
with patch("deerflow.community.tavily.tools._get_tavily_client", return_value=client):
result = web_search_tool.invoke({"query": "latest releases", "time_range": "month"})
assert json.loads(result)[0]["title"] == "Release notes"
client.search.assert_called_once_with("latest releases", max_results=5, time_range="month")
def test_web_search_omits_time_range_from_default_tavily_call() -> None:
client = MagicMock()
client.search.return_value = _tavily_response()
with patch("deerflow.community.tavily.tools.get_app_config") as mock_config:
mock_config.return_value.get_tool_config.return_value = None
with patch("deerflow.community.tavily.tools._get_tavily_client", return_value=client):
web_search_tool.invoke({"query": "stable documentation"})
client.search.assert_called_once_with("stable documentation", max_results=5)

View File

@ -0,0 +1,26 @@
"""Shared contract tests for provider-native web-search recency filtering."""
import pytest
from langchain_core.utils.function_calling import convert_to_openai_tool
from deerflow.community.brave.tools import web_search_tool as brave_web_search
from deerflow.community.ddg_search.tools import web_search_tool as ddg_web_search
from deerflow.community.searxng.tools import web_search_tool as searxng_web_search
from deerflow.community.tavily.tools import web_search_tool as tavily_web_search
EXPECTED_TIME_RANGES = {"day", "week", "month", "year"}
@pytest.mark.parametrize(
"tool_obj",
[ddg_web_search, brave_web_search, tavily_web_search, searxng_web_search],
ids=["ddg", "brave", "tavily", "searxng"],
)
def test_web_search_time_range_schema_is_consistent(tool_obj) -> None:
parameters = convert_to_openai_tool(tool_obj)["function"]["parameters"]
time_range_schema = parameters["properties"]["time_range"]
branches = time_range_schema.get("anyOf", [time_range_schema])
enum_values = next(branch["enum"] for branch in branches if "enum" in branch)
assert set(enum_values) == EXPECTED_TIME_RANGES
assert "time_range" not in parameters.get("required", [])