From a23dbdd837141e3b5e782a32e3e884b266926d17 Mon Sep 17 00:00:00 2001 From: Weng Qiang <92919224+Tsubaki-01@users.noreply.github.com> Date: Fri, 18 Sep 2026 07:25:49 +0800 Subject: [PATCH] feat(tavily): support configured search domain filters (#5513) --- README.md | 8 ++++ backend/docs/CONFIGURATION.md | 15 ++++++ .../deerflow/community/tavily/AGENTS.md | 10 ++++ .../deerflow/community/tavily/tools.py | 6 +++ backend/tests/test_tavily_tools.py | 46 +++++++++++++++++++ config.example.yaml | 5 ++ 6 files changed, 90 insertions(+) diff --git a/README.md b/README.md index a79f1c017..79bb158c4 100644 --- a/README.md +++ b/README.md @@ -1074,6 +1074,14 @@ Public-skill CI waivers are exact, expiring exceptions in `.github/skill-review- Tools follow the same philosophy. DeerFlow comes with a core toolset — web search, web fetch, rendered web capture, file operations, bash execution — and supports custom tools via MCP servers and Python functions. The bundled DDG, Brave, Tavily, and SearXNG search providers accept an optional `time_range` of `day`, `week`, `month`, or `year`; omitting it preserves existing search behavior. For DDG recency searches, DeerFlow excludes DDGS backends that ignore time limits. Swap anything. Add anything. +Tavily `web_search` also accepts optional `include_domains` and `exclude_domains` +lists in its `config.yaml` tool entry to control search sources. Non-empty +`include_domains` uses Tavily's `filter` mode to restrict results to those domains. +These are deployment settings; the model still supplies only `query` and optional +`time_range`. Omitted filters preserve the existing SDK request; an explicit +empty list is forwarded and imposes no restriction of that kind. See the +[tool configuration example](backend/docs/CONFIGURATION.md#tools). + When using Tavily for `web_fetch`, extracted pages without a title use their URL as the heading; their content remains available to the agent. Tavily search and fetch each read `api_key` from their own tool entry in diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 112e4b0e9..ac3171d0f 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -485,9 +485,24 @@ tools: group: web use: deerflow.community.tavily.tools:web_search_tool max_results: 5 + include_domains: # Optional: limit search sources to these domains + - docs.python.org + - developer.mozilla.org + exclude_domains: [] # Optional: domains to exclude from search results # api_key: $TAVILY_API_KEY # Optional ``` +For Tavily, `include_domains` and `exclude_domains` are deployment-only options +read from the `web_search` tool entry and passed directly to `TavilyClient.search`. +For a non-empty `include_domains`, DeerFlow also sends `include_domains_mode: filter` +so Tavily restricts results to those domains rather than merely boosting them. +Either list may be configured independently. Omitted options are not added to the SDK +call; explicit empty lists are forwarded as `[]`, meaning no inclusion restriction +or no excluded domains, respectively. No `include_domains_mode` is sent for an +empty or omitted `include_domains`. These filters compose with `max_results` +and the model's optional `time_range`. The model-visible arguments remain `query` +and `time_range`; the filters do not apply to `web_fetch` or other search providers. + **Built-in Tools**: - `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Serply, Exa, InfoQuest, Tencent Cloud WSA, Firecrawl, fastCRW, GroundRoute, Sofya) - `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless, Sofya) diff --git a/backend/packages/harness/deerflow/community/tavily/AGENTS.md b/backend/packages/harness/deerflow/community/tavily/AGENTS.md index c346cc7c7..908f14fd5 100644 --- a/backend/packages/harness/deerflow/community/tavily/AGENTS.md +++ b/backend/packages/harness/deerflow/community/tavily/AGENTS.md @@ -4,3 +4,13 @@ The client helper selects credentials by tool name: search defaults to `web_search`, fetch passes `web_fetch`, and an omitted key uses the SDK's `TAVILY_API_KEY` fallback. Keep credential regressions in `backend/tests/test_tavily_tools.py` on the real helper and SDK constructor, mocking only search/extract calls. + +Search forwards `include_domains` and `exclude_domains` only when present in +the `web_search` tool config's `model_extra`, alongside `max_results` and optional +`time_range`. Preserve explicit empty lists (no restriction of that kind) and +omit absent keys. These are deployment-only search-source settings; keep the +model-visible signature limited to `query` and `time_range`. Offline regressions +in `backend/tests/test_tavily_tools.py` pin SDK arguments and the tool schema. +For non-empty `include_domains`, explicitly send `include_domains_mode="filter"` +through the SDK's keyword arguments; inclusion must restrict sources rather than +boost them. Omit the mode when the include list is absent or empty. diff --git a/backend/packages/harness/deerflow/community/tavily/tools.py b/backend/packages/harness/deerflow/community/tavily/tools.py index 4b40a2801..c33546b2c 100644 --- a/backend/packages/harness/deerflow/community/tavily/tools.py +++ b/backend/packages/harness/deerflow/community/tavily/tools.py @@ -30,6 +30,12 @@ def web_search_tool(query: str, time_range: SearchTimeRange | None = None) -> st client = _get_tavily_client() search_kwargs: dict[str, object] = {"max_results": max_results} + if config is not None: + for key in ("include_domains", "exclude_domains"): + if key in config.model_extra: + search_kwargs[key] = config.model_extra[key] + if search_kwargs.get("include_domains"): + search_kwargs["include_domains_mode"] = "filter" if time_range is not None: search_kwargs["time_range"] = time_range res = client.search(query, **search_kwargs) diff --git a/backend/tests/test_tavily_tools.py b/backend/tests/test_tavily_tools.py index 07de72bf7..5fd78a353 100644 --- a/backend/tests/test_tavily_tools.py +++ b/backend/tests/test_tavily_tools.py @@ -4,6 +4,7 @@ import json from unittest.mock import MagicMock, patch import pytest +from langchain_core.utils.function_calling import convert_to_openai_tool from tavily import TavilyClient from deerflow.community.tavily.tools import web_fetch_tool, web_search_tool @@ -96,6 +97,51 @@ def test_web_search_omits_time_range_from_default_tavily_call() -> None: client.search.assert_called_once_with("stable documentation", max_results=5) +@pytest.mark.parametrize("time_range", [None, "week"]) +@pytest.mark.parametrize( + "domain_config", + [ + pytest.param({}, id="omitted"), + pytest.param({"include_domains": ["docs.example.com", "reference.example.org"], "exclude_domains": ["archive.example.com"]}, id="both"), + pytest.param({"include_domains": ["docs.example.com"]}, id="include-only"), + pytest.param({"exclude_domains": ["archive.example.com"]}, id="exclude-only"), + pytest.param({"include_domains": [], "exclude_domains": []}, id="empty-both"), + pytest.param({"include_domains": []}, id="empty-include"), + pytest.param({"exclude_domains": []}, id="empty-exclude"), + ], +) +def test_web_search_forwards_configured_domains(domain_config, time_range) -> None: + configs = { + "web_search": ToolConfig(name="web_search", group="web", use="deerflow.community.tavily.tools:web_search_tool", api_key="search-key", max_results=3, **domain_config), + "web_fetch": ToolConfig(name="web_fetch", group="web", use="deerflow.community.tavily.tools:web_fetch_tool", include_domains=["fetch.example.com"], exclude_domains=["other.example.org"]), + } + tool_args = {"query": "documentation"} + expected_kwargs = {"max_results": 3, **domain_config} + if domain_config.get("include_domains"): + expected_kwargs["include_domains_mode"] = "filter" + if time_range is not None: + tool_args["time_range"] = time_range + expected_kwargs["time_range"] = time_range + + with ( + patch("deerflow.community.tavily.tools.get_app_config") as mock_config, + patch.object(TavilyClient, "search", autospec=True, return_value=_tavily_response()) as search, + ): + mock_config.return_value.get_tool_config.side_effect = configs.get + result = web_search_tool.invoke(tool_args) + + client = search.call_args.args[0] + search.assert_called_once_with(client, "documentation", **expected_kwargs) + assert json.loads(result) == [{"title": "Release notes", "url": "https://example.com/releases", "snippet": "A recent release."}] + + +def test_web_search_keeps_domain_filters_out_of_model_schema() -> None: + parameters = convert_to_openai_tool(web_search_tool)["function"]["parameters"] + + assert set(parameters["properties"]) == {"query", "time_range"} + assert parameters["required"] == ["query"] + + @pytest.mark.parametrize("title", [None, "", "Report title"]) def test_web_fetch_accepts_extract_results_with_optional_title(title) -> None: result = {"url": "https://example.com/report", "raw_content": "Important findings."} diff --git a/config.example.yaml b/config.example.yaml index 61e725398..db423f97d 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -854,6 +854,11 @@ tools: # group: web # use: deerflow.community.tavily.tools:web_search_tool # max_results: 5 + # # Optional search-source filters; omit or use [] for no restriction of that kind. + # # include_domains: + # # - docs.python.org + # # - developer.mozilla.org + # # exclude_domains: [] # # api_key: $TAVILY_API_KEY # Set if needed # Web search tool (uses InfoQuest, requires InfoQuest API key)