From dde131a808980fdd5b2bad8c258f1df9cb73a127 Mon Sep 17 00:00:00 2001 From: tiammomo <26957354+tiammomo@users.noreply.github.com> Date: Tue, 8 Sep 2026 17:12:50 +0800 Subject: [PATCH] fix(tavily): handle Extract responses without a title (#5280) * fix(tavily): handle Extract responses without a title Closes #5270 Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> * docs(tavily): keep extraction guidance within instruction budget Keep the approved AGENTS file layout and inherited size limits. Follow-up for #5280; refs #5270. Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --------- Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --- README.md | 3 + backend/AGENTS.md | 4 ++ .../deerflow/community/tavily/tools.py | 4 +- backend/tests/test_tavily_tools.py | 58 ++++++++++++++++++- 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 583b3e4ec..429e70fd9 100644 --- a/README.md +++ b/README.md @@ -997,6 +997,9 @@ Public-skill CI waivers are exact, expiring exceptions in `.github/skill-review- Tools follow the same philosophy. DeerFlow comes with a core toolset — web search, web fetch, rendered web capture, file operations, bash execution — and supports custom tools via MCP servers and Python functions. The bundled DDG, Brave, Tavily, and SearXNG search providers accept an optional `time_range` of `day`, `week`, `month`, or `year`; omitting it preserves existing search behavior. For DDG recency searches, DeerFlow excludes DDGS backends that ignore time limits. Swap anything. Add anything. +When using Tavily for `web_fetch`, extracted pages without a title use their URL +as the heading; their content remains available to the agent. + Advanced deployments can enable pluggable authorization with `authorization.enabled` in `config.yaml`. A configured `AuthorizationProvider` filters denied tools before they reach the model or deferred-tool catalog, then the same provider is checked again before every business-tool execution through the existing guardrail middleware. Gateway `threads:*` and `runs:*` route permissions are derived from the same provider, while existing owner checks and admin-only management gates remain in force. Every HTTP route that starts or enables a future Agent run requires `runs:create`: this includes the stateless `POST /api/runs/stream` and `POST /api/runs/wait` endpoints plus scheduled-task create, update, resume, and manual-trigger mutations. Scheduled-task mutations retain their existing `threads:write` requirement, and the stateless routes separately enforce ownership when the optional thread ID is supplied in the request body. A generated `tool_search` may bypass the second tool check only when it fronts the current build's already-filtered deferred catalog. Model access follows the same provider: the Gateway `models` list is filtered per principal, `model:use` is enforced on model detail requests and again when the runtime resolves the agent's model, and a denied default model falls back to the first remaining candidate that also passes `model:use`. The built-in RBAC provider supports per-role `tools`, `routes`, `models`, `skills`, and `sandbox` allow/deny policies and validates that `default_role` names a configured role; authorization is disabled by default. See `config.example.yaml` and the [authorization RFC](docs/plans/2026-07-10-pluggable-authorization-rfc.md). Advanced deployments can also extend the agent runtime itself by declaring zero-argument `AgentMiddleware` classes under `extensions.middlewares` in `config.yaml` or `extensions_config.json`. DeerFlow loads the same configured class list into the lead-agent and subagent pipelines after their built-in runtime middlewares and loop/token guards, but before the terminal-response/safety/clarification tail, so enterprise forks can add domain guardrails, tool-call governance, or observability hooks without patching the built-in middleware builders. Missing packages, invalid classes, and broken modules fail loudly at agent creation. Treat `config.yaml` and `extensions_config.json` as trusted operator-controlled files: middleware paths are code execution, just like custom tool, model, sandbox, guardrail, MCP server, and MCP interceptor declarations. Gateway skill/MCP toggle endpoints preserve this field but do not expose an API write path for `extensions.middlewares`. Per-context parameterization and separate lead-only/subagent-only middleware lists are not supported yet. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index c93caccad..4bb978f13 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -318,6 +318,10 @@ For recency, DDGS 9.14.1 uses only enabled Brave, DuckDuckGo, and Yahoo engines that honor `timelimit`: `auto`/`all` resolves to this set, incompatible configured engines are removed, and an empty set falls back to it. Re-check on DDGS upgrades. +### Tavily Fetch + +Title fallback: result URL, then request URL. + ### File Upload Multi-file uploads convert documents; outlines skip fenced code: diff --git a/backend/packages/harness/deerflow/community/tavily/tools.py b/backend/packages/harness/deerflow/community/tavily/tools.py index 973b44cb4..11770db29 100644 --- a/backend/packages/harness/deerflow/community/tavily/tools.py +++ b/backend/packages/harness/deerflow/community/tavily/tools.py @@ -62,6 +62,8 @@ def web_fetch_tool(url: str) -> str: return f"Error: {res['failed_results'][0]['error']}" elif "results" in res and len(res["results"]) > 0: result = res["results"][0] - return f"# {result['title']}\n\n{result['raw_content'][:4096]}" + # Extract results guarantee a URL and content, but not a page title. + title = result.get("title") or result.get("url") or url + return f"# {title}\n\n{result['raw_content'][:4096]}" else: return "Error: No results found" diff --git a/backend/tests/test_tavily_tools.py b/backend/tests/test_tavily_tools.py index 8503f6ec1..2b591e2e1 100644 --- a/backend/tests/test_tavily_tools.py +++ b/backend/tests/test_tavily_tools.py @@ -1,9 +1,11 @@ -"""Unit tests for the Tavily community web search tool.""" +"""Unit tests for the Tavily community search and fetch tools.""" import json from unittest.mock import MagicMock, patch -from deerflow.community.tavily.tools import web_search_tool +import pytest + +from deerflow.community.tavily.tools import web_fetch_tool, web_search_tool def _tavily_response() -> dict: @@ -41,3 +43,55 @@ def test_web_search_omits_time_range_from_default_tavily_call() -> None: web_search_tool.invoke({"query": "stable documentation"}) client.search.assert_called_once_with("stable documentation", max_results=5) + + +@pytest.mark.parametrize("title", [None, "", "Report title"]) +def test_web_fetch_accepts_extract_results_with_optional_title(title) -> None: + result = {"url": "https://example.com/report", "raw_content": "Important findings."} + if title is not None: + result["title"] = title + client = MagicMock() + client.extract.return_value = {"results": [result], "failed_results": []} + + with patch("deerflow.community.tavily.tools._get_tavily_client", return_value=client): + output = web_fetch_tool.invoke({"url": "https://example.com/requested"}) + + assert output == f"# {title or result['url']}\n\nImportant findings." + client.extract.assert_called_once_with(["https://example.com/requested"]) + + +def test_web_fetch_falls_back_to_requested_url_without_result_metadata() -> None: + client = MagicMock() + client.extract.return_value = {"results": [{"title": None, "url": None, "raw_content": "Important findings."}]} + + with patch("deerflow.community.tavily.tools._get_tavily_client", return_value=client): + output = web_fetch_tool.invoke({"url": "https://example.com/requested"}) + + assert output == "# https://example.com/requested\n\nImportant findings." + + +def test_web_fetch_preserves_content_limit_without_title() -> None: + client = MagicMock() + client.extract.return_value = {"results": [{"url": "https://example.com/report", "raw_content": "x" * 5000}]} + + with patch("deerflow.community.tavily.tools._get_tavily_client", return_value=client): + output = web_fetch_tool.invoke({"url": "https://example.com/report"}) + + assert output == "# https://example.com/report\n\n" + "x" * 4096 + + +@pytest.mark.parametrize( + ("response", "expected"), + [ + ({"failed_results": [{"error": "Extraction failed"}]}, "Error: Extraction failed"), + ({"results": [], "failed_results": []}, "Error: No results found"), + ], +) +def test_web_fetch_preserves_unsuccessful_extract_results(response, expected) -> None: + client = MagicMock() + client.extract.return_value = response + + with patch("deerflow.community.tavily.tools._get_tavily_client", return_value=client): + output = web_fetch_tool.invoke({"url": "https://example.com/report"}) + + assert output == expected