From d5e0a0a269ba5fca7efb481f2ae93c537b714e8b Mon Sep 17 00:00:00 2001 From: Jason Cat Date: Sun, 11 Oct 2026 15:53:01 +0800 Subject: [PATCH] fix(tavily): contain malformed web search and fetch payloads (#6645) * fix(tavily): contain malformed web search and fetch payloads web_search and web_fetch normalized the SDK's decoded 200 body without validating its shape: a `results`/`failed_results` container of the wrong type, a non-object entry, or a missing leaf field escaped as a raw TypeError/KeyError/AttributeError instead of the structured format error brave, serper, serply, sofya, tencent_wsa, unbrowse and groundroute return. Validate the container with one shared helper, skip non-object entries while preserving valid objects in order, normalize missing leaf fields to empty text, and coerce a non-string raw_content in web_fetch before truncating. Diagnostics carry only the malformed container's name and type, never the query, URL, key or payload values. backend/tests/test_tavily_response_shapes.py drives the real tools and SDK over an offline HTTPX MockTransport; it fails 32 of 36 cases on main and passes with this change, while test_tavily_tools.py stays green. * docs(readme): note Tavily payload containment Follow the Brave and GroundRoute README entries in the setup wizard step; the containment contract itself lives in community/tavily/AGENTS.md. * fix(tavily): normalize null search fields and test mixed fetch lists --------- Co-authored-by: Willem Jiang --- README.md | 2 + .../deerflow/community/tavily/AGENTS.md | 19 ++ .../deerflow/community/tavily/tools.py | 64 ++++- backend/tests/test_tavily_response_shapes.py | 270 ++++++++++++++++++ 4 files changed, 343 insertions(+), 12 deletions(-) create mode 100644 backend/tests/test_tavily_response_shapes.py diff --git a/README.md b/README.md index e5981e872a..a9a9be7ccc 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,8 @@ streams simulate chunks from a non-streaming response, while no-tool streams sta GroundRoute web search and fetch contain malformed HTTP-success payloads the same way: a non-object payload, a `results` container that is not a list, or a non-empty list with no object entries returns the provider's format error, while non-object entries in an otherwise valid list are skipped in order. Missing, null, or empty results keep the existing "No results found" response. + Tavily web search and fetch contain malformed HTTP-success payloads the same way: a non-object payload, or a `results`/`failed_results` container that is not a list, returns the provider's format error, and non-object entries in an otherwise valid list are skipped in order. Missing, null, or empty results keep the existing response (`[]` for search, `Error: No results found` for fetch); missing or null search fields (`title`, `url`, `content`) normalize to empty text, and a non-string fetch `raw_content` is coerced to text before the truncation limit. + Jina, Browserless, and InfoQuest web fetches resolve relative links and image sources using the requested page URL (or a usable HTML base URL), so returned Markdown includes complete destinations. Link resolution preserves the surrounding HTML source, including malformed-page formatting. Self-hosted Browserless, Crawl4AI, Firecrawl, and fastCRW fetch backends require outbound isolation from private, loopback, link-local, and cloud-metadata networks before setting `network_isolation_confirmed: true` on each tool. DeerFlow rejects private or unverifiable backends by default because those services handle redirects, DNS resolution, and subresources themselves. `allow_private_addresses` permits intentional internal target URLs and does not bypass the backend check. See [delegated fetch deployment](backend/docs/CONFIGURATION.md#delegated-fetch-backend-isolation); `make doctor` reports backend configurations that would be refused. diff --git a/backend/packages/harness/deerflow/community/tavily/AGENTS.md b/backend/packages/harness/deerflow/community/tavily/AGENTS.md index 8f86c6fb4d..c6c043db32 100644 --- a/backend/packages/harness/deerflow/community/tavily/AGENTS.md +++ b/backend/packages/harness/deerflow/community/tavily/AGENTS.md @@ -18,3 +18,22 @@ in `backend/tests/test_tavily_tools.py` pin SDK arguments and the tool schema. For non-empty `include_domains`, explicitly send `include_domains_mode="filter"` through the SDK's keyword arguments; inclusion must restrict sources rather than boost them. Omit the mode when the include list is absent or empty. + +Validate the HTTP-success payload before normalizing it. A top-level that is not an +object, a `results`/`failed_results` container that is not a list, or a non-empty +list whose entries hold no objects is a format error, not a legitimate empty result: +search returns the structured `{"error": ..., "query": ...}` response and fetch +returns the `Error: ...` string. Non-object entries are skipped while valid objects +are preserved in order, and missing or null search leaf fields (`title`, `url`, +`content`) normalize to empty text. A missing, null, or empty `results` keeps +search's `[]` and fetch's "Error: No results found"; a failed extraction whose object has no `error` +message reports "Error: Extraction failed". `web_fetch` coerces non-string +`raw_content` to text before truncating to 4096 characters. Log the malformed +container's name and type, or that no usable result objects remain, before +returning — never the query, URL, credentials, or payload values. + +`backend/tests/test_tavily_response_shapes.py` drives the real tools and the real +AsyncTavilyClient through an offline HTTPX MockTransport; non-object top-level +payloads stub the client because the SDK's public wrappers reject that shape before +the tools see it. Cover null search fields individually and together, and mixed +fetch lists so the first retained object supplies the result or extraction error. diff --git a/backend/packages/harness/deerflow/community/tavily/tools.py b/backend/packages/harness/deerflow/community/tavily/tools.py index 256029f6f0..8be6a3b00c 100644 --- a/backend/packages/harness/deerflow/community/tavily/tools.py +++ b/backend/packages/harness/deerflow/community/tavily/tools.py @@ -1,4 +1,5 @@ import json +import logging from langchain.tools import tool from tavily import AsyncTavilyClient @@ -6,6 +7,10 @@ from tavily import AsyncTavilyClient from deerflow.community.search_time_range import SearchTimeRange from deerflow.config import get_app_config +logger = logging.getLogger(__name__) + +_FORMAT_ERROR = "Tavily returned an unexpected response format" + def _get_tavily_client(tool_name: str = "web_search") -> AsyncTavilyClient: config = get_app_config().get_tool_config(tool_name) @@ -15,6 +20,29 @@ def _get_tavily_client(tool_name: str = "web_search") -> AsyncTavilyClient: return AsyncTavilyClient(api_key=api_key) +def _response_objects(data: object, container: str) -> list[dict] | None: + """Return the object entries of a Tavily response container, or None if malformed. + + A missing or null container is empty; a container that is not a list, or a + non-empty list holding no objects, is malformed: the caller reports a format + error rather than returning a result the agent would misread. + """ + if not isinstance(data, dict): + logger.error("Tavily returned unexpected payload type: %s", type(data).__name__) + return None + value = data.get(container) + if value is None: + return [] + if not isinstance(value, list): + logger.error("Tavily returned unexpected '%s' payload type: %s", container, type(value).__name__) + return None + objects = [item for item in value if isinstance(item, dict)] + if value and not objects: + logger.error("Tavily returned '%s' with no usable result objects", container) + return None + return objects + + @tool("web_search", parse_docstring=True) async def web_search_tool(query: str, time_range: SearchTimeRange | None = None) -> str: """Search the web. @@ -42,13 +70,16 @@ async def web_search_tool(query: str, time_range: SearchTimeRange | None = None) res = await client.search(query, **search_kwargs) finally: await client.close() + results = _response_objects(res, "results") + if results is None: + return json.dumps({"error": _FORMAT_ERROR, "query": query}, ensure_ascii=False) normalized_results = [ { - "title": result["title"], - "url": result["url"], - "snippet": result["content"], + "title": result.get("title") or "", + "url": result.get("url") or "", + "snippet": result.get("content") or "", } - for result in res["results"] + for result in results ] json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False) return json_results @@ -70,12 +101,21 @@ async def web_fetch_tool(url: str) -> str: res = await client.extract([url]) finally: await client.close() - if "failed_results" in res and len(res["failed_results"]) > 0: - return f"Error: {res['failed_results'][0]['error']}" - elif "results" in res and len(res["results"]) > 0: - result = res["results"][0] - # Extract results guarantee a URL and content, but not a page title. - title = result.get("title") or result.get("url") or url - return f"# {title}\n\n{result['raw_content'][:4096]}" - else: + failed = _response_objects(res, "failed_results") + if failed is None: + return f"Error: {_FORMAT_ERROR}" + if failed: + error = failed[0].get("error") + return f"Error: {error}" if error else "Error: Extraction failed" + results = _response_objects(res, "results") + if results is None: + return f"Error: {_FORMAT_ERROR}" + if not results: return "Error: No results found" + result = results[0] + # Extract results guarantee a URL and content, but not a page title. + title = result.get("title") or result.get("url") or url + raw_content = result.get("raw_content") + if not isinstance(raw_content, str): + raw_content = "" if raw_content is None else str(raw_content) + return f"# {title}\n\n{raw_content[:4096]}" diff --git a/backend/tests/test_tavily_response_shapes.py b/backend/tests/test_tavily_response_shapes.py new file mode 100644 index 0000000000..2f385cf808 --- /dev/null +++ b/backend/tests/test_tavily_response_shapes.py @@ -0,0 +1,270 @@ +"""Tavily web_search/web_fetch must contain malformed HTTP-success payloads. + +These cases drive the real tools and the real AsyncTavilyClient through an offline +HTTPX MockTransport, so each one exercises the response-decoding path rather than a +mocked normalizer. +""" + +import json +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from tavily import AsyncTavilyClient + +_LOGGER_NAME = "deerflow.community.tavily.tools" +_FORMAT_ERROR = "Tavily returned an unexpected response format" +_SEARCH_QUERY = "docs search" +_FETCH_URL = "https://example.com/docs" + + +@pytest.fixture(autouse=True) +def app_config(monkeypatch): + from deerflow.community.tavily import tools + + config = SimpleNamespace(get_tool_config=lambda _: None) + monkeypatch.setattr(tools, "get_app_config", lambda: config) + + +def _transport(payload, status_code=200): + requests = [] + + def respond(request): + requests.append(request) + content = payload if isinstance(payload, str) else json.dumps(payload) + return httpx.Response(status_code, content=content.encode("utf-8"), headers={"Content-Type": "application/json"}) + + return httpx.MockTransport(respond), requests + + +async def _invoke(monkeypatch, tool, arguments, payload, status_code=200): + from deerflow.community.tavily import tools + + transport, requests = _transport(payload, status_code) + http_client = httpx.AsyncClient(transport=transport, base_url="https://api.tavily.test") + client = AsyncTavilyClient(api_key="synthetic-test-key", client=http_client) + monkeypatch.setattr(tools, "_get_tavily_client", lambda *args, **kwargs: client) + try: + result = await tool.ainvoke(arguments) + finally: + await http_client.aclose() + return result, requests + + +async def _invoke_search(monkeypatch, payload, status_code=200): + from deerflow.community.tavily.tools import web_search_tool + + return await _invoke(monkeypatch, web_search_tool, {"query": _SEARCH_QUERY}, payload, status_code) + + +async def _invoke_fetch(monkeypatch, payload, status_code=200): + from deerflow.community.tavily.tools import web_fetch_tool + + return await _invoke(monkeypatch, web_fetch_tool, {"url": _FETCH_URL}, payload, status_code) + + +async def _invoke_with_stubbed_client(monkeypatch, tool_name, method, payload, arguments): + """Exercise a top-level shape the real SDK cannot return (it indexes the parsed body).""" + from deerflow.community.tavily import tools + + client = MagicMock(spec=AsyncTavilyClient) + setattr(client, method, AsyncMock(return_value=payload)) + monkeypatch.setattr(tools, "_get_tavily_client", lambda *args, **kwargs: client) + tool = tools.web_search_tool if tool_name == "web_search" else tools.web_fetch_tool + return await tool.ainvoke(arguments) + + +def _error_logs(caplog): + return [record.getMessage() for record in caplog.records if record.name == _LOGGER_NAME and record.levelno == logging.ERROR] + + +def _assert_no_payload_values_leaked(caplog): + assert "provider-private-payload" not in caplog.text + assert "synthetic-test-key" not in caplog.text + assert "example.com" not in caplog.text + + +@pytest.mark.anyio +@pytest.mark.parametrize("payload", [None, ["provider-private-payload"], "provider-private-payload"]) +async def test_search_non_object_payload_returns_format_error(monkeypatch, payload, caplog): + result = await _invoke_with_stubbed_client(monkeypatch, "web_search", "search", payload, {"query": _SEARCH_QUERY}) + + assert json.loads(result) == {"error": _FORMAT_ERROR, "query": _SEARCH_QUERY} + assert _error_logs(caplog) == [f"Tavily returned unexpected payload type: {type(payload).__name__}"] + _assert_no_payload_values_leaked(caplog) + + +@pytest.mark.anyio +@pytest.mark.parametrize("results", [False, 0, "", "provider-private-payload", {}, {"0": "provider-private-payload"}]) +async def test_search_malformed_results_container_returns_format_error(monkeypatch, results, caplog): + result, _ = await _invoke_search(monkeypatch, {"results": results}) + + assert json.loads(result) == {"error": _FORMAT_ERROR, "query": _SEARCH_QUERY} + assert _error_logs(caplog) == [f"Tavily returned unexpected 'results' payload type: {type(results).__name__}"] + _assert_no_payload_values_leaked(caplog) + + +@pytest.mark.anyio +@pytest.mark.parametrize("results", [[None], [False, 0, "provider-private-payload", []]]) +async def test_search_all_malformed_entries_return_format_error(monkeypatch, results, caplog): + result, _ = await _invoke_search(monkeypatch, {"results": results}) + + assert json.loads(result) == {"error": _FORMAT_ERROR, "query": _SEARCH_QUERY} + assert _error_logs(caplog) == ["Tavily returned 'results' with no usable result objects"] + _assert_no_payload_values_leaked(caplog) + + +@pytest.mark.anyio +async def test_search_mixed_entries_preserve_valid_results_in_order(monkeypatch, caplog): + payload = {"results": [None, {"title": "First", "url": "https://ok.example/one", "content": "snippet"}, "invalid", [], {"title": "Second"}]} + + result, _ = await _invoke_search(monkeypatch, payload) + + assert json.loads(result) == [ + {"title": "First", "url": "https://ok.example/one", "snippet": "snippet"}, + {"title": "Second", "url": "", "snippet": ""}, + ] + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +@pytest.mark.parametrize("fields", [{}, {"title": None, "url": None, "content": None}], ids=["missing", "null"]) +async def test_search_missing_or_null_leaf_fields_normalize_to_empty_text(monkeypatch, fields, caplog): + result, requests = await _invoke_search(monkeypatch, {"results": [fields]}) + + assert json.loads(result) == [{"title": "", "url": "", "snippet": ""}] + assert len(requests) == 1 + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +@pytest.mark.parametrize(("field", "output_field"), [("title", "title"), ("url", "url"), ("content", "snippet")]) +async def test_search_null_leaf_does_not_change_other_fields(monkeypatch, field, output_field, caplog): + entry = {"title": "Report", "url": "https://example.com/report", "content": "Important findings."} + entry[field] = None + expected = {"title": "Report", "url": "https://example.com/report", "snippet": "Important findings."} + expected[output_field] = "" + + result, _ = await _invoke_search(monkeypatch, {"results": [entry]}) + + assert json.loads(result) == [expected] + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +@pytest.mark.parametrize("payload", [{}, {"results": None}, {"results": []}]) +async def test_search_missing_or_empty_results_keep_empty_list(monkeypatch, payload, caplog): + result, _ = await _invoke_search(monkeypatch, payload) + + assert json.loads(result) == [] + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +async def test_search_valid_payload_keeps_existing_normalization(monkeypatch, caplog): + payload = { + "query": _SEARCH_QUERY, + "results": [{"title": "Release notes", "url": "https://example.com/releases", "content": "A recent release."}], + } + + result, requests = await _invoke_search(monkeypatch, payload) + + assert json.loads(result) == [{"title": "Release notes", "url": "https://example.com/releases", "snippet": "A recent release."}] + assert len(requests) == 1 + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +@pytest.mark.parametrize("payload", [None, ["provider-private-payload"], "provider-private-payload"]) +async def test_fetch_non_object_payload_returns_format_error(monkeypatch, payload, caplog): + result = await _invoke_with_stubbed_client(monkeypatch, "web_fetch", "extract", payload, {"url": _FETCH_URL}) + + assert result == "Error: Tavily returned an unexpected response format" + assert _error_logs(caplog) == [f"Tavily returned unexpected payload type: {type(payload).__name__}"] + _assert_no_payload_values_leaked(caplog) + + +@pytest.mark.anyio +@pytest.mark.parametrize("failed", [False, 0, "", "provider-private-payload", {"0": "provider-private-payload"}]) +async def test_fetch_malformed_failed_results_container_returns_format_error(monkeypatch, failed, caplog): + result, _ = await _invoke_fetch(monkeypatch, {"failed_results": failed}) + + assert result == "Error: Tavily returned an unexpected response format" + assert _error_logs(caplog) == [f"Tavily returned unexpected 'failed_results' payload type: {type(failed).__name__}"] + _assert_no_payload_values_leaked(caplog) + + +@pytest.mark.anyio +@pytest.mark.parametrize("failed", [[None], [False, "provider-private-payload"]]) +async def test_fetch_failed_results_without_objects_returns_format_error(monkeypatch, failed, caplog): + result, _ = await _invoke_fetch(monkeypatch, {"failed_results": failed}) + + assert result == "Error: Tavily returned an unexpected response format" + assert _error_logs(caplog) == ["Tavily returned 'failed_results' with no usable result objects"] + _assert_no_payload_values_leaked(caplog) + + +@pytest.mark.anyio +async def test_fetch_failed_result_without_error_returns_generic_error(monkeypatch, caplog): + result, _ = await _invoke_fetch(monkeypatch, {"failed_results": [{"url": _FETCH_URL}]}) + + assert result == "Error: Extraction failed" + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +@pytest.mark.parametrize("results", [False, {}, "provider-private-payload", [None], [False, 0]]) +async def test_fetch_malformed_results_returns_format_error(monkeypatch, results, caplog): + result, _ = await _invoke_fetch(monkeypatch, {"failed_results": [], "results": results}) + + assert result == "Error: Tavily returned an unexpected response format" + assert len(_error_logs(caplog)) == 1 + _assert_no_payload_values_leaked(caplog) + + +@pytest.mark.anyio +@pytest.mark.parametrize( + ("container", "first", "second", "expected"), + [ + ("results", {"title": "First", "raw_content": "First content."}, {"title": "Second", "raw_content": "Second content."}, "# First\n\nFirst content."), + ("failed_results", {"error": "First failure"}, {"error": "Second failure"}, "Error: First failure"), + ], +) +async def test_fetch_mixed_entries_use_first_object_in_order(monkeypatch, container, first, second, expected, caplog): + payload = {container: [None, "invalid", [], first, False, second]} + + result, requests = await _invoke_fetch(monkeypatch, payload) + + assert result == expected + assert len(requests) == 1 + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +async def test_fetch_result_without_raw_content_returns_title_only(monkeypatch, caplog): + result, _ = await _invoke_fetch(monkeypatch, {"results": [{"url": "https://example.com/report"}]}) + + assert result == "# https://example.com/report\n\n" + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +@pytest.mark.parametrize("raw_content", [123, {"text": "provider-private-payload"}]) +async def test_fetch_coerces_non_string_raw_content(monkeypatch, raw_content, caplog): + result, _ = await _invoke_fetch(monkeypatch, {"results": [{"title": "Report", "raw_content": raw_content}]}) + + assert result == f"# Report\n\n{raw_content if isinstance(raw_content, str) else str(raw_content)}" + assert _error_logs(caplog) == [] + + +@pytest.mark.anyio +async def test_fetch_valid_payload_keeps_existing_normalization(monkeypatch, caplog): + payload = {"failed_results": [], "results": [{"title": "Report title", "url": "https://example.com/report", "raw_content": "Important findings."}]} + + result, requests = await _invoke_fetch(monkeypatch, payload) + + assert result == "# Report title\n\nImportant findings." + assert len(requests) == 1 + assert _error_logs(caplog) == []