diff --git a/README.md b/README.md index 5641643634..2798ed4a65 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ streams simulate chunks from a non-streaming response, while no-tool streams sta InfoQuest web and image search contain malformed HTTP-success payloads the same way: a `search_result` envelope that is not an object, or an `organic`/`top_stories.items`/`images_results` container that is not a list or that holds no usable entry, returns the provider's format error, while malformed entries in an otherwise valid payload are skipped so the usable rows survive. Missing, null, or empty containers keep the existing empty result. Format errors log the container or entry position only, without queries or payload values. + InfoQuest web and image search screen result URLs through the shared public-URL policy and omit rejected rows. Responses without the required `search_result` envelope return a format error instead of exposing the raw provider payload. + Jina, Browserless, and InfoQuest web fetches resolve relative links and image sources using the requested page URL (or a usable HTML base URL), so returned Markdown includes complete destinations. Link resolution preserves the surrounding HTML source, including malformed-page formatting. Self-hosted Browserless, Crawl4AI, Firecrawl, and fastCRW fetch backends require outbound isolation from private, loopback, link-local, and cloud-metadata networks before setting `network_isolation_confirmed: true` on each tool. DeerFlow rejects private or unverifiable backends by default because those services handle redirects, DNS resolution, and subresources themselves. `allow_private_addresses` permits intentional internal target URLs and does not bypass the backend check. See [delegated fetch deployment](backend/docs/CONFIGURATION.md#delegated-fetch-backend-isolation); `make doctor` reports backend configurations that would be refused. diff --git a/backend/packages/harness/deerflow/community/infoquest/AGENTS.md b/backend/packages/harness/deerflow/community/infoquest/AGENTS.md new file mode 100644 index 0000000000..979cd90be2 --- /dev/null +++ b/backend/packages/harness/deerflow/community/infoquest/AGENTS.md @@ -0,0 +1,21 @@ +# InfoQuest result safety + +Result URLs in `organic[].url`, `top_stories.items[].url`, and +`images_results[].original` use +`community.url_safety.safe_public_result_url` (imported as `_safe_public_url`); +do not add a provider-local copy. Return the screened value and deduplicate +on that value. Drop rejected rows while retaining valid rows in order. + +Web and image searches require a `search_result` envelope. Unknown response +shapes return the provider's format error; never return the raw payload as a +fallback, because that bypasses result-URL screening. Missing, null, or empty +result containers within a valid envelope retain their empty-result behavior. +Log missing envelope names without queries, credentials, or payload values. + +Screening is string-only and performs no DNS lookup. Downloaders must still +validate resolved addresses at connection time through the shared fetch policy. +Keep fetch behavior and HTTP timeout handling unchanged. + +Tests: `backend/tests/test_infoquest_result_url_safety.py`, +`backend/tests/test_image_result_url_safety.py`, and +`backend/tests/test_infoquest_client.py` (all offline). diff --git a/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py b/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py index 3f4afa6f84..24a1088075 100644 --- a/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py +++ b/backend/packages/harness/deerflow/community/infoquest/infoquest_client.py @@ -11,6 +11,8 @@ from typing import Any import httpx +from deerflow.community.url_safety import safe_public_result_url as _safe_public_url + logger = logging.getLogger(__name__) # httpx defaults to a 5s timeout. This bounds local connect/read inactivity; @@ -272,12 +274,13 @@ class InfoQuestClient: clean_result["desc"] = result["desc"] clean_result["snippet"] = result["desc"] if "url" in result: - clean_result["url"] = result["url"] - url = clean_result["url"] - if isinstance(url, str) and url and url not in seen_urls: - seen_urls.add(url) - clean_results.append(clean_result) - counts["pages"] += 1 + url = _safe_public_url(result["url"]) + if not url or url in seen_urls: + continue + clean_result["url"] = url + seen_urls.add(url) + clean_results.append(clean_result) + counts["pages"] += 1 top_stories = results.get("top_stories") if isinstance(top_stories, dict): @@ -291,12 +294,12 @@ class InfoQuestClient: if "source" in obj: clean_result["source"] = obj["source"] title = obj.get("title") - url = obj.get("url") + url = _safe_public_url(obj.get("url")) if title: clean_result["title"] = title if url: clean_result["url"] = url - if title and isinstance(url, str) and url and url not in seen_urls: + if title and url and url not in seen_urls: seen_urls.add(url) clean_results.append(clean_result) counts["news"] += 1 @@ -358,9 +361,9 @@ class InfoQuestClient: logger.error("web search API return wrong format, no search_result nor content field found in JSON response, content: %s", raw_results["content"]) return f"Error: {error_message}" else: - # If neither field exists, return the original response - logger.warning("InfoQuest Web-Search - Neither search_result nor content field found in JSON response") - return json.dumps(raw_results, indent=2, ensure_ascii=False) + # Unknown envelopes must not bypass result-URL screening. + logger.warning("InfoQuest Web-Search - Missing search_result envelope") + return "Error: InfoQuest returned an unexpected response format" except InfoQuestResponseError: return "Error: InfoQuest returned an unexpected response format" @@ -382,12 +385,13 @@ class InfoQuestClient: for result in cls._rows(results.get("images_results"), "images_results"): clean_result = {} if "original" in result: - clean_result["image_url"] = result["original"] - url = clean_result["image_url"] - if isinstance(url, str) and url and url not in seen_urls: - seen_urls.add(url) - clean_results.append(clean_result) - counts["images"] += 1 + image_url = _safe_public_url(result["original"]) + if not image_url or image_url in seen_urls: + continue + clean_result["image_url"] = image_url + seen_urls.add(image_url) + clean_results.append(clean_result) + counts["images"] += 1 if "title" in result: clean_result["title"] = result["title"] logger.debug(f"Results processing completed | total_results={len(clean_results)} | images={counts['images']} | unique_urls={len(seen_urls)}") @@ -484,9 +488,9 @@ class InfoQuestClient: logger.error("image search API return wrong format, no search_result nor content field found in JSON response, content: %s", raw_results["content"]) return f"Error: {error_message}" else: - # If neither field exists, return the original response - logger.warning("InfoQuest Image Search - Neither search_result nor content field found in JSON response") - return json.dumps(raw_results, indent=2, ensure_ascii=False) + # Unknown envelopes must not bypass result-URL screening. + logger.warning("InfoQuest Image Search - Missing search_result envelope") + return "Error: InfoQuest returned an unexpected response format" except InfoQuestResponseError: return "Error: InfoQuest returned an unexpected response format" diff --git a/backend/tests/test_agent_guidance_check.py b/backend/tests/test_agent_guidance_check.py index bc03343062..4a3b3be6b8 100644 --- a/backend/tests/test_agent_guidance_check.py +++ b/backend/tests/test_agent_guidance_check.py @@ -23,6 +23,7 @@ EXPECTED_GUIDANCE_PATHS = { "backend/packages/harness/deerflow/community/brave/AGENTS.md", "backend/packages/harness/deerflow/community/groundroute/AGENTS.md", "backend/packages/harness/deerflow/community/image_search/AGENTS.md", + "backend/packages/harness/deerflow/community/infoquest/AGENTS.md", "backend/packages/harness/deerflow/community/jina_ai/AGENTS.md", "backend/packages/harness/deerflow/community/ragflow/AGENTS.md", "backend/packages/harness/deerflow/community/serper/AGENTS.md", diff --git a/backend/tests/test_image_result_url_safety.py b/backend/tests/test_image_result_url_safety.py index 9ac03eeafc..be59fc7512 100644 --- a/backend/tests/test_image_result_url_safety.py +++ b/backend/tests/test_image_result_url_safety.py @@ -6,7 +6,7 @@ from unittest.mock import patch import pytest -@pytest.fixture(params=["url_safety", "brave.tools", "serper.tools"]) +@pytest.fixture(params=["url_safety", "brave.tools", "serper.tools", "infoquest.infoquest_client"]) def screen(request): module = importlib.import_module(f"deerflow.community.{request.param}") return module.safe_public_result_url if request.param == "url_safety" else module._safe_public_url diff --git a/backend/tests/test_infoquest_result_url_safety.py b/backend/tests/test_infoquest_result_url_safety.py new file mode 100644 index 0000000000..583a8a61d6 --- /dev/null +++ b/backend/tests/test_infoquest_result_url_safety.py @@ -0,0 +1,167 @@ +"""InfoQuest screens every untrusted result URL through the shared owner. + +``community/url_safety.safe_public_result_url`` is the single owner for the +result-URL policy the sibling providers already follow (brave and serper import +it under the ``_safe_public_url`` alias, and their ``AGENTS.md`` says "do not +add a provider-local copy"). InfoQuest registers the ``image_search`` tool too, +so the policy covers its three result fields: ``organic[].url``, +``top_stories.items[].url`` and ``images_results[].original``. + +A row whose URL is not a public HTTP(S) result is dropped rather than raising: +one hostile entry must not turn an otherwise valid search into the provider's +format error, which is the containment rule #6932 established for this client. +""" + +import asyncio +import json +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest + +from deerflow.community import url_safety +from deerflow.community.infoquest import infoquest_client as infoquest_module +from deerflow.community.infoquest.infoquest_client import InfoQuestClient + +NON_PUBLIC_URLS: list[Any] = [ + "http://127.0.0.1/record", + "http://localhost./record", + "http://metadata.google.internal/record", + "http://169.254.169.254/latest/meta-data", + "http://224.0.0.1/record", + "http://240.0.0.1/record", + "http://3758096385/record", + "http://0x7f000001/record", + "http://0177.0.0.1/record", + "http://[::ffff:127.0.0.1]/record", + "http://[64:ff9b::7f00:1]/record", + "http://[2002:7f00:1::]/record", + "http://[ff02::1]/record", + "http://127。0。0。1/record", + "file:///etc/passwd", + "", + " ", + None, + {"url": "https://example.com"}, +] + +PUBLIC_URL = "https://example.com/keep" + + +def _envelope(**sections: Any) -> list[dict]: + return [{"content": {"results": sections}}] + + +def _rows_for(field: str, url: Any) -> list[dict]: + if field == "images_results": + return [{"original": url, "title": "Row"}] + return [{"url": url, "title": "Row"}] + + +def _clean(field: str, url: Any) -> list[dict]: + if field == "images_results": + return InfoQuestClient.clean_results_with_image_search(_envelope(images_results=_rows_for(field, url))) + if field == "top_stories": + return InfoQuestClient.clean_results(_envelope(top_stories={"items": [{"url": url, "title": "Row", "source": "wire"}]})) + return InfoQuestClient.clean_results(_envelope(organic=[{"url": url, "title": "Row"}])) + + +def test_client_uses_the_shared_owner_without_a_local_copy(): + assert infoquest_module._safe_public_url is url_safety.safe_public_result_url + + +@pytest.mark.parametrize("field", ["organic", "images_results", "top_stories"]) +@pytest.mark.parametrize("url", NON_PUBLIC_URLS) +def test_non_public_result_urls_are_dropped(field, url): + assert _clean(field, url) == [] + + +@pytest.mark.parametrize("field", ["organic", "images_results", "top_stories"]) +def test_public_result_urls_are_kept(field): + cleaned = _clean(field, PUBLIC_URL) + + assert len(cleaned) == 1 + assert PUBLIC_URL in json.dumps(cleaned, ensure_ascii=False) + + +def test_screened_value_is_the_one_returned(): + # Returning the raw string after screening only the decision would hand the + # model a URL the owner never vetted. + organic = InfoQuestClient.clean_results(_envelope(organic=[{"url": " https://example.com/p ", "title": "Row"}])) + images = InfoQuestClient.clean_results_with_image_search(_envelope(images_results=[{"original": " https://example.com/p ", "title": "Row"}])) + news = InfoQuestClient.clean_results(_envelope(top_stories={"items": [{"url": " https://example.com/p ", "title": "Row"}]})) + + assert [item["url"] for item in organic] == ["https://example.com/p"] + assert [item["image_url"] for item in images] == ["https://example.com/p"] + assert [item["url"] for item in news] == ["https://example.com/p"] + + +def test_whitespace_variant_collapses_onto_the_same_url_for_deduplication(): + cleaned = InfoQuestClient.clean_results(_envelope(organic=[{"url": "https://example.com/dup", "title": "A"}, {"url": " https://example.com/dup ", "title": "B"}])) + + assert [item["title"] for item in cleaned] == ["A"] + + +def test_one_hostile_row_does_not_sink_the_rest_of_the_payload(): + cleaned = InfoQuestClient.clean_results(_envelope(organic=[{"url": "http://127.0.0.1/x", "title": "Private"}, {"url": "https://example.com/y", "title": "Public"}])) + + assert [(item["type"], item["url"]) for item in cleaned] == [("page", "https://example.com/y")] + + +def test_rows_without_a_url_are_still_dropped(): + assert InfoQuestClient.clean_results(_envelope(organic=[{"title": "No url"}])) == [] + assert InfoQuestClient.clean_results(_envelope(top_stories={"items": [{"title": "No url news"}]})) == [] + assert InfoQuestClient.clean_results_with_image_search(_envelope(images_results=[{"title": "No original"}])) == [] + + +def test_other_fields_survive_screening_of_the_url(): + cleaned = InfoQuestClient.clean_results(_envelope(organic=[{"url": PUBLIC_URL, "title": "Title", "desc": "Snippet"}])) + + assert cleaned[0]["title"] == "Title" + assert cleaned[0]["snippet"] == "Snippet" + + +def test_web_search_output_never_carries_a_non_public_result_url(): + hostile = "http://169.254.169.254/latest/meta-data" + payload = {"search_result": {"results": _envelope(organic=[{"url": hostile, "title": "Private"}, {"url": PUBLIC_URL, "title": "Public"}])}} + client = InfoQuestClient() + + with patch.object(type(client), "web_search_raw_results", new=AsyncMock(return_value=payload)): + output = asyncio.run(client.web_search("query")) + + assert json.loads(output) == [{"type": "page", "title": "Public", "url": PUBLIC_URL}] + assert hostile not in output + + +def test_image_search_output_never_carries_a_non_public_result_url(): + hostile = "http://169.254.169.254/latest/meta-data" + payload = {"search_result": {"results": _envelope(images_results=[{"original": hostile, "title": "Private"}, {"original": PUBLIC_URL, "title": "Public"}])}} + client = InfoQuestClient() + + with patch.object(type(client), "image_search_raw_results", new=AsyncMock(return_value=payload)): + output = asyncio.run(client.image_search("query")) + + assert json.loads(output) == [{"image_url": PUBLIC_URL, "title": "Public"}] + assert hostile not in output + + +@pytest.mark.parametrize("method", ["web_search", "image_search"]) +@pytest.mark.parametrize( + "payload", + [ + {}, + {"url": "http://169.254.169.254/latest/meta-data", "marker": "untrusted-provider-payload"}, + {"results": [{"url": PUBLIC_URL}, {"url": "http://127.0.0.1/admin"}]}, + {"unexpected": {"images_results": [{"original": "http://169.254.169.254/latest/meta-data"}]}}, + ], +) +def test_search_without_result_envelope_never_returns_raw_payload(method, payload): + client = InfoQuestClient() + with patch.object(type(client), f"{method}_raw_results", new=AsyncMock(return_value=payload)) as request: + output = asyncio.run(getattr(client, method)("query")) + + assert output == "Error: InfoQuest returned an unexpected response format" + assert "169.254.169.254" not in output + assert "127.0.0.1" not in output + assert "untrusted-provider-payload" not in output + request.assert_awaited_once()