From ec81b4d8dd99bb0dac79630b5dad3816f9a135fe Mon Sep 17 00:00:00 2001 From: huyengiang101086-spec Date: Wed, 14 Oct 2026 08:22:30 +0800 Subject: [PATCH] fix(image-search): normalize Unicode host separators (#6819) Co-authored-by: huyengiang101086-spec <270684046+huyengiang101086-spec@users.noreply.github.com> --- README.md | 4 ++ .../deerflow/community/image_search/AGENTS.md | 4 ++ .../harness/deerflow/community/url_safety.py | 5 +- backend/tests/test_image_result_idna_hosts.py | 52 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_image_result_idna_hosts.py diff --git a/README.md b/README.md index a952fd733c..d06b67f09a 100644 --- a/README.md +++ b/README.md @@ -1533,6 +1533,10 @@ Tools follow the same philosophy. DeerFlow comes with a core toolset — web sea For DDG web search and DDG image search, `max_results` in `config.yaml` can be a positive integer or an environment-variable reference such as `max_results: $DDG_MAX_RESULTS` with `DDG_MAX_RESULTS=5`. The configured value takes precedence over the tool call's `max_results` argument. Invalid values (for example, `abc`, an empty string, or `3.5`), zero, and negative counts produce a warning and fall back to the default of 5 results. DDG, Brave, and Serper image search share one result-URL screen that omits non-HTTP(S), localhost, cloud-metadata hostnames, and non-public address literals, including multicast and IPv6 spellings embedding private IPv4 addresses. The screen does not resolve DNS; downloaders must validate the resolved address when connecting. A result with neither a safe full image URL nor a safe thumbnail is omitted. +Image result screening also recognizes Unicode hostname separators (`。`, `．`, +`｡`) as dots, matching IDNA client normalization before classifying local/private +hosts. Accepted result URLs retain their original spelling. + Stdio MCP servers can set `cwd` in `extensions_config.json` when their entrypoint or data files depend on a specific working directory. The setting applies to both discovery and tool calls; see [MCP configuration](backend/docs/MCP_SERVER.md#stdio-working-directory). diff --git a/backend/packages/harness/deerflow/community/image_search/AGENTS.md b/backend/packages/harness/deerflow/community/image_search/AGENTS.md index b816dd24b4..000a0c8d54 100644 --- a/backend/packages/harness/deerflow/community/image_search/AGENTS.md +++ b/backend/packages/harness/deerflow/community/image_search/AGENTS.md @@ -7,3 +7,7 @@ address policy. The screen reuses `is_blocked_address` for literal and embedded IPv4 addresses, including multicast/reserved forms. Shared regression coverage is in `tests/test_image_result_url_safety.py`; DDG tool coverage is in `tests/test_ddg_image_search_urls.py`. + +Normalize IDNA's Unicode dot separators before checking blocked hostnames or IP +literals; preserve the original URL for allowed results. Client-normalization +regressions are in `tests/test_image_result_idna_hosts.py`. diff --git a/backend/packages/harness/deerflow/community/url_safety.py b/backend/packages/harness/deerflow/community/url_safety.py index 2c4b71b3e4..c54863a1a4 100644 --- a/backend/packages/harness/deerflow/community/url_safety.py +++ b/backend/packages/harness/deerflow/community/url_safety.py @@ -8,6 +8,7 @@ from collections.abc import Callable from urllib.parse import urlparse _BLOCKED_HOSTNAMES = {"localhost", "metadata.google.internal"} +_IDNA_DOT_TRANSLATION = str.maketrans("\u3002\uff0e\uff61", "...") def safe_public_result_url(value: object) -> str: @@ -26,7 +27,9 @@ def safe_public_result_url(value: object) -> str: return "" if parsed.scheme not in {"http", "https"} or not parsed.netloc or not host: return "" - host = host.lower().rstrip(".") + # IDNA clients treat these Unicode label separators as ASCII dots. Screen + # the normalized host so an IP literal cannot masquerade as a DNS name. + host = host.translate(_IDNA_DOT_TRANSLATION).lower().rstrip(".") if not host or host in _BLOCKED_HOSTNAMES or host.endswith(".localhost"): return "" diff --git a/backend/tests/test_image_result_idna_hosts.py b/backend/tests/test_image_result_idna_hosts.py new file mode 100644 index 0000000000..07168abd49 --- /dev/null +++ b/backend/tests/test_image_result_idna_hosts.py @@ -0,0 +1,52 @@ +"""Image result screening must classify the host clients use after IDNA.""" + +import importlib +import json +from unittest.mock import patch + +import httpx +import pytest + +from deerflow.community.image_search import tools + + +@pytest.fixture(params=["url_safety", "brave.tools", "serper.tools"]) +def screen(request): + module = importlib.import_module(f"deerflow.community.{request.param}") + return module.safe_public_result_url if request.param == "url_safety" else module._safe_public_url + + +@pytest.mark.parametrize( + ("url", "normalized_host"), + [ + ("http://127\u30020\u30020\u30021/image", "127.0.0.1"), + ("http://127\uff0e0\uff0e0\uff0e1/image", "127.0.0.1"), + ("http://127\uff610\uff610\uff611/image", "127.0.0.1"), + ("http://169\u3002254\u3002169\u3002254/image", "169.254.169.254"), + ("http://10\u30020\u30020\u30021/image", "10.0.0.1"), + ("http://localhost\u3002/image", "localhost."), + ("http://metadata\u3002google\u3002internal/image", "metadata.google.internal"), + ], +) +def test_image_results_reject_idna_normalized_private_hosts(screen, url, normalized_host): + # Parse with the real client, without connecting, to pin the normalization + # that makes these spellings equivalent to the blocked ASCII hosts. + assert httpx.URL(url).host == normalized_host + assert screen(url) == "" + + +@pytest.mark.parametrize("url", ["https://images\u3002example/full.jpg", "https://\u4f8b\u3048.\u30c6\u30b9\u30c8/full.jpg", "https://images.example/full.jpg"]) +def test_public_idna_result_urls_keep_the_original_string_without_dns(screen, url): + with patch("socket.getaddrinfo", side_effect=AssertionError("Result screening must not resolve DNS")): + assert screen(url) == url + + +@pytest.mark.asyncio +async def test_ddg_image_search_keeps_safe_thumbnail_when_idna_image_is_private(): + rows = [{"title": "reference", "image": "http://127\u30020\u30020\u30021/full.jpg", "thumbnail": "https://images.example/thumb.jpg"}] + with patch.object(tools, "_search_images", return_value=rows), patch.object(tools, "get_app_config") as config: + config.return_value.get_tool_config.return_value = None + result = json.loads(await tools.image_search_tool.ainvoke({"query": "reference"})) + + assert result["total_results"] == 1 + assert result["results"] == [{"title": "reference", "image_url": "", "thumbnail_url": "https://images.example/thumb.jpg"}]