fix(unbrowse): bound target error final URLs (#7399)

* fix(unbrowse): bound target error final URLs

* fix(unbrowse): redact final URL userinfo

* fix(unbrowse): remove credentials from final URL diagnostics

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

README.md

@@ -188,7 +188,7 @@ fences, including unclosed fences, in complete and streamed replies.
 
    Serper web and image search preserve valid entries in mixed result lists. A malformed `organic`/`images` container, or a non-empty list holding no object entries, returns Serper's format error; missing, null, or empty containers keep the existing "No results found" / "No images found" response. Format errors log the container field name only, without queries, credentials, or payload values.
 
-   Unbrowse web fetch preserves the extracted text but prefixes a warning when the target page's HTTP status is 400–599, even when the scraping service itself returns a successful response. Empty-page errors retain that status too, and both messages include the reported final URL when available.
+   Unbrowse web fetch preserves the extracted text but prefixes a warning when the target page's HTTP status is 400–599, even when the scraping service itself returns a successful response. Empty-page errors retain that status too. Both messages include up to 500 characters of the reported final URL after removing any username and password; final URLs that cannot be parsed as HTTP(S) URLs with a host are omitted.
 
    Tencent Cloud WSA web search contains malformed HTTP-success payloads the same way: a `Pages` container that is not a list, or a non-empty container holding no usable page, returns the provider's format error, while malformed page entries in an otherwise valid container are skipped so the valid rows survive. Missing, null, or empty `Pages` keep the existing "No results found" response.
 

backend/packages/harness/deerflow/community/unbrowse/tools.py

@@ -12,6 +12,7 @@ https://unbrowse.ai/app.
 import json
 import logging
 import os
+from urllib.parse import urlsplit, urlunsplit
 
 import httpx
 from langchain.tools import tool
@@ -24,6 +25,7 @@ _UNBROWSE_MCP_URL = "https://unbrowse.ai/api/mcp"
 _UNBROWSE_SCRAPE_TOOL = "unbrowse.scrape"
 _UNBROWSE_TIMEOUT = 90
 _UNBROWSE_FETCH_MAX_CHARS = 4096
+_UNBROWSE_FINAL_URL_MAX_CHARS = 500
 _DEFAULT_RENDER = "auto"
 _RENDER_MODES = ("auto", "never", "always")
 _api_key_warned: set[str] = set()
@@ -141,6 +143,20 @@ def _clip(value: object, limit: int) -> str:
     return text if limit <= 0 else text[:limit]
 
 
+def _strip_url_userinfo(url: str) -> str:
+    """Remove URL credentials; omit unparseable HTTP(S) destinations."""
+    try:
+        parsed = urlsplit(url)
+        if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+            return ""
+    except ValueError:
+        # Never echo raw credentials when parsing the authority fails.
+        return ""
+    if "@" not in parsed.netloc:
+        return url
+    return urlunsplit(parsed._replace(netloc=parsed.netloc.rsplit("@", 1)[-1]))
+
+
 @tool("web_fetch", parse_docstring=True)
 def web_fetch_tool(url: str) -> str:
     """Fetch the contents of a web page at a given URL.
@@ -171,8 +187,8 @@ def web_fetch_tool(url: str) -> str:
     if isinstance(status, int) and not isinstance(status, bool) and 400 <= status <= 599:
         failure = f"target page responded with HTTP {status}"
         final_url = data.get("finalUrl")
-        if isinstance(final_url, str) and final_url.strip():
-            failure += f" (final URL: {final_url.strip()})"
+        if isinstance(final_url, str) and (safe_final_url := _strip_url_userinfo(final_url.strip())):
+            failure += f" (final URL: {_clip(safe_final_url, _UNBROWSE_FINAL_URL_MAX_CHARS)})"
 
     content = _clip(data.get("markdown") or "", _UNBROWSE_FETCH_MAX_CHARS)
     if not content:

backend/tests/AGENTS.md

@@ -21,6 +21,12 @@ capture handler with fixture-scoped cleanup. Expect scheme/host context with
 redacted paths, so isolated runs and full-suite logging configuration agree;
 keep payload omission and tool-result assertions intact.
 
+Unbrowse final-URL regressions invoke the real fetch tool through mocked HTTP,
+covering text/structured MCP payloads and warning/empty-page diagnostics. Strip
+userinfo before the 500-character bound; keep ordinary URL spelling and `@` in
+paths/queries. Invalid authorities must not echo raw credentials. Include long
+userinfo and IPv6 cases in `test_unbrowse_target_status.py`.
+
 `.github/workflows/backend-windows-live-policy.yml` runs the child-env contract and all
 four live-test collection policy cases on Windows/Python 3.12, weekly, manually,
 and for relevant PR changes. These children only collect live tests. Keep the

backend/tests/test_unbrowse_target_status.py

@@ -31,6 +31,36 @@ def test_fetch_reports_redirect_destination_for_target_failure(monkeypatch, stru
         assert result == f"Error: No content found ({detail})"
 
 
+@pytest.mark.parametrize("structured", [False, True], ids=["text", "structured"])
+def test_fetch_bounds_redirect_destination(monkeypatch, structured):
+    final_url = "https://example.invalid/?next=" + "x" * 10_000
+
+    result = _fetch(monkeypatch, structured, {"statusCode": 404}, finalUrl=final_url)
+
+    assert final_url[:500] in result
+    assert final_url[:501] not in result
+
+
+@pytest.mark.parametrize("structured", [False, True], ids=["text", "structured"])
+@pytest.mark.parametrize("markdown", ["Body", ""], ids=["warning", "error"])
+@pytest.mark.parametrize(
+    ("final_url", "safe_url", "secrets"),
+    [
+        ("https://alice:password@example.invalid/private", "https://example.invalid/private", ("alice", "password")),
+        (
+            "https://bob:p%40ss@[2001:db8::1]:8443/private",
+            "https://[2001:db8::1]:8443/private",
+            ("bob", "p%40ss"),
+        ),
+    ],
+)
+def test_fetch_strips_userinfo_from_failure_redirect_destination(monkeypatch, structured, markdown, final_url, safe_url, secrets):
+    result = _fetch(monkeypatch, structured, {"statusCode": 404}, markdown=markdown, finalUrl=final_url)
+
+    assert f"final URL: {safe_url}" in result
+    assert all(secret not in result for secret in secrets)
+
+
 @pytest.mark.parametrize("structured", [False, True], ids=["text", "structured"])
 @pytest.mark.parametrize("final_url", [None, "", "   ", 42, {}, []])
 def test_fetch_ignores_empty_or_non_string_final_url(monkeypatch, structured, final_url):
@@ -51,6 +81,62 @@ def test_fetch_preserves_response_without_valid_error_status(monkeypatch, struct
     assert _fetch(monkeypatch, structured, metadata) == "# Untitled\n\nBody"
 
 
+@pytest.mark.parametrize("structured", [False, True], ids=["text", "structured"])
+@pytest.mark.parametrize("markdown", ["Body", "", None], ids=["warning", "empty", "null"])
+@pytest.mark.parametrize(
+    "final_url,safe_url,secret",
+    [
+        ("https://alice:synthetic-password@example.invalid:8443/missing?next=a%2Fb#part", "https://example.invalid:8443/missing?next=a%2Fb#part", "synthetic-password"),
+        ("https://alice@example.invalid/missing", "https://example.invalid/missing", "alice"),
+        ("https://alice:synthetic%40password@example.invalid/missing", "https://example.invalid/missing", "synthetic%40password"),
+        ("https://alice:synthetic@password@example.invalid/missing", "https://example.invalid/missing", "synthetic@password"),
+        ("https://alice:synthetic-password@[2001:db8::1]:8443/missing", "https://[2001:db8::1]:8443/missing", "synthetic-password"),
+    ],
+    ids=["password", "username-only", "encoded", "at-in-password", "ipv6"],
+)
+def test_fetch_removes_final_url_userinfo(monkeypatch, structured, markdown, final_url, safe_url, secret):
+    result = _fetch(monkeypatch, structured, {"title": "Page", "statusCode": 404}, markdown=markdown, finalUrl=final_url)
+    detail = f"target page responded with HTTP 404 (final URL: {safe_url})"
+    expected = f"Warning: {detail}.\n\n# Page\n\nBody" if markdown else f"Error: No content found ({detail})"
+    assert result == expected
+    assert secret not in result
+
+
+@pytest.mark.parametrize("structured", [False, True], ids=["text", "structured"])
+@pytest.mark.parametrize("markdown", ["Body", ""], ids=["warning", "empty"])
+def test_fetch_strips_userinfo_before_clipping_final_url(monkeypatch, structured, markdown):
+    secret = "synthetic-password-" + "s" * 600
+    safe_url = "https://example.invalid/missing?next=" + "x" * 10_000
+    final_url = "https://alice:" + secret + "@" + safe_url.removeprefix("https://")
+    result = _fetch(monkeypatch, structured, {"statusCode": 404}, markdown=markdown, finalUrl=final_url)
+    assert secret[:500] not in result
+    assert "alice:" not in result
+    assert safe_url[:500] in result
+    assert safe_url[:501] not in result
+
+
+@pytest.mark.parametrize("structured", [False, True], ids=["text", "structured"])
+@pytest.mark.parametrize("markdown", ["Body", ""], ids=["warning", "empty"])
+@pytest.mark.parametrize(
+    "final_url", ["https://alice:synthetic-password@[broken/missing", "https:/alice:synthetic-password@example.invalid/missing", "https://alice:synthetic-password@/missing"], ids=["invalid-brackets", "missing-authority", "missing-host"]
+)
+def test_fetch_omits_unparseable_final_url_without_echoing_credentials(monkeypatch, structured, markdown, final_url):
+    result = _fetch(monkeypatch, structured, {"statusCode": 404}, markdown=markdown, finalUrl=final_url)
+    detail = "target page responded with HTTP 404"
+    expected = f"Warning: {detail}.\n\n# Untitled\n\nBody" if markdown else f"Error: No content found ({detail})"
+    assert result == expected
+    assert "synthetic-password" not in result
+
+
+@pytest.mark.parametrize("structured", [False, True], ids=["text", "structured"])
+@pytest.mark.parametrize("length", [499, 500, 501], ids=["below-limit", "at-limit", "over-limit"])
+def test_fetch_keeps_userinfo_free_final_urls_unchanged_up_to_limit(monkeypatch, structured, length):
+    prefix = "HTTPS://Example.invalid:8443/path@team?email=person@example.invalid&next="
+    final_url = prefix + "x" * (length - len(prefix))
+    result = _fetch(monkeypatch, structured, {"statusCode": 404}, finalUrl="  " + final_url + "  ")
+    assert result == f"Warning: target page responded with HTTP 404 (final URL: {final_url[:500]}).\n\n# Untitled\n\nBody"
+
+
 def _fetch(monkeypatch, structured, metadata, markdown="Body", **fields):
     payload = {"markdown": markdown, "metadata": metadata, **fields}
     result = {"content": [{"type": "text", "text": json.dumps(payload)}], "isError": False}