fix(web-fetch): resolve relative URLs in extracted Markdown (#5310)

* fix(web-fetch): resolve relative URLs in extracted Markdown

Pass the request URL through Jina and Browserless extraction and resolve
link/image destinations before Readability removes document base tags.
Preserve the optional legacy API and fallback text behavior.

Fixes #5307

Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com>

* fix(web-fetch): address provider and base URL review feedback

Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com>

* fix(web-fetch): preserve HTML source when resolving destinations

Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com>

---------

Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com>

README.md

@@ -129,6 +129,8 @@ That prompt is intended for coding agents. It tells the agent to clone the repo
 
    The wizard also lets you configure an optional web search provider, or skip it for now.
 
+   Jina, Browserless, and InfoQuest web fetches resolve relative links and image sources using the requested page URL (or a usable HTML base URL), so returned Markdown includes complete destinations. Link resolution preserves the surrounding HTML source, including malformed-page formatting.
+
    Run `make doctor` at any time to verify your setup and get actionable fix hints.
    If you are opening a GitHub issue about a local setup or runtime problem, run
    `make support-bundle`. The command prints reporter next steps, writes a

backend/AGENTS.md

@@ -265,8 +265,9 @@ Direct pytest collection or execution of `tests/test_client_live.py` remains
 skipped unless `DEER_FLOW_RUN_LIVE_TESTS=1` is set. Do not add that opt-in to
 default CI workflows.
 
-Jina logging tests isolate missing-key warnings with dummy keys (`tests/test_jina_client.py`).
+Jina logging tests use dummy keys (`tests/test_jina_client.py`).
-InfoQuest HTTP calls share a 30s connect/read inactivity timeout, separate from remote crawl timeouts; see `tests/test_infoquest_http_timeout.py`.
+Jina/Browserless/InfoQuest resolve URLs without rebuilding HTML.
+InfoQuest connect/read timeout is 30s, separate from crawl timeouts (`tests/test_infoquest_http_timeout.py`).
 
 ### Running the Full Application
 

backend/packages/harness/deerflow/community/browserless/tools.py

@@ -271,7 +271,7 @@ async def web_fetch_tool(url: str) -> str:
         if isinstance(result, str):
             return result
 
-        article = await asyncio.to_thread(_readability_extractor.extract_article, result.html)
+        article = await asyncio.to_thread(_readability_extractor.extract_article, result.html, url=url)
         return f"{article.to_markdown()[:4096]}{_target_status_warning(result)}"
 
     except Exception as e:

backend/packages/harness/deerflow/community/infoquest/tools.py

@@ -70,7 +70,7 @@ def web_fetch_tool(url: str) -> str:
     result = client.fetch(url)
     if result.startswith("Error: "):
         return result
-    article = readability_extractor.extract_article(result)
+    article = readability_extractor.extract_article(result, url=url)
     return article.to_markdown()[:4096]
 
 

backend/packages/harness/deerflow/community/jina_ai/tools.py

@@ -64,5 +64,5 @@ async def web_fetch_tool(url: str) -> str:
     html_content = await jina_client.crawl(url, return_format="html", timeout=timeout, proxy=proxy, trust_env=trust_env)
     if isinstance(html_content, str) and html_content.startswith("Error:"):
         return html_content
-    article = await asyncio.to_thread(readability_extractor.extract_article, html_content)
+    article = await asyncio.to_thread(readability_extractor.extract_article, html_content, url=url)
     return article.to_markdown()[:4096]

backend/packages/harness/deerflow/utils/readability.py

@@ -1,8 +1,11 @@
 import logging
 import re
 import subprocess
-from urllib.parse import urljoin
+from html import escape, unescape
+from html.parser import HTMLParser
+from urllib.parse import urljoin, urlparse, uses_relative
 
+from bs4 import BeautifulSoup
 from markdownify import markdownify as md
 from readabilipy import simple_json_from_html_string
 
@@ -55,8 +58,101 @@ class Article:
         return content
 
 
+_BASE_TAG_RE = re.compile(r"<base", re.IGNORECASE)
+
+
+def _resolve_html_urls(html: str, url: str) -> str:
+    """Resolve destinations before extraction can discard the document's base tag."""
+    # A base element requires a literal start-tag prefix. False positives in
+    # comments or text elements still go through HTML5 tree construction.
+    base = BeautifulSoup(html, "html5lib").find("base", href=True) if _BASE_TAG_RE.search(html) else None
+    base_url = url
+    if base is not None:
+        try:
+            candidate = urljoin(url, str(base["href"]).strip())
+            # Keep only bases urljoin can resolve relative paths against.
+            # Opaque bases fall back to the fetched URL; hierarchical FTP remains valid.
+            if urlparse(candidate).scheme in uses_relative:
+                base_url = candidate
+        except ValueError:
+            pass  # An invalid base must not prevent extraction of the page.
+    resolver = _DestinationRewriter(html, base_url)
+    resolver.feed(html)
+    resolver.close()
+    return resolver.result()
+
+
+# Tokenize attributes only inside a start tag identified by HTMLParser. Keeping
+# source spans avoids rebuilding malformed markup before jsdom parses it.
+_ATTRIBUTE_RE = re.compile(r"""([^\s/>=]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]*))?""")
+
+
+class _DestinationRewriter(HTMLParser):
+    # Treat link examples inside text-only elements as data, including nested
+    # script-looking text; only the matching closing tag resumes tokenization.
+    CDATA_CONTENT_ELEMENTS = ("script", "style", "textarea", "title", "xmp", "iframe", "noembed", "noframes", "plaintext")
+
+    def __init__(self, html: str, base_url: str):
+        super().__init__(convert_charrefs=False)
+        self.html = html
+        self.base_url = base_url
+        self.text_element: str | None = None
+        self.line_offsets = [0, *(match.end() for match in re.finditer("\n", html))]
+        self.replacements: list[tuple[int, int, str]] = []
+
+    def handle_starttag(self, tag, attrs):
+        if self.text_element is not None:
+            return
+        if tag in {"textarea", "title", "xmp", "iframe", "noembed", "noframes", "plaintext"}:
+            self.text_element = tag
+            return
+        attribute = {"a": "href", "img": "src"}.get(tag)
+        if attribute is None:
+            return
+        raw = self.get_starttag_text()
+        tag_end = re.match(r"<[^\s/>]+", raw).end()
+        for match in _ATTRIBUTE_RE.finditer(raw, tag_end):
+            if match.group(1).lower() != attribute:
+                continue
+            value = match.group(2)
+            if value is not None:
+                original = unescape(value[1:-1] if value.startswith(('"', "'")) else value)
+                try:
+                    resolved = urljoin(self.base_url, original.strip())
+                except ValueError:
+                    return
+                if resolved != original:
+                    line, column = self.getpos()
+                    offset = self.line_offsets[line - 1] + column
+                    self.replacements.append((offset + match.start(2), offset + match.end(2), '"' + escape(resolved, quote=True) + '"'))
+            else:
+                line, column = self.getpos()
+                offset = self.line_offsets[line - 1] + column + match.end(1)
+                self.replacements.append((offset, offset, '="' + escape(self.base_url, quote=True) + '"'))
+            # Browsers use the first duplicate attribute, including a bare one.
+            return
+
+    def handle_endtag(self, tag):
+        if tag == self.text_element and tag != "plaintext":
+            self.text_element = None
+
+    def handle_startendtag(self, tag, attrs):
+        self.handle_starttag(tag, attrs)
+
+    def result(self) -> str:
+        parts = []
+        cursor = 0
+        for start, end, value in self.replacements:
+            parts.extend((self.html[cursor:start], value))
+            cursor = end
+        parts.append(self.html[cursor:])
+        return "".join(parts)
+
+
 class ReadabilityExtractor:
-    def extract_article(self, html: str) -> Article:
+    def extract_article(self, html: str, *, url: str | None = None) -> Article:
+        if url:
+            html = _resolve_html_urls(html, url)
         try:
             article = simple_json_from_html_string(html, use_readability=True)
         except (subprocess.CalledProcessError, FileNotFoundError) as exc:

backend/packages/harness/pyproject.toml

@@ -6,6 +6,8 @@ requires-python = ">=3.12"
 dependencies = [
     "agent-client-protocol>=0.4.0",
     "agent-sandbox>=0.0.30",
+    "beautifulsoup4>=4.12",
+    "html5lib>=1.1",
     "croniter>=6.0.0",
     # Exact pin by design (extension-system version contract): the host pins
     # the contract version it implements, extensions declare ranges. A range

backend/tests/test_web_fetch_relative_links.py

@@ -0,0 +1,211 @@
+"""Fetched HTML must preserve usable destinations in model-visible Markdown."""
+
+import importlib
+from ipaddress import ip_address
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from deerflow.community.browserless.browserless_client import BrowserlessFetchResult
+from deerflow.utils.readability import ReadabilityExtractor
+
+PAGE_URL = "https://example.com/docs/current"
+
+
+def _article(links: str, *, head: str = "") -> str:
+    paragraph = "<p>This article explains the documentation in detail, with enough ordinary prose for the real readability extractor to retain the content and its related references.</p>"
+    return f"<html><head><title>Guide</title>{head}</head><body><article>{paragraph * 5}<p>{links}</p></article></body></html>"
+
+
+@pytest.mark.parametrize("provider", ["jina_ai", "browserless", "infoquest"])
+@pytest.mark.anyio
+async def test_web_fetch_resolves_relative_links_through_real_extraction(monkeypatch, provider):
+    module = importlib.import_module(f"deerflow.community.{provider}.tools")
+    html = _article('<a href="../next">Next</a> <a href="/reference">Reference</a>')
+    monkeypatch.setattr(module, "get_app_config", lambda: SimpleNamespace(get_tool_config=lambda name: None))
+    if provider == "jina_ai":
+        monkeypatch.setattr(module.JinaClient, "crawl", AsyncMock(return_value=html))
+    elif provider == "infoquest":
+        monkeypatch.setattr(module, "_get_infoquest_client", lambda: SimpleNamespace(fetch=lambda url: html))
+    else:
+        client = SimpleNamespace(fetch_html_with_status=AsyncMock(return_value=BrowserlessFetchResult(html, "200", "OK")))
+        monkeypatch.setattr(module, "_get_browserless_client", lambda name: client)
+        monkeypatch.setattr(module, "_resolve_host_addresses", lambda host: [ip_address("93.184.216.34")])
+    result = await module.web_fetch_tool.ainvoke({"url": PAGE_URL})
+    assert "[Next](https://example.com/next)" in result
+    assert "[Reference](https://example.com/reference)" in result
+
+
+@pytest.mark.parametrize(
+    ("destination", "expected"),
+    [
+        ("../next", "https://example.com/next"),
+        ("/reference", "https://example.com/reference"),
+        ("?page=2", "https://example.com/docs/current?page=2"),
+        ("#section", "https://example.com/docs/current#section"),
+        ("//cdn.example.com/file", "https://cdn.example.com/file"),
+        ("https://other.example.com/file", "https://other.example.com/file"),
+        ("mailto:help@example.com", "mailto:help@example.com"),
+    ],
+)
+def test_extract_article_resolves_link_destinations(destination, expected):
+    article = ReadabilityExtractor().extract_article(_article(f'<a href="{destination}">Reference</a>'), url=PAGE_URL)
+    assert f"[Reference]({expected})" in article.to_markdown()
+
+
+def test_extract_article_resolves_images_and_relative_document_base():
+    article = ReadabilityExtractor().extract_article(
+        _article('<a href="next">Next</a> <img src="images/chart.png" alt="Chart">', head='<base href="../assets/">'),
+        url=PAGE_URL,
+    )
+    markdown = article.to_markdown()
+    assert "[Next](https://example.com/assets/next)" in markdown
+    assert "![Chart](https://example.com/assets/images/chart.png)" in markdown
+
+
+def test_extract_article_without_url_preserves_legacy_relative_links():
+    article = ReadabilityExtractor().extract_article(_article('<a href="../next">Next</a>'))
+    assert "[Next](../next)" in article.to_markdown()
+
+
+@pytest.mark.parametrize(
+    ("base", "expected"),
+    [
+        ("https://cdn.example.com/assets/", "https://cdn.example.com/assets/next"),
+        ("//cdn.example.com/assets/", "https://cdn.example.com/assets/next"),
+        ("ftp://files.example.com/assets/", "ftp://files.example.com/assets/next"),
+    ],
+)
+def test_extract_article_uses_first_document_base(base, expected):
+    article = ReadabilityExtractor().extract_article(
+        _article('<a href="next">Next</a>', head=f'<base href="{base}"><base href="https://other.example.com/">'),
+        url=PAGE_URL,
+    )
+    assert f"[Next]({expected})" in article.to_markdown()
+
+
+def test_python_extraction_fallback_preserves_article_text(monkeypatch):
+    import subprocess
+
+    from deerflow.utils import readability
+
+    original = readability.simple_json_from_html_string
+
+    def extract(html, *, use_readability):
+        if use_readability:
+            raise subprocess.CalledProcessError(1, "node")
+        return original(html, use_readability=False)
+
+    monkeypatch.setattr(readability, "simple_json_from_html_string", extract)
+    article = ReadabilityExtractor().extract_article(_article('<a href="../next">Next</a>'), url=PAGE_URL)
+    # The existing Python fallback strips link markup; preserve its text contract.
+    assert "Next" in article.to_markdown()
+    assert "This article explains the documentation" in article.to_markdown()
+
+
+@pytest.mark.parametrize("base", ["http://[broken", "data:text/plain,invalid", "javascript:void(0)", "about:blank", "mailto:help@example.com", "blob:https://example.com/id"])
+def test_invalid_document_base_does_not_lose_valid_relative_links(base):
+    article = ReadabilityExtractor().extract_article(
+        _article('<a href="../next">Next</a>', head=f'<base href="{base}">'),
+        url=PAGE_URL,
+    )
+    assert "[Next](https://example.com/next)" in article.to_markdown()
+
+
+@pytest.mark.parametrize(
+    "fragment",
+    [
+        "<b>Bold <i>mixed</b> italics</i>",
+        "<p>Before<div>Block</div>after</p>",
+        "<table>Outside<tr><td>Cell</td></tr></table>",
+        '<a href="https://example.com/next">Outer <a href="https://example.com/inner">Inner</a> Tail</a>',
+    ],
+)
+def test_url_resolution_preserves_malformed_markup_extraction(fragment):
+    html = _article(fragment + '<a href="../next">Next</a>')
+    extractor = ReadabilityExtractor()
+    assert extractor.extract_article(html, url=PAGE_URL).to_markdown() == extractor.extract_article(html).to_markdown().replace("(../next)", "(https://example.com/next)")
+
+
+def test_document_base_skips_target_only_base():
+    html = _article('<a href="next">Next</a>', head='<base target="_blank"><base href="https://cdn.example.com/assets/">')
+    assert "[Next](https://cdn.example.com/assets/next)" in ReadabilityExtractor().extract_article(html, url=PAGE_URL).to_markdown()
+
+
+@pytest.mark.parametrize("destination", ["href=../next", "HREF='../next'", 'href="../next?x=1&amp;y=2"', 'href = "../next" href="/ignored"'])
+def test_rewriter_changes_only_destination_values(destination):
+    from deerflow.utils.readability import _resolve_html_urls
+
+    html = """<!-- <a href="/comment"> -->\n<script>const sample = "<a href=/script>";</script>\n""" + f"<p><b>Misnested <i>text</b> tail</i> <a {destination}>Next</a></p>"
+    result = _resolve_html_urls(html, PAGE_URL)
+    assert result.startswith(html[: html.index("<p>")])
+    assert "<p><b>Misnested <i>text</b> tail</i>" in result
+    assert '"https://example.com/next' in result
+    if 'href="/ignored"' in html:
+        assert 'href="/ignored"' in result
+
+
+@pytest.mark.parametrize("tag", ["textarea", "title", "xmp", "iframe", "noembed", "noframes"])
+def test_rewriter_preserves_link_examples_in_text_elements(tag):
+    from deerflow.utils.readability import _resolve_html_urls
+
+    example = f'<{tag}><a href="/literal">Example</a></{tag}>'
+    html = example + '<a href="../next">Next</a>'
+    assert _resolve_html_urls(html, PAGE_URL) == example + '<a href="https://example.com/next">Next</a>'
+
+
+@pytest.mark.parametrize("attribute", ["href", 'href=""', "href=''", 'href href="/ignored"'])
+def test_empty_destination_uses_document_base(attribute):
+    from deerflow.utils.readability import _resolve_html_urls
+
+    html = f"<a {attribute}>Current</a>"
+    assert f'href="{PAGE_URL}"' in _resolve_html_urls(html, PAGE_URL)
+
+
+def test_textarea_with_script_example_does_not_hide_following_links():
+    from deerflow.utils.readability import _resolve_html_urls
+
+    example = '<textarea><script><a href="/literal"></textarea>'
+    html = example + '<a href="../next">Next</a>'
+    assert _resolve_html_urls(html, PAGE_URL) == example + '<a href="https://example.com/next">Next</a>'
+
+
+def test_pages_without_base_skip_html5_tree_construction(monkeypatch):
+    from deerflow.utils import readability
+
+    def unexpected_parse(*args, **kwargs):
+        pytest.fail("A page without a base prefix must not build an HTML5 tree")
+
+    monkeypatch.setattr(readability, "BeautifulSoup", unexpected_parse)
+    html = '<p>Guide</p><a href="../next">Next</a>'
+    assert readability._resolve_html_urls(html, PAGE_URL) == '<p>Guide</p><a href="https://example.com/next">Next</a>'
+
+
+@pytest.mark.parametrize(
+    "head",
+    [
+        '<BaSe href="https://cdn.example.com/">',
+        '<!-- <base href="/ignored/"> --><BASE href="https://cdn.example.com/">',
+    ],
+)
+def test_base_precheck_retains_case_insensitive_tree_selection(head):
+    from deerflow.utils.readability import _resolve_html_urls
+
+    html = _article('<a href="next">Next</a>', head=head)
+    assert '<a href="https://cdn.example.com/next">Next</a>' in _resolve_html_urls(html, PAGE_URL)
+
+
+@pytest.mark.parametrize(
+    "head",
+    [
+        '<!-- <base href="/ignored/"> -->',
+        """<script>const sample = '<base href="/ignored/">';</script>""",
+        '<title>Example &lt;base href="/ignored/"&gt;</title>',
+    ],
+)
+def test_base_precheck_false_positives_do_not_override_page_url(head):
+    from deerflow.utils.readability import _resolve_html_urls
+
+    html = _article('<a href="../next">Next</a>', head=head)
+    assert '<a href="https://example.com/next">Next</a>' in _resolve_html_urls(html, PAGE_URL)

backend/uv.lock

@@ -926,6 +926,7 @@ dependencies = [
     { name = "agent-sandbox" },
     { name = "aiosqlite" },
     { name = "alembic" },
+    { name = "beautifulsoup4" },
     { name = "croniter" },
     { name = "cryptography" },
     { name = "ddgs" },
@@ -935,6 +936,7 @@ dependencies = [
     { name = "e2b-code-interpreter" },
     { name = "exa-py" },
     { name = "firecrawl-py" },
+    { name = "html5lib" },
     { name = "httpx" },
     { name = "kubernetes" },
     { name = "langchain" },
@@ -1008,6 +1010,7 @@ requires-dist = [
     { name = "aiosqlite", specifier = ">=0.19" },
     { name = "alembic", specifier = ">=1.13" },
     { name = "asyncpg", marker = "extra == 'postgres'", specifier = ">=0.29" },
+    { name = "beautifulsoup4", specifier = ">=4.12" },
     { name = "boxlite", marker = "extra == 'boxlite'", specifier = ">=0.9.7" },
     { name = "croniter", specifier = ">=6.0.0" },
     { name = "cryptography", specifier = ">=50.0.0" },
@@ -1018,6 +1021,7 @@ requires-dist = [
     { name = "e2b-code-interpreter", specifier = ">=2.8.0" },
     { name = "exa-py", specifier = ">=1.0.0" },
     { name = "firecrawl-py", specifier = ">=1.15.0" },
+    { name = "html5lib", specifier = ">=1.1" },
     { name = "httpx", specifier = ">=0.28.0" },
     { name = "jieba", marker = "extra == 'memory-zh'", specifier = ">=0.42.1" },
     { name = "kubernetes", specifier = ">=30.0.0" },