From b809b7bc7de43b4f0122eb26d7713e27aa1d36aa Mon Sep 17 00:00:00 2001 From: tiammomo <26957354+tiammomo@users.noreply.github.com> Date: Fri, 11 Sep 2026 18:36:58 +0800 Subject: [PATCH] fix(web-fetch): resolve relative URLs in extracted Markdown (#5310) * fix(web-fetch): resolve relative URLs in extracted Markdown Pass the request URL through Jina and Browserless extraction and resolve link/image destinations before Readability removes document base tags. Preserve the optional legacy API and fallback text behavior. Fixes #5307 Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> * fix(web-fetch): address provider and base URL review feedback Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> * fix(web-fetch): preserve HTML source when resolving destinations Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --------- Signed-off-by: tiammomo <26957354+tiammomo@users.noreply.github.com> --- README.md | 2 + backend/AGENTS.md | 5 +- .../deerflow/community/browserless/tools.py | 2 +- .../deerflow/community/infoquest/tools.py | 2 +- .../deerflow/community/jina_ai/tools.py | 2 +- .../harness/deerflow/utils/readability.py | 100 ++++++++- backend/packages/harness/pyproject.toml | 2 + .../tests/test_web_fetch_relative_links.py | 211 ++++++++++++++++++ backend/uv.lock | 4 + 9 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 backend/tests/test_web_fetch_relative_links.py diff --git a/README.md b/README.md index 35b845f75..b16b568d4 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,8 @@ That prompt is intended for coding agents. It tells the agent to clone the repo The wizard also lets you configure an optional web search provider, or skip it for now. + Jina, Browserless, and InfoQuest web fetches resolve relative links and image sources using the requested page URL (or a usable HTML base URL), so returned Markdown includes complete destinations. Link resolution preserves the surrounding HTML source, including malformed-page formatting. + Run `make doctor` at any time to verify your setup and get actionable fix hints. If you are opening a GitHub issue about a local setup or runtime problem, run `make support-bundle`. The command prints reporter next steps, writes a diff --git a/backend/AGENTS.md b/backend/AGENTS.md index a0fc7d93f..c95ed29f7 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -265,8 +265,9 @@ Direct pytest collection or execution of `tests/test_client_live.py` remains skipped unless `DEER_FLOW_RUN_LIVE_TESTS=1` is set. Do not add that opt-in to default CI workflows. -Jina logging tests isolate missing-key warnings with dummy keys (`tests/test_jina_client.py`). -InfoQuest HTTP calls share a 30s connect/read inactivity timeout, separate from remote crawl timeouts; see `tests/test_infoquest_http_timeout.py`. +Jina logging tests use dummy keys (`tests/test_jina_client.py`). +Jina/Browserless/InfoQuest resolve URLs without rebuilding HTML. +InfoQuest connect/read timeout is 30s, separate from crawl timeouts (`tests/test_infoquest_http_timeout.py`). ### Running the Full Application diff --git a/backend/packages/harness/deerflow/community/browserless/tools.py b/backend/packages/harness/deerflow/community/browserless/tools.py index 7fd981c94..d2784d5a8 100644 --- a/backend/packages/harness/deerflow/community/browserless/tools.py +++ b/backend/packages/harness/deerflow/community/browserless/tools.py @@ -271,7 +271,7 @@ async def web_fetch_tool(url: str) -> str: if isinstance(result, str): return result - article = await asyncio.to_thread(_readability_extractor.extract_article, result.html) + article = await asyncio.to_thread(_readability_extractor.extract_article, result.html, url=url) return f"{article.to_markdown()[:4096]}{_target_status_warning(result)}" except Exception as e: diff --git a/backend/packages/harness/deerflow/community/infoquest/tools.py b/backend/packages/harness/deerflow/community/infoquest/tools.py index 49fa1de52..c4cecb7cd 100644 --- a/backend/packages/harness/deerflow/community/infoquest/tools.py +++ b/backend/packages/harness/deerflow/community/infoquest/tools.py @@ -70,7 +70,7 @@ def web_fetch_tool(url: str) -> str: result = client.fetch(url) if result.startswith("Error: "): return result - article = readability_extractor.extract_article(result) + article = readability_extractor.extract_article(result, url=url) return article.to_markdown()[:4096] diff --git a/backend/packages/harness/deerflow/community/jina_ai/tools.py b/backend/packages/harness/deerflow/community/jina_ai/tools.py index 81c8370d6..913f420bb 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/tools.py +++ b/backend/packages/harness/deerflow/community/jina_ai/tools.py @@ -64,5 +64,5 @@ async def web_fetch_tool(url: str) -> str: html_content = await jina_client.crawl(url, return_format="html", timeout=timeout, proxy=proxy, trust_env=trust_env) if isinstance(html_content, str) and html_content.startswith("Error:"): return html_content - article = await asyncio.to_thread(readability_extractor.extract_article, html_content) + article = await asyncio.to_thread(readability_extractor.extract_article, html_content, url=url) return article.to_markdown()[:4096] diff --git a/backend/packages/harness/deerflow/utils/readability.py b/backend/packages/harness/deerflow/utils/readability.py index e905f71ee..e25093aa8 100644 --- a/backend/packages/harness/deerflow/utils/readability.py +++ b/backend/packages/harness/deerflow/utils/readability.py @@ -1,8 +1,11 @@ import logging import re import subprocess -from urllib.parse import urljoin +from html import escape, unescape +from html.parser import HTMLParser +from urllib.parse import urljoin, urlparse, uses_relative +from bs4 import BeautifulSoup from markdownify import markdownify as md from readabilipy import simple_json_from_html_string @@ -55,8 +58,101 @@ class Article: return content +_BASE_TAG_RE = re.compile(r" str: + """Resolve destinations before extraction can discard the document's base tag.""" + # A base element requires a literal start-tag prefix. False positives in + # comments or text elements still go through HTML5 tree construction. + base = BeautifulSoup(html, "html5lib").find("base", href=True) if _BASE_TAG_RE.search(html) else None + base_url = url + if base is not None: + try: + candidate = urljoin(url, str(base["href"]).strip()) + # Keep only bases urljoin can resolve relative paths against. + # Opaque bases fall back to the fetched URL; hierarchical FTP remains valid. + if urlparse(candidate).scheme in uses_relative: + base_url = candidate + except ValueError: + pass # An invalid base must not prevent extraction of the page. + resolver = _DestinationRewriter(html, base_url) + resolver.feed(html) + resolver.close() + return resolver.result() + + +# Tokenize attributes only inside a start tag identified by HTMLParser. Keeping +# source spans avoids rebuilding malformed markup before jsdom parses it. +_ATTRIBUTE_RE = re.compile(r"""([^\s/>=]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]*))?""") + + +class _DestinationRewriter(HTMLParser): + # Treat link examples inside text-only elements as data, including nested + # script-looking text; only the matching closing tag resumes tokenization. + CDATA_CONTENT_ELEMENTS = ("script", "style", "textarea", "title", "xmp", "iframe", "noembed", "noframes", "plaintext") + + def __init__(self, html: str, base_url: str): + super().__init__(convert_charrefs=False) + self.html = html + self.base_url = base_url + self.text_element: str | None = None + self.line_offsets = [0, *(match.end() for match in re.finditer("\n", html))] + self.replacements: list[tuple[int, int, str]] = [] + + def handle_starttag(self, tag, attrs): + if self.text_element is not None: + return + if tag in {"textarea", "title", "xmp", "iframe", "noembed", "noframes", "plaintext"}: + self.text_element = tag + return + attribute = {"a": "href", "img": "src"}.get(tag) + if attribute is None: + return + raw = self.get_starttag_text() + tag_end = re.match(r"<[^\s/>]+", raw).end() + for match in _ATTRIBUTE_RE.finditer(raw, tag_end): + if match.group(1).lower() != attribute: + continue + value = match.group(2) + if value is not None: + original = unescape(value[1:-1] if value.startswith(('"', "'")) else value) + try: + resolved = urljoin(self.base_url, original.strip()) + except ValueError: + return + if resolved != original: + line, column = self.getpos() + offset = self.line_offsets[line - 1] + column + self.replacements.append((offset + match.start(2), offset + match.end(2), '"' + escape(resolved, quote=True) + '"')) + else: + line, column = self.getpos() + offset = self.line_offsets[line - 1] + column + match.end(1) + self.replacements.append((offset, offset, '="' + escape(self.base_url, quote=True) + '"')) + # Browsers use the first duplicate attribute, including a bare one. + return + + def handle_endtag(self, tag): + if tag == self.text_element and tag != "plaintext": + self.text_element = None + + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + + def result(self) -> str: + parts = [] + cursor = 0 + for start, end, value in self.replacements: + parts.extend((self.html[cursor:start], value)) + cursor = end + parts.append(self.html[cursor:]) + return "".join(parts) + + class ReadabilityExtractor: - def extract_article(self, html: str) -> Article: + def extract_article(self, html: str, *, url: str | None = None) -> Article: + if url: + html = _resolve_html_urls(html, url) try: article = simple_json_from_html_string(html, use_readability=True) except (subprocess.CalledProcessError, FileNotFoundError) as exc: diff --git a/backend/packages/harness/pyproject.toml b/backend/packages/harness/pyproject.toml index 08d4c5688..aa98c70e5 100644 --- a/backend/packages/harness/pyproject.toml +++ b/backend/packages/harness/pyproject.toml @@ -6,6 +6,8 @@ requires-python = ">=3.12" dependencies = [ "agent-client-protocol>=0.4.0", "agent-sandbox>=0.0.30", + "beautifulsoup4>=4.12", + "html5lib>=1.1", "croniter>=6.0.0", # Exact pin by design (extension-system version contract): the host pins # the contract version it implements, extensions declare ranges. A range diff --git a/backend/tests/test_web_fetch_relative_links.py b/backend/tests/test_web_fetch_relative_links.py new file mode 100644 index 000000000..4693ca6bf --- /dev/null +++ b/backend/tests/test_web_fetch_relative_links.py @@ -0,0 +1,211 @@ +"""Fetched HTML must preserve usable destinations in model-visible Markdown.""" + +import importlib +from ipaddress import ip_address +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from deerflow.community.browserless.browserless_client import BrowserlessFetchResult +from deerflow.utils.readability import ReadabilityExtractor + +PAGE_URL = "https://example.com/docs/current" + + +def _article(links: str, *, head: str = "") -> str: + paragraph = "

This article explains the documentation in detail, with enough ordinary prose for the real readability extractor to retain the content and its related references.

" + return f"Guide{head}
{paragraph * 5}

{links}

" + + +@pytest.mark.parametrize("provider", ["jina_ai", "browserless", "infoquest"]) +@pytest.mark.anyio +async def test_web_fetch_resolves_relative_links_through_real_extraction(monkeypatch, provider): + module = importlib.import_module(f"deerflow.community.{provider}.tools") + html = _article('Next Reference') + monkeypatch.setattr(module, "get_app_config", lambda: SimpleNamespace(get_tool_config=lambda name: None)) + if provider == "jina_ai": + monkeypatch.setattr(module.JinaClient, "crawl", AsyncMock(return_value=html)) + elif provider == "infoquest": + monkeypatch.setattr(module, "_get_infoquest_client", lambda: SimpleNamespace(fetch=lambda url: html)) + else: + client = SimpleNamespace(fetch_html_with_status=AsyncMock(return_value=BrowserlessFetchResult(html, "200", "OK"))) + monkeypatch.setattr(module, "_get_browserless_client", lambda name: client) + monkeypatch.setattr(module, "_resolve_host_addresses", lambda host: [ip_address("93.184.216.34")]) + result = await module.web_fetch_tool.ainvoke({"url": PAGE_URL}) + assert "[Next](https://example.com/next)" in result + assert "[Reference](https://example.com/reference)" in result + + +@pytest.mark.parametrize( + ("destination", "expected"), + [ + ("../next", "https://example.com/next"), + ("/reference", "https://example.com/reference"), + ("?page=2", "https://example.com/docs/current?page=2"), + ("#section", "https://example.com/docs/current#section"), + ("//cdn.example.com/file", "https://cdn.example.com/file"), + ("https://other.example.com/file", "https://other.example.com/file"), + ("mailto:help@example.com", "mailto:help@example.com"), + ], +) +def test_extract_article_resolves_link_destinations(destination, expected): + article = ReadabilityExtractor().extract_article(_article(f'Reference'), url=PAGE_URL) + assert f"[Reference]({expected})" in article.to_markdown() + + +def test_extract_article_resolves_images_and_relative_document_base(): + article = ReadabilityExtractor().extract_article( + _article('Next Chart', head=''), + url=PAGE_URL, + ) + markdown = article.to_markdown() + assert "[Next](https://example.com/assets/next)" in markdown + assert "![Chart](https://example.com/assets/images/chart.png)" in markdown + + +def test_extract_article_without_url_preserves_legacy_relative_links(): + article = ReadabilityExtractor().extract_article(_article('Next')) + assert "[Next](../next)" in article.to_markdown() + + +@pytest.mark.parametrize( + ("base", "expected"), + [ + ("https://cdn.example.com/assets/", "https://cdn.example.com/assets/next"), + ("//cdn.example.com/assets/", "https://cdn.example.com/assets/next"), + ("ftp://files.example.com/assets/", "ftp://files.example.com/assets/next"), + ], +) +def test_extract_article_uses_first_document_base(base, expected): + article = ReadabilityExtractor().extract_article( + _article('Next', head=f''), + url=PAGE_URL, + ) + assert f"[Next]({expected})" in article.to_markdown() + + +def test_python_extraction_fallback_preserves_article_text(monkeypatch): + import subprocess + + from deerflow.utils import readability + + original = readability.simple_json_from_html_string + + def extract(html, *, use_readability): + if use_readability: + raise subprocess.CalledProcessError(1, "node") + return original(html, use_readability=False) + + monkeypatch.setattr(readability, "simple_json_from_html_string", extract) + article = ReadabilityExtractor().extract_article(_article('Next'), url=PAGE_URL) + # The existing Python fallback strips link markup; preserve its text contract. + assert "Next" in article.to_markdown() + assert "This article explains the documentation" in article.to_markdown() + + +@pytest.mark.parametrize("base", ["http://[broken", "data:text/plain,invalid", "javascript:void(0)", "about:blank", "mailto:help@example.com", "blob:https://example.com/id"]) +def test_invalid_document_base_does_not_lose_valid_relative_links(base): + article = ReadabilityExtractor().extract_article( + _article('Next', head=f''), + url=PAGE_URL, + ) + assert "[Next](https://example.com/next)" in article.to_markdown() + + +@pytest.mark.parametrize( + "fragment", + [ + "Bold mixed italics", + "

Before

Block
after

", + "Outside
Cell
", + 'Outer Inner Tail', + ], +) +def test_url_resolution_preserves_malformed_markup_extraction(fragment): + html = _article(fragment + 'Next') + extractor = ReadabilityExtractor() + assert extractor.extract_article(html, url=PAGE_URL).to_markdown() == extractor.extract_article(html).to_markdown().replace("(../next)", "(https://example.com/next)") + + +def test_document_base_skips_target_only_base(): + html = _article('Next', head='') + assert "[Next](https://cdn.example.com/assets/next)" in ReadabilityExtractor().extract_article(html, url=PAGE_URL).to_markdown() + + +@pytest.mark.parametrize("destination", ["href=../next", "HREF='../next'", 'href="../next?x=1&y=2"', 'href = "../next" href="/ignored"']) +def test_rewriter_changes_only_destination_values(destination): + from deerflow.utils.readability import _resolve_html_urls + + html = """\n\n""" + f"

Misnested text tail Next

" + result = _resolve_html_urls(html, PAGE_URL) + assert result.startswith(html[: html.index("

")]) + assert "

Misnested text tail" in result + assert '"https://example.com/next' in result + if 'href="/ignored"' in html: + assert 'href="/ignored"' in result + + +@pytest.mark.parametrize("tag", ["textarea", "title", "xmp", "iframe", "noembed", "noframes"]) +def test_rewriter_preserves_link_examples_in_text_elements(tag): + from deerflow.utils.readability import _resolve_html_urls + + example = f'<{tag}>Example' + html = example + 'Next' + assert _resolve_html_urls(html, PAGE_URL) == example + 'Next' + + +@pytest.mark.parametrize("attribute", ["href", 'href=""', "href=''", 'href href="/ignored"']) +def test_empty_destination_uses_document_base(attribute): + from deerflow.utils.readability import _resolve_html_urls + + html = f"Current" + assert f'href="{PAGE_URL}"' in _resolve_html_urls(html, PAGE_URL) + + +def test_textarea_with_script_example_does_not_hide_following_links(): + from deerflow.utils.readability import _resolve_html_urls + + example = '' + html = example + 'Next' + assert _resolve_html_urls(html, PAGE_URL) == example + 'Next' + + +def test_pages_without_base_skip_html5_tree_construction(monkeypatch): + from deerflow.utils import readability + + def unexpected_parse(*args, **kwargs): + pytest.fail("A page without a base prefix must not build an HTML5 tree") + + monkeypatch.setattr(readability, "BeautifulSoup", unexpected_parse) + html = '

Guide

Next' + assert readability._resolve_html_urls(html, PAGE_URL) == '

Guide

Next' + + +@pytest.mark.parametrize( + "head", + [ + '', + '', + ], +) +def test_base_precheck_retains_case_insensitive_tree_selection(head): + from deerflow.utils.readability import _resolve_html_urls + + html = _article('Next', head=head) + assert 'Next' in _resolve_html_urls(html, PAGE_URL) + + +@pytest.mark.parametrize( + "head", + [ + '', + """""", + 'Example <base href="/ignored/">', + ], +) +def test_base_precheck_false_positives_do_not_override_page_url(head): + from deerflow.utils.readability import _resolve_html_urls + + html = _article('Next', head=head) + assert 'Next' in _resolve_html_urls(html, PAGE_URL) diff --git a/backend/uv.lock b/backend/uv.lock index 565eaf73e..44ad076f6 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -926,6 +926,7 @@ dependencies = [ { name = "agent-sandbox" }, { name = "aiosqlite" }, { name = "alembic" }, + { name = "beautifulsoup4" }, { name = "croniter" }, { name = "cryptography" }, { name = "ddgs" }, @@ -935,6 +936,7 @@ dependencies = [ { name = "e2b-code-interpreter" }, { name = "exa-py" }, { name = "firecrawl-py" }, + { name = "html5lib" }, { name = "httpx" }, { name = "kubernetes" }, { name = "langchain" }, @@ -1008,6 +1010,7 @@ requires-dist = [ { name = "aiosqlite", specifier = ">=0.19" }, { name = "alembic", specifier = ">=1.13" }, { name = "asyncpg", marker = "extra == 'postgres'", specifier = ">=0.29" }, + { name = "beautifulsoup4", specifier = ">=4.12" }, { name = "boxlite", marker = "extra == 'boxlite'", specifier = ">=0.9.7" }, { name = "croniter", specifier = ">=6.0.0" }, { name = "cryptography", specifier = ">=50.0.0" }, @@ -1018,6 +1021,7 @@ requires-dist = [ { name = "e2b-code-interpreter", specifier = ">=2.8.0" }, { name = "exa-py", specifier = ">=1.0.0" }, { name = "firecrawl-py", specifier = ">=1.15.0" }, + { name = "html5lib", specifier = ">=1.1" }, { name = "httpx", specifier = ">=0.28.0" }, { name = "jieba", marker = "extra == 'memory-zh'", specifier = ">=0.42.1" }, { name = "kubernetes", specifier = ">=30.0.0" },