"""Fetched HTML must preserve usable destinations in model-visible Markdown."""
import importlib
from ipaddress import ip_address
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from deerflow.community.browserless.browserless_client import BrowserlessFetchResult
from deerflow.utils.readability import ReadabilityExtractor
PAGE_URL = "https://example.com/docs/current"
def _article(links: str, *, head: str = "") -> str:
paragraph = "This article explains the documentation in detail, with enough ordinary prose for the real readability extractor to retain the content and its related references.
"
return f"Guide{head}{paragraph * 5}{links}
"
@pytest.mark.parametrize("provider", ["jina_ai", "browserless", "infoquest"])
@pytest.mark.anyio
async def test_web_fetch_resolves_relative_links_through_real_extraction(monkeypatch, provider):
module = importlib.import_module(f"deerflow.community.{provider}.tools")
html = _article('Next Reference')
monkeypatch.setattr(module, "get_app_config", lambda: SimpleNamespace(get_tool_config=lambda name: None))
if provider == "jina_ai":
monkeypatch.setattr(module.JinaClient, "crawl", AsyncMock(return_value=html))
elif provider == "infoquest":
monkeypatch.setattr(module, "_get_infoquest_client", lambda: SimpleNamespace(fetch=lambda url: html))
else:
client = SimpleNamespace(fetch_html_with_status=AsyncMock(return_value=BrowserlessFetchResult(html, "200", "OK")))
monkeypatch.setattr(module, "_get_browserless_client", lambda name: client)
monkeypatch.setattr(module, "_resolve_host_addresses", lambda host: [ip_address("93.184.216.34")])
result = await module.web_fetch_tool.ainvoke({"url": PAGE_URL})
assert "[Next](https://example.com/next)" in result
assert "[Reference](https://example.com/reference)" in result
@pytest.mark.parametrize(
("destination", "expected"),
[
("../next", "https://example.com/next"),
("/reference", "https://example.com/reference"),
("?page=2", "https://example.com/docs/current?page=2"),
("#section", "https://example.com/docs/current#section"),
("//cdn.example.com/file", "https://cdn.example.com/file"),
("https://other.example.com/file", "https://other.example.com/file"),
("mailto:help@example.com", "mailto:help@example.com"),
],
)
def test_extract_article_resolves_link_destinations(destination, expected):
article = ReadabilityExtractor().extract_article(_article(f'Reference'), url=PAGE_URL)
assert f"[Reference]({expected})" in article.to_markdown()
def test_extract_article_resolves_images_and_relative_document_base():
article = ReadabilityExtractor().extract_article(
_article('Next 
', head=''),
url=PAGE_URL,
)
markdown = article.to_markdown()
assert "[Next](https://example.com/assets/next)" in markdown
assert "" in markdown
def test_extract_article_without_url_preserves_legacy_relative_links():
article = ReadabilityExtractor().extract_article(_article('Next'))
assert "[Next](../next)" in article.to_markdown()
@pytest.mark.parametrize(
("base", "expected"),
[
("https://cdn.example.com/assets/", "https://cdn.example.com/assets/next"),
("//cdn.example.com/assets/", "https://cdn.example.com/assets/next"),
("ftp://files.example.com/assets/", "ftp://files.example.com/assets/next"),
],
)
def test_extract_article_uses_first_document_base(base, expected):
article = ReadabilityExtractor().extract_article(
_article('Next', head=f''),
url=PAGE_URL,
)
assert f"[Next]({expected})" in article.to_markdown()
def test_python_extraction_fallback_preserves_article_text(monkeypatch):
import subprocess
from deerflow.utils import readability
original = readability.simple_json_from_html_string
def extract(html, *, use_readability):
if use_readability:
raise subprocess.CalledProcessError(1, "node")
return original(html, use_readability=False)
monkeypatch.setattr(readability, "simple_json_from_html_string", extract)
article = ReadabilityExtractor().extract_article(_article('Next'), url=PAGE_URL)
# The existing Python fallback strips link markup; preserve its text contract.
assert "Next" in article.to_markdown()
assert "This article explains the documentation" in article.to_markdown()
@pytest.mark.parametrize("base", ["http://[broken", "data:text/plain,invalid", "javascript:void(0)", "about:blank", "mailto:help@example.com", "blob:https://example.com/id"])
def test_invalid_document_base_does_not_lose_valid_relative_links(base):
article = ReadabilityExtractor().extract_article(
_article('Next', head=f''),
url=PAGE_URL,
)
assert "[Next](https://example.com/next)" in article.to_markdown()
@pytest.mark.parametrize(
"fragment",
[
"Bold mixed italics",
"Before
Block
after",
"",
'Outer Inner Tail',
],
)
def test_url_resolution_preserves_malformed_markup_extraction(fragment):
html = _article(fragment + 'Next')
extractor = ReadabilityExtractor()
assert extractor.extract_article(html, url=PAGE_URL).to_markdown() == extractor.extract_article(html).to_markdown().replace("(../next)", "(https://example.com/next)")
def test_document_base_skips_target_only_base():
html = _article('Next', head='')
assert "[Next](https://cdn.example.com/assets/next)" in ReadabilityExtractor().extract_article(html, url=PAGE_URL).to_markdown()
@pytest.mark.parametrize("destination", ["href=../next", "HREF='../next'", 'href="../next?x=1&y=2"', 'href = "../next" href="/ignored"'])
def test_rewriter_changes_only_destination_values(destination):
from deerflow.utils.readability import _resolve_html_urls
html = """\n\n""" + f"Misnested text tail Next
"
result = _resolve_html_urls(html, PAGE_URL)
assert result.startswith(html[: html.index("")])
assert "
Misnested text tail" in result
assert '"https://example.com/next' in result
if 'href="/ignored"' in html:
assert 'href="/ignored"' in result
@pytest.mark.parametrize("tag", ["textarea", "title", "xmp", "iframe", "noembed", "noframes"])
def test_rewriter_preserves_link_examples_in_text_elements(tag):
from deerflow.utils.readability import _resolve_html_urls
example = f'<{tag}>Example'
html = example + 'Next'
assert _resolve_html_urls(html, PAGE_URL) == example + 'Next'
@pytest.mark.parametrize("attribute", ["href", 'href=""', "href=''", 'href href="/ignored"'])
def test_empty_destination_uses_document_base(attribute):
from deerflow.utils.readability import _resolve_html_urls
html = f"Current"
assert f'href="{PAGE_URL}"' in _resolve_html_urls(html, PAGE_URL)
def test_textarea_with_script_example_does_not_hide_following_links():
from deerflow.utils.readability import _resolve_html_urls
example = ''
html = example + 'Next'
assert _resolve_html_urls(html, PAGE_URL) == example + 'Next'
def test_pages_without_base_skip_html5_tree_construction(monkeypatch):
from deerflow.utils import readability
def unexpected_parse(*args, **kwargs):
pytest.fail("A page without a base prefix must not build an HTML5 tree")
monkeypatch.setattr(readability, "BeautifulSoup", unexpected_parse)
html = '
Guide
Next'
assert readability._resolve_html_urls(html, PAGE_URL) == 'Guide
Next'
@pytest.mark.parametrize(
"head",
[
'',
'',
],
)
def test_base_precheck_retains_case_insensitive_tree_selection(head):
from deerflow.utils.readability import _resolve_html_urls
html = _article('Next', head=head)
assert 'Next' in _resolve_html_urls(html, PAGE_URL)
@pytest.mark.parametrize(
"head",
[
'',
"""""",
'Example <base href="/ignored/">',
],
)
def test_base_precheck_false_positives_do_not_override_page_url(head):
from deerflow.utils.readability import _resolve_html_urls
html = _article('Next', head=head)
assert 'Next' in _resolve_html_urls(html, PAGE_URL)