From 54a09c467ed983a65f3898d7e72a427df0e97464 Mon Sep 17 00:00:00 2001 From: Tsai Yuan <136825049+nothingyuancando@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:33:41 +0800 Subject: [PATCH] fix(readability): cache unavailable Node extraction (#7617) Co-authored-by: Willem Jiang --- CHANGELOG.md | 7 + CHANGELOG_zh.md | 6 + README.md | 7 + .../packages/harness/deerflow/utils/AGENTS.md | 15 ++ .../harness/deerflow/utils/readability.py | 76 ++++++++-- backend/tests/test_readability.py | 134 ++++++++++++++++++ .../tests/test_web_fetch_relative_links.py | 3 + 7 files changed, 235 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba13c6219e..13bb77f71c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1402,6 +1402,12 @@ As of 2026-10-21, the milestone includes **829 merged pull requests**. ### Fixed +- **web fetch:** Cache unavailable Readability.js probes and stop retrying the + Node extractor after a process-launch failure. Installations without a + working Node/npm toolchain now use the existing link-preserving Python + fallback on later fetches without repeating the failed subprocess probe; + document-specific non-zero exits still retry Node on later pages. ([#5973]) + - **scripts:** `make dev` and `make start` no longer run the checkout `.env` as a shell script. `serve.sh` sourced it, so `$$` became the shell's PID, backticks and `$(...)` ran commands, and an unquoted value with a space ran @@ -12602,6 +12608,7 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#5965]: https://github.com/bytedance/deer-flow/pull/5965 [#5966]: https://github.com/bytedance/deer-flow/pull/5966 [#5968]: https://github.com/bytedance/deer-flow/pull/5968 +[#5973]: https://github.com/bytedance/deer-flow/issues/5973 [#5977]: https://github.com/bytedance/deer-flow/pull/5977 [#5978]: https://github.com/bytedance/deer-flow/pull/5978 [#5980]: https://github.com/bytedance/deer-flow/pull/5980 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index a310aac91f..30f7f5afd4 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -1116,6 +1116,11 @@ ### 修复 +- **网页抓取：** 缓存 Readability.js 不可用的探测结果，并在 Node 抽取器首次出现 + 进程启动失败后停止重试。没有可用 Node/npm 工具链的安装环境会在后续抓取中直接使用 + 现有且保留链接的 Python 回退，不再重复执行失败的子进程探测；单篇文档导致的非零退出 + 仍会在后续页面重试 Node。([#5973]) + - **脚本：** `make dev` 和 `make start` 不再把检出目录的 `.env` 当作 shell 脚本执行。此前 `serve.sh` 用 `source` 读取它，`$$` 会变成 shell 的 PID，反引号和 `$(...)` 会执行命令，未加引号 且含空格的值会把第二个词当作命令运行，启动脚本以退出码 127 中止。这些值还会覆盖 Gateway @@ -9926,6 +9931,7 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#5965]: https://github.com/bytedance/deer-flow/pull/5965 [#5966]: https://github.com/bytedance/deer-flow/pull/5966 [#5968]: https://github.com/bytedance/deer-flow/pull/5968 +[#5973]: https://github.com/bytedance/deer-flow/issues/5973 [#5977]: https://github.com/bytedance/deer-flow/pull/5977 [#5978]: https://github.com/bytedance/deer-flow/pull/5978 [#5980]: https://github.com/bytedance/deer-flow/pull/5980 diff --git a/README.md b/README.md index bab1317d11..97151467a9 100644 --- a/README.md +++ b/README.md @@ -2041,6 +2041,13 @@ Extra requests may consume quota or incur cost. See When using Tavily for `web_fetch`, extracted pages without a title use their URL as the heading; their content remains available to the agent. +For providers that use the shared Readability extractor, Readability.js is +optional. DeerFlow caches an unavailable Node.js toolchain or failed process +launch and uses the existing Python fallback for later fetches instead of +repeatedly starting a missing toolchain. A non-zero extraction exit falls back +only for that document, because malformed or oversized input can fail while +the toolchain remains healthy. The fallback keeps resolved article links in +the returned Markdown. Chat tool-step titles accept leading blank lines and up to three spaces before a page's first H1 heading. Indented code, including mixed spaces and tabs, is not used as a title; the tool step falls back to the URL. diff --git a/backend/packages/harness/deerflow/utils/AGENTS.md b/backend/packages/harness/deerflow/utils/AGENTS.md index 56bcc771df..777c9eccab 100644 --- a/backend/packages/harness/deerflow/utils/AGENTS.md +++ b/backend/packages/harness/deerflow/utils/AGENTS.md @@ -1,3 +1,18 @@ +### Readability Fallback + +`readability.py` caches the optional Readability.js availability probe. If the +Node extractor is unavailable, its probe raises, or an attempted extraction +cannot launch Node, subsequent calls in that process use readabilipy's Python +fallback without retrying Node. A non-zero Node exit can be document-specific, +so it falls back only for that document and later documents retry Node. If a +future readabilipy version removes the private availability probe, log that +once at import and attempt extraction instead of silently disabling Node. Keep +successful Node extraction unchanged and allow unexpected parser/programming +exceptions to propagate. The resolved-body fallback preserves article links +when Python extraction returns the original document wrapper. Regression +coverage lives in `tests/test_readability.py` and +`tests/test_web_fetch_relative_links.py`. + ### Atomic Byte Publication `atomic_write.publish_bytes_atomically` owns browser screenshot and Browserless diff --git a/backend/packages/harness/deerflow/utils/readability.py b/backend/packages/harness/deerflow/utils/readability.py index 5c94fb33da..fe195af384 100644 --- a/backend/packages/harness/deerflow/utils/readability.py +++ b/backend/packages/harness/deerflow/utils/readability.py @@ -3,6 +3,7 @@ import re import subprocess from html import escape, unescape from html.parser import HTMLParser +from threading import Lock from urllib.parse import urljoin, urlparse, uses_relative from bs4 import BeautifulSoup @@ -11,6 +12,45 @@ from readabilipy import simple_json_from_html_string logger = logging.getLogger(__name__) +try: + from readabilipy.simple_json import have_node as _readabilipy_have_node +except ImportError: # pragma: no cover - compatibility with future readabilipy layouts + logger.warning( + "Cannot import readabilipy's Node availability probe; attempting Readability.js extraction directly", + exc_info=True, + ) + _readabilipy_have_node = None + +_readability_js_launch_failed = False +_readability_js_available_cached: bool | None = None +_readability_js_probe_lock = Lock() + + +def _readability_js_available() -> bool: + """Cache whether the optional Node extractor can be used.""" + global _readability_js_available_cached + + if _readability_js_launch_failed: + return False + if _readabilipy_have_node is None: + return True + if _readability_js_available_cached is not None: + return _readability_js_available_cached + with _readability_js_probe_lock: + if _readability_js_launch_failed: + return False + if _readability_js_available_cached is not None: + return _readability_js_available_cached + try: + _readability_js_available_cached = bool(_readabilipy_have_node()) + except (subprocess.SubprocessError, OSError, ValueError): + logger.warning("Readability.js availability probe failed; using the Python fallback", exc_info=True) + _readability_js_available_cached = False + return False + if not _readability_js_available_cached: + logger.warning("Readability.js is unavailable; using the Python fallback") + return _readability_js_available_cached + class Article: url: str @@ -156,21 +196,31 @@ class _DestinationRewriter(HTMLParser): class ReadabilityExtractor: def extract_article(self, html: str, *, url: str | None = None) -> Article: + global _readability_js_launch_failed + if url: html = _resolve_html_urls(html, url) - try: - article = simple_json_from_html_string(html, use_readability=True) - except (subprocess.CalledProcessError, FileNotFoundError) as exc: - stderr = getattr(exc, "stderr", None) - if isinstance(stderr, bytes): - stderr = stderr.decode(errors="replace") - stderr_info = f"; stderr={stderr.strip()}" if isinstance(stderr, str) and stderr.strip() else "" - logger.warning( - "Readability.js extraction failed with %s%s; falling back to pure-Python extraction", - type(exc).__name__, - stderr_info, - exc_info=True, - ) + article = None + if _readability_js_available(): + try: + article = simple_json_from_html_string(html, use_readability=True) + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + # A missing executable is an environment-level failure. A + # non-zero exit can be document-specific (OOM, stack depth, + # malformed input), so only disable Node for the former. + if isinstance(exc, FileNotFoundError): + _readability_js_launch_failed = True + stderr = getattr(exc, "stderr", None) + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + stderr_info = f"; stderr={stderr.strip()}" if isinstance(stderr, str) and stderr.strip() else "" + logger.warning( + "Readability.js extraction failed with %s%s; falling back to pure-Python extraction", + type(exc).__name__, + stderr_info, + exc_info=True, + ) + if article is None: article = simple_json_from_html_string(html, use_readability=False) html_content = article.get("content") diff --git a/backend/tests/test_readability.py b/backend/tests/test_readability.py index 3a2341e8c1..47602f393a 100644 --- a/backend/tests/test_readability.py +++ b/backend/tests/test_readability.py @@ -7,6 +7,15 @@ import pytest from deerflow.utils.readability import Article, ReadabilityExtractor +@pytest.fixture(autouse=True) +def _reset_readability_js_state(monkeypatch): + from deerflow.utils import readability + + monkeypatch.setattr(readability, "_readabilipy_have_node", lambda: True) + monkeypatch.setattr(readability, "_readability_js_launch_failed", False) + monkeypatch.setattr(readability, "_readability_js_available_cached", None) + + def test_extract_article_falls_back_when_readability_js_fails(monkeypatch): """When Node-based readability fails, extraction should fall back to Python mode.""" @@ -55,6 +64,131 @@ def test_extract_article_re_raises_unexpected_exception(monkeypatch): assert calls == [True] +def test_unavailable_readability_uses_cached_link_preserving_fallback(monkeypatch): + from deerflow.utils import readability + + probe_calls = 0 + extraction_modes: list[bool] = [] + original = readability.simple_json_from_html_string + + def unavailable(): + nonlocal probe_calls + probe_calls += 1 + return False + + def extract(html: str, use_readability: bool = False): + extraction_modes.append(use_readability) + return original(html, use_readability=use_readability) + + monkeypatch.setattr(readability, "_readabilipy_have_node", unavailable) + monkeypatch.setattr(readability, "simple_json_from_html_string", extract) + html = 'Guide

Body

Next
' + + first = ReadabilityExtractor().extract_article(html, url="https://example.com/current") + second = ReadabilityExtractor().extract_article(html, url="https://example.com/current") + + assert "[Next](https://example.com/next)" in first.to_markdown() + assert second.to_markdown() == first.to_markdown() + assert probe_calls == 1 + assert extraction_modes == [False, False] + + +def test_readability_probe_failure_is_cached(monkeypatch): + from deerflow.utils import readability + + probe_calls = 0 + extraction_modes: list[bool] = [] + + def fail_probe(): + nonlocal probe_calls + probe_calls += 1 + raise FileNotFoundError("npm") + + def extract(_html: str, use_readability: bool = False): + extraction_modes.append(use_readability) + return {"title": "Guide", "content": "

Body

"} + + monkeypatch.setattr(readability, "_readabilipy_have_node", fail_probe) + monkeypatch.setattr(readability, "simple_json_from_html_string", extract) + + ReadabilityExtractor().extract_article("Body") + ReadabilityExtractor().extract_article("Body") + + assert probe_calls == 1 + assert extraction_modes == [False, False] + + +def test_document_runtime_failure_retries_node_for_later_documents(monkeypatch): + from deerflow.utils import readability + + probe_calls = 0 + node_attempts = 0 + extraction_modes: list[bool] = [] + + def available(): + nonlocal probe_calls + probe_calls += 1 + return True + + def extract(_html: str, use_readability: bool = False): + nonlocal node_attempts + extraction_modes.append(use_readability) + if use_readability: + node_attempts += 1 + if node_attempts == 1: + raise subprocess.CalledProcessError(1, "node", stderr="document OOM") + return {"title": "Recovered", "content": "

Node Body

"} + return {"title": "Fallback", "content": "

Python Body

"} + + monkeypatch.setattr(readability, "_readabilipy_have_node", available) + monkeypatch.setattr(readability, "simple_json_from_html_string", extract) + + first = ReadabilityExtractor().extract_article("Body") + second = ReadabilityExtractor().extract_article("Body") + + assert first.title == "Fallback" + assert second.title == "Recovered" + assert probe_calls == 1 + assert extraction_modes == [True, False, True] + + +def test_missing_node_during_extraction_disables_repeated_node_attempts(monkeypatch): + from deerflow.utils import readability + + extraction_modes: list[bool] = [] + + def extract(_html: str, use_readability: bool = False): + extraction_modes.append(use_readability) + if use_readability: + raise FileNotFoundError("node") + return {"title": "Guide", "content": "

Body

"} + + monkeypatch.setattr(readability, "simple_json_from_html_string", extract) + + ReadabilityExtractor().extract_article("One") + ReadabilityExtractor().extract_article("Two") + + assert extraction_modes == [True, False, False] + + +def test_missing_probe_api_still_attempts_node_extraction(monkeypatch): + from deerflow.utils import readability + + extraction_modes: list[bool] = [] + + def extract(_html: str, use_readability: bool = False): + extraction_modes.append(use_readability) + return {"title": "Guide", "content": "

Body

"} + + monkeypatch.setattr(readability, "_readabilipy_have_node", None) + monkeypatch.setattr(readability, "simple_json_from_html_string", extract) + + ReadabilityExtractor().extract_article("One") + ReadabilityExtractor().extract_article("Two") + + assert extraction_modes == [True, True] + + def test_article_to_message_with_images(): """Article.to_message should handle articles containing images without AttributeError.""" # Absolute image URL without source url diff --git a/backend/tests/test_web_fetch_relative_links.py b/backend/tests/test_web_fetch_relative_links.py index 0a6a8c5996..1f266069e9 100644 --- a/backend/tests/test_web_fetch_relative_links.py +++ b/backend/tests/test_web_fetch_relative_links.py @@ -99,6 +99,9 @@ def test_python_extraction_fallback_preserves_article_text(monkeypatch): return original(html, use_readability=False) monkeypatch.setattr(readability, "simple_json_from_html_string", extract) + monkeypatch.setattr(readability, "_readabilipy_have_node", lambda: True) + monkeypatch.setattr(readability, "_readability_js_launch_failed", False) + monkeypatch.setattr(readability, "_readability_js_available_cached", None) article = ReadabilityExtractor().extract_article(_article('Next'), url=PAGE_URL) # The existing Python fallback strips link markup; preserve its text contract. assert "Next" in article.to_markdown()