mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-04-25 11:18:22 +00:00
1df389b9d0
* fix: wrap blocking readability call with asyncio.to_thread in web_fetch The readability extractor internally spawns a Node.js subprocess via readabilipy, which blocks the async event loop and causes a BlockingError when web_fetch is invoked inside LangGraph's async runtime. Wrap the synchronous extract_article call with asyncio.to_thread to offload it to a thread pool, unblocking the event loop. Note: community/infoquest/tools.py has the same latent issue and should be addressed in a follow-up PR. Closes #2152 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * test: verify web_fetch offloads extraction via asyncio.to_thread Add a regression test that monkeypatches asyncio.to_thread to confirm readability extraction is offloaded to a worker thread, preventing future refactors from reintroducing the blocking call. Addresses Copilot review feedback on #2157. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
33 lines
1.4 KiB
Python
33 lines
1.4 KiB
Python
import asyncio
|
|
|
|
from langchain.tools import tool
|
|
|
|
from deerflow.community.jina_ai.jina_client import JinaClient
|
|
from deerflow.config import get_app_config
|
|
from deerflow.utils.readability import ReadabilityExtractor
|
|
|
|
readability_extractor = ReadabilityExtractor()
|
|
|
|
|
|
@tool("web_fetch", parse_docstring=True)
|
|
async def web_fetch_tool(url: str) -> str:
|
|
"""Fetch the contents of a web page at a given URL.
|
|
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
|
|
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
|
|
Do NOT add www. to URLs that do NOT have them.
|
|
URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
|
|
|
|
Args:
|
|
url: The URL to fetch the contents of.
|
|
"""
|
|
jina_client = JinaClient()
|
|
timeout = 10
|
|
config = get_app_config().get_tool_config("web_fetch")
|
|
if config is not None and "timeout" in config.model_extra:
|
|
timeout = config.model_extra.get("timeout")
|
|
html_content = await jina_client.crawl(url, return_format="html", timeout=timeout)
|
|
if isinstance(html_content, str) and html_content.startswith("Error:"):
|
|
return html_content
|
|
article = await asyncio.to_thread(readability_extractor.extract_article, html_content)
|
|
return article.to_markdown()[:4096]
|