github/deer-flow
1
0
Fork 0
mirror of https://github.com/bytedance/deer-flow.git synced 2026-10-27 05:30:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(community): migrate firecrawl tools to AsyncFirecrawlApp (#5786)

Browse Source 
Replace the synchronous FirecrawlApp with AsyncFirecrawlApp in the
firecrawl community tools, continuing the TODO.md planned feature
('Replace sync requests with httpx.AsyncClient in community tools').

web_search_tool and web_fetch_tool become async functions and await
client.search()/client.scrape(). Return values (SearchData.web,
Document.markdown/metadata) are identical between the sync and async
clients, so callers see no change; error handling and the api_key /
api_url construction path are unchanged.

Tests now invoke the async tools via ainvoke under @pytest.mark.anyio,
and the client search/scrape methods are mocked with AsyncMock.
This commit is contained in:
yc2bgr8 2026-09-28 18:08:46 +08:00 committed by GitHub
parent 50d2d68556
commit 712a11c0cd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 30 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
backend/packages/harness/deerflow/community/firecrawl/tools.py
View File

@ -1,12 +1,12 @@
import json
from firecrawl import FirecrawlApp
from firecrawl import AsyncFirecrawlApp
from langchain.tools import tool
from deerflow.config import get_app_config
def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp:
def _get_firecrawl_client(tool_name: str = "web_search") -> AsyncFirecrawlApp:
config = get_app_config().get_tool_config(tool_name)
api_key = None
api_url = None
@ -18,11 +18,11 @@ def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp:
kwargs = {"api_key": api_key}
if api_url:
kwargs["api_url"] = api_url
return FirecrawlApp(**kwargs) # type: ignore[arg-type]
return AsyncFirecrawlApp(**kwargs) # type: ignore[arg-type]
@tool("web_search", parse_docstring=True)
def web_search_tool(query: str) -> str:
async def web_search_tool(query: str) -> str:
"""Search the web.
Args:
@ -35,7 +35,7 @@ def web_search_tool(query: str) -> str:
max_results = config.model_extra.get("max_results", max_results)
client = _get_firecrawl_client("web_search")
result = client.search(query, limit=max_results)
result = await client.search(query, limit=max_results)
# result.web contains list of SearchResultWeb objects
web_results = result.web or []
@ -54,7 +54,7 @@ def web_search_tool(query: str) -> str:
@tool("web_fetch", parse_docstring=True)
def web_fetch_tool(url: str) -> str:
async def web_fetch_tool(url: str) -> str:
"""Fetch the contents of a web page at a given URL.
Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
@ -66,7 +66,7 @@ def web_fetch_tool(url: str) -> str:
"""
try:
client = _get_firecrawl_client("web_fetch")
result = client.scrape(url, formats=["markdown"])
result = await client.scrape(url, formats=["markdown"])
markdown_content = result.markdown or ""
metadata = result.metadata

40
backend/tests/test_firecrawl_tools.py
View File

@ -1,13 +1,16 @@
"""Unit tests for the Firecrawl community tools."""
import json
from unittest.mock import MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
class TestWebSearchTool:
@patch("deerflow.community.firecrawl.tools.FirecrawlApp")
@patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp")
@patch("deerflow.community.firecrawl.tools.get_app_config")
def test_search_uses_web_search_config(self, mock_get_app_config, mock_firecrawl_cls):
@pytest.mark.anyio
async def test_search_uses_web_search_config(self, mock_get_app_config, mock_firecrawl_cls):
search_config = MagicMock()
search_config.model_extra = {"api_key": "firecrawl-search-key", "max_results": 7}
mock_get_app_config.return_value.get_tool_config.return_value = search_config
@ -16,11 +19,11 @@ class TestWebSearchTool:
mock_result.web = [
MagicMock(title="Result", url="https://example.com", description="Snippet"),
]
mock_firecrawl_cls.return_value.search.return_value = mock_result
mock_firecrawl_cls.return_value.search = AsyncMock(return_value=mock_result)
from deerflow.community.firecrawl.tools import web_search_tool
result = web_search_tool.invoke({"query": "test query"})
result = await web_search_tool.ainvoke({"query": "test query"})
assert json.loads(result) == [
{
@ -35,9 +38,10 @@ class TestWebSearchTool:
class TestWebFetchTool:
@patch("deerflow.community.firecrawl.tools.FirecrawlApp")
@patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp")
@patch("deerflow.community.firecrawl.tools.get_app_config")
def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_firecrawl_cls):
@pytest.mark.anyio
async def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_firecrawl_cls):
fetch_config = MagicMock()
fetch_config.model_extra = {"api_key": "firecrawl-fetch-key"}
@ -51,11 +55,11 @@ class TestWebFetchTool:
mock_scrape_result = MagicMock()
mock_scrape_result.markdown = "Fetched markdown"
mock_scrape_result.metadata = MagicMock(title="Fetched Page")
mock_firecrawl_cls.return_value.scrape.return_value = mock_scrape_result
mock_firecrawl_cls.return_value.scrape = AsyncMock(return_value=mock_scrape_result)
from deerflow.community.firecrawl.tools import web_fetch_tool
result = web_fetch_tool.invoke({"url": "https://example.com"})
result = await web_fetch_tool.ainvoke({"url": "https://example.com"})
assert result == "# Fetched Page\n\nFetched markdown"
mock_get_app_config.return_value.get_tool_config.assert_any_call("web_fetch")
@ -67,9 +71,10 @@ class TestWebFetchTool:
class TestFirecrawlBaseUrl:
@patch("deerflow.community.firecrawl.tools.FirecrawlApp")
@patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp")
@patch("deerflow.community.firecrawl.tools.get_app_config")
def test_fetch_passes_base_url_as_api_url(self, mock_get_app_config, mock_firecrawl_cls):
@pytest.mark.anyio
async def test_fetch_passes_base_url_as_api_url(self, mock_get_app_config, mock_firecrawl_cls):
fetch_config = MagicMock()
fetch_config.model_extra = {"base_url": "http://192.168.0.47:3002"}
@ -83,18 +88,19 @@ class TestFirecrawlBaseUrl:
mock_scrape_result = MagicMock()
mock_scrape_result.markdown = "Fetched markdown"
mock_scrape_result.metadata = MagicMock(title="Fetched Page")
mock_firecrawl_cls.return_value.scrape.return_value = mock_scrape_result
mock_firecrawl_cls.return_value.scrape = AsyncMock(return_value=mock_scrape_result)
from deerflow.community.firecrawl.tools import web_fetch_tool
result = web_fetch_tool.invoke({"url": "https://example.com"})
result = await web_fetch_tool.ainvoke({"url": "https://example.com"})
assert result == "# Fetched Page\n\nFetched markdown"
mock_firecrawl_cls.assert_called_once_with(api_key=None, api_url="http://192.168.0.47:3002")
@patch("deerflow.community.firecrawl.tools.FirecrawlApp")
@patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp")
@patch("deerflow.community.firecrawl.tools.get_app_config")
def test_search_passes_base_url_and_api_key(self, mock_get_app_config, mock_firecrawl_cls):
@pytest.mark.anyio
async def test_search_passes_base_url_and_api_key(self, mock_get_app_config, mock_firecrawl_cls):
search_config = MagicMock()
search_config.model_extra = {
"api_key": "firecrawl-key",
@ -105,10 +111,10 @@ class TestFirecrawlBaseUrl:
mock_result = MagicMock()
mock_result.web = []
mock_firecrawl_cls.return_value.search.return_value = mock_result
mock_firecrawl_cls.return_value.search = AsyncMock(return_value=mock_result)
from deerflow.community.firecrawl.tools import web_search_tool
web_search_tool.invoke({"query": "test query"})
await web_search_tool.ainvoke({"query": "test query"})
mock_firecrawl_cls.assert_called_once_with(api_key="firecrawl-key", api_url="http://192.168.0.47:3002")