From 712a11c0cda0a7c9441c9afd68c1e854d38103c6 Mon Sep 17 00:00:00 2001 From: yc2bgr8 <119500159+yc2bgr8@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:08:46 +0800 Subject: [PATCH] feat(community): migrate firecrawl tools to AsyncFirecrawlApp (#5786) Replace the synchronous FirecrawlApp with AsyncFirecrawlApp in the firecrawl community tools, continuing the TODO.md planned feature ('Replace sync requests with httpx.AsyncClient in community tools'). web_search_tool and web_fetch_tool become async functions and await client.search()/client.scrape(). Return values (SearchData.web, Document.markdown/metadata) are identical between the sync and async clients, so callers see no change; error handling and the api_key / api_url construction path are unchanged. Tests now invoke the async tools via ainvoke under @pytest.mark.anyio, and the client search/scrape methods are mocked with AsyncMock. --- .../deerflow/community/firecrawl/tools.py | 14 +++---- backend/tests/test_firecrawl_tools.py | 40 +++++++++++-------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/backend/packages/harness/deerflow/community/firecrawl/tools.py b/backend/packages/harness/deerflow/community/firecrawl/tools.py index f46d9ddf12..c7af25147c 100644 --- a/backend/packages/harness/deerflow/community/firecrawl/tools.py +++ b/backend/packages/harness/deerflow/community/firecrawl/tools.py @@ -1,12 +1,12 @@ import json -from firecrawl import FirecrawlApp +from firecrawl import AsyncFirecrawlApp from langchain.tools import tool from deerflow.config import get_app_config -def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp: +def _get_firecrawl_client(tool_name: str = "web_search") -> AsyncFirecrawlApp: config = get_app_config().get_tool_config(tool_name) api_key = None api_url = None @@ -18,11 +18,11 @@ def _get_firecrawl_client(tool_name: str = "web_search") -> FirecrawlApp: kwargs = {"api_key": api_key} if api_url: kwargs["api_url"] = api_url - return FirecrawlApp(**kwargs) # type: ignore[arg-type] + return AsyncFirecrawlApp(**kwargs) # type: ignore[arg-type] @tool("web_search", parse_docstring=True) -def web_search_tool(query: str) -> str: +async def web_search_tool(query: str) -> str: """Search the web. Args: @@ -35,7 +35,7 @@ def web_search_tool(query: str) -> str: max_results = config.model_extra.get("max_results", max_results) client = _get_firecrawl_client("web_search") - result = client.search(query, limit=max_results) + result = await client.search(query, limit=max_results) # result.web contains list of SearchResultWeb objects web_results = result.web or [] @@ -54,7 +54,7 @@ def web_search_tool(query: str) -> str: @tool("web_fetch", parse_docstring=True) -def web_fetch_tool(url: str) -> str: +async def web_fetch_tool(url: str) -> str: """Fetch the contents of a web page at a given URL. Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools. This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls. @@ -66,7 +66,7 @@ def web_fetch_tool(url: str) -> str: """ try: client = _get_firecrawl_client("web_fetch") - result = client.scrape(url, formats=["markdown"]) + result = await client.scrape(url, formats=["markdown"]) markdown_content = result.markdown or "" metadata = result.metadata diff --git a/backend/tests/test_firecrawl_tools.py b/backend/tests/test_firecrawl_tools.py index 4e0720e566..eb9deb6f86 100644 --- a/backend/tests/test_firecrawl_tools.py +++ b/backend/tests/test_firecrawl_tools.py @@ -1,13 +1,16 @@ """Unit tests for the Firecrawl community tools.""" import json -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest class TestWebSearchTool: - @patch("deerflow.community.firecrawl.tools.FirecrawlApp") + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_search_uses_web_search_config(self, mock_get_app_config, mock_firecrawl_cls): + @pytest.mark.anyio + async def test_search_uses_web_search_config(self, mock_get_app_config, mock_firecrawl_cls): search_config = MagicMock() search_config.model_extra = {"api_key": "firecrawl-search-key", "max_results": 7} mock_get_app_config.return_value.get_tool_config.return_value = search_config @@ -16,11 +19,11 @@ class TestWebSearchTool: mock_result.web = [ MagicMock(title="Result", url="https://example.com", description="Snippet"), ] - mock_firecrawl_cls.return_value.search.return_value = mock_result + mock_firecrawl_cls.return_value.search = AsyncMock(return_value=mock_result) from deerflow.community.firecrawl.tools import web_search_tool - result = web_search_tool.invoke({"query": "test query"}) + result = await web_search_tool.ainvoke({"query": "test query"}) assert json.loads(result) == [ { @@ -35,9 +38,10 @@ class TestWebSearchTool: class TestWebFetchTool: - @patch("deerflow.community.firecrawl.tools.FirecrawlApp") + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_firecrawl_cls): + @pytest.mark.anyio + async def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_firecrawl_cls): fetch_config = MagicMock() fetch_config.model_extra = {"api_key": "firecrawl-fetch-key"} @@ -51,11 +55,11 @@ class TestWebFetchTool: mock_scrape_result = MagicMock() mock_scrape_result.markdown = "Fetched markdown" mock_scrape_result.metadata = MagicMock(title="Fetched Page") - mock_firecrawl_cls.return_value.scrape.return_value = mock_scrape_result + mock_firecrawl_cls.return_value.scrape = AsyncMock(return_value=mock_scrape_result) from deerflow.community.firecrawl.tools import web_fetch_tool - result = web_fetch_tool.invoke({"url": "https://example.com"}) + result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) assert result == "# Fetched Page\n\nFetched markdown" mock_get_app_config.return_value.get_tool_config.assert_any_call("web_fetch") @@ -67,9 +71,10 @@ class TestWebFetchTool: class TestFirecrawlBaseUrl: - @patch("deerflow.community.firecrawl.tools.FirecrawlApp") + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_fetch_passes_base_url_as_api_url(self, mock_get_app_config, mock_firecrawl_cls): + @pytest.mark.anyio + async def test_fetch_passes_base_url_as_api_url(self, mock_get_app_config, mock_firecrawl_cls): fetch_config = MagicMock() fetch_config.model_extra = {"base_url": "http://192.168.0.47:3002"} @@ -83,18 +88,19 @@ class TestFirecrawlBaseUrl: mock_scrape_result = MagicMock() mock_scrape_result.markdown = "Fetched markdown" mock_scrape_result.metadata = MagicMock(title="Fetched Page") - mock_firecrawl_cls.return_value.scrape.return_value = mock_scrape_result + mock_firecrawl_cls.return_value.scrape = AsyncMock(return_value=mock_scrape_result) from deerflow.community.firecrawl.tools import web_fetch_tool - result = web_fetch_tool.invoke({"url": "https://example.com"}) + result = await web_fetch_tool.ainvoke({"url": "https://example.com"}) assert result == "# Fetched Page\n\nFetched markdown" mock_firecrawl_cls.assert_called_once_with(api_key=None, api_url="http://192.168.0.47:3002") - @patch("deerflow.community.firecrawl.tools.FirecrawlApp") + @patch("deerflow.community.firecrawl.tools.AsyncFirecrawlApp") @patch("deerflow.community.firecrawl.tools.get_app_config") - def test_search_passes_base_url_and_api_key(self, mock_get_app_config, mock_firecrawl_cls): + @pytest.mark.anyio + async def test_search_passes_base_url_and_api_key(self, mock_get_app_config, mock_firecrawl_cls): search_config = MagicMock() search_config.model_extra = { "api_key": "firecrawl-key", @@ -105,10 +111,10 @@ class TestFirecrawlBaseUrl: mock_result = MagicMock() mock_result.web = [] - mock_firecrawl_cls.return_value.search.return_value = mock_result + mock_firecrawl_cls.return_value.search = AsyncMock(return_value=mock_result) from deerflow.community.firecrawl.tools import web_search_tool - web_search_tool.invoke({"query": "test query"}) + await web_search_tool.ainvoke({"query": "test query"}) mock_firecrawl_cls.assert_called_once_with(api_key="firecrawl-key", api_url="http://192.168.0.47:3002")