diff --git a/backend/CONTRIBUTING.md b/backend/CONTRIBUTING.md index f7ef58447..e4467b6db 100644 --- a/backend/CONTRIBUTING.md +++ b/backend/CONTRIBUTING.md @@ -114,6 +114,7 @@ backend/src/ │ ├── tavily/ # Tavily web search │ ├── jina/ # Jina web fetch │ ├── firecrawl/ # Firecrawl scraping +│ ├── fastcrw/ # fastCRW scraping (Firecrawl-compatible) │ └── aio_sandbox/ # Docker sandbox │ ├── reflection/ # Dynamic loading diff --git a/backend/README.md b/backend/README.md index 0c4724692..22eb8f71d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -101,7 +101,7 @@ LLM-powered persistent context retention across conversations: |----------|-------| | **Sandbox** | `bash`, `ls`, `read_file`, `write_file`, `str_replace` | | **Built-in** | `present_files`, `ask_clarification`, `view_image`, `task` (subagent) | -| **Community** | Tavily (web search), Jina AI (web fetch), Firecrawl (scraping), DuckDuckGo (image search) | +| **Community** | Tavily (web search), Jina AI (web fetch), Firecrawl (scraping), fastCRW (scraping), DuckDuckGo (image search) | | **MCP** | Any Model Context Protocol server (stdio, SSE, HTTP transports) | | **Skills** | Domain-specific workflows injected via system prompt | diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 3417e70bd..30973c482 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -234,8 +234,8 @@ tools: ``` **Built-in Tools**: -- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Firecrawl) -- `web_fetch` - Fetch web pages (Jina AI, Exa, InfoQuest, Firecrawl) +- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Firecrawl, fastCRW) +- `web_fetch` - Fetch web pages (Jina AI, Exa, InfoQuest, Firecrawl, fastCRW) - `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper) - `ls` - List directory contents - `read_file` - Read file contents diff --git a/backend/packages/harness/deerflow/community/fastcrw/tools.py b/backend/packages/harness/deerflow/community/fastcrw/tools.py new file mode 100644 index 000000000..f38fc6319 --- /dev/null +++ b/backend/packages/harness/deerflow/community/fastcrw/tools.py @@ -0,0 +1,88 @@ +import json +import os + +from firecrawl import FirecrawlApp +from langchain.tools import tool + +from deerflow.config import get_app_config + +# fastCRW is a Firecrawl-compatible web data engine (single Rust binary; self-host +# or cloud). Because the REST API is Firecrawl-compatible, this provider reuses the +# Firecrawl client and only swaps the base URL. Cloud default points at the managed +# service; override `base_url` in the tool config (or set CRW_API_URL) for self-host. +DEFAULT_BASE_URL = "https://fastcrw.com/api" + + +def _get_fastcrw_client(tool_name: str = "web_search") -> FirecrawlApp: + config = get_app_config().get_tool_config(tool_name) + api_key = None + base_url = None + if config is not None: + if "api_key" in config.model_extra: + api_key = config.model_extra.get("api_key") + if "base_url" in config.model_extra: + base_url = config.model_extra.get("base_url") + if api_key is None: + api_key = os.getenv("CRW_API_KEY") + if base_url is None: + base_url = os.getenv("CRW_API_URL", DEFAULT_BASE_URL) + return FirecrawlApp(api_key=api_key, api_url=base_url) # type: ignore[arg-type] + + +@tool("web_search", parse_docstring=True) +def web_search_tool(query: str) -> str: + """Search the web. + + Args: + query: The query to search for. + """ + try: + config = get_app_config().get_tool_config("web_search") + max_results = 5 + if config is not None: + max_results = config.model_extra.get("max_results", max_results) + + client = _get_fastcrw_client("web_search") + result = client.search(query, limit=max_results) + + # result.web contains list of SearchResultWeb objects + web_results = result.web or [] + normalized_results = [ + { + "title": getattr(item, "title", "") or "", + "url": getattr(item, "url", "") or "", + "snippet": getattr(item, "description", "") or "", + } + for item in web_results + ] + json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False) + return json_results + except Exception as e: + return f"Error: {str(e)}" + + +@tool("web_fetch", parse_docstring=True) +def web_fetch_tool(url: str) -> str: + """Fetch the contents of a web page at a given URL. + Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools. + This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls. + Do NOT add www. to URLs that do NOT have them. + URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL. + + Args: + url: The URL to fetch the contents of. + """ + try: + client = _get_fastcrw_client("web_fetch") + result = client.scrape(url, formats=["markdown"]) + + markdown_content = result.markdown or "" + metadata = result.metadata + title = metadata.title if metadata and metadata.title else "Untitled" + + if not markdown_content: + return "Error: No content found" + except Exception as e: + return f"Error: {str(e)}" + + return f"# {title}\n\n{markdown_content[:4096]}" diff --git a/backend/tests/test_fastcrw_tools.py b/backend/tests/test_fastcrw_tools.py new file mode 100644 index 000000000..9d63585e1 --- /dev/null +++ b/backend/tests/test_fastcrw_tools.py @@ -0,0 +1,123 @@ +"""Unit tests for the fastCRW community tools.""" + +import json +from unittest.mock import MagicMock, patch + + +class TestWebSearchTool: + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + def test_search_uses_web_search_config(self, mock_get_app_config, mock_fastcrw_cls): + search_config = MagicMock() + search_config.model_extra = {"api_key": "fastcrw-search-key", "max_results": 7} + mock_get_app_config.return_value.get_tool_config.return_value = search_config + + mock_result = MagicMock() + mock_result.web = [ + MagicMock(title="Result", url="https://example.com", description="Snippet"), + ] + mock_fastcrw_cls.return_value.search.return_value = mock_result + + from deerflow.community.fastcrw.tools import web_search_tool + + result = web_search_tool.invoke({"query": "test query"}) + + assert json.loads(result) == [ + { + "title": "Result", + "url": "https://example.com", + "snippet": "Snippet", + } + ] + mock_get_app_config.return_value.get_tool_config.assert_called_with("web_search") + mock_fastcrw_cls.assert_called_once_with(api_key="fastcrw-search-key", api_url="https://fastcrw.com/api") + mock_fastcrw_cls.return_value.search.assert_called_once_with("test query", limit=7) + + @patch.dict("os.environ", {"CRW_API_KEY": "env-key", "CRW_API_URL": "http://self-hosted:3000"}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + def test_search_falls_back_to_env_and_default_max_results(self, mock_get_app_config, mock_fastcrw_cls): + mock_get_app_config.return_value.get_tool_config.return_value = None + + mock_result = MagicMock() + mock_result.web = [] + mock_fastcrw_cls.return_value.search.return_value = mock_result + + from deerflow.community.fastcrw.tools import web_search_tool + + result = web_search_tool.invoke({"query": "q"}) + + assert result == "[]" + mock_fastcrw_cls.assert_called_once_with(api_key="env-key", api_url="http://self-hosted:3000") + mock_fastcrw_cls.return_value.search.assert_called_once_with("q", limit=5) + + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + def test_search_returns_error_string_on_exception(self, mock_get_app_config, mock_fastcrw_cls): + mock_get_app_config.return_value.get_tool_config.return_value = None + mock_fastcrw_cls.return_value.search.side_effect = RuntimeError("boom") + + from deerflow.community.fastcrw.tools import web_search_tool + + assert web_search_tool.invoke({"query": "q"}) == "Error: boom" + + +class TestWebFetchTool: + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_fastcrw_cls): + fetch_config = MagicMock() + fetch_config.model_extra = {"api_key": "fastcrw-fetch-key", "base_url": "http://localhost:3000"} + + def get_tool_config(name): + if name == "web_fetch": + return fetch_config + return None + + mock_get_app_config.return_value.get_tool_config.side_effect = get_tool_config + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "Fetched markdown" + mock_scrape_result.metadata = MagicMock(title="Fetched Page") + mock_fastcrw_cls.return_value.scrape.return_value = mock_scrape_result + + from deerflow.community.fastcrw.tools import web_fetch_tool + + result = web_fetch_tool.invoke({"url": "https://example.com"}) + + assert result == "# Fetched Page\n\nFetched markdown" + mock_get_app_config.return_value.get_tool_config.assert_any_call("web_fetch") + mock_fastcrw_cls.assert_called_once_with(api_key="fastcrw-fetch-key", api_url="http://localhost:3000") + mock_fastcrw_cls.return_value.scrape.assert_called_once_with( + "https://example.com", + formats=["markdown"], + ) + + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + def test_fetch_returns_error_when_no_content(self, mock_get_app_config, mock_fastcrw_cls): + mock_get_app_config.return_value.get_tool_config.return_value = None + + mock_scrape_result = MagicMock() + mock_scrape_result.markdown = "" + mock_scrape_result.metadata = MagicMock(title="Empty") + mock_fastcrw_cls.return_value.scrape.return_value = mock_scrape_result + + from deerflow.community.fastcrw.tools import web_fetch_tool + + assert web_fetch_tool.invoke({"url": "https://example.com"}) == "Error: No content found" + + @patch.dict("os.environ", {}, clear=True) + @patch("deerflow.community.fastcrw.tools.FirecrawlApp") + @patch("deerflow.community.fastcrw.tools.get_app_config") + def test_fetch_returns_error_string_on_exception(self, mock_get_app_config, mock_fastcrw_cls): + mock_get_app_config.return_value.get_tool_config.return_value = None + mock_fastcrw_cls.return_value.scrape.side_effect = RuntimeError("scrape failed") + + from deerflow.community.fastcrw.tools import web_fetch_tool + + assert web_fetch_tool.invoke({"url": "https://example.com"}) == "Error: scrape failed" diff --git a/config.example.yaml b/config.example.yaml index 24b65bbcb..75d82c882 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -570,6 +570,15 @@ tools: # max_results: 5 # # api_key: $FIRECRAWL_API_KEY + # Web search tool (uses fastCRW - Firecrawl-compatible web scraper, single binary, + # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.) + # - name: web_search + # group: web + # use: deerflow.community.fastcrw.tools:web_search_tool + # max_results: 5 + # # api_key: $CRW_API_KEY + # # base_url: https://fastcrw.com/api # default cloud; set to e.g. http://localhost:3000 for self-host + # Web fetch tool (uses Browserless - headless Chrome, self-hosted or cloud) # Browserless renders pages with a real headless Chrome, ideal for JavaScript-heavy # sites and SPAs. Deploy your own: https://github.com/browserless/browserless @@ -621,6 +630,16 @@ tools: # use: deerflow.community.firecrawl.tools:web_fetch_tool # # api_key: $FIRECRAWL_API_KEY + # Web fetch tool (uses fastCRW - Firecrawl-compatible web scraper, single binary, + # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.) + # NOTE: Only one web_fetch provider can be active at a time. + # Comment out the Jina AI web_fetch entry above before enabling this one. + # - name: web_fetch + # group: web + # use: deerflow.community.fastcrw.tools:web_fetch_tool + # # api_key: $CRW_API_KEY + # # base_url: https://fastcrw.com/api # default cloud; set to e.g. http://localhost:3000 for self-host + # Image search tool (uses DuckDuckGo) # Use this to find reference images before image generation - name: image_search diff --git a/scripts/doctor.py b/scripts/doctor.py index 9ad4d3bb9..20fc4642e 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -468,6 +468,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes "infoquest": "INFOQUEST_API_KEY", "exa": "EXA_API_KEY", "firecrawl": "FIRECRAWL_API_KEY", + "fastcrw": "CRW_API_KEY", "brave": "BRAVE_SEARCH_API_KEY", "serper": "SERPER_API_KEY", }, @@ -475,6 +476,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes "infoquest": "INFOQUEST_API_KEY", "exa": "EXA_API_KEY", "firecrawl": "FIRECRAWL_API_KEY", + "fastcrw": "CRW_API_KEY", }, "image_search": { "infoquest": "INFOQUEST_API_KEY", diff --git a/scripts/wizard/providers.py b/scripts/wizard/providers.py index eb38996cf..ee57bbecf 100644 --- a/scripts/wizard/providers.py +++ b/scripts/wizard/providers.py @@ -510,6 +510,14 @@ SEARCH_PROVIDERS: list[SearchProvider] = [ env_var="FIRECRAWL_API_KEY", extra_config={"max_results": 5}, ), + SearchProvider( + name="fastcrw", + display_name="fastCRW", + description="Firecrawl-compatible web scraper, single binary, self-host or cloud", + use="deerflow.community.fastcrw.tools:web_search_tool", + env_var="CRW_API_KEY", + extra_config={"max_results": 5}, + ), SearchProvider( name="brave", display_name="Brave Search", @@ -555,4 +563,12 @@ WEB_FETCH_PROVIDERS: list[WebProvider] = [ env_var="FIRECRAWL_API_KEY", tool_name="web_fetch", ), + WebProvider( + name="fastcrw", + display_name="fastCRW", + description="Firecrawl-compatible web scraper with markdown output, self-host or cloud", + use="deerflow.community.fastcrw.tools:web_fetch_tool", + env_var="CRW_API_KEY", + tool_name="web_fetch", + ), ]