feat: add fastCRW provider (#3585)

* feat: add fastCRW provider

* test(fastcrw): fix env isolation and cover error, no-content, and env-fallback paths

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>

backend/CONTRIBUTING.md

@@ -114,6 +114,7 @@ backend/src/
 │   ├── tavily/             # Tavily web search
 │   ├── jina/               # Jina web fetch
 │   ├── firecrawl/          # Firecrawl scraping
+│   ├── fastcrw/            # fastCRW scraping (Firecrawl-compatible)
 │   └── aio_sandbox/        # Docker sandbox
 │
 ├── reflection/              # Dynamic loading

backend/README.md

@@ -101,7 +101,7 @@ LLM-powered persistent context retention across conversations:
 |----------|-------|
 | **Sandbox** | `bash`, `ls`, `read_file`, `write_file`, `str_replace` |
 | **Built-in** | `present_files`, `ask_clarification`, `view_image`, `task` (subagent) |
-| **Community** | Tavily (web search), Jina AI (web fetch), Firecrawl (scraping), DuckDuckGo (image search) |
+| **Community** | Tavily (web search), Jina AI (web fetch), Firecrawl (scraping), fastCRW (scraping), DuckDuckGo (image search) |
 | **MCP** | Any Model Context Protocol server (stdio, SSE, HTTP transports) |
 | **Skills** | Domain-specific workflows injected via system prompt |
 

backend/docs/CONFIGURATION.md

@@ -234,8 +234,8 @@ tools:
 ```
 
 **Built-in Tools**:
-- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Firecrawl)
-- `web_fetch` - Fetch web pages (Jina AI, Exa, InfoQuest, Firecrawl)
+- `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Firecrawl, fastCRW)
+- `web_fetch` - Fetch web pages (Jina AI, Exa, InfoQuest, Firecrawl, fastCRW)
 - `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper)
 - `ls` - List directory contents
 - `read_file` - Read file contents

backend/packages/harness/deerflow/community/fastcrw/tools.py

@@ -0,0 +1,88 @@
+import json
+import os
+
+from firecrawl import FirecrawlApp
+from langchain.tools import tool
+
+from deerflow.config import get_app_config
+
+# fastCRW is a Firecrawl-compatible web data engine (single Rust binary; self-host
+# or cloud). Because the REST API is Firecrawl-compatible, this provider reuses the
+# Firecrawl client and only swaps the base URL. Cloud default points at the managed
+# service; override `base_url` in the tool config (or set CRW_API_URL) for self-host.
+DEFAULT_BASE_URL = "https://fastcrw.com/api"
+
+
+def _get_fastcrw_client(tool_name: str = "web_search") -> FirecrawlApp:
+    config = get_app_config().get_tool_config(tool_name)
+    api_key = None
+    base_url = None
+    if config is not None:
+        if "api_key" in config.model_extra:
+            api_key = config.model_extra.get("api_key")
+        if "base_url" in config.model_extra:
+            base_url = config.model_extra.get("base_url")
+    if api_key is None:
+        api_key = os.getenv("CRW_API_KEY")
+    if base_url is None:
+        base_url = os.getenv("CRW_API_URL", DEFAULT_BASE_URL)
+    return FirecrawlApp(api_key=api_key, api_url=base_url)  # type: ignore[arg-type]
+
+
+@tool("web_search", parse_docstring=True)
+def web_search_tool(query: str) -> str:
+    """Search the web.
+
+    Args:
+        query: The query to search for.
+    """
+    try:
+        config = get_app_config().get_tool_config("web_search")
+        max_results = 5
+        if config is not None:
+            max_results = config.model_extra.get("max_results", max_results)
+
+        client = _get_fastcrw_client("web_search")
+        result = client.search(query, limit=max_results)
+
+        # result.web contains list of SearchResultWeb objects
+        web_results = result.web or []
+        normalized_results = [
+            {
+                "title": getattr(item, "title", "") or "",
+                "url": getattr(item, "url", "") or "",
+                "snippet": getattr(item, "description", "") or "",
+            }
+            for item in web_results
+        ]
+        json_results = json.dumps(normalized_results, indent=2, ensure_ascii=False)
+        return json_results
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+
+@tool("web_fetch", parse_docstring=True)
+def web_fetch_tool(url: str) -> str:
+    """Fetch the contents of a web page at a given URL.
+    Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools.
+    This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls.
+    Do NOT add www. to URLs that do NOT have them.
+    URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL.
+
+    Args:
+        url: The URL to fetch the contents of.
+    """
+    try:
+        client = _get_fastcrw_client("web_fetch")
+        result = client.scrape(url, formats=["markdown"])
+
+        markdown_content = result.markdown or ""
+        metadata = result.metadata
+        title = metadata.title if metadata and metadata.title else "Untitled"
+
+        if not markdown_content:
+            return "Error: No content found"
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+    return f"# {title}\n\n{markdown_content[:4096]}"

backend/tests/test_fastcrw_tools.py

@@ -0,0 +1,123 @@
+"""Unit tests for the fastCRW community tools."""
+
+import json
+from unittest.mock import MagicMock, patch
+
+
+class TestWebSearchTool:
+    @patch.dict("os.environ", {}, clear=True)
+    @patch("deerflow.community.fastcrw.tools.FirecrawlApp")
+    @patch("deerflow.community.fastcrw.tools.get_app_config")
+    def test_search_uses_web_search_config(self, mock_get_app_config, mock_fastcrw_cls):
+        search_config = MagicMock()
+        search_config.model_extra = {"api_key": "fastcrw-search-key", "max_results": 7}
+        mock_get_app_config.return_value.get_tool_config.return_value = search_config
+
+        mock_result = MagicMock()
+        mock_result.web = [
+            MagicMock(title="Result", url="https://example.com", description="Snippet"),
+        ]
+        mock_fastcrw_cls.return_value.search.return_value = mock_result
+
+        from deerflow.community.fastcrw.tools import web_search_tool
+
+        result = web_search_tool.invoke({"query": "test query"})
+
+        assert json.loads(result) == [
+            {
+                "title": "Result",
+                "url": "https://example.com",
+                "snippet": "Snippet",
+            }
+        ]
+        mock_get_app_config.return_value.get_tool_config.assert_called_with("web_search")
+        mock_fastcrw_cls.assert_called_once_with(api_key="fastcrw-search-key", api_url="https://fastcrw.com/api")
+        mock_fastcrw_cls.return_value.search.assert_called_once_with("test query", limit=7)
+
+    @patch.dict("os.environ", {"CRW_API_KEY": "env-key", "CRW_API_URL": "http://self-hosted:3000"}, clear=True)
+    @patch("deerflow.community.fastcrw.tools.FirecrawlApp")
+    @patch("deerflow.community.fastcrw.tools.get_app_config")
+    def test_search_falls_back_to_env_and_default_max_results(self, mock_get_app_config, mock_fastcrw_cls):
+        mock_get_app_config.return_value.get_tool_config.return_value = None
+
+        mock_result = MagicMock()
+        mock_result.web = []
+        mock_fastcrw_cls.return_value.search.return_value = mock_result
+
+        from deerflow.community.fastcrw.tools import web_search_tool
+
+        result = web_search_tool.invoke({"query": "q"})
+
+        assert result == "[]"
+        mock_fastcrw_cls.assert_called_once_with(api_key="env-key", api_url="http://self-hosted:3000")
+        mock_fastcrw_cls.return_value.search.assert_called_once_with("q", limit=5)
+
+    @patch.dict("os.environ", {}, clear=True)
+    @patch("deerflow.community.fastcrw.tools.FirecrawlApp")
+    @patch("deerflow.community.fastcrw.tools.get_app_config")
+    def test_search_returns_error_string_on_exception(self, mock_get_app_config, mock_fastcrw_cls):
+        mock_get_app_config.return_value.get_tool_config.return_value = None
+        mock_fastcrw_cls.return_value.search.side_effect = RuntimeError("boom")
+
+        from deerflow.community.fastcrw.tools import web_search_tool
+
+        assert web_search_tool.invoke({"query": "q"}) == "Error: boom"
+
+
+class TestWebFetchTool:
+    @patch.dict("os.environ", {}, clear=True)
+    @patch("deerflow.community.fastcrw.tools.FirecrawlApp")
+    @patch("deerflow.community.fastcrw.tools.get_app_config")
+    def test_fetch_uses_web_fetch_config(self, mock_get_app_config, mock_fastcrw_cls):
+        fetch_config = MagicMock()
+        fetch_config.model_extra = {"api_key": "fastcrw-fetch-key", "base_url": "http://localhost:3000"}
+
+        def get_tool_config(name):
+            if name == "web_fetch":
+                return fetch_config
+            return None
+
+        mock_get_app_config.return_value.get_tool_config.side_effect = get_tool_config
+
+        mock_scrape_result = MagicMock()
+        mock_scrape_result.markdown = "Fetched markdown"
+        mock_scrape_result.metadata = MagicMock(title="Fetched Page")
+        mock_fastcrw_cls.return_value.scrape.return_value = mock_scrape_result
+
+        from deerflow.community.fastcrw.tools import web_fetch_tool
+
+        result = web_fetch_tool.invoke({"url": "https://example.com"})
+
+        assert result == "# Fetched Page\n\nFetched markdown"
+        mock_get_app_config.return_value.get_tool_config.assert_any_call("web_fetch")
+        mock_fastcrw_cls.assert_called_once_with(api_key="fastcrw-fetch-key", api_url="http://localhost:3000")
+        mock_fastcrw_cls.return_value.scrape.assert_called_once_with(
+            "https://example.com",
+            formats=["markdown"],
+        )
+
+    @patch.dict("os.environ", {}, clear=True)
+    @patch("deerflow.community.fastcrw.tools.FirecrawlApp")
+    @patch("deerflow.community.fastcrw.tools.get_app_config")
+    def test_fetch_returns_error_when_no_content(self, mock_get_app_config, mock_fastcrw_cls):
+        mock_get_app_config.return_value.get_tool_config.return_value = None
+
+        mock_scrape_result = MagicMock()
+        mock_scrape_result.markdown = ""
+        mock_scrape_result.metadata = MagicMock(title="Empty")
+        mock_fastcrw_cls.return_value.scrape.return_value = mock_scrape_result
+
+        from deerflow.community.fastcrw.tools import web_fetch_tool
+
+        assert web_fetch_tool.invoke({"url": "https://example.com"}) == "Error: No content found"
+
+    @patch.dict("os.environ", {}, clear=True)
+    @patch("deerflow.community.fastcrw.tools.FirecrawlApp")
+    @patch("deerflow.community.fastcrw.tools.get_app_config")
+    def test_fetch_returns_error_string_on_exception(self, mock_get_app_config, mock_fastcrw_cls):
+        mock_get_app_config.return_value.get_tool_config.return_value = None
+        mock_fastcrw_cls.return_value.scrape.side_effect = RuntimeError("scrape failed")
+
+        from deerflow.community.fastcrw.tools import web_fetch_tool
+
+        assert web_fetch_tool.invoke({"url": "https://example.com"}) == "Error: scrape failed"

config.example.yaml

@@ -570,6 +570,15 @@ tools:
   #   max_results: 5
   #   # api_key: $FIRECRAWL_API_KEY
 
+  # Web search tool (uses fastCRW - Firecrawl-compatible web scraper, single binary,
+  # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.)
+  # - name: web_search
+  #   group: web
+  #   use: deerflow.community.fastcrw.tools:web_search_tool
+  #   max_results: 5
+  #   # api_key: $CRW_API_KEY
+  #   # base_url: https://fastcrw.com/api   # default cloud; set to e.g. http://localhost:3000 for self-host
+
   # Web fetch tool (uses Browserless - headless Chrome, self-hosted or cloud)
   # Browserless renders pages with a real headless Chrome, ideal for JavaScript-heavy
   # sites and SPAs. Deploy your own: https://github.com/browserless/browserless
@@ -621,6 +630,16 @@ tools:
   #   use: deerflow.community.firecrawl.tools:web_fetch_tool
   #   # api_key: $FIRECRAWL_API_KEY
 
+  # Web fetch tool (uses fastCRW - Firecrawl-compatible web scraper, single binary,
+  # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.)
+  # NOTE: Only one web_fetch provider can be active at a time.
+  # Comment out the Jina AI web_fetch entry above before enabling this one.
+  # - name: web_fetch
+  #   group: web
+  #   use: deerflow.community.fastcrw.tools:web_fetch_tool
+  #   # api_key: $CRW_API_KEY
+  #   # base_url: https://fastcrw.com/api   # default cloud; set to e.g. http://localhost:3000 for self-host
+
   # Image search tool (uses DuckDuckGo)
   # Use this to find reference images before image generation
   - name: image_search

scripts/doctor.py

@@ -468,6 +468,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes
                 "infoquest": "INFOQUEST_API_KEY",
                 "exa": "EXA_API_KEY",
                 "firecrawl": "FIRECRAWL_API_KEY",
+                "fastcrw": "CRW_API_KEY",
                 "brave": "BRAVE_SEARCH_API_KEY",
                 "serper": "SERPER_API_KEY",
             },
@@ -475,6 +476,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes
                 "infoquest": "INFOQUEST_API_KEY",
                 "exa": "EXA_API_KEY",
                 "firecrawl": "FIRECRAWL_API_KEY",
+                "fastcrw": "CRW_API_KEY",
             },
             "image_search": {
                 "infoquest": "INFOQUEST_API_KEY",

scripts/wizard/providers.py

@@ -510,6 +510,14 @@ SEARCH_PROVIDERS: list[SearchProvider] = [
         env_var="FIRECRAWL_API_KEY",
         extra_config={"max_results": 5},
     ),
+    SearchProvider(
+        name="fastcrw",
+        display_name="fastCRW",
+        description="Firecrawl-compatible web scraper, single binary, self-host or cloud",
+        use="deerflow.community.fastcrw.tools:web_search_tool",
+        env_var="CRW_API_KEY",
+        extra_config={"max_results": 5},
+    ),
     SearchProvider(
         name="brave",
         display_name="Brave Search",
@@ -555,4 +563,12 @@ WEB_FETCH_PROVIDERS: list[WebProvider] = [
         env_var="FIRECRAWL_API_KEY",
         tool_name="web_fetch",
     ),
+    WebProvider(
+        name="fastcrw",
+        display_name="fastCRW",
+        description="Firecrawl-compatible web scraper with markdown output, self-host or cloud",
+        use="deerflow.community.fastcrw.tools:web_fetch_tool",
+        env_var="CRW_API_KEY",
+        tool_name="web_fetch",
+    ),
 ]