From 1f74082987e983ea7b9e60853291abacf9af1568 Mon Sep 17 00:00:00 2001 From: ajayr Date: Thu, 2 Jul 2026 04:22:42 +0100 Subject: [PATCH] feat(community): add Crawl4AI web_fetch provider (#3821) * feat(community): add Crawl4AI web_fetch provider Crawl4AI is a self-hosted, no-API-key web fetcher: it runs headless Chromium and returns server-cleaned "fit" markdown directly via its POST /md endpoint, so no client-side readability extraction is needed. It sits alongside the existing self-hosted Browserless provider. - deerflow.community.crawl4ai: async Crawl4AiClient + web_fetch_tool (reads base_url/timeout_s/token/filter from config; "Error:" string convention; 4096-char cap), mirroring the browserless provider - tests: 17 unit cases (success, HTTP error, success:false, empty, timeout, request error, token header, truncation, config reads) - config.example.yaml: commented web_fetch example - doctor: register as a no-key (free) web_fetch provider - setup wizard: add to WEB_FETCH_PROVIDERS (no API key) - docs: README, CONTRIBUTING, CONFIGURATION, AGENTS provider lists Co-Authored-By: Claude Opus 4.8 (1M context) * fix(community): address Crawl4AI provider review feedback - timeout: robust _coerce_timeout (bool / non-numeric -> default) mirroring jina, so 'timeout: off' no longer becomes 0.0 and times out every request - read web_fetch config once per invocation and pass values into the client, so a concurrent hot-reload can't split base_url from filter - rename config key timeout_s -> timeout to match jina/infoquest (the default providers); update config.example.yaml + setup wizard - validate + normalize the markdown filter against {fit,raw,bm25,llm}; unknown values fall back to fit with a warning instead of an opaque HTTP 400 - client: a non-JSON 200 body (reverse proxy / auth wall) now reports the content-type + snippet instead of a generic JSONDecodeError - tests: 22 cases (added non-JSON-200, _coerce_timeout, _coerce_filter, invalid-filter fallback, read-config-once) Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: DanielWalnut <45447813+hetaoBackend@users.noreply.github.com> Co-authored-by: Willem Jiang --- backend/AGENTS.md | 2 +- backend/CONTRIBUTING.md | 1 + backend/README.md | 2 +- backend/docs/CONFIGURATION.md | 2 +- .../deerflow/community/crawl4ai/__init__.py | 4 + .../community/crawl4ai/crawl4ai_client.py | 63 ++++ .../deerflow/community/crawl4ai/tools.py | 100 ++++++ backend/tests/test_crawl4ai_tools.py | 296 ++++++++++++++++++ config.example.yaml | 15 + scripts/doctor.py | 2 +- scripts/wizard/providers.py | 9 + 11 files changed, 492 insertions(+), 4 deletions(-) create mode 100644 backend/packages/harness/deerflow/community/crawl4ai/__init__.py create mode 100644 backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py create mode 100644 backend/packages/harness/deerflow/community/crawl4ai/tools.py create mode 100644 backend/tests/test_crawl4ai_tools.py diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 720c4e257..b2a1b99c4 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -354,7 +354,7 @@ Proxied through nginx: `/api/langgraph/*` → Gateway LangGraph-compatible runti - `image_search/` - Image search - `aio_sandbox/` - Docker-based isolation (`AioSandboxProvider`) -Additional providers also live here (`brave`, `browserless`, `ddg_search`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`); see each subpackage for specifics. +Additional providers also live here (`brave`, `browserless`, `crawl4ai`, `ddg_search`, `exa`, `fastcrw`, `groundroute`, `infoquest`, `searxng`, `serper`); see each subpackage for specifics. **ACP agent tools**: - `invoke_acp_agent` - Invokes external ACP-compatible agents from `config.yaml` diff --git a/backend/CONTRIBUTING.md b/backend/CONTRIBUTING.md index e4467b6db..fb80615ba 100644 --- a/backend/CONTRIBUTING.md +++ b/backend/CONTRIBUTING.md @@ -115,6 +115,7 @@ backend/src/ │ ├── jina/ # Jina web fetch │ ├── firecrawl/ # Firecrawl scraping │ ├── fastcrw/ # fastCRW scraping (Firecrawl-compatible) +│ ├── crawl4ai/ # Crawl4AI web fetch (self-hosted, no key) │ └── aio_sandbox/ # Docker sandbox │ ├── reflection/ # Dynamic loading diff --git a/backend/README.md b/backend/README.md index 51a79a37a..13eb130f9 100644 --- a/backend/README.md +++ b/backend/README.md @@ -101,7 +101,7 @@ LLM-powered persistent context retention across conversations: |----------|-------| | **Sandbox** | `bash`, `ls`, `read_file`, `write_file`, `str_replace` | | **Built-in** | `present_files`, `ask_clarification`, `view_image`, `task` (subagent) | -| **Community** | Tavily (web search), Jina AI (web fetch), Firecrawl (scraping), fastCRW (scraping), DuckDuckGo (image search) | +| **Community** | Tavily (web search), Jina AI (web fetch), Crawl4AI (web fetch), Firecrawl (scraping), fastCRW (scraping), DuckDuckGo (image search) | | **MCP** | Any Model Context Protocol server (stdio, SSE, HTTP transports) | | **Skills** | Domain-specific workflows injected via system prompt | diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 415e725ca..3071fcd70 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -235,7 +235,7 @@ tools: **Built-in Tools**: - `web_search` - Search the web (DuckDuckGo, Tavily, Brave, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute) -- `web_fetch` - Fetch web pages (Jina AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless) +- `web_fetch` - Fetch web pages (Jina AI, Crawl4AI, Exa, InfoQuest, Firecrawl, fastCRW, GroundRoute, Browserless) - `web_capture` - Capture rendered webpage screenshots as artifacts (Browserless) - `image_search` - Search for reference images (DuckDuckGo, InfoQuest, Serper, Brave) - `ls` - List directory contents diff --git a/backend/packages/harness/deerflow/community/crawl4ai/__init__.py b/backend/packages/harness/deerflow/community/crawl4ai/__init__.py new file mode 100644 index 000000000..3c5116fcc --- /dev/null +++ b/backend/packages/harness/deerflow/community/crawl4ai/__init__.py @@ -0,0 +1,4 @@ +from .crawl4ai_client import Crawl4AiClient +from .tools import web_fetch_tool + +__all__ = ["Crawl4AiClient", "web_fetch_tool"] diff --git a/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py b/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py new file mode 100644 index 000000000..0a97de809 --- /dev/null +++ b/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py @@ -0,0 +1,63 @@ +import json +import logging +from typing import Any + +import httpx + +logger = logging.getLogger(__name__) + + +class Crawl4AiClient: + """Client for a self-hosted Crawl4AI Docker server (POST /md).""" + + def __init__(self, base_url: str, token: str = "", timeout_s: float = 30.0) -> None: + self.base_url = base_url.rstrip("/") + self.token = token + self.timeout_s = timeout_s + + async def fetch_markdown(self, url: str, filter_mode: str = "fit") -> str: + """Fetch a page's clean markdown via Crawl4AI's POST /md endpoint. + + Args: + url: The URL to fetch. + filter_mode: Crawl4AI markdown filter ("fit", "raw", "bm25", "llm"). + + Returns: + Markdown content, or an "Error: ..." string on failure. + """ + payload: dict[str, Any] = {"url": url, "f": filter_mode} + headers = {"Content-Type": "application/json"} + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + + logger.debug(f"Fetching URL via Crawl4AI: {url}") + try: + async with httpx.AsyncClient(timeout=self.timeout_s) as client: + resp = await client.post(f"{self.base_url}/md", json=payload, headers=headers) + + if resp.status_code != 200: + return f"Error: Crawl4AI HTTP {resp.status_code}: {resp.text[:200]}" + + try: + data = resp.json() + except (json.JSONDecodeError, ValueError): + content_type = resp.headers.get("content-type", "unknown") + return f"Error: Crawl4AI returned a non-JSON 200 response (content-type: {content_type}): {resp.text[:200]}" + + if not data.get("success", False): + return f"Error: Crawl4AI reported failure for {url}" + + markdown = data.get("markdown") or "" + if not markdown.strip(): + return "Error: Crawl4AI returned empty markdown" + + return markdown + + except httpx.TimeoutException: + return f"Error: Crawl4AI request timed out after {self.timeout_s}s" + except httpx.RequestError as e: + logger.error(f"Crawl4AI request failed: {e}") + return f"Error: Crawl4AI request failed: {e!s}" + except Exception as e: + logger.error(f"Crawl4AI fetch failed: {e}") + return f"Error: Crawl4AI fetch failed: {e!s}" diff --git a/backend/packages/harness/deerflow/community/crawl4ai/tools.py b/backend/packages/harness/deerflow/community/crawl4ai/tools.py new file mode 100644 index 000000000..5971c892e --- /dev/null +++ b/backend/packages/harness/deerflow/community/crawl4ai/tools.py @@ -0,0 +1,100 @@ +import logging + +from langchain.tools import tool + +from deerflow.config import get_app_config + +from .crawl4ai_client import Crawl4AiClient + +logger = logging.getLogger(__name__) + +DEFAULT_BASE_URL = "http://localhost:11235" +DEFAULT_TIMEOUT_S = 30 +DEFAULT_FILTER = "fit" +VALID_FILTERS = ("fit", "raw", "bm25", "llm") + + +def _get_tool_config(tool_name: str) -> dict | None: + """Return the tool's config extras (model_extra) dict, or None if unconfigured.""" + config = get_app_config().get_tool_config(tool_name) + if config is None: + return None + extras = config.model_extra + return extras if extras is not None else {} + + +def _coerce_timeout(value: object, default: int) -> float: + """Coerce a config timeout into seconds, falling back to ``default`` on bad input. + + Mirrors ``jina_ai._coerce_timeout``: booleans and non-numeric strings fall + back to the default so e.g. ``timeout: off`` (YAML ``False``) does not become + ``0.0`` and time out every request against a healthy server. + """ + if isinstance(value, bool): + return float(default) + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + logger.warning("Crawl4AI web_fetch: invalid timeout %r in config; using %ss", value, default) + return float(default) + + +def _coerce_filter(value: object) -> str: + """Normalize and validate the markdown filter, falling back to the default. + + Catches typos / stale values (e.g. ``FIt``, ``fit_content``) at config-read + time instead of letting them reach the server as an opaque HTTP 400. + """ + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in VALID_FILTERS: + return normalized + logger.warning("Crawl4AI web_fetch: unknown filter %r in config; using %r (valid: %s)", value, DEFAULT_FILTER, ", ".join(VALID_FILTERS)) + return DEFAULT_FILTER + + +def _build_client(cfg: dict | None) -> Crawl4AiClient: + """Build a ``Crawl4AiClient`` from an already-read ``web_fetch`` config dict. + + Takes the config as an argument (rather than reading it again) so a single + invocation reads ``get_app_config()`` exactly once and cannot split across a + concurrent hot-reload. + """ + base_url = DEFAULT_BASE_URL + token = "" + timeout_s: float = float(DEFAULT_TIMEOUT_S) + if cfg is not None: + base_url = cfg.get("base_url", base_url) + token = cfg.get("token", token) + timeout_s = _coerce_timeout(cfg.get("timeout"), DEFAULT_TIMEOUT_S) + return Crawl4AiClient(base_url=base_url, token=token, timeout_s=timeout_s) + + +@tool("web_fetch", parse_docstring=True) +async def web_fetch_tool(url: str) -> str: + """Fetch the contents of a web page at a given URL. + Only fetch EXACT URLs that have been provided directly by the user or have been returned in results from the web_search and web_fetch tools. + This tool can NOT access content that requires authentication, such as private Google Docs or pages behind login walls. + Do NOT add www. to URLs that do NOT have them. + URLs must include the schema: https://example.com is a valid URL while example.com is an invalid URL. + + Args: + url: The URL to fetch the contents of. + """ + try: + cfg = _get_tool_config("web_fetch") # read config once; pass the values down + filter_mode = _coerce_filter(cfg.get("filter") if cfg is not None else None) + client = _build_client(cfg) + markdown = await client.fetch_markdown(url, filter_mode=filter_mode) + + if markdown.startswith("Error:"): + return markdown + + return markdown[:4096] + + except Exception as e: + logger.error(f"Error in web_fetch_tool: {e}") + return f"Error: {str(e)}" diff --git a/backend/tests/test_crawl4ai_tools.py b/backend/tests/test_crawl4ai_tools.py new file mode 100644 index 000000000..b1a4a57bc --- /dev/null +++ b/backend/tests/test_crawl4ai_tools.py @@ -0,0 +1,296 @@ +"""Tests for Crawl4AI community tools.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from deerflow.community.crawl4ai.crawl4ai_client import Crawl4AiClient + + +class AsyncMock(MagicMock): + """Mock that supports async call.""" + + async def __call__(self, *args, **kwargs): + return super().__call__(*args, **kwargs) + + +@pytest.mark.asyncio +class TestCrawl4AiClient: + """Tests for the Crawl4AiClient class.""" + + async def test_fetch_markdown_success(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"markdown": "# Title\n\nHello", "success": True} + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + + assert result == "# Title\n\nHello" + call = mock_ctx.post.call_args + assert call.args[0].endswith("/md") + assert call.kwargs["json"]["url"] == "https://example.com" + assert call.kwargs["json"]["f"] == "fit" + + async def test_fetch_markdown_strips_trailing_slash_in_base_url(self): + client = Crawl4AiClient(base_url="http://crawl4ai:11235/") + assert client.base_url == "http://crawl4ai:11235" + + async def test_fetch_markdown_http_error(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 502 + mock_resp.text = "Bad Gateway" + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + assert "Error: Crawl4AI HTTP 502" in result + + async def test_fetch_markdown_success_false(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"markdown": "", "success": False} + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + assert result.startswith("Error:") + + async def test_fetch_markdown_empty(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"markdown": " ", "success": True} + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + assert result == "Error: Crawl4AI returned empty markdown" + + async def test_fetch_markdown_timeout(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + import httpx + + mock_ctx.post = AsyncMock(side_effect=httpx.TimeoutException("Timed out")) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235", timeout_s=10) + result = await client.fetch_markdown("https://example.com") + assert "timed out" in result.lower() or "timeout" in result.lower() + + async def test_fetch_markdown_with_token(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"markdown": "ok", "success": True} + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235", token="secret") + await client.fetch_markdown("https://example.com") + + headers = mock_ctx.post.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer secret" + + async def test_fetch_markdown_no_token_header_when_unset(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.return_value = {"markdown": "ok", "success": True} + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + await client.fetch_markdown("https://example.com") + + headers = mock_ctx.post.call_args.kwargs["headers"] + assert "Authorization" not in headers + + async def test_fetch_markdown_request_error(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + import httpx + + mock_ctx.post = AsyncMock(side_effect=httpx.ConnectError("connection refused")) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + assert result.startswith("Error: Crawl4AI request failed") + + async def test_fetch_markdown_non_json_200(self): + with patch("deerflow.community.crawl4ai.crawl4ai_client.httpx.AsyncClient") as mock_cls: + mock_ctx = MagicMock() + mock_cls.return_value.__aenter__.return_value = mock_ctx + + mock_resp = MagicMock() + mock_resp.status_code = 200 + mock_resp.json.side_effect = json.JSONDecodeError("Expecting value", "doc", 0) + mock_resp.headers = {"content-type": "text/html"} + mock_resp.text = "login wall" + mock_ctx.post = AsyncMock(return_value=mock_resp) + + client = Crawl4AiClient(base_url="http://crawl4ai:11235") + result = await client.fetch_markdown("https://example.com") + assert result.startswith("Error: Crawl4AI returned a non-JSON 200 response") + assert "text/html" in result + + +@pytest.mark.asyncio +class TestCrawl4AiTools: + """Tests for the Crawl4AI tool functions.""" + + @patch("deerflow.community.crawl4ai.tools._build_client") + async def test_web_fetch_tool_success(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="# Title\n\nContent") + mock_build.return_value = mock_client + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value=None): + result = await tools.web_fetch_tool.ainvoke("https://example.com/article") + + assert result == "# Title\n\nContent" + assert "Error:" not in result + + @patch("deerflow.community.crawl4ai.tools._build_client") + async def test_web_fetch_tool_truncates_to_4096(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="x" * 5000) + mock_build.return_value = mock_client + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value=None): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert len(result) == 4096 + + @patch("deerflow.community.crawl4ai.tools._build_client") + async def test_web_fetch_tool_error_passthrough(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="Error: Crawl4AI returned empty markdown") + mock_build.return_value = mock_client + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value=None): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert result.startswith("Error:") + + @patch("deerflow.community.crawl4ai.tools._build_client") + async def test_web_fetch_tool_exception(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(side_effect=Exception("boom")) + mock_build.return_value = mock_client + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value=None): + result = await tools.web_fetch_tool.ainvoke("https://example.com") + + assert result.startswith("Error:") + + @patch("deerflow.community.crawl4ai.tools._build_client") + async def test_web_fetch_tool_reads_config_once(self, mock_build): + """Config is read exactly once per invocation (no split read on hot-reload).""" + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="# ok") + mock_build.return_value = mock_client + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value={}) as mock_cfg: + await tools.web_fetch_tool.ainvoke("https://example.com") + + mock_cfg.assert_called_once_with("web_fetch") + + @patch("deerflow.community.crawl4ai.tools._build_client") + async def test_web_fetch_tool_passes_configured_filter(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="# ok") + mock_build.return_value = mock_client + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value={"filter": "raw"}): + await tools.web_fetch_tool.ainvoke("https://example.com") + + mock_client.fetch_markdown.assert_called_once() + assert mock_client.fetch_markdown.call_args.kwargs.get("filter_mode") == "raw" + + @patch("deerflow.community.crawl4ai.tools._build_client") + async def test_web_fetch_tool_invalid_filter_falls_back_to_fit(self, mock_build): + from deerflow.community.crawl4ai import tools + + mock_client = MagicMock() + mock_client.fetch_markdown = AsyncMock(return_value="# ok") + mock_build.return_value = mock_client + + with patch("deerflow.community.crawl4ai.tools._get_tool_config", return_value={"filter": "BOGUS"}): + await tools.web_fetch_tool.ainvoke("https://example.com") + + assert mock_client.fetch_markdown.call_args.kwargs.get("filter_mode") == "fit" + + async def test_build_client_reads_config(self): + from deerflow.community.crawl4ai import tools + + client = tools._build_client({"base_url": "http://host.docker.internal:11235", "timeout": 45}) + assert client.base_url == "http://host.docker.internal:11235" + assert client.timeout_s == 45.0 + + async def test_build_client_defaults_when_unconfigured(self): + from deerflow.community.crawl4ai import tools + + client = tools._build_client(None) + assert client.base_url == "http://localhost:11235" + assert client.timeout_s == 30.0 + assert client.token == "" + + async def test_build_client_reads_token(self): + from deerflow.community.crawl4ai import tools + + client = tools._build_client({"token": "secret-token"}) + assert client.token == "secret-token" + + async def test_coerce_timeout_handles_bool_and_bad_values(self): + from deerflow.community.crawl4ai import tools + + assert tools._coerce_timeout(True, 30) == 30.0 + assert tools._coerce_timeout(False, 30) == 30.0 + assert tools._coerce_timeout("thirty", 30) == 30.0 + assert tools._coerce_timeout("45", 30) == 45.0 + assert tools._coerce_timeout(20, 30) == 20.0 + assert tools._coerce_timeout(12.5, 30) == 12.5 + + async def test_coerce_filter_validates_and_normalizes(self): + from deerflow.community.crawl4ai import tools + + assert tools._coerce_filter("raw") == "raw" + assert tools._coerce_filter(" FIt ") == "fit" + assert tools._coerce_filter("bogus") == "fit" + assert tools._coerce_filter(None) == "fit" diff --git a/config.example.yaml b/config.example.yaml index f6cf154c1..207bc7034 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -663,6 +663,21 @@ tools: # # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds # # wait_for_selector: "article" # CSS selector to wait for before returning + # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no API key) + # Crawl4AI returns server-cleaned "fit" markdown directly (no readability step needed), + # ideal for JavaScript-heavy sites. Self-host (JWT auth is off by default, no key): + # docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.8.6 + # For Docker deployments, use the Docker service name instead of localhost. + # NOTE: Only one web_fetch provider can be active at a time. + # Comment out the Jina AI web_fetch entry below before enabling this one. + # - name: web_fetch + # group: web + # use: deerflow.community.crawl4ai.tools:web_fetch_tool + # base_url: http://localhost:11235 # Crawl4AI server URL (Docker: http://crawl4ai:11235) + # timeout: 30 # Request timeout in seconds + # # filter: fit # Markdown filter: fit (default) | raw | bm25 | llm + # # token: $CRAWL4AI_TOKEN # Bearer token (only if the server has JWT auth enabled) + # Web capture tool (uses Browserless /screenshot to render a page as an artifact) # Browserless captures JavaScript-heavy pages with a real headless Chrome and # writes the screenshot into the current thread's outputs. It can run against diff --git a/scripts/doctor.py b/scripts/doctor.py index 854e1e738..33dd0a911 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -459,7 +459,7 @@ def check_web_tool(config_path: Path, *, tool_name: str, label: str) -> CheckRes free_providers = { "web_search": {"ddg_search": "DuckDuckGo (no key needed)"}, - "web_fetch": {"jina_ai": "Jina AI Reader (no key needed)"}, + "web_fetch": {"jina_ai": "Jina AI Reader (no key needed)", "crawl4ai": "Crawl4AI (self-hosted, no key needed)"}, "image_search": {"deerflow.community.image_search.tools": "DuckDuckGo Images (no key needed)"}, } key_providers = { diff --git a/scripts/wizard/providers.py b/scripts/wizard/providers.py index 0a81db47b..cae28b3fa 100644 --- a/scripts/wizard/providers.py +++ b/scripts/wizard/providers.py @@ -587,4 +587,13 @@ WEB_FETCH_PROVIDERS: list[WebProvider] = [ env_var="CRW_API_KEY", tool_name="web_fetch", ), + WebProvider( + name="crawl4ai", + display_name="Crawl4AI", + description="Self-hosted headless Chromium with markdown output, no API key required", + use="deerflow.community.crawl4ai.tools:web_fetch_tool", + env_var=None, + tool_name="web_fetch", + extra_config={"base_url": "http://localhost:11235", "timeout": 30}, + ), ]