mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-08-16 17:58:43 +00:00
1f74082987
* feat(community): add Crawl4AI web_fetch provider
Crawl4AI is a self-hosted, no-API-key web fetcher: it runs headless
Chromium and returns server-cleaned "fit" markdown directly via its
POST /md endpoint, so no client-side readability extraction is needed.
It sits alongside the existing self-hosted Browserless provider.
- deerflow.community.crawl4ai: async Crawl4AiClient + web_fetch_tool
(reads base_url/timeout_s/token/filter from config; "Error:" string
convention; 4096-char cap), mirroring the browserless provider
- tests: 17 unit cases (success, HTTP error, success:false, empty,
timeout, request error, token header, truncation, config reads)
- config.example.yaml: commented web_fetch example
- doctor: register as a no-key (free) web_fetch provider
- setup wizard: add to WEB_FETCH_PROVIDERS (no API key)
- docs: README, CONTRIBUTING, CONFIGURATION, AGENTS provider lists
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
* fix(community): address Crawl4AI provider review feedback
- timeout: robust _coerce_timeout (bool / non-numeric -> default) mirroring
jina, so 'timeout: off' no longer becomes 0.0 and times out every request
- read web_fetch config once per invocation and pass values into the client,
so a concurrent hot-reload can't split base_url from filter
- rename config key timeout_s -> timeout to match jina/infoquest (the
default providers); update config.example.yaml + setup wizard
- validate + normalize the markdown filter against {fit,raw,bm25,llm};
unknown values fall back to fit with a warning instead of an opaque HTTP 400
- client: a non-JSON 200 body (reverse proxy / auth wall) now reports the
content-type + snippet instead of a generic JSONDecodeError
- tests: 22 cases (added non-JSON-200, _coerce_timeout, _coerce_filter,
invalid-filter fallback, read-config-once)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: DanielWalnut <45447813+hetaoBackend@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
64 lines
2.4 KiB
Python
64 lines
2.4 KiB
Python
import json
|
|
import logging
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class Crawl4AiClient:
|
|
"""Client for a self-hosted Crawl4AI Docker server (POST /md)."""
|
|
|
|
def __init__(self, base_url: str, token: str = "", timeout_s: float = 30.0) -> None:
|
|
self.base_url = base_url.rstrip("/")
|
|
self.token = token
|
|
self.timeout_s = timeout_s
|
|
|
|
async def fetch_markdown(self, url: str, filter_mode: str = "fit") -> str:
|
|
"""Fetch a page's clean markdown via Crawl4AI's POST /md endpoint.
|
|
|
|
Args:
|
|
url: The URL to fetch.
|
|
filter_mode: Crawl4AI markdown filter ("fit", "raw", "bm25", "llm").
|
|
|
|
Returns:
|
|
Markdown content, or an "Error: ..." string on failure.
|
|
"""
|
|
payload: dict[str, Any] = {"url": url, "f": filter_mode}
|
|
headers = {"Content-Type": "application/json"}
|
|
if self.token:
|
|
headers["Authorization"] = f"Bearer {self.token}"
|
|
|
|
logger.debug(f"Fetching URL via Crawl4AI: {url}")
|
|
try:
|
|
async with httpx.AsyncClient(timeout=self.timeout_s) as client:
|
|
resp = await client.post(f"{self.base_url}/md", json=payload, headers=headers)
|
|
|
|
if resp.status_code != 200:
|
|
return f"Error: Crawl4AI HTTP {resp.status_code}: {resp.text[:200]}"
|
|
|
|
try:
|
|
data = resp.json()
|
|
except (json.JSONDecodeError, ValueError):
|
|
content_type = resp.headers.get("content-type", "unknown")
|
|
return f"Error: Crawl4AI returned a non-JSON 200 response (content-type: {content_type}): {resp.text[:200]}"
|
|
|
|
if not data.get("success", False):
|
|
return f"Error: Crawl4AI reported failure for {url}"
|
|
|
|
markdown = data.get("markdown") or ""
|
|
if not markdown.strip():
|
|
return "Error: Crawl4AI returned empty markdown"
|
|
|
|
return markdown
|
|
|
|
except httpx.TimeoutException:
|
|
return f"Error: Crawl4AI request timed out after {self.timeout_s}s"
|
|
except httpx.RequestError as e:
|
|
logger.error(f"Crawl4AI request failed: {e}")
|
|
return f"Error: Crawl4AI request failed: {e!s}"
|
|
except Exception as e:
|
|
logger.error(f"Crawl4AI fetch failed: {e}")
|
|
return f"Error: Crawl4AI fetch failed: {e!s}"
|