From 28553040feffd645b8a99393f5fa0f54f73f01cb Mon Sep 17 00:00:00 2001 From: ajayr Date: Tue, 28 Jul 2026 00:44:59 +0100 Subject: [PATCH] docs(config): Crawl4AI >= 0.9 requires a bearer token (#4518) The commented Crawl4AI web_fetch example still describes the pre-0.9 server: it says JWT auth is off by default, marks `token:` as only needed "if the server has JWT auth enabled", and pins 0.8.6 in the docker run line. Crawl4AI 0.9.0 made the Docker API server secure-by-default. Auth is on for every request except GET /health, and a server started without CRAWL4AI_API_TOKEN binds 127.0.0.1 only. Following the current example against a current image therefore yields either HTTP 401 on every fetch, or a server a containerised DeerFlow cannot reach -- with no hint that auth is the cause. 0.8.6 is also worth moving off: 0.8.7 fixed two pre-auth RCEs (CVSS 9.8), and 0.8.8/0.8.9 closed SSRF gaps in the same server. No code change is needed -- Crawl4AiClient already sends `Authorization: Bearer ` whenever `token` is set, so this is purely the example catching up with the upstream server. Comment-only, so config_version is unchanged. Co-authored-by: Claude Opus 5 --- config.example.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 81ef4f7fb..a7ab5a555 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -763,10 +763,16 @@ tools: # # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds # # wait_for_selector: "article" # CSS selector to wait for before returning - # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no API key) + # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no third-party API key) # Crawl4AI returns server-cleaned "fit" markdown directly (no readability step needed), - # ideal for JavaScript-heavy sites. Self-host (JWT auth is off by default, no key): - # docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.8.6 + # ideal for JavaScript-heavy sites. Self-host it: + # docker run -d -p 11235:11235 --shm-size=1g \ + # -e CRAWL4AI_API_TOKEN=$CRAWL4AI_TOKEN unclecode/crawl4ai:0.9.2 + # Crawl4AI >= 0.9 is secure-by-default: a bearer token is REQUIRED on every request + # except GET /health, and a server started without CRAWL4AI_API_TOKEN binds 127.0.0.1 + # only (so a container/remote DeerFlow cannot reach it at all). Set the same value in + # the server env and in `token:` below, or requests fail with HTTP 401. + # Use >= 0.8.7: earlier images, including 0.8.6, carry known pre-auth RCEs. # For Docker deployments, use the Docker service name instead of localhost. # NOTE: Only one web_fetch provider can be active at a time. # Comment out the Jina AI web_fetch entry below before enabling this one. @@ -774,10 +780,10 @@ tools: # group: web # use: deerflow.community.crawl4ai.tools:web_fetch_tool # base_url: http://localhost:11235 # Crawl4AI server URL (Docker: http://crawl4ai:11235) + # token: $CRAWL4AI_TOKEN # Bearer token; required by Crawl4AI >= 0.9 # timeout: 30 # Request timeout in seconds # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets. # # filter: fit # Markdown filter: fit (default) | raw | bm25 | llm - # # token: $CRAWL4AI_TOKEN # Bearer token (only if the server has JWT auth enabled) # Web capture tool (uses Browserless /screenshot to render a page as an artifact) # Browserless captures JavaScript-heavy pages with a real headless Chrome and