diff --git a/config.example.yaml b/config.example.yaml index 81ef4f7fb..a7ab5a555 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -763,10 +763,16 @@ tools: # # wait_for_timeout_ms: 2000 # Extra wait after page load in milliseconds # # wait_for_selector: "article" # CSS selector to wait for before returning - # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no API key) + # Web fetch tool (uses Crawl4AI - self-hosted headless Chromium, no third-party API key) # Crawl4AI returns server-cleaned "fit" markdown directly (no readability step needed), - # ideal for JavaScript-heavy sites. Self-host (JWT auth is off by default, no key): - # docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.8.6 + # ideal for JavaScript-heavy sites. Self-host it: + # docker run -d -p 11235:11235 --shm-size=1g \ + # -e CRAWL4AI_API_TOKEN=$CRAWL4AI_TOKEN unclecode/crawl4ai:0.9.2 + # Crawl4AI >= 0.9 is secure-by-default: a bearer token is REQUIRED on every request + # except GET /health, and a server started without CRAWL4AI_API_TOKEN binds 127.0.0.1 + # only (so a container/remote DeerFlow cannot reach it at all). Set the same value in + # the server env and in `token:` below, or requests fail with HTTP 401. + # Use >= 0.8.7: earlier images, including 0.8.6, carry known pre-auth RCEs. # For Docker deployments, use the Docker service name instead of localhost. # NOTE: Only one web_fetch provider can be active at a time. # Comment out the Jina AI web_fetch entry below before enabling this one. @@ -774,10 +780,10 @@ tools: # group: web # use: deerflow.community.crawl4ai.tools:web_fetch_tool # base_url: http://localhost:11235 # Crawl4AI server URL (Docker: http://crawl4ai:11235) + # token: $CRAWL4AI_TOKEN # Bearer token; required by Crawl4AI >= 0.9 # timeout: 30 # Request timeout in seconds # # allow_private_addresses: false # SSRF guard: keep false in production. Set true ONLY for intentional internal targets. # # filter: fit # Markdown filter: fit (default) | raw | bm25 | llm - # # token: $CRAWL4AI_TOKEN # Bearer token (only if the server has JWT auth enabled) # Web capture tool (uses Browserless /screenshot to render a page as an artifact) # Browserless captures JavaScript-heavy pages with a real headless Chrome and