fix(crawl4ai): construct the fetch client off the event loop (#7906)

* fix(crawl4ai): construct the fetch client off the event loop

Crawl4AiClient builds an httpx.AsyncClient per fetch, and AsyncClient
construction loads the TLS trust store synchronously: on the agent event
loop that blocks every other request for the CA bundle read. Construct it
through asyncio.to_thread, the same rule the RAGFlow/LightRAG and
You.com/GroundRoute/Webz clients already follow; the request and cleanup
stay on the loop.

* docs(agents): list Crawl4AI among the off-loop HTTPX clients

#7889 extended the same sentence for its providers; keep the named list in
the same change set as the client, and record the clients that still
construct on the loop so the next migration has a checklist.

backend/AGENTS.md

@@ -136,8 +136,11 @@ Read the nearest scoped `AGENTS.md` for subsystem ownership.
 
 Read-only knowledge: [contract](docs/KNOWLEDGE.md) covers independent retrieval
 tools, scope intersection and extension-owned providers.
-HTTPX clients for RAGFlow, LightRAG, You.com, GroundRoute and Webz use
+HTTPX clients for RAGFlow, LightRAG, You.com, GroundRoute, Webz and Crawl4AI use
 `asyncio.to_thread` for synchronous TLS setup; requests and cleanup stay on-loop.
+The list is non-exhaustive: jsonify, SearXNG, Browserless, Jina and InfoQuest
+still construct their clients on the loop and should follow the same rule when
+touched.
 
 ### Harness / App Split
 

backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 from typing import Any
@@ -32,7 +33,8 @@ class Crawl4AiClient:
 
         logger.debug(f"Fetching URL via Crawl4AI: {url}")
         try:
-            async with httpx.AsyncClient(timeout=self.timeout_s) as client:
+            # Client construction loads TLS trust synchronously; keep it off the loop.
+            async with await asyncio.to_thread(httpx.AsyncClient, timeout=self.timeout_s) as client:
                 resp = await client.post(f"{self.base_url}/md", json=payload, headers=headers)
 
                 if resp.status_code != 200:

backend/tests/blocking_io/test_crawl4ai_client_ssl.py

@@ -0,0 +1,41 @@
+"""Crawl4AI's HTTPX client must load TLS trust off the agent event loop."""
+
+import ssl
+import threading
+
+import httpx
+import pytest
+
+from deerflow.community.crawl4ai.crawl4ai_client import Crawl4AiClient
+
+
+@pytest.mark.asyncio
+async def test_crawl4ai_client_tls_setup_runs_off_loop(monkeypatch):
+    loop_thread = threading.get_ident()
+    original_context = ssl.create_default_context
+    contexts = []
+    clients = []
+
+    def create_context(*args, **kwargs):
+        assert threading.get_ident() != loop_thread, "TLS trust loading blocks the event loop"
+        context = original_context(*args, **kwargs)
+        contexts.append(context)
+        return context
+
+    async def send(self, request, **kwargs):
+        assert threading.get_ident() == loop_thread
+        clients.append(self)
+        assert request.url.host == "crawl4ai.example"
+        assert request.url.path == "/md"
+        return httpx.Response(200, request=request, json={"success": True, "markdown": "synthetic page"})
+
+    monkeypatch.setattr(ssl, "create_default_context", create_context)
+    monkeypatch.setattr(httpx.AsyncClient, "send", send)
+
+    client = Crawl4AiClient("https://crawl4ai.example")
+    result = await client.fetch_markdown("https://example.com")
+
+    assert result == "synthetic page"
+    assert contexts
+    assert len(clients) == 1
+    assert clients[0].is_closed