From bebbd2cb35313321f90df6f99a46c85461c0fd44 Mon Sep 17 00:00:00 2001
From: yetuge <182123929+yetuge@users.noreply.github.com>
Date: Thu, 5 Nov 2026 14:24:21 +0800
Subject: [PATCH] fix(crawl4ai): construct the fetch client off the event loop
 (#7906)

* fix(crawl4ai): construct the fetch client off the event loop

Crawl4AiClient builds an httpx.AsyncClient per fetch, and AsyncClient
construction loads the TLS trust store synchronously: on the agent event
loop that blocks every other request for the CA bundle read. Construct it
through asyncio.to_thread, the same rule the RAGFlow/LightRAG and
You.com/GroundRoute/Webz clients already follow; the request and cleanup
stay on the loop.

* docs(agents): list Crawl4AI among the off-loop HTTPX clients

#7889 extended the same sentence for its providers; keep the named list in
the same change set as the client, and record the clients that still
construct on the loop so the next migration has a checklist.
---
 backend/AGENTS.md                             |  5 ++-
 .../community/crawl4ai/crawl4ai_client.py     |  4 +-
 .../blocking_io/test_crawl4ai_client_ssl.py   | 41 +++++++++++++++++++
 3 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 backend/tests/blocking_io/test_crawl4ai_client_ssl.py

diff --git a/backend/AGENTS.md b/backend/AGENTS.md
index 6d8d2bcb13..d0d1bd7700 100644
--- a/backend/AGENTS.md
+++ b/backend/AGENTS.md
@@ -136,8 +136,11 @@ Read the nearest scoped `AGENTS.md` for subsystem ownership.
 
 Read-only knowledge: [contract](docs/KNOWLEDGE.md) covers independent retrieval
 tools, scope intersection and extension-owned providers.
-HTTPX clients for RAGFlow, LightRAG, You.com, GroundRoute and Webz use
+HTTPX clients for RAGFlow, LightRAG, You.com, GroundRoute, Webz and Crawl4AI use
 `asyncio.to_thread` for synchronous TLS setup; requests and cleanup stay on-loop.
+The list is non-exhaustive: jsonify, SearXNG, Browserless, Jina and InfoQuest
+still construct their clients on the loop and should follow the same rule when
+touched.
 
 ### Harness / App Split
 
diff --git a/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py b/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py
index eaa68e971f..c9e64893ad 100644
--- a/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py
+++ b/backend/packages/harness/deerflow/community/crawl4ai/crawl4ai_client.py
@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 from typing import Any
@@ -32,7 +33,8 @@ class Crawl4AiClient:
 
         logger.debug(f"Fetching URL via Crawl4AI: {url}")
         try:
-            async with httpx.AsyncClient(timeout=self.timeout_s) as client:
+            # Client construction loads TLS trust synchronously; keep it off the loop.
+            async with await asyncio.to_thread(httpx.AsyncClient, timeout=self.timeout_s) as client:
                 resp = await client.post(f"{self.base_url}/md", json=payload, headers=headers)
 
                 if resp.status_code != 200:
diff --git a/backend/tests/blocking_io/test_crawl4ai_client_ssl.py b/backend/tests/blocking_io/test_crawl4ai_client_ssl.py
new file mode 100644
index 0000000000..bd36c5e619
--- /dev/null
+++ b/backend/tests/blocking_io/test_crawl4ai_client_ssl.py
@@ -0,0 +1,41 @@
+"""Crawl4AI's HTTPX client must load TLS trust off the agent event loop."""
+
+import ssl
+import threading
+
+import httpx
+import pytest
+
+from deerflow.community.crawl4ai.crawl4ai_client import Crawl4AiClient
+
+
+@pytest.mark.asyncio
+async def test_crawl4ai_client_tls_setup_runs_off_loop(monkeypatch):
+    loop_thread = threading.get_ident()
+    original_context = ssl.create_default_context
+    contexts = []
+    clients = []
+
+    def create_context(*args, **kwargs):
+        assert threading.get_ident() != loop_thread, "TLS trust loading blocks the event loop"
+        context = original_context(*args, **kwargs)
+        contexts.append(context)
+        return context
+
+    async def send(self, request, **kwargs):
+        assert threading.get_ident() == loop_thread
+        clients.append(self)
+        assert request.url.host == "crawl4ai.example"
+        assert request.url.path == "/md"
+        return httpx.Response(200, request=request, json={"success": True, "markdown": "synthetic page"})
+
+    monkeypatch.setattr(ssl, "create_default_context", create_context)
+    monkeypatch.setattr(httpx.AsyncClient, "send", send)
+
+    client = Crawl4AiClient("https://crawl4ai.example")
+    result = await client.fetch_markdown("https://example.com")
+
+    assert result == "synthetic page"
+    assert contexts
+    assert len(clients) == 1
+    assert clients[0].is_closed