mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-08 03:30:34 +00:00
bebbd2cb35
* fix(crawl4ai): construct the fetch client off the event loop Crawl4AiClient builds an httpx.AsyncClient per fetch, and AsyncClient construction loads the TLS trust store synchronously: on the agent event loop that blocks every other request for the CA bundle read. Construct it through asyncio.to_thread, the same rule the RAGFlow/LightRAG and You.com/GroundRoute/Webz clients already follow; the request and cleanup stay on the loop. * docs(agents): list Crawl4AI among the off-loop HTTPX clients #7889 extended the same sentence for its providers; keep the named list in the same change set as the client, and record the clients that still construct on the loop so the next migration has a checklist.
42 lines
1.3 KiB
Python
42 lines
1.3 KiB
Python
"""Crawl4AI's HTTPX client must load TLS trust off the agent event loop."""
|
|
|
|
import ssl
|
|
import threading
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from deerflow.community.crawl4ai.crawl4ai_client import Crawl4AiClient
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_crawl4ai_client_tls_setup_runs_off_loop(monkeypatch):
|
|
loop_thread = threading.get_ident()
|
|
original_context = ssl.create_default_context
|
|
contexts = []
|
|
clients = []
|
|
|
|
def create_context(*args, **kwargs):
|
|
assert threading.get_ident() != loop_thread, "TLS trust loading blocks the event loop"
|
|
context = original_context(*args, **kwargs)
|
|
contexts.append(context)
|
|
return context
|
|
|
|
async def send(self, request, **kwargs):
|
|
assert threading.get_ident() == loop_thread
|
|
clients.append(self)
|
|
assert request.url.host == "crawl4ai.example"
|
|
assert request.url.path == "/md"
|
|
return httpx.Response(200, request=request, json={"success": True, "markdown": "synthetic page"})
|
|
|
|
monkeypatch.setattr(ssl, "create_default_context", create_context)
|
|
monkeypatch.setattr(httpx.AsyncClient, "send", send)
|
|
|
|
client = Crawl4AiClient("https://crawl4ai.example")
|
|
result = await client.fetch_markdown("https://example.com")
|
|
|
|
assert result == "synthetic page"
|
|
assert contexts
|
|
assert len(clients) == 1
|
|
assert clients[0].is_closed
|