mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-04-25 11:18:22 +00:00
e8572b9d0c
The exception handler in JinaClient.crawl used logger.exception, which emits an ERROR-level record with the full httpx/httpcore/anyio traceback for every transient network failure (timeout, connection refused). Other search/crawl providers in the project log the same class of recoverable failures as a single line. One offline/slow-network session could produce dozens of multi-frame ERROR stack traces, drowning out real problems. Switch to logger.warning with a concise message that includes the exception type and its str, matching the style used elsewhere for recoverable transient failures (aio_sandbox, ddg, etc.). The exception type now also surfaces into the returned "Error: ..." string so callers retain diagnostic signal. Adds a regression test that asserts the log record is WARNING, carries no exc_info, and includes the exception class name. Co-authored-by: voidborne-d <voidborne-d@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
44 lines
1.6 KiB
Python
44 lines
1.6 KiB
Python
import logging
|
|
import os
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_api_key_warned = False
|
|
|
|
|
|
class JinaClient:
|
|
async def crawl(self, url: str, return_format: str = "html", timeout: int = 10) -> str:
|
|
global _api_key_warned
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"X-Return-Format": return_format,
|
|
"X-Timeout": str(timeout),
|
|
}
|
|
if os.getenv("JINA_API_KEY"):
|
|
headers["Authorization"] = f"Bearer {os.getenv('JINA_API_KEY')}"
|
|
elif not _api_key_warned:
|
|
_api_key_warned = True
|
|
logger.warning("Jina API key is not set. Provide your own key to access a higher rate limit. See https://jina.ai/reader for more information.")
|
|
data = {"url": url}
|
|
try:
|
|
async with httpx.AsyncClient() as client:
|
|
response = await client.post("https://r.jina.ai/", headers=headers, json=data, timeout=timeout)
|
|
|
|
if response.status_code != 200:
|
|
error_message = f"Jina API returned status {response.status_code}: {response.text}"
|
|
logger.error(error_message)
|
|
return f"Error: {error_message}"
|
|
|
|
if not response.text or not response.text.strip():
|
|
error_message = "Jina API returned empty response"
|
|
logger.error(error_message)
|
|
return f"Error: {error_message}"
|
|
|
|
return response.text
|
|
except Exception as e:
|
|
error_message = f"Request to Jina API failed: {type(e).__name__}: {e}"
|
|
logger.warning(error_message)
|
|
return f"Error: {error_message}"
|