mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-05-24 17:43:45 +00:00
84dccef230
Finish Phase 2 of the config refactor: production code no longer calls AppConfig.current() anywhere. AppConfig now flows as an explicit parameter down every consumer lane. Call-site migrations -------------------- - Memory subsystem (queue/updater/storage): MemoryConfig captured at enqueue time so the Timer closure survives the ContextVar boundary. - Sandbox layer: tools.py, security.py, sandbox_provider.py, local_sandbox_provider, aio_sandbox_provider all take app_config explicitly. Module-level caching in tools.py's path helpers is removed — pure parameter flow. - Skills layer: manager.py + loader.py + lead_agent.prompt cache refresh all thread app_config; cache worker closes over it. - Community tools (tavily, jina, firecrawl, exa, ddg, image_search, infoquest, aio_sandbox): read runtime.context.app_config. - Subagents registry: get_subagent_config / list_subagents / get_available_subagent_names require app_config. - Runtime worker: requires RunContext.app_config; no fallback. - Gateway routers (uploads, skills): add Depends(get_config). - Channels feishu: uses AppConfig.from_file() (pure) at its sync boundary. - LangGraph Server bootstrap (make_lead_agent): falls back to AppConfig.from_file() — pure load, not ambient lookup. Context resolution ------------------ - resolve_context(runtime) now raises on non-DeerFlowContext runtime.context. Every entry point attaches typed context; dict/None shapes are rejected loudly instead of being papered over with an ambient AppConfig lookup. AppConfig lifecycle ------------------- - AppConfig.current() kept as a deprecated slot that raises RuntimeError, purely so legacy tests that still run `patch.object(AppConfig, "current")` don't trip AttributeError at teardown. Production never calls it. - conftest autouse fixture no longer monkey-patches `current` — it only stubs `from_file()` so tests don't need a real config.yaml. Design refs ----------- - docs/plans/2026-04-12-config-refactor-plan.md (Phase 2: P2-6..P2-10) - docs/plans/2026-04-12-config-refactor-design.md §8 All 2338 non-e2e tests pass. Zero AppConfig.current() call sites remain in backend/packages or backend/app (docstrings in deps.py excepted).
189 lines
7.2 KiB
Python
189 lines
7.2 KiB
Python
"""Tests for JinaClient async crawl method."""
|
|
|
|
import logging
|
|
from unittest.mock import MagicMock
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
import deerflow.community.jina_ai.jina_client as jina_client_module
|
|
from deerflow.community.jina_ai.jina_client import JinaClient
|
|
from deerflow.community.jina_ai.tools import web_fetch_tool
|
|
from deerflow.config.app_config import AppConfig
|
|
|
|
# --- Phase 2 test helper: injected runtime for community tools ---
|
|
from types import SimpleNamespace as _P2NS
|
|
from deerflow.config.app_config import AppConfig as _P2AppConfig
|
|
from deerflow.config.sandbox_config import SandboxConfig as _P2SandboxConfig
|
|
from deerflow.config.deer_flow_context import DeerFlowContext as _P2Ctx
|
|
_P2_APP_CONFIG = _P2AppConfig(sandbox=_P2SandboxConfig(use="test"))
|
|
_P2_RUNTIME = _P2NS(context=_P2Ctx(app_config=_P2_APP_CONFIG, thread_id="test-thread"))
|
|
# -------------------------------------------------------------------
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
def jina_client():
|
|
return JinaClient()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_success(jina_client, monkeypatch):
|
|
"""Test successful crawl returns response text."""
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
return httpx.Response(200, text="<html><body>Hello</body></html>", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
result = await jina_client.crawl("https://example.com")
|
|
assert result == "<html><body>Hello</body></html>"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_non_200_status(jina_client, monkeypatch):
|
|
"""Test that non-200 status returns error message."""
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
return httpx.Response(429, text="Rate limited", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
result = await jina_client.crawl("https://example.com")
|
|
assert result.startswith("Error:")
|
|
assert "429" in result
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_empty_response(jina_client, monkeypatch):
|
|
"""Test that empty response returns error message."""
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
return httpx.Response(200, text="", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
result = await jina_client.crawl("https://example.com")
|
|
assert result.startswith("Error:")
|
|
assert "empty" in result.lower()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_whitespace_only_response(jina_client, monkeypatch):
|
|
"""Test that whitespace-only response returns error message."""
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
return httpx.Response(200, text=" \n ", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
result = await jina_client.crawl("https://example.com")
|
|
assert result.startswith("Error:")
|
|
assert "empty" in result.lower()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_network_error(jina_client, monkeypatch):
|
|
"""Test that network errors are handled gracefully."""
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
raise httpx.ConnectError("Connection refused")
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
result = await jina_client.crawl("https://example.com")
|
|
assert result.startswith("Error:")
|
|
assert "failed" in result.lower()
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_passes_headers(jina_client, monkeypatch):
|
|
"""Test that correct headers are sent."""
|
|
captured_headers = {}
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
captured_headers.update(kwargs.get("headers", {}))
|
|
return httpx.Response(200, text="ok", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
await jina_client.crawl("https://example.com", return_format="markdown", timeout=30)
|
|
assert captured_headers["X-Return-Format"] == "markdown"
|
|
assert captured_headers["X-Timeout"] == "30"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_includes_api_key_when_set(jina_client, monkeypatch):
|
|
"""Test that Authorization header is set when JINA_API_KEY is available."""
|
|
captured_headers = {}
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
captured_headers.update(kwargs.get("headers", {}))
|
|
return httpx.Response(200, text="ok", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
monkeypatch.setenv("JINA_API_KEY", "test-key-123")
|
|
await jina_client.crawl("https://example.com")
|
|
assert captured_headers["Authorization"] == "Bearer test-key-123"
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_warns_once_when_api_key_missing(jina_client, monkeypatch, caplog):
|
|
"""Test that the missing API key warning is logged only once."""
|
|
jina_client_module._api_key_warned = False
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
return httpx.Response(200, text="ok", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
monkeypatch.delenv("JINA_API_KEY", raising=False)
|
|
|
|
with caplog.at_level(logging.WARNING, logger="deerflow.community.jina_ai.jina_client"):
|
|
await jina_client.crawl("https://example.com")
|
|
await jina_client.crawl("https://example.com")
|
|
|
|
warning_count = sum(1 for record in caplog.records if "Jina API key is not set" in record.message)
|
|
assert warning_count == 1
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_crawl_no_auth_header_without_api_key(jina_client, monkeypatch):
|
|
"""Test that no Authorization header is set when JINA_API_KEY is not available."""
|
|
jina_client_module._api_key_warned = False
|
|
captured_headers = {}
|
|
|
|
async def mock_post(self, url, **kwargs):
|
|
captured_headers.update(kwargs.get("headers", {}))
|
|
return httpx.Response(200, text="ok", request=httpx.Request("POST", url))
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "post", mock_post)
|
|
monkeypatch.delenv("JINA_API_KEY", raising=False)
|
|
await jina_client.crawl("https://example.com")
|
|
assert "Authorization" not in captured_headers
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_web_fetch_tool_returns_error_on_crawl_failure(monkeypatch):
|
|
"""Test that web_fetch_tool short-circuits and returns the error string when crawl fails."""
|
|
|
|
async def mock_crawl(self, url, **kwargs):
|
|
return "Error: Jina API returned status 429: Rate limited"
|
|
|
|
mock_config = MagicMock()
|
|
mock_config.get_tool_config.return_value = None
|
|
monkeypatch.setattr(AppConfig, "current", staticmethod(lambda: mock_config))
|
|
monkeypatch.setattr(JinaClient, "crawl", mock_crawl)
|
|
result = await web_fetch_tool.coroutine(url="https://example.com", runtime=_P2_RUNTIME)
|
|
assert result.startswith("Error:")
|
|
assert "429" in result
|
|
|
|
|
|
@pytest.mark.anyio
|
|
async def test_web_fetch_tool_returns_markdown_on_success(monkeypatch):
|
|
"""Test that web_fetch_tool returns extracted markdown on successful crawl."""
|
|
|
|
async def mock_crawl(self, url, **kwargs):
|
|
return "<html><body><p>Hello world</p></body></html>"
|
|
|
|
mock_config = MagicMock()
|
|
mock_config.get_tool_config.return_value = None
|
|
monkeypatch.setattr(AppConfig, "current", staticmethod(lambda: mock_config))
|
|
monkeypatch.setattr(JinaClient, "crawl", mock_crawl)
|
|
result = await web_fetch_tool.coroutine(url="https://example.com", runtime=_P2_RUNTIME)
|
|
assert "Hello world" in result
|
|
assert not result.startswith("Error:")
|