diff --git a/backend/AGENTS.md b/backend/AGENTS.md index e77e931722..391ea2d581 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -276,6 +276,7 @@ Keep live tests opt-in via `DEER_FLOW_RUN_LIVE_TESTS=1`; guard POSIX-only markers with `os.name` for Windows collection. Jina logging tests use dummy keys (`tests/test_jina_client.py`). +The Jina API redirect regression uses an offline `httpx.MockTransport` and checks that temporary redirects keep the POST method. Jina/Browserless/InfoQuest resolve URLs without rebuilding HTML. InfoQuest connect/read timeout is 30s, separate from crawl timeouts (`tests/test_infoquest_http_timeout.py`). diff --git a/backend/README.md b/backend/README.md index 800a227721..59b59cac83 100644 --- a/backend/README.md +++ b/backend/README.md @@ -118,6 +118,8 @@ LLM-powered persistent context retention across conversations: | **MCP** | Any Model Context Protocol server (stdio, SSE, HTTP transports) | | **Skills** | Domain-specific workflows injected via system prompt | +Jina AI web fetch follows HTTP redirects from its API endpoint, including 307/308 redirects, which retain the POST method and body. + ### Run Event Storage For direct `RunEventStore.list_messages` callers, `after_seq` and `before_seq` diff --git a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py index 856a96c33b..a9f74035d2 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py +++ b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py @@ -89,7 +89,7 @@ class JinaClient: # bound the complete request sequence (including waits and cleanup). deadline = asyncio.get_running_loop().time() + retry_budget_seconds if max_retries else None async with asyncio.timeout_at(deadline): - client_kwargs: dict[str, object] = {"trust_env": trust_env} + client_kwargs: dict[str, object] = {"trust_env": trust_env, "follow_redirects": True} if proxy: client_kwargs["proxy"] = proxy async with httpx.AsyncClient(**client_kwargs) as client: diff --git a/backend/tests/test_jina_client.py b/backend/tests/test_jina_client.py index 8c4c30a715..e386ad9e22 100644 --- a/backend/tests/test_jina_client.py +++ b/backend/tests/test_jina_client.py @@ -1,5 +1,6 @@ """Tests for JinaClient async crawl method.""" +import json import logging from unittest.mock import MagicMock @@ -33,6 +34,57 @@ async def test_crawl_success(jina_client, monkeypatch): assert result == "Hello" +@pytest.mark.anyio +@pytest.mark.parametrize("max_retries", [0, 1]) +async def test_crawl_follows_jina_api_redirect(jina_client, monkeypatch, max_retries): + """A redirected Jina API endpoint should still return fetched content.""" + requests = [] + + def handle(request: httpx.Request) -> httpx.Response: + assert json.loads(request.read()) == {"url": "https://example.com"} + requests.append(request) + if request.url.path == "/": + return httpx.Response(307, headers={"Location": "https://r.jina.ai/reader"}) + return httpx.Response(200, text="Fetched page") + + original_client = httpx.AsyncClient + transport = httpx.MockTransport(handle) + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: original_client(transport=transport, **kwargs)) + + result = await jina_client.crawl("https://example.com", max_retries=max_retries) + + assert result == "Fetched page" + assert [request.url.path for request in requests] == ["/", "/reader"] + assert all(request.method == "POST" for request in requests) + + +@pytest.mark.anyio +@pytest.mark.parametrize("max_retries", [0, 1]) +async def test_crawl_strips_api_key_on_cross_host_redirect(jina_client, monkeypatch, max_retries): + """A redirected host must not receive the Jina bearer credential.""" + requests: list[httpx.Request] = [] + + def handle(request: httpx.Request) -> httpx.Response: + assert json.loads(request.read()) == {"url": "https://example.com"} + requests.append(request) + if request.url.host == "r.jina.ai": + return httpx.Response(307, headers={"Location": "https://redirect.example/final"}) + return httpx.Response(200, text="Fetched page") + + original_client = httpx.AsyncClient + transport = httpx.MockTransport(handle) + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: original_client(transport=transport, **kwargs)) + monkeypatch.setenv("JINA_API_KEY", "test-key") + + result = await jina_client.crawl("https://example.com", max_retries=max_retries) + + assert result == "Fetched page" + assert [request.url.host for request in requests] == ["r.jina.ai", "redirect.example"] + assert [request.method for request in requests] == ["POST", "POST"] + assert requests[0].headers["Authorization"] == "Bearer test-key" + assert "Authorization" not in requests[1].headers + + @pytest.mark.anyio async def test_crawl_non_200_status(jina_client, monkeypatch): """Test that non-200 status returns error message.""" @@ -150,6 +202,7 @@ async def test_crawl_passes_proxy_to_httpx_client(jina_client, monkeypatch): assert result == "ok" assert captured_client_kwargs["proxy"] == "http://127.0.0.1:7890" assert captured_client_kwargs["trust_env"] is True + assert captured_client_kwargs["follow_redirects"] is True @pytest.mark.anyio @@ -175,7 +228,7 @@ async def test_crawl_can_disable_trust_env(jina_client, monkeypatch): result = await jina_client.crawl("https://example.com", trust_env=False) assert result == "ok" - assert captured_client_kwargs == {"trust_env": False} + assert captured_client_kwargs == {"trust_env": False, "follow_redirects": True} @pytest.mark.anyio