From 127c2c220c30d875b2995e95608b98d86db6bc13 Mon Sep 17 00:00:00 2001 From: yc2bgr8 <119500159+yc2bgr8@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:23:51 +0800 Subject: [PATCH] fix(community): follow Jina API redirects during web fetch (#5808) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(community): 跟随 Jina API 重定向 * test(community): 覆盖 Jina 跨域重定向凭据边界 * test(community): assert Jina redirects retain request body --- backend/AGENTS.md | 1 + backend/README.md | 2 + .../deerflow/community/jina_ai/jina_client.py | 2 +- backend/tests/test_jina_client.py | 55 ++++++++++++++++++- 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/backend/AGENTS.md b/backend/AGENTS.md index e77e931722..391ea2d581 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -276,6 +276,7 @@ Keep live tests opt-in via `DEER_FLOW_RUN_LIVE_TESTS=1`; guard POSIX-only markers with `os.name` for Windows collection. Jina logging tests use dummy keys (`tests/test_jina_client.py`). +The Jina API redirect regression uses an offline `httpx.MockTransport` and checks that temporary redirects keep the POST method. Jina/Browserless/InfoQuest resolve URLs without rebuilding HTML. InfoQuest connect/read timeout is 30s, separate from crawl timeouts (`tests/test_infoquest_http_timeout.py`). diff --git a/backend/README.md b/backend/README.md index 800a227721..59b59cac83 100644 --- a/backend/README.md +++ b/backend/README.md @@ -118,6 +118,8 @@ LLM-powered persistent context retention across conversations: | **MCP** | Any Model Context Protocol server (stdio, SSE, HTTP transports) | | **Skills** | Domain-specific workflows injected via system prompt | +Jina AI web fetch follows HTTP redirects from its API endpoint, including 307/308 redirects, which retain the POST method and body. + ### Run Event Storage For direct `RunEventStore.list_messages` callers, `after_seq` and `before_seq` diff --git a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py index 856a96c33b..a9f74035d2 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py +++ b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py @@ -89,7 +89,7 @@ class JinaClient: # bound the complete request sequence (including waits and cleanup). deadline = asyncio.get_running_loop().time() + retry_budget_seconds if max_retries else None async with asyncio.timeout_at(deadline): - client_kwargs: dict[str, object] = {"trust_env": trust_env} + client_kwargs: dict[str, object] = {"trust_env": trust_env, "follow_redirects": True} if proxy: client_kwargs["proxy"] = proxy async with httpx.AsyncClient(**client_kwargs) as client: diff --git a/backend/tests/test_jina_client.py b/backend/tests/test_jina_client.py index 8c4c30a715..e386ad9e22 100644 --- a/backend/tests/test_jina_client.py +++ b/backend/tests/test_jina_client.py @@ -1,5 +1,6 @@ """Tests for JinaClient async crawl method.""" +import json import logging from unittest.mock import MagicMock @@ -33,6 +34,57 @@ async def test_crawl_success(jina_client, monkeypatch): assert result == "Hello" +@pytest.mark.anyio +@pytest.mark.parametrize("max_retries", [0, 1]) +async def test_crawl_follows_jina_api_redirect(jina_client, monkeypatch, max_retries): + """A redirected Jina API endpoint should still return fetched content.""" + requests = [] + + def handle(request: httpx.Request) -> httpx.Response: + assert json.loads(request.read()) == {"url": "https://example.com"} + requests.append(request) + if request.url.path == "/": + return httpx.Response(307, headers={"Location": "https://r.jina.ai/reader"}) + return httpx.Response(200, text="Fetched page") + + original_client = httpx.AsyncClient + transport = httpx.MockTransport(handle) + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: original_client(transport=transport, **kwargs)) + + result = await jina_client.crawl("https://example.com", max_retries=max_retries) + + assert result == "Fetched page" + assert [request.url.path for request in requests] == ["/", "/reader"] + assert all(request.method == "POST" for request in requests) + + +@pytest.mark.anyio +@pytest.mark.parametrize("max_retries", [0, 1]) +async def test_crawl_strips_api_key_on_cross_host_redirect(jina_client, monkeypatch, max_retries): + """A redirected host must not receive the Jina bearer credential.""" + requests: list[httpx.Request] = [] + + def handle(request: httpx.Request) -> httpx.Response: + assert json.loads(request.read()) == {"url": "https://example.com"} + requests.append(request) + if request.url.host == "r.jina.ai": + return httpx.Response(307, headers={"Location": "https://redirect.example/final"}) + return httpx.Response(200, text="Fetched page") + + original_client = httpx.AsyncClient + transport = httpx.MockTransport(handle) + monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: original_client(transport=transport, **kwargs)) + monkeypatch.setenv("JINA_API_KEY", "test-key") + + result = await jina_client.crawl("https://example.com", max_retries=max_retries) + + assert result == "Fetched page" + assert [request.url.host for request in requests] == ["r.jina.ai", "redirect.example"] + assert [request.method for request in requests] == ["POST", "POST"] + assert requests[0].headers["Authorization"] == "Bearer test-key" + assert "Authorization" not in requests[1].headers + + @pytest.mark.anyio async def test_crawl_non_200_status(jina_client, monkeypatch): """Test that non-200 status returns error message.""" @@ -150,6 +202,7 @@ async def test_crawl_passes_proxy_to_httpx_client(jina_client, monkeypatch): assert result == "ok" assert captured_client_kwargs["proxy"] == "http://127.0.0.1:7890" assert captured_client_kwargs["trust_env"] is True + assert captured_client_kwargs["follow_redirects"] is True @pytest.mark.anyio @@ -175,7 +228,7 @@ async def test_crawl_can_disable_trust_env(jina_client, monkeypatch): result = await jina_client.crawl("https://example.com", trust_env=False) assert result == "ok" - assert captured_client_kwargs == {"trust_env": False} + assert captured_client_kwargs == {"trust_env": False, "follow_redirects": True} @pytest.mark.anyio