diff --git a/README.md b/README.md index ce62a6084b..419487df59 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ For Google's official Gemini OpenAI-compatible endpoint, use the Jina, Browserless, and InfoQuest web fetches resolve relative links and image sources using the requested page URL (or a usable HTML base URL), so returned Markdown includes complete destinations. Link resolution preserves the surrounding HTML source, including malformed-page formatting. - Jina fetches support opt-in bounded retries via `max_retries` (default `0`) and `retry_budget_seconds` (default `30`) in the tool configuration. Retry waits are randomized within the time budget. Retries may increase upstream requests and cost; see [Jina fetch retries](backend/docs/CONFIGURATION.md#jina-fetch-retries). + Jina fetches support opt-in bounded retries via `max_retries` (default `0`) and `retry_budget_seconds` (default `30`) in the tool configuration. Valid `Retry-After` hints set a minimum wait for HTTP 429/503; 429 without a valid hint stays terminal. Hints that cannot fit the remaining budget stop retries. Local backoff remains randomized. Retries may increase upstream requests and cost; see [Jina fetch retries](backend/docs/CONFIGURATION.md#jina-fetch-retries). Run `make doctor` at any time to verify your setup and get actionable fix hints. If you are opening a GitHub issue about a local setup or runtime problem, run diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index a34dad587a..69718c7a31 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -617,13 +617,29 @@ request timeout is capped by the remaining budget; the outer deadline also bound responses that keep delivering data. The existing `timeout` remains Jina's `X-Timeout` header and the per-request HTTP timeout limit. -Only HTTP 502/503/504 and HTTPX connection-establishment errors (`ConnectError`, -`ConnectTimeout`) are retried. Authentication/client errors, 429, other statuses, -empty successful responses, read/write timeouts and arbitrary exceptions are not -retried. `Retry-After` is not interpreted. Backoff ceilings start at 0.5 seconds, -double to 1 and 2 seconds, then stay at 4 seconds. Each asynchronous wait caps its -ceiling by the remaining budget and independently samples a uniform factor from -0.5 to 1.0, reducing synchronized retries without increasing the wait cap. +HTTP 502/503/504 and HTTPX connection-establishment errors (`ConnectError`, +`ConnectTimeout`) are retryable. HTTP 429 is retried **only** with a valid +`Retry-After` header; missing or malformed hints leave it terminal. HTTP 503 +uses the same hints, falling back to local backoff when they are absent or invalid. +Authentication, payment/credit and other statuses remain terminal even with hints; +error-body prose never enables retries. Empty successful responses, read/write +timeouts and arbitrary exceptions are not retried. + +`Retry-After` accepts non-negative ASCII integer seconds or an HTTP-date +(including obsolete HTTP date forms); past dates mean a zero server floor. +Signed/fractional delays and non-HTTP dates are invalid. Local backoff ceilings +start at 0.5 seconds, double to 1 and 2, then stay at 4. Each wait caps the local +ceiling by the remaining budget and samples a uniform factor from 0.5 to 1.0. +The actual wait is the greater of that local pacing and the server floor. A +server floor is never reduced by jitter or the 4-second local ceiling. If the +hinted wait equals or exceeds the remaining budget (including enormous valid +integers), the last HTTP status error is returned without another request. +Dates use wall time to compute a delay; requests and waits share one monotonic +deadline. Each attempt uses only its own hint. + +Offline mocked regressions cover these policies; they do not establish that +Jina's hosted service always supplies recovery hints. No paid-provider testing +is required to enable this option. Cancellation propagates during requests and waits. This stops local work; it cannot cancel work already started by Jina. Enabling retries can send up to `1 + max_retries` upstream requests and incur additional cost. Successful content and final `Error:` results retain diff --git a/backend/packages/harness/deerflow/community/jina_ai/AGENTS.md b/backend/packages/harness/deerflow/community/jina_ai/AGENTS.md index a7987ab2a2..d3f2452083 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/AGENTS.md +++ b/backend/packages/harness/deerflow/community/jina_ai/AGENTS.md @@ -1,9 +1,10 @@ # Jina web fetch -Jina opt-in retries stay provider-local: `max_retries=0` preserves one attempt; -`retry_budget_seconds` bounds enabled request sequences and asynchronous backoff -with one asyncio deadline. Only 502/503/504 and connection-establishment failures -retry; cancellation propagates. Backoff ceilings double from 0.5 to 4 seconds. -Each wait caps the ceiling by the remaining budget, then multiplies it by a fresh -uniform random factor from 0.5 to 1.0 to avoid synchronized retries. -Offline coverage: `tests/test_jina_retries.py`. +Retries stay provider-local; default `max_retries=0` sends once. One monotonic +budget bounds requests/waits; cancellation propagates. Retry 502/503/504 and +connection-establishment failures; 429 requires valid Retry-After. Parse ASCII +integer seconds or HTTP dates (including obsolete forms); past dates floor at +zero. Wait max(server floor, existing budget-capped jittered 0.5–4s backoff). +Never reduce server floors; unfit waits return the HTTP error. Reset hints each +attempt. Auth/payment errors stay terminal. Tests: `test_jina_retries.py` and +`test_jina_retry_after.py` (offline, not hosted-provider validation). diff --git a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py index bcf0053f15..e2673b5823 100644 --- a/backend/packages/harness/deerflow/community/jina_ai/jina_client.py +++ b/backend/packages/harness/deerflow/community/jina_ai/jina_client.py @@ -3,6 +3,10 @@ import logging import math import os import random +import re +import time +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime import httpx @@ -11,6 +15,51 @@ logger = logging.getLogger(__name__) _api_key_warned = False +_DAY = r"(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)" +_MONTH = r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)" +_CLOCK = r"[0-9]{2}:[0-9]{2}:[0-9]{2}" +_HTTP_DATE = re.compile( + rf"(?:{_DAY}, [0-9]{{2}} {_MONTH} [0-9]{{4}} {_CLOCK} GMT" + rf"|(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), [0-9]{{2}}-{_MONTH}-[0-9]{{2}} {_CLOCK} GMT" + rf"|{_DAY} {_MONTH} (?:[0-9]{{2}}| [0-9]) {_CLOCK} [0-9]{{4}})" +) + + +def _retry_after(value: str | None) -> float | None: + """Return a server floor, or None for an invalid HTTP Retry-After value.""" + if value is None: + return None + value = value.strip(" \t") + if value and value.isascii() and value.isdecimal(): + digits = value.lstrip("0") or "0" + # Bound integer conversion work. Infinity is a valid, unfit floor, not + # a parsing failure that could cause an early fallback retry. + if len(digits) > 309: + return math.inf + seconds = int(digits) + try: + floor = float(seconds) + except OverflowError: + return math.inf + return math.nextafter(floor, math.inf) if floor < seconds else floor + if not _HTTP_DATE.fullmatch(value): + return None + try: + date = parsedate_to_datetime(value).replace(tzinfo=UTC) + now = time.time() + if "-" in value: + # RFC 850 two-digit years: choose the most recent matching year + # no more than 50 years in the future (RFC 9110 section 5.6.7). + current = datetime.fromtimestamp(now, UTC) + year = (current.year + 50) // 100 * 100 + date.year % 100 + if (year, date.month, date.day, date.hour, date.minute, date.second) > (current.year + 50, current.month, current.day, current.hour, current.minute, current.second): + year -= 100 + date = date.replace(year=year) + return max(0.0, date.timestamp() - now) + except (ValueError, OverflowError): + return None + + class JinaClient: async def crawl(self, url: str, return_format: str = "html", timeout: int = 10, proxy: str | None = None, trust_env: bool = True, *, max_retries: int = 0, retry_budget_seconds: float = 30.0) -> str: """Fetch with optional bounded retries; cancellation always propagates.""" @@ -46,6 +95,7 @@ class JinaClient: if remaining is not None and remaining <= 0: raise TimeoutError request_timeout = min(timeout, remaining) if remaining is not None else timeout + server_floor = None try: response = await client.post("https://r.jina.ai/", headers=headers, json=data, timeout=request_timeout) except (httpx.ConnectError, httpx.ConnectTimeout): @@ -58,16 +108,26 @@ class JinaClient: error_message = "Jina API returned empty response" logger.error(error_message) return f"Error: {error_message}" - if response.status_code not in {502, 503, 504} or attempt == max_retries: + if response.status_code in {429, 503} and attempt < max_retries: + server_floor = _retry_after(response.headers.get("Retry-After")) + retryable = response.status_code in {502, 503, 504} or (response.status_code == 429 and server_floor is not None) + if not retryable or attempt == max_retries: error_message = f"Jina API returned status {response.status_code}: {response.text}" logger.error(error_message) return f"Error: {error_message}" # Only allowlisted failures reach the non-blocking wait. remaining = deadline - asyncio.get_running_loop().time() + if server_floor is not None and server_floor >= remaining: + return f"Error: Jina API returned status {response.status_code}: {response.text}" if remaining <= 0: raise TimeoutError - await asyncio.sleep(min(delay, remaining) * random.uniform(0.5, 1.0)) + wait = min(delay, remaining) * random.uniform(0.5, 1.0) + if server_floor is not None: + wait = max(wait, server_floor) + if wait >= remaining: + return f"Error: Jina API returned status {response.status_code}: {response.text}" + await asyncio.sleep(wait) delay = min(delay * 2, 4.0) except Exception as e: if isinstance(e, TimeoutError) and max_retries and asyncio.get_running_loop().time() >= deadline: diff --git a/backend/tests/test_jina_retries.py b/backend/tests/test_jina_retries.py index 9a37d565c4..4a21dad431 100644 --- a/backend/tests/test_jina_retries.py +++ b/backend/tests/test_jina_retries.py @@ -100,8 +100,17 @@ async def test_backoff_jitter_is_capped_by_remaining_budget(requests, monkeypatc @pytest.mark.parametrize("during_backoff", [False, True]) async def test_shared_deadline(requests, monkeypatch, during_backoff): monkeypatch.setattr(random, "uniform", lambda low, high: 1.0) + loop = asyncio.get_running_loop() + real_time = loop.time + start = 0.0 + # Freeze only mock-client setup. Once the request starts, the actual asyncio + # deadline runs for 50ms; host scheduling cannot consume it before this test + # reaches the request/backoff operation it intends to exercise. + monkeypatch.setattr(loop, "time", lambda: start) async def post(*args, **kwargs): + offset = real_time() - start + monkeypatch.setattr(loop, "time", lambda: real_time() - offset) assert 0 < kwargs["timeout"] <= 0.05 if not during_backoff: await asyncio.Event().wait() diff --git a/backend/tests/test_jina_retry_after.py b/backend/tests/test_jina_retry_after.py new file mode 100644 index 0000000000..69edf2a0df --- /dev/null +++ b/backend/tests/test_jina_retry_after.py @@ -0,0 +1,217 @@ +"""Deterministic, offline provider-directed retry regressions.""" + +import asyncio +import time +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx +import pytest + +from deerflow.community.jina_ai import jina_client, tools + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def anyio_backend(): + return "asyncio" + + +@pytest.fixture +async def rig(monkeypatch): + loop = asyncio.get_running_loop() + now = [1000.0] # Exact origin keeps simulated budget arithmetic reproducible. + waits = [] + post = AsyncMock() + client = AsyncMock() + client.__aenter__.return_value = client + client.post = post + options = [] + + def factory(**kwargs): + options.append(kwargs) + return client + + async def sleep(delay): + waits.append(delay) + now[0] += delay + + monkeypatch.setattr(loop, "time", lambda: now[0]) + monkeypatch.setattr(time, "time", lambda: 784111777.0) # Sun, 06 Nov 1994 08:49:37 GMT + monkeypatch.setattr(asyncio, "sleep", sleep) + monkeypatch.setattr(jina_client.random, "uniform", lambda low, high: 1.0) + monkeypatch.setattr(httpx, "AsyncClient", factory) + monkeypatch.setenv("JINA_API_KEY", "offline-test") + return SimpleNamespace(post=post, client=client, waits=waits, now=now, options=options) + + +def response(status, hint=None): + return httpx.Response(status, text="provider result", headers={} if hint is None else {"Retry-After": hint}) + + +@pytest.mark.parametrize("status", [429, 503]) +@pytest.mark.parametrize("hint,wait", [("7", 7), ("0007", 7), (" 7\t", 7), ("0", 0.5), ("Sun, 06 Nov 1994 08:49:44 GMT", 7), ("Sunday, 06-Nov-94 08:49:44 GMT", 7), ("Sun Nov 6 08:49:44 1994", 7), ("Sun, 06 Nov 1994 08:49:30 GMT", 0.5)]) +async def test_recovery(rig, status, hint, wait): + rig.post.side_effect = [response(status, hint), response(200)] + assert await jina_client.JinaClient().crawl("https://example.com", max_retries=1) == "provider result" + assert rig.waits == [wait] + assert rig.post.await_count == 2 + + +@pytest.mark.parametrize("status", [429, 503]) +@pytest.mark.parametrize("hint", [None, "", "-1", "+1", "1.5", "NaN", "inf", "1e2", "tomorrow", "06 Nov 1994 08:49:44 GMT", "Sun, 06 Nov 1994 08:49:44 +0000", "Sun, 06 Nov 1994 08:49:44 GMT junk", "Sun, 31 Feb 1994 08:49:44 GMT"]) +async def test_missing_or_malformed(rig, status, hint): + rig.post.side_effect = [response(status, hint), response(200)] + result = await jina_client.JinaClient().crawl("https://example.com", max_retries=1) + assert rig.post.await_count == (1 if status == 429 else 2) + assert result == ("Error: Jina API returned status 429: provider result" if status == 429 else "provider result") + assert rig.waits == ([] if status == 429 else [0.5]) + + +@pytest.mark.parametrize("status", [429, 503]) +@pytest.mark.parametrize("hint", ["10", "11", "9" * 5000]) +async def test_cannot_fit_budget(rig, status, hint): + rig.post.return_value = response(status, hint) + result = await jina_client.JinaClient().crawl("https://example.com", max_retries=2, retry_budget_seconds=10) + assert result == f"Error: Jina API returned status {status}: provider result" + assert rig.post.await_count == 1 + assert rig.waits == [] + + +@pytest.mark.parametrize("elapsed", [3, 10]) +async def test_request_time_consumes_budget(rig, elapsed): + async def post(*args, **kwargs): + rig.now[0] += elapsed + return response(503, "7") + + rig.post.side_effect = post + assert "status 503" in await jina_client.JinaClient().crawl("https://example.com", max_retries=2, retry_budget_seconds=10) + assert rig.post.await_count == 1 + assert not rig.waits + + +@pytest.mark.parametrize("status", [429, 503]) +async def test_exhaustion_and_defaults(rig, status): + rig.post.return_value = response(status, "7") + assert "Error:" in await jina_client.JinaClient().crawl("https://example.com") + assert rig.post.await_count == 1 + assert not rig.waits + rig.post.reset_mock() + assert f"status {status}" in await jina_client.JinaClient().crawl("https://example.com", max_retries=2) + assert rig.post.await_count == 3 + assert rig.waits == [7, 7] + + +@pytest.mark.parametrize("status", [502, 504]) +async def test_other_retryable_statuses_ignore_retry_after(rig, status): + rig.post.side_effect = [response(status, "7"), response(200)] + assert await jina_client.JinaClient().crawl("https://example.com", max_retries=1) == "provider result" + assert rig.waits == [0.5] + assert rig.post.await_count == 2 + + +@pytest.mark.parametrize("status", [401, 402, 403, 409, 500, 418]) +async def test_other_statuses_terminal(rig, status): + rig.post.return_value = response(status, "1") + assert f"status {status}" in await jina_client.JinaClient().crawl("https://example.com", max_retries=2) + assert rig.post.await_count == 1 + assert not rig.waits + + +async def test_changing_hints_and_connection_failure(rig): + rig.post.side_effect = [response(503, "7"), httpx.ConnectError("offline"), response(429, "3"), response(503), response(200)] + assert await jina_client.JinaClient().crawl("https://example.com", max_retries=4, retry_budget_seconds=20) == "provider result" + assert rig.waits == [7, 1, 3, 4] + assert [call.kwargs["timeout"] for call in rig.post.await_args_list] == [10, 10, 10, 9, 5] + + +@pytest.mark.parametrize("during_sleep", [True, False]) +async def test_cancellation(rig, monkeypatch, during_sleep): + async def cancel(*args, **kwargs): + raise asyncio.CancelledError + + if during_sleep: + rig.post.return_value = response(429, "7") + monkeypatch.setattr(asyncio, "sleep", cancel) + else: + rig.post.side_effect = cancel + with pytest.raises(asyncio.CancelledError): + await jina_client.JinaClient().crawl("https://example.com", max_retries=2) + assert rig.post.await_count == 1 + + +async def test_web_fetch_integration(rig, monkeypatch): + config = SimpleNamespace(model_extra={"max_retries": 1, "retry_budget_seconds": 8, "proxy": "http://proxy:8080", "trust_env": False, "timeout": 6}) + monkeypatch.setattr(tools, "get_app_config", lambda: SimpleNamespace(get_tool_config=lambda name: config)) + rig.post.side_effect = [response(429, "7"), response(402, "1")] + assert await tools.web_fetch_tool.ainvoke({"url": "https://example.com"}) == "Error: Jina API returned status 402: provider result" + assert rig.waits == [7] + assert rig.options == [{"proxy": "http://proxy:8080", "trust_env": False}] + assert [call.kwargs["timeout"] for call in rig.post.await_args_list] == [6, 1] + assert set(tools.web_fetch_tool.args) == {"url"} + + +@pytest.mark.parametrize("value", ["١", "１", "²", "\n7", "7\n"]) +async def test_non_ascii_or_non_http_whitespace(value): + assert jina_client._retry_after(value) is None + + +async def test_obsolete_year_window(rig): + # At the fixture's 1994 wall time, 00 means 2000, not 1900. + assert jina_client._retry_after("Saturday, 01-Jan-00 00:00:00 GMT") > 0 + assert jina_client._retry_after("Sunday, 06-Nov-94 08:49:44 GMT") == 7 + assert jina_client._retry_after("Monday, 01-Jan-45 00:00:00 GMT") == 0 + + +async def test_local_pacing_cannot_fit_hinted_budget(rig): + rig.post.return_value = response(503, "0") + assert "status 503" in await jina_client.JinaClient().crawl("https://example.com", max_retries=1, retry_budget_seconds=0.25) + assert not rig.waits + assert rig.post.await_count == 1 + + +async def test_later_429_without_hint_is_terminal(rig): + rig.post.side_effect = [response(503, "7"), response(429)] + assert "status 429" in await jina_client.JinaClient().crawl("https://example.com", max_retries=3) + assert rig.waits == [7] + assert rig.post.await_count == 2 + + +async def test_future_date_cannot_fit_budget(rig): + rig.post.return_value = response(503, "Sun, 06 Nov 1994 08:49:47 GMT") + assert "status 503" in await jina_client.JinaClient().crawl("https://example.com", max_retries=1, retry_budget_seconds=10) + assert rig.post.await_count == 1 + assert not rig.waits + + +async def test_oversleep_does_not_send_after_deadline(rig, monkeypatch): + async def oversleep(delay): + rig.now[0] += 10 + + monkeypatch.setattr(asyncio, "sleep", oversleep) + rig.post.return_value = response(429, "7") + assert "budget exhausted" in await jina_client.JinaClient().crawl("https://example.com", max_retries=2, retry_budget_seconds=10) + assert rig.post.await_count == 1 + + +@pytest.mark.parametrize("status", [429, 503]) +@pytest.mark.parametrize("jitter", [0.5, 0.75]) +async def test_jitter_never_shortens_server_floor(rig, monkeypatch, status, jitter): + monkeypatch.setattr(jina_client.random, "uniform", lambda low, high: jitter) + rig.post.side_effect = [response(status, "7"), response(200)] + assert await jina_client.JinaClient().crawl("https://example.com", max_retries=1) == "provider result" + assert rig.waits == [7] + assert rig.post.await_count == 2 + + +async def test_client_setup_consumes_shared_budget(rig): + async def enter(): + rig.now[0] += 10 + return rig.client + + rig.client.__aenter__.side_effect = enter + result = await jina_client.JinaClient().crawl("https://example.com", max_retries=1, retry_budget_seconds=10) + assert "budget exhausted" in result + rig.post.assert_not_awaited() + assert not rig.waits