From 1278c5192b40406f570193b7c7b051d5d8ce2192 Mon Sep 17 00:00:00 2001 From: derrickLzw <78075015+DerrickJc@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:21:44 -0400 Subject: [PATCH] fix(models): honor HTTP-date Retry-After hints in Claude retries (#7295) * fix(models): honor HTTP-date Retry-After hints in Claude retries * refactor(models): share bounded HTTP-date retry parsing * test(agents): move retry date regression to middleware suite --- README.md | 2 + .../llm_error_handling_middleware.py | 10 +-- .../harness/deerflow/models/AGENTS.md | 8 ++- .../deerflow/models/claude_provider.py | 6 +- .../deerflow/models/openai_codex_provider.py | 13 +--- .../packages/harness/deerflow/utils/AGENTS.md | 4 ++ .../harness/deerflow/utils/retry_after.py | 13 ++++ .../tests/test_claude_retry_after_dates.py | 68 +++++++++++++++++++ .../test_llm_error_handling_middleware.py | 10 +++ 9 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 backend/tests/test_claude_retry_after_dates.py diff --git a/README.md b/README.md index da9ae0eece..cfe19aa2c4 100644 --- a/README.md +++ b/README.md @@ -409,6 +409,8 @@ streams simulate chunks from a non-streaming response, while no-tool streams sta - Codex honors `tool_choice` in `bind_tools` and direct calls: `auto`, `none`, `required`, or a function name/selection. `any` and `True` mean `required`; `None` and `False` keep the provider default. Per-call choices override the bound policy. Bound choices are validated when binding; direct calls and overrides are validated before HTTP. `required` needs at least one function tool, and named selections must match a supplied function. `auto` and `none` also work without tools. - Completed Codex responses still return their text and tool calls when token usage is null, omitted, or empty; usage metadata remains unavailable. - The Codex model provider returns completed responses without waiting for the SSE connection to close. Failed or incomplete responses report the provider's error or reason; partial output is not returned as a successful answer. Non-object error details are reported as text. + - `ClaudeChatModel` honors integer-second and HTTP-date `Retry-After` hints in its provider retry loop. Dates use UTC; past dates retry immediately, while invalid hints or delays beyond 24 hours retain local backoff. + - Claude Code accepts `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_AUTH_TOKEN`, `CLAUDE_CODE_CREDENTIALS_PATH`, or `~/.claude/.credentials.json` - `CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR` accepts a UTF-8 token handoff and reuses it for later model instances in the same process. Undecodable handoffs are skipped so Claude Code can still try its override or default credentials file. - CLI credential JSON files accept UTF-8 with or without a BOM, independently of the host locale. `make doctor` accepts the same files when checking CLI authentication. Invalid text encoding is treated as an unreadable source; Claude Code can still try its default file after an invalid override. diff --git a/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py index 8d451d5293..83a05223c3 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/llm_error_handling_middleware.py @@ -11,7 +11,6 @@ import time from collections import deque from collections.abc import Awaitable, Callable from dataclasses import dataclass -from email.utils import parsedate_to_datetime from typing import Any, override from langchain.agents import AgentState @@ -28,7 +27,7 @@ from deerflow.agents.middlewares.model_response import append_visible_text, fini from deerflow.config.app_config import AppConfig from deerflow.models.request_admission import AdmissionError from deerflow.utils.custom_events import aemit_custom_event, emit_custom_event -from deerflow.utils.retry_after import bounded_retry_after_ms +from deerflow.utils.retry_after import bounded_retry_after_ms, parse_retry_after_date_ms logger = logging.getLogger(__name__) @@ -1180,12 +1179,7 @@ def _extract_retry_after_ms(exc: BaseException) -> int | None: multiplier = 1 if "ms" in header_name.lower() else 1000 return bounded_retry_after_ms(float(raw) * multiplier) except (TypeError, ValueError, OverflowError): - try: - target = parsedate_to_datetime(str(raw)) - delta = target.timestamp() - time.time() - return bounded_retry_after_ms(delta * 1000) - except (TypeError, ValueError, OverflowError): - return None + return parse_retry_after_date_ms(str(raw), now=time.time()) def _extract_error_detail(exc: BaseException) -> str: diff --git a/backend/packages/harness/deerflow/models/AGENTS.md b/backend/packages/harness/deerflow/models/AGENTS.md index 37f2ec88ab..49bc0a28cd 100644 --- a/backend/packages/harness/deerflow/models/AGENTS.md +++ b/backend/packages/harness/deerflow/models/AGENTS.md @@ -155,9 +155,11 @@ in `tests/test_codex_retry_after.py` includes public sync/async HTTP retries. ### Claude Retry Delays (`claude_provider.py`) `_calc_backoff_ms` uses `utils.retry_after.bounded_retry_after_ms` for integer-second -provider hints. Invalid or over-24-hour values retain local exponential backoff; -finite negative values clamp to zero. Sync/async 429/500 regressions live in -`tests/test_claude_provider_retry_after.py` and use offline provider doubles. +provider hints and HTTP-date hints (including obsolete forms interpreted in UTC). +Invalid or over-24-hour values retain local exponential backoff; finite negative +values and past dates clamp to zero. Sync/async 429/500 regressions live in +`tests/test_claude_provider_retry_after.py`; `tests/test_claude_retry_after_dates.py` +exercises public invocation through real SDK clients with offline HTTP transports. ### Provider Retry Admission (`claude_provider.py`, `openai_codex_provider.py`) diff --git a/backend/packages/harness/deerflow/models/claude_provider.py b/backend/packages/harness/deerflow/models/claude_provider.py index 2ffbf52112..76cf63a83d 100644 --- a/backend/packages/harness/deerflow/models/claude_provider.py +++ b/backend/packages/harness/deerflow/models/claude_provider.py @@ -30,7 +30,7 @@ from langchain_anthropic import ChatAnthropic from langchain_core.messages import BaseMessage from pydantic import PrivateAttr -from deerflow.utils.retry_after import bounded_retry_after_ms +from deerflow.utils.retry_after import bounded_retry_after_ms, parse_retry_after_date_ms logger = logging.getLogger(__name__) @@ -440,6 +440,8 @@ class ClaudeChatModel(ChatAnthropic): if hint_ms is not None: total_ms = hint_ms except (ValueError, TypeError, OverflowError): - pass + hint_ms = parse_retry_after_date_ms(retry_after, now=time.time()) + if hint_ms is not None: + total_ms = hint_ms return total_ms diff --git a/backend/packages/harness/deerflow/models/openai_codex_provider.py b/backend/packages/harness/deerflow/models/openai_codex_provider.py index b8c072aa99..952bc6d553 100644 --- a/backend/packages/harness/deerflow/models/openai_codex_provider.py +++ b/backend/packages/harness/deerflow/models/openai_codex_provider.py @@ -14,8 +14,6 @@ Supports: import json import logging import time -from datetime import UTC -from email.utils import parsedate_to_datetime from typing import Any import httpx @@ -25,7 +23,7 @@ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, System from langchain_core.outputs import ChatGeneration, ChatResult from deerflow.models.credential_loader import CodexCliCredential, load_codex_cli_credential -from deerflow.utils.retry_after import bounded_retry_after_ms +from deerflow.utils.retry_after import bounded_retry_after_ms, parse_retry_after_date_ms logger = logging.getLogger(__name__) @@ -55,13 +53,8 @@ def _retry_after_ms(error: Exception) -> int | None: except (TypeError, ValueError, OverflowError): if "ms" in header.lower(): continue - try: - target = parsedate_to_datetime(str(raw)) - # The obsolete asctime HTTP-date form has no timezone; it is UTC. - if target.tzinfo is None: - target = target.replace(tzinfo=UTC) - milliseconds = (target.timestamp() - time.time()) * 1000 - except (TypeError, ValueError, OverflowError): + milliseconds = parse_retry_after_date_ms(str(raw), now=time.time()) + if milliseconds is None: continue bounded = bounded_retry_after_ms(milliseconds) if bounded is not None: diff --git a/backend/packages/harness/deerflow/utils/AGENTS.md b/backend/packages/harness/deerflow/utils/AGENTS.md index 6d86d476e4..6b0757132b 100644 --- a/backend/packages/harness/deerflow/utils/AGENTS.md +++ b/backend/packages/harness/deerflow/utils/AGENTS.md @@ -171,3 +171,7 @@ Keep per-caller policies: Lark has a 15-minute Windows file-lock deadline, extension management waits indefinitely, and skill export retains its existing deadline, 0.02-second cadence and cancellation check. Clocks and sleeps are injectable; tests must not patch the shared stdlib time module. + +`parse_retry_after_date_ms(raw, now=...)` owns HTTP-date parsing for Claude, Codex +and LLM error middleware. Interpret timezone-less asctime as UTC and apply the +shared delay bound. Callers retain numeric/header precedence and local backoff. diff --git a/backend/packages/harness/deerflow/utils/retry_after.py b/backend/packages/harness/deerflow/utils/retry_after.py index 7a56796f77..356115450c 100644 --- a/backend/packages/harness/deerflow/utils/retry_after.py +++ b/backend/packages/harness/deerflow/utils/retry_after.py @@ -1,6 +1,8 @@ """Bounds for provider-supplied model retry delays.""" import math +from datetime import UTC +from email.utils import parsedate_to_datetime MAX_RETRY_AFTER_MS = 24 * 60 * 60 * 1000 @@ -12,3 +14,14 @@ def bounded_retry_after_ms(delay_ms: int | float) -> int | None: if delay_ms > MAX_RETRY_AFTER_MS: return None return max(0, int(delay_ms)) + + +def parse_retry_after_date_ms(raw: str, *, now: float) -> int | None: + """Parse an HTTP-date in UTC, bounded by the common provider-hint policy.""" + try: + target = parsedate_to_datetime(raw) + if target.tzinfo is None: + target = target.replace(tzinfo=UTC) + return bounded_retry_after_ms((target.timestamp() - now) * 1000) + except (TypeError, ValueError, OverflowError, OSError): + return None diff --git a/backend/tests/test_claude_retry_after_dates.py b/backend/tests/test_claude_retry_after_dates.py new file mode 100644 index 0000000000..2ef796bbbb --- /dev/null +++ b/backend/tests/test_claude_retry_after_dates.py @@ -0,0 +1,68 @@ +"""HTTP-date retry hints through the public Claude model and real SDK.""" + +import asyncio +from datetime import UTC, datetime, timedelta +from email.utils import format_datetime +from types import SimpleNamespace + +import anthropic +import httpx +import pytest + +from deerflow.models import claude_provider as provider + + +@pytest.fixture +def fixed_clock(monkeypatch): + now = datetime(2026, 1, 1, tzinfo=UTC) + monkeypatch.setattr(provider, "time", SimpleNamespace(time=lambda: now.timestamp())) + return now + + +@pytest.mark.parametrize("seconds, expected", [(-60, 0), (0, 0), (60, 60000), (86400, 86400000), (86401, 2400)]) +def test_date_delay_bounds(fixed_clock, seconds, expected): + raw = format_datetime(fixed_clock + timedelta(seconds=seconds), usegmt=True) + error = SimpleNamespace(response=httpx.Response(429, headers={"Retry-After": raw})) + assert provider.ClaudeChatModel._calc_backoff_ms(1, error) == expected + + +@pytest.mark.parametrize("raw", ["Thu, 99 Jan 2026 00:10:00 GMT", "Fri, 31 Dec 9999 23:59:59 GMT", "not a date"]) +def test_unusable_dates_keep_local_backoff(fixed_clock, raw): + error = SimpleNamespace(response=httpx.Response(503, headers={"Retry-After": raw})) + assert provider.ClaudeChatModel._calc_backoff_ms(2, error) == 4800 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("status", [429, 503]) +@pytest.mark.parametrize("raw", ["Thu, 01 Jan 2026 00:01:00 GMT", "Thursday, 01-Jan-26 00:01:00 GMT", "Thu Jan 1 00:01:00 2026"]) +async def test_public_call_waits_for_http_date(monkeypatch, fixed_clock, mode, status, raw): + requests = [] + waits = [] + + def respond(request): + requests.append(request) + if len(requests) == 1: + return httpx.Response(status, headers={"Retry-After": raw}, json={"type": "error", "error": {"type": "rate_limit_error" if status == 429 else "api_error", "message": "Try later"}}) + return httpx.Response( + 200, json={"id": "msg-test", "type": "message", "role": "assistant", "model": "claude-sonnet-4-6", "content": [{"type": "text", "text": "recovered"}], "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1}} + ) + + async def sleep(delay): + waits.append(delay) + + provider.time.sleep = waits.append + monkeypatch.setattr(asyncio, "sleep", sleep) + model = provider.ClaudeChatModel(model="claude-sonnet-4-6", anthropic_api_key="sk-ant-fake", max_tokens=1024, max_retries=0, retry_max_attempts=2) + transport = httpx.MockTransport(respond) + if mode == "sync": + with anthropic.Anthropic(api_key="sk-ant-fake", max_retries=0, http_client=httpx.Client(transport=transport)) as client: + model._client = client + result = model.invoke("hi") + else: + async with anthropic.AsyncAnthropic(api_key="sk-ant-fake", max_retries=0, http_client=httpx.AsyncClient(transport=transport)) as client: + model._async_client = client + result = await model.ainvoke("hi") + assert result.content == "recovered" + assert len(requests) == 2 + assert waits == [60.0] diff --git a/backend/tests/test_llm_error_handling_middleware.py b/backend/tests/test_llm_error_handling_middleware.py index 03191ff742..19d793e75c 100644 --- a/backend/tests/test_llm_error_handling_middleware.py +++ b/backend/tests/test_llm_error_handling_middleware.py @@ -2028,6 +2028,16 @@ def test_retry_after_date_boundaries(monkeypatch: pytest.MonkeyPatch, delta_seco assert _extract_retry_after_ms(exc) == expected_ms +@pytest.mark.parametrize("raw", ["Thu, 01 Jan 2026 00:01:00 GMT", "Thursday, 01-Jan-26 00:01:00 GMT", "Thu Jan 1 00:01:00 2026"]) +def test_retry_after_date_formats_use_utc(monkeypatch: pytest.MonkeyPatch, raw: str) -> None: + from deerflow.agents.middlewares import llm_error_handling_middleware as middleware + + now = datetime(2026, 1, 1, tzinfo=UTC) + monkeypatch.setattr(middleware, "time", SimpleNamespace(time=lambda: now.timestamp())) + exc = FakeError("rate limited", status_code=429, headers={"Retry-After": raw}) + assert _extract_retry_after_ms(exc) == 60000 + + @pytest.mark.anyio @pytest.mark.parametrize("mode", ["sync", "async"]) @pytest.mark.parametrize("header_name", ["Retry-After", "retry-after-ms"])