fix(models): honor HTTP-date Retry-After hints in Claude retries (#7295)

* fix(models): honor HTTP-date Retry-After hints in Claude retries

* refactor(models): share bounded HTTP-date retry parsing

* test(agents): move retry date regression to middleware suite
This commit is contained in:
derrickLzw 2026-10-23 03:21:44 -04:00 committed by GitHub
parent 5839822aa1
commit 1278c5192b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 111 additions and 23 deletions

View File

@ -409,6 +409,8 @@ streams simulate chunks from a non-streaming response, while no-tool streams sta
- Codex honors `tool_choice` in `bind_tools` and direct calls: `auto`, `none`, `required`, or a function name/selection. `any` and `True` mean `required`; `None` and `False` keep the provider default. Per-call choices override the bound policy. Bound choices are validated when binding; direct calls and overrides are validated before HTTP. `required` needs at least one function tool, and named selections must match a supplied function. `auto` and `none` also work without tools.
- Completed Codex responses still return their text and tool calls when token usage is null, omitted, or empty; usage metadata remains unavailable.
- The Codex model provider returns completed responses without waiting for the SSE connection to close. Failed or incomplete responses report the provider's error or reason; partial output is not returned as a successful answer. Non-object error details are reported as text.
- `ClaudeChatModel` honors integer-second and HTTP-date `Retry-After` hints in its provider retry loop. Dates use UTC; past dates retry immediately, while invalid hints or delays beyond 24 hours retain local backoff.
- Claude Code accepts `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_AUTH_TOKEN`, `CLAUDE_CODE_CREDENTIALS_PATH`, or `~/.claude/.credentials.json`
- `CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR` accepts a UTF-8 token handoff and reuses it for later model instances in the same process. Undecodable handoffs are skipped so Claude Code can still try its override or default credentials file.
- CLI credential JSON files accept UTF-8 with or without a BOM, independently of the host locale. `make doctor` accepts the same files when checking CLI authentication. Invalid text encoding is treated as an unreadable source; Claude Code can still try its default file after an invalid override.

View File

@ -11,7 +11,6 @@ import time
from collections import deque
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from email.utils import parsedate_to_datetime
from typing import Any, override
from langchain.agents import AgentState
@ -28,7 +27,7 @@ from deerflow.agents.middlewares.model_response import append_visible_text, fini
from deerflow.config.app_config import AppConfig
from deerflow.models.request_admission import AdmissionError
from deerflow.utils.custom_events import aemit_custom_event, emit_custom_event
from deerflow.utils.retry_after import bounded_retry_after_ms
from deerflow.utils.retry_after import bounded_retry_after_ms, parse_retry_after_date_ms
logger = logging.getLogger(__name__)
@ -1180,12 +1179,7 @@ def _extract_retry_after_ms(exc: BaseException) -> int | None:
multiplier = 1 if "ms" in header_name.lower() else 1000
return bounded_retry_after_ms(float(raw) * multiplier)
except (TypeError, ValueError, OverflowError):
try:
target = parsedate_to_datetime(str(raw))
delta = target.timestamp() - time.time()
return bounded_retry_after_ms(delta * 1000)
except (TypeError, ValueError, OverflowError):
return None
return parse_retry_after_date_ms(str(raw), now=time.time())
def _extract_error_detail(exc: BaseException) -> str:

View File

@ -155,9 +155,11 @@ in `tests/test_codex_retry_after.py` includes public sync/async HTTP retries.
### Claude Retry Delays (`claude_provider.py`)
`_calc_backoff_ms` uses `utils.retry_after.bounded_retry_after_ms` for integer-second
provider hints. Invalid or over-24-hour values retain local exponential backoff;
finite negative values clamp to zero. Sync/async 429/500 regressions live in
`tests/test_claude_provider_retry_after.py` and use offline provider doubles.
provider hints and HTTP-date hints (including obsolete forms interpreted in UTC).
Invalid or over-24-hour values retain local exponential backoff; finite negative
values and past dates clamp to zero. Sync/async 429/500 regressions live in
`tests/test_claude_provider_retry_after.py`; `tests/test_claude_retry_after_dates.py`
exercises public invocation through real SDK clients with offline HTTP transports.
### Provider Retry Admission (`claude_provider.py`, `openai_codex_provider.py`)

View File

@ -30,7 +30,7 @@ from langchain_anthropic import ChatAnthropic
from langchain_core.messages import BaseMessage
from pydantic import PrivateAttr
from deerflow.utils.retry_after import bounded_retry_after_ms
from deerflow.utils.retry_after import bounded_retry_after_ms, parse_retry_after_date_ms
logger = logging.getLogger(__name__)
@ -440,6 +440,8 @@ class ClaudeChatModel(ChatAnthropic):
if hint_ms is not None:
total_ms = hint_ms
except (ValueError, TypeError, OverflowError):
pass
hint_ms = parse_retry_after_date_ms(retry_after, now=time.time())
if hint_ms is not None:
total_ms = hint_ms
return total_ms

View File

@ -14,8 +14,6 @@ Supports:
import json
import logging
import time
from datetime import UTC
from email.utils import parsedate_to_datetime
from typing import Any
import httpx
@ -25,7 +23,7 @@ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, System
from langchain_core.outputs import ChatGeneration, ChatResult
from deerflow.models.credential_loader import CodexCliCredential, load_codex_cli_credential
from deerflow.utils.retry_after import bounded_retry_after_ms
from deerflow.utils.retry_after import bounded_retry_after_ms, parse_retry_after_date_ms
logger = logging.getLogger(__name__)
@ -55,13 +53,8 @@ def _retry_after_ms(error: Exception) -> int | None:
except (TypeError, ValueError, OverflowError):
if "ms" in header.lower():
continue
try:
target = parsedate_to_datetime(str(raw))
# The obsolete asctime HTTP-date form has no timezone; it is UTC.
if target.tzinfo is None:
target = target.replace(tzinfo=UTC)
milliseconds = (target.timestamp() - time.time()) * 1000
except (TypeError, ValueError, OverflowError):
milliseconds = parse_retry_after_date_ms(str(raw), now=time.time())
if milliseconds is None:
continue
bounded = bounded_retry_after_ms(milliseconds)
if bounded is not None:

View File

@ -171,3 +171,7 @@ Keep per-caller policies: Lark has a 15-minute Windows file-lock deadline,
extension management waits indefinitely, and skill export retains its existing
deadline, 0.02-second cadence and cancellation check. Clocks and sleeps are
injectable; tests must not patch the shared stdlib time module.
`parse_retry_after_date_ms(raw, now=...)` owns HTTP-date parsing for Claude, Codex
and LLM error middleware. Interpret timezone-less asctime as UTC and apply the
shared delay bound. Callers retain numeric/header precedence and local backoff.

View File

@ -1,6 +1,8 @@
"""Bounds for provider-supplied model retry delays."""
import math
from datetime import UTC
from email.utils import parsedate_to_datetime
MAX_RETRY_AFTER_MS = 24 * 60 * 60 * 1000
@ -12,3 +14,14 @@ def bounded_retry_after_ms(delay_ms: int | float) -> int | None:
if delay_ms > MAX_RETRY_AFTER_MS:
return None
return max(0, int(delay_ms))
def parse_retry_after_date_ms(raw: str, *, now: float) -> int | None:
"""Parse an HTTP-date in UTC, bounded by the common provider-hint policy."""
try:
target = parsedate_to_datetime(raw)
if target.tzinfo is None:
target = target.replace(tzinfo=UTC)
return bounded_retry_after_ms((target.timestamp() - now) * 1000)
except (TypeError, ValueError, OverflowError, OSError):
return None

View File

@ -0,0 +1,68 @@
"""HTTP-date retry hints through the public Claude model and real SDK."""
import asyncio
from datetime import UTC, datetime, timedelta
from email.utils import format_datetime
from types import SimpleNamespace
import anthropic
import httpx
import pytest
from deerflow.models import claude_provider as provider
@pytest.fixture
def fixed_clock(monkeypatch):
now = datetime(2026, 1, 1, tzinfo=UTC)
monkeypatch.setattr(provider, "time", SimpleNamespace(time=lambda: now.timestamp()))
return now
@pytest.mark.parametrize("seconds, expected", [(-60, 0), (0, 0), (60, 60000), (86400, 86400000), (86401, 2400)])
def test_date_delay_bounds(fixed_clock, seconds, expected):
raw = format_datetime(fixed_clock + timedelta(seconds=seconds), usegmt=True)
error = SimpleNamespace(response=httpx.Response(429, headers={"Retry-After": raw}))
assert provider.ClaudeChatModel._calc_backoff_ms(1, error) == expected
@pytest.mark.parametrize("raw", ["Thu, 99 Jan 2026 00:10:00 GMT", "Fri, 31 Dec 9999 23:59:59 GMT", "not a date"])
def test_unusable_dates_keep_local_backoff(fixed_clock, raw):
error = SimpleNamespace(response=httpx.Response(503, headers={"Retry-After": raw}))
assert provider.ClaudeChatModel._calc_backoff_ms(2, error) == 4800
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["sync", "async"])
@pytest.mark.parametrize("status", [429, 503])
@pytest.mark.parametrize("raw", ["Thu, 01 Jan 2026 00:01:00 GMT", "Thursday, 01-Jan-26 00:01:00 GMT", "Thu Jan 1 00:01:00 2026"])
async def test_public_call_waits_for_http_date(monkeypatch, fixed_clock, mode, status, raw):
requests = []
waits = []
def respond(request):
requests.append(request)
if len(requests) == 1:
return httpx.Response(status, headers={"Retry-After": raw}, json={"type": "error", "error": {"type": "rate_limit_error" if status == 429 else "api_error", "message": "Try later"}})
return httpx.Response(
200, json={"id": "msg-test", "type": "message", "role": "assistant", "model": "claude-sonnet-4-6", "content": [{"type": "text", "text": "recovered"}], "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1}}
)
async def sleep(delay):
waits.append(delay)
provider.time.sleep = waits.append
monkeypatch.setattr(asyncio, "sleep", sleep)
model = provider.ClaudeChatModel(model="claude-sonnet-4-6", anthropic_api_key="sk-ant-fake", max_tokens=1024, max_retries=0, retry_max_attempts=2)
transport = httpx.MockTransport(respond)
if mode == "sync":
with anthropic.Anthropic(api_key="sk-ant-fake", max_retries=0, http_client=httpx.Client(transport=transport)) as client:
model._client = client
result = model.invoke("hi")
else:
async with anthropic.AsyncAnthropic(api_key="sk-ant-fake", max_retries=0, http_client=httpx.AsyncClient(transport=transport)) as client:
model._async_client = client
result = await model.ainvoke("hi")
assert result.content == "recovered"
assert len(requests) == 2
assert waits == [60.0]

View File

@ -2028,6 +2028,16 @@ def test_retry_after_date_boundaries(monkeypatch: pytest.MonkeyPatch, delta_seco
assert _extract_retry_after_ms(exc) == expected_ms
@pytest.mark.parametrize("raw", ["Thu, 01 Jan 2026 00:01:00 GMT", "Thursday, 01-Jan-26 00:01:00 GMT", "Thu Jan 1 00:01:00 2026"])
def test_retry_after_date_formats_use_utc(monkeypatch: pytest.MonkeyPatch, raw: str) -> None:
from deerflow.agents.middlewares import llm_error_handling_middleware as middleware
now = datetime(2026, 1, 1, tzinfo=UTC)
monkeypatch.setattr(middleware, "time", SimpleNamespace(time=lambda: now.timestamp()))
exc = FakeError("rate limited", status_code=429, headers={"Retry-After": raw})
assert _extract_retry_after_ms(exc) == 60000
@pytest.mark.anyio
@pytest.mark.parametrize("mode", ["sync", "async"])
@pytest.mark.parametrize("header_name", ["Retry-After", "retry-after-ms"])