mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
5e5913f4cf
* fix(codex): honor Retry-After in the provider retry loop * style(codex): format retry-after regression helper * fix(codex): honor HTTP-date Retry-After hints --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
251 lines
8.8 KiB
Python
251 lines
8.8 KiB
Python
"""Retry pacing for deerflow.models.openai_codex_provider.CodexChatModel.
|
|
|
|
The provider's retry loop used to pace every 429/500/529 on a hard-coded exponential
|
|
schedule, ignoring the server's hint. These tests pin the helper that now decides the
|
|
wait, and the loop that consumes it, without any network access.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import UTC, datetime, timedelta
|
|
from email.utils import format_datetime
|
|
|
|
import httpx
|
|
import pytest
|
|
from langchain_core.messages import HumanMessage
|
|
|
|
from deerflow.models.openai_codex_provider import _calc_backoff_ms
|
|
from deerflow.utils.retry_after import MAX_RETRY_AFTER_MS
|
|
|
|
|
|
def _status_error(status: int, headers: dict[str, str] | None = None) -> httpx.HTTPStatusError:
|
|
request = httpx.Request("POST", "https://example.invalid/responses")
|
|
return httpx.HTTPStatusError(
|
|
"boom",
|
|
request=request,
|
|
response=httpx.Response(status, headers=headers or {}, request=request),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_without_a_response_it_keeps_the_local_schedule():
|
|
error = RuntimeError("no response here")
|
|
assert _calc_backoff_ms(1, error) == 2000
|
|
assert _calc_backoff_ms(3, error) == 8000
|
|
|
|
|
|
def test_without_an_error_it_keeps_the_local_schedule():
|
|
assert _calc_backoff_ms(1) == 2000
|
|
assert _calc_backoff_ms(2) == 4000
|
|
|
|
|
|
def test_retry_after_seconds_is_honoured():
|
|
error = _status_error(429, {"Retry-After": "45"})
|
|
assert _calc_backoff_ms(1, error) == 45_000
|
|
|
|
|
|
def test_retry_after_ms_is_preferred_over_seconds():
|
|
error = _status_error(429, {"retry-after-ms": "1500", "Retry-After": "45"})
|
|
assert _calc_backoff_ms(1, error) == 1500
|
|
|
|
|
|
def test_header_lookup_is_case_insensitive():
|
|
error = _status_error(429, {"RETRY-AFTER": "7"})
|
|
assert _calc_backoff_ms(1, error) == 7000
|
|
|
|
|
|
@pytest.mark.parametrize("value", ["not-a-number", "", "nan", "inf"])
|
|
def test_unusable_hints_fall_back_to_the_local_schedule(value: str):
|
|
error = _status_error(429, {"Retry-After": value})
|
|
assert _calc_backoff_ms(2, error) == 4000
|
|
|
|
|
|
def test_a_negative_hint_clamps_to_zero_like_the_claude_provider():
|
|
# utils.retry_after treats a finite negative value as 0 rather than dropping the hint;
|
|
# the Codex provider now shares that contract instead of inventing its own.
|
|
error = _status_error(429, {"Retry-After": "-3"})
|
|
assert _calc_backoff_ms(2, error) == 0
|
|
|
|
|
|
def test_a_hint_beyond_the_bound_falls_back_to_the_local_schedule():
|
|
# 25 hours, one past the 24h bound.
|
|
error = _status_error(429, {"Retry-After": str(25 * 60 * 60)})
|
|
assert _calc_backoff_ms(1, error) == 2000
|
|
|
|
|
|
def test_the_bound_itself_is_accepted():
|
|
seconds = MAX_RETRY_AFTER_MS // 1000
|
|
error = _status_error(429, {"Retry-After": str(seconds)})
|
|
assert _calc_backoff_ms(1, error) == MAX_RETRY_AFTER_MS
|
|
|
|
|
|
def test_a_zero_hint_means_retry_immediately():
|
|
error = _status_error(429, {"Retry-After": "0"})
|
|
assert _calc_backoff_ms(1, error) == 0
|
|
|
|
|
|
@pytest.fixture
|
|
def fixed_clock(monkeypatch):
|
|
now = datetime(2026, 1, 1, tzinfo=UTC)
|
|
monkeypatch.setattr("deerflow.models.openai_codex_provider.time.time", lambda: now.timestamp())
|
|
return now
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("delta_seconds", "expected_ms"),
|
|
[(-60, 0), (0, 0), (600, 600_000), (86400, MAX_RETRY_AFTER_MS), (86401, 2000)],
|
|
)
|
|
def test_http_date_retry_after_obeys_delay_boundaries(fixed_clock, delta_seconds, expected_ms):
|
|
raw = format_datetime(fixed_clock + timedelta(seconds=delta_seconds), usegmt=True)
|
|
error = _status_error(429, {"Retry-After": raw})
|
|
assert _calc_backoff_ms(1, error) == expected_ms
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw",
|
|
[
|
|
"Thu, 01 Jan 2026 00:10:00 GMT",
|
|
"Thursday, 01-Jan-26 00:10:00 GMT",
|
|
"Thu Jan 1 00:10:00 2026",
|
|
],
|
|
)
|
|
def test_http_date_retry_after_accepts_http_date_formats_in_utc(fixed_clock, raw):
|
|
error = _status_error(429, {"RETRY-AFTER": raw})
|
|
assert _calc_backoff_ms(1, error) == 600_000
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw",
|
|
["Thu, 99 Jan 2026 00:10:00 GMT", "Thu, 01 Jan 2026 99:10:00 GMT", "Fri, 31 Dec 9999 23:59:59 GMT"],
|
|
)
|
|
def test_invalid_or_unbounded_http_dates_keep_local_backoff(fixed_clock, raw):
|
|
error = _status_error(429, {"Retry-After": raw})
|
|
assert _calc_backoff_ms(2, error) == 4000
|
|
|
|
|
|
@pytest.mark.parametrize(("milliseconds", "expected_ms"), [("1500", 1500), ("invalid", 600_000)])
|
|
def test_http_date_retry_after_preserves_millisecond_hint_precedence(fixed_clock, milliseconds, expected_ms):
|
|
raw = format_datetime(fixed_clock + timedelta(minutes=10), usegmt=True)
|
|
error = _status_error(429, {"retry-after-ms": milliseconds, "Retry-After": raw})
|
|
assert _calc_backoff_ms(1, error) == expected_ms
|
|
|
|
|
|
def test_http_dates_are_not_accepted_as_millisecond_hints(fixed_clock):
|
|
raw = format_datetime(fixed_clock + timedelta(minutes=10), usegmt=True)
|
|
error = _status_error(429, {"retry-after-ms": raw})
|
|
assert _calc_backoff_ms(1, error) == 2000
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The loop, through the public model surface
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_model(**kwargs):
|
|
from unittest.mock import patch
|
|
|
|
from deerflow.models.credential_loader import CodexCliCredential
|
|
from deerflow.models.openai_codex_provider import CodexChatModel
|
|
|
|
cred = CodexCliCredential(access_token="tok-test", account_id="acc-test")
|
|
with patch(
|
|
"deerflow.models.openai_codex_provider.load_codex_cli_credential",
|
|
return_value=cred,
|
|
):
|
|
return CodexChatModel(model="gpt-5.4", reasoning_effort="medium", **kwargs)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("mode", ["sync", "async"])
|
|
@pytest.mark.parametrize("status", [429, 500, 529])
|
|
async def test_public_invocation_waits_for_http_date_retry_after(monkeypatch, fixed_clock, mode, status):
|
|
from deerflow.models import openai_codex_provider as provider
|
|
|
|
model = _make_model(retry_max_attempts=2)
|
|
calls = []
|
|
sleeps = []
|
|
raw = format_datetime(fixed_clock + timedelta(minutes=10), usegmt=True)
|
|
|
|
def respond(request):
|
|
calls.append(request)
|
|
if len(calls) == 1:
|
|
return httpx.Response(status, headers={"Retry-After": raw}, json={"error": "synthetic throttle"})
|
|
event = {
|
|
"type": "response.completed",
|
|
"response": {"output": [{"type": "message", "content": [{"type": "output_text", "text": "recovered"}]}]},
|
|
}
|
|
return httpx.Response(200, headers={"content-type": "text/event-stream"}, text=f"data: {json.dumps(event)}\n\n")
|
|
|
|
client_class = httpx.Client
|
|
monkeypatch.setattr(provider.httpx, "Client", lambda **kwargs: client_class(transport=httpx.MockTransport(respond), trust_env=False, **kwargs))
|
|
monkeypatch.setattr(provider.time, "sleep", sleeps.append)
|
|
|
|
result = model.invoke("hi") if mode == "sync" else await model.ainvoke("hi")
|
|
|
|
assert result.content == "recovered"
|
|
assert len(calls) == 2
|
|
assert sleeps == [600.0]
|
|
|
|
|
|
def test_loop_sleeps_for_the_server_hint(monkeypatch):
|
|
model = _make_model(retry_max_attempts=2)
|
|
sleeps: list[float] = []
|
|
errors = [_status_error(429, {"Retry-After": "45"})]
|
|
|
|
def fake_stream(headers, payload):
|
|
if errors:
|
|
raise errors.pop()
|
|
return {"ok": True}
|
|
|
|
monkeypatch.setattr(model, "_stream_response", fake_stream)
|
|
monkeypatch.setattr(
|
|
"deerflow.models.openai_codex_provider.time.sleep",
|
|
lambda seconds: sleeps.append(seconds),
|
|
)
|
|
|
|
assert model._call_codex_api([HumanMessage(content="hi")]) == {"ok": True}
|
|
assert sleeps == [45.0]
|
|
|
|
|
|
def test_loop_uses_local_backoff_without_a_hint(monkeypatch):
|
|
model = _make_model(retry_max_attempts=2)
|
|
sleeps: list[float] = []
|
|
errors = [_status_error(429)]
|
|
|
|
def fake_stream(headers, payload):
|
|
if errors:
|
|
raise errors.pop()
|
|
return {"ok": True}
|
|
|
|
monkeypatch.setattr(model, "_stream_response", fake_stream)
|
|
monkeypatch.setattr(
|
|
"deerflow.models.openai_codex_provider.time.sleep",
|
|
lambda seconds: sleeps.append(seconds),
|
|
)
|
|
|
|
assert model._call_codex_api([HumanMessage(content="hi")]) == {"ok": True}
|
|
assert sleeps == [2.0]
|
|
|
|
|
|
def test_loop_does_not_retry_a_400(monkeypatch):
|
|
model = _make_model(retry_max_attempts=3)
|
|
sleeps: list[float] = []
|
|
|
|
def fake_stream(headers, payload):
|
|
raise _status_error(400)
|
|
|
|
monkeypatch.setattr(model, "_stream_response", fake_stream)
|
|
monkeypatch.setattr(
|
|
"deerflow.models.openai_codex_provider.time.sleep",
|
|
lambda seconds: sleeps.append(seconds),
|
|
)
|
|
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
model._call_codex_api([HumanMessage(content="hi")])
|
|
assert sleeps == []
|