ChatDev/execorchestra/tests/test_proxy_retry.py
2026-11-05 16:22:12 +08:00

100 lines
5.4 KiB
Python

"""Exercise the cluster cooldown with the normal SDK and Profiler retry owner."""
import json
import os
import threading
import time
import unittest
from http.server import BaseHTTPRequestHandler, HTTPServer
from pathlib import Path
from unittest.mock import patch
import httpx
from openai import APIConnectionError, OpenAI
from sgar_mvp.src import proxy_retry
from sgar_mvp.src.direct_network import direct_sync_http_client
from sgar_mvp.src.model_request_budget import ModelRequestBudget, model_request_budget
from sgar_mvp.src.model_transport import ProviderEndpointIdentity, SyncModelTransportPort
class ProxyRetryTests(unittest.TestCase):
def test_profiler_recovers_after_proxy_starts_listening(self):
template = Path(__file__).resolve().parents[1] / "configs/retrieval_policy.example.json"
with patch.dict(os.environ, {"SGAR_RETRIEVAL_POLICY_PATH": str(template),
"SGAR_SEALED_LOCAL_VALIDATION": "1"}):
import retrieve
posts = []
profile = {"think": "The question needs published hours.",
"capability_text": "Look up library opening hours.",
"constraint_text": "Use the official source."}
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
posts.append(json.loads(self.rfile.read(int(self.headers["Content-Length"]))))
response = json.dumps({"choices": [{"finish_reason": "stop", "message": {
"role": "assistant", "content": json.dumps(profile)}}]}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def log_message(self, *args):
pass
with HTTPServer(("127.0.0.1", 0), Handler, bind_and_activate=False) as proxy:
proxy.server_bind()
proxy.timeout = 3
worker = threading.Thread(target=proxy.handle_request, daemon=True)
def recover(delay):
self.assertTrue(5 <= delay <= 6)
proxy.server_activate()
worker.start()
# No external name resolution: this HTTP proxy itself serves the SDK response.
base = "http://unreachable.invalid/v1"
endpoint = ProviderEndpointIdentity.create(provider="test", base_url=base,
credential_environment_variable="DUMMY_KEY", timeout_seconds=4)
with patch.dict(os.environ, {"SGAR_MODEL_PROXY": f"http://127.0.0.1:{proxy.server_port}"}):
with OpenAI(api_key="test", base_url=base, max_retries=0, timeout=4,
http_client=direct_sync_http_client()) as sdk:
port = SyncModelTransportPort.from_sdk_client(client=sdk, endpoint_identity=endpoint)
def request():
return retrieve._generate_hyde_result("Find the library's opening hours.",
transport=port, model_attempt_limit=1, allow_direct_fallback=False)
with patch.dict(os.environ, {"SGAR_PROXY_CONNECT_BACKOFF":"0"}):
with self.assertRaises(retrieve.HyDEGenerationError) as before:
request()
self.assertEqual(before.exception.transport_attempts,
retrieve.RETRIEVAL_POLICY.hyde.max_transport_attempts_per_model_attempt)
with patch.dict(os.environ, {"SGAR_PROXY_CONNECT_BACKOFF":"1"}), patch.object(proxy_retry, "time") as clock:
clock.sleep.side_effect = recover
result = request()
clock.sleep.assert_called_once()
worker.join(3)
self.assertEqual(len(posts), 1)
self.assertEqual(result["capability_text"], profile["capability_text"])
self.assertEqual(result["metadata"]["transport_attempt_count"], 2)
self.assertEqual(len(set(result["metadata"]["transport_request_hashes"])), 1)
def test_no_wait_on_response_loss_or_unrelated_connection_errors(self):
req = httpx.Request("POST", "https://invalid.test")
with patch.dict(os.environ, {"SGAR_PROXY_CONNECT_BACKOFF":"1", "SGAR_MODEL_PROXY":"http://127.0.0.1:9999"}), patch.object(proxy_retry, "time") as clock:
for cause in (httpx.RemoteProtocolError("reply lost"), httpx.ReadTimeout("read"),
httpx.ConnectError("TLS error"), httpx.ProxyError("tunnel rejected")):
error = APIConnectionError(request=req)
error.__cause__ = cause
self.assertTrue(proxy_retry.wait_for_proxy_retry(error, 1))
clock.sleep.assert_not_called()
def test_insufficient_time_preserves_reserve(self):
error = httpx.ConnectError("connection failed")
error.__cause__ = ConnectionRefusedError()
budget = ModelRequestBudget(deadline_monotonic=time.monotonic()+20,
minimum_attempt_seconds=15, downstream_reserve_seconds=4)
with patch.dict(os.environ, {"SGAR_PROXY_CONNECT_BACKOFF":"1", "SGAR_MODEL_PROXY":"http://127.0.0.1:9999"}), model_request_budget(budget), patch.object(proxy_retry, "time") as clock:
self.assertFalse(proxy_retry.wait_for_proxy_retry(error, 2))
clock.sleep.assert_not_called()