perf(tests): eliminate 140s of idle waits and SIGTERM grace timeouts across 7 slow suites (#8374)

* perf(tests): eliminate 140s of idle waits and SIGTERM grace timeouts across 7 slow suites

* test(backend): address review feedback on heartbeat sync, container cleanup, and deadline patch

* test(backend): observe artifact deadline extension at transfer

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
Xinmin Zeng 2026-11-19 04:16:16 -08:00 committed by GitHub
parent 3b1578fd91
commit dab27449b3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 120 additions and 72 deletions

View File

@ -253,6 +253,7 @@ async def test_initialization_keeps_renewing_for_other_workers(repository, monke
started, release = threading.Event(), threading.Event()
setup_executor(monkeypatch, started, release)
service = make_service(repository)
service._config = SubagentBatchesConfig.model_construct(lease_seconds=3)
renewals, twice = [], asyncio.Event()
original = repository.renew_item_lease
@ -274,8 +275,8 @@ async def test_initialization_keeps_renewing_for_other_workers(repository, monke
except TimeoutError:
pytest.fail("a blocked tool assembly received no periodic lease heartbeat")
# Just beyond the first renewal's expiry, but before the second one's.
peer_time = renewals[0] + timedelta(seconds=10, milliseconds=1)
assert await repository.claim_items(now=peer_time, lease_owner="peer", lease_seconds=10, limit=1) == []
peer_time = renewals[0] + timedelta(seconds=3, milliseconds=1)
assert await repository.claim_items(now=peer_time, lease_owner="peer", lease_seconds=3, limit=1) == []
finally:
release.set()
await asyncio.wait_for(asyncio.gather(*tasks), 3)
@ -358,7 +359,7 @@ async def test_lost_heartbeat_cancels_execution_before_a_long_poll_interval(repo
release.set()
setup_executor(monkeypatch, started, release)
service = make_service(repository)
service._config = SubagentBatchesConfig(lease_seconds=10, poll_interval_seconds=60)
service._config = SubagentBatchesConfig.model_construct(lease_seconds=3, poll_interval_seconds=60)
running, cancelled = asyncio.Event(), asyncio.Event()
mark_running = repository.mark_item_running

View File

@ -2,6 +2,7 @@
from __future__ import annotations
import functools
import os
import subprocess
import sys
@ -110,6 +111,7 @@ def _assert_collection_skipped(result: subprocess.CompletedProcess[str], reason:
assert all(node_id not in output for node_id in REPRESENTATIVE_LIVE_NODE_IDS)
@functools.cache
def _dry_run_make_target(target: str, *extra: str) -> str:
result = subprocess.run(
["make", "-n", target, *extra],

View File

@ -86,10 +86,10 @@ class SlowDriver:
return TaskSnapshot(status=TaskStatus.CANCELLED)
def make_service(repo, driver):
def make_service(repo, driver, *, lease_seconds: int = 5):
drivers = McpTaskDriverRegistry()
drivers.register("fake", driver)
return McpTaskService(repository=repo, drivers=drivers, poll_interval_seconds=1, lease_seconds=5, max_concurrent_polls=2)
return McpTaskService(repository=repo, drivers=drivers, poll_interval_seconds=1, lease_seconds=lease_seconds, max_concurrent_polls=2)
async def read_task(repo):
@ -140,21 +140,30 @@ async def test_cancel_snapshot_discard_is_logged_without_changing_peer_or_retry_
@pytest.mark.asyncio
@pytest.mark.parametrize("phase,expected", [("poll", "completed"), ("cancel", "cancelled")])
async def test_slow_control_call_keeps_claim_and_persists_terminal_result(task_repo, phase, expected):
async def test_slow_control_call_keeps_claim_and_persists_terminal_result(task_repo, monkeypatch, phase, expected):
repo = task_repo
if phase == "cancel":
await repo.request_cancel("task-1", user_id="user-1", thread_id="thread-1", thread_incarnation=None, requested_at=Clock.current)
driver = SlowDriver()
service = make_service(repo, driver)
heartbeat_committed = asyncio.Event()
real_renew = repo.renew_control_claim
async def observe_heartbeat_renewal(*args, **kwargs):
renewed = await real_renew(*args, **kwargs)
if driver.started.is_set() and renewed:
heartbeat_committed.set()
return renewed
monkeypatch.setattr(repo, "renew_control_claim", observe_heartbeat_renewal)
service = make_service(repo, driver, lease_seconds=2)
cycle = asyncio.create_task(service.run_once(now=Clock.current))
try:
await asyncio.wait_for(driver.started.wait(), timeout=3)
Clock.current += timedelta(seconds=4)
# One real heartbeat interval (5/3s), with a controlled durable clock.
await asyncio.sleep(2)
Clock.current += timedelta(seconds=2)
Clock.current += timedelta(seconds=1.5)
await asyncio.wait_for(heartbeat_committed.wait(), timeout=3)
Clock.current += timedelta(seconds=1)
claim = repo.claim_due_tasks if phase == "poll" else repo.claim_cancel_requests
peer = await claim(now=Clock.current, lease_owner="peer", lease_seconds=5, limit=1)
peer = await claim(now=Clock.current, lease_owner="peer", lease_seconds=2, limit=1)
assert peer == [], "a live slow control call lost its lease and was reclaimed"
driver.finish.set()
await asyncio.wait_for(cycle, timeout=3)
@ -299,7 +308,7 @@ async def test_failed_renewal_stops_remote_call_without_recording_task_failure(t
monkeypatch.setattr(repo, "renew_control_claim", unavailable_after_admission)
driver = SlowDriver()
cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current))
cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=1).run_once(now=Clock.current))
try:
await asyncio.wait_for(driver.started.wait(), timeout=3)
await asyncio.wait_for(driver.stopped.wait(), timeout=3)
@ -319,23 +328,32 @@ async def test_heartbeat_covers_snapshot_persistence_after_remote_completion(tas
repo = task_repo
applying = asyncio.Event()
commit = asyncio.Event()
heartbeat_committed = asyncio.Event()
apply = repo.apply_snapshot
real_renew = repo.renew_control_claim
async def delayed_write(*args, **kwargs):
applying.set()
await commit.wait()
return await apply(*args, **kwargs)
async def observe_heartbeat_renewal(*args, **kwargs):
renewed = await real_renew(*args, **kwargs)
if applying.is_set() and renewed:
heartbeat_committed.set()
return renewed
monkeypatch.setattr(repo, "apply_snapshot", delayed_write)
monkeypatch.setattr(repo, "renew_control_claim", observe_heartbeat_renewal)
driver = SlowDriver()
driver.finish.set()
cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current))
cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=2).run_once(now=Clock.current))
try:
await asyncio.wait_for(applying.wait(), timeout=3)
Clock.current += timedelta(seconds=4)
await asyncio.sleep(2)
Clock.current += timedelta(seconds=2)
assert await repo.claim_due_tasks(now=Clock.current, lease_owner="peer", lease_seconds=5, limit=1) == []
Clock.current += timedelta(seconds=1.5)
await asyncio.wait_for(heartbeat_committed.wait(), timeout=3)
Clock.current += timedelta(seconds=1)
assert await repo.claim_due_tasks(now=Clock.current, lease_owner="peer", lease_seconds=2, limit=1) == []
commit.set()
await asyncio.wait_for(cycle, timeout=3)
assert (await read_task(repo))["status"] == "completed"
@ -443,7 +461,7 @@ async def test_hung_renewal_stops_control_call_before_lease_expires(task_repo, m
monkeypatch.setattr(repo, "renew_control_claim", hang_after_admission)
driver = SlowDriver()
cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current))
cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=1).run_once(now=Clock.current))
try:
await asyncio.wait_for(driver.started.wait(), timeout=3)
await asyncio.wait_for(renew_started.wait(), timeout=3)
@ -501,7 +519,7 @@ async def test_uncertain_renewal_cleanup_hands_off_without_stalling_other_tasks(
monkeypatch.setattr(repo, "renew_control_claim", uncooperative_renewal)
monkeypatch.setattr(service_module, "_CANCELLATION_DRAIN_TIMEOUT_SECONDS", 0.05)
driver = DrainingDriver()
service = make_service(repo, driver)
service = make_service(repo, driver, lease_seconds=1)
cycle = asyncio.create_task(service.run_once(now=Clock.current))
try:
await asyncio.wait_for(renewal_started.wait(), timeout=3)

View File

@ -175,7 +175,14 @@ class TestFileMemoryStorage:
cached = storage.load()
assert cached["user"]["workContext"]["summary"] == "original"
with patch("builtins.open", side_effect=OSError("disk full")):
real_open = open
def fail_write_open(file, mode="r", *args, **kwargs):
if str(file).endswith(".lock") or not any(flag in mode for flag in ("w", "a", "x", "+")):
return real_open(file, mode, *args, **kwargs)
raise OSError("disk full")
with patch("builtins.open", side_effect=fail_write_open):
changed = create_empty_memory()
changed["user"]["workContext"]["summary"] = "mutated"
result = storage.save(changed)

View File

@ -16,6 +16,7 @@ Requires: Docker running locally
import subprocess
import time
import uuid
import pytest
@ -38,13 +39,37 @@ def _container_running(container_name: str) -> bool:
return result.returncode == 0 and result.stdout.strip().lower() == "true"
def _stop_container(container_name: str) -> None:
subprocess.run(["docker", "stop", container_name], capture_output=True, timeout=15)
def _stop_container(container_id: str) -> None:
subprocess.run(["docker", "rm", "-f", container_id], capture_output=True, timeout=15)
# Use a lightweight image for testing to avoid pulling the heavy sandbox image
def _start_container(container_name: str) -> str:
result = subprocess.run(
["docker", "run", "--rm", "-d", "--name", container_name, E2E_TEST_IMAGE, *_SLEEP_CMD],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, f"Failed to start {container_name}: {result.stderr}"
container_id = result.stdout.strip()
assert container_id, f"Docker returned empty container ID for {container_name}"
return container_id
def _wait_container_stopped(container_name: str, *, timeout: float = 5.0) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if not _container_running(container_name):
return
time.sleep(0.05)
# Use a lightweight image for testing to avoid pulling the heavy sandbox image.
# Trap SIGTERM in PID 1 so `docker stop` inside `backend.destroy()` exits cleanly
# instead of blocking for Docker's 10-second SIGTERM grace timeout per container.
E2E_TEST_IMAGE = "busybox:latest"
E2E_PREFIX = "deer-flow-sandbox-e2e-test"
_SLEEP_CMD = ["sh", "-c", "trap 'exit 0' TERM; sleep 3600 & wait"]
@pytest.fixture(autouse=True)
@ -78,17 +103,11 @@ class TestOrphanReconciliationE2E:
4. Simulate _reconcile_orphans() logic → container should be destroyed
"""
container_name = f"{E2E_PREFIX}-orphan01"
# Step 1: Start a container (simulating previous process lifecycle)
result = subprocess.run(
["docker", "run", "--rm", "-d", "--name", container_name, E2E_TEST_IMAGE, "sleep", "3600"],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, f"Failed to start test container: {result.stderr}"
created_ids: list[str] = []
try:
# Step 1: Start a container (simulating previous process lifecycle)
created_ids.append(_start_container(container_name))
assert _container_running(container_name), "Test container should be running"
# Step 2: Create backend and list running containers
@ -115,30 +134,25 @@ class TestOrphanReconciliationE2E:
# Destroy it (simulating what _reconcile_orphans does for old containers)
backend.destroy(orphan_info)
# Give Docker a moment to stop the container
time.sleep(1)
_wait_container_stopped(container_name)
# Verify container is gone
assert not _container_running(container_name), "Orphan container should be stopped after destroy"
finally:
# Safety cleanup
_stop_container(container_name)
# Safety cleanup only for containers started by this test
for cid in created_ids:
_stop_container(cid)
def test_multiple_orphans_all_cleaned(self):
"""Multiple orphaned containers are all found and can be cleaned up."""
containers = []
containers: list[str] = []
created_ids: list[str] = []
try:
# Start 3 containers
for i in range(3):
name = f"{E2E_PREFIX}-multi{i:02d}"
result = subprocess.run(
["docker", "run", "--rm", "-d", "--name", name, E2E_TEST_IMAGE, "sleep", "3600"],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, f"Failed to start {name}: {result.stderr}"
created_ids.append(_start_container(name))
containers.append(name)
from deerflow.community.aio_sandbox.local_backend import LocalContainerBackend
@ -162,34 +176,28 @@ class TestOrphanReconciliationE2E:
for info in running:
backend.destroy(info)
time.sleep(1)
for name in containers:
_wait_container_stopped(name)
# Verify all gone
for name in containers:
assert not _container_running(name), f"{name} should be stopped"
finally:
for name in containers:
_stop_container(name)
for cid in created_ids:
_stop_container(cid)
def test_list_running_ignores_unrelated_containers(self):
"""Containers with different prefixes should not be listed."""
unrelated_name = "unrelated-test-container"
unrelated_suffix = uuid.uuid4().hex[:8]
unrelated_name = f"unrelated-e2e-{unrelated_suffix}"
our_name = f"{E2E_PREFIX}-ours001"
created_ids: list[str] = []
try:
# Start an unrelated container
subprocess.run(
["docker", "run", "--rm", "-d", "--name", unrelated_name, E2E_TEST_IMAGE, "sleep", "3600"],
capture_output=True,
timeout=30,
)
# Start our container
subprocess.run(
["docker", "run", "--rm", "-d", "--name", our_name, E2E_TEST_IMAGE, "sleep", "3600"],
capture_output=True,
timeout=30,
)
# Start an unrelated container and our container; both starts must succeed.
created_ids.append(_start_container(unrelated_name))
created_ids.append(_start_container(our_name))
from deerflow.community.aio_sandbox.local_backend import LocalContainerBackend
@ -206,10 +214,9 @@ class TestOrphanReconciliationE2E:
# Should find ours but not unrelated
assert "ours001" in found_ids
# "unrelated-test-container" doesn't match "deer-flow-sandbox-e2e-test-" prefix
for info in running:
assert not info.sandbox_id.startswith("unrelated")
finally:
_stop_container(unrelated_name)
_stop_container(our_name)
for cid in created_ids:
_stop_container(cid)

View File

@ -98,7 +98,7 @@ def _graph(checkpointer, mode, answer):
@pytest.mark.parametrize("origin", ["remote", "local"])
async def test_slow_late_rollback_renews_slot_and_cannot_erase_peer_turn(database_runtime, monkeypatch, mode, origin):
store, checkpointer = database_runtime
ownership = RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=5, grace_seconds=0)
ownership = RunOwnershipConfig.model_construct(heartbeat_enabled=True, lease_seconds=2, grace_seconds=0)
owner = RunManager(store=store, worker_id="owner", run_ownership_config=ownership)
peer = RunManager(store=store, worker_id="peer", run_ownership_config=ownership)
metadata = {"scheduled_task_id": "task", "scheduled_task_run_id": "occurrence", "scheduled_goal_objective": "Report"}

View File

@ -247,6 +247,8 @@ async def test_resumed_task_registers_files_with_its_active_execution_key(plugin
@pytest.mark.asyncio
@pytest.mark.parametrize("operation", ["import", "capture"])
async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatch, operation):
from deerflow_extension_agent_teams import artifacts
if operation == "import":
_, _, service, runs, team, _, target = await ready_attachment(plugin)
method = "import_artifact"
@ -258,16 +260,27 @@ async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatc
runs.finish(target["id"])
method = "read_artifact"
transfer = getattr(runs, method)
calls = 0
advance = service.advance
poll_deadline = None
transfer_deadlines = []
async def slow_transfer(**kwargs):
nonlocal calls
calls += 1
await asyncio.sleep(10.1)
async def observe_poll_deadline(*args, **kwargs):
nonlocal poll_deadline
timeout = artifacts.worker_deadline.get()
poll_deadline = timeout.when() if timeout is not None else None
return await advance(*args, **kwargs)
async def observe_transfer_deadline(**kwargs):
timeout = artifacts.worker_deadline.get()
transfer_deadlines.append(timeout.when() if timeout is not None else None)
return await transfer(**kwargs)
monkeypatch.setattr(runs, method, slow_transfer)
monkeypatch.setattr(service, "advance", observe_poll_deadline)
monkeypatch.setattr(runs, method, observe_transfer_deadline)
await service.tick()
assert poll_deadline is not None
(transfer_deadline,) = transfer_deadlines
assert transfer_deadline is not None and transfer_deadline > poll_deadline
saved = await service.db("get", "alice", team["id"])
job = next(j for j in saved["jobs"] if j["id"] == target["id"])
if operation == "import":
@ -275,7 +288,7 @@ async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatc
else:
assert job["status"] == "completed" and saved["artifacts"][0]["status"] == "ready"
await service.tick()
assert calls == 1
assert len(transfer_deadlines) == 1
@pytest.mark.asyncio