From dab27449b3d8e6de0eb3f6c12b6c013ddb4ea79c Mon Sep 17 00:00:00 2001 From: Xinmin Zeng <135568692+fancyboi999@users.noreply.github.com> Date: Thu, 19 Nov 2026 04:16:16 -0800 Subject: [PATCH] perf(tests): eliminate 140s of idle waits and SIGTERM grace timeouts across 7 slow suites (#8374) * perf(tests): eliminate 140s of idle waits and SIGTERM grace timeouts across 7 slow suites * test(backend): address review feedback on heartbeat sync, container cleanup, and deadline patch * test(backend): observe artifact deadline extension at transfer --------- Co-authored-by: Willem Jiang --- backend/tests/test_batch_lease_lifecycle.py | 7 +- backend/tests/test_client_live_policy.py | 2 + .../tests/test_mcp_task_lease_lifecycle.py | 52 +++++++---- backend/tests/test_memory_storage.py | 9 +- .../test_sandbox_orphan_reconciliation_e2e.py | 93 ++++++++++--------- .../test_scheduled_goal_slow_rollback.py | 2 +- backend/tests/test_team_artifact_execution.py | 27 ++++-- 7 files changed, 120 insertions(+), 72 deletions(-) diff --git a/backend/tests/test_batch_lease_lifecycle.py b/backend/tests/test_batch_lease_lifecycle.py index 0f67226d1a..bf435c75c9 100644 --- a/backend/tests/test_batch_lease_lifecycle.py +++ b/backend/tests/test_batch_lease_lifecycle.py @@ -253,6 +253,7 @@ async def test_initialization_keeps_renewing_for_other_workers(repository, monke started, release = threading.Event(), threading.Event() setup_executor(monkeypatch, started, release) service = make_service(repository) + service._config = SubagentBatchesConfig.model_construct(lease_seconds=3) renewals, twice = [], asyncio.Event() original = repository.renew_item_lease @@ -274,8 +275,8 @@ async def test_initialization_keeps_renewing_for_other_workers(repository, monke except TimeoutError: pytest.fail("a blocked tool assembly received no periodic lease heartbeat") # Just beyond the first renewal's expiry, but before the second one's. - peer_time = renewals[0] + timedelta(seconds=10, milliseconds=1) - assert await repository.claim_items(now=peer_time, lease_owner="peer", lease_seconds=10, limit=1) == [] + peer_time = renewals[0] + timedelta(seconds=3, milliseconds=1) + assert await repository.claim_items(now=peer_time, lease_owner="peer", lease_seconds=3, limit=1) == [] finally: release.set() await asyncio.wait_for(asyncio.gather(*tasks), 3) @@ -358,7 +359,7 @@ async def test_lost_heartbeat_cancels_execution_before_a_long_poll_interval(repo release.set() setup_executor(monkeypatch, started, release) service = make_service(repository) - service._config = SubagentBatchesConfig(lease_seconds=10, poll_interval_seconds=60) + service._config = SubagentBatchesConfig.model_construct(lease_seconds=3, poll_interval_seconds=60) running, cancelled = asyncio.Event(), asyncio.Event() mark_running = repository.mark_item_running diff --git a/backend/tests/test_client_live_policy.py b/backend/tests/test_client_live_policy.py index 96f20f2367..8176de38e0 100644 --- a/backend/tests/test_client_live_policy.py +++ b/backend/tests/test_client_live_policy.py @@ -2,6 +2,7 @@ from __future__ import annotations +import functools import os import subprocess import sys @@ -110,6 +111,7 @@ def _assert_collection_skipped(result: subprocess.CompletedProcess[str], reason: assert all(node_id not in output for node_id in REPRESENTATIVE_LIVE_NODE_IDS) +@functools.cache def _dry_run_make_target(target: str, *extra: str) -> str: result = subprocess.run( ["make", "-n", target, *extra], diff --git a/backend/tests/test_mcp_task_lease_lifecycle.py b/backend/tests/test_mcp_task_lease_lifecycle.py index 46d360aaa9..65608a671b 100644 --- a/backend/tests/test_mcp_task_lease_lifecycle.py +++ b/backend/tests/test_mcp_task_lease_lifecycle.py @@ -86,10 +86,10 @@ class SlowDriver: return TaskSnapshot(status=TaskStatus.CANCELLED) -def make_service(repo, driver): +def make_service(repo, driver, *, lease_seconds: int = 5): drivers = McpTaskDriverRegistry() drivers.register("fake", driver) - return McpTaskService(repository=repo, drivers=drivers, poll_interval_seconds=1, lease_seconds=5, max_concurrent_polls=2) + return McpTaskService(repository=repo, drivers=drivers, poll_interval_seconds=1, lease_seconds=lease_seconds, max_concurrent_polls=2) async def read_task(repo): @@ -140,21 +140,30 @@ async def test_cancel_snapshot_discard_is_logged_without_changing_peer_or_retry_ @pytest.mark.asyncio @pytest.mark.parametrize("phase,expected", [("poll", "completed"), ("cancel", "cancelled")]) -async def test_slow_control_call_keeps_claim_and_persists_terminal_result(task_repo, phase, expected): +async def test_slow_control_call_keeps_claim_and_persists_terminal_result(task_repo, monkeypatch, phase, expected): repo = task_repo if phase == "cancel": await repo.request_cancel("task-1", user_id="user-1", thread_id="thread-1", thread_incarnation=None, requested_at=Clock.current) driver = SlowDriver() - service = make_service(repo, driver) + heartbeat_committed = asyncio.Event() + real_renew = repo.renew_control_claim + + async def observe_heartbeat_renewal(*args, **kwargs): + renewed = await real_renew(*args, **kwargs) + if driver.started.is_set() and renewed: + heartbeat_committed.set() + return renewed + + monkeypatch.setattr(repo, "renew_control_claim", observe_heartbeat_renewal) + service = make_service(repo, driver, lease_seconds=2) cycle = asyncio.create_task(service.run_once(now=Clock.current)) try: await asyncio.wait_for(driver.started.wait(), timeout=3) - Clock.current += timedelta(seconds=4) - # One real heartbeat interval (5/3s), with a controlled durable clock. - await asyncio.sleep(2) - Clock.current += timedelta(seconds=2) + Clock.current += timedelta(seconds=1.5) + await asyncio.wait_for(heartbeat_committed.wait(), timeout=3) + Clock.current += timedelta(seconds=1) claim = repo.claim_due_tasks if phase == "poll" else repo.claim_cancel_requests - peer = await claim(now=Clock.current, lease_owner="peer", lease_seconds=5, limit=1) + peer = await claim(now=Clock.current, lease_owner="peer", lease_seconds=2, limit=1) assert peer == [], "a live slow control call lost its lease and was reclaimed" driver.finish.set() await asyncio.wait_for(cycle, timeout=3) @@ -299,7 +308,7 @@ async def test_failed_renewal_stops_remote_call_without_recording_task_failure(t monkeypatch.setattr(repo, "renew_control_claim", unavailable_after_admission) driver = SlowDriver() - cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current)) + cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=1).run_once(now=Clock.current)) try: await asyncio.wait_for(driver.started.wait(), timeout=3) await asyncio.wait_for(driver.stopped.wait(), timeout=3) @@ -319,23 +328,32 @@ async def test_heartbeat_covers_snapshot_persistence_after_remote_completion(tas repo = task_repo applying = asyncio.Event() commit = asyncio.Event() + heartbeat_committed = asyncio.Event() apply = repo.apply_snapshot + real_renew = repo.renew_control_claim async def delayed_write(*args, **kwargs): applying.set() await commit.wait() return await apply(*args, **kwargs) + async def observe_heartbeat_renewal(*args, **kwargs): + renewed = await real_renew(*args, **kwargs) + if applying.is_set() and renewed: + heartbeat_committed.set() + return renewed + monkeypatch.setattr(repo, "apply_snapshot", delayed_write) + monkeypatch.setattr(repo, "renew_control_claim", observe_heartbeat_renewal) driver = SlowDriver() driver.finish.set() - cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current)) + cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=2).run_once(now=Clock.current)) try: await asyncio.wait_for(applying.wait(), timeout=3) - Clock.current += timedelta(seconds=4) - await asyncio.sleep(2) - Clock.current += timedelta(seconds=2) - assert await repo.claim_due_tasks(now=Clock.current, lease_owner="peer", lease_seconds=5, limit=1) == [] + Clock.current += timedelta(seconds=1.5) + await asyncio.wait_for(heartbeat_committed.wait(), timeout=3) + Clock.current += timedelta(seconds=1) + assert await repo.claim_due_tasks(now=Clock.current, lease_owner="peer", lease_seconds=2, limit=1) == [] commit.set() await asyncio.wait_for(cycle, timeout=3) assert (await read_task(repo))["status"] == "completed" @@ -443,7 +461,7 @@ async def test_hung_renewal_stops_control_call_before_lease_expires(task_repo, m monkeypatch.setattr(repo, "renew_control_claim", hang_after_admission) driver = SlowDriver() - cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current)) + cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=1).run_once(now=Clock.current)) try: await asyncio.wait_for(driver.started.wait(), timeout=3) await asyncio.wait_for(renew_started.wait(), timeout=3) @@ -501,7 +519,7 @@ async def test_uncertain_renewal_cleanup_hands_off_without_stalling_other_tasks( monkeypatch.setattr(repo, "renew_control_claim", uncooperative_renewal) monkeypatch.setattr(service_module, "_CANCELLATION_DRAIN_TIMEOUT_SECONDS", 0.05) driver = DrainingDriver() - service = make_service(repo, driver) + service = make_service(repo, driver, lease_seconds=1) cycle = asyncio.create_task(service.run_once(now=Clock.current)) try: await asyncio.wait_for(renewal_started.wait(), timeout=3) diff --git a/backend/tests/test_memory_storage.py b/backend/tests/test_memory_storage.py index 571aaee42e..06e6054352 100644 --- a/backend/tests/test_memory_storage.py +++ b/backend/tests/test_memory_storage.py @@ -175,7 +175,14 @@ class TestFileMemoryStorage: cached = storage.load() assert cached["user"]["workContext"]["summary"] == "original" - with patch("builtins.open", side_effect=OSError("disk full")): + real_open = open + + def fail_write_open(file, mode="r", *args, **kwargs): + if str(file).endswith(".lock") or not any(flag in mode for flag in ("w", "a", "x", "+")): + return real_open(file, mode, *args, **kwargs) + raise OSError("disk full") + + with patch("builtins.open", side_effect=fail_write_open): changed = create_empty_memory() changed["user"]["workContext"]["summary"] = "mutated" result = storage.save(changed) diff --git a/backend/tests/test_sandbox_orphan_reconciliation_e2e.py b/backend/tests/test_sandbox_orphan_reconciliation_e2e.py index 07f11eddd7..a84b9f94f2 100644 --- a/backend/tests/test_sandbox_orphan_reconciliation_e2e.py +++ b/backend/tests/test_sandbox_orphan_reconciliation_e2e.py @@ -16,6 +16,7 @@ Requires: Docker running locally import subprocess import time +import uuid import pytest @@ -38,13 +39,37 @@ def _container_running(container_name: str) -> bool: return result.returncode == 0 and result.stdout.strip().lower() == "true" -def _stop_container(container_name: str) -> None: - subprocess.run(["docker", "stop", container_name], capture_output=True, timeout=15) +def _stop_container(container_id: str) -> None: + subprocess.run(["docker", "rm", "-f", container_id], capture_output=True, timeout=15) -# Use a lightweight image for testing to avoid pulling the heavy sandbox image +def _start_container(container_name: str) -> str: + result = subprocess.run( + ["docker", "run", "--rm", "-d", "--name", container_name, E2E_TEST_IMAGE, *_SLEEP_CMD], + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, f"Failed to start {container_name}: {result.stderr}" + container_id = result.stdout.strip() + assert container_id, f"Docker returned empty container ID for {container_name}" + return container_id + + +def _wait_container_stopped(container_name: str, *, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not _container_running(container_name): + return + time.sleep(0.05) + + +# Use a lightweight image for testing to avoid pulling the heavy sandbox image. +# Trap SIGTERM in PID 1 so `docker stop` inside `backend.destroy()` exits cleanly +# instead of blocking for Docker's 10-second SIGTERM grace timeout per container. E2E_TEST_IMAGE = "busybox:latest" E2E_PREFIX = "deer-flow-sandbox-e2e-test" +_SLEEP_CMD = ["sh", "-c", "trap 'exit 0' TERM; sleep 3600 & wait"] @pytest.fixture(autouse=True) @@ -78,17 +103,11 @@ class TestOrphanReconciliationE2E: 4. Simulate _reconcile_orphans() logic → container should be destroyed """ container_name = f"{E2E_PREFIX}-orphan01" - - # Step 1: Start a container (simulating previous process lifecycle) - result = subprocess.run( - ["docker", "run", "--rm", "-d", "--name", container_name, E2E_TEST_IMAGE, "sleep", "3600"], - capture_output=True, - text=True, - timeout=30, - ) - assert result.returncode == 0, f"Failed to start test container: {result.stderr}" + created_ids: list[str] = [] try: + # Step 1: Start a container (simulating previous process lifecycle) + created_ids.append(_start_container(container_name)) assert _container_running(container_name), "Test container should be running" # Step 2: Create backend and list running containers @@ -115,30 +134,25 @@ class TestOrphanReconciliationE2E: # Destroy it (simulating what _reconcile_orphans does for old containers) backend.destroy(orphan_info) - # Give Docker a moment to stop the container - time.sleep(1) + _wait_container_stopped(container_name) # Verify container is gone assert not _container_running(container_name), "Orphan container should be stopped after destroy" finally: - # Safety cleanup - _stop_container(container_name) + # Safety cleanup only for containers started by this test + for cid in created_ids: + _stop_container(cid) def test_multiple_orphans_all_cleaned(self): """Multiple orphaned containers are all found and can be cleaned up.""" - containers = [] + containers: list[str] = [] + created_ids: list[str] = [] try: # Start 3 containers for i in range(3): name = f"{E2E_PREFIX}-multi{i:02d}" - result = subprocess.run( - ["docker", "run", "--rm", "-d", "--name", name, E2E_TEST_IMAGE, "sleep", "3600"], - capture_output=True, - text=True, - timeout=30, - ) - assert result.returncode == 0, f"Failed to start {name}: {result.stderr}" + created_ids.append(_start_container(name)) containers.append(name) from deerflow.community.aio_sandbox.local_backend import LocalContainerBackend @@ -162,34 +176,28 @@ class TestOrphanReconciliationE2E: for info in running: backend.destroy(info) - time.sleep(1) + for name in containers: + _wait_container_stopped(name) # Verify all gone for name in containers: assert not _container_running(name), f"{name} should be stopped" finally: - for name in containers: - _stop_container(name) + for cid in created_ids: + _stop_container(cid) def test_list_running_ignores_unrelated_containers(self): """Containers with different prefixes should not be listed.""" - unrelated_name = "unrelated-test-container" + unrelated_suffix = uuid.uuid4().hex[:8] + unrelated_name = f"unrelated-e2e-{unrelated_suffix}" our_name = f"{E2E_PREFIX}-ours001" + created_ids: list[str] = [] try: - # Start an unrelated container - subprocess.run( - ["docker", "run", "--rm", "-d", "--name", unrelated_name, E2E_TEST_IMAGE, "sleep", "3600"], - capture_output=True, - timeout=30, - ) - # Start our container - subprocess.run( - ["docker", "run", "--rm", "-d", "--name", our_name, E2E_TEST_IMAGE, "sleep", "3600"], - capture_output=True, - timeout=30, - ) + # Start an unrelated container and our container; both starts must succeed. + created_ids.append(_start_container(unrelated_name)) + created_ids.append(_start_container(our_name)) from deerflow.community.aio_sandbox.local_backend import LocalContainerBackend @@ -206,10 +214,9 @@ class TestOrphanReconciliationE2E: # Should find ours but not unrelated assert "ours001" in found_ids - # "unrelated-test-container" doesn't match "deer-flow-sandbox-e2e-test-" prefix for info in running: assert not info.sandbox_id.startswith("unrelated") finally: - _stop_container(unrelated_name) - _stop_container(our_name) + for cid in created_ids: + _stop_container(cid) diff --git a/backend/tests/test_scheduled_goal_slow_rollback.py b/backend/tests/test_scheduled_goal_slow_rollback.py index 7dd0c0e410..8af6627f24 100644 --- a/backend/tests/test_scheduled_goal_slow_rollback.py +++ b/backend/tests/test_scheduled_goal_slow_rollback.py @@ -98,7 +98,7 @@ def _graph(checkpointer, mode, answer): @pytest.mark.parametrize("origin", ["remote", "local"]) async def test_slow_late_rollback_renews_slot_and_cannot_erase_peer_turn(database_runtime, monkeypatch, mode, origin): store, checkpointer = database_runtime - ownership = RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=5, grace_seconds=0) + ownership = RunOwnershipConfig.model_construct(heartbeat_enabled=True, lease_seconds=2, grace_seconds=0) owner = RunManager(store=store, worker_id="owner", run_ownership_config=ownership) peer = RunManager(store=store, worker_id="peer", run_ownership_config=ownership) metadata = {"scheduled_task_id": "task", "scheduled_task_run_id": "occurrence", "scheduled_goal_objective": "Report"} diff --git a/backend/tests/test_team_artifact_execution.py b/backend/tests/test_team_artifact_execution.py index b5f2098b75..c68722a321 100644 --- a/backend/tests/test_team_artifact_execution.py +++ b/backend/tests/test_team_artifact_execution.py @@ -247,6 +247,8 @@ async def test_resumed_task_registers_files_with_its_active_execution_key(plugin @pytest.mark.asyncio @pytest.mark.parametrize("operation", ["import", "capture"]) async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatch, operation): + from deerflow_extension_agent_teams import artifacts + if operation == "import": _, _, service, runs, team, _, target = await ready_attachment(plugin) method = "import_artifact" @@ -258,16 +260,27 @@ async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatc runs.finish(target["id"]) method = "read_artifact" transfer = getattr(runs, method) - calls = 0 + advance = service.advance + poll_deadline = None + transfer_deadlines = [] - async def slow_transfer(**kwargs): - nonlocal calls - calls += 1 - await asyncio.sleep(10.1) + async def observe_poll_deadline(*args, **kwargs): + nonlocal poll_deadline + timeout = artifacts.worker_deadline.get() + poll_deadline = timeout.when() if timeout is not None else None + return await advance(*args, **kwargs) + + async def observe_transfer_deadline(**kwargs): + timeout = artifacts.worker_deadline.get() + transfer_deadlines.append(timeout.when() if timeout is not None else None) return await transfer(**kwargs) - monkeypatch.setattr(runs, method, slow_transfer) + monkeypatch.setattr(service, "advance", observe_poll_deadline) + monkeypatch.setattr(runs, method, observe_transfer_deadline) await service.tick() + assert poll_deadline is not None + (transfer_deadline,) = transfer_deadlines + assert transfer_deadline is not None and transfer_deadline > poll_deadline saved = await service.db("get", "alice", team["id"]) job = next(j for j in saved["jobs"] if j["id"] == target["id"]) if operation == "import": @@ -275,7 +288,7 @@ async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatc else: assert job["status"] == "completed" and saved["artifacts"][0]["status"] == "ready" await service.tick() - assert calls == 1 + assert len(transfer_deadlines) == 1 @pytest.mark.asyncio