mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
perf(tests): eliminate 140s of idle waits and SIGTERM grace timeouts across 7 slow suites (#8374)
* perf(tests): eliminate 140s of idle waits and SIGTERM grace timeouts across 7 slow suites * test(backend): address review feedback on heartbeat sync, container cleanup, and deadline patch * test(backend): observe artifact deadline extension at transfer --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
parent
3b1578fd91
commit
dab27449b3
@ -253,6 +253,7 @@ async def test_initialization_keeps_renewing_for_other_workers(repository, monke
|
||||
started, release = threading.Event(), threading.Event()
|
||||
setup_executor(monkeypatch, started, release)
|
||||
service = make_service(repository)
|
||||
service._config = SubagentBatchesConfig.model_construct(lease_seconds=3)
|
||||
renewals, twice = [], asyncio.Event()
|
||||
original = repository.renew_item_lease
|
||||
|
||||
@ -274,8 +275,8 @@ async def test_initialization_keeps_renewing_for_other_workers(repository, monke
|
||||
except TimeoutError:
|
||||
pytest.fail("a blocked tool assembly received no periodic lease heartbeat")
|
||||
# Just beyond the first renewal's expiry, but before the second one's.
|
||||
peer_time = renewals[0] + timedelta(seconds=10, milliseconds=1)
|
||||
assert await repository.claim_items(now=peer_time, lease_owner="peer", lease_seconds=10, limit=1) == []
|
||||
peer_time = renewals[0] + timedelta(seconds=3, milliseconds=1)
|
||||
assert await repository.claim_items(now=peer_time, lease_owner="peer", lease_seconds=3, limit=1) == []
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.wait_for(asyncio.gather(*tasks), 3)
|
||||
@ -358,7 +359,7 @@ async def test_lost_heartbeat_cancels_execution_before_a_long_poll_interval(repo
|
||||
release.set()
|
||||
setup_executor(monkeypatch, started, release)
|
||||
service = make_service(repository)
|
||||
service._config = SubagentBatchesConfig(lease_seconds=10, poll_interval_seconds=60)
|
||||
service._config = SubagentBatchesConfig.model_construct(lease_seconds=3, poll_interval_seconds=60)
|
||||
running, cancelled = asyncio.Event(), asyncio.Event()
|
||||
mark_running = repository.mark_item_running
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
@ -110,6 +111,7 @@ def _assert_collection_skipped(result: subprocess.CompletedProcess[str], reason:
|
||||
assert all(node_id not in output for node_id in REPRESENTATIVE_LIVE_NODE_IDS)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _dry_run_make_target(target: str, *extra: str) -> str:
|
||||
result = subprocess.run(
|
||||
["make", "-n", target, *extra],
|
||||
|
||||
@ -86,10 +86,10 @@ class SlowDriver:
|
||||
return TaskSnapshot(status=TaskStatus.CANCELLED)
|
||||
|
||||
|
||||
def make_service(repo, driver):
|
||||
def make_service(repo, driver, *, lease_seconds: int = 5):
|
||||
drivers = McpTaskDriverRegistry()
|
||||
drivers.register("fake", driver)
|
||||
return McpTaskService(repository=repo, drivers=drivers, poll_interval_seconds=1, lease_seconds=5, max_concurrent_polls=2)
|
||||
return McpTaskService(repository=repo, drivers=drivers, poll_interval_seconds=1, lease_seconds=lease_seconds, max_concurrent_polls=2)
|
||||
|
||||
|
||||
async def read_task(repo):
|
||||
@ -140,21 +140,30 @@ async def test_cancel_snapshot_discard_is_logged_without_changing_peer_or_retry_
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("phase,expected", [("poll", "completed"), ("cancel", "cancelled")])
|
||||
async def test_slow_control_call_keeps_claim_and_persists_terminal_result(task_repo, phase, expected):
|
||||
async def test_slow_control_call_keeps_claim_and_persists_terminal_result(task_repo, monkeypatch, phase, expected):
|
||||
repo = task_repo
|
||||
if phase == "cancel":
|
||||
await repo.request_cancel("task-1", user_id="user-1", thread_id="thread-1", thread_incarnation=None, requested_at=Clock.current)
|
||||
driver = SlowDriver()
|
||||
service = make_service(repo, driver)
|
||||
heartbeat_committed = asyncio.Event()
|
||||
real_renew = repo.renew_control_claim
|
||||
|
||||
async def observe_heartbeat_renewal(*args, **kwargs):
|
||||
renewed = await real_renew(*args, **kwargs)
|
||||
if driver.started.is_set() and renewed:
|
||||
heartbeat_committed.set()
|
||||
return renewed
|
||||
|
||||
monkeypatch.setattr(repo, "renew_control_claim", observe_heartbeat_renewal)
|
||||
service = make_service(repo, driver, lease_seconds=2)
|
||||
cycle = asyncio.create_task(service.run_once(now=Clock.current))
|
||||
try:
|
||||
await asyncio.wait_for(driver.started.wait(), timeout=3)
|
||||
Clock.current += timedelta(seconds=4)
|
||||
# One real heartbeat interval (5/3s), with a controlled durable clock.
|
||||
await asyncio.sleep(2)
|
||||
Clock.current += timedelta(seconds=2)
|
||||
Clock.current += timedelta(seconds=1.5)
|
||||
await asyncio.wait_for(heartbeat_committed.wait(), timeout=3)
|
||||
Clock.current += timedelta(seconds=1)
|
||||
claim = repo.claim_due_tasks if phase == "poll" else repo.claim_cancel_requests
|
||||
peer = await claim(now=Clock.current, lease_owner="peer", lease_seconds=5, limit=1)
|
||||
peer = await claim(now=Clock.current, lease_owner="peer", lease_seconds=2, limit=1)
|
||||
assert peer == [], "a live slow control call lost its lease and was reclaimed"
|
||||
driver.finish.set()
|
||||
await asyncio.wait_for(cycle, timeout=3)
|
||||
@ -299,7 +308,7 @@ async def test_failed_renewal_stops_remote_call_without_recording_task_failure(t
|
||||
|
||||
monkeypatch.setattr(repo, "renew_control_claim", unavailable_after_admission)
|
||||
driver = SlowDriver()
|
||||
cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current))
|
||||
cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=1).run_once(now=Clock.current))
|
||||
try:
|
||||
await asyncio.wait_for(driver.started.wait(), timeout=3)
|
||||
await asyncio.wait_for(driver.stopped.wait(), timeout=3)
|
||||
@ -319,23 +328,32 @@ async def test_heartbeat_covers_snapshot_persistence_after_remote_completion(tas
|
||||
repo = task_repo
|
||||
applying = asyncio.Event()
|
||||
commit = asyncio.Event()
|
||||
heartbeat_committed = asyncio.Event()
|
||||
apply = repo.apply_snapshot
|
||||
real_renew = repo.renew_control_claim
|
||||
|
||||
async def delayed_write(*args, **kwargs):
|
||||
applying.set()
|
||||
await commit.wait()
|
||||
return await apply(*args, **kwargs)
|
||||
|
||||
async def observe_heartbeat_renewal(*args, **kwargs):
|
||||
renewed = await real_renew(*args, **kwargs)
|
||||
if applying.is_set() and renewed:
|
||||
heartbeat_committed.set()
|
||||
return renewed
|
||||
|
||||
monkeypatch.setattr(repo, "apply_snapshot", delayed_write)
|
||||
monkeypatch.setattr(repo, "renew_control_claim", observe_heartbeat_renewal)
|
||||
driver = SlowDriver()
|
||||
driver.finish.set()
|
||||
cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current))
|
||||
cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=2).run_once(now=Clock.current))
|
||||
try:
|
||||
await asyncio.wait_for(applying.wait(), timeout=3)
|
||||
Clock.current += timedelta(seconds=4)
|
||||
await asyncio.sleep(2)
|
||||
Clock.current += timedelta(seconds=2)
|
||||
assert await repo.claim_due_tasks(now=Clock.current, lease_owner="peer", lease_seconds=5, limit=1) == []
|
||||
Clock.current += timedelta(seconds=1.5)
|
||||
await asyncio.wait_for(heartbeat_committed.wait(), timeout=3)
|
||||
Clock.current += timedelta(seconds=1)
|
||||
assert await repo.claim_due_tasks(now=Clock.current, lease_owner="peer", lease_seconds=2, limit=1) == []
|
||||
commit.set()
|
||||
await asyncio.wait_for(cycle, timeout=3)
|
||||
assert (await read_task(repo))["status"] == "completed"
|
||||
@ -443,7 +461,7 @@ async def test_hung_renewal_stops_control_call_before_lease_expires(task_repo, m
|
||||
|
||||
monkeypatch.setattr(repo, "renew_control_claim", hang_after_admission)
|
||||
driver = SlowDriver()
|
||||
cycle = asyncio.create_task(make_service(repo, driver).run_once(now=Clock.current))
|
||||
cycle = asyncio.create_task(make_service(repo, driver, lease_seconds=1).run_once(now=Clock.current))
|
||||
try:
|
||||
await asyncio.wait_for(driver.started.wait(), timeout=3)
|
||||
await asyncio.wait_for(renew_started.wait(), timeout=3)
|
||||
@ -501,7 +519,7 @@ async def test_uncertain_renewal_cleanup_hands_off_without_stalling_other_tasks(
|
||||
monkeypatch.setattr(repo, "renew_control_claim", uncooperative_renewal)
|
||||
monkeypatch.setattr(service_module, "_CANCELLATION_DRAIN_TIMEOUT_SECONDS", 0.05)
|
||||
driver = DrainingDriver()
|
||||
service = make_service(repo, driver)
|
||||
service = make_service(repo, driver, lease_seconds=1)
|
||||
cycle = asyncio.create_task(service.run_once(now=Clock.current))
|
||||
try:
|
||||
await asyncio.wait_for(renewal_started.wait(), timeout=3)
|
||||
|
||||
@ -175,7 +175,14 @@ class TestFileMemoryStorage:
|
||||
cached = storage.load()
|
||||
assert cached["user"]["workContext"]["summary"] == "original"
|
||||
|
||||
with patch("builtins.open", side_effect=OSError("disk full")):
|
||||
real_open = open
|
||||
|
||||
def fail_write_open(file, mode="r", *args, **kwargs):
|
||||
if str(file).endswith(".lock") or not any(flag in mode for flag in ("w", "a", "x", "+")):
|
||||
return real_open(file, mode, *args, **kwargs)
|
||||
raise OSError("disk full")
|
||||
|
||||
with patch("builtins.open", side_effect=fail_write_open):
|
||||
changed = create_empty_memory()
|
||||
changed["user"]["workContext"]["summary"] = "mutated"
|
||||
result = storage.save(changed)
|
||||
|
||||
@ -16,6 +16,7 @@ Requires: Docker running locally
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
@ -38,13 +39,37 @@ def _container_running(container_name: str) -> bool:
|
||||
return result.returncode == 0 and result.stdout.strip().lower() == "true"
|
||||
|
||||
|
||||
def _stop_container(container_name: str) -> None:
|
||||
subprocess.run(["docker", "stop", container_name], capture_output=True, timeout=15)
|
||||
def _stop_container(container_id: str) -> None:
|
||||
subprocess.run(["docker", "rm", "-f", container_id], capture_output=True, timeout=15)
|
||||
|
||||
|
||||
# Use a lightweight image for testing to avoid pulling the heavy sandbox image
|
||||
def _start_container(container_name: str) -> str:
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "-d", "--name", container_name, E2E_TEST_IMAGE, *_SLEEP_CMD],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == 0, f"Failed to start {container_name}: {result.stderr}"
|
||||
container_id = result.stdout.strip()
|
||||
assert container_id, f"Docker returned empty container ID for {container_name}"
|
||||
return container_id
|
||||
|
||||
|
||||
def _wait_container_stopped(container_name: str, *, timeout: float = 5.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if not _container_running(container_name):
|
||||
return
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
# Use a lightweight image for testing to avoid pulling the heavy sandbox image.
|
||||
# Trap SIGTERM in PID 1 so `docker stop` inside `backend.destroy()` exits cleanly
|
||||
# instead of blocking for Docker's 10-second SIGTERM grace timeout per container.
|
||||
E2E_TEST_IMAGE = "busybox:latest"
|
||||
E2E_PREFIX = "deer-flow-sandbox-e2e-test"
|
||||
_SLEEP_CMD = ["sh", "-c", "trap 'exit 0' TERM; sleep 3600 & wait"]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@ -78,17 +103,11 @@ class TestOrphanReconciliationE2E:
|
||||
4. Simulate _reconcile_orphans() logic → container should be destroyed
|
||||
"""
|
||||
container_name = f"{E2E_PREFIX}-orphan01"
|
||||
|
||||
# Step 1: Start a container (simulating previous process lifecycle)
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "-d", "--name", container_name, E2E_TEST_IMAGE, "sleep", "3600"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == 0, f"Failed to start test container: {result.stderr}"
|
||||
created_ids: list[str] = []
|
||||
|
||||
try:
|
||||
# Step 1: Start a container (simulating previous process lifecycle)
|
||||
created_ids.append(_start_container(container_name))
|
||||
assert _container_running(container_name), "Test container should be running"
|
||||
|
||||
# Step 2: Create backend and list running containers
|
||||
@ -115,30 +134,25 @@ class TestOrphanReconciliationE2E:
|
||||
# Destroy it (simulating what _reconcile_orphans does for old containers)
|
||||
backend.destroy(orphan_info)
|
||||
|
||||
# Give Docker a moment to stop the container
|
||||
time.sleep(1)
|
||||
_wait_container_stopped(container_name)
|
||||
|
||||
# Verify container is gone
|
||||
assert not _container_running(container_name), "Orphan container should be stopped after destroy"
|
||||
|
||||
finally:
|
||||
# Safety cleanup
|
||||
_stop_container(container_name)
|
||||
# Safety cleanup only for containers started by this test
|
||||
for cid in created_ids:
|
||||
_stop_container(cid)
|
||||
|
||||
def test_multiple_orphans_all_cleaned(self):
|
||||
"""Multiple orphaned containers are all found and can be cleaned up."""
|
||||
containers = []
|
||||
containers: list[str] = []
|
||||
created_ids: list[str] = []
|
||||
try:
|
||||
# Start 3 containers
|
||||
for i in range(3):
|
||||
name = f"{E2E_PREFIX}-multi{i:02d}"
|
||||
result = subprocess.run(
|
||||
["docker", "run", "--rm", "-d", "--name", name, E2E_TEST_IMAGE, "sleep", "3600"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == 0, f"Failed to start {name}: {result.stderr}"
|
||||
created_ids.append(_start_container(name))
|
||||
containers.append(name)
|
||||
|
||||
from deerflow.community.aio_sandbox.local_backend import LocalContainerBackend
|
||||
@ -162,34 +176,28 @@ class TestOrphanReconciliationE2E:
|
||||
for info in running:
|
||||
backend.destroy(info)
|
||||
|
||||
time.sleep(1)
|
||||
for name in containers:
|
||||
_wait_container_stopped(name)
|
||||
|
||||
# Verify all gone
|
||||
for name in containers:
|
||||
assert not _container_running(name), f"{name} should be stopped"
|
||||
|
||||
finally:
|
||||
for name in containers:
|
||||
_stop_container(name)
|
||||
for cid in created_ids:
|
||||
_stop_container(cid)
|
||||
|
||||
def test_list_running_ignores_unrelated_containers(self):
|
||||
"""Containers with different prefixes should not be listed."""
|
||||
unrelated_name = "unrelated-test-container"
|
||||
unrelated_suffix = uuid.uuid4().hex[:8]
|
||||
unrelated_name = f"unrelated-e2e-{unrelated_suffix}"
|
||||
our_name = f"{E2E_PREFIX}-ours001"
|
||||
created_ids: list[str] = []
|
||||
|
||||
try:
|
||||
# Start an unrelated container
|
||||
subprocess.run(
|
||||
["docker", "run", "--rm", "-d", "--name", unrelated_name, E2E_TEST_IMAGE, "sleep", "3600"],
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
)
|
||||
# Start our container
|
||||
subprocess.run(
|
||||
["docker", "run", "--rm", "-d", "--name", our_name, E2E_TEST_IMAGE, "sleep", "3600"],
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
)
|
||||
# Start an unrelated container and our container; both starts must succeed.
|
||||
created_ids.append(_start_container(unrelated_name))
|
||||
created_ids.append(_start_container(our_name))
|
||||
|
||||
from deerflow.community.aio_sandbox.local_backend import LocalContainerBackend
|
||||
|
||||
@ -206,10 +214,9 @@ class TestOrphanReconciliationE2E:
|
||||
|
||||
# Should find ours but not unrelated
|
||||
assert "ours001" in found_ids
|
||||
# "unrelated-test-container" doesn't match "deer-flow-sandbox-e2e-test-" prefix
|
||||
for info in running:
|
||||
assert not info.sandbox_id.startswith("unrelated")
|
||||
|
||||
finally:
|
||||
_stop_container(unrelated_name)
|
||||
_stop_container(our_name)
|
||||
for cid in created_ids:
|
||||
_stop_container(cid)
|
||||
|
||||
@ -98,7 +98,7 @@ def _graph(checkpointer, mode, answer):
|
||||
@pytest.mark.parametrize("origin", ["remote", "local"])
|
||||
async def test_slow_late_rollback_renews_slot_and_cannot_erase_peer_turn(database_runtime, monkeypatch, mode, origin):
|
||||
store, checkpointer = database_runtime
|
||||
ownership = RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=5, grace_seconds=0)
|
||||
ownership = RunOwnershipConfig.model_construct(heartbeat_enabled=True, lease_seconds=2, grace_seconds=0)
|
||||
owner = RunManager(store=store, worker_id="owner", run_ownership_config=ownership)
|
||||
peer = RunManager(store=store, worker_id="peer", run_ownership_config=ownership)
|
||||
metadata = {"scheduled_task_id": "task", "scheduled_task_run_id": "occurrence", "scheduled_goal_objective": "Report"}
|
||||
|
||||
@ -247,6 +247,8 @@ async def test_resumed_task_registers_files_with_its_active_execution_key(plugin
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("operation", ["import", "capture"])
|
||||
async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatch, operation):
|
||||
from deerflow_extension_agent_teams import artifacts
|
||||
|
||||
if operation == "import":
|
||||
_, _, service, runs, team, _, target = await ready_attachment(plugin)
|
||||
method = "import_artifact"
|
||||
@ -258,16 +260,27 @@ async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatc
|
||||
runs.finish(target["id"])
|
||||
method = "read_artifact"
|
||||
transfer = getattr(runs, method)
|
||||
calls = 0
|
||||
advance = service.advance
|
||||
poll_deadline = None
|
||||
transfer_deadlines = []
|
||||
|
||||
async def slow_transfer(**kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
await asyncio.sleep(10.1)
|
||||
async def observe_poll_deadline(*args, **kwargs):
|
||||
nonlocal poll_deadline
|
||||
timeout = artifacts.worker_deadline.get()
|
||||
poll_deadline = timeout.when() if timeout is not None else None
|
||||
return await advance(*args, **kwargs)
|
||||
|
||||
async def observe_transfer_deadline(**kwargs):
|
||||
timeout = artifacts.worker_deadline.get()
|
||||
transfer_deadlines.append(timeout.when() if timeout is not None else None)
|
||||
return await transfer(**kwargs)
|
||||
|
||||
monkeypatch.setattr(runs, method, slow_transfer)
|
||||
monkeypatch.setattr(service, "advance", observe_poll_deadline)
|
||||
monkeypatch.setattr(runs, method, observe_transfer_deadline)
|
||||
await service.tick()
|
||||
assert poll_deadline is not None
|
||||
(transfer_deadline,) = transfer_deadlines
|
||||
assert transfer_deadline is not None and transfer_deadline > poll_deadline
|
||||
saved = await service.db("get", "alice", team["id"])
|
||||
job = next(j for j in saved["jobs"] if j["id"] == target["id"])
|
||||
if operation == "import":
|
||||
@ -275,7 +288,7 @@ async def test_artifact_transfer_can_exceed_the_poll_deadline(plugin, monkeypatc
|
||||
else:
|
||||
assert job["status"] == "completed" and saved["artifacts"][0]["status"] == "ready"
|
||||
await service.tick()
|
||||
assert calls == 1
|
||||
assert len(transfer_deadlines) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user