diff --git a/README.md b/README.md index 73913eaf1..32e5a140c 100644 --- a/README.md +++ b/README.md @@ -1123,6 +1123,26 @@ Sub-agents are an optimization, not the default response to a complex request. The lead agent can spawn sub-agents on the fly — each with its own scoped context, tools, and termination conditions — when delegation has clear net benefit from real parallel latency, specialist capability, or context isolation. It keeps interdependent scopes and overlapping side effects out of parallel dispatch; a bounded sequential chain can still run in one sub-agent when specialist or context-isolation benefit clearly wins. The lead uses the fewest useful sub-agents and re-evaluates later batches instead of fanning out solely because a task is large or multi-step. Sub-agents report back structured results, and the lead agent verifies and synthesizes them into a coherent output. Deterministic tool receipts cover both direct tool messages and state-updating `Command` results such as delegated `task` responses; when the receipt ledger reaches its context budget, it retains the newest actions and their original receipt IDs. Operators can disable this provenance layer with `verification.receipts_enabled: false`. Their configured skills are resolved from the same user-scoped catalog as the lead agent, so user-owned custom skills remain available without exposing another user's version. Their internal AI and tool messages stay scoped to the delegated graph instead of entering the parent chat stream. Reloaded thread history enforces the same boundary: callback-captured sub-agent AI responses remain available in run-event diagnostics but are excluded from the parent transcript, while the parent `task` result remains attached to its subtask card. Long-running sub-agents compact older history when summarization is enabled and re-inject the summary as guarded, hidden durable context before continuing, so recent assistant/tool activity remains grounded in the task. Provider/model request failures are reported as failed sub-agent tasks rather than successful results, so the lead agent and Web UI can react to them correctly. Concurrent parent runs also receive independent server-side sub-agent execution IDs, so a provider that reuses a tool-call ID cannot make one run poll, cancel, or clean up another run's background task. Collapsed sub-agent cards show the effective model and, when the provider returns usage metadata, a cumulative token total that updates after each completed sub-agent LLM call and persists after a reload. When token usage tracking is enabled, completed sub-agent usage is attributed back to the dispatching step from that run's terminal tool-message metadata rather than a process-global provider-ID cache. +Ordinary `task` delegation and explicit durable `batch_task` execution share the startup-scoped `subagent_runtime` process capacity. Batch mode keeps large independent item sets in SQL with separate total, live, and running limits, restart recovery, bounded results, and a thread-scoped Web UI panel. The panel pages through bounded previews on demand; full stored result text is available only through the owner-scoped JSONL export, while internal execution and authorization context never enters owner-facing responses. If the batch worker is later stopped or disabled, threads with persisted batches retain read-only item inspection and JSONL export; execution controls remain disabled until the worker is running again. See `config.example.yaml` and [the implementation contract](docs/plans/2026-08-24-subagent-batch-capacity-implementation.md) for limits and recovery semantics. + +Direct `create_deerflow_agent(...)` integrations can own the same boundary explicitly instead of relying on Gateway startup. Construct one `SubagentRuntime` and share it across every graph in that application; its `max_running`, ordinary per-run total, bound `task` tool, and optional durable-batch tools then use the same caller-owned snapshot and execution controller. A runtime with a batch repository owns a worker and must be started before graph construction and stopped during application shutdown: + +```python +from deerflow.agents import RuntimeFeatures, create_deerflow_agent +from deerflow.subagents import SubagentRuntime + +runtime = SubagentRuntime.from_app_config(app_config, batch_repository=batch_repository) +async with runtime: + graph = create_deerflow_agent( + model, + features=RuntimeFeatures(subagent=True), + subagent_runtime=runtime, + ) + # Serve or invoke graph while the durable worker is running. +``` + +The factory still does not load YAML or create SQL infrastructure: the caller supplies the config snapshot, repository, and lifecycle. Because it accepts a caller-owned `system_prompt`, direct integrations also own any model-visible wording about those limits; the default middleware enforces the runtime limits regardless. The factory does not mount the Gateway owner-scoped HTTP routes or Web UI, so direct applications must expose their own result API/UI if they need those surfaces. For ordinary delegation only, `SubagentRuntime(...)` needs no asynchronous startup. + Administrators can add, edit, disable, and delete reusable worker definitions from **Settings → Subagents**. Built-in and `config.yaml` definitions remain visible there as read-only entries. The default Lead Agent can use every enabled runtime sub-agent; each page-created Custom Agent can instead allow all, none, or a selected set. That selection is enforced both in the model-visible directory and by the server-side `task` tool. Managed definitions are deployment-wide in this version and follow `agent_storage.backend`: atomic files for a local deployment or the shared application database for multiple instances. For example, independent read-only research can run concurrently when the wall-clock savings outweigh duplicated discovery and synthesis cost, while a repository refactor with shared files and sequential test feedback remains with the lead agent. When `max_concurrent_subagents` is `1`, parallel and multi-batch routing guidance is disabled; delegation remains available only for material specialist or context-isolation benefit. diff --git a/backend/app/gateway/AGENTS.md b/backend/app/gateway/AGENTS.md index 1a4de6a38..bdb0f5428 100644 --- a/backend/app/gateway/AGENTS.md +++ b/backend/app/gateway/AGENTS.md @@ -44,7 +44,7 @@ reads/searches. | Router | Endpoints | |--------|-----------| | **Models** (`/api/models`) | `GET /` - list models; `GET /{name}` - model details | -| **Features** (`/api/features`) | `GET /` - report feature availability for frontend UI gating: hot-reloaded `agents_api`, guarded browser capability, and the startup-scoped durable MCP task capability (enabled config plus SQL repository) | +| **Features** (`/api/features`) | `GET /` - UI capabilities: hot-reloaded agents, guarded browser, startup MCP tasks, and separate batch repository/worker states so history stays readable without a worker | | **Console** (`/api/console`) | Read-only cross-thread observability for the current user (the data layer for an operations dashboard or external monitoring): `GET /stats` - headline counters (runs/threads/agents/tokens/cost); `GET /runs` - paginated run history joined with thread titles (per-run cost); `GET /usage` - zero-filled daily token series + per-model breakdown with spend. Queries `runs`/`threads_meta` directly as a reporting layer (no new `RunStore` methods); requires a SQL database backend — returns 503 on `database.backend: memory`. Real-cost estimation reads optional `models[*].pricing` (`currency`, `input_per_million`, `output_per_million`, `input_cache_hit_per_million`; `ModelConfig` is `extra="allow"`, so no schema change) and prices each run from its `token_usage_by_model` input/output split. Pricing is **cache-aware**: `RunJournal` accumulates prompt-cache hits from `usage_metadata.input_token_details.cache_read` into a sparse `cache_read_tokens` bucket key (also threaded through `SubagentTokenCollector` → `record_external_llm_usage_records`), and cache-hit input tokens are billed at `input_cache_hit_per_million` (omitted → billed at the miss price, a conservative upper bound). All priced models must use one currency; mixed currencies disable cost reporting and leave cost/currency fields null instead of producing invalid aggregates. Legacy rows fall back to run-level totals at `model_name`; unpriced models yield `cost: null` and cost fields are null when no pricing is configured | | **MCP** (`/api/mcp`) | `GET /config` - get config; `PUT /config` - replace the full config with whole-payload stdio validation; `PATCH /config` - toggle one server while preserving the raw extensions config and validating only an enabled target; both writes reload config and reset the process-local MCP cache | | **MCP Tasks** (`/api/threads/{id}/mcp-tasks`) | `GET /` - current user's durable tasks for one owned thread; `GET /{task_id}` - bounded result/input/status-error/cancellation-error detail, including cancellation attempt count, without remote task IDs or driver configuration | diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 97cc04e8f..1ce4559a4 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -34,6 +34,7 @@ from app.gateway.routers import ( runs, scheduled_tasks, skills, + subagent_batches, subagents, suggestions, thread_runs, @@ -202,6 +203,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: # snapshot on `app.state` to keep that contract enforceable. try: startup_config = get_app_config() + from deerflow.config.subagent_batches_config import SubagentBatchesConfig + from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig + from deerflow.subagents.capacity import configure_subagent_execution_capacity + + subagent_runtime_config = getattr(startup_config, "subagent_runtime", None) + if not isinstance(subagent_runtime_config, SubagentRuntimeConfig): + subagent_runtime_config = SubagentRuntimeConfig() + subagent_batches_config = getattr(startup_config, "subagent_batches", None) + if not isinstance(subagent_batches_config, SubagentBatchesConfig): + subagent_batches_config = SubagentBatchesConfig() + configure_subagent_execution_capacity(subagent_runtime_config) configure_logging(startup_config) ensure_browser_runtime_available(startup_config) logger.info("Configuration loaded successfully") @@ -395,6 +407,26 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: set_mcp_task_submitter(mcp_task_service) app.state.mcp_tasks_available = True + from app.subagent_batches import SubagentBatchService + from deerflow.subagents.batch_runtime import set_subagent_batch_submitter + + batch_repo = getattr(app.state, "subagent_batch_repo", None) + app.state.subagent_batches_available = False + set_subagent_batch_submitter(None) + if subagent_batches_config.enabled and batch_repo is None: + raise RuntimeError("subagent_batches.enabled requires database.backend sqlite or postgres") + if batch_repo is not None: + batch_service = SubagentBatchService( + repository=batch_repo, + config=subagent_batches_config, + runtime_config=subagent_runtime_config, + ) + app.state.subagent_batch_service = batch_service + if subagent_batches_config.enabled: + await batch_service.start() + set_subagent_batch_submitter(batch_service) + app.state.subagent_batches_available = True + yield try: @@ -438,6 +470,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: set_mcp_task_config_snapshot(None) + if getattr(app.state, "subagent_batch_service", None) is not None: + app.state.subagent_batches_available = False + try: + await app.state.subagent_batch_service.stop() + except Exception: + logger.exception("Failed to stop subagent batch service") + finally: + from deerflow.subagents.batch_runtime import set_subagent_batch_submitter + + set_subagent_batch_submitter(None) + try: from deerflow.community.browser_automation import get_browser_session_manager @@ -748,6 +791,7 @@ This gateway provides runtime endpoints for agent runs plus custom endpoints for # Durable MCP tasks are scoped to their owning thread. app.include_router(mcp_tasks.router) + app.include_router(subagent_batches.router) # Memory API is mounted at /api/memory app.include_router(memory.router) diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index d81a72eba..aba322aea 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -501,6 +501,7 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen ScheduledTaskRunRepository, ) from deerflow.persistence.scheduled_tasks import ScheduledTaskRepository + from deerflow.persistence.subagent_batches import SubagentBatchRepository app.state.scheduled_task_repo = ScheduledTaskRepository( sf, @@ -511,8 +512,10 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen run_repository=app.state.run_store, ) app.state.mcp_task_repo = McpTaskRepository(sf) + app.state.subagent_batch_repo = SubagentBatchRepository(sf) else: app.state.mcp_task_repo = None + app.state.subagent_batch_repo = None app.state.scheduled_task_repo = None app.state.scheduled_task_run_repo = None @@ -679,6 +682,20 @@ def get_mcp_task_service(request: Request): return val +def get_subagent_batch_repo(request: Request): + val = getattr(request.app.state, "subagent_batch_repo", None) + if val is None: + raise HTTPException(status_code=503, detail="Subagent batch repository not available") + return val + + +def get_subagent_batch_service(request: Request): + val = getattr(request.app.state, "subagent_batch_service", None) + if val is None: + raise HTTPException(status_code=503, detail="Subagent batch service not available") + return val + + def get_run_context(request: Request) -> RunContext: """Build a :class:`RunContext` from ``app.state`` singletons. diff --git a/backend/app/gateway/routers/__init__.py b/backend/app/gateway/routers/__init__.py index a948c522b..87db39b38 100644 --- a/backend/app/gateway/routers/__init__.py +++ b/backend/app/gateway/routers/__init__.py @@ -7,6 +7,7 @@ from . import ( models, scheduled_tasks, skills, + subagent_batches, suggestions, thread_runs, threads, @@ -22,6 +23,7 @@ __all__ = [ "models", "scheduled_tasks", "skills", + "subagent_batches", "suggestions", "threads", "thread_runs", diff --git a/backend/app/gateway/routers/features.py b/backend/app/gateway/routers/features.py index 886726df9..97b009bbd 100644 --- a/backend/app/gateway/routers/features.py +++ b/backend/app/gateway/routers/features.py @@ -13,6 +13,7 @@ from pydantic import BaseModel, Field from app.gateway.browser_capability import browser_capability from app.gateway.deps import get_config from deerflow.config.app_config import AppConfig +from deerflow.subagents.capacity import configured_subagent_max_running router = APIRouter(prefix="/api", tags=["features"]) @@ -35,12 +36,22 @@ class McpTasksFeature(BaseModel): enabled: bool = Field(..., description="Whether durable MCP task APIs and UI are available") +class SubagentBatchesFeature(BaseModel): + """Persistence, worker, and process capacity for native-subagent batches.""" + + enabled: bool = Field(..., description="Compatibility alias for worker_running") + repository_available: bool = Field(..., description="Whether durable batch history APIs are available") + worker_running: bool = Field(..., description="Whether this Gateway process is executing durable batch work") + max_running: int = Field(..., description="Native subagent execution slots in this Gateway process") + + class FeaturesResponse(BaseModel): """Frontend-facing feature availability flags.""" agents_api: AgentsApiFeature browser_control: BrowserControlFeature mcp_tasks: McpTasksFeature + subagent_batches: SubagentBatchesFeature @router.get( @@ -52,6 +63,7 @@ class FeaturesResponse(BaseModel): async def list_features(request: Request, config: AppConfig = Depends(get_config)) -> FeaturesResponse: """Return availability of optional frontend features.""" browser = browser_capability(config) + subagent_batch_worker_running = bool(getattr(request.app.state, "subagent_batches_available", False)) return FeaturesResponse( agents_api=AgentsApiFeature(enabled=config.agents_api.enabled), browser_control=BrowserControlFeature(enabled=browser.available), @@ -59,4 +71,13 @@ async def list_features(request: Request, config: AppConfig = Depends(get_config # capability that actually started rather than a hot-reloaded config # value that would require a Gateway restart to take effect. mcp_tasks=McpTasksFeature(enabled=bool(getattr(request.app.state, "mcp_tasks_available", False))), + subagent_batches=SubagentBatchesFeature( + # Keep the historical `enabled` field as a compatibility alias + # while exposing read persistence independently from execution. + # A stopped/disabled worker must not hide durable history/export. + enabled=subagent_batch_worker_running, + repository_available=getattr(request.app.state, "subagent_batch_repo", None) is not None, + worker_running=subagent_batch_worker_running, + max_running=configured_subagent_max_running(), + ), ) diff --git a/backend/app/gateway/routers/subagent_batches.py b/backend/app/gateway/routers/subagent_batches.py new file mode 100644 index 000000000..705aff3b2 --- /dev/null +++ b/backend/app/gateway/routers/subagent_batches.py @@ -0,0 +1,130 @@ +"""Owner-scoped progress and control API for durable subagent batches.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator + +from fastapi import APIRouter, HTTPException, Query, Request +from fastapi.responses import StreamingResponse + +from app.gateway.authz import require_permission +from app.gateway.deps import ( + get_current_user, + get_subagent_batch_repo, + get_subagent_batch_service, +) +from deerflow.utils.thread_id import ThreadId + +router = APIRouter(prefix="/api/threads/{thread_id}/subagent-batches", tags=["subagent-batches"]) +_ITEM_STATUSES = {"pending", "queued", "leased", "running", "succeeded", "failed", "cancelled"} + + +async def _user_id(request: Request) -> str: + user_id = await get_current_user(request) + if user_id is None: + raise HTTPException(status_code=401, detail="Authentication required") + return user_id + + +async def _owned_batch(request: Request, thread_id: str, batch_id: str) -> tuple[object, str, dict]: + repo = get_subagent_batch_repo(request) + user_id = await _user_id(request) + batch = await repo.get_batch(batch_id, user_id=user_id) + if batch is None or batch["thread_id"] != thread_id: + raise HTTPException(status_code=404, detail="Subagent batch not found") + return repo, user_id, batch + + +@router.get("") +@require_permission("threads", "read", owner_check=True) +async def list_batches(thread_id: ThreadId, request: Request, limit: int = Query(20, ge=1, le=100)) -> list[dict]: + repo = get_subagent_batch_repo(request) + return await repo.list_by_thread(thread_id, user_id=await _user_id(request), limit=limit) + + +@router.get("/{batch_id}") +@require_permission("threads", "read", owner_check=True) +async def get_batch(thread_id: ThreadId, batch_id: str, request: Request) -> dict: + _repo, _user_id_value, batch = await _owned_batch(request, thread_id, batch_id) + return batch + + +@router.get("/{batch_id}/items") +@require_permission("threads", "read", owner_check=True) +async def list_batch_items( + thread_id: ThreadId, + batch_id: str, + request: Request, + offset: int = Query(0, ge=0), + limit: int = Query(100, ge=1, le=500), + status: str | None = Query(None), +) -> list[dict]: + if status is not None and status not in _ITEM_STATUSES: + raise HTTPException(status_code=422, detail="Unknown batch item status") + repo, user_id, _batch = await _owned_batch(request, thread_id, batch_id) + return await repo.list_items(batch_id, user_id=user_id, offset=offset, limit=limit, status=status) or [] + + +@router.post("/{batch_id}/pause") +@require_permission("threads", "write", owner_check=True) +async def pause_batch(thread_id: ThreadId, batch_id: str, request: Request) -> dict: + repo, user_id, _batch = await _owned_batch(request, thread_id, batch_id) + return await repo.pause_batch(batch_id, user_id=user_id) + + +@router.post("/{batch_id}/resume") +@require_permission("threads", "write", owner_check=True) +async def resume_batch(thread_id: ThreadId, batch_id: str, request: Request) -> dict: + repo, user_id, _batch = await _owned_batch(request, thread_id, batch_id) + return await repo.resume_batch(batch_id, user_id=user_id) + + +@router.post("/{batch_id}/cancel") +@require_permission("threads", "write", owner_check=True) +async def cancel_batch(thread_id: ThreadId, batch_id: str, request: Request) -> dict: + if not getattr(request.app.state, "subagent_batches_available", False): + raise HTTPException(status_code=503, detail="Subagent batch worker is not running") + _repo, user_id, _batch = await _owned_batch(request, thread_id, batch_id) + result = await get_subagent_batch_service(request).cancel_batch(batch_id=batch_id, user_id=user_id) + if result is None: + raise HTTPException(status_code=404, detail="Subagent batch not found") + return result + + +@router.post("/{batch_id}/items/{item_id}/retry") +@require_permission("threads", "write", owner_check=True) +async def retry_batch_item(thread_id: ThreadId, batch_id: str, item_id: str, request: Request) -> dict: + repo, user_id, _batch = await _owned_batch(request, thread_id, batch_id) + item = await repo.retry_item(batch_id, item_id, user_id=user_id) + if item is None: + raise HTTPException(status_code=409, detail="Only failed items can be retried") + return item + + +@router.get("/{batch_id}/results.jsonl") +@require_permission("threads", "read", owner_check=True) +async def export_batch_results(thread_id: ThreadId, batch_id: str, request: Request) -> StreamingResponse: + repo, user_id, _batch = await _owned_batch(request, thread_id, batch_id) + + async def lines() -> AsyncIterator[bytes]: + offset = 0 + while True: + page = await repo.list_items( + batch_id, + user_id=user_id, + offset=offset, + limit=500, + include_result=True, + ) + if not page: + break + for item in page: + yield (json.dumps(item, ensure_ascii=False, default=str) + "\n").encode() + offset += len(page) + + return StreamingResponse( + lines(), + media_type="application/x-ndjson", + headers={"Content-Disposition": f'attachment; filename="{batch_id}-results.jsonl"'}, + ) diff --git a/backend/app/subagent_batches/__init__.py b/backend/app/subagent_batches/__init__.py new file mode 100644 index 000000000..c7cdd68df --- /dev/null +++ b/backend/app/subagent_batches/__init__.py @@ -0,0 +1,3 @@ +from deerflow.subagents.batch_service import SubagentBatchService + +__all__ = ["SubagentBatchService"] diff --git a/backend/app/subagent_batches/service.py b/backend/app/subagent_batches/service.py new file mode 100644 index 000000000..2f3538e6c --- /dev/null +++ b/backend/app/subagent_batches/service.py @@ -0,0 +1,5 @@ +"""Compatibility import for the harness-owned subagent batch service.""" + +from deerflow.subagents.batch_service import SubagentBatchService + +__all__ = ["SubagentBatchService"] diff --git a/backend/docs/rfc-create-deerflow-agent.md b/backend/docs/rfc-create-deerflow-agent.md index cfb5be2f1..688a676ac 100644 --- a/backend/docs/rfc-create-deerflow-agent.md +++ b/backend/docs/rfc-create-deerflow-agent.md @@ -144,12 +144,17 @@ def create_deerflow_agent( state_schema: type | None = None, checkpointer: BaseCheckpointSaver | None = None, name: str = "default", + subagent_runtime: SubagentRuntime | None = None, ) -> CompiledStateGraph: ... ``` `DeerFlowClient` 内部调用此函数。 +直接集成需要原生子智能体时，调用方可以显式传入一个 `SubagentRuntime`。同一应用内的多个 graph 应复用同一实例，让 middleware 限制、`task` 工具、真实执行槽位和可选批处理 worker 共享同一份容量快照。这个参数不改变“纯参数工厂”边界：工厂不读取 YAML、不创建 SQL repository，也不启动后台 worker。 + +当 runtime 持有批处理 repository 时，调用方必须在构建 graph 前执行 `await runtime.start()`（或使用 `async with runtime`），并在应用停机时 `stop()`。直接工厂只绑定执行工具和 worker，不会自动挂载 Gateway 的 HTTP API 或前端面板。只使用普通 `task` 时不需要异步启停。 + ### 3.3 `RuntimeFeatures` — 内置 Middleware 替换 只做一件事：用自定义实例替换内置 middleware。不管配置参数（参数走 `config` dict）。 diff --git a/backend/packages/harness/deerflow/agents/AGENTS.md b/backend/packages/harness/deerflow/agents/AGENTS.md index f0548f685..af664f960 100644 --- a/backend/packages/harness/deerflow/agents/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/AGENTS.md @@ -41,3 +41,5 @@ Gateway and `DeerFlowClient.stream()` always provide the runtime `run_id`; custom graph integrations must do the same. If it is absent, enforcement deliberately counts the thread's full delegation ledger (fail-restrictive) and emits a warning. + +**Direct subagent runtime**: `create_deerflow_agent(..., subagent_runtime=runtime)` is the explicit dependency-injection path for direct graph callers. Reuse one `deerflow.subagents.SubagentRuntime` across every graph that belongs to the same application capacity boundary. With the default subagent feature it binds middleware concurrency/total limits, the ordinary `task` tool, one real execution controller, and any active durable-batch submitter to the same snapshot. A caller-owned batch repository requires `await runtime.start()` (or `async with runtime`) before graph construction and `stop()` at shutdown; the factory fails closed while that worker is stopped, and already-built bound batch tools must fail unavailable after it stops rather than falling through to another process-global submitter. The factory never creates SQL infrastructure, renders the caller-owned `system_prompt`, or mounts Gateway API/UI routes. Full middleware takeover cannot be combined with this runtime; direct callers and custom subagent middleware remain responsible for model-visible call-policy wording. diff --git a/backend/packages/harness/deerflow/agents/factory.py b/backend/packages/harness/deerflow/agents/factory.py index 9270d3428..df2c64894 100644 --- a/backend/packages/harness/deerflow/agents/factory.py +++ b/backend/packages/harness/deerflow/agents/factory.py @@ -1,13 +1,14 @@ """Pure-argument factory for DeerFlow agents. -``create_deerflow_agent`` accepts plain Python arguments — no YAML files, no -global singletons. It is the SDK-level entry point sitting between the raw +``create_deerflow_agent`` accepts plain Python arguments — it does not load +YAML or install process-global runtime dependencies. It is the SDK-level entry +point sitting between the raw ``langchain.agents.create_agent`` primitive and the config-driven ``make_lead_agent`` application factory. -Note: the factory assembly itself is config-free, but some injected runtime -components (e.g. ``task_tool`` for subagent) may still read global config at -invocation time. Full config-free runtime is a Phase 2 goal. +Direct callers that need an isolated native-subagent capacity or a durable +batch worker pass a caller-owned ``SubagentRuntime`` explicitly. When omitted, +subagent tools retain their application-compatible process-global fallback. """ from __future__ import annotations @@ -33,6 +34,7 @@ if TYPE_CHECKING: from langgraph.graph.state import CompiledStateGraph from deerflow.config.memory_config import MemoryConfig + from deerflow.subagents.runtime import SubagentRuntime logger = logging.getLogger(__name__) @@ -75,12 +77,13 @@ def create_deerflow_agent( checkpoint_snapshot_frequency: int | None = None, checkpointer: BaseCheckpointSaver | None = None, name: str = "default", + subagent_runtime: SubagentRuntime | None = None, ) -> CompiledStateGraph: """Create a DeerFlow agent from plain Python arguments. - The factory assembly itself reads no config files. Some injected runtime - components (e.g. ``task_tool``) may still depend on global config at - invocation time — see Phase 2 roadmap for full config-free runtime. + The factory assembly itself reads no config files. Pass ``subagent_runtime`` + when direct SDK-created graphs must share an explicit native-subagent + capacity or caller-managed durable batch worker. Parameters ---------- @@ -116,6 +119,11 @@ def create_deerflow_agent( Optional persistence backend. name: Agent name (passed to middleware that cares, e.g. ``MemoryMiddleware``). + subagent_runtime: + Explicit process runtime shared by direct SDK-created graphs. Required + only when the caller needs non-default native-subagent capacity or a + caller-managed durable batch worker without Gateway/DeerFlowClient + startup. Requires ``features.subagent`` to be enabled. Raises ------ @@ -134,6 +142,10 @@ def create_deerflow_agent( ) if middleware is not None and extra_middleware: raise ValueError("Cannot use 'extra_middleware' with 'middleware' (full takeover).") + if subagent_runtime is not None and (middleware is not None or features is None or features.subagent is False): + raise ValueError("subagent_runtime requires features.subagent to be enabled; it cannot be used with middleware full takeover") + if subagent_runtime is not None and subagent_runtime.batch_config is not None and subagent_runtime.batch_submitter is None: + raise RuntimeError("The explicit durable batch worker is not running; await subagent_runtime.start() or enter it with 'async with' before calling create_deerflow_agent") if extra_middleware: for mw in extra_middleware: if not isinstance(mw, AgentMiddleware): @@ -151,6 +163,7 @@ def create_deerflow_agent( name=name, plan_mode=plan_mode, extra_middleware=extra_middleware or [], + subagent_runtime=subagent_runtime, ) # Deduplicate by tool name — user-provided tools take priority. existing_names = {t.name for t in effective_tools} @@ -187,6 +200,7 @@ def _assemble_from_features( name: str = "default", plan_mode: bool = False, extra_middleware: list[AgentMiddleware] | None = None, + subagent_runtime: SubagentRuntime | None = None, ) -> tuple[list[AgentMiddleware], list[BaseTool]]: """Build an ordered middleware chain + extra tools from *feat*. @@ -316,11 +330,47 @@ def _assemble_from_features( chain.append(feat.subagent) else: from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware + from deerflow.config.subagents_config import DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN + from deerflow.subagents.capacity import configured_subagent_max_running - chain.append(SubagentLimitMiddleware()) + max_concurrent = subagent_runtime.config.max_running if subagent_runtime is not None else configured_subagent_max_running() + max_total = subagent_runtime.max_total_per_run if subagent_runtime is not None else DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN + chain.append( + SubagentLimitMiddleware( + max_concurrent=max_concurrent, + max_total=max_total, + ) + ) from deerflow.tools.builtins import task_tool - extra_tools.append(task_tool) + if subagent_runtime is None: + extra_tools.append(task_tool) + else: + from deerflow.tools.builtins.task_tool import bind_task_tool + + extra_tools.append( + bind_task_tool( + subagent_runtime.execution_capacity, + app_config=subagent_runtime.app_config, + ) + ) + + if subagent_runtime is not None and subagent_runtime.batch_submitter is not None: + from deerflow.tools.builtins.batch_task_tool import bind_batch_tools + + extra_tools.extend( + bind_batch_tools( + submitter_provider=lambda: subagent_runtime.batch_submitter, + app_config=subagent_runtime.app_config, + ) + ) + elif subagent_runtime is None: + from deerflow.subagents.batch_runtime import is_subagent_batch_runtime_available + + if is_subagent_batch_runtime_available(): + from deerflow.tools.builtins import batch_status, batch_task, cancel_batch + + extra_tools.extend((batch_task, batch_status, cancel_batch)) # --- [12] LoopDetection --- if feat.loop_detection is not False: diff --git a/backend/packages/harness/deerflow/agents/lead_agent/agent.py b/backend/packages/harness/deerflow/agents/lead_agent/agent.py index 49af99403..b72c5423e 100644 --- a/backend/packages/harness/deerflow/agents/lead_agent/agent.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/agent.py @@ -57,7 +57,10 @@ from deerflow.authz.tool_filter import apply_tool_authorization from deerflow.config.agents_config import load_agent_config, validate_agent_name from deerflow.config.app_config import AppConfig, get_app_config from deerflow.config.memory_config import should_use_memory_tools -from deerflow.config.subagents_config import DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN +from deerflow.config.subagents_config import ( + DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN, + effective_subagent_concurrency, +) from deerflow.models import create_chat_model from deerflow.runtime.checkpoint_mode import ( INTERNAL_CHECKPOINT_MODE_KEY, @@ -67,6 +70,7 @@ from deerflow.runtime.checkpoint_mode import ( inject_checkpoint_mode, ) from deerflow.skills.types import Skill +from deerflow.subagents.capacity import configured_subagent_max_running from deerflow.tracing import build_tracing_callbacks logger = logging.getLogger(__name__) @@ -463,6 +467,7 @@ def build_middlewares( user_id: str | None = None, authorization_provider=None, extensions=None, + subagent_execution_capacity: int | None = None, ): """Build the lead-agent middleware chain based on runtime configuration. @@ -485,6 +490,8 @@ def build_middlewares( to ``SkillActivationMiddleware`` so it can resolve per-user custom skills. authorization_provider: Provider already resolved for assembly-time filtering. Reused by the execution-time authorization middleware. + subagent_execution_capacity: Startup-frozen process capacity used to + keep advertised and enforced task concurrency aligned after reloads. extensions: Loaded extensions whose middleware contributions are merged into the final stack. Defaults to the process-wide set. @@ -625,7 +632,11 @@ def build_middlewares( subagent_enabled = cfg.get("subagent_enabled", False) effective_max_subagents_per_run: int | None = None if subagent_enabled: - max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3) + max_concurrent_subagents = effective_subagent_concurrency( + cfg.get("max_concurrent_subagents"), + resolved_app_config, + execution_capacity=subagent_execution_capacity, + ) max_total_subagents = cfg.get("max_total_subagents", _default_max_total_subagents(resolved_app_config)) effective_max_subagents_per_run = max_total_subagents middlewares.append(SubagentLimitMiddleware(max_concurrent=max_concurrent_subagents, max_total=max_total_subagents)) @@ -870,7 +881,12 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le requested_model_name: str | None = cfg.get("model_name") or cfg.get("model") is_plan_mode = cfg.get("is_plan_mode", False) requested_subagent_enabled = cfg.get("subagent_enabled", False) - max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3) + subagent_execution_capacity = configured_subagent_max_running() + max_concurrent_subagents = effective_subagent_concurrency( + cfg.get("max_concurrent_subagents"), + resolved_app_config, + execution_capacity=subagent_execution_capacity, + ) max_total_subagents = cfg.get("max_total_subagents", _default_max_total_subagents(resolved_app_config)) is_bootstrap = cfg.get("is_bootstrap", False) non_interactive = bool(cfg.get("non_interactive", False)) @@ -1014,6 +1030,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le mcp_routing_middleware=mcp_routing_middleware, user_id=resolved_user_id, authorization_provider=_authz_provider, + subagent_execution_capacity=subagent_execution_capacity, ) system_prompt = apply_prompt_template( subagent_enabled=subagent_enabled, @@ -1025,6 +1042,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le user_id=resolved_user_id, skill_names=skill_setup.skill_names or None, allowed_subagents=allowed_subagents, + subagent_execution_capacity=subagent_execution_capacity, ) graph = create_agent( model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, app_config=resolved_app_config, attach_tracing=False), @@ -1128,6 +1146,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le mcp_routing_middleware=mcp_routing_middleware, user_id=resolved_user_id, authorization_provider=_authz_provider, + subagent_execution_capacity=subagent_execution_capacity, ) system_prompt = apply_prompt_template( subagent_enabled=subagent_enabled, @@ -1141,6 +1160,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le user_id=resolved_user_id, skill_names=skill_setup.skill_names or None, allowed_subagents=allowed_subagents, + subagent_execution_capacity=subagent_execution_capacity, ) graph = create_agent( model=create_chat_model(name=model_name, thinking_enabled=thinking_enabled, reasoning_effort=reasoning_effort, app_config=resolved_app_config, attach_tracing=False, model_overrides=agent_model_overrides), diff --git a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py index 6ee451672..658c094aa 100644 --- a/backend/packages/harness/deerflow/agents/lead_agent/prompt.py +++ b/backend/packages/harness/deerflow/agents/lead_agent/prompt.py @@ -14,6 +14,7 @@ from deerflow.config.subagents_config import ( DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN, clamp_subagent_concurrency, clamp_total_subagents_per_run, + effective_subagent_concurrency, ) from deerflow.constants import DEFAULT_SKILLS_CONTAINER_PATH from deerflow.skills.storage import get_or_new_skill_storage, get_or_new_user_skill_storage @@ -344,6 +345,7 @@ def _build_subagent_section( *, app_config: AppConfig | None = None, allowed_subagents: list[str] | None = None, + batch_enabled: bool = False, ) -> str: """Build the subagent system prompt section with dynamic subagent limits. @@ -426,6 +428,24 @@ A single subagent is justified only by material specialist or context-isolation - Wait for the batch, then re-evaluate the remaining work and net benefit. - **Batch 2** may launch the next scopes if it still wins; otherwise continue directly. - **Synthesize all retained results** at the end. +""" + durable_batch_guidance = "" + if batch_enabled: + durable_batch_guidance = """ +## Explicit durable batch mode + +`batch_task` is a separate execution mode for a large collection of independent, +idempotent or read-only items. It returns a durable batch id immediately and does +not consume the ordinary `task` per-run total. Never infer batch mode from item +count and never emulate it by repeatedly calling `task`. + +- Every item must be self-contained and must not depend on another item's output. +- Give every item a stable unique key; retries reuse that key as idempotency identity. +- Set total, live-window, and running concurrency separately. A high total never + implies that all items become live or run at once. +- Use `batch_status` for compact progress and `cancel_batch` for cancellation. +- Do not wait for or paste all item results into this run. The Web UI and results + export API own progress and result inspection. """ return f""" ## Subagent Routing: Delegate Only for Clear Net Benefit @@ -476,6 +496,7 @@ Otherwise execute directly using available tools ({direct_tool_examples}): ``` The `task` tool waits for the subagent and returns its result directly; no polling is needed. +{durable_batch_guidance} """ @@ -1010,15 +1031,38 @@ def apply_prompt_template( user_id: str | None = None, skill_names: frozenset[str] | None = None, allowed_subagents: list[str] | None = None, + subagent_execution_capacity: int | None = None, ) -> str: # Include subagent section only if enabled (from runtime parameter) - n = clamp_subagent_concurrency(max_concurrent_subagents) + n = ( + effective_subagent_concurrency( + max_concurrent_subagents, + app_config, + execution_capacity=subagent_execution_capacity, + ) + if app_config is not None + else clamp_subagent_concurrency( + max_concurrent_subagents, + execution_capacity=subagent_execution_capacity, + ) + ) total = max_total_subagents if total is None: subagents_config = getattr(app_config, "subagents", None) if app_config is not None else None total = getattr(subagents_config, "max_total_per_run", DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN) total = clamp_total_subagents_per_run(total) - subagent_section = _build_subagent_section(n, total, app_config=app_config, allowed_subagents=allowed_subagents) if subagent_enabled else "" + if subagent_enabled: + from deerflow.subagents.batch_runtime import is_subagent_batch_runtime_available + + subagent_section = _build_subagent_section( + n, + total, + app_config=app_config, + allowed_subagents=allowed_subagents, + batch_enabled=is_subagent_batch_runtime_available(), + ) + else: + subagent_section = "" # Add subagent reminder to critical_reminders if enabled reminder_benefits = "specialist capability or context isolation" if n == 1 else "real parallel latency, specialist capability, or context isolation" diff --git a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md index 066304a66..79d8b26ca 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md @@ -88,7 +88,7 @@ Before changing a later authorization phase, read the [authorization RFC](../../ 24. **McpRoutingMiddleware** - *(optional, if `tool_search.enabled` and PR1 MCP routing metadata produce a routing index)* Auto-promotes matching deferred MCP tool schemas before the model call by writing a minimal `promoted` state update. It matches only the latest real `HumanMessage`, uses the global `tool_search.auto_promote_top_k` limit (default 3, clamped to 1..5), never executes tools, and must be installed before `DeferredToolFilterMiddleware` 25. **DeferredToolFilterMiddleware** - *(optional, if `tool_search.enabled`)* Hides deferred (MCP) tool schemas from the bound model until `tool_search` or `McpRoutingMiddleware` promotes them (reads per-thread promotions from `ThreadState.promoted`, hash-scoped) 26. **SystemMessageCoalescingMiddleware** - Merges every SystemMessage into a single leading SystemMessage per request; provider-agnostic fix for strict backends (vLLM/SGLang/Qwen/Anthropic) that reject non-leading system messages. Touches the per-request payload only (checkpoint state unchanged); on midnight crossings only the latest `dynamic_context_reminder` SystemMessage survives. The subagent builder places its date-only context middleware immediately before this coalescer, so the built-in subagent prompt and hidden date reminder still reach providers as one leading system block -27. **SubagentLimitMiddleware** - *(optional, if `subagent_enabled`)* Truncates excess `task` tool calls to enforce both the per-response concurrency limit (`max_concurrent_subagents`, clamped to 1-4) and the per-run total delegation cap (`max_total_subagents` runtime override or `subagents.max_total_per_run`, default 6, clamped to 1-50). The total cap counts current-run entries in the durable delegation ledger (entries are tagged with `run_id` when captured), so repeated planning checkpoints in one run cannot keep launching legal-sized batches indefinitely, while later user turns in the same thread get a fresh run budget. If the cap is exhausted, the middleware strips remaining `task` calls, forces `finish_reason="stop"`, and appends a visible limit note so the run can synthesize existing results instead of ending with an empty tool-call response. +27. **SubagentLimitMiddleware** - *(optional, if `subagent_enabled`)* Truncates excess ordinary `task` tool calls to enforce both the per-response concurrency limit (`max_concurrent_subagents`, resolved against startup `subagent_runtime.max_running` and the 1-64 safety range before construction) and the per-run total delegation cap (`max_total_subagents` runtime override or `subagents.max_total_per_run`, default 6, clamped to 1-50). The total cap counts current-run entries in the durable delegation ledger (entries are tagged with `run_id` when captured), so repeated planning checkpoints in one run cannot keep launching legal-sized batches indefinitely, while later user turns in the same thread get a fresh run budget. Explicit durable `batch_task` calls are a separate mode with persisted total/live/running limits and are not rewritten into ordinary ledger entries. If the ordinary cap is exhausted, the middleware strips remaining `task` calls, forces `finish_reason="stop"`, and appends a visible limit note so the run can synthesize existing results instead of ending with an empty tool-call response. 28. **LoopDetectionMiddleware** - *(optional, if `loop_detection.enabled`)* Detects repeated tool-call loops; hard-stop clears both structured `tool_calls` and raw provider tool-call metadata before forcing a final text answer; stamps `loop_capped` via `consume_stop_reason` (#3875 Phase 2), symmetric to `TokenBudgetMiddleware` 29. **TokenBudgetMiddleware** - *(optional, if `token_budget.enabled`)* Enforces per-run token limits 30. **Custom middlewares** - *(optional)* Any `custom_middlewares` passed to `build_middlewares` are injected here, before config-declared extensions and the terminal-response/safety/clarification tail diff --git a/backend/packages/harness/deerflow/agents/middlewares/subagent_limit_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/subagent_limit_middleware.py index 35ae913bf..5e852fdcc 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/subagent_limit_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/subagent_limit_middleware.py @@ -36,7 +36,7 @@ _TOTAL_LIMIT_STOP_MSG = ( def _clamp_subagent_limit(value: int) -> int: - """Clamp subagent limit to valid range [1, 4].""" + """Clamp subagent limit to the hard safety range [1, 64].""" return clamp_subagent_concurrency(value) @@ -104,7 +104,8 @@ class SubagentLimitMiddleware(AgentMiddleware[AgentState]): Args: max_concurrent: Maximum number of concurrent subagent calls allowed. - Defaults to MAX_CONCURRENT_SUBAGENTS (3). Clamped to [1, 4]. + Defaults to MAX_CONCURRENT_SUBAGENTS (3). Callers pass the value + already clamped to the configured process execution capacity. max_total: Maximum number of subagent calls allowed across the run. Defaults to 6. Clamped to [1, 50]. """ diff --git a/backend/packages/harness/deerflow/client.py b/backend/packages/harness/deerflow/client.py index 47446829a..a5cdd5ffe 100644 --- a/backend/packages/harness/deerflow/client.py +++ b/backend/packages/harness/deerflow/client.py @@ -49,6 +49,7 @@ from deerflow.config.extensions_config import ( reload_extensions_config, ) from deerflow.config.paths import get_paths +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig from deerflow.models import create_chat_model from deerflow.runtime import CheckpointStateAccessor from deerflow.runtime.checkpoint_mode import ( @@ -61,6 +62,7 @@ from deerflow.runtime.goal import DEFAULT_MAX_GOAL_CONTINUATIONS, build_goal_sta from deerflow.runtime.user_context import get_effective_user_id from deerflow.skills.describe import build_skill_search_setup from deerflow.skills.storage import get_or_new_user_skill_storage +from deerflow.subagents.capacity import configure_subagent_execution_capacity from deerflow.tools.builtins.tool_search import assemble_deferred_tools, build_mcp_routing_middleware, get_mcp_routing_hints_prompt_section from deerflow.trace_context import DEERFLOW_TRACE_METADATA_KEY, generate_trace_id, get_current_trace_id, reset_current_trace_id, set_current_trace_id from deerflow.tracing import build_tracing_callbacks, inject_langfuse_metadata @@ -199,6 +201,13 @@ class DeerFlowClient: if config_path is not None: reload_app_config(config_path) self._app_config = get_app_config() + runtime_config = getattr(self._app_config, "subagent_runtime", None) + if not isinstance(runtime_config, SubagentRuntimeConfig): + # Preserve compatibility with lightweight embedded/test configs + # created before the startup-only section existed. + runtime_config = SubagentRuntimeConfig() + configure_subagent_execution_capacity(runtime_config) + self._subagent_execution_capacity = runtime_config.max_running self._checkpoint_channel_mode = freeze_checkpoint_channel_mode(self._app_config.database.checkpoint_channel_mode) self._checkpoint_snapshot_frequency = freeze_checkpoint_snapshot_frequency(self._app_config.database.checkpoint_delta.snapshot_frequency) @@ -303,7 +312,22 @@ class DeerFlowClient: model_name = self._app_config.models[0].name model_name = _authorize_model_name(model_name, context=cfg, app_config=self._app_config) subagent_enabled = cfg.get("subagent_enabled", False) - max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3) + from deerflow.config.subagents_config import effective_subagent_concurrency + + # Lightweight integrations and older tests may construct a client via + # ``__new__`` and inject only ``_app_config``. Production clients keep + # the startup snapshot set by ``__init__``; the fallback preserves the + # pre-snapshot construction contract without consulting global state. + subagent_execution_capacity = getattr( + self, + "_subagent_execution_capacity", + int(getattr(getattr(self._app_config, "subagent_runtime", None), "max_running", 3)), + ) + max_concurrent_subagents = effective_subagent_concurrency( + cfg.get("max_concurrent_subagents"), + self._app_config, + execution_capacity=subagent_execution_capacity, + ) max_total_subagents = cfg.get("max_total_subagents", self._app_config.subagents.max_total_per_run) tools = self._get_tools(model_name=model_name, subagent_enabled=subagent_enabled) @@ -363,6 +387,7 @@ class DeerFlowClient: mcp_routing_middleware=mcp_routing_middleware, user_id=effective_user_id, authorization_provider=_authz_provider, + subagent_execution_capacity=subagent_execution_capacity, ), self._checkpoint_channel_mode, self._checkpoint_snapshot_frequency, @@ -378,6 +403,7 @@ class DeerFlowClient: mcp_routing_hints_section=mcp_routing_hints_section, user_id=effective_user_id, skill_names=skill_setup.skill_names or None, + subagent_execution_capacity=subagent_execution_capacity, ), "state_schema": get_thread_state_schema(self._checkpoint_channel_mode, self._checkpoint_snapshot_frequency), } diff --git a/backend/packages/harness/deerflow/config/AGENTS.md b/backend/packages/harness/deerflow/config/AGENTS.md index e0da87b13..dbe6715a1 100644 --- a/backend/packages/harness/deerflow/config/AGENTS.md +++ b/backend/packages/harness/deerflow/config/AGENTS.md @@ -10,7 +10,7 @@ Setup: Copy `config.example.yaml` to `config.yaml` in the **project root** direc **Config Hot-Reload Boundary**: Gateway dependencies route through `get_app_config()` on every request, so per-run fields like `models[*].max_tokens`, `summarization.*`, `title.*`, `memory.*`, `subagents.*`, `verification.*`, `tools[*]`, and the agent system prompt pick up `config.yaml` edits on the next message. `AppConfig` is intentionally **not** cached on `app.state` — `lifespan()` keeps a local `startup_config` variable for one-shot bootstrap work and passes it to `langgraph_runtime(app, startup_config)`. -Infrastructure fields are **restart-required**. The authoritative list lives in `packages/harness/deerflow/config/reload_boundary.py::STARTUP_ONLY_FIELDS` and is mirrored by the standardised `"startup-only:"` prefix on the corresponding `Field(description=...)` in `AppConfig`, so IDE hover on those fields surfaces the reason inline (no need to context-switch into this table). Currently registered: `plugins`, `database`, `checkpointer`, `run_events`, `stream_bridge`, `sandbox`, `log_level`, `logging`, `channels`, `channel_connections`, `scheduler`, `mcp_tasks`, `run_ownership`. Adding a new restart-required field requires updating the registry; drift is pinned by `tests/test_reload_boundary.py`. `scheduler.recursion_limit` is the exception inside that section: it is read from `get_app_config()` at each scheduled dispatch, so a YAML edit applies to the next run without restarting the poller. +Infrastructure fields are **restart-required**. The authoritative list lives in `packages/harness/deerflow/config/reload_boundary.py::STARTUP_ONLY_FIELDS` and is mirrored by the standardised `"startup-only:"` prefix on the corresponding `Field(description=...)` in `AppConfig`, so IDE hover on those fields surfaces the reason inline (no need to context-switch into this table). Currently registered: `plugins`, `database`, `checkpointer`, `run_events`, `stream_bridge`, `sandbox`, `log_level`, `logging`, `channels`, `channel_connections`, `scheduler`, `mcp_tasks`, `subagent_runtime`, `subagent_batches`, `run_ownership`. Adding a new restart-required field requires updating the registry; drift is pinned by `tests/test_reload_boundary.py`. `scheduler.recursion_limit` is the exception inside that section: it is read from `get_app_config()` at each scheduled dispatch, so a YAML edit applies to the next run without restarting the poller. **Persistence backend resolution**: the unified `database` section selects the Gateway's LangGraph checkpointer, LangGraph Store, and DeerFlow SQL repositories. @@ -58,6 +58,8 @@ Extensions are optional only in the fallback *search* mode (priority 3-4 above): - `title` - Auto-title generation (enabled, max_words, max_chars, model_name; null model_name uses fast local fallback, explicit model_name uses the prompt_template LLM path) - `summarization` - Context summarization (enabled, trigger conditions, keep policy) - `subagents.enabled` - Master switch for subagent delegation +- `subagent_runtime` - Startup-only shared process admission (`max_running`, bounded async wait queue, queue/reject policy, and queue timeout) for ordinary and durable-batch native subagents +- `subagent_batches` - Startup-only explicit durable batch scheduler limits (disabled by default), including separate total, live, and running dimensions plus leases/retries/result bounds - `memory` - Memory system (enabled, storage_path, debounce_seconds, shutdown_flush_timeout_seconds, model_name, max_facts, fact_confidence_threshold, injection_enabled, max_injection_tokens, staleness_review_enabled, staleness_age_days, staleness_min_candidates, staleness_max_removals_per_cycle, staleness_protected_categories, staleness_max_lifetime_multiplier, staleness_max_extension_days) **`extensions_config.json`**: diff --git a/backend/packages/harness/deerflow/config/app_config.py b/backend/packages/harness/deerflow/config/app_config.py index 129186791..5af8109b6 100644 --- a/backend/packages/harness/deerflow/config/app_config.py +++ b/backend/packages/harness/deerflow/config/app_config.py @@ -39,6 +39,8 @@ from deerflow.config.skill_evolution_config import SkillEvolutionConfig from deerflow.config.skill_scan_config import SkillScanConfig from deerflow.config.skills_config import SkillsConfig from deerflow.config.stream_bridge_config import StreamBridgeConfig, load_stream_bridge_config_from_dict +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig from deerflow.config.subagents_config import SubagentsAppConfig, load_subagents_config_from_dict from deerflow.config.suggestions_config import SuggestionsConfig from deerflow.config.summarization_config import SummarizationConfig, load_summarization_config_from_dict @@ -300,6 +302,20 @@ class AppConfig(BaseModel): field_doc="Long-running MCP task persistence and background polling runtime.", ), ) + subagent_runtime: SubagentRuntimeConfig = Field( + default_factory=SubagentRuntimeConfig, + description=format_field_description( + "subagent_runtime", + field_doc="Process-local admission and execution capacity shared by ordinary and batch subagents.", + ), + ) + subagent_batches: SubagentBatchesConfig = Field( + default_factory=SubagentBatchesConfig, + description=format_field_description( + "subagent_batches", + field_doc="Durable native-subagent batch scheduling, lease, and recovery configuration.", + ), + ) checkpointer: CheckpointerConfig | None = Field( default=None, description=format_field_description( diff --git a/backend/packages/harness/deerflow/config/reload_boundary.py b/backend/packages/harness/deerflow/config/reload_boundary.py index 5f8dbaffa..31488d063 100644 --- a/backend/packages/harness/deerflow/config/reload_boundary.py +++ b/backend/packages/harness/deerflow/config/reload_boundary.py @@ -76,6 +76,8 @@ STARTUP_ONLY_FIELDS: dict[str, str] = { "McpTaskService is constructed and started once during Gateway lifespan startup; enabled, poll_interval_seconds, lease_seconds, " "and max_concurrent_polls are captured into the service instance and the background poller task is not rebuilt on config.yaml edits." ), + "subagent_runtime": ("the shared native-subagent admission controller and isolated execution loop are configured once during Gateway lifespan startup; changing process slots, queue policy, or queue bounds requires a restart."), + "subagent_batches": ("the durable subagent batch service is constructed and started once during Gateway lifespan startup; scheduler limits, leases, and recovery behavior are captured by that service instance."), "run_ownership": ( "RunOwnershipConfig is captured once into RunManager at langgraph_runtime() startup; the lease heartbeat background task is created and " "started there, and heartbeat_enabled / lease_seconds / grace_seconds are not re-read on config.yaml edits." diff --git a/backend/packages/harness/deerflow/config/subagent_batches_config.py b/backend/packages/harness/deerflow/config/subagent_batches_config.py new file mode 100644 index 000000000..c42f4d698 --- /dev/null +++ b/backend/packages/harness/deerflow/config/subagent_batches_config.py @@ -0,0 +1,31 @@ +"""Startup-only configuration for durable native-subagent batches.""" + +from pydantic import BaseModel, Field, model_validator + + +class SubagentBatchesConfig(BaseModel): + """Durable batch scheduler limits and recovery settings.""" + + enabled: bool = Field(default=False) + poll_interval_seconds: float = Field(default=1.0, ge=0.1, le=60.0) + lease_seconds: int = Field(default=120, ge=10, le=3600) + max_items_per_batch: int = Field(default=5_000, ge=1, le=100_000) + default_max_live_items: int = Field(default=100, ge=1, le=10_000) + max_live_items_per_batch: int = Field(default=1_000, ge=1, le=100_000) + default_max_running_items: int = Field(default=3, ge=1, le=64) + max_running_items_per_batch: int = Field(default=64, ge=1, le=1_000) + max_attempts: int = Field(default=3, ge=1, le=10) + max_result_chars: int = Field(default=100_000, ge=1_000, le=1_000_000) + result_preview_max_chars: int = Field(default=2_000, ge=64, le=100_000) + + @model_validator(mode="after") + def validate_default_limits(self) -> "SubagentBatchesConfig": + if self.default_max_live_items > self.max_live_items_per_batch: + raise ValueError("default_max_live_items must not exceed max_live_items_per_batch") + if self.default_max_running_items > self.max_running_items_per_batch: + raise ValueError("default_max_running_items must not exceed max_running_items_per_batch") + if self.default_max_running_items > self.default_max_live_items: + raise ValueError("default_max_running_items must not exceed default_max_live_items") + if self.result_preview_max_chars > self.max_result_chars: + raise ValueError("result_preview_max_chars must not exceed max_result_chars") + return self diff --git a/backend/packages/harness/deerflow/config/subagent_runtime_config.py b/backend/packages/harness/deerflow/config/subagent_runtime_config.py new file mode 100644 index 000000000..a6301f1ec --- /dev/null +++ b/backend/packages/harness/deerflow/config/subagent_runtime_config.py @@ -0,0 +1,32 @@ +"""Startup-only process capacity for native subagent execution.""" + +from typing import Literal + +from pydantic import BaseModel, Field + + +class SubagentRuntimeConfig(BaseModel): + """Process-local admission and execution limits shared by all subagents.""" + + max_running: int = Field( + default=3, + ge=1, + le=64, + description="Maximum native subagents that may execute concurrently in one Gateway process.", + ) + max_queued: int = Field( + default=64, + ge=0, + le=10_000, + description="Maximum native subagents waiting for a process execution slot.", + ) + admission_policy: Literal["queue", "reject"] = Field( + default="queue", + description="Whether a full execution pool queues work or rejects it immediately.", + ) + queue_timeout_seconds: int = Field( + default=300, + ge=1, + le=86_400, + description="Maximum wait for a queued native subagent before it fails admission.", + ) diff --git a/backend/packages/harness/deerflow/config/subagents_config.py b/backend/packages/harness/deerflow/config/subagents_config.py index 699669e6c..8125783be 100644 --- a/backend/packages/harness/deerflow/config/subagents_config.py +++ b/backend/packages/harness/deerflow/config/subagents_config.py @@ -12,12 +12,28 @@ DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN = 6 MIN_TOTAL_SUBAGENTS_PER_RUN = 1 MAX_TOTAL_SUBAGENTS_PER_RUN = 50 MIN_CONCURRENT_SUBAGENT_CALLS = 1 -MAX_CONCURRENT_SUBAGENT_CALLS = 4 +MAX_CONCURRENT_SUBAGENT_CALLS = 64 -def clamp_subagent_concurrency(value: int) -> int: - """Clamp per-response task call concurrency to the enforced middleware range.""" - return max(MIN_CONCURRENT_SUBAGENT_CALLS, min(MAX_CONCURRENT_SUBAGENT_CALLS, value)) +def clamp_subagent_concurrency(value: int, *, execution_capacity: int | None = None) -> int: + """Clamp task-call concurrency to both the safety ceiling and real slots.""" + upper = MAX_CONCURRENT_SUBAGENT_CALLS + if execution_capacity is not None: + upper = min(upper, max(MIN_CONCURRENT_SUBAGENT_CALLS, execution_capacity)) + return max(MIN_CONCURRENT_SUBAGENT_CALLS, min(upper, value)) + + +def effective_subagent_concurrency( + value: int | None, + app_config: object, + *, + execution_capacity: int | None = None, +) -> int: + """Resolve one value for prompt, middleware, and process execution capacity.""" + runtime = getattr(app_config, "subagent_runtime", None) + capacity = int(execution_capacity if execution_capacity is not None else getattr(runtime, "max_running", 3)) + requested = capacity if value is None else int(value) + return clamp_subagent_concurrency(requested, execution_capacity=capacity) def clamp_total_subagents_per_run(value: int) -> int: diff --git a/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md b/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md index 7b69fcdbb..99dcbf1f2 100644 --- a/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md +++ b/backend/packages/harness/deerflow/persistence/migrations/AGENTS.md @@ -85,6 +85,7 @@ on installs that never enabled it. The convention is: - `migrations/versions/0013_mcp_task_notifications.py` — adds durable Agent-run notification snapshots, delivery leases, idempotency fields, and the separate bounded-retry attempt counter - `migrations/versions/0014_managed_subagents.py` — creates the deployment-level managed Subagent catalog table - `migrations/versions/0015_scheduled_task_enqueue.py` — interrupts legacy transient queued rows, adds durable scheduled-run launch leases and attempt counts, expands the one-active-occurrence index to `queued`/`launching`/`running`, and migrates the overlap policy from `skip` to `enqueue`; chains after `0014_managed_subagents` +- `migrations/versions/0016_subagent_batches.py` — creates durable native-subagent batch and item tables, including owner/submission idempotency, item identity, lease/recovery state, and result fields - `persistence/bootstrap.py` — `bootstrap_schema(engine, backend=...)`, the three-branch decision + locking - `extensions/loader.py::load_extensions` — registers each spec's `table_prefix` with `register_extension_table_prefix()` - Tests: `tests/test_persistence_bootstrap.py` (branches), `tests/test_persistence_bootstrap_concurrency.py` (concurrency), `tests/test_persistence_bootstrap_regression.py` (issue #3682), `tests/test_persistence_migrations_env.py` (filter, including extension-owned tables), `tests/test_extension_loader.py::TestTablePrefixRegistration` (spec-to-filter wiring), `tests/blocking_io/test_persistence_bootstrap.py` (asyncio.to_thread anchor), `tests/test_migration_0004_run_ownership_dedupe.py` + `tests/test_migration_0007_scheduled_run_active_dedupe.py` (dedupe-before-unique-index pre-steps) diff --git a/backend/packages/harness/deerflow/persistence/migrations/versions/0016_subagent_batches.py b/backend/packages/harness/deerflow/persistence/migrations/versions/0016_subagent_batches.py new file mode 100644 index 000000000..52baf2ebb --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/migrations/versions/0016_subagent_batches.py @@ -0,0 +1,96 @@ +"""durable native-subagent batches. + +Revision ID: 0016_subagent_batches +Revises: 0015_scheduled_task_enqueue +Create Date: 2026-08-24 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "0016_subagent_batches" +down_revision: str | Sequence[str] | None = "0015_scheduled_task_enqueue" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table("subagent_batches"): + op.create_table( + "subagent_batches", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("user_id", sa.String(length=64), nullable=False), + sa.Column("thread_id", sa.String(length=64), nullable=False), + sa.Column("run_id", sa.String(length=64), nullable=True), + sa.Column("tool_call_id", sa.String(length=128), nullable=True), + sa.Column("submission_key", sa.String(length=256), nullable=False), + sa.Column("title", sa.String(length=256), nullable=False), + sa.Column("subagent_type", sa.String(length=128), nullable=False), + sa.Column("status", sa.String(length=24), nullable=False), + sa.Column("total_items", sa.Integer(), nullable=False), + sa.Column("max_live_items", sa.Integer(), nullable=False), + sa.Column("max_running_items", sa.Integer(), nullable=False), + sa.Column("max_attempts", sa.Integer(), nullable=False), + sa.Column("execution_spec", sa.JSON(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "submission_key", name="uq_subagent_batches_user_submission"), + ) + op.create_index("ix_subagent_batches_user_id", "subagent_batches", ["user_id"]) + op.create_index("ix_subagent_batches_thread_id", "subagent_batches", ["thread_id"]) + op.create_index("ix_subagent_batches_status", "subagent_batches", ["status"]) + op.create_index("ix_subagent_batches_thread_created", "subagent_batches", ["thread_id", "created_at"]) + + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table("subagent_batch_items"): + op.create_table( + "subagent_batch_items", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("batch_id", sa.String(length=64), nullable=False), + sa.Column("item_key", sa.String(length=128), nullable=False), + sa.Column("position", sa.Integer(), nullable=False), + sa.Column("prompt", sa.Text(), nullable=False), + sa.Column("status", sa.String(length=24), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=False), + sa.Column("lease_owner", sa.String(length=128), nullable=True), + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("cancel_requested_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("model_name", sa.String(length=128), nullable=True), + sa.Column("result", sa.Text(), nullable=True), + sa.Column("result_preview", sa.Text(), nullable=True), + sa.Column("result_truncated", sa.Boolean(), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("stop_reason", sa.String(length=64), nullable=True), + sa.Column("token_usage", sa.JSON(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["batch_id"], ["subagent_batches.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("batch_id", "item_key", name="uq_subagent_batch_items_key"), + sa.UniqueConstraint("batch_id", "position", name="uq_subagent_batch_items_position"), + ) + op.create_index("ix_subagent_batch_items_batch_id", "subagent_batch_items", ["batch_id"]) + op.create_index("ix_subagent_batch_items_status", "subagent_batch_items", ["status"]) + op.create_index( + "ix_subagent_batch_items_claim", + "subagent_batch_items", + ["status", "lease_expires_at", "batch_id"], + ) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if inspector.has_table("subagent_batch_items"): + op.drop_table("subagent_batch_items") + inspector = sa.inspect(op.get_bind()) + if inspector.has_table("subagent_batches"): + op.drop_table("subagent_batches") diff --git a/backend/packages/harness/deerflow/persistence/models/__init__.py b/backend/packages/harness/deerflow/persistence/models/__init__.py index 21b981e87..0732a68a3 100644 --- a/backend/packages/harness/deerflow/persistence/models/__init__.py +++ b/backend/packages/harness/deerflow/persistence/models/__init__.py @@ -28,6 +28,7 @@ from deerflow.persistence.models.run_event import RunEventRow from deerflow.persistence.run.model import RunRow from deerflow.persistence.scheduled_task_runs.model import ScheduledTaskRunRow from deerflow.persistence.scheduled_tasks.model import ScheduledTaskRow +from deerflow.persistence.subagent_batches.model import SubagentBatchItemRow, SubagentBatchRow from deerflow.persistence.thread_meta.model import ThreadMetaRow from deerflow.persistence.user.model import UserRow from deerflow.persistence.webhook_delivery.model import WebhookDeliveryRow @@ -45,6 +46,8 @@ __all__ = [ "RunRow", "ScheduledTaskRow", "ScheduledTaskRunRow", + "SubagentBatchRow", + "SubagentBatchItemRow", "ThreadMetaRow", "UserRow", "WebhookDeliveryRow", diff --git a/backend/packages/harness/deerflow/persistence/subagent_batches/__init__.py b/backend/packages/harness/deerflow/persistence/subagent_batches/__init__.py new file mode 100644 index 000000000..4b8b6bcaf --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/subagent_batches/__init__.py @@ -0,0 +1,4 @@ +from deerflow.persistence.subagent_batches.model import SubagentBatchItemRow, SubagentBatchRow +from deerflow.persistence.subagent_batches.sql import SubagentBatchRepository + +__all__ = ["SubagentBatchItemRow", "SubagentBatchRepository", "SubagentBatchRow"] diff --git a/backend/packages/harness/deerflow/persistence/subagent_batches/model.py b/backend/packages/harness/deerflow/persistence/subagent_batches/model.py new file mode 100644 index 000000000..4c4b9da5e --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/subagent_batches/model.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy import JSON, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from deerflow.persistence.base import Base + + +class SubagentBatchRow(Base): + __tablename__ = "subagent_batches" + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + user_id: Mapped[str] = mapped_column(String(64), index=True) + thread_id: Mapped[str] = mapped_column(String(64), index=True) + run_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + tool_call_id: Mapped[str | None] = mapped_column(String(128), nullable=True) + submission_key: Mapped[str] = mapped_column(String(256)) + title: Mapped[str] = mapped_column(String(256)) + subagent_type: Mapped[str] = mapped_column(String(128)) + status: Mapped[str] = mapped_column(String(24), index=True) + total_items: Mapped[int] = mapped_column(Integer) + max_live_items: Mapped[int] = mapped_column(Integer) + max_running_items: Mapped[int] = mapped_column(Integer) + max_attempts: Mapped[int] = mapped_column(Integer) + execution_spec: Mapped[dict[str, Any]] = mapped_column(JSON) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + __table_args__ = ( + UniqueConstraint("user_id", "submission_key", name="uq_subagent_batches_user_submission"), + Index("ix_subagent_batches_thread_created", "thread_id", "created_at"), + ) + + +class SubagentBatchItemRow(Base): + __tablename__ = "subagent_batch_items" + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + batch_id: Mapped[str] = mapped_column( + ForeignKey("subagent_batches.id", ondelete="CASCADE"), + index=True, + ) + item_key: Mapped[str] = mapped_column(String(128)) + position: Mapped[int] = mapped_column(Integer) + prompt: Mapped[str] = mapped_column(Text) + status: Mapped[str] = mapped_column(String(24), index=True) + attempt: Mapped[int] = mapped_column(Integer, default=0) + lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True) + lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + cancel_requested_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + model_name: Mapped[str | None] = mapped_column(String(128), nullable=True) + result: Mapped[str | None] = mapped_column(Text, nullable=True) + result_preview: Mapped[str | None] = mapped_column(Text, nullable=True) + result_truncated: Mapped[bool] = mapped_column(default=False) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + stop_reason: Mapped[str | None] = mapped_column(String(64), nullable=True) + token_usage: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(UTC)) + + __table_args__ = ( + UniqueConstraint("batch_id", "item_key", name="uq_subagent_batch_items_key"), + UniqueConstraint("batch_id", "position", name="uq_subagent_batch_items_position"), + Index("ix_subagent_batch_items_claim", "status", "lease_expires_at", "batch_id"), + ) diff --git a/backend/packages/harness/deerflow/persistence/subagent_batches/sql.py b/backend/packages/harness/deerflow/persistence/subagent_batches/sql.py new file mode 100644 index 000000000..9775c469d --- /dev/null +++ b/backend/packages/harness/deerflow/persistence/subagent_batches/sql.py @@ -0,0 +1,576 @@ +from __future__ import annotations + +import uuid +from collections import Counter +from datetime import UTC, datetime, timedelta +from typing import Any + +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from deerflow.persistence.subagent_batches.model import SubagentBatchItemRow, SubagentBatchRow +from deerflow.utils.time import coerce_iso + +BATCH_ACTIVE_STATUSES = ("queued", "running", "paused") +BATCH_TERMINAL_STATUSES = ("completed", "failed", "cancelled") +ITEM_ACTIVE_STATUSES = ("queued", "leased", "running") +ITEM_TERMINAL_STATUSES = ("succeeded", "failed", "cancelled") +_BATCH_PUBLIC_FIELDS = ( + "id", + "thread_id", + "title", + "subagent_type", + "status", + "total_items", + "max_live_items", + "max_running_items", + "max_attempts", + "created_at", + "updated_at", + "completed_at", +) +_BATCH_TIMESTAMP_FIELDS = ("created_at", "updated_at", "completed_at") +_ITEM_PUBLIC_FIELDS = ( + "id", + "batch_id", + "item_key", + "position", + "status", + "attempt", + "model_name", + "result_preview", + "result_truncated", + "error", + "stop_reason", + "token_usage", + "started_at", + "completed_at", + "created_at", + "updated_at", +) +_ITEM_TIMESTAMP_FIELDS = ("started_at", "completed_at", "created_at", "updated_at") + + +class SubagentBatchRepository: + """Durable batch/item state with lease-based multi-worker claiming.""" + + def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + self._sf = session_factory + + @staticmethod + def _batch_dict(row: SubagentBatchRow) -> dict[str, Any]: + """Return the stable owner-facing projection, never execution context.""" + data = {key: getattr(row, key) for key in _BATCH_PUBLIC_FIELDS} + for key in _BATCH_TIMESTAMP_FIELDS: + if data.get(key) is not None: + data[key] = coerce_iso(data[key]) + return data + + @staticmethod + def _execution_batch_dict(row: SubagentBatchRow) -> dict[str, Any]: + """Return worker-only fields required to reconstruct an execution.""" + return { + "id": row.id, + "user_id": row.user_id, + "thread_id": row.thread_id, + "run_id": row.run_id, + "execution_spec": row.execution_spec, + } + + @staticmethod + def _item_dict(row: SubagentBatchItemRow, *, include_result: bool = False) -> dict[str, Any]: + data = {key: getattr(row, key) for key in _ITEM_PUBLIC_FIELDS} + if include_result: + data["result"] = row.result + for key in _ITEM_TIMESTAMP_FIELDS: + if data.get(key) is not None: + data[key] = coerce_iso(data[key]) + return data + + async def create_batch( + self, + *, + batch_id: str, + user_id: str, + thread_id: str, + run_id: str | None, + tool_call_id: str | None, + submission_key: str, + title: str, + subagent_type: str, + items: list[dict[str, str]], + max_live_items: int, + max_running_items: int, + max_attempts: int, + execution_spec: dict[str, Any], + ) -> dict[str, Any]: + now = datetime.now(UTC) + batch = SubagentBatchRow( + id=batch_id, + user_id=user_id, + thread_id=thread_id, + run_id=run_id, + tool_call_id=tool_call_id, + submission_key=submission_key, + title=title, + subagent_type=subagent_type, + status="queued", + total_items=len(items), + max_live_items=max_live_items, + max_running_items=max_running_items, + max_attempts=max_attempts, + execution_spec=execution_spec, + created_at=now, + updated_at=now, + ) + rows = [ + SubagentBatchItemRow( + id=f"batch-item-{uuid.uuid4().hex}", + batch_id=batch_id, + item_key=item["key"], + position=position, + prompt=item["prompt"], + status="pending", + attempt=0, + result_truncated=False, + created_at=now, + updated_at=now, + ) + for position, item in enumerate(items) + ] + async with self._sf() as session: + try: + session.add(batch) + # The models intentionally do not declare an ORM relationship; + # flush the parent explicitly so SQLite's immediate FK check + # never observes item inserts before their batch row. Keep the + # flush inside the idempotency handler: a duplicate submission + # key can fail here before commit. + await session.flush() + session.add_all(rows) + await session.commit() + except IntegrityError: + await session.rollback() + existing = ( + await session.execute( + select(SubagentBatchRow).where( + SubagentBatchRow.user_id == user_id, + SubagentBatchRow.submission_key == submission_key, + ) + ) + ).scalar_one_or_none() + if existing is not None: + return await self._with_counts(session, existing) + raise + return await self._with_counts(session, batch) + + async def _counts(self, session: AsyncSession, batch_id: str) -> Counter[str]: + rows = await session.execute(select(SubagentBatchItemRow.status, func.count()).where(SubagentBatchItemRow.batch_id == batch_id).group_by(SubagentBatchItemRow.status)) + return Counter({status: int(count) for status, count in rows}) + + async def _with_counts(self, session: AsyncSession, batch: SubagentBatchRow) -> dict[str, Any]: + counts = await self._counts(session, batch.id) + data = self._batch_dict(batch) + data["counts"] = {status: counts.get(status, 0) for status in ("pending", "queued", "leased", "running", "succeeded", "failed", "cancelled")} + return data + + async def get_batch(self, batch_id: str, *, user_id: str) -> dict[str, Any] | None: + async with self._sf() as session: + batch = await session.get(SubagentBatchRow, batch_id) + if batch is None or batch.user_id != user_id: + return None + return await self._with_counts(session, batch) + + async def list_by_thread(self, thread_id: str, *, user_id: str, limit: int = 20) -> list[dict[str, Any]]: + async with self._sf() as session: + rows = list( + ( + await session.execute( + select(SubagentBatchRow) + .where( + SubagentBatchRow.thread_id == thread_id, + SubagentBatchRow.user_id == user_id, + ) + .order_by(SubagentBatchRow.created_at.desc(), SubagentBatchRow.id.desc()) + .limit(limit) + ) + ).scalars() + ) + return [await self._with_counts(session, row) for row in rows] + + async def list_items( + self, + batch_id: str, + *, + user_id: str, + offset: int = 0, + limit: int = 100, + status: str | None = None, + include_prompt: bool = False, + include_result: bool = False, + ) -> list[dict[str, Any]] | None: + async with self._sf() as session: + batch = await session.get(SubagentBatchRow, batch_id) + if batch is None or batch.user_id != user_id: + return None + stmt = select(SubagentBatchItemRow).where(SubagentBatchItemRow.batch_id == batch_id) + if status is not None: + stmt = stmt.where(SubagentBatchItemRow.status == status) + stmt = stmt.order_by(SubagentBatchItemRow.position).offset(offset).limit(limit) + rows = list((await session.execute(stmt)).scalars()) + values = [] + for row in rows: + value = self._item_dict(row, include_result=include_result) + if include_prompt: + value["prompt"] = row.prompt + values.append(value) + return values + + async def claim_items( + self, + *, + now: datetime, + lease_owner: str, + lease_seconds: int, + limit: int, + ) -> list[dict[str, Any]]: + """Promote pending work and atomically claim runnable items.""" + if limit <= 0: + return [] + claimed: list[dict[str, Any]] = [] + async with self._sf() as session: + batches = list((await session.execute(select(SubagentBatchRow).where(SubagentBatchRow.status.in_(("queued", "running"))).order_by(SubagentBatchRow.created_at, SubagentBatchRow.id).with_for_update(skip_locked=True))).scalars()) + for batch in batches: + if len(claimed) >= limit: + break + + expired = list( + ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.batch_id == batch.id, + SubagentBatchItemRow.status.in_(("leased", "running")), + SubagentBatchItemRow.lease_expires_at < now, + ) + .with_for_update(skip_locked=True) + ) + ).scalars() + ) + for item in expired: + item.lease_owner = None + item.lease_expires_at = None + item.updated_at = now + if item.cancel_requested_at is not None: + item.status = "cancelled" + item.completed_at = now + elif item.attempt >= batch.max_attempts: + item.status = "failed" + item.error = item.error or "Execution lease expired after the maximum retry count" + item.completed_at = now + else: + item.status = "queued" + item.error = "Previous worker lease expired; retrying" + + counts = await self._counts(session, batch.id) + live = counts["queued"] + counts["leased"] + counts["running"] + promote_count = max(0, batch.max_live_items - live) + if promote_count: + pending = list( + ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.batch_id == batch.id, + SubagentBatchItemRow.status == "pending", + ) + .order_by(SubagentBatchItemRow.position) + .limit(promote_count) + .with_for_update(skip_locked=True) + ) + ).scalars() + ) + for item in pending: + item.status = "queued" + item.updated_at = now + + counts = await self._counts(session, batch.id) + batch_available = max(0, batch.max_running_items - counts["leased"] - counts["running"]) + take = min(limit - len(claimed), batch_available) + if take <= 0: + continue + runnable = list( + ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.batch_id == batch.id, + SubagentBatchItemRow.status == "queued", + SubagentBatchItemRow.cancel_requested_at.is_(None), + ) + .order_by(SubagentBatchItemRow.position) + .limit(take) + .with_for_update(skip_locked=True) + ) + ).scalars() + ) + expires_at = now + timedelta(seconds=lease_seconds) + for item in runnable: + item.status = "leased" + item.attempt += 1 + item.lease_owner = lease_owner + item.lease_expires_at = expires_at + item.started_at = now + item.updated_at = now + item.error = None + value = self._item_dict(item) + value["prompt"] = item.prompt + value["batch"] = self._execution_batch_dict(batch) + claimed.append(value) + if runnable: + batch.status = "running" + batch.updated_at = now + await session.commit() + return claimed + + async def renew_item_lease( + self, + item_id: str, + *, + lease_owner: str, + lease_seconds: int, + now: datetime, + ) -> dict[str, bool]: + async with self._sf() as session: + item = ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.id == item_id, + SubagentBatchItemRow.status.in_(("leased", "running")), + SubagentBatchItemRow.lease_owner == lease_owner, + ) + .with_for_update() + ) + ).scalar_one_or_none() + if item is None: + return {"valid": False, "cancel_requested": True} + batch = await session.get(SubagentBatchRow, item.batch_id) + cancel_requested = item.cancel_requested_at is not None or batch is None or batch.status == "cancelled" + if not cancel_requested: + item.lease_expires_at = now + timedelta(seconds=lease_seconds) + item.updated_at = now + await session.commit() + return {"valid": not cancel_requested, "cancel_requested": cancel_requested} + + async def mark_item_running(self, item_id: str, *, lease_owner: str, now: datetime) -> bool: + async with self._sf() as session: + item = ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.id == item_id, + SubagentBatchItemRow.status == "leased", + SubagentBatchItemRow.lease_owner == lease_owner, + ) + .with_for_update() + ) + ).scalar_one_or_none() + if item is None or item.cancel_requested_at is not None: + return False + item.status = "running" + item.started_at = now + item.updated_at = now + await session.commit() + return True + + async def finalize_item( + self, + item_id: str, + *, + lease_owner: str, + succeeded: bool, + result: str | None, + result_preview: str | None, + result_truncated: bool, + error: str | None, + stop_reason: str | None, + token_usage: dict[str, Any] | None, + model_name: str | None, + completed_at: datetime, + ) -> bool: + async with self._sf() as session: + item = ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.id == item_id, + SubagentBatchItemRow.status.in_(("leased", "running")), + SubagentBatchItemRow.lease_owner == lease_owner, + ) + .with_for_update() + ) + ).scalar_one_or_none() + if item is None: + return False + batch = await session.get(SubagentBatchRow, item.batch_id, with_for_update=True) + cancelled = item.cancel_requested_at is not None or batch is None or batch.status == "cancelled" + item.lease_owner = None + item.lease_expires_at = None + item.model_name = model_name + item.stop_reason = stop_reason + item.token_usage = token_usage + item.updated_at = completed_at + if cancelled: + item.status = "cancelled" + item.error = "Cancelled by user" + item.completed_at = completed_at + elif succeeded: + item.status = "succeeded" + item.result = result + item.result_preview = result_preview + item.result_truncated = result_truncated + item.error = None + item.completed_at = completed_at + elif item.attempt < batch.max_attempts: + item.status = "queued" + item.error = error + item.started_at = None + else: + item.status = "failed" + item.error = error + item.completed_at = completed_at + if batch is not None: + await self._refresh_batch_status(session, batch, now=completed_at) + await session.commit() + return True + + async def requeue_item_after_admission_failure( + self, + item_id: str, + *, + lease_owner: str, + error: str | None, + now: datetime, + ) -> bool: + """Undo a claim rejected before execution admission. + + Claiming increments ``attempt`` so crash recovery can bound real + executions. A process-wide capacity rejection happens before an + execution starts, so it must release the lease and restore that + attempt instead of consuming the batch's retry budget. + """ + async with self._sf() as session: + item = ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.id == item_id, + SubagentBatchItemRow.status.in_(("leased", "running")), + SubagentBatchItemRow.lease_owner == lease_owner, + ) + .with_for_update() + ) + ).scalar_one_or_none() + if item is None: + return False + batch = await session.get(SubagentBatchRow, item.batch_id, with_for_update=True) + cancelled = item.cancel_requested_at is not None or batch is None or batch.status == "cancelled" + item.lease_owner = None + item.lease_expires_at = None + item.updated_at = now + if cancelled: + item.status = "cancelled" + item.error = "Cancelled by user" + item.completed_at = now + else: + item.status = "queued" + item.attempt = max(0, item.attempt - 1) + item.started_at = None + item.error = error + if batch is not None: + await self._refresh_batch_status(session, batch, now=now) + await session.commit() + return True + + async def _refresh_batch_status(self, session: AsyncSession, batch: SubagentBatchRow, *, now: datetime) -> None: + counts = await self._counts(session, batch.id) + terminal = sum(counts[state] for state in ITEM_TERMINAL_STATUSES) + if terminal >= batch.total_items: + if batch.status != "cancelled": + batch.status = "failed" if counts["failed"] > 0 and counts["succeeded"] == 0 else "completed" + batch.completed_at = now + elif batch.status not in ("paused", "cancelled"): + batch.status = "running" + batch.updated_at = now + + async def pause_batch(self, batch_id: str, *, user_id: str) -> dict[str, Any] | None: + return await self._set_control(batch_id, user_id=user_id, action="pause") + + async def resume_batch(self, batch_id: str, *, user_id: str) -> dict[str, Any] | None: + return await self._set_control(batch_id, user_id=user_id, action="resume") + + async def cancel_batch(self, batch_id: str, *, user_id: str) -> dict[str, Any] | None: + return await self._set_control(batch_id, user_id=user_id, action="cancel") + + async def _set_control(self, batch_id: str, *, user_id: str, action: str) -> dict[str, Any] | None: + now = datetime.now(UTC) + async with self._sf() as session: + batch = await session.get(SubagentBatchRow, batch_id, with_for_update=True) + if batch is None or batch.user_id != user_id: + return None + if action == "pause" and batch.status in ("queued", "running"): + batch.status = "paused" + elif action == "resume" and batch.status == "paused": + batch.status = "queued" + elif action == "cancel" and batch.status not in BATCH_TERMINAL_STATUSES: + batch.status = "cancelled" + batch.completed_at = now + items = list( + ( + await session.execute( + select(SubagentBatchItemRow) + .where( + SubagentBatchItemRow.batch_id == batch_id, + SubagentBatchItemRow.status.not_in(ITEM_TERMINAL_STATUSES), + ) + .with_for_update() + ) + ).scalars() + ) + for item in items: + item.cancel_requested_at = now + item.updated_at = now + item.status = "cancelled" + item.error = "Cancelled by user" + item.lease_owner = None + item.lease_expires_at = None + item.completed_at = now + batch.updated_at = now + await session.commit() + return await self._with_counts(session, batch) + + async def retry_item(self, batch_id: str, item_id: str, *, user_id: str) -> dict[str, Any] | None: + now = datetime.now(UTC) + async with self._sf() as session: + batch = await session.get(SubagentBatchRow, batch_id, with_for_update=True) + if batch is None or batch.user_id != user_id: + return None + item = await session.get(SubagentBatchItemRow, item_id, with_for_update=True) + if item is None or item.batch_id != batch_id or item.status != "failed": + return None + item.status = "pending" + item.attempt = 0 + item.error = None + item.result = None + item.result_preview = None + item.result_truncated = False + item.completed_at = None + item.cancel_requested_at = None + item.updated_at = now + batch.status = "queued" + batch.completed_at = None + batch.updated_at = now + await session.commit() + return self._item_dict(item) diff --git a/backend/packages/harness/deerflow/subagents/AGENTS.md b/backend/packages/harness/deerflow/subagents/AGENTS.md index b5457eef1..b1742413a 100644 --- a/backend/packages/harness/deerflow/subagents/AGENTS.md +++ b/backend/packages/harness/deerflow/subagents/AGENTS.md @@ -5,9 +5,9 @@ **Benefit-based routing policy**: Enabling subagents exposes delegation as an optimization, not a default response to complexity. The lead prompt defaults to direct execution and permits `task` only when parallel latency, specialist capability, or context-isolation benefit clearly exceeds startup, duplicate-discovery, synthesis, state-conflict, and side-effect costs. Inter-agent output dependencies and overlapping mutable state are hard vetoes for parallel dispatch, while duplicate discovery and a cheap direct path remain costs rather than categorical vetoes; a bounded sequential chain may run in one subagent when specialist or context-isolation benefit clearly wins. Parallel scopes must be independent and non-overlapping, the lead uses the fewest useful subagents, and every later batch is re-evaluated while retaining any within-batch parallel benefit. When the enforced per-response limit is 1, the rendered prompt removes parallel and multi-batch benefit guidance and permits delegation only for material specialist or context-isolation benefit. Keep this policy aligned across `lead_agent/prompt.py`, the `task` tool description, and both built-in role descriptions; routing regressions are pinned in `tests/test_subagent_routing_prompt.py`, `tests/test_subagent_prompt_security.py`, and `tests/test_lead_agent_prompt.py`. **User-scoped Skills**: Subagents resolve their configured skills through `get_or_new_user_skill_storage(user_id)` using the parent runtime identity, with `DEFAULT_USER_ID` only when no identity is available. This keeps custom-skill shadowing and visibility aligned with the lead agent instead of reading the global-only catalog. **Date context (#4781)**: Every built-in subagent execution registers `SubagentDateContextMiddleware` immediately before `SystemMessageCoalescingMiddleware`. Its one-time `before_agent` hook adds a hidden framework-owned `SystemMessage` containing only `` before the first model call; it does not read `AppConfig.memory`, call the memory manager, rewrite the task `HumanMessage`, or inherit the lead agent's frozen-conversation/midnight lifecycle. The coalescer merges that reminder with the subagent's static prompt so strict providers still receive exactly one leading `SystemMessage`. The lead-only `DynamicContextMiddleware` registration and its date, optional-memory, and midnight-update behavior remain unchanged. -**Execution**: Dual thread pool - `_scheduler_pool` (3 workers) + `_execution_pool` (3 workers) -**Concurrency and total delegation cap**: `MAX_CONCURRENT_SUBAGENTS = 3` is enforced by `SubagentLimitMiddleware` (truncates excess tool calls in `after_model`; runtime `max_concurrent_subagents` is clamped to 1-4). The same middleware also enforces `subagents.max_total_per_run` (default 6, config schema 1-50, runtime override `max_total_subagents` clamped to the same range) against current-run entries in the durable delegation ledger, so a long lead-agent run cannot bypass concurrency limits by launching repeated legal-sized batches at each planning checkpoint, but historical delegations from previous runs in the same thread do not consume the new run's budget. The lead-agent prompt uses the same clamped values, so model-visible limits match enforcement. Gateway `run_agent()` and embedded `DeerFlowClient.stream()` both provide a per-invocation `run_id` in runtime context; `DeerFlowClient.stream()` also tags its input `HumanMessage` with that same id so durable-context capture can identify the current request boundary. Gateway resume paths may not append a new `HumanMessage`, so the worker also exposes the pre-run checkpoint's message ids in runtime context; durable-context capture uses that as the current-run boundary and never re-tags older task calls as the resumed run. When no delegation slots remain, task calls are stripped, provider raw tool-call metadata is synced, `finish_reason` is forced to `stop`, and a visible "subagent delegation limit" note is appended so the agent can synthesize already-collected results. Default subagent timeout `subagents.timeout_seconds=1800` (30 min) and built-in `general-purpose` `max_turns=150` (raised from 100/15-min so deep-research subtasks stop hitting `GraphRecursionError` out of the box) -**Flow**: `task()` tool → `SubagentExecutor` → background thread → poll 5s → SSE events → result. `task_started` carries the resolved effective model name. The per-subagent `SubagentTokenCollector` publishes a cumulative usage snapshot to the shared `SubagentResult` after every completed LLM response; the next `task_running` event carries that snapshot, so collapsed workspace cards can update without re-accounting parent-run totals. Terminal ToolMessage metadata (`subagent_model_name`, `subagent_token_usage`) and the persisted `subagent.end` event retain the model/usage after reload; absent provider usage stays absent rather than being estimated as zero. +**Execution**: Ordinary and durable-batch native subagents submit coroutines directly to one persistent isolated event loop. Gateway/embedded startup installs one process-wide async FIFO admission controller (default 3 running, bounded queue). Direct `create_deerflow_agent` callers can instead pass a caller-owned `SubagentRuntime`; reuse the same instance across graphs so its bound `task`, optional batch tools/service, middleware limits, and `SubagentExecutor` all share one controller without reading global YAML. An owned batch service must be started before graph construction and stopped at application shutdown. Waiters hold no scheduler thread, and cancellation/timeout release queue/slot ownership. +**Concurrency and total delegation cap**: Ordinary `task` concurrency is resolved once as the minimum of the per-run request, the startup-frozen `subagent_runtime.max_running`, and the schema safety ceiling (1-64), then shared by the lead prompt and `SubagentLimitMiddleware`. Hot reloads must not make either layer advertise more capacity than the already-created process controller; a changed startup-only value takes effect only after restart. The same middleware separately enforces `subagents.max_total_per_run` (default 6, config schema 1-50, runtime override `max_total_subagents` clamped to the same range) against current-run entries in the durable delegation ledger, so a long lead-agent run cannot bypass concurrency limits by launching repeated legal-sized batches at each planning checkpoint, but historical delegations from previous runs in the same thread do not consume the new run's budget. Explicit `batch_task` work does not consume or relax that ordinary-run ledger: its persisted total/live/running limits live under `subagent_batches`. Gateway `run_agent()` and embedded `DeerFlowClient.stream()` both provide a per-invocation `run_id` in runtime context; `DeerFlowClient.stream()` also tags its input `HumanMessage` with that same id so durable-context capture can identify the current request boundary. Gateway resume paths may not append a new `HumanMessage`, so the worker also exposes the pre-run checkpoint's message ids in runtime context; durable-context capture uses that as the current-run boundary and never re-tags older task calls as the resumed run. When no delegation slots remain, task calls are stripped, provider raw tool-call metadata is synced, `finish_reason` is forced to `stop`, and a visible "subagent delegation limit" note is appended so the agent can synthesize already-collected results. Default subagent timeout `subagents.timeout_seconds=1800` (30 min) and built-in `general-purpose` `max_turns=150`. +**Flow**: Ordinary `task()` → `SubagentExecutor` → shared process slot → result polling/SSE. Explicit `batch_task()` → durable batch/item rows → lease-based batch service (`subagents/batch_service.py`, started by Gateway or an explicit direct runtime) → the same `SubagentExecutor`/process slots → bounded stored result and owner-scoped API/JSONL export. Batch mode is selected only by the explicit tool, never inferred from prompt size. Executor queue rejection/timeout occurs before model execution and therefore releases the durable lease without consuming an item attempt; real execution failure and expired leases still consume the retry budget. User cancellation terminalizes every nonterminal item immediately and clears its lease, fencing any stale worker completion. Direct runtimes provide the tools and worker but not Gateway's HTTP/UI surface. `task_started` carries the resolved effective model name. The per-subagent `SubagentTokenCollector` publishes a cumulative usage snapshot to the shared `SubagentResult` after every completed LLM response; the next `task_running` event carries that snapshot, so collapsed workspace cards can update without re-accounting parent-run totals. Terminal ToolMessage metadata (`subagent_model_name`, `subagent_token_usage`) and the persisted `subagent.end` event retain the model/usage after reload; absent provider usage stays absent rather than being estimated as zero. **Events**: `task_started`, `task_running`, `task_completed`/`task_failed`/`task_timed_out` **Handled LLM failures**: `LLMErrorHandlingMiddleware` deliberately converts provider/model exceptions into an `AIMessage` so the graph can end cleanly, stamping `additional_kwargs.deerflow_error_fallback=true` plus error metadata. Clean graph termination does not imply subagent success: `SubagentExecutor` inspects the last assistant message at terminalization and maps a marked fallback to `SubagentStatus.FAILED`, which then emits `task_failed` and the existing structured `subagent_error`. Only the marker is authoritative — error-looking assistant prose without it remains a normal completed result, so neither the executor nor frontend parses display text as a status protocol. **Guardrail caps & `stop_reason` (#3875 Phase 2)**: three independent axes can end a subagent run early, and all now surface *why* through one additive field rather than a new status enum. **Turn axis**: `recursion_limit` on the subagent `run_config` equals `max_turns`, so exhausting the turn budget raises `GraphRecursionError` from `agent.astream`; `executor.py::_aexecute` catches it specifically (before the generic `except Exception`). **Token axis**: `TokenBudgetMiddleware` is attached per-agent via `build_subagent_runtime_middlewares` from `subagents.token_budget` (default `max_tokens` **coupled to `summarization.enabled`** — 1,000,000 when subagent summarization is on, 2,000,000 when off, warn at 0.7, hard-stop at 1.0; a user-set budget always wins regardless of the switch — #3875 Phase 3; a backstop against a subagent that burns tokens on trivial work). It does *not* raise: at the hard-stop threshold it strips the in-flight turn's tool calls, forces `finish_reason="stop"`, and lets the run complete naturally with a final answer. **Loop axis**: `LoopDetectionMiddleware` (attached at the same point) catches repeated identical tool-call sets — or one tool *type* called many times with varying args — and its hard-stop likewise strips `tool_calls` and forces a final answer without raising, recording `loop_capped`. Each guard exposes its cap on a per-`run_id` `consume_stop_reason(run_id)` accessor; `_aexecute` collects **every** middleware with that method (duck-typed via `hasattr`, so the executor has no import coupling to the guard classes) and surfaces the first non-`None` reason — adding a future guard needs no executor change. **Surfacing**: whichever axis fired, `_aexecute` stamps a normal status plus an additive reason — `completed` + `stop_reason=token_capped|turn_capped|loop_capped` when a usable final answer (or partial recovered from the last streamed chunk via `_extract_final_result` → `utils/messages.py::message_content_to_text`, returning a `"No response Generated"` sentinel when no text survived) was produced; `failed` + `stop_reason=turn_capped` when nothing usable survived. `SubagentResult.stop_reason` flows through `task_tool.py::_task_result_command` → `format_subagent_result_message` (renders `Task Succeeded (capped: ...)` / `Task failed (capped: ...)`) and `make_subagent_additional_kwargs`, which stamps the additive `subagent_stop_reason` key alongside the normal `subagent_status`. **Why additive, not an enum**: a new status value would break v1 consumers; an optional field is ignored by older frontends and ledger readers, so the cross-language contract (`contracts/subagent_status_contract.json` v2 + `subagents/status_contract.py` + `frontend/.../subtask-result.ts`, pinned by `test_status_values_match_contract` / `test_stop_reason_values_match_contract`) stays backward-compatible. The durable delegation ledger captures `stop_reason` onto the entry and renders model-facing guidance ("hit a guardrail cap with a partial result; reuse it, retry tighter, or raise the per-agent budget (`max_turns` / `token_budget`)") so the lead reuses a capped completion knowingly instead of mistaking it for a clean one. (Phase 1 shipped this surfacing as a `MAX_TURNS_REACHED` status enum in #3949; Phase 2 replaced that enum with the additive `stop_reason` field per the agreed design — the `max_turns_reached` status value and `SubagentStatus.MAX_TURNS_REACHED` are gone.) diff --git a/backend/packages/harness/deerflow/subagents/__init__.py b/backend/packages/harness/deerflow/subagents/__init__.py index 29f30a004..d0f4a0e18 100644 --- a/backend/packages/harness/deerflow/subagents/__init__.py +++ b/backend/packages/harness/deerflow/subagents/__init__.py @@ -5,6 +5,7 @@ __all__ = [ "SubagentConfig", "SubagentExecutor", "SubagentResult", + "SubagentRuntime", "get_available_subagent_names", "get_subagent_config", "list_subagents", @@ -21,4 +22,9 @@ def __getattr__(name: str): } globals().update(exports) return exports[name] + if name == "SubagentRuntime": + from .runtime import SubagentRuntime + + globals()[name] = SubagentRuntime + return SubagentRuntime raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/backend/packages/harness/deerflow/subagents/batch_runtime.py b/backend/packages/harness/deerflow/subagents/batch_runtime.py new file mode 100644 index 000000000..7ce53522d --- /dev/null +++ b/backend/packages/harness/deerflow/subagents/batch_runtime.py @@ -0,0 +1,49 @@ +"""Process-local bridge from harness tools to the Gateway batch service.""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass +from typing import Any, Protocol + + +@dataclass(frozen=True) +class BatchSubmitRequest: + user_id: str + thread_id: str + run_id: str | None + tool_call_id: str + submission_key: str + title: str + subagent_type: str + items: list[dict[str, str]] + max_live_items: int | None + max_running_items: int | None + execution_spec: dict[str, Any] + + +class SubagentBatchSubmitter(Protocol): + async def submit(self, request: BatchSubmitRequest) -> dict[str, Any]: ... + + async def get_batch(self, *, batch_id: str, user_id: str) -> dict[str, Any] | None: ... + + async def cancel_batch(self, *, batch_id: str, user_id: str) -> dict[str, Any] | None: ... + + +_submitter: SubagentBatchSubmitter | None = None +_lock = threading.Lock() + + +def set_subagent_batch_submitter(submitter: SubagentBatchSubmitter | None) -> None: + global _submitter + with _lock: + _submitter = submitter + + +def get_subagent_batch_submitter() -> SubagentBatchSubmitter | None: + with _lock: + return _submitter + + +def is_subagent_batch_runtime_available() -> bool: + return get_subagent_batch_submitter() is not None diff --git a/backend/packages/harness/deerflow/subagents/batch_service.py b/backend/packages/harness/deerflow/subagents/batch_service.py new file mode 100644 index 000000000..ed1810b04 --- /dev/null +++ b/backend/packages/harness/deerflow/subagents/batch_service.py @@ -0,0 +1,320 @@ +from __future__ import annotations + +import asyncio +import logging +import socket +import uuid +from datetime import UTC, datetime +from typing import Any + +from deerflow.config.app_config import AppConfig, get_app_config +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.subagents.batch_runtime import BatchSubmitRequest +from deerflow.subagents.capacity import SubagentExecutionCapacity +from deerflow.subagents.config import SubagentConfig, resolve_subagent_model_name +from deerflow.subagents.executor import ( + SubagentExecutor, + SubagentStatus, + cleanup_background_task, + get_background_task_result, + request_cancel_background_task, +) + +logger = logging.getLogger(__name__) + + +def _usage(records: list[dict[str, Any]] | None) -> dict[str, int] | None: + if not records: + return None + return { + "input_tokens": sum(int(row.get("input_tokens") or 0) for row in records), + "output_tokens": sum(int(row.get("output_tokens") or 0) for row in records), + "total_tokens": sum(int(row.get("total_tokens") or 0) for row in records), + } + + +class SubagentBatchService: + """Lease, execute, and recover durable native-subagent batch items.""" + + def __init__( + self, + *, + repository, + config: SubagentBatchesConfig, + runtime_config: SubagentRuntimeConfig, + app_config: AppConfig | None = None, + execution_capacity: SubagentExecutionCapacity | None = None, + ) -> None: + self._repository = repository + self._config = config + self._runtime_config = runtime_config + self._app_config = app_config + self._execution_capacity = execution_capacity + self._lease_owner = f"{socket.gethostname()}:{uuid.uuid4().hex}" + self._stop = asyncio.Event() + self._poller: asyncio.Task[None] | None = None + self._executions: dict[str, asyncio.Task[None]] = {} + self._execution_ids: dict[str, str] = {} + self._item_batches: dict[str, str] = {} + + async def start(self) -> None: + if self._poller is not None: + return + self._stop.clear() + self._poller = asyncio.create_task(self._run(), name="subagent-batch-poller") + + async def stop(self) -> None: + self._stop.set() + poller = self._poller + self._poller = None + if poller is not None: + poller.cancel() + await asyncio.gather(poller, return_exceptions=True) + execution_ids = list(self._execution_ids.values()) + for execution_id in execution_ids: + request_cancel_background_task(execution_id) + tasks = list(self._executions.values()) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + self._executions.clear() + self._execution_ids.clear() + self._item_batches.clear() + + async def _run(self) -> None: + while not self._stop.is_set(): + try: + await self.run_once(now=datetime.now(UTC)) + except asyncio.CancelledError: + raise + except Exception: + logger.exception("Subagent batch scheduler pass failed") + try: + await asyncio.wait_for( + self._stop.wait(), + timeout=self._config.poll_interval_seconds, + ) + except TimeoutError: + pass + + async def run_once(self, *, now: datetime) -> None: + available = max(0, self._runtime_config.max_running - len(self._executions)) + if available <= 0: + return + items = await self._repository.claim_items( + now=now, + lease_owner=self._lease_owner, + lease_seconds=self._config.lease_seconds, + limit=available, + ) + for item in items: + item_id = item["id"] + if item_id in self._executions: + continue + task = asyncio.create_task( + self._execute_item(item), + name=f"subagent-batch-item-{item_id}", + ) + self._executions[item_id] = task + task.add_done_callback( + lambda _task, current_id=item_id: self._executions.pop( + current_id, + None, + ) + ) + + async def submit(self, request: BatchSubmitRequest) -> dict[str, Any]: + total = len(request.items) + if total < 1 or total > self._config.max_items_per_batch: + raise ValueError(f"Batch item count must be between 1 and {self._config.max_items_per_batch}") + max_live = request.max_live_items or self._config.default_max_live_items + max_running = request.max_running_items or self._config.default_max_running_items + if not 1 <= max_live <= self._config.max_live_items_per_batch: + raise ValueError(f"max_live_items must be between 1 and {self._config.max_live_items_per_batch}") + if not 1 <= max_running <= self._config.max_running_items_per_batch: + raise ValueError(f"max_running_items must be between 1 and {self._config.max_running_items_per_batch}") + if max_running > max_live: + raise ValueError("max_running_items must not exceed max_live_items") + return await self._repository.create_batch( + batch_id=f"subagent-batch-{uuid.uuid4().hex}", + user_id=request.user_id, + thread_id=request.thread_id, + run_id=request.run_id, + tool_call_id=request.tool_call_id, + submission_key=request.submission_key, + title=request.title, + subagent_type=request.subagent_type, + items=request.items, + max_live_items=max_live, + max_running_items=max_running, + max_attempts=self._config.max_attempts, + execution_spec=request.execution_spec, + ) + + async def get_batch( + self, + *, + batch_id: str, + user_id: str, + ) -> dict[str, Any] | None: + return await self._repository.get_batch(batch_id, user_id=user_id) + + async def cancel_batch( + self, + *, + batch_id: str, + user_id: str, + ) -> dict[str, Any] | None: + batch = await self._repository.cancel_batch(batch_id, user_id=user_id) + if batch is None: + return None + for item_id, execution_id in list(self._execution_ids.items()): + if self._item_batches.get(item_id) == batch_id: + request_cancel_background_task(execution_id) + # Normal ids are not prefixed; the renew loop observes the durable + # cancellation within lease_seconds/3. Keeping cancellation durable is + # what lets another worker own the HTTP control request safely. + return batch + + async def _execute_item(self, item: dict[str, Any]) -> None: + item_id = item["id"] + execution_id: str | None = None + try: + batch = item["batch"] + self._item_batches[item_id] = batch["id"] + spec = batch["execution_spec"] + config = SubagentConfig(**spec["subagent_config"]) + app_config = self._app_config or get_app_config() + from deerflow.tools import get_available_tools + + effective_model = resolve_subagent_model_name( + config, + spec.get("parent_model"), + app_config=app_config, + ) + tools = get_available_tools( + groups=spec.get("tool_groups"), + model_name=effective_model, + subagent_enabled=False, + include_upload_tool=False, + app_config=app_config, + ) + executor = SubagentExecutor( + config=config, + tools=tools, + app_config=app_config, + parent_model=spec.get("parent_model"), + thread_id=batch["thread_id"], + user_id=batch["user_id"], + user_role=spec.get("user_role"), + oauth_provider=spec.get("oauth_provider"), + oauth_id=spec.get("oauth_id"), + run_id=batch.get("run_id"), + channel_user_id=spec.get("channel_user_id"), + is_internal=spec.get("is_internal") is True, + authz_attributes=spec.get("authz_attributes"), + execution_capacity=self._execution_capacity, + ) + prompt = f"Durable batch item key: {item['item_key']}\nThis item may be retried after a worker crash. Keep side effects idempotent and use the item key as the idempotency identity.\n\n{item['prompt']}" + execution_id = executor.execute_async(prompt, task_id=item_id) + self._execution_ids[item_id] = execution_id + marked_running = False + renew_every = max(1.0, self._config.lease_seconds / 3) + status_poll_every = min( + self._config.poll_interval_seconds, + renew_every, + ) + loop = asyncio.get_running_loop() + next_renew_at = loop.time() + renew_every + while True: + result = get_background_task_result(execution_id) + if result is None: + raise RuntimeError("Native subagent execution disappeared") + if result.status is SubagentStatus.RUNNING and not marked_running: + marked_running = await self._repository.mark_item_running( + item_id, + lease_owner=self._lease_owner, + now=datetime.now(UTC), + ) + if not marked_running: + request_cancel_background_task(execution_id) + if result.status.is_terminal: + break + now_monotonic = loop.time() + if now_monotonic >= next_renew_at: + lease = await self._repository.renew_item_lease( + item_id, + lease_owner=self._lease_owner, + lease_seconds=self._config.lease_seconds, + now=datetime.now(UTC), + ) + next_renew_at = loop.time() + renew_every + if not lease["valid"]: + request_cancel_background_task(execution_id) + try: + until_renew = max(0.0, next_renew_at - loop.time()) + await asyncio.wait_for( + self._stop.wait(), + timeout=min(status_poll_every, until_renew), + ) + if self._stop.is_set(): + raise asyncio.CancelledError + except TimeoutError: + pass + + raw_result = result.result or "" + if getattr(result, "admission_failure", False): + await self._repository.requeue_item_after_admission_failure( + item_id, + lease_owner=self._lease_owner, + error=result.error, + now=datetime.now(UTC), + ) + return + truncated = len(raw_result) > self._config.max_result_chars + stored_result = raw_result[: self._config.max_result_chars] if raw_result else None + preview = raw_result[: self._config.result_preview_max_chars] if raw_result else None + await self._repository.finalize_item( + item_id, + lease_owner=self._lease_owner, + succeeded=result.status is SubagentStatus.COMPLETED, + result=stored_result, + result_preview=preview, + result_truncated=truncated, + error=result.error, + stop_reason=result.stop_reason, + token_usage=_usage(result.token_usage_records), + model_name=effective_model, + completed_at=datetime.now(UTC), + ) + except asyncio.CancelledError: + if execution_id is not None: + request_cancel_background_task(execution_id) + # Do not finalize on process shutdown. The durable lease expires and + # another worker reclaims the same stable item key. + raise + except Exception as exc: + logger.exception( + "Durable subagent batch item failed (item_id=%s)", + item_id, + ) + await self._repository.finalize_item( + item_id, + lease_owner=self._lease_owner, + succeeded=False, + result=None, + result_preview=None, + result_truncated=False, + error=str(exc)[:4_000], + stop_reason=None, + token_usage=None, + model_name=None, + completed_at=datetime.now(UTC), + ) + finally: + self._execution_ids.pop(item_id, None) + self._item_batches.pop(item_id, None) + if execution_id is not None: + cleanup_background_task(execution_id) diff --git a/backend/packages/harness/deerflow/subagents/capacity.py b/backend/packages/harness/deerflow/subagents/capacity.py new file mode 100644 index 000000000..5a5091b1d --- /dev/null +++ b/backend/packages/harness/deerflow/subagents/capacity.py @@ -0,0 +1,159 @@ +"""Shared process-local admission control for native subagent execution.""" + +from __future__ import annotations + +import asyncio +import threading +from collections import deque +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass + +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig + + +class SubagentCapacityError(RuntimeError): + """Base class for explicit admission failures.""" + + +class SubagentCapacityRejected(SubagentCapacityError): + """The process queue is full or configured to reject when saturated.""" + + +class SubagentCapacityTimeout(SubagentCapacityError): + """A queued execution did not receive a slot before its deadline.""" + + +@dataclass(frozen=True) +class SubagentCapacitySnapshot: + max_running: int + running: int + max_queued: int + queued: int + admission_policy: str + + +class SubagentExecutionCapacity: + """FIFO async capacity controller; queued work never owns a thread.""" + + def __init__(self, config: SubagentRuntimeConfig) -> None: + self._config = config + self._lock = asyncio.Lock() + self._running = 0 + self._waiters: deque[asyncio.Future[None]] = deque() + + def snapshot(self) -> SubagentCapacitySnapshot: + return SubagentCapacitySnapshot( + max_running=self._config.max_running, + running=self._running, + max_queued=self._config.max_queued, + queued=sum(not waiter.done() for waiter in self._waiters), + admission_policy=self._config.admission_policy, + ) + + async def _acquire(self) -> None: + waiter: asyncio.Future[None] | None = None + async with self._lock: + if self._running < self._config.max_running: + self._running += 1 + return + queued = sum(not candidate.done() for candidate in self._waiters) + if self._config.admission_policy == "reject" or queued >= self._config.max_queued: + raise SubagentCapacityRejected(f"Subagent execution capacity is full ({self._config.max_running} running, {queued} queued)") + waiter = asyncio.get_running_loop().create_future() + self._waiters.append(waiter) + + try: + await asyncio.wait_for( + waiter, + timeout=self._config.queue_timeout_seconds, + ) + except (TimeoutError, asyncio.CancelledError) as exc: + async with self._lock: + try: + self._waiters.remove(waiter) + except ValueError: + # A release transferred the slot just as the timeout fired. + # If the future completed, this caller owns that transfer and + # must release it before reporting the timeout. + if waiter.done() and not waiter.cancelled(): + self._release_locked() + if isinstance(exc, asyncio.CancelledError): + raise + raise SubagentCapacityTimeout(f"Timed out after {self._config.queue_timeout_seconds}s waiting for a subagent execution slot") from exc + + def _release_locked(self) -> None: + while self._waiters: + waiter = self._waiters.popleft() + if waiter.done(): + continue + # Transfer the existing slot; _running intentionally stays flat. + waiter.set_result(None) + return + if self._running <= 0: + raise RuntimeError("Subagent execution capacity released without an owner") + self._running -= 1 + + async def _release(self) -> None: + async with self._lock: + self._release_locked() + + @asynccontextmanager + async def slot(self) -> AsyncIterator[None]: + await self._acquire() + try: + yield + finally: + await self._release() + + +_config = SubagentRuntimeConfig() +_controller: SubagentExecutionCapacity | None = None +_controller_loop: asyncio.AbstractEventLoop | None = None +_state_lock = threading.Lock() + + +def configure_subagent_execution_capacity(config: SubagentRuntimeConfig) -> None: + """Install the startup snapshot used by the lazily-created loop controller.""" + global _config, _controller, _controller_loop + candidate = config.model_copy(deep=True) + with _state_lock: + # Gateway startup and embedded clients may initialize the same process. + # Treat installing the same frozen startup configuration as a no-op so + # those entry points cannot reset a live queue. + if _config == candidate: + return + if _controller is not None: + snapshot = _controller.snapshot() + if snapshot.running or snapshot.queued: + raise RuntimeError("Cannot reconfigure subagent capacity while executions are active") + _config = candidate + _controller = None + _controller_loop = None + + +def get_subagent_execution_capacity() -> SubagentExecutionCapacity: + """Return the controller bound to the current execution loop.""" + global _controller, _controller_loop + loop = asyncio.get_running_loop() + with _state_lock: + if _controller is None: + _controller = SubagentExecutionCapacity(_config) + _controller_loop = loop + elif _controller_loop is not loop: + snapshot = _controller.snapshot() + if snapshot.running or snapshot.queued: + raise RuntimeError("Native subagent capacity cannot move event loops while executions are active") + # Direct async consumers (notably embedded callers and tests) may + # legitimately use a new event loop after the previous idle loop + # has closed. Rebind only while completely idle; production sync + # and background paths still share the persistent isolated loop. + _controller = SubagentExecutionCapacity(_config) + _controller_loop = loop + return _controller + + +def configured_subagent_max_running() -> int: + """Return the startup snapshot without requiring an event loop.""" + with _state_lock: + return _config.max_running diff --git a/backend/packages/harness/deerflow/subagents/executor.py b/backend/packages/harness/deerflow/subagents/executor.py index 8eaae8622..e3e7832f2 100644 --- a/backend/packages/harness/deerflow/subagents/executor.py +++ b/backend/packages/harness/deerflow/subagents/executor.py @@ -7,7 +7,7 @@ import os import threading import uuid from collections.abc import Callable, Coroutine, Mapping -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import Future from concurrent.futures import TimeoutError as FuturesTimeoutError from contextvars import Context, copy_context from dataclasses import dataclass, field @@ -30,6 +30,11 @@ from deerflow.config.app_config import AppConfig from deerflow.models import create_chat_model from deerflow.runtime.user_context import DEFAULT_USER_ID from deerflow.skills.types import Skill +from deerflow.subagents.capacity import ( + SubagentCapacityError, + SubagentExecutionCapacity, + get_subagent_execution_capacity, +) from deerflow.subagents.config import SubagentConfig, resolve_subagent_model_name from deerflow.subagents.step_events import capture_new_step_messages from deerflow.subagents.token_collector import SubagentTokenCollector @@ -96,6 +101,7 @@ class SubagentResult: started_at: When execution started. completed_at: When execution completed. ai_messages: List of complete AI messages (as dicts) generated during execution. + admission_failure: Whether capacity rejected/timed out before execution started. """ task_id: str @@ -110,6 +116,7 @@ class SubagentResult: ai_messages: list[dict[str, Any]] | None = None token_usage_records: list[dict[str, int | str | None]] = field(default_factory=list) usage_reported: bool = False + admission_failure: bool = False cancel_event: threading.Event = field(default_factory=threading.Event, repr=False) _state_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False) @@ -134,6 +141,7 @@ class SubagentResult: completed_at: datetime | None = None, ai_messages: list[dict[str, Any]] | None = None, token_usage_records: list[dict[str, int | str | None]] | None = None, + admission_failure: bool = False, ) -> bool: """Set a terminal status exactly once. @@ -158,6 +166,7 @@ class SubagentResult: self.ai_messages = ai_messages if token_usage_records is not None: self.token_usage_records = token_usage_records + self.admission_failure = admission_failure self.completed_at = completed_at or datetime.now() self.status = status return True @@ -261,8 +270,7 @@ def _extract_llm_error_fallback(final_state: Any) -> str | None: _background_tasks: dict[str, SubagentResult] = {} _background_tasks_lock = threading.Lock() -# Thread pool for background task scheduling and orchestration -_scheduler_pool = ThreadPoolExecutor(max_workers=3, thread_name_prefix="subagent-scheduler-") +_background_futures: dict[str, Future[SubagentResult]] = {} # Persistent event loop for isolated subagent executions triggered from an # already-running parent loop. Reusing one long-lived loop avoids creating a @@ -458,6 +466,7 @@ class SubagentExecutor: authz_attributes: Mapping[str, Any] | None = None, deerflow_trace_id: str | None = None, extensions: Any | None = None, + execution_capacity: SubagentExecutionCapacity | None = None, ): """Initialize the executor. @@ -486,6 +495,10 @@ class SubagentExecutor: captured at ``task_tool`` dispatch. When None (embedded client, standalone LangGraph Server), ``_aexecute`` falls back to the process-wide singleton. + execution_capacity: Optional explicitly shared admission controller. + Direct ``create_deerflow_agent`` callers pass one through their + ``SubagentRuntime``; application factories fall back to the + startup-configured process singleton. """ self.config = config self.app_config = app_config @@ -524,6 +537,7 @@ class SubagentExecutor: # the lead run's start and this subagent's execution must not swap the # generation underneath the delegated work. self.extensions = extensions + self.execution_capacity = execution_capacity self._base_tools = _filter_tools( tools, @@ -883,6 +897,31 @@ class SubagentExecutor: return state, final_tools, deferred_setup async def _aexecute(self, task: str, result_holder: SubagentResult | None = None) -> SubagentResult: + """Execute after acquiring the process-wide native-subagent slot.""" + result = result_holder + if result is None: + result = SubagentResult( + task_id=str(uuid.uuid4())[:8], + trace_id=self.trace_id, + status=SubagentStatus.PENDING, + ) + try: + capacity = self.execution_capacity or get_subagent_execution_capacity() + async with capacity.slot(): + with result._state_lock: + if not result.status.is_terminal: + result.status = SubagentStatus.RUNNING + result.started_at = datetime.now() + return await self._aexecute_admitted(task, result) + except SubagentCapacityError as exc: + result.try_set_terminal( + SubagentStatus.FAILED, + error=str(exc), + admission_failure=True, + ) + return result + + async def _aexecute_admitted(self, task: str, result_holder: SubagentResult | None = None) -> SubagentResult: """Execute a task asynchronously. Args: @@ -958,6 +997,9 @@ class SubagentExecutor: task_info, timeout=_EXTENSION_TASK_NOTIFY_TIMEOUT_SECONDS, ) + if result.cancel_event.is_set(): + result.try_set_terminal(SubagentStatus.CANCELLED, error="Cancelled by user") + return result state, final_tools, deferred_setup = await self._build_initial_state(task) agent = self._create_agent( @@ -1240,13 +1282,9 @@ class SubagentExecutor: def execute(self, task: str, result_holder: SubagentResult | None = None) -> SubagentResult: """Execute a task synchronously (wrapper around async execution). - This method runs the async execution in a new event loop, allowing - asynchronous tools (like MCP tools) to be used within the thread pool. - - When called from within an already-running event loop (e.g., when the - parent agent is async), this method synchronously waits on the - persistent isolated loop to avoid event loop conflicts with shared - async primitives like httpx clients. + All sync executions use the persistent isolated event loop. This keeps + shared async clients and the process-wide admission controller bound to + one long-lived loop instead of creating a short-lived loop per call. Args: task: The task description for the subagent. @@ -1256,17 +1294,7 @@ class SubagentExecutor: SubagentResult with the execution result. """ try: - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = None - - if loop is not None and loop.is_running(): - logger.debug(f"[trace={self.trace_id}] Subagent {self.config.name} detected running event loop, using isolated loop") - return self._execute_in_isolated_loop(task, result_holder) - - # Standard path: no running event loop, use asyncio.run - return asyncio.run(self._aexecute(task, result_holder)) + return self._execute_in_isolated_loop(task, result_holder) except Exception as e: logger.exception(f"[trace={self.trace_id}] Subagent {self.config.name} execution failed") # Create a result with error if we don't have one @@ -1317,36 +1345,37 @@ class SubagentExecutor: parent_context = _copy_isolated_subagent_context() - # Submit to scheduler pool - def run_task(): - with _background_tasks_lock: - result.status = SubagentStatus.RUNNING - result.started_at = datetime.now() - + async def run_with_timeout() -> SubagentResult: try: - # Submit execution directly to the persistent isolated loop so the - # background path does not create a temporary loop via execute(). - execution_future = _submit_to_isolated_loop_in_context( - parent_context, - lambda: self._aexecute(task, result), + return await asyncio.wait_for( + self._aexecute(task, result), + timeout=self.config.timeout_seconds, ) - try: - # Wait for execution with timeout - execution_future.result(timeout=self.config.timeout_seconds) - except FuturesTimeoutError: - logger.error(f"[trace={self.trace_id}] Subagent {self.config.name} execution timed out after {self.config.timeout_seconds}s") - # Signal cooperative cancellation and cancel the future - result.cancel_event.set() - result.try_set_terminal( - SubagentStatus.TIMED_OUT, - error=f"Execution timed out after {self.config.timeout_seconds} seconds", - ) - execution_future.cancel() - except Exception as e: - logger.exception(f"[trace={self.trace_id}] Subagent {self.config.name} async execution failed") - result.try_set_terminal(SubagentStatus.FAILED, error=str(e)) + except TimeoutError: + result.cancel_event.set() + result.try_set_terminal( + SubagentStatus.TIMED_OUT, + error=f"Execution timed out after {self.config.timeout_seconds} seconds", + ) + return result + except asyncio.CancelledError: + result.cancel_event.set() + result.try_set_terminal(SubagentStatus.CANCELLED, error="Cancelled by user") + return result + except Exception as exc: + logger.exception("[trace=%s] Subagent %s async execution failed", self.trace_id, self.config.name) + result.try_set_terminal(SubagentStatus.FAILED, error=str(exc)) + return result - _scheduler_pool.submit(run_task) + execution_future = _submit_to_isolated_loop_in_context(parent_context, run_with_timeout) + with _background_tasks_lock: + _background_futures[execution_id] = execution_future + + def forget_future(_future: Future[SubagentResult]) -> None: + with _background_tasks_lock: + _background_futures.pop(execution_id, None) + + execution_future.add_done_callback(forget_future) return execution_id @@ -1368,6 +1397,9 @@ def request_cancel_background_task(execution_id: str) -> None: result = _background_tasks.get(execution_id) if result is not None: result.cancel_event.set() + future = _background_futures.get(execution_id) + if future is not None: + future.cancel() logger.info("Requested cancellation for background execution %s", execution_id) @@ -1417,6 +1449,7 @@ def cleanup_background_task(execution_id: str) -> None: # the background executor still updating the task entry. if result.status.is_terminal or result.completed_at is not None: del _background_tasks[execution_id] + _background_futures.pop(execution_id, None) logger.debug("Cleaned up background execution: %s", execution_id) else: logger.debug( diff --git a/backend/packages/harness/deerflow/subagents/runtime.py b/backend/packages/harness/deerflow/subagents/runtime.py new file mode 100644 index 000000000..ddb9ced50 --- /dev/null +++ b/backend/packages/harness/deerflow/subagents/runtime.py @@ -0,0 +1,147 @@ +"""Explicit runtime dependencies for direct ``create_deerflow_agent`` use.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Any + +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.config.subagents_config import ( + DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN, + MAX_TOTAL_SUBAGENTS_PER_RUN, + MIN_TOTAL_SUBAGENTS_PER_RUN, +) +from deerflow.subagents.batch_runtime import SubagentBatchSubmitter +from deerflow.subagents.capacity import SubagentExecutionCapacity + +if TYPE_CHECKING: + from deerflow.config.app_config import AppConfig + + +class SubagentRuntime: + """Share native-subagent capacity and optional durable batches across graphs. + + Application entry points install equivalent process-global dependencies at + startup. Direct graph factories instead receive this object explicitly, so + multiple graphs can share one real execution ceiling. Supplying + ``app_config`` also keeps their subagent registry, model, and tool + resolution on the same caller-owned snapshot instead of global YAML. + + When ``batch_repository`` is supplied, the runtime owns a durable batch + worker. Start it before constructing the graph (or use ``async with``) so + ``create_deerflow_agent`` can expose the bound batch tools, and stop it + during application shutdown. + """ + + def __init__( + self, + config: SubagentRuntimeConfig | None = None, + *, + max_total_per_run: int = DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN, + batch_submitter: SubagentBatchSubmitter | None = None, + batch_repository: Any | None = None, + batch_config: SubagentBatchesConfig | None = None, + app_config: AppConfig | None = None, + ) -> None: + if not MIN_TOTAL_SUBAGENTS_PER_RUN <= max_total_per_run <= MAX_TOTAL_SUBAGENTS_PER_RUN: + raise ValueError(f"max_total_per_run must be between {MIN_TOTAL_SUBAGENTS_PER_RUN} and {MAX_TOTAL_SUBAGENTS_PER_RUN}") + if batch_submitter is not None and batch_repository is not None: + raise ValueError("Provide either batch_submitter or batch_repository, not both") + if batch_repository is not None and not bool(getattr(batch_config, "enabled", False)): + raise ValueError("batch_repository requires batch_config.enabled=true") + if batch_repository is not None and app_config is None: + raise ValueError("batch_repository requires an explicit app_config snapshot") + if batch_config is not None and batch_repository is None: + raise ValueError("batch_config requires batch_repository") + + self.config = (config or SubagentRuntimeConfig()).model_copy(deep=True) + self.max_total_per_run = max_total_per_run + self.app_config = app_config + self.execution_capacity = SubagentExecutionCapacity(self.config) + self.batch_config = batch_config.model_copy(deep=True) if batch_config is not None else None + self._external_batch_submitter = batch_submitter + self._owned_batch_service = None + self._batch_started = False + self._lifecycle_lock = asyncio.Lock() + + if batch_repository is not None: + from deerflow.subagents.batch_service import SubagentBatchService + + self._owned_batch_service = SubagentBatchService( + repository=batch_repository, + config=self.batch_config, + runtime_config=self.config, + app_config=app_config, + execution_capacity=self.execution_capacity, + ) + + @classmethod + def from_app_config( + cls, + app_config: AppConfig, + *, + batch_repository: Any | None = None, + ) -> SubagentRuntime: + """Build explicit SDK dependencies from a caller-owned config snapshot.""" + + runtime_config = getattr(app_config, "subagent_runtime", None) + if not isinstance(runtime_config, SubagentRuntimeConfig): + runtime_config = SubagentRuntimeConfig() + max_total_per_run = int( + getattr( + getattr(app_config, "subagents", None), + "max_total_per_run", + DEFAULT_MAX_TOTAL_SUBAGENTS_PER_RUN, + ) + ) + batch_config = None + if batch_repository is not None: + configured_batches = getattr(app_config, "subagent_batches", None) + if not isinstance(configured_batches, SubagentBatchesConfig): + configured_batches = SubagentBatchesConfig() + batch_config = configured_batches + return cls( + runtime_config, + max_total_per_run=max_total_per_run, + batch_repository=batch_repository, + batch_config=batch_config, + app_config=app_config, + ) + + @property + def batch_submitter(self) -> SubagentBatchSubmitter | None: + if self._external_batch_submitter is not None: + return self._external_batch_submitter + if self._batch_started: + return self._owned_batch_service + return None + + async def start(self) -> None: + """Start the owned durable batch worker, if configured.""" + + if self._owned_batch_service is None: + return + async with self._lifecycle_lock: + if self._batch_started: + return + await self._owned_batch_service.start() + self._batch_started = True + + async def stop(self) -> None: + """Stop the owned worker and hide its bound tools from new graphs.""" + + if self._owned_batch_service is None: + return + async with self._lifecycle_lock: + if not self._batch_started: + return + self._batch_started = False + await self._owned_batch_service.stop() + + async def __aenter__(self) -> SubagentRuntime: + await self.start() + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.stop() diff --git a/backend/packages/harness/deerflow/tools/AGENTS.md b/backend/packages/harness/deerflow/tools/AGENTS.md index d39cf58eb..428ba497e 100644 --- a/backend/packages/harness/deerflow/tools/AGENTS.md +++ b/backend/packages/harness/deerflow/tools/AGENTS.md @@ -11,6 +11,8 @@ - `update_agent` - Custom-agent-only: persist self-updates to the current agent's `SOUL.md` / `config.yaml` from inside a normal chat (partial update + atomic write). Bound when `agent_name` is set and `is_bootstrap=False`. 4. **Subagent tool** (if enabled): - `task` - Delegate to subagent (description, prompt, subagent_type) + - `batch_task`, `batch_status`, `cancel_batch` - Explicit durable batch submission/progress/cancellation. Added only while the startup SQL-backed batch submitter is installed; large results stay in the owner-scoped API/JSONL export rather than the lead context. + - Direct `create_deerflow_agent` integrations receive cloned tools bound to their explicit `SubagentRuntime`. The bound `task` forwards that runtime's exact execution controller and optional caller-owned `AppConfig` into registry/model/tool resolution and `SubagentExecutor`; bound batch tools use the same config snapshot and resolve only that runtime's submitter before falling back to no other application's active worker. Keep the original tool name/schema unchanged so model contracts and user-tool deduplication remain stable. Scheduled-task runtime note: - Scheduled background runs set `context.non_interactive=true` and therefore exclude `ask_clarification` from the lead-agent tool list. This keeps scheduler-triggered runs from stalling on human confirmation mid-execution. `non_interactive` is an internal-only context key: it is merged from `body.context` only when the request authenticated as the process-internal user (the scheduler path), never from arbitrary HTTP/IM clients. diff --git a/backend/packages/harness/deerflow/tools/builtins/__init__.py b/backend/packages/harness/deerflow/tools/builtins/__init__.py index 3cce6ca96..c004af741 100644 --- a/backend/packages/harness/deerflow/tools/builtins/__init__.py +++ b/backend/packages/harness/deerflow/tools/builtins/__init__.py @@ -1,4 +1,5 @@ from .background_tasks_tool import cancel_background_task, list_background_tasks +from .batch_task_tool import batch_status, batch_task, cancel_batch from .clarification_tool import ask_clarification_tool from .list_uploaded_files_tool import list_uploaded_files from .present_file_tool import present_file_tool @@ -16,6 +17,9 @@ __all__ = [ "ask_clarification_tool", "view_image_tool", "task_tool", + "batch_task", + "batch_status", + "cancel_batch", "list_uploaded_files", "list_background_tasks", "cancel_background_task", diff --git a/backend/packages/harness/deerflow/tools/builtins/batch_task_tool.py b/backend/packages/harness/deerflow/tools/builtins/batch_task_tool.py new file mode 100644 index 000000000..31e255a9d --- /dev/null +++ b/backend/packages/harness/deerflow/tools/builtins/batch_task_tool.py @@ -0,0 +1,267 @@ +"""Explicit durable batch mode for many independent native-subagent items.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from contextvars import ContextVar +from dataclasses import asdict, replace +from typing import Annotated, Any, cast + +from langchain.tools import InjectedToolCallId, tool +from langchain_core.messages import ToolMessage +from langgraph.types import Command +from pydantic import BaseModel, Field + +from deerflow.authz.principal import normalize_authz_attributes +from deerflow.runtime.user_context import resolve_runtime_user_id +from deerflow.subagents.batch_runtime import ( + BatchSubmitRequest, + SubagentBatchSubmitter, + get_subagent_batch_submitter, +) +from deerflow.subagents.registry import get_available_subagent_names, get_subagent_config +from deerflow.tools.types import Runtime + + +class BatchTaskItem(BaseModel): + key: str = Field(min_length=1, max_length=128) + prompt: str = Field(min_length=1, max_length=100_000) + + +_NO_EXPLICIT_BATCH_SUBMITTER = object() +_explicit_batch_submitter: ContextVar[SubagentBatchSubmitter | None | object] = ContextVar( + "deerflow_explicit_subagent_batch_submitter", + default=_NO_EXPLICIT_BATCH_SUBMITTER, +) +_explicit_batch_app_config: ContextVar[Any | None] = ContextVar( + "deerflow_explicit_subagent_batch_app_config", + default=None, +) + + +def _batch_submitter() -> SubagentBatchSubmitter | None: + explicit = _explicit_batch_submitter.get() + if explicit is not _NO_EXPLICIT_BATCH_SUBMITTER: + return cast(SubagentBatchSubmitter | None, explicit) + return get_subagent_batch_submitter() + + +def _batch_app_config(runtime: Runtime) -> Any | None: + explicit = _explicit_batch_app_config.get() + if explicit is not None: + return explicit + context = runtime.context if runtime is not None and isinstance(runtime.context, dict) else {} + return context.get("app_config") + + +def _bind_batch_tool( + tool, + submitter_provider: Callable[[], SubagentBatchSubmitter | None], + app_config: Any | None, +): + original_coroutine = tool.coroutine + if original_coroutine is None: # pragma: no cover - all batch tools are async + raise RuntimeError(f"{tool.name} has no async implementation") + + async def bound_coroutine(**kwargs): + submitter_token = _explicit_batch_submitter.set(submitter_provider()) + config_token = _explicit_batch_app_config.set(app_config) + try: + return await original_coroutine(**kwargs) + finally: + _explicit_batch_app_config.reset(config_token) + _explicit_batch_submitter.reset(submitter_token) + + return tool.model_copy(update={"coroutine": bound_coroutine}) + + +def bind_batch_tools( + submitter: SubagentBatchSubmitter | None = None, + *, + submitter_provider: Callable[[], SubagentBatchSubmitter | None] | None = None, + app_config: Any | None = None, +): + """Return batch tools bound to an explicit SDK runtime submitter. + + A provider preserves runtime lifecycle semantics for already-compiled + graphs: after their owned worker stops, the tools report unavailable and + never fall through to another application's process-global submitter. + """ + + if (submitter is None) == (submitter_provider is None): + raise ValueError("Provide exactly one of submitter or submitter_provider") + provider = submitter_provider if submitter_provider is not None else lambda: submitter + + return tuple(_bind_batch_tool(tool, provider, app_config) for tool in (batch_task, batch_status, cancel_batch)) + + +def _result(tool_call_id: str, *, content: str, batch: dict[str, Any] | None = None, error: bool = False) -> Command: + metadata: dict[str, Any] = {"subagent_batch_error": error} + if batch is not None: + metadata.update( + { + "subagent_batch_id": batch["id"], + "subagent_batch_status": batch["status"], + "subagent_batch_total_items": batch["total_items"], + } + ) + return Command( + update={ + "messages": [ + ToolMessage( + content=content, + tool_call_id=tool_call_id, + name="batch_task", + status="error" if error else "success", + additional_kwargs=metadata, + ) + ] + } + ) + + +def _merge_skill_allowlists(parent: list[str] | None, child: list[str] | None) -> list[str] | None: + if parent is None: + return child + if child is None: + return list(parent) + allowed = set(parent) + return [name for name in child if name in allowed] + + +@tool("batch_task", parse_docstring=True) +async def batch_task( + runtime: Runtime, + title: str, + items: list[BatchTaskItem], + subagent_type: str, + tool_call_id: Annotated[str, InjectedToolCallId], + max_live_items: int | None = None, + max_running_items: int | None = None, +) -> Command: + """Submit many independent items to DeerFlow's explicit durable batch mode. + + Use this only when every item is independent, idempotent or read-only, and + can be completed without another item's output. This tool returns a batch + identifier immediately; it never inserts thousands of results into the lead + agent context. Use ``batch_status`` for a compact progress snapshot. + + Args: + title: Short batch name shown to the user. + items: Stable item keys and self-contained prompts. + subagent_type: Native subagent definition used for every item. + max_live_items: Optional queued-plus-running item window. + max_running_items: Optional per-batch real execution concurrency. + """ + submitter = _batch_submitter() + if submitter is None: + return _result( + tool_call_id, + content="Durable subagent batches are unavailable. Enable subagent_batches with a SQL database and restart Gateway.", + error=True, + ) + if not items: + return _result(tool_call_id, content="A batch must contain at least one item.", error=True) + keys = [item.key for item in items] + if len(set(keys)) != len(keys): + return _result(tool_call_id, content="Batch item keys must be unique.", error=True) + + context = runtime.context if runtime is not None and isinstance(runtime.context, dict) else {} + metadata = runtime.config.get("metadata", {}) if runtime is not None else {} + app_config = _batch_app_config(runtime) + allowed_subagents = metadata.get("allowed_subagents") + available = get_available_subagent_names(app_config=app_config, allowed_subagents=allowed_subagents) + config = get_subagent_config(subagent_type, app_config=app_config) + if config is None or subagent_type not in available: + names = ", ".join(available) if available else "none" + return _result( + tool_call_id, + content=f"Unknown or disallowed subagent type {subagent_type!r}. Available: {names}", + error=True, + ) + + parent_skills = metadata.get("available_skills") + if parent_skills is not None: + config = replace(config, skills=_merge_skill_allowlists(list(parent_skills), config.skills)) + + thread_id = context.get("thread_id") or runtime.config.get("configurable", {}).get("thread_id") + if not thread_id: + return _result(tool_call_id, content="Durable batches require a thread_id.", error=True) + user_id = resolve_runtime_user_id(runtime) + run_id = context.get("run_id") + submission_key = f"{run_id or thread_id}:{tool_call_id}" + execution_spec = { + "subagent_config": asdict(config), + "parent_model": metadata.get("model_name"), + "tool_groups": metadata.get("tool_groups"), + "user_role": context.get("user_role"), + "oauth_provider": context.get("oauth_provider"), + "oauth_id": context.get("oauth_id"), + "channel_user_id": context.get("channel_user_id"), + "is_internal": context.get("is_internal") is True, + "authz_attributes": normalize_authz_attributes(context.get("authz_attributes")), + } + try: + batch = await submitter.submit( + BatchSubmitRequest( + user_id=user_id, + thread_id=str(thread_id), + run_id=str(run_id) if run_id else None, + tool_call_id=tool_call_id, + submission_key=submission_key, + title=title.strip()[:256] or "Subagent batch", + subagent_type=subagent_type, + items=[item.model_dump() for item in items], + max_live_items=max_live_items, + max_running_items=max_running_items, + execution_spec=execution_spec, + ) + ) + except Exception as exc: + return _result(tool_call_id, content=f"Batch submission failed: {exc}", error=True) + return _result( + tool_call_id, + batch=batch, + content=(f"Batch {batch['id']} accepted with {batch['total_items']} items. It is running independently and survives Gateway restarts. Use batch_status for progress; do not launch ordinary task calls for these items."), + ) + + +@tool("batch_status", parse_docstring=True) +async def batch_status(runtime: Runtime, batch_id: str) -> str: + """Return a compact durable batch progress snapshot. + + Args: + batch_id: Server batch identifier returned by ``batch_task``. + """ + submitter = _batch_submitter() + if submitter is None: + return "Durable subagent batches are unavailable." + batch = await submitter.get_batch(batch_id=batch_id, user_id=resolve_runtime_user_id(runtime)) + if batch is None: + return "Batch not found." + return json.dumps( + { + "batch_id": batch["id"], + "status": batch["status"], + "total_items": batch["total_items"], + "counts": batch["counts"], + }, + ensure_ascii=False, + ) + + +@tool("cancel_batch", parse_docstring=True) +async def cancel_batch(runtime: Runtime, batch_id: str) -> str: + """Cancel pending and running work in one durable subagent batch. + + Args: + batch_id: Server batch identifier returned by ``batch_task``. + """ + submitter = _batch_submitter() + if submitter is None: + return "Durable subagent batches are unavailable." + batch = await submitter.cancel_batch(batch_id=batch_id, user_id=resolve_runtime_user_id(runtime)) + if batch is None: + return "Batch not found." + return f"Batch {batch_id} cancellation requested." diff --git a/backend/packages/harness/deerflow/tools/builtins/task_tool.py b/backend/packages/harness/deerflow/tools/builtins/task_tool.py index 8400b2997..ad2631aef 100644 --- a/backend/packages/harness/deerflow/tools/builtins/task_tool.py +++ b/backend/packages/harness/deerflow/tools/builtins/task_tool.py @@ -3,6 +3,7 @@ import asyncio import logging import uuid +from contextvars import ContextVar from dataclasses import replace from typing import TYPE_CHECKING, Annotated, Any, cast @@ -18,6 +19,7 @@ from deerflow.extensions import resolve_run_extensions from deerflow.runtime.user_context import resolve_runtime_user_id from deerflow.sandbox.security import LOCAL_BASH_SUBAGENT_DISABLED_MESSAGE, is_host_bash_allowed from deerflow.subagents import SubagentExecutor, get_available_subagent_names, get_subagent_config +from deerflow.subagents.capacity import SubagentExecutionCapacity from deerflow.subagents.config import resolve_subagent_model_name from deerflow.subagents.executor import ( SubagentStatus, @@ -40,6 +42,15 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_explicit_execution_capacity: ContextVar[SubagentExecutionCapacity | None] = ContextVar( + "deerflow_explicit_subagent_execution_capacity", + default=None, +) +_explicit_app_config: ContextVar[Any | None] = ContextVar( + "deerflow_explicit_subagent_app_config", + default=None, +) + def _is_subagent_terminal(result: Any) -> bool: """Return whether a background subagent result is safe to clean up.""" @@ -87,6 +98,35 @@ def _log_cleanup_failure(cleanup_task: asyncio.Task[None], *, trace_id: str, exe _deferred_cleanup_tasks: set[asyncio.Task[None]] = set() +def bind_task_tool( + execution_capacity: SubagentExecutionCapacity, + *, + app_config: "AppConfig | None" = None, +): + """Return a task tool bound to one explicit SDK runtime capacity. + + The copied tool keeps the original name, description, and argument schema; + only its coroutine is wrapped. ``ContextVar`` keeps concurrent direct + factories isolated while the resolved capacity is passed into the executor + before work crosses to the persistent subagent event loop. + """ + + original_coroutine = task_tool.coroutine + if original_coroutine is None: # pragma: no cover - task_tool is async by contract + raise RuntimeError("task tool has no async implementation") + + async def bound_coroutine(**kwargs): + capacity_token = _explicit_execution_capacity.set(execution_capacity) + config_token = _explicit_app_config.set(app_config) + try: + return await original_coroutine(**kwargs) + finally: + _explicit_app_config.reset(config_token) + _explicit_execution_capacity.reset(capacity_token) + + return task_tool.model_copy(update={"coroutine": bound_coroutine}) + + def _schedule_deferred_subagent_cleanup(execution_id: str, trace_id: str, max_polls: int) -> asyncio.Task[None]: logger.debug(f"[trace={trace_id}] Scheduling deferred cleanup for cancelled execution {execution_id}") cleanup_task = asyncio.create_task(_deferred_cleanup_subagent_task(execution_id, trace_id, max_polls)) @@ -161,6 +201,9 @@ def _report_subagent_usage(runtime: Any, result: Any) -> None: def _get_runtime_app_config(runtime: Any) -> "AppConfig | None": + explicit = _explicit_app_config.get() + if explicit is not None: + return cast("AppConfig", explicit) context = getattr(runtime, "context", None) if isinstance(context, dict): app_config = context.get("app_config") @@ -414,6 +457,9 @@ async def task_tool( executor_kwargs["app_config"] = resolved_app_config if run_extensions is not None: executor_kwargs["extensions"] = run_extensions + explicit_capacity = _explicit_execution_capacity.get() + if explicit_capacity is not None: + executor_kwargs["execution_capacity"] = explicit_capacity executor = SubagentExecutor(**executor_kwargs) # Keep the provider tool-call ID for stream/message correlation, but use a diff --git a/backend/packages/harness/deerflow/tools/tools.py b/backend/packages/harness/deerflow/tools/tools.py index c586a41c3..77e9bed57 100644 --- a/backend/packages/harness/deerflow/tools/tools.py +++ b/backend/packages/harness/deerflow/tools/tools.py @@ -7,9 +7,13 @@ from deerflow.config.app_config import AppConfig from deerflow.mcp.tasks.runtime import is_mcp_task_runtime_available from deerflow.reflection import resolve_variable from deerflow.sandbox.security import is_host_bash_allowed +from deerflow.subagents.batch_runtime import is_subagent_batch_runtime_available from deerflow.tools.builtins import ( ask_clarification_tool, + batch_status, + batch_task, cancel_background_task, + cancel_batch, list_background_tasks, list_uploaded_files, present_file_tool, @@ -117,7 +121,9 @@ def get_available_tools( # Add subagent tools only if enabled via runtime parameter if subagent_enabled: builtin_tools.extend(SUBAGENT_TOOLS) - logger.info("Including subagent tools (task)") + if is_subagent_batch_runtime_available(): + builtin_tools.extend((batch_task, batch_status, cancel_batch)) + logger.info("Including native subagent tools") # If no model_name specified, use the first model (default) if model_name is None and config.models: diff --git a/backend/tests/test_batch_task_tool.py b/backend/tests/test_batch_task_tool.py new file mode 100644 index 000000000..75d31cba3 --- /dev/null +++ b/backend/tests/test_batch_task_tool.py @@ -0,0 +1,200 @@ +import importlib +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from langchain_core.messages import ToolMessage +from langgraph.types import Command + +from deerflow.subagents.config import SubagentConfig +from deerflow.tools.builtins.batch_task_tool import BatchTaskItem + +tool_module = importlib.import_module("deerflow.tools.builtins.batch_task_tool") + + +def _runtime(): + return SimpleNamespace( + state={}, + context={ + "thread_id": "thread-1", + "run_id": "run-1", + "user_id": "user-1", + "user_role": "member", + }, + config={ + "metadata": { + "model_name": "model-a", + "allowed_subagents": ["general-purpose"], + "tool_groups": ["web"], + }, + "configurable": {"thread_id": "thread-1"}, + }, + ) + + +def _message(command: Command) -> ToolMessage: + messages = command.update["messages"] + assert len(messages) == 1 and isinstance(messages[0], ToolMessage) + return messages[0] + + +@pytest.mark.asyncio +async def test_batch_task_is_explicit_idempotent_submission(monkeypatch) -> None: + submitter = AsyncMock() + submitter.submit.return_value = { + "id": "subagent-batch-1", + "status": "queued", + "total_items": 2, + } + monkeypatch.setattr(tool_module, "get_subagent_batch_submitter", lambda: submitter) + monkeypatch.setattr( + tool_module, + "get_available_subagent_names", + lambda **_kwargs: ["general-purpose"], + ) + monkeypatch.setattr( + tool_module, + "get_subagent_config", + lambda *_args, **_kwargs: SubagentConfig( + name="general-purpose", + description="General purpose", + ), + ) + + command = await tool_module.batch_task.coroutine( + runtime=_runtime(), + title="Process records", + items=[ + BatchTaskItem(key="record-1", prompt="Process one"), + BatchTaskItem(key="record-2", prompt="Process two"), + ], + subagent_type="general-purpose", + tool_call_id="call-1", + max_live_items=20, + max_running_items=5, + ) + + message = _message(command) + request = submitter.submit.await_args.args[0] + assert request.submission_key == "run-1:call-1" + assert request.user_id == "user-1" + assert [item["key"] for item in request.items] == ["record-1", "record-2"] + assert request.max_live_items == 20 + assert request.max_running_items == 5 + assert message.additional_kwargs["subagent_batch_id"] == "subagent-batch-1" + assert "running independently" in message.content + + +@pytest.mark.asyncio +async def test_batch_task_rejects_duplicate_item_keys_without_submitting(monkeypatch) -> None: + submitter = AsyncMock() + monkeypatch.setattr(tool_module, "get_subagent_batch_submitter", lambda: submitter) + + command = await tool_module.batch_task.coroutine( + runtime=_runtime(), + title="Duplicates", + items=[ + BatchTaskItem(key="same", prompt="one"), + BatchTaskItem(key="same", prompt="two"), + ], + subagent_type="general-purpose", + tool_call_id="call-1", + ) + + message = _message(command) + assert message.status == "error" + assert "unique" in message.content + submitter.submit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_bound_batch_tools_use_the_explicit_submitter(monkeypatch) -> None: + explicit = AsyncMock() + explicit.get_batch.return_value = { + "id": "subagent-batch-explicit", + "status": "running", + "total_items": 2, + "counts": {"running": 1, "succeeded": 1}, + } + fallback = AsyncMock() + monkeypatch.setattr(tool_module, "get_subagent_batch_submitter", lambda: fallback) + + tools = {tool.name: tool for tool in tool_module.bind_batch_tools(explicit)} + result = await tools["batch_status"].coroutine( + runtime=_runtime(), + batch_id="subagent-batch-explicit", + ) + + assert "subagent-batch-explicit" in result + explicit.get_batch.assert_awaited_once_with( + batch_id="subagent-batch-explicit", + user_id="user-1", + ) + fallback.get_batch.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_bound_batch_task_uses_the_explicit_app_config(monkeypatch) -> None: + app_config = object() + captured = {} + submitter = AsyncMock() + submitter.submit.return_value = { + "id": "subagent-batch-explicit", + "status": "queued", + "total_items": 1, + } + + def available_names(*, app_config, allowed_subagents): + captured["names"] = (app_config, allowed_subagents) + return ["general-purpose"] + + def subagent_config(name, *, app_config): + captured["config"] = (name, app_config) + return SubagentConfig( + name="general-purpose", + description="General purpose", + ) + + monkeypatch.setattr(tool_module, "get_available_subagent_names", available_names) + monkeypatch.setattr(tool_module, "get_subagent_config", subagent_config) + + tools = { + tool.name: tool + for tool in tool_module.bind_batch_tools( + submitter, + app_config=app_config, + ) + } + await tools["batch_task"].coroutine( + runtime=_runtime(), + title="Explicit config", + items=[BatchTaskItem(key="record-1", prompt="Process one")], + subagent_type="general-purpose", + tool_call_id="call-explicit", + max_live_items=None, + max_running_items=None, + ) + + assert captured["names"] == (app_config, ["general-purpose"]) + assert captured["config"] == ("general-purpose", app_config) + submitter.submit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_bound_batch_tools_do_not_fall_back_after_runtime_stops(monkeypatch) -> None: + fallback = AsyncMock() + monkeypatch.setattr(tool_module, "get_subagent_batch_submitter", lambda: fallback) + + tools = { + tool.name: tool + for tool in tool_module.bind_batch_tools( + submitter_provider=lambda: None, + ) + } + result = await tools["batch_status"].coroutine( + runtime=_runtime(), + batch_id="subagent-batch-stopped", + ) + + assert result == "Durable subagent batches are unavailable." + fallback.get_batch.assert_not_awaited() diff --git a/backend/tests/test_client.py b/backend/tests/test_client.py index f4122735e..392158a42 100644 --- a/backend/tests/test_client.py +++ b/backend/tests/test_client.py @@ -26,6 +26,7 @@ from deerflow.client import DeerFlowClient from deerflow.config.authorization_config import AuthorizationConfig, AuthorizationProviderConfig from deerflow.config.extensions_config import ExtensionsConfig, McpServerConfig from deerflow.config.paths import Paths +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig from deerflow.skills.types import SkillCategory from deerflow.tools.mcp_metadata import tag_mcp_tool from deerflow.uploads.manager import PathTraversalError @@ -122,6 +123,16 @@ class TestClientInit: DeerFlowClient(config_path="/tmp/custom.yaml") mock_reload.assert_called_once_with("/tmp/custom.yaml") + def test_installs_process_subagent_capacity_from_frozen_config(self, mock_app_config): + runtime_config = SubagentRuntimeConfig(max_running=7) + mock_app_config.subagent_runtime = runtime_config + with ( + patch("deerflow.client.get_app_config", return_value=mock_app_config), + patch("deerflow.client.configure_subagent_execution_capacity") as configure, + ): + DeerFlowClient() + configure.assert_called_once_with(runtime_config) + def test_checkpointer_stored(self, mock_app_config): cp = MagicMock() with patch("deerflow.client.get_app_config", return_value=mock_app_config): diff --git a/backend/tests/test_create_deerflow_agent.py b/backend/tests/test_create_deerflow_agent.py index 700de696c..55c27c1ea 100644 --- a/backend/tests/test_create_deerflow_agent.py +++ b/backend/tests/test_create_deerflow_agent.py @@ -14,6 +14,9 @@ from deerflow.agents.factory import create_deerflow_agent from deerflow.agents.features import Next, Prev, RuntimeFeatures from deerflow.agents.middlewares.view_image_middleware import ViewImageMiddleware from deerflow.agents.thread_state import DeltaThreadState, ThreadState +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.subagents import SubagentRuntime def _make_mock_model(): @@ -241,6 +244,57 @@ def test_subagent_injects_task_tool(mock_create_agent): assert "task" in tool_names +@patch("deerflow.agents.factory.create_agent") +def test_explicit_subagent_runtime_aligns_factory_middleware_and_tools(mock_create_agent): + mock_create_agent.return_value = MagicMock() + submitter = MagicMock() + runtime = SubagentRuntime( + SubagentRuntimeConfig(max_running=7), + max_total_per_run=12, + batch_submitter=submitter, + ) + + create_deerflow_agent( + _make_mock_model(), + features=RuntimeFeatures(subagent=True, sandbox=False), + subagent_runtime=runtime, + ) + + call_kwargs = mock_create_agent.call_args.kwargs + limit = next(middleware for middleware in call_kwargs["middleware"] if type(middleware).__name__ == "SubagentLimitMiddleware") + assert limit.max_concurrent == 7 + assert limit.max_total == 12 + tool_names = {tool.name for tool in call_kwargs["tools"]} + assert {"task", "batch_task", "batch_status", "cancel_batch"} <= tool_names + + +def test_explicit_subagent_runtime_requires_the_subagent_feature() -> None: + runtime = SubagentRuntime(SubagentRuntimeConfig(max_running=4)) + + with pytest.raises(ValueError, match="subagent_runtime.*features.subagent"): + create_deerflow_agent( + _make_mock_model(), + features=RuntimeFeatures(subagent=False, sandbox=False), + subagent_runtime=runtime, + ) + + +def test_factory_rejects_configured_batch_runtime_before_worker_start() -> None: + runtime = SubagentRuntime( + SubagentRuntimeConfig(max_running=4), + batch_repository=MagicMock(), + batch_config=SubagentBatchesConfig(enabled=True), + app_config=MagicMock(), + ) + + with pytest.raises(RuntimeError, match="await subagent_runtime.start"): + create_deerflow_agent( + _make_mock_model(), + features=RuntimeFeatures(subagent=True, sandbox=False), + subagent_runtime=runtime, + ) + + # --------------------------------------------------------------------------- # 9. Middleware ordering — ClarificationMiddleware always last # --------------------------------------------------------------------------- diff --git a/backend/tests/test_features_router.py b/backend/tests/test_features_router.py index 2336316b0..d78de64d3 100644 --- a/backend/tests/test_features_router.py +++ b/backend/tests/test_features_router.py @@ -14,9 +14,15 @@ def _app_with_config( browser_enabled: bool = False, browser_extra: dict | None = None, mcp_tasks_available: bool = False, + subagent_batches_available: bool = False, + subagent_batch_repo_available: bool | None = None, ) -> FastAPI: app = FastAPI() app.state.mcp_tasks_available = mcp_tasks_available + app.state.subagent_batches_available = subagent_batches_available + if subagent_batch_repo_available is None: + subagent_batch_repo_available = subagent_batches_available + app.state.subagent_batch_repo = object() if subagent_batch_repo_available else None app.include_router(features.router) tools = ( [ @@ -25,7 +31,11 @@ def _app_with_config( if browser_enabled else [] ) - fake_config = SimpleNamespace(agents_api=SimpleNamespace(enabled=agents_api_enabled), tools=tools) + fake_config = SimpleNamespace( + agents_api=SimpleNamespace(enabled=agents_api_enabled), + tools=tools, + subagent_runtime=SimpleNamespace(max_running=3), + ) app.dependency_overrides[get_config] = lambda: fake_config return app @@ -38,6 +48,12 @@ def test_features_reports_agents_api_enabled() -> None: "agents_api": {"enabled": True}, "browser_control": {"enabled": False}, "mcp_tasks": {"enabled": False}, + "subagent_batches": { + "enabled": False, + "repository_available": False, + "worker_running": False, + "max_running": 3, + }, } @@ -49,6 +65,12 @@ def test_features_reports_agents_api_disabled() -> None: "agents_api": {"enabled": False}, "browser_control": {"enabled": False}, "mcp_tasks": {"enabled": False}, + "subagent_batches": { + "enabled": False, + "repository_available": False, + "worker_running": False, + "max_running": 3, + }, } @@ -59,6 +81,41 @@ def test_features_reports_mcp_tasks_startup_capability() -> None: assert response.json()["mcp_tasks"] == {"enabled": True} +def test_features_reports_subagent_batch_startup_capability() -> None: + with TestClient( + _app_with_config( + agents_api_enabled=True, + subagent_batches_available=True, + ) + ) as client: + response = client.get("/api/features") + assert response.status_code == 200 + assert response.json()["subagent_batches"] == { + "enabled": True, + "repository_available": True, + "worker_running": True, + "max_running": 3, + } + + +def test_features_distinguishes_batch_history_from_worker_availability() -> None: + with TestClient( + _app_with_config( + agents_api_enabled=True, + subagent_batches_available=False, + subagent_batch_repo_available=True, + ) + ) as client: + response = client.get("/api/features") + assert response.status_code == 200 + assert response.json()["subagent_batches"] == { + "enabled": False, + "repository_available": True, + "worker_running": False, + "max_running": 3, + } + + def test_features_reports_browser_control_enabled_when_configured_and_runtime_available() -> None: with ( patch("app.gateway.browser_capability.importlib.util.find_spec", return_value=object()), diff --git a/backend/tests/test_lead_agent_model_resolution.py b/backend/tests/test_lead_agent_model_resolution.py index a19f00c38..db35e7284 100644 --- a/backend/tests/test_lead_agent_model_resolution.py +++ b/backend/tests/test_lead_agent_model_resolution.py @@ -642,6 +642,30 @@ def test_build_middlewares_uses_resolved_model_name_for_vision(monkeypatch): assert isinstance(middlewares[-1], ClarificationMiddleware) +def test_build_middlewares_prefers_startup_execution_capacity_after_reload(monkeypatch): + app_config = _make_app_config([_make_model("safe-model", supports_thinking=False)]) + app_config.subagent_runtime.max_running = 12 + monkeypatch.setattr(lead_agent_module, "_create_summarization_middleware", lambda **kwargs: None) + monkeypatch.setattr(lead_agent_module, "_create_todo_list_middleware", lambda is_plan_mode: None) + + middlewares = lead_agent_module.build_middlewares( + { + "configurable": { + "model_name": "safe-model", + "is_plan_mode": False, + "subagent_enabled": True, + "max_concurrent_subagents": 10, + } + }, + model_name="safe-model", + app_config=app_config, + subagent_execution_capacity=3, + ) + + limiter = next(middleware for middleware in middlewares if isinstance(middleware, lead_agent_module.SubagentLimitMiddleware)) + assert limiter.max_concurrent == 3 + + def test_build_middlewares_passes_explicit_app_config_to_shared_factory(monkeypatch): app_config = _make_app_config([_make_model("safe-model", supports_thinking=False)]) captured: dict[str, object] = {} diff --git a/backend/tests/test_lead_agent_prompt.py b/backend/tests/test_lead_agent_prompt.py index 5ad449d57..1344494b2 100644 --- a/backend/tests/test_lead_agent_prompt.py +++ b/backend/tests/test_lead_agent_prompt.py @@ -215,6 +215,7 @@ def test_apply_prompt_template_includes_subagent_total_limit(monkeypatch): mounts=[], ), subagents=SubagentsAppConfig(), + subagent_runtime=SimpleNamespace(max_running=4), skills=SimpleNamespace(container_path="/mnt/skills", use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage", get_skills_path=lambda: Path("/tmp/skills")), skill_evolution=SimpleNamespace(enabled=False), tool_search=SimpleNamespace(enabled=False), @@ -248,6 +249,7 @@ def test_apply_prompt_template_clamps_subagent_limits_to_enforced_bounds(monkeyp mounts=[], ), subagents=SubagentsAppConfig(), + subagent_runtime=SimpleNamespace(max_running=4), skills=SimpleNamespace(container_path="/mnt/skills", use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage", get_skills_path=lambda: Path("/tmp/skills")), skill_evolution=SimpleNamespace(enabled=False), tool_search=SimpleNamespace(enabled=False), @@ -269,6 +271,39 @@ def test_apply_prompt_template_clamps_subagent_limits_to_enforced_bounds(monkeyp assert "MAXIMUM 50 `task` CALLS PER RUN" in prompt +def test_apply_prompt_template_prefers_startup_execution_capacity_after_reload(monkeypatch): + explicit_config = SimpleNamespace( + sandbox=SimpleNamespace( + use="deerflow.sandbox.local:LocalSandboxProvider", + allow_host_bash=False, + mounts=[], + ), + subagents=SubagentsAppConfig(), + subagent_runtime=SimpleNamespace(max_running=12), + skills=SimpleNamespace( + container_path="/mnt/skills", + use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage", + get_skills_path=lambda: Path("/tmp/skills"), + ), + skill_evolution=SimpleNamespace(enabled=False), + tool_search=SimpleNamespace(enabled=False), + memory=SimpleNamespace(enabled=False, injection_enabled=True, max_injection_tokens=2000), + acp_agents={}, + ) + monkeypatch.setattr(prompt_module, "get_or_new_skill_storage", lambda app_config=None: SimpleNamespace(load_skills=lambda enabled_only=True: [])) + monkeypatch.setattr(prompt_module, "get_agent_soul", lambda agent_name=None, **kwargs: "") + + prompt = prompt_module.apply_prompt_template( + subagent_enabled=True, + max_concurrent_subagents=10, + app_config=explicit_config, + subagent_execution_capacity=3, + ) + + assert "MAXIMUM 3 `task` CALLS PER RESPONSE" in prompt + assert "MAXIMUM 10 `task` CALLS PER RESPONSE" not in prompt + + def test_apply_prompt_template_single_subagent_limit_matches_middleware(monkeypatch): """Regression test for single-subagent mode (MIN_CONCURRENT_SUBAGENT_CALLS = 1). diff --git a/backend/tests/test_migration_0004_run_ownership_dedupe.py b/backend/tests/test_migration_0004_run_ownership_dedupe.py index 0c71a25cc..dbda04f7f 100644 --- a/backend/tests/test_migration_0004_run_ownership_dedupe.py +++ b/backend/tests/test_migration_0004_run_ownership_dedupe.py @@ -157,7 +157,7 @@ async def test_migration_dedupes_duplicate_active_rows_before_unique_index(tmp_p with sqlite3.connect(db_path) as raw: version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() # Bootstrap upgrades through the later revisions after 0004. - assert version_row[0] == "0015_scheduled_task_enqueue" + assert version_row[0] == "0016_subagent_batches" # Sanity: the invariant the index enforces is now true — at most one # active row per thread. diff --git a/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py b/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py index 6d7f686c3..aabd312e5 100644 --- a/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py +++ b/backend/tests/test_migration_0007_scheduled_run_active_dedupe.py @@ -173,7 +173,7 @@ async def test_migration_supersedes_duplicate_active_runs_before_unique_index(tm with sqlite3.connect(db_path) as raw: version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() - assert version_row[0] == "0015_scheduled_task_enqueue" + assert version_row[0] == "0016_subagent_batches" # Sanity: the invariant the index enforces now holds — at most one # active row per task_id. diff --git a/backend/tests/test_migration_0015_scheduled_task_enqueue.py b/backend/tests/test_migration_0015_scheduled_task_enqueue.py index 8a327fc49..513ac7767 100644 --- a/backend/tests/test_migration_0015_scheduled_task_enqueue.py +++ b/backend/tests/test_migration_0015_scheduled_task_enqueue.py @@ -55,7 +55,9 @@ async def test_migration_interrupts_legacy_queue_and_adds_claim_fields(tmp_path: legacy_run = (await conn.execute(sa.text("SELECT status, error, finished_at FROM scheduled_task_runs WHERE id = 'run-legacy-queued'"))).one() index_sql = await conn.scalar(sa.text("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = 'uq_scheduled_task_run_active'")) - assert version == "0015_scheduled_task_enqueue" + # Bootstrap always advances to the repository head after exercising + # the 0015 migration behavior below. + assert version == "0016_subagent_batches" assert {"lease_owner", "lease_expires_at", "attempt_count"} <= columns.keys() assert columns["attempt_count"]["nullable"] is False assert overlap_policy == "enqueue" diff --git a/backend/tests/test_persistence_bootstrap.py b/backend/tests/test_persistence_bootstrap.py index ec93b1408..23548e636 100644 --- a/backend/tests/test_persistence_bootstrap.py +++ b/backend/tests/test_persistence_bootstrap.py @@ -48,7 +48,7 @@ from deerflow.persistence.migrations._helpers import _normalize_default asyncio_test = pytest.mark.asyncio -HEAD = "0015_scheduled_task_enqueue" +HEAD = "0016_subagent_batches" BASELINE = "0001_baseline" diff --git a/backend/tests/test_persistence_bootstrap_concurrency.py b/backend/tests/test_persistence_bootstrap_concurrency.py index 620cff96e..36f703fb5 100644 --- a/backend/tests/test_persistence_bootstrap_concurrency.py +++ b/backend/tests/test_persistence_bootstrap_concurrency.py @@ -28,7 +28,7 @@ from deerflow.persistence.bootstrap import bootstrap_schema pytestmark = pytest.mark.asyncio -HEAD = "0015_scheduled_task_enqueue" +HEAD = "0016_subagent_batches" def _url(tmp_path: Path) -> str: diff --git a/backend/tests/test_persistence_bootstrap_regression.py b/backend/tests/test_persistence_bootstrap_regression.py index aa0bff019..46aded16a 100644 --- a/backend/tests/test_persistence_bootstrap_regression.py +++ b/backend/tests/test_persistence_bootstrap_regression.py @@ -76,7 +76,7 @@ async def test_legacy_database_recovers_token_usage_column(tmp_path: Path) -> No cols = {row[1] for row in raw.execute("PRAGMA table_info(runs)").fetchall()} assert "token_usage_by_model" in cols version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() - assert version_row[0] == "0015_scheduled_task_enqueue" + assert version_row[0] == "0016_subagent_batches" # And the read path that originally 500'd must now succeed. sf = get_session_factory() @@ -116,6 +116,6 @@ async def test_legacy_database_with_manual_alter_still_bootstraps(tmp_path: Path # No duplicate column -- list, not set, to catch dupes. assert cols.count("token_usage_by_model") == 1 version_row = raw.execute("SELECT version_num FROM alembic_version").fetchone() - assert version_row[0] == "0015_scheduled_task_enqueue" + assert version_row[0] == "0016_subagent_batches" finally: await close_engine() diff --git a/backend/tests/test_subagent_batch_repository.py b/backend/tests/test_subagent_batch_repository.py new file mode 100644 index 000000000..ebc25301b --- /dev/null +++ b/backend/tests/test_subagent_batch_repository.py @@ -0,0 +1,301 @@ +from datetime import UTC, datetime, timedelta + +import pytest +import pytest_asyncio + +from deerflow.config.database_config import DatabaseConfig +from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config +from deerflow.persistence.subagent_batches import SubagentBatchRepository + + +@pytest_asyncio.fixture(autouse=True) +async def _close_engine() -> None: + yield + await close_engine() + + +async def _repo(tmp_path) -> SubagentBatchRepository: + await init_engine_from_config(DatabaseConfig(backend="sqlite", sqlite_dir=str(tmp_path))) + sf = get_session_factory() + assert sf is not None + return SubagentBatchRepository(sf) + + +async def _create( + repo: SubagentBatchRepository, + *, + count: int = 4, + max_live: int = 2, + max_running: int = 1, + max_attempts: int = 2, +) -> dict: + return await repo.create_batch( + batch_id="batch-1", + user_id="user-1", + thread_id="thread-1", + run_id="run-1", + tool_call_id="call-1", + submission_key="run-1:call-1", + title="Research records", + subagent_type="general-purpose", + items=[{"key": f"item-{i}", "prompt": f"Process {i}"} for i in range(count)], + max_live_items=max_live, + max_running_items=max_running, + max_attempts=max_attempts, + execution_spec={ + "subagent_config": { + "name": "general-purpose", + "description": "test", + "system_prompt": "private instructions", + }, + "authz_attributes": {"tenant": "private-tenant"}, + }, + ) + + +@pytest.mark.asyncio +async def test_claim_separates_total_live_leased_and_running(tmp_path) -> None: + repo = await _repo(tmp_path) + created = await _create(repo) + assert created["counts"]["pending"] == 4 + + now = datetime.now(UTC) + claimed = await repo.claim_items(now=now, lease_owner="worker-1", lease_seconds=60, limit=10) + assert len(claimed) == 1 + assert claimed[0]["status"] == "leased" + + batch = await repo.get_batch("batch-1", user_id="user-1") + assert batch is not None + assert batch["counts"] == { + "pending": 2, + "queued": 1, + "leased": 1, + "running": 0, + "succeeded": 0, + "failed": 0, + "cancelled": 0, + } + assert await repo.mark_item_running(claimed[0]["id"], lease_owner="worker-1", now=now) + + while_full = await repo.claim_items( + now=now + timedelta(seconds=1), + lease_owner="worker-2", + lease_seconds=60, + limit=10, + ) + assert while_full == [] + + +@pytest.mark.asyncio +async def test_expired_lease_is_recovered_with_stable_item_identity(tmp_path) -> None: + repo = await _repo(tmp_path) + await _create(repo, count=1, max_live=1, max_running=1) + now = datetime.now(UTC) + first = await repo.claim_items(now=now, lease_owner="worker-1", lease_seconds=30, limit=1) + + reclaimed = await repo.claim_items( + now=now + timedelta(seconds=31), + lease_owner="worker-2", + lease_seconds=30, + limit=1, + ) + assert len(reclaimed) == 1 + assert reclaimed[0]["id"] == first[0]["id"] + assert reclaimed[0]["item_key"] == "item-0" + assert reclaimed[0]["attempt"] == 2 + + +@pytest.mark.asyncio +async def test_finalize_retries_then_terminalizes_and_completes_batch(tmp_path) -> None: + repo = await _repo(tmp_path) + await _create(repo, count=1, max_live=1, max_running=1) + now = datetime.now(UTC) + first = (await repo.claim_items(now=now, lease_owner="worker-1", lease_seconds=60, limit=1))[0] + await repo.finalize_item( + first["id"], + lease_owner="worker-1", + succeeded=False, + result=None, + result_preview=None, + result_truncated=False, + error="temporary", + stop_reason=None, + token_usage=None, + model_name="model-a", + completed_at=now, + ) + item = (await repo.list_items("batch-1", user_id="user-1"))[0] + assert item["status"] == "queued" + + second = (await repo.claim_items(now=now + timedelta(seconds=1), lease_owner="worker-2", lease_seconds=60, limit=1))[0] + await repo.finalize_item( + second["id"], + lease_owner="worker-2", + succeeded=True, + result="done", + result_preview="done", + result_truncated=False, + error=None, + stop_reason=None, + token_usage={"total_tokens": 12}, + model_name="model-a", + completed_at=now + timedelta(seconds=2), + ) + batch = await repo.get_batch("batch-1", user_id="user-1") + assert batch is not None + assert batch["status"] == "completed" + assert batch["counts"]["succeeded"] == 1 + + +@pytest.mark.asyncio +async def test_pause_resume_cancel_and_owner_scope(tmp_path) -> None: + repo = await _repo(tmp_path) + await _create(repo, count=2, max_live=2, max_running=1) + paused = await repo.pause_batch("batch-1", user_id="user-1") + assert paused is not None and paused["status"] == "paused" + assert await repo.claim_items(now=datetime.now(UTC), lease_owner="worker", lease_seconds=60, limit=1) == [] + resumed = await repo.resume_batch("batch-1", user_id="user-1") + assert resumed is not None and resumed["status"] == "queued" + cancelled = await repo.cancel_batch("batch-1", user_id="user-1") + assert cancelled is not None and cancelled["status"] == "cancelled" + assert cancelled["counts"]["cancelled"] == 2 + assert await repo.get_batch("batch-1", user_id="other") is None + + +@pytest.mark.asyncio +async def test_cancel_terminalizes_in_flight_items_and_fences_stale_completion(tmp_path) -> None: + repo = await _repo(tmp_path) + await _create(repo, count=1, max_live=1, max_running=1) + now = datetime.now(UTC) + claimed = (await repo.claim_items(now=now, lease_owner="worker-1", lease_seconds=60, limit=1))[0] + assert await repo.mark_item_running(claimed["id"], lease_owner="worker-1", now=now) + + cancelled = await repo.cancel_batch("batch-1", user_id="user-1") + + assert cancelled is not None + assert cancelled["counts"]["cancelled"] == 1 + item = (await repo.list_items("batch-1", user_id="user-1"))[0] + assert item["status"] == "cancelled" + assert not await repo.finalize_item( + claimed["id"], + lease_owner="worker-1", + succeeded=True, + result="late result", + result_preview="late result", + result_truncated=False, + error=None, + stop_reason=None, + token_usage=None, + model_name="model-a", + completed_at=now + timedelta(seconds=1), + ) + + +@pytest.mark.asyncio +async def test_executor_admission_failure_requeues_without_consuming_attempt(tmp_path) -> None: + repo = await _repo(tmp_path) + await _create(repo, count=1, max_live=1, max_running=1) + now = datetime.now(UTC) + first = (await repo.claim_items(now=now, lease_owner="worker-1", lease_seconds=60, limit=1))[0] + assert first["attempt"] == 1 + + assert await repo.requeue_item_after_admission_failure( + first["id"], + lease_owner="worker-1", + error="Process-wide subagent capacity is full", + now=now, + ) + queued = (await repo.list_items("batch-1", user_id="user-1"))[0] + assert queued["status"] == "queued" + assert queued["attempt"] == 0 + + second = (await repo.claim_items(now=now + timedelta(seconds=1), lease_owner="worker-2", lease_seconds=60, limit=1))[0] + assert second["attempt"] == 1 + + +@pytest.mark.asyncio +async def test_all_failed_items_mark_batch_failed(tmp_path) -> None: + repo = await _repo(tmp_path) + await _create(repo, count=1, max_live=1, max_running=1, max_attempts=1) + now = datetime.now(UTC) + item = (await repo.claim_items(now=now, lease_owner="worker-1", lease_seconds=60, limit=1))[0] + + assert await repo.finalize_item( + item["id"], + lease_owner="worker-1", + succeeded=False, + result=None, + result_preview=None, + result_truncated=False, + error="permanent", + stop_reason=None, + token_usage=None, + model_name="model-a", + completed_at=now, + ) + batch = await repo.get_batch("batch-1", user_id="user-1") + assert batch is not None + assert batch["status"] == "failed" + + +@pytest.mark.asyncio +async def test_public_projections_omit_execution_context_and_full_results(tmp_path) -> None: + repo = await _repo(tmp_path) + created = await _create(repo, count=1, max_live=1, max_running=1) + assert "execution_spec" not in created + assert "user_id" not in created + assert "submission_key" not in created + assert "run_id" not in created + assert "tool_call_id" not in created + + now = datetime.now(UTC) + item = (await repo.claim_items(now=now, lease_owner="worker-1", lease_seconds=60, limit=1))[0] + assert await repo.finalize_item( + item["id"], + lease_owner="worker-1", + succeeded=True, + result="full private result", + result_preview="preview", + result_truncated=False, + error=None, + stop_reason=None, + token_usage=None, + model_name="model-a", + completed_at=now, + ) + + public_batch = await repo.get_batch("batch-1", user_id="user-1") + assert public_batch is not None + assert "execution_spec" not in public_batch + public_item = (await repo.list_items("batch-1", user_id="user-1"))[0] + assert public_item["result_preview"] == "preview" + assert "result" not in public_item + assert "lease_owner" not in public_item + export_item = (await repo.list_items("batch-1", user_id="user-1", include_result=True))[0] + assert export_item["result"] == "full private result" + + +@pytest.mark.asyncio +async def test_duplicate_submission_key_returns_original_batch(tmp_path) -> None: + repo = await _repo(tmp_path) + original = await _create(repo, count=1) + duplicate = await repo.create_batch( + batch_id="batch-2", + user_id="user-1", + thread_id="thread-1", + run_id="run-1", + tool_call_id="call-1", + submission_key="run-1:call-1", + title="Duplicate retry", + subagent_type="general-purpose", + items=[{"key": "different", "prompt": "Must not be inserted"}], + max_live_items=1, + max_running_items=1, + max_attempts=2, + execution_spec={"subagent_config": {"name": "general-purpose", "description": "test"}}, + ) + + assert duplicate["id"] == original["id"] == "batch-1" + items = await repo.list_items("batch-1", user_id="user-1", include_prompt=True) + assert items is not None + assert [item["item_key"] for item in items] == ["item-0"] diff --git a/backend/tests/test_subagent_batch_service.py b/backend/tests/test_subagent_batch_service.py new file mode 100644 index 000000000..6a1aab722 --- /dev/null +++ b/backend/tests/test_subagent_batch_service.py @@ -0,0 +1,292 @@ +import asyncio +from enum import Enum +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.subagents import batch_service as service_module +from deerflow.subagents.batch_runtime import BatchSubmitRequest +from deerflow.subagents.batch_service import SubagentBatchService +from deerflow.subagents.capacity import SubagentExecutionCapacity + + +class FakeStatus(Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + @property + def is_terminal(self) -> bool: + return self in {FakeStatus.COMPLETED, FakeStatus.FAILED} + + +def _request(**overrides) -> BatchSubmitRequest: + values = { + "user_id": "user-1", + "thread_id": "thread-1", + "run_id": "run-1", + "tool_call_id": "call-1", + "submission_key": "run-1:call-1", + "title": "Records", + "subagent_type": "general-purpose", + "items": [{"key": "record-1", "prompt": "Process record 1"}], + "max_live_items": None, + "max_running_items": None, + "execution_spec": { + "subagent_config": { + "name": "general-purpose", + "description": "General purpose", + "system_prompt": "Work carefully.", + }, + "parent_model": "model-a", + }, + } + values.update(overrides) + return BatchSubmitRequest(**values) + + +@pytest.mark.asyncio +async def test_submit_keeps_batch_running_limit_separate_from_one_process_capacity() -> None: + repository = SimpleNamespace(create_batch=AsyncMock(return_value={"id": "batch-1"})) + service = SubagentBatchService( + repository=repository, + config=SubagentBatchesConfig(max_running_items_per_batch=32), + runtime_config=SubagentRuntimeConfig(max_running=3), + ) + + result = await service.submit(_request(max_live_items=20, max_running_items=10)) + + assert result == {"id": "batch-1"} + assert repository.create_batch.await_args.kwargs["max_running_items"] == 10 + + +@pytest.mark.asyncio +async def test_execute_item_marks_real_running_then_persists_terminal_result(monkeypatch) -> None: + result = SimpleNamespace( + status=FakeStatus.RUNNING, + result=None, + error=None, + stop_reason=None, + token_usage_records=None, + ) + + class Repository: + def __init__(self) -> None: + self.marked_running = False + self.finalized = None + + async def claim_items(self, **_kwargs): + return [ + { + "id": "item-1", + "item_key": "record-1", + "prompt": "Process record 1", + "batch": { + "id": "batch-1", + "thread_id": "thread-1", + "user_id": "user-1", + "run_id": "run-1", + "execution_spec": _request().execution_spec, + }, + } + ] + + async def mark_item_running(self, *_args, **_kwargs): + self.marked_running = True + result.status = FakeStatus.COMPLETED + result.result = "done" + return True + + async def finalize_item(self, *_args, **kwargs): + self.finalized = kwargs + return True + + execution_capacity = SubagentExecutionCapacity(SubagentRuntimeConfig(max_running=1)) + executor_kwargs = {} + + class Executor: + def __init__(self, **kwargs) -> None: + executor_kwargs.update(kwargs) + + def execute_async(self, _prompt, task_id=None): + assert task_id == "item-1" + return "execution-1" + + repository = Repository() + monkeypatch.setattr(service_module, "get_app_config", lambda: SimpleNamespace()) + monkeypatch.setattr(service_module, "resolve_subagent_model_name", lambda *_args, **_kwargs: "model-a") + monkeypatch.setattr(service_module, "SubagentExecutor", Executor) + monkeypatch.setattr(service_module, "SubagentStatus", FakeStatus) + monkeypatch.setattr(service_module, "get_background_task_result", lambda _execution_id: result) + monkeypatch.setattr(service_module, "cleanup_background_task", lambda _execution_id: None) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **_kwargs: []) + service = SubagentBatchService( + repository=repository, + config=SubagentBatchesConfig(), + runtime_config=SubagentRuntimeConfig(max_running=1), + execution_capacity=execution_capacity, + ) + + await service.run_once(now=service_module.datetime.now(service_module.UTC)) + await asyncio.gather(*list(service._executions.values())) + + assert repository.marked_running is True + assert repository.finalized is not None + assert repository.finalized["succeeded"] is True + assert repository.finalized["result"] == "done" + assert executor_kwargs["execution_capacity"] is execution_capacity + + +@pytest.mark.asyncio +async def test_execute_item_polls_completion_without_waiting_for_lease_renewal(monkeypatch) -> None: + result = SimpleNamespace( + status=FakeStatus.PENDING, + result=None, + error=None, + stop_reason=None, + token_usage_records=None, + ) + reads = 0 + + class Repository: + def __init__(self) -> None: + self.finalized = None + + async def claim_items(self, **_kwargs): + return [ + { + "id": "item-1", + "item_key": "record-1", + "prompt": "Process record 1", + "batch": { + "id": "batch-1", + "thread_id": "thread-1", + "user_id": "user-1", + "run_id": "run-1", + "execution_spec": _request().execution_spec, + }, + } + ] + + async def mark_item_running(self, *_args, **_kwargs): + raise AssertionError("a task that completes between polls need not expose running") + + async def renew_item_lease(self, *_args, **_kwargs): + raise AssertionError("short completion must not wait for lease renewal") + + async def finalize_item(self, *_args, **kwargs): + self.finalized = kwargs + return True + + class Executor: + def __init__(self, **_kwargs) -> None: + pass + + def execute_async(self, _prompt, task_id=None): + assert task_id == "item-1" + return "execution-1" + + def read_result(_execution_id): + nonlocal reads + reads += 1 + if reads > 1: + result.status = FakeStatus.COMPLETED + result.result = "fast result" + return result + + repository = Repository() + monkeypatch.setattr(service_module, "get_app_config", lambda: SimpleNamespace()) + monkeypatch.setattr(service_module, "resolve_subagent_model_name", lambda *_args, **_kwargs: "model-a") + monkeypatch.setattr(service_module, "SubagentExecutor", Executor) + monkeypatch.setattr(service_module, "SubagentStatus", FakeStatus) + monkeypatch.setattr(service_module, "get_background_task_result", read_result) + monkeypatch.setattr(service_module, "cleanup_background_task", lambda _execution_id: None) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **_kwargs: []) + service = SubagentBatchService( + repository=repository, + config=SubagentBatchesConfig(poll_interval_seconds=0.1, lease_seconds=120), + runtime_config=SubagentRuntimeConfig(max_running=1), + ) + + await service.run_once(now=service_module.datetime.now(service_module.UTC)) + await asyncio.wait_for( + asyncio.gather(*list(service._executions.values())), + timeout=1, + ) + + assert repository.finalized is not None + assert repository.finalized["result"] == "fast result" + + +@pytest.mark.asyncio +async def test_executor_admission_failure_requeues_instead_of_finalizing(monkeypatch) -> None: + result = SimpleNamespace( + status=FakeStatus.FAILED, + result=None, + error="Process-wide subagent capacity is full", + stop_reason=None, + token_usage_records=None, + admission_failure=True, + ) + + class Repository: + def __init__(self) -> None: + self.requeued = None + self.finalized = False + + async def claim_items(self, **_kwargs): + return [ + { + "id": "item-1", + "item_key": "record-1", + "prompt": "Process record 1", + "batch": { + "id": "batch-1", + "thread_id": "thread-1", + "user_id": "user-1", + "run_id": "run-1", + "execution_spec": _request().execution_spec, + }, + } + ] + + async def requeue_item_after_admission_failure(self, item_id, **kwargs): + self.requeued = (item_id, kwargs) + return True + + async def finalize_item(self, *_args, **_kwargs): + self.finalized = True + return True + + class Executor: + def __init__(self, **_kwargs) -> None: + pass + + def execute_async(self, _prompt, task_id=None): + assert task_id == "item-1" + return "execution-1" + + repository = Repository() + monkeypatch.setattr(service_module, "get_app_config", lambda: SimpleNamespace()) + monkeypatch.setattr(service_module, "resolve_subagent_model_name", lambda *_args, **_kwargs: "model-a") + monkeypatch.setattr(service_module, "SubagentExecutor", Executor) + monkeypatch.setattr(service_module, "SubagentStatus", FakeStatus) + monkeypatch.setattr(service_module, "get_background_task_result", lambda _execution_id: result) + monkeypatch.setattr(service_module, "cleanup_background_task", lambda _execution_id: None) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **_kwargs: []) + service = SubagentBatchService( + repository=repository, + config=SubagentBatchesConfig(), + runtime_config=SubagentRuntimeConfig(max_running=1), + ) + + await service.run_once(now=service_module.datetime.now(service_module.UTC)) + await asyncio.gather(*list(service._executions.values())) + + assert repository.requeued is not None + assert repository.requeued[0] == "item-1" + assert repository.finalized is False diff --git a/backend/tests/test_subagent_batches_router.py b/backend/tests/test_subagent_batches_router.py new file mode 100644 index 000000000..b15a02ad2 --- /dev/null +++ b/backend/tests/test_subagent_batches_router.py @@ -0,0 +1,126 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fastapi import HTTPException + +from app.gateway.app import create_app +from app.gateway.routers import subagent_batches + + +def _batch(**overrides): + return { + "id": "batch-1", + "user_id": "user-1", + "thread_id": "thread-1", + "status": "running", + "total_items": 2, + "counts": {"running": 1, "pending": 1}, + **overrides, + } + + +class Repository: + def __init__(self) -> None: + self.batch = _batch() + self.items = [ + {"id": "item-1", "batch_id": "batch-1", "item_key": "one", "status": "succeeded", "result": "done"}, + {"id": "item-2", "batch_id": "batch-1", "item_key": "two", "status": "failed", "error": "bad"}, + ] + self.include_result_calls = [] + + async def get_batch(self, batch_id, *, user_id): + if batch_id != self.batch["id"] or user_id != self.batch["user_id"]: + return None + return self.batch + + async def list_by_thread(self, thread_id, *, user_id, limit): + assert (thread_id, user_id, limit) == ("thread-1", "user-1", 20) + return [self.batch] + + async def list_items(self, batch_id, *, user_id, offset=0, limit=100, status=None, include_result=False): + assert batch_id == "batch-1" and user_id == "user-1" + self.include_result_calls.append(include_result) + values = self.items + if status is not None: + values = [item for item in values if item["status"] == status] + return values[offset : offset + limit] + + +def _request(repo, *, available=True, service=None): + return SimpleNamespace( + app=SimpleNamespace( + state=SimpleNamespace( + subagent_batch_repo=repo, + subagent_batch_service=service, + subagent_batches_available=available, + ) + ) + ) + + +def test_gateway_mounts_subagent_batch_routes() -> None: + paths = {route.path for route in create_app().routes} + assert "/api/threads/{thread_id}/subagent-batches" in paths + assert "/api/threads/{thread_id}/subagent-batches/{batch_id}/items" in paths + assert "/api/threads/{thread_id}/subagent-batches/{batch_id}/results.jsonl" in paths + + +@pytest.mark.asyncio +async def test_list_and_detail_are_owner_scoped(monkeypatch) -> None: + repo = Repository() + request = _request(repo) + monkeypatch.setattr(subagent_batches, "get_current_user", AsyncMock(return_value="user-1")) + + listed = await subagent_batches.list_batches.__wrapped__(thread_id="thread-1", request=request, limit=20) + detail = await subagent_batches.get_batch.__wrapped__(thread_id="thread-1", batch_id="batch-1", request=request) + + assert listed == [repo.batch] + assert detail == repo.batch + + with pytest.raises(HTTPException) as cross_thread: + await subagent_batches.get_batch.__wrapped__(thread_id="thread-2", batch_id="batch-1", request=request) + assert cross_thread.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_cancel_requires_running_worker_and_exact_owner(monkeypatch) -> None: + repo = Repository() + service = AsyncMock() + service.cancel_batch.return_value = _batch(status="cancelled") + monkeypatch.setattr(subagent_batches, "get_current_user", AsyncMock(return_value="user-1")) + + result = await subagent_batches.cancel_batch.__wrapped__( + thread_id="thread-1", + batch_id="batch-1", + request=_request(repo, service=service), + ) + assert result["status"] == "cancelled" + service.cancel_batch.assert_awaited_once_with(batch_id="batch-1", user_id="user-1") + + with pytest.raises(HTTPException) as unavailable: + await subagent_batches.cancel_batch.__wrapped__( + thread_id="thread-1", + batch_id="batch-1", + request=_request(repo, available=False, service=service), + ) + assert unavailable.value.status_code == 503 + + +@pytest.mark.asyncio +async def test_jsonl_export_streams_item_results(monkeypatch) -> None: + repo = Repository() + monkeypatch.setattr(subagent_batches, "get_current_user", AsyncMock(return_value="user-1")) + + response = await subagent_batches.export_batch_results.__wrapped__( + thread_id="thread-1", + batch_id="batch-1", + request=_request(repo), + ) + payload = b"".join([chunk async for chunk in response.body_iterator]).decode() + rows = [json.loads(line) for line in payload.splitlines()] + + assert [row["id"] for row in rows] == ["item-1", "item-2"] + assert rows[0]["result"] == "done" + assert repo.include_result_calls and all(repo.include_result_calls) diff --git a/backend/tests/test_subagent_capacity_config.py b/backend/tests/test_subagent_capacity_config.py new file mode 100644 index 000000000..b0e0388fd --- /dev/null +++ b/backend/tests/test_subagent_capacity_config.py @@ -0,0 +1,58 @@ +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from deerflow.config.app_config import AppConfig +from deerflow.config.reload_boundary import STARTUP_ONLY_FIELDS, STARTUP_ONLY_PREFIX +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.config.subagents_config import effective_subagent_concurrency + + +def test_subagent_runtime_defaults_are_safe_and_bounded() -> None: + config = SubagentRuntimeConfig() + assert config.max_running == 3 + assert config.max_queued == 64 + assert config.admission_policy == "queue" + assert config.queue_timeout_seconds == 300 + + with pytest.raises(ValidationError): + SubagentRuntimeConfig(max_running=0) + with pytest.raises(ValidationError): + SubagentRuntimeConfig(max_running=65) + with pytest.raises(ValidationError): + SubagentRuntimeConfig(max_queued=10_001) + + +def test_subagent_batch_defaults_separate_total_live_and_running() -> None: + config = SubagentBatchesConfig() + assert config.enabled is False + assert config.max_items_per_batch == 5_000 + assert config.default_max_live_items == 100 + assert config.default_max_running_items == 3 + + with pytest.raises(ValidationError): + SubagentBatchesConfig(default_max_live_items=5, default_max_running_items=6) + with pytest.raises(ValidationError): + SubagentBatchesConfig(max_live_items_per_batch=4, default_max_live_items=5) + + +def test_subagent_capacity_sections_are_startup_only() -> None: + for name in ("subagent_runtime", "subagent_batches"): + assert name in STARTUP_ONLY_FIELDS + description = AppConfig.model_fields[name].description or "" + assert description.startswith(STARTUP_ONLY_PREFIX) + + +def test_ordinary_task_limit_never_advertises_more_than_real_process_slots() -> None: + config = SimpleNamespace(subagent_runtime=SubagentRuntimeConfig(max_running=8)) + assert effective_subagent_concurrency(None, config) == 8 + assert effective_subagent_concurrency(4, config) == 4 + assert effective_subagent_concurrency(50, config) == 8 + + +def test_ordinary_task_limit_uses_frozen_execution_capacity_after_reload() -> None: + reloaded_config = SimpleNamespace(subagent_runtime=SubagentRuntimeConfig(max_running=12)) + assert effective_subagent_concurrency(None, reloaded_config, execution_capacity=3) == 3 + assert effective_subagent_concurrency(10, reloaded_config, execution_capacity=3) == 3 diff --git a/backend/tests/test_subagent_execution_capacity.py b/backend/tests/test_subagent_execution_capacity.py new file mode 100644 index 000000000..70e14947a --- /dev/null +++ b/backend/tests/test_subagent_execution_capacity.py @@ -0,0 +1,130 @@ +import asyncio + +import pytest + +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.subagents.capacity import ( + SubagentCapacityRejected, + SubagentCapacityTimeout, + configure_subagent_execution_capacity, + configured_subagent_max_running, + get_subagent_execution_capacity, +) + + +@pytest.mark.asyncio +async def test_capacity_queues_without_starting_more_than_configured_slots() -> None: + configure_subagent_execution_capacity(SubagentRuntimeConfig(max_running=1, max_queued=2, queue_timeout_seconds=5)) + capacity = get_subagent_execution_capacity() + release = asyncio.Event() + started: list[str] = [] + + async def work(name: str) -> None: + async with capacity.slot(): + started.append(name) + if name == "first": + await release.wait() + + first = asyncio.create_task(work("first")) + await asyncio.sleep(0) + second = asyncio.create_task(work("second")) + await asyncio.sleep(0) + + assert started == ["first"] + assert capacity.snapshot().running == 1 + assert capacity.snapshot().queued == 1 + + release.set() + await asyncio.gather(first, second) + assert started == ["first", "second"] + assert capacity.snapshot().running == 0 + + +@pytest.mark.asyncio +async def test_capacity_rejects_immediately_when_configured() -> None: + configure_subagent_execution_capacity(SubagentRuntimeConfig(max_running=1, max_queued=10, admission_policy="reject")) + capacity = get_subagent_execution_capacity() + async with capacity.slot(): + with pytest.raises(SubagentCapacityRejected, match="capacity is full"): + async with capacity.slot(): + raise AssertionError("unreachable") + + +@pytest.mark.asyncio +async def test_capacity_enforces_queue_bound() -> None: + configure_subagent_execution_capacity(SubagentRuntimeConfig(max_running=1, max_queued=1, queue_timeout_seconds=5)) + capacity = get_subagent_execution_capacity() + release = asyncio.Event() + + async def holder() -> None: + async with capacity.slot(): + await release.wait() + + first = asyncio.create_task(holder()) + await asyncio.sleep(0) + queued = asyncio.create_task(holder()) + await asyncio.sleep(0) + with pytest.raises(SubagentCapacityRejected, match="1 queued"): + async with capacity.slot(): + raise AssertionError("unreachable") + release.set() + await asyncio.gather(first, queued) + + +@pytest.mark.asyncio +async def test_capacity_timeout_removes_waiter_and_releases_slot() -> None: + configure_subagent_execution_capacity(SubagentRuntimeConfig(max_running=1, max_queued=1, queue_timeout_seconds=1)) + capacity = get_subagent_execution_capacity() + async with capacity.slot(): + with pytest.raises(SubagentCapacityTimeout, match="Timed out"): + async with capacity.slot(): + raise AssertionError("unreachable") + assert capacity.snapshot().queued == 0 + assert capacity.snapshot().running == 0 + + +@pytest.mark.asyncio +async def test_capacity_cancelled_waiter_does_not_leak_queue_or_slot() -> None: + configure_subagent_execution_capacity(SubagentRuntimeConfig(max_running=1, max_queued=2, queue_timeout_seconds=5)) + capacity = get_subagent_execution_capacity() + release = asyncio.Event() + + async def holder() -> None: + async with capacity.slot(): + await release.wait() + + first = asyncio.create_task(holder()) + await asyncio.sleep(0) + waiting = asyncio.create_task(holder()) + await asyncio.sleep(0) + waiting.cancel() + with pytest.raises(asyncio.CancelledError): + await waiting + assert capacity.snapshot().queued == 0 + release.set() + await first + assert capacity.snapshot().running == 0 + + +@pytest.mark.asyncio +async def test_installing_same_startup_config_does_not_reset_live_capacity() -> None: + config = SubagentRuntimeConfig(max_running=1, max_queued=2, queue_timeout_seconds=5) + configure_subagent_execution_capacity(config) + capacity = get_subagent_execution_capacity() + + async with capacity.slot(): + configure_subagent_execution_capacity(config) + assert get_subagent_execution_capacity() is capacity + assert capacity.snapshot().running == 1 + + assert configured_subagent_max_running() == 1 + + +@pytest.mark.asyncio +async def test_installing_different_config_while_active_is_rejected() -> None: + configure_subagent_execution_capacity(SubagentRuntimeConfig(max_running=1)) + capacity = get_subagent_execution_capacity() + + async with capacity.slot(): + with pytest.raises(RuntimeError, match="while executions are active"): + configure_subagent_execution_capacity(SubagentRuntimeConfig(max_running=2)) diff --git a/backend/tests/test_subagent_executor.py b/backend/tests/test_subagent_executor.py index 153a9017c..1ec7f8ed7 100644 --- a/backend/tests/test_subagent_executor.py +++ b/backend/tests/test_subagent_executor.py @@ -18,6 +18,7 @@ import asyncio import importlib import sys import threading +import time from datetime import datetime from importlib.metadata import version as package_version from pathlib import Path @@ -28,6 +29,7 @@ import pytest from packaging.version import Version from deerflow.skills.types import Skill +from deerflow.subagents.capacity import SubagentCapacityRejected # Module names that need to be mocked to break circular imports _MOCKED_MODULE_NAMES = [ @@ -861,6 +863,35 @@ class TestAsyncExecutionPath: assert result.started_at is not None assert result.completed_at is not None + @pytest.mark.anyio + async def test_aexecute_marks_capacity_rejection_as_admission_failure(self, classes, base_config): + SubagentExecutor = classes["SubagentExecutor"] + SubagentStatus = classes["SubagentStatus"] + + class RejectingSlot: + async def __aenter__(self): + raise SubagentCapacityRejected("Process-wide subagent capacity is full") + + async def __aexit__(self, exc_type, exc, traceback): + return False + + class RejectingCapacity: + def slot(self): + return RejectingSlot() + + executor = SubagentExecutor( + config=base_config, + tools=[], + thread_id="test-thread", + execution_capacity=RejectingCapacity(), + ) + + result = await executor._aexecute("Do something") + + assert result.status == SubagentStatus.FAILED + assert result.admission_failure is True + assert "capacity is full" in result.error + @pytest.mark.anyio async def test_aexecute_marks_structured_llm_error_fallback_as_failed(self, classes, base_config, mock_agent, msg): """A handled provider error is still a failed delegated task. @@ -2305,10 +2336,10 @@ class TestCooperativeCancellation: SubagentResult = classes["SubagentResult"] SubagentStatus = classes["SubagentStatus"] - def run_inline(fn, *args, **kwargs): + def run_coroutine(context, coroutine_factory): future = concurrent.futures.Future() try: - future.set_result(fn(*args, **kwargs)) + future.set_result(context.run(lambda: asyncio.run(coroutine_factory()))) except Exception as exc: future.set_exception(exc) return future @@ -2332,7 +2363,7 @@ class TestCooperativeCancellation: ) with ( - patch.object(executor_module._scheduler_pool, "submit", side_effect=run_inline), + patch.object(executor_module, "_submit_to_isolated_loop_in_context", side_effect=run_coroutine), patch.object(executor, "_aexecute", side_effect=fake_aexecute), patch.object(executor, "execute", side_effect=AssertionError("execute() should not be called by execute_async")), ): @@ -2350,16 +2381,11 @@ class TestCooperativeCancellation: SubagentExecutor = classes["SubagentExecutor"] SubagentStatus = classes["SubagentStatus"] - scheduled: list = [] - def capture_submission(fn, *args, **kwargs): - scheduled.append(lambda: fn(*args, **kwargs)) - return concurrent.futures.Future() - - def run_coroutine(_context, coroutine_factory): + def run_coroutine(context, coroutine_factory): future = concurrent.futures.Future() try: - future.set_result(asyncio.run(coroutine_factory())) + future.set_result(context.run(lambda: asyncio.run(coroutine_factory()))) except Exception as exc: future.set_exception(exc) return future @@ -2390,7 +2416,6 @@ class TestCooperativeCancellation: ) with ( - patch.object(executor_module._scheduler_pool, "submit", side_effect=capture_submission), patch.object(executor_module, "_submit_to_isolated_loop_in_context", side_effect=run_coroutine), patch.object(executor_a, "_aexecute", side_effect=complete_a), patch.object(executor_b, "_aexecute", side_effect=complete_b), @@ -2402,9 +2427,6 @@ class TestCooperativeCancellation: assert executor_module._background_tasks[execution_a].trace_id == "trace-a" assert executor_module._background_tasks[execution_b].trace_id == "trace-b" - for run_task in scheduled: - run_task() - assert executor_module._background_tasks[execution_a].result == "done-a" assert executor_module._background_tasks[execution_b].result == "done-b" @@ -2446,19 +2468,24 @@ class TestCooperativeCancellation: trace_id="test-trace", ) - scheduler = concurrent.futures.ThreadPoolExecutor(max_workers=1) + def run_coroutine(context, coroutine_factory): + future = concurrent.futures.Future() + try: + future.set_result(context.run(lambda: asyncio.run(coroutine_factory()))) + except Exception as exc: + future.set_exception(exc) + return future + token = set_current_user(SimpleNamespace(id="alice")) try: with ( - patch.object(executor_module, "_scheduler_pool", scheduler), + patch.object(executor_module, "_submit_to_isolated_loop_in_context", side_effect=run_coroutine), patch.object(executor, "_aexecute", side_effect=fake_aexecute), patch.object(executor, "execute", side_effect=AssertionError("execute() should not be called by execute_async")), ): task_id = executor.execute_async("Task") - executor_module._scheduler_pool.shutdown(wait=True) finally: reset_current_user(token) - scheduler.shutdown(wait=False, cancel_futures=True) result = executor_module._background_tasks.get(task_id) assert result is not None @@ -2496,7 +2523,14 @@ class TestCooperativeCancellation: trace_id="test-trace", ) - scheduler = concurrent.futures.ThreadPoolExecutor(max_workers=1) + def run_coroutine(context, coroutine_factory): + future = concurrent.futures.Future() + try: + future.set_result(context.run(lambda: asyncio.run(coroutine_factory()))) + except Exception as exc: + future.set_exception(exc) + return future + token = var_child_runnable_config.set( { "callbacks": [parent_callback, stream_callback], @@ -2508,14 +2542,12 @@ class TestCooperativeCancellation: ) try: with ( - patch.object(executor_module, "_scheduler_pool", scheduler), + patch.object(executor_module, "_submit_to_isolated_loop_in_context", side_effect=run_coroutine), patch.object(executor, "_aexecute", side_effect=fake_aexecute), ): executor.execute_async("Task") - scheduler.shutdown(wait=True) finally: var_child_runnable_config.reset(token) - scheduler.shutdown(wait=False, cancel_futures=True) assert child_callback in observed["callbacks"] assert parent_callback not in observed["callbacks"] @@ -2570,7 +2602,6 @@ class TestCooperativeCancellation: # Synchronisation primitives execute_entered = threading.Event() # signals that _aexecute() has started - run_task_done = threading.Event() # signals that run_task() has finished # A blocking _aexecute() replacement so we control the timing exactly. async def blocking_aexecute(task, result_holder=None): @@ -2584,19 +2615,7 @@ class TestCooperativeCancellation: trace_id="test-trace", ) - # Wrap _scheduler_pool.submit so we know when run_task finishes - original_scheduler_submit = executor_module._scheduler_pool.submit - - def tracked_submit(fn, *args, **kwargs): - def wrapper(): - try: - fn(*args, **kwargs) - finally: - run_task_done.set() - - return original_scheduler_submit(wrapper) - - with patch.object(executor, "_aexecute", side_effect=blocking_aexecute), patch.object(executor_module._scheduler_pool, "submit", tracked_submit): + with patch.object(executor, "_aexecute", side_effect=blocking_aexecute): task_id = executor.execute_async("Task") # Wait until _aexecute() is entered on the persistent loop. @@ -2609,9 +2628,10 @@ class TestCooperativeCancellation: executor_module._background_tasks[task_id].error = "Cancelled by user" executor_module._background_tasks[task_id].completed_at = datetime.now() - # Wait for run_task to finish — the FuturesTimeoutError handler has - # now executed and (should have) left CANCELLED intact. - assert run_task_done.wait(timeout=5), "run_task() did not finish" + deadline = time.monotonic() + 5 + while task_id in executor_module._background_futures and time.monotonic() < deadline: + time.sleep(0.01) + assert task_id not in executor_module._background_futures, "background coroutine did not finish" result = executor_module._background_tasks.get(task_id) assert result is not None diff --git a/backend/tests/test_subagent_limit_middleware.py b/backend/tests/test_subagent_limit_middleware.py index 0d336fc8f..2940824b0 100644 --- a/backend/tests/test_subagent_limit_middleware.py +++ b/backend/tests/test_subagent_limit_middleware.py @@ -57,7 +57,7 @@ class TestClampSubagentLimit: # Both consumers (SubagentLimitMiddleware.__init__ and the prompt path) # share this floor via clamp_subagent_concurrency in subagents_config.py. assert MIN_SUBAGENT_LIMIT == 1 - assert MAX_SUBAGENT_LIMIT == 4 + assert MAX_SUBAGENT_LIMIT == 64 def test_below_min_clamped_to_one(self): assert _clamp_subagent_limit(0) == 1 @@ -67,9 +67,10 @@ class TestClampSubagentLimit: # Previously 1 clamped up to 2; it must now pass through as 1. assert _clamp_subagent_limit(1) == 1 - def test_above_max_clamped_to_four(self): - assert _clamp_subagent_limit(5) == 4 - assert _clamp_subagent_limit(10) == MAX_SUBAGENT_LIMIT + def test_above_hard_max_clamped(self): + assert _clamp_subagent_limit(5) == 5 + assert _clamp_subagent_limit(10) == 10 + assert _clamp_subagent_limit(65) == MAX_SUBAGENT_LIMIT assert _clamp_subagent_limit(100) == MAX_SUBAGENT_LIMIT def test_within_range_unchanged(self): @@ -88,7 +89,7 @@ class TestSubagentLimitMiddlewareInit: mw = SubagentLimitMiddleware(max_concurrent=1) assert mw.max_concurrent == MIN_SUBAGENT_LIMIT - mw = SubagentLimitMiddleware(max_concurrent=10) + mw = SubagentLimitMiddleware(max_concurrent=100) assert mw.max_concurrent == MAX_SUBAGENT_LIMIT diff --git a/backend/tests/test_subagent_runtime.py b/backend/tests/test_subagent_runtime.py new file mode 100644 index 000000000..a2d304326 --- /dev/null +++ b/backend/tests/test_subagent_runtime.py @@ -0,0 +1,77 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.subagents import SubagentRuntime + + +def test_runtime_rejects_batch_repository_without_enabled_batch_config() -> None: + with pytest.raises(ValueError, match="batch_config.enabled"): + SubagentRuntime( + SubagentRuntimeConfig(), + batch_repository=MagicMock(), + batch_config=SubagentBatchesConfig(enabled=False), + ) + + +def test_runtime_rejects_batch_repository_without_app_config_snapshot() -> None: + with pytest.raises(ValueError, match="explicit app_config snapshot"): + SubagentRuntime( + SubagentRuntimeConfig(), + batch_repository=MagicMock(), + batch_config=SubagentBatchesConfig(enabled=True), + ) + + +def test_runtime_uses_one_caller_owned_app_config_snapshot() -> None: + app_config = SimpleNamespace( + subagent_runtime=SubagentRuntimeConfig(max_running=11), + subagents=SimpleNamespace(max_total_per_run=14), + subagent_batches=SubagentBatchesConfig(enabled=False), + ) + + runtime = SubagentRuntime.from_app_config(app_config) + + assert runtime.config.max_running == 11 + assert runtime.max_total_per_run == 14 + assert runtime.app_config is app_config + assert runtime.batch_submitter is None + + +@pytest.mark.asyncio +async def test_runtime_owns_batch_worker_lifecycle_and_shared_capacity() -> None: + service = MagicMock() + service.start = AsyncMock() + service.stop = AsyncMock() + repository = MagicMock() + app_config = MagicMock() + + with patch( + "deerflow.subagents.batch_service.SubagentBatchService", + return_value=service, + ) as service_type: + runtime = SubagentRuntime( + SubagentRuntimeConfig(max_running=9), + batch_repository=repository, + batch_config=SubagentBatchesConfig(enabled=True), + app_config=app_config, + ) + assert runtime.batch_submitter is None + + async with runtime: + assert runtime.batch_submitter is service + + assert runtime.batch_submitter is None + + service_type.assert_called_once_with( + repository=repository, + config=runtime.batch_config, + runtime_config=runtime.config, + app_config=app_config, + execution_capacity=runtime.execution_capacity, + ) + service.start.assert_awaited_once_with() + service.stop.assert_awaited_once_with() diff --git a/backend/tests/test_task_tool_core_logic.py b/backend/tests/test_task_tool_core_logic.py index 8a63e8f1f..29322d4b4 100644 --- a/backend/tests/test_task_tool_core_logic.py +++ b/backend/tests/test_task_tool_core_logic.py @@ -13,7 +13,9 @@ import pytest from langchain_core.messages import ToolMessage from langgraph.types import Command +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig from deerflow.sandbox.security import LOCAL_BASH_SUBAGENT_DISABLED_MESSAGE +from deerflow.subagents.capacity import SubagentExecutionCapacity from deerflow.subagents.config import SubagentConfig from deerflow.subagents.status_contract import ( SUBAGENT_ERROR_KEY, @@ -373,6 +375,49 @@ def test_task_tool_omits_extensions_without_a_run_snapshot(monkeypatch): assert "extensions" not in captured["executor_kwargs"] +def test_bound_task_tool_forwards_explicit_execution_capacity(monkeypatch): + runtime = _make_runtime() + captured = {} + capacity = SubagentExecutionCapacity(SubagentRuntimeConfig(max_running=7)) + app_config = object() + + class DummyExecutor: + def __init__(self, **kwargs): + captured["executor_kwargs"] = kwargs + + def execute_async(self, prompt, task_id=None): + return task_id or "generated-task-id" + + monkeypatch.setattr(task_tool_module, "SubagentStatus", FakeSubagentStatus) + monkeypatch.setattr(task_tool_module, "SubagentExecutor", DummyExecutor) + monkeypatch.setattr(task_tool_module, "get_available_subagent_names", lambda **_kwargs: ["general-purpose"]) + monkeypatch.setattr(task_tool_module, "get_subagent_config", lambda _name, **_kwargs: _make_subagent_config()) + monkeypatch.setattr( + task_tool_module, + "get_background_task_result", + lambda _: _make_result(FakeSubagentStatus.COMPLETED, result="done"), + ) + monkeypatch.setattr(task_tool_module, "get_stream_writer", lambda: lambda _event: None) + monkeypatch.setattr(task_tool_module.asyncio, "sleep", _no_sleep) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: []) + + bound_tool = task_tool_module.bind_task_tool(capacity, app_config=app_config) + coroutine = getattr(bound_tool, "coroutine", None) + assert coroutine is not None + asyncio.run( + coroutine( + runtime=runtime, + description="test", + prompt="p", + subagent_type="general-purpose", + tool_call_id="tc-capacity", + ) + ) + + assert captured["executor_kwargs"]["execution_capacity"] is capacity + assert captured["executor_kwargs"]["app_config"] is app_config + + def test_task_tool_forwards_channel_user_id_to_executor(monkeypatch): """The IM-channel sender identity must survive delegation: in group chats one thread serves many senders, so a subagent's bash commands need the diff --git a/config.example.yaml b/config.example.yaml index 2b64aa9f9..eeb611395 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -15,7 +15,7 @@ # ============================================================================ # Bump this number when the config schema changes. # Run `make config-upgrade` to merge new fields into your local config.yaml. -config_version: 35 +config_version: 36 # ============================================================================ # Logging @@ -1437,6 +1437,15 @@ sandbox: # Configure timeouts for subagent execution # Subagents are background workers delegated tasks by the lead agent +# Process-wide execution capacity. These fields are restart-required and are +# shared by ordinary `task` calls and durable batches. Waiting work is async; +# queued subagents do not occupy a scheduler thread. +subagent_runtime: + max_running: 3 + max_queued: 64 + admission_policy: queue # queue or reject when all slots are occupied + queue_timeout_seconds: 300 + # subagents: # # Default timeout (seconds) for built-in subagents (default: 1800 = 30 min). # # Custom agents use their own timeout_seconds (default 900) unless overridden. @@ -1508,6 +1517,25 @@ sandbox: # # Set `model` to use a different model (e.g., a local Ollama model for cost savings). # # The model name must match a name defined in the `models:` section above. +# Durable native-subagent batches. Disabled by default because enabling it can +# materially increase model usage and requires database.backend sqlite/postgres. +# The three limits are intentionally separate: +# - total: all persisted items in one batch +# - live: pending work admitted as queued/running at one time +# - running: items from one batch allowed to hold real execution slots +subagent_batches: + enabled: false + poll_interval_seconds: 1 + lease_seconds: 120 + max_items_per_batch: 5000 + default_max_live_items: 100 + max_live_items_per_batch: 1000 + default_max_running_items: 3 + max_running_items_per_batch: 64 + max_attempts: 3 + max_result_chars: 100000 + result_preview_max_chars: 2000 + # ============================================================================ # Tool Result Verification # ============================================================================ diff --git a/deploy/helm/deer-flow/README.md b/deploy/helm/deer-flow/README.md index d907e3eda..684251ae7 100644 --- a/deploy/helm/deer-flow/README.md +++ b/deploy/helm/deer-flow/README.md @@ -124,7 +124,7 @@ they resolve from the `secrets` map): ```yaml config: | - config_version: 35 + config_version: 36 models: - name: gpt-4 use: langchain_openai:ChatOpenAI diff --git a/deploy/helm/deer-flow/values.yaml b/deploy/helm/deer-flow/values.yaml index 70a106c83..940f76ebc 100644 --- a/deploy/helm/deer-flow/values.yaml +++ b/deploy/helm/deer-flow/values.yaml @@ -243,7 +243,7 @@ ingress: # -- DeerFlow config.yaml content. Secrets MUST stay as $VAR references — never # inline literal secret values here. The default enables provisioner sandbox. config: | - config_version: 35 + config_version: 36 log_level: info models: [] diff --git a/docs/plans/2026-08-24-subagent-batch-capacity-implementation.md b/docs/plans/2026-08-24-subagent-batch-capacity-implementation.md new file mode 100644 index 000000000..7195b3fd8 --- /dev/null +++ b/docs/plans/2026-08-24-subagent-batch-capacity-implementation.md @@ -0,0 +1,260 @@ +# Subagent Capacity and Durable Batch Execution + +## Status + +Implemented in the `feat/subagent-batch-capacity` worktree. The implementation keeps ordinary delegation bounded, makes its advertised and real concurrency consistent, and adds an explicit durable batch path for large collections of independent items. + +This is an implementation document, not an RFC. It describes the behavior and operational contract delivered by the code. + +## Problem + +DeerFlow previously exposed three different concepts as if they were one limit: + +- `max_concurrent_subagents` controlled how many `task` calls the lead agent could emit in one model response. +- `subagents.max_total_per_run` limited cumulative ordinary delegations in one lead-agent run (default `6`, hard range `1`–`50`). +- the executor had a fixed process-local capacity of three native subagents. + +Changing only the model-visible limit did not change the executor, while raising only the executor capacity could let the model and middleware make different promises. Neither change makes a request containing thousands of independent records durable: ordinary `task` calls still depend on the lead run, keep their task state in process memory, and return their results to the lead context. + +The implementation therefore delivers both the shared capacity foundation and a separate durable batch execution mode. + +## Stage 1: one process-wide execution capacity + +### Startup-only configuration + +`subagent_runtime` is loaded once during Gateway startup: + +```yaml +subagent_runtime: + max_running: 3 + max_queued: 64 + admission_policy: queue # queue or reject + queue_timeout_seconds: 300 +``` + +The schema enforces bounded values. `max_running` accepts `1`–`64`; `max_queued` accepts `0`–`10000`. The default remains three, so existing deployments do not increase model, sandbox, or database load after upgrading. + +Configuration edits require a Gateway restart. Hot reload must not change a live process's semaphore while work owns slots. + +### One value across prompt, middleware, and executor + +For an ordinary lead-agent run, the effective `max_concurrent_subagents` is: + +```text +min(requested task-call concurrency, subagent_runtime.max_running, hard safety maximum) +``` + +The same resolved value is used by: + +- the generated lead-agent prompt on Gateway/embedded-client paths; +- `SubagentLimitMiddleware` tool-call truncation; +- Gateway, embedded client, and direct `create_deerflow_agent` construction; and +- the real process-wide execution controller. + +The hard schema maximum is now `64`, but that number is not an instruction to run 64 workers. A deployment must explicitly raise `subagent_runtime.max_running`, and every ordinary request remains capped by that real process capacity. + +### Admission behavior + +All native subagents, including ordinary `task` calls and durable batch items, acquire the same asynchronous FIFO execution slot. + +- A slot holder is counted as running. +- A waiter owns no scheduler thread. +- `queue` admits waiters up to `max_queued` and applies `queue_timeout_seconds`. +- `reject` fails immediately while saturated. +- cancellation and timeout remove the waiter and cannot leak a slot. +- task status remains pending until a real slot has been acquired. +- a durable item rejected or timed out at this admission boundary returns to + `queued` without consuming an execution attempt. + +The previous scheduler thread pool was removed. Background execution submits a coroutine directly to the existing persistent isolated event loop; increasing the queue no longer creates the same number of long-lived blocked threads. + +### Direct factory ownership + +Gateway and `DeerFlowClient` install their startup dependencies for callers. A direct `create_deerflow_agent(...)` integration instead passes one explicit `SubagentRuntime` to every graph that must share a capacity boundary. The runtime snapshots `SubagentRuntimeConfig`, the ordinary `max_total_per_run`, one `SubagentExecutionCapacity`, an optional caller-owned `AppConfig` used for subagent registry/model/tool resolution, and an optional batch submitter or owned durable worker. This keeps the factory pure-argument: it does not load `config.yaml`, create a SQL repository, or silently start background work. + +If the runtime owns a batch repository, the caller must start it before graph construction and stop it at application shutdown; `async with runtime` provides that lifecycle. Graph construction fails closed while the configured worker is stopped, so a graph cannot advertise batch tools backed by no worker. The bound ordinary and batch tools use that runtime's exact capacity/submitter even if another Gateway runtime exists in the same Python process. + +The direct factory accepts a caller-provided `system_prompt` and does not render DeerFlow's lead prompt. Such callers own any model-visible wording about delegation capacity; the default middleware still enforces the runtime's real limit regardless of that wording. The direct factory path also does not mount Gateway HTTP routes or the Web UI panel. Callers that need owner-scoped item browsing or export must expose those application surfaces themselves. Repository-free runtimes support ordinary delegation without an asynchronous lifecycle. + +### Ordinary runaway protection remains + +`subagents.max_total_per_run` still protects the iterative lead-agent loop. Its default remains `6` and its hard range remains `1`–`50`. + +This limit is not batch capacity. It prevents an ordinary conversational run from repeatedly emitting legal-sized `task` groups at successive planning checkpoints. Removing it or setting it to thousands would make accidental recursive or unproductive delegation much more expensive without adding persistence, recovery, or result collection. + +## Stage 2: explicit durable batch mode + +### Mode selection is explicit + +A job is a batch only when the lead agent or another authorized caller invokes `batch_task`. DeerFlow does not infer batch mode from prompt wording, item count, or frontend state. + +The tool is exposed only when all of the following are true: + +- native subagents are enabled for the lead agent; +- `subagent_batches.enabled` was true at Gateway startup; and +- SQL persistence is available. + +Ordinary `task` retains its existing wait-for-result conversational semantics and per-run ledger. `batch_task` returns a durable batch receipt immediately and tells the lead agent not to re-submit those items as ordinary tasks. Compact progress is available through `batch_status`; results are read through the owner-scoped API or JSONL export instead of being appended wholesale to the model context. + +### Separate total, live, and running limits + +```yaml +subagent_batches: + enabled: false + poll_interval_seconds: 1 + lease_seconds: 120 + max_items_per_batch: 5000 + default_max_live_items: 100 + max_live_items_per_batch: 1000 + default_max_running_items: 3 + max_running_items_per_batch: 64 + max_attempts: 3 + max_result_chars: 100000 + result_preview_max_chars: 2000 +``` + +The three capacity dimensions are intentionally independent: + +| Dimension | Meaning | Enforcement | +| --- | --- | --- | +| Total | All durable items belonging to the batch | Submission is rejected above `max_items_per_batch`. | +| Live | Items promoted from durable pending storage into queued, leased, or running work | The repository promotes only enough pending rows to fill `max_live_items`. | +| Running | Maximum execution admissions owned by one batch across workers | Database claiming counts leased and running rows conservatively; real execution also requires a process-wide slot. | + +`max_running_items` is not clamped to one process's capacity. In a multi-worker Postgres deployment, several processes can contribute execution slots while the database enforces the batch-wide ceiling. Within every process, ordinary and batch work still share `subagent_runtime.max_running`. + +### Durable state model + +The migration creates `subagent_batches` and `subagent_batch_items`. + +Batch states: + +```text +queued -> running <-> paused -> completed + \-------> cancelled +``` + +Item states: + +```text +pending -> queued -> leased -> running -> succeeded + ^ | |----> failed + | | \----> cancelled + \----------/ retry while attempts remain +``` + +- `pending` is durable backlog outside the live window. +- `queued` is admitted batch work not owned by a worker. +- `leased` means one worker owns recovery responsibility but the native executor has not necessarily acquired a process slot. +- `running` is written only after the executor reports real execution. +- terminal rows retain bounded result, preview, error, model, stop reason, and aggregate token usage. + +### Recovery and delivery semantics + +Workers claim rows using database locks and a lease owner. A worker renews the lease while an item is leased or running. If the process exits without finalizing: + +1. the lease expires; +2. another scheduler pass returns the same item row to the queue; +3. the attempt counter advances; and +4. the item reaches terminal failure when `max_attempts` is exhausted. + +Gateway shutdown cancels local native executions but intentionally does not falsely finalize their durable rows; the expired lease is the recovery handoff. + +User cancellation is different from process shutdown: it atomically marks every +nonterminal item `cancelled`, clears active leases, and fences late completion +writes from workers that were already running. A terminal batch is `failed` when +all finished work failed and no item succeeded; mixed success/failure is +`completed`, preserving item-level failure counts for partial-result consumers. + +Submission is idempotent per `(user_id, submission_key)`, where model submissions use the stable `run_id:tool_call_id` identity. Item keys must be unique within a batch and survive retries. + +Execution is **at least once**, not exactly once. An external side effect can complete immediately before a worker crashes and before DeerFlow commits the result. Batch items therefore must be read-only or use their stable item key as an idempotency key at the external system. + +### Authorization snapshot + +Batch submission validates the requested subagent against the caller's effective allowlist. The durable execution specification records the selected subagent definition, parent model, tool groups, skill intersection, role, channel identity, and authorization attributes needed to reconstruct the same delegated execution boundary after restart. Subagents cannot recursively enable subagent tools. + +### Owner-scoped HTTP API + +The Gateway exposes: + +```text +GET /api/threads/{thread_id}/subagent-batches +GET /api/threads/{thread_id}/subagent-batches/{batch_id} +GET /api/threads/{thread_id}/subagent-batches/{batch_id}/items +POST /api/threads/{thread_id}/subagent-batches/{batch_id}/pause +POST /api/threads/{thread_id}/subagent-batches/{batch_id}/resume +POST /api/threads/{thread_id}/subagent-batches/{batch_id}/cancel +POST /api/threads/{thread_id}/subagent-batches/{batch_id}/items/{item_id}/retry +GET /api/threads/{thread_id}/subagent-batches/{batch_id}/results.jsonl +``` + +Every lookup uses both authenticated owner and thread scope. Owner-facing batch +responses are explicit projections and never expose `execution_spec`, submission +identity, authorization context, prompts, lease internals, or full result text. +Paged item reads return bounded previews; only JSONL export explicitly includes +the stored full result. Read and export remain possible for historical batches; +live cancellation requires the startup batch worker to be available. + +### Frontend behavior + +The frontend does not decide whether a prompt is “Swarm-like.” It reads separate SQL-repository and worker-runtime capabilities from `/api/features`. A running worker exposes the batch panel immediately; when the worker is stopped or disabled, the panel remains available in read-only mode only for threads with durable history. This preserves item inspection and JSONL export without exposing an unused panel on deployments that have never enabled batches. + +The panel provides: + +- active-batch count and progress; +- total, live, running, failed, and terminal counts; +- pause, resume, and cancel controls; +- item status, result preview, error, and failed-item retry; and +- JSONL result export. + +Worker-dependent mutations are disabled while the worker is unavailable. Progress rendering clamps malformed persisted totals to a bounded `0`–`100` percentage so an invalid or manually edited row cannot pass `NaN`/`Infinity` into the UI primitive. + +The panel fetches one bounded item page at a time and exposes an explicit +load-more control until the final page. Large result sets therefore stay outside +the React tree and model transcript until requested, while JSONL remains the +full-result export path. + +## Why this matches the useful part of OpenClaw's design + +OpenClaw does not treat unrestricted ordinary delegation as its large fan-out solution. Its ordinary subagent path has a global concurrent lane and a per-agent active-child limit. Its explicit opt-in Swarm path separately configures: + +- `maxConcurrent` (running collectors); +- `maxChildrenPerGroup` (live collectors); and +- `maxTotalPerGroup` (lifetime runaway backstop). + +Accepted collectors above concurrency queue FIFO inside the global subagent lane. This is the same important separation used here: explicit mode selection plus total, live, batch-running, and process-running boundaries. DeerFlow additionally persists each item and lease because issue #4993 requires long-running bulk work to survive Gateway restart rather than only organizing a conversational fan-out. + +Comparison was verified against OpenClaw commit `65bcdf2f`; future OpenClaw behavior may change. + +## Operational limits + +This implementation makes a 5,000-item batch representable and recoverable. It does not promise that 5,000 agents start simultaneously or finish within a fixed wall-clock target. + +End-to-end throughput remains bounded by: + +- model-provider RPM and TPM; +- DeerFlow's LLM limiter and provider retry behavior; +- sandbox CPU, memory, startup latency, and external tool quotas; +- SQL connection pool and write throughput; +- result size; and +- the number of Gateway processes and their `subagent_runtime.max_running` values. + +Operators should raise capacity gradually, observe provider throttling and resource saturation, and keep batch items independent. A value such as `max_running: 500` is rejected by the schema; scaling to hundreds of real concurrent items requires multiple appropriately provisioned workers and a shared Postgres database. + +## Related work + +- [#4993](https://github.com/bytedance/deer-flow/issues/4993) — bulk subagent capacity request addressed by this implementation. +- [#3099](https://github.com/bytedance/deer-flow/issues/3099) and [PR #3415](https://github.com/bytedance/deer-flow/pull/3415) — model-visible and executor concurrency inconsistency. +- [#2670](https://github.com/bytedance/deer-flow/issues/2670), [#1319](https://github.com/bytedance/deer-flow/issues/1319), and [#1339](https://github.com/bytedance/deer-flow/issues/1339) — concurrency, queueing, and subagent execution pressure. +- [#3857](https://github.com/bytedance/deer-flow/issues/3857), [#4290](https://github.com/bytedance/deer-flow/issues/4290), and [#4560](https://github.com/bytedance/deer-flow/issues/4560) — runaway protection, token/cost bounds, and long-running execution safety. +- [#3948](https://github.com/bytedance/deer-flow/issues/3948) and [#1223](https://github.com/bytedance/deer-flow/issues/1223) — persistent background work and recoverable task state. + +## Validation contract + +The implementation is covered at four boundaries: + +- capacity configuration, startup reload boundaries, queue/reject/timeout/cancel, and slot release; +- ordinary prompt/middleware/executor consistency, explicit direct-factory runtime/config binding, and executor regression coverage; +- migration parity, durable repository idempotency, live-window claiming, lease recovery, retries, controls, ownership, service execution, and JSONL routes; and +- frontend type checking, linting, API/type unit tests, and feature-gated chat integration. diff --git a/frontend/src/AGENTS.md b/frontend/src/AGENTS.md index d4a1fcf35..49a6af1fd 100644 --- a/frontend/src/AGENTS.md +++ b/frontend/src/AGENTS.md @@ -35,6 +35,19 @@ bounded error and attempt count; retryable failures use backend backoff, while a permanent rejection or exhausted five-attempt budget is shown as stopped rather than implying that retries will continue. + Explicit durable native-subagent batches use `core/subagent-batches` and + `ThreadSubagentBatches`. `/api/features` reports SQL-repository availability + separately from the startup worker. A running worker exposes the trigger on + both default and Custom Agent chat pages; a stopped worker keeps threads with + durable history visible in read-only mode for inspection and JSONL export, + while deployments with neither a worker nor history keep the trigger hidden. + The panel renders bounded progress and incrementally paged item previews, controls pause/resume/cancel, + retries failed items, and exports JSONL. Worker-dependent mutations stay + disabled in read-only history mode, and persisted progress is normalized to + a bounded percentage before reaching the UI primitive. Item pagination uses a + fixed page size and an explicit load-more control; full results remain available + only through JSONL export. The panel must not infer batch mode from prompt text + or inject the complete result set into chat state. Settings > Integrations uses a local generation only to suppress stale React callbacks; server-issued Lark flow generations must be passed through every config/auth completion and across switch-or-register to authorization chains diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx index 327045d5e..1e777efb9 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx @@ -28,6 +28,7 @@ import { SidecarTrigger, } from "@/components/workspace/sidecar"; import { ThreadBackgroundTasks } from "@/components/workspace/thread-background-tasks"; +import { ThreadSubagentBatches } from "@/components/workspace/thread-subagent-batches"; import { ThreadTitle } from "@/components/workspace/thread-title"; import { TodoList } from "@/components/workspace/todo-list"; import { TokenUsageIndicator } from "@/components/workspace/token-usage-indicator"; @@ -282,6 +283,11 @@ export default function AgentChatPage() { env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY !== "true" && ( )} + {!isNewThread && + !isMock && + env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY !== "true" && ( + + )} + + + + + + {t.subagentBatches.title} + + {t.subagentBatches.description} + +
+ {!workerRunning && ( +
+ {t.subagentBatches.workerUnavailable} +
+ )} + {batchesQuery.isLoading ? ( +
+ + {t.common.loading} +
+ ) : batchesQuery.isError ? ( +
+

+ {t.subagentBatches.loadFailed} +

+

+ {batchesQuery.error.message} +

+
+ ) : batches.length === 0 ? ( +
+ +

+ {t.subagentBatches.empty} +

+

{t.subagentBatches.emptyHint}

+
+ ) : ( +
+ {batches.map((batch) => ( + + control.mutate({ batchId: batch.id, action }) + } + /> + ))} +
+ )} +
+ + + ); +} + +function BatchCard({ + threadId, + batch, + workerRunning, + controlling, + onControl, +}: { + threadId: string; + batch: SubagentBatch; + workerRunning: boolean; + controlling: boolean; + onControl: (action: "pause" | "resume" | "cancel") => void; +}) { + const { t } = useI18n(); + const [open, setOpen] = useState(false); + const completed = completedSubagentBatchItems(batch); + const active = isActiveSubagentBatch(batch); + const labels = t.subagentBatches.status; + + return ( +
+
+
+

+ {batch.title} +

+

+ {batch.subagent_type} ·{" "} + {t.subagentBatches.limits( + batch.max_live_items, + batch.max_running_items, + )} +

+
+ {labels[batch.status]} +
+ +
+ {t.subagentBatches.progress(completed, batch.total_items)} + + {batch.counts.running} {labels.running.toLowerCase()} + + {batch.counts.failed > 0 && ( + + {batch.counts.failed} {labels.failed.toLowerCase()} + + )} +
+
+ + {batch.status === "running" || batch.status === "queued" ? ( + + ) : batch.status === "paused" ? ( + + ) : null} + {active && ( + + )} + +
+ {open && ( + + )} +
+ ); +} + +function BatchItems({ + threadId, + batch, + workerRunning, +}: { + threadId: string; + batch: SubagentBatch; + workerRunning: boolean; +}) { + const { t } = useI18n(); + const query = useSubagentBatchItems(threadId, batch.id, { + polling: workerRunning, + }); + const retry = useRetrySubagentBatchItem(threadId, batch.id); + if (query.isLoading) { + return ( +
+ {t.common.loading} +
+ ); + } + if (query.isError) { + return ( +
+ {t.subagentBatches.itemsFailed}: {query.error.message} +
+ ); + } + return ( +
+ {(query.data?.pages.flat() ?? []).map((item) => ( + retry.mutate(item.id)} + /> + ))} + {query.hasNextPage && ( + + )} +
+ ); +} + +function BatchItemRow({ + item, + workerRunning, + retrying, + onRetry, +}: { + item: SubagentBatchItem; + workerRunning: boolean; + retrying: boolean; + onRetry: () => void; +}) { + const { t } = useI18n(); + return ( +
+
+ + {item.item_key} + + {item.status} +
+ {item.result_preview && ( +

+ {item.result_preview} +

+ )} + {item.error && ( +

{item.error}

+ )} + {item.status === "failed" && ( + + )} +
+ ); +} diff --git a/frontend/src/core/features/api.ts b/frontend/src/core/features/api.ts index 658b51d60..d28aabd31 100644 --- a/frontend/src/core/features/api.ts +++ b/frontend/src/core/features/api.ts @@ -5,6 +5,18 @@ export interface FeaturesResponse { agents_api: { enabled: boolean }; browser_control?: { enabled: boolean }; mcp_tasks?: { enabled: boolean }; + subagent_batches?: { + enabled?: boolean; + repository_available?: boolean; + worker_running?: boolean; + max_running?: number; + }; +} + +export interface SubagentBatchesCapability { + repositoryAvailable: boolean; + workerRunning: boolean; + maxRunning: number; } export async function fetchFeatures(): Promise { @@ -26,3 +38,13 @@ export async function fetchBrowserControlEnabled(): Promise { export async function fetchMcpTasksEnabled(): Promise { return (await fetchFeatures()).mcp_tasks?.enabled ?? false; } + +export async function fetchSubagentBatchesCapability(): Promise { + const feature = (await fetchFeatures()).subagent_batches; + const legacyEnabled = feature?.enabled ?? false; + return { + repositoryAvailable: feature?.repository_available ?? legacyEnabled, + workerRunning: feature?.worker_running ?? legacyEnabled, + maxRunning: feature?.max_running ?? 0, + }; +} diff --git a/frontend/src/core/features/hooks.ts b/frontend/src/core/features/hooks.ts index fb7a87864..5ab574078 100644 --- a/frontend/src/core/features/hooks.ts +++ b/frontend/src/core/features/hooks.ts @@ -1,6 +1,10 @@ import { useQuery } from "@tanstack/react-query"; -import { fetchBrowserControlEnabled, fetchMcpTasksEnabled } from "./api"; +import { + fetchBrowserControlEnabled, + fetchMcpTasksEnabled, + fetchSubagentBatchesCapability, +} from "./api"; export function useBrowserControlEnabled() { const { data, isPending } = useQuery({ @@ -31,3 +35,19 @@ export function useMcpTasksEnabled() { isLoading: isPending, }; } + +export function useSubagentBatchesCapability() { + const { data, isPending } = useQuery({ + queryKey: ["features", "subagent_batches"], + queryFn: () => fetchSubagentBatchesCapability(), + staleTime: 0, + refetchOnMount: true, + retry: false, + }); + return { + repositoryAvailable: data?.repositoryAvailable ?? false, + workerRunning: data?.workerRunning ?? false, + maxRunning: data?.maxRunning ?? 0, + isLoading: isPending, + }; +} diff --git a/frontend/src/core/i18n/locales/en-US.ts b/frontend/src/core/i18n/locales/en-US.ts index e36070dfa..bcd60cc75 100644 --- a/frontend/src/core/i18n/locales/en-US.ts +++ b/frontend/src/core/i18n/locales/en-US.ts @@ -345,6 +345,37 @@ export const enUS: Translations = { }, }, + subagentBatches: { + label: "Batches", + title: "Subagent batches", + description: "Durable, restart-safe work for many independent items.", + workerUnavailable: + "The batch worker is not running. Historical batches remain available in read-only mode.", + empty: "No subagent batches yet", + emptyHint: "Explicit batch_task submissions in this chat will appear here.", + loadFailed: "Couldn't load subagent batches", + active: "Active", + recent: "Recent", + pause: "Pause", + resume: "Resume", + cancel: "Cancel", + retryItem: "Retry", + exportResults: "Export JSONL", + viewItems: "View items", + hideItems: "Hide items", + itemsFailed: "Couldn't load batch items", + progress: (completed, total) => `${completed} of ${total} terminal`, + limits: (live, running) => `Live ${live} · running ${running}`, + status: { + queued: "Queued", + running: "Running", + paused: "Paused", + completed: "Completed", + failed: "Failed", + cancelled: "Cancelled", + }, + }, + // Scheduled tasks scheduledTasks: { scheduleType: { diff --git a/frontend/src/core/i18n/locales/types.ts b/frontend/src/core/i18n/locales/types.ts index e33aae719..bfe668fe6 100644 --- a/frontend/src/core/i18n/locales/types.ts +++ b/frontend/src/core/i18n/locales/types.ts @@ -267,6 +267,36 @@ export interface Translations { }; }; + subagentBatches: { + label: string; + title: string; + description: string; + workerUnavailable: string; + empty: string; + emptyHint: string; + loadFailed: string; + active: string; + recent: string; + pause: string; + resume: string; + cancel: string; + retryItem: string; + exportResults: string; + viewItems: string; + hideItems: string; + itemsFailed: string; + progress: (completed: number, total: number) => string; + limits: (live: number, running: number) => string; + status: { + queued: string; + running: string; + paused: string; + completed: string; + failed: string; + cancelled: string; + }; + }; + // Scheduled tasks scheduledTasks: { scheduleType: { cron: string; once: string }; diff --git a/frontend/src/core/i18n/locales/zh-CN.ts b/frontend/src/core/i18n/locales/zh-CN.ts index f64be61d7..d8511b4da 100644 --- a/frontend/src/core/i18n/locales/zh-CN.ts +++ b/frontend/src/core/i18n/locales/zh-CN.ts @@ -327,6 +327,37 @@ export const zhCN: Translations = { }, }, + subagentBatches: { + label: "批处理", + title: "子智能体批处理", + description: "面向大量独立条目的持久化、可恢复执行。", + workerUnavailable: + "批处理 worker 未运行。历史批次仍可查看和导出，当前为只读模式。", + empty: "暂无子智能体批处理", + emptyHint: "当前对话通过 batch_task 提交的批处理会显示在这里。", + loadFailed: "无法加载子智能体批处理", + active: "进行中", + recent: "最近任务", + pause: "暂停", + resume: "继续", + cancel: "取消", + retryItem: "重试", + exportResults: "导出 JSONL", + viewItems: "查看条目", + hideItems: "收起条目", + itemsFailed: "无法加载批处理条目", + progress: (completed, total) => `${completed}/${total} 已结束`, + limits: (live, running) => `存活 ${live} · 运行 ${running}`, + status: { + queued: "排队中", + running: "运行中", + paused: "已暂停", + completed: "已完成", + failed: "已失败", + cancelled: "已取消", + }, + }, + // 定时任务 scheduledTasks: { scheduleType: { diff --git a/frontend/src/core/subagent-batches/api.ts b/frontend/src/core/subagent-batches/api.ts new file mode 100644 index 000000000..ddcfd0445 --- /dev/null +++ b/frontend/src/core/subagent-batches/api.ts @@ -0,0 +1,83 @@ +import { throwGatewayApiError } from "@/core/api/errors"; +import { fetch } from "@/core/api/fetcher"; +import { getBackendBaseURL } from "@/core/config"; + +import type { SubagentBatch, SubagentBatchItem } from "./types"; + +function batchUrl(threadId: string, path = ""): string { + return `${getBackendBaseURL()}/api/threads/${encodeURIComponent(threadId)}/subagent-batches${path}`; +} + +async function json(response: Response, fallback: string): Promise { + if (!response.ok) await throwGatewayApiError(response, fallback); + return response.json() as Promise; +} + +export async function fetchSubagentBatches( + threadId: string, +): Promise { + return json( + await fetch(`${batchUrl(threadId)}?limit=20`), + "Failed to load subagent batches", + ); +} + +export async function fetchSubagentBatchItems( + threadId: string, + batchId: string, + options: { + offset?: number; + limit?: number; + status?: SubagentBatchItem["status"]; + } = {}, +): Promise { + const params = new URLSearchParams({ + offset: String(options.offset ?? 0), + limit: String(options.limit ?? 100), + }); + if (options.status) params.set("status", options.status); + return json( + await fetch( + batchUrl(threadId, `/${encodeURIComponent(batchId)}/items?${params}`), + ), + "Failed to load batch items", + ); +} + +export async function controlSubagentBatch( + threadId: string, + batchId: string, + action: "pause" | "resume" | "cancel", +): Promise { + return json( + await fetch( + batchUrl(threadId, `/${encodeURIComponent(batchId)}/${action}`), + { method: "POST" }, + ), + `Failed to ${action} subagent batch`, + ); +} + +export async function retrySubagentBatchItem( + threadId: string, + batchId: string, + itemId: string, +): Promise { + return json( + await fetch( + batchUrl( + threadId, + `/${encodeURIComponent(batchId)}/items/${encodeURIComponent(itemId)}/retry`, + ), + { method: "POST" }, + ), + "Failed to retry subagent batch item", + ); +} + +export function subagentBatchResultsUrl( + threadId: string, + batchId: string, +): string { + return batchUrl(threadId, `/${encodeURIComponent(batchId)}/results.jsonl`); +} diff --git a/frontend/src/core/subagent-batches/hooks.ts b/frontend/src/core/subagent-batches/hooks.ts new file mode 100644 index 000000000..07c8b20fa --- /dev/null +++ b/frontend/src/core/subagent-batches/hooks.ts @@ -0,0 +1,99 @@ +import { + useInfiniteQuery, + useMutation, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import { toast } from "sonner"; + +import { + controlSubagentBatch, + fetchSubagentBatchItems, + fetchSubagentBatches, + retrySubagentBatchItem, +} from "./api"; +import { isActiveSubagentBatch } from "./types"; + +export const subagentBatchesKey = (threadId: string) => + ["subagent-batches", threadId] as const; +export const subagentBatchItemsKey = (threadId: string, batchId: string) => + [...subagentBatchesKey(threadId), batchId, "items"] as const; +const SUBAGENT_BATCH_ITEMS_PAGE_SIZE = 100; + +export function useSubagentBatches( + threadId: string, + options: { enabled?: boolean; polling?: boolean } = {}, +) { + return useQuery({ + queryKey: subagentBatchesKey(threadId), + queryFn: () => fetchSubagentBatches(threadId), + enabled: options.enabled !== false && Boolean(threadId), + refetchInterval: (query) => { + if (options.polling === false) return false; + return query.state.data?.some(isActiveSubagentBatch) ? 2000 : 15000; + }, + refetchIntervalInBackground: false, + }); +} + +export function useSubagentBatchItems( + threadId: string, + batchId: string, + options: { enabled?: boolean; polling?: boolean } = {}, +) { + return useInfiniteQuery({ + queryKey: subagentBatchItemsKey(threadId, batchId), + queryFn: ({ pageParam }) => + fetchSubagentBatchItems(threadId, batchId, { + offset: pageParam, + limit: SUBAGENT_BATCH_ITEMS_PAGE_SIZE, + }), + initialPageParam: 0, + getNextPageParam: (lastPage, allPages) => + lastPage.length === SUBAGENT_BATCH_ITEMS_PAGE_SIZE + ? allPages.reduce((total, page) => total + page.length, 0) + : undefined, + enabled: options.enabled !== false && Boolean(threadId) && Boolean(batchId), + // React Query refetches every loaded infinite page. Keep live polling for + // the bounded first page, then stop automatic fan-out after the user loads + // more; window-focus/manual invalidation still refreshes the loaded pages. + refetchInterval: (query) => { + if (options.polling === false) return false; + return (query.state.data?.pages.length ?? 0) <= 1 ? 3000 : false; + }, + refetchIntervalInBackground: false, + }); +} + +export function useControlSubagentBatch(threadId: string) { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: ({ + batchId, + action, + }: { + batchId: string; + action: "pause" | "resume" | "cancel"; + }) => controlSubagentBatch(threadId, batchId, action), + onSuccess: () => + queryClient.invalidateQueries({ queryKey: subagentBatchesKey(threadId) }), + onError: (error: Error) => toast.error(error.message), + }); +} + +export function useRetrySubagentBatchItem(threadId: string, batchId: string) { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (itemId: string) => + retrySubagentBatchItem(threadId, batchId, itemId), + onSuccess: () => { + void queryClient.invalidateQueries({ + queryKey: subagentBatchesKey(threadId), + }); + void queryClient.invalidateQueries({ + queryKey: subagentBatchItemsKey(threadId, batchId), + }); + }, + onError: (error: Error) => toast.error(error.message), + }); +} diff --git a/frontend/src/core/subagent-batches/index.ts b/frontend/src/core/subagent-batches/index.ts new file mode 100644 index 000000000..0733bf1a4 --- /dev/null +++ b/frontend/src/core/subagent-batches/index.ts @@ -0,0 +1,3 @@ +export * from "./api"; +export * from "./hooks"; +export * from "./types"; diff --git a/frontend/src/core/subagent-batches/types.ts b/frontend/src/core/subagent-batches/types.ts new file mode 100644 index 000000000..b25f9808d --- /dev/null +++ b/frontend/src/core/subagent-batches/types.ts @@ -0,0 +1,67 @@ +export type SubagentBatchStatus = + | "queued" + | "running" + | "paused" + | "completed" + | "failed" + | "cancelled"; + +export type SubagentBatchItemStatus = + | "pending" + | "queued" + | "leased" + | "running" + | "succeeded" + | "failed" + | "cancelled"; + +export type SubagentBatchCounts = Record; + +export type SubagentBatch = { + id: string; + title: string; + subagent_type: string; + status: SubagentBatchStatus; + total_items: number; + max_live_items: number; + max_running_items: number; + max_attempts: number; + counts: SubagentBatchCounts; + created_at: string; + updated_at: string; + completed_at: string | null; +}; + +export type SubagentBatchItem = { + id: string; + batch_id: string; + item_key: string; + position: number; + status: SubagentBatchItemStatus; + attempt: number; + model_name: string | null; + result_preview: string | null; + result_truncated: boolean; + error: string | null; + stop_reason: string | null; + token_usage: Record | null; + started_at: string | null; + completed_at: string | null; + created_at: string; + updated_at: string; +}; + +export function isActiveSubagentBatch(batch: SubagentBatch): boolean { + return ["queued", "running", "paused"].includes(batch.status); +} + +export function completedSubagentBatchItems(batch: SubagentBatch): number { + return batch.counts.succeeded + batch.counts.failed + batch.counts.cancelled; +} + +export function subagentBatchProgress(batch: SubagentBatch): number { + if (!Number.isFinite(batch.total_items) || batch.total_items <= 0) return 0; + const completed = completedSubagentBatchItems(batch); + if (!Number.isFinite(completed)) return 0; + return Math.min(100, Math.max(0, (completed / batch.total_items) * 100)); +} diff --git a/frontend/tests/unit/components/workspace/thread-subagent-batches.dom.test.tsx b/frontend/tests/unit/components/workspace/thread-subagent-batches.dom.test.tsx new file mode 100644 index 000000000..02fb88fd7 --- /dev/null +++ b/frontend/tests/unit/components/workspace/thread-subagent-batches.dom.test.tsx @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, it, rs } from "@rstest/core"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; + +const featureState = rs.hoisted(() => ({ + repositoryAvailable: true, + workerRunning: false, +})); +const batchState = rs.hoisted(() => ({ + batches: [] as Array>, + control: rs.fn(), + itemPages: [] as Array>>, + fetchNextPage: rs.fn(), + hasNextPage: false, +})); + +rs.mock("@/core/features", () => ({ + useSubagentBatchesCapability: () => ({ + ...featureState, + maxRunning: 3, + isLoading: false, + }), +})); + +rs.mock("@/core/i18n/hooks", () => ({ + useI18n: () => ({ + t: { + common: { loading: "Loading", loadMore: "Load more" }, + subagentBatches: { + label: "Batches", + title: "Subagent batches", + description: "Durable batch work", + workerUnavailable: + "The batch worker is not running. Historical batches are read-only.", + empty: "No batches", + emptyHint: "Submit a batch", + loadFailed: "Load failed", + pause: "Pause", + resume: "Resume", + cancel: "Cancel", + retryItem: "Retry", + exportResults: "Export JSONL", + viewItems: "View items", + hideItems: "Hide items", + itemsFailed: "Items failed", + progress: (completed: number, total: number) => + `${completed} of ${total}`, + limits: (live: number, running: number) => + `Live ${live} running ${running}`, + status: { + queued: "Queued", + running: "Running", + paused: "Paused", + completed: "Completed", + failed: "Failed", + cancelled: "Cancelled", + }, + }, + }, + }), +})); + +rs.mock("@/core/subagent-batches", () => ({ + completedSubagentBatchItems: () => 1, + isActiveSubagentBatch: (batch: { status: string }) => + ["queued", "running", "paused"].includes(batch.status), + subagentBatchProgress: () => 50, + subagentBatchResultsUrl: () => "/results.jsonl", + useControlSubagentBatch: () => ({ + isPending: false, + variables: undefined, + mutate: batchState.control, + }), + useRetrySubagentBatchItem: () => ({ + isPending: false, + variables: undefined, + mutate: rs.fn(), + }), + useSubagentBatchItems: () => ({ + data: { pages: batchState.itemPages }, + isLoading: false, + isError: false, + hasNextPage: batchState.hasNextPage, + isFetchingNextPage: false, + fetchNextPage: batchState.fetchNextPage, + }), + useSubagentBatches: () => ({ + data: batchState.batches, + isLoading: false, + isError: false, + }), +})); + +import { ThreadSubagentBatches } from "@/components/workspace/thread-subagent-batches"; + +const HISTORICAL_BATCH = { + id: "batch-1", + title: "Historical records", + subagent_type: "general-purpose", + status: "running", + total_items: 2, + max_live_items: 2, + max_running_items: 1, + max_attempts: 3, + counts: { + pending: 0, + queued: 0, + leased: 0, + running: 1, + succeeded: 1, + failed: 0, + cancelled: 0, + }, + created_at: "2026-08-24T00:00:00Z", + updated_at: "2026-08-24T00:01:00Z", + completed_at: null, +}; + +afterEach(() => { + cleanup(); + featureState.repositoryAvailable = true; + featureState.workerRunning = false; + batchState.batches = []; + batchState.control.mockReset(); + batchState.itemPages = []; + batchState.fetchNextPage.mockReset(); + batchState.hasNextPage = false; +}); + +describe("ThreadSubagentBatches capability gating", () => { + it("keeps historical batches readable when the worker is stopped", async () => { + batchState.batches = [HISTORICAL_BATCH]; + render(); + + fireEvent.click(screen.getByRole("button", { name: "Batches" })); + + expect( + await screen.findByText( + "The batch worker is not running. Historical batches are read-only.", + ), + ).toBeDefined(); + expect(screen.getByRole("button", { name: "Pause" })).toHaveProperty( + "disabled", + true, + ); + expect(screen.getByRole("button", { name: "Cancel" })).toHaveProperty( + "disabled", + true, + ); + expect( + screen.getByRole("link", { name: "Export JSONL" }).getAttribute("href"), + ).toBe("/results.jsonl"); + }); + + it("hides an unused batch surface when neither worker nor history exists", () => { + render(); + expect(screen.queryByRole("button", { name: "Batches" })).toBeNull(); + }); + + it("loads the next page of batch items", async () => { + batchState.batches = [HISTORICAL_BATCH]; + batchState.itemPages = [ + [ + { + id: "item-1", + item_key: "record-1", + status: "succeeded", + result_preview: "done", + }, + ], + ]; + batchState.hasNextPage = true; + render(); + + fireEvent.click(screen.getByRole("button", { name: "Batches" })); + fireEvent.click(screen.getByRole("button", { name: "View items" })); + fireEvent.click(await screen.findByRole("button", { name: "Load more" })); + + expect(batchState.fetchNextPage).toHaveBeenCalledTimes(1); + }); +}); diff --git a/frontend/tests/unit/core/features/api.test.ts b/frontend/tests/unit/core/features/api.test.ts new file mode 100644 index 000000000..3c80810b5 --- /dev/null +++ b/frontend/tests/unit/core/features/api.test.ts @@ -0,0 +1,57 @@ +import { beforeEach, describe, expect, it, rs } from "@rstest/core"; + +rs.mock("@/core/api/fetcher", () => ({ fetch: rs.fn() })); +rs.mock("@/core/config", () => ({ getBackendBaseURL: () => "" })); + +import { fetch } from "@/core/api/fetcher"; +import { fetchSubagentBatchesCapability } from "@/core/features/api"; + +const mockedFetch = rs.mocked(fetch); + +function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +beforeEach(() => { + mockedFetch.mockReset(); +}); + +describe("subagent batch feature capability", () => { + it("keeps repository and worker availability independent", async () => { + mockedFetch.mockResolvedValueOnce( + jsonResponse({ + agents_api: { enabled: true }, + subagent_batches: { + enabled: false, + repository_available: true, + worker_running: false, + max_running: 3, + }, + }), + ); + + await expect(fetchSubagentBatchesCapability()).resolves.toEqual({ + repositoryAvailable: true, + workerRunning: false, + maxRunning: 3, + }); + }); + + it("falls back to the legacy enabled flag during rolling upgrades", async () => { + mockedFetch.mockResolvedValueOnce( + jsonResponse({ + agents_api: { enabled: true }, + subagent_batches: { enabled: true, max_running: 4 }, + }), + ); + + await expect(fetchSubagentBatchesCapability()).resolves.toEqual({ + repositoryAvailable: true, + workerRunning: true, + maxRunning: 4, + }); + }); +}); diff --git a/frontend/tests/unit/core/subagent-batches/api.test.ts b/frontend/tests/unit/core/subagent-batches/api.test.ts new file mode 100644 index 000000000..68f0a1cc0 --- /dev/null +++ b/frontend/tests/unit/core/subagent-batches/api.test.ts @@ -0,0 +1,70 @@ +import { beforeEach, describe, expect, it, rs } from "@rstest/core"; + +rs.mock("@/core/api/fetcher", () => ({ fetch: rs.fn() })); +rs.mock("@/core/config", () => ({ getBackendBaseURL: () => "" })); + +import { fetch } from "@/core/api/fetcher"; +import { + controlSubagentBatch, + fetchSubagentBatchItems, + fetchSubagentBatches, + retrySubagentBatchItem, + subagentBatchResultsUrl, +} from "@/core/subagent-batches/api"; + +const mockedFetch = rs.mocked(fetch); + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +beforeEach(() => { + mockedFetch.mockReset(); +}); + +describe("subagent batch API", () => { + it("loads thread batches and items through encoded local ids", async () => { + mockedFetch.mockResolvedValueOnce(jsonResponse([])); + await fetchSubagentBatches("thread / 1"); + expect(mockedFetch).toHaveBeenLastCalledWith( + "/api/threads/thread%20%2F%201/subagent-batches?limit=20", + ); + + mockedFetch.mockResolvedValueOnce(jsonResponse([])); + await fetchSubagentBatchItems("thread / 1", "batch / 1", { + offset: 100, + limit: 50, + status: "failed", + }); + expect(mockedFetch).toHaveBeenLastCalledWith( + "/api/threads/thread%20%2F%201/subagent-batches/batch%20%2F%201/items?offset=100&limit=50&status=failed", + ); + }); + + it("posts control and failed-item retry actions", async () => { + mockedFetch + .mockResolvedValueOnce(jsonResponse({})) + .mockResolvedValueOnce(jsonResponse({})); + + await controlSubagentBatch("thread-1", "batch-1", "pause"); + expect(mockedFetch).toHaveBeenLastCalledWith( + "/api/threads/thread-1/subagent-batches/batch-1/pause", + { method: "POST" }, + ); + + await retrySubagentBatchItem("thread-1", "batch-1", "item / 1"); + expect(mockedFetch).toHaveBeenLastCalledWith( + "/api/threads/thread-1/subagent-batches/batch-1/items/item%20%2F%201/retry", + { method: "POST" }, + ); + }); + + it("builds a JSONL export URL without putting results in chat context", () => { + expect(subagentBatchResultsUrl("thread-1", "batch-1")).toBe( + "/api/threads/thread-1/subagent-batches/batch-1/results.jsonl", + ); + }); +}); diff --git a/frontend/tests/unit/core/subagent-batches/types.test.ts b/frontend/tests/unit/core/subagent-batches/types.test.ts new file mode 100644 index 000000000..428d881bc --- /dev/null +++ b/frontend/tests/unit/core/subagent-batches/types.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "@rstest/core"; + +import { + completedSubagentBatchItems, + isActiveSubagentBatch, + subagentBatchProgress, + type SubagentBatch, +} from "@/core/subagent-batches/types"; + +const BATCH: SubagentBatch = { + id: "batch-1", + title: "Records", + subagent_type: "general-purpose", + status: "running", + total_items: 10, + max_live_items: 5, + max_running_items: 2, + max_attempts: 3, + counts: { + pending: 2, + queued: 2, + leased: 1, + running: 1, + succeeded: 2, + failed: 1, + cancelled: 1, + }, + created_at: "2026-08-24T00:00:00Z", + updated_at: "2026-08-24T00:01:00Z", + completed_at: null, +}; + +describe("subagent batch progress", () => { + it("counts only terminal items as completed progress", () => { + expect(completedSubagentBatchItems(BATCH)).toBe(4); + expect(subagentBatchProgress(BATCH)).toBe(40); + }); + + it("returns bounded progress for malformed persisted totals or counts", () => { + expect(subagentBatchProgress({ ...BATCH, total_items: 0 })).toBe(0); + expect( + subagentBatchProgress({ + ...BATCH, + total_items: 1, + counts: { ...BATCH.counts, succeeded: 10 }, + }), + ).toBe(100); + }); + + it.each(["queued", "running", "paused"] as const)( + "treats %s as active", + (status) => expect(isActiveSubagentBatch({ ...BATCH, status })).toBe(true), + ); + + it.each(["completed", "failed", "cancelled"] as const)( + "treats %s as terminal", + (status) => expect(isActiveSubagentBatch({ ...BATCH, status })).toBe(false), + ); +});