From c5d072af91af11a01d2d4ef3a51fab9b4bd5f560 Mon Sep 17 00:00:00 2001 From: Aurora_ <247336952+xypang33-sketch@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:29:26 +0800 Subject: [PATCH] feat(subagents): add optional SQLite checkpoint history (#8265) * feat(subagents): add optional SQLite checkpoint history * docs(subagents): clarify checkpoint history limits * fix(subagents): resolve checkpoint factory at startup --------- Co-authored-by: Nan Gao --- README.md | 2 +- backend/app/gateway/deps.py | 5 + .../deerflow/config/reload_boundary.py | 5 +- .../config/subagent_runtime_config.py | 9 +- .../harness/deerflow/subagents/AGENTS.md | 5 +- .../harness/deerflow/subagents/checkpoints.py | 228 ++++++ .../harness/deerflow/subagents/executor.py | 66 +- .../test_subagent_checkpoints_offloop.py | 40 + .../test_subagent_checkpoint_persistence.py | 692 ++++++++++++++++++ .../test_subagent_checkpointer_isolation.py | 10 +- config.example.yaml | 4 + docs/guide/subagents.md | 79 ++ 12 files changed, 1124 insertions(+), 21 deletions(-) create mode 100644 backend/packages/harness/deerflow/subagents/checkpoints.py create mode 100644 backend/tests/blocking_io/test_subagent_checkpoints_offloop.py create mode 100644 backend/tests/test_subagent_checkpoint_persistence.py diff --git a/README.md b/README.md index 6e0075c77d..bd4117e9b6 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Each guide is the full reference for its topic. Open the one you need. | Sandbox modes, file system, uploads and outputs | [Sandbox](docs/guide/sandbox.md) | | Skills, built-in tools, script workflows, Claude Code integration | [Skills & Tools](docs/guide/skills-and-tools.md) | | MCP servers | [MCP Server](docs/guide/mcp.md) | -| Sub-agents and task delegation | [Sub-Agents](docs/guide/subagents.md), [Tool restrictions](docs/guide/subagents.md#tool-restrictions) | +| Sub-agents, task delegation and optional SQLite checkpoint history | [Sub-Agents](docs/guide/subagents.md), [Tool restrictions](docs/guide/subagents.md#tool-restrictions) | | Long-term memory | [Long-Term Memory](docs/guide/memory.md) | | Session goals, context compaction, task notes, evidence-based decisions | [Context, Goals and Agent Decisions](docs/guide/context.md) | | Private knowledge retrieval (RAGFlow) | [Private Knowledge Retrieval](docs/guide/knowledge-retrieval.md) | diff --git a/backend/app/gateway/deps.py b/backend/app/gateway/deps.py index 989d4cee38..b4b6c06e72 100644 --- a/backend/app/gateway/deps.py +++ b/backend/app/gateway/deps.py @@ -873,6 +873,11 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen await init_engine_from_config(config.database) app.state.checkpointer = await stack.enter_async_context(make_checkpointer(config)) + # The peer is closed before the parent saver, including partial + # startup failure. Its own shutdown fences and drains isolated tasks. + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + + await stack.enter_async_context(subagent_checkpoint_runtime(config, app.state.checkpointer)) app.state.store = await stack.enter_async_context(make_store(config)) # Record the checkpointer/Store backend and the provisioner endpoint diff --git a/backend/packages/harness/deerflow/config/reload_boundary.py b/backend/packages/harness/deerflow/config/reload_boundary.py index c38dfe7dcd..6226190b5d 100644 --- a/backend/packages/harness/deerflow/config/reload_boundary.py +++ b/backend/packages/harness/deerflow/config/reload_boundary.py @@ -85,7 +85,10 @@ STARTUP_ONLY_FIELDS: dict[str, str] = { "McpTaskService is constructed and started once during Gateway lifespan startup; enabled, poll_interval_seconds, lease_seconds, " "and max_concurrent_polls are captured into the service instance and the background poller task is not rebuilt on config.yaml edits." ), - "subagent_runtime": ("the shared native-subagent admission controller and isolated execution loop are configured once during Gateway lifespan startup; changing process slots, queue policy, or queue bounds requires a restart."), + "subagent_runtime": ( + "the shared native-subagent admission controller, isolated execution loop, and optional checkpoint saver are configured once during Gateway lifespan startup; " + "changing checkpoint persistence, process slots, queue policy, or queue bounds requires a restart." + ), "subagent_batches": ("the durable subagent batch service is constructed and started once during Gateway lifespan startup; scheduler limits, leases, and recovery behavior are captured by that service instance."), "deployment": ( "langgraph_runtime() evaluates deployment.multi_instance once at startup inside the multi-process safety gate (_enforce_postgres_for_multi_worker); " diff --git a/backend/packages/harness/deerflow/config/subagent_runtime_config.py b/backend/packages/harness/deerflow/config/subagent_runtime_config.py index a6301f1ec3..2ee73d6f07 100644 --- a/backend/packages/harness/deerflow/config/subagent_runtime_config.py +++ b/backend/packages/harness/deerflow/config/subagent_runtime_config.py @@ -1,4 +1,4 @@ -"""Startup-only process capacity for native subagent execution.""" +"""Startup-only capacity and checkpoint settings for native subagents.""" from typing import Literal @@ -6,7 +6,12 @@ from pydantic import BaseModel, Field class SubagentRuntimeConfig(BaseModel): - """Process-local admission and execution limits shared by all subagents.""" + """Process-local execution capacity and optional native task history.""" + + checkpoint_enabled: bool = Field( + default=False, + description="Persist native task checkpoint history through a loop-owned saver. Requires file-backed SQLite checkpoint storage and a Gateway restart.", + ) max_running: int = Field( default=3, diff --git a/backend/packages/harness/deerflow/subagents/AGENTS.md b/backend/packages/harness/deerflow/subagents/AGENTS.md index 9f996dd45f..3213d59867 100644 --- a/backend/packages/harness/deerflow/subagents/AGENTS.md +++ b/backend/packages/harness/deerflow/subagents/AGENTS.md @@ -78,9 +78,8 @@ from `deerflow.utils.messages`. Keep delayed Steer reports as sanitized history. - **Sandbox use is leased.** An admitted subagent carries a stable lease owner ID and command scope ID derived from its task, so the shared sandbox can stop only that subagent's processes. -- **All native subagents share one persistent event loop.** Keep loop and thread - ownership until the worker has stopped. A bounded join does not prove that it - stopped. +- **Retain loop/thread ownership until subagents stop; a bounded join is insufficient.** + See [checkpoint lifetimes](../../../../../docs/guide/subagents.md#checkpoint-resource-ownership). - **Delegation is not the default.** The routing policy permits `task` only when parallel latency, a specialist skill or context isolation clearly outweighs the startup cost and the duplicated discovery. diff --git a/backend/packages/harness/deerflow/subagents/checkpoints.py b/backend/packages/harness/deerflow/subagents/checkpoints.py new file mode 100644 index 0000000000..ea28e411be --- /dev/null +++ b/backend/packages/harness/deerflow/subagents/checkpoints.py @@ -0,0 +1,228 @@ +"""Loop-owned SQLite checkpoint resources for opt-in native task history. + +The registry associates a parent storage owner with its isolated-loop peer. It +does not infer a database from a request or hot-reload infrastructure settings. +""" + +from __future__ import annotations + +import asyncio +import threading +from collections.abc import AsyncIterator, Callable, Coroutine +from contextlib import AsyncExitStack, asynccontextmanager +from contextvars import Context, ContextVar +from typing import Any + +from langchain_core.runnables.config import var_child_runnable_config +from langgraph._internal._constants import CONFIG_KEY_CHECKPOINTER, NS_SEP +from langgraph.checkpoint.base import BaseCheckpointSaver +from langgraph.checkpoint.memory import InMemorySaver + +from deerflow.config.app_config import AppConfig +from deerflow.runtime.checkpointer import async_provider +from deerflow.utils.file_io import await_drained + +_runtimes: dict[int, SubagentCheckpointRuntime] = {} +_registry_lock = threading.Lock() +_captured_runtime: ContextVar[SubagentCheckpointRuntime | None] = ContextVar("subagent_checkpoint_runtime", default=None) +_active_runtime: ContextVar[SubagentCheckpointRuntime | None] = ContextVar("active_subagent_checkpoint_runtime", default=None) + + +class SubagentCheckpointError(RuntimeError): + """Checkpoint admission failed before a native graph could start.""" + + +async def _on_isolated_loop[T](factory: Callable[[], Coroutine[Any, Any, T]], *, loop: asyncio.AbstractEventLoop | None = None) -> T: + # Infrastructure startup/teardown must not inherit a run's graph context. + from deerflow.subagents.executor import _submit_to_isolated_loop_in_context + + if loop is None: + # First use may start a thread and wait for its loop-ready handshake. + future = await asyncio.to_thread(_submit_to_isolated_loop_in_context, Context(), factory) + else: + if loop.is_closed() or not loop.is_running(): + raise RuntimeError("Subagent checkpoint owner loop stopped before its resources were drained") + + def submit(): + coroutine = factory() + try: + return asyncio.run_coroutine_threadsafe(coroutine, loop) + except BaseException: + coroutine.close() + raise + + future = Context().run(submit) + return await asyncio.wrap_future(future) + + +class SubagentCheckpointRuntime: + """One saver shared by native children on their persistent execution loop.""" + + def __init__(self, config: AppConfig, parent_saver: BaseCheckpointSaver) -> None: + self.config = config.model_copy(deep=True) + self.parent_saver = parent_saver + self.mode = config.database.checkpoint_channel_mode + self.loop: asyncio.AbstractEventLoop | None = None + self.saver: BaseCheckpointSaver | None = None + self.closed = False + self.closing = False + self._stack: AsyncExitStack | None = None + self._executions: set[asyncio.Task] = set() + self._stop_task: asyncio.Task | None = None + + def owns_parent_saver(self, saver: BaseCheckpointSaver | None) -> bool: + # Strict msgpack support makes shallow saver clones with a derived + # serializer. They still share the parent's connection and lock. + source = getattr(saver, "source_saver", saver) + owner = getattr(self.parent_saver, "source_saver", self.parent_saver) + if source is owner: + return True + connection = getattr(source, "conn", None) + return connection is not None and connection is getattr(owner, "conn", None) + + async def start(self) -> None: + self.loop = asyncio.get_running_loop() + self._stack = AsyncExitStack() + self.saver = await self._stack.enter_async_context(async_provider.make_checkpointer(self.config)) + + async def stop(self) -> None: + if self.closed: + return + if asyncio.get_running_loop() is not self.loop: + raise RuntimeError("Subagent checkpoint resources must close on their owning event loop") + self.closing = True + if self._stop_task is None: + self._stop_task = asyncio.create_task(self._stop()) + await asyncio.shield(self._stop_task) + + async def _stop(self) -> None: + # These tasks own their complete stream/checkpoint cleanup tails, not + # just their public terminal status. Do not close storage until they exit. + tasks = tuple(self._executions) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + if self._stack is not None: + await self._stack.aclose() + self.closed = True + + +@asynccontextmanager +async def subagent_checkpoint_runtime(config: AppConfig, parent_saver: BaseCheckpointSaver) -> AsyncIterator[SubagentCheckpointRuntime | None]: + """Own the isolated saver inside the parent saver's application lifetime. + + Admission is captured at dispatch; shutdown fences new leases, cancels and + drains existing executions, and closes the peer on its owning loop. The + parent connection must remain open until this context exits. + """ + if getattr(getattr(config, "subagent_runtime", None), "checkpoint_enabled", False) is not True: + yield None + return + source = getattr(parent_saver, "source_saver", parent_saver) + legacy = config.checkpointer + backend = legacy.type if legacy is not None else config.database.backend + if backend != "sqlite" or isinstance(source, InMemorySaver): + raise ValueError("subagent_runtime.checkpoint_enabled only supports file-backed SQLite checkpoint storage") + if legacy is not None and legacy.type == "sqlite" and legacy.connection_string == ":memory:": + raise ValueError("subagent_runtime.checkpoint_enabled requires file-backed SQLite, not :memory:") + + runtime = SubagentCheckpointRuntime(config, parent_saver) + registered = False + try: + await await_drained(_on_isolated_loop(runtime.start)) + with _registry_lock: + if id(parent_saver) in _runtimes: + raise RuntimeError("Subagent checkpoint runtime is already installed for this parent saver") + _runtimes[id(parent_saver)] = runtime + registered = True + yield runtime + finally: + try: + if runtime.loop is not None and not runtime.closed: + await await_drained(_on_isolated_loop(runtime.stop, loop=runtime.loop)) + finally: + if registered and runtime.closed: + with _registry_lock: + _runtimes.pop(id(parent_saver), None) + + +def has_checkpoint_runtimes(loop: asyncio.AbstractEventLoop) -> bool: + with _registry_lock: + return any(runtime.loop is loop and not runtime.closed for runtime in _runtimes.values()) + + +async def drain_checkpoint_runtimes() -> None: + """Last-resort loop shutdown still drains its registered storage owners.""" + loop = asyncio.get_running_loop() + with _registry_lock: + runtimes = [runtime for runtime in _runtimes.values() if runtime.loop is loop] + for runtime in runtimes: + await runtime.stop() + + +def capture_checkpoint_runtime(context: Context) -> None: + """Pin an installed runtime before a copied context crosses to the worker.""" + inherited = context.get(var_child_runnable_config) or {} + parent = inherited.get("configurable", {}).get(CONFIG_KEY_CHECKPOINTER) + with _registry_lock: + runtime = _runtimes.get(id(parent)) + if runtime is None: + runtime = next((candidate for candidate in _runtimes.values() if candidate.owns_parent_saver(parent)), None) + captured = context.get(_captured_runtime) + if runtime is None and captured is not None and captured.owns_parent_saver(parent): + runtime = captured + # A captured runtime remains fenced after its registration is removed. + context.run(_captured_runtime.set, runtime) + + +def checkpoint_runtime() -> SubagentCheckpointRuntime | None: + """Return only a runtime leased by the current native execution.""" + return _active_runtime.get() + + +@asynccontextmanager +async def subagent_checkpoint_scope(*, execution_id: str, run_id: str | None, tool_call_id: str | None, agent_name: str) -> AsyncIterator[None]: + runtime = _captured_runtime.get() + if runtime is None: + yield + return + if runtime.closing or runtime.closed: + raise SubagentCheckpointError("Subagent checkpoint runtime is shutting down") + if asyncio.get_running_loop() is not runtime.loop: + raise SubagentCheckpointError("Subagent checkpoint runtime must be used on its owning event loop") + inherited = var_child_runnable_config.get() or {} + configurable = inherited.get("configurable", {}) + if not runtime.owns_parent_saver(configurable.get(CONFIG_KEY_CHECKPOINTER)): + raise SubagentCheckpointError("Subagent checkpoint runtime does not match the inherited parent saver") + if not configurable.get("thread_id") or not configurable.get("checkpoint_ns"): + raise SubagentCheckpointError("Subagent checkpoint history requires an inherited parent graph namespace") + + # Replace the inherited saver, not just the compile-time argument: + # LangGraph gives CONFIG_KEY_CHECKPOINTER precedence over that argument. + # A server execution ID also gives each independent attempt a fresh history, + # even if a provider reuses a tool ID or a parent node is replayed. + child_configurable = { + **configurable, + CONFIG_KEY_CHECKPOINTER: runtime.saver, + "checkpoint_ns": NS_SEP.join((configurable["checkpoint_ns"], f"deerflow_subagent:{execution_id}")), + } + metadata = { + **inherited.get("metadata", {}), + "deerflow_subagent_execution_id": execution_id, + "deerflow_subagent_name": agent_name, + } + if run_id: + metadata["deerflow_parent_run_id"] = run_id + if tool_call_id: + metadata["deerflow_subagent_tool_call_id"] = tool_call_id + config_token = var_child_runnable_config.set({**inherited, "configurable": child_configurable, "metadata": metadata}) + runtime_token = _active_runtime.set(runtime) + task = asyncio.current_task() + runtime._executions.add(task) + try: + yield + finally: + runtime._executions.discard(task) + _active_runtime.reset(runtime_token) + var_child_runnable_config.reset(config_token) diff --git a/backend/packages/harness/deerflow/subagents/executor.py b/backend/packages/harness/deerflow/subagents/executor.py index 04e6af7958..f7c56e832b 100644 --- a/backend/packages/harness/deerflow/subagents/executor.py +++ b/backend/packages/harness/deerflow/subagents/executor.py @@ -704,7 +704,27 @@ def _shutdown_isolated_subagent_loop(*, only_if_pending: bool = False) -> None: return if loop.is_running(): - loop.call_soon_threadsafe(loop.stop) + from deerflow.subagents.checkpoints import drain_checkpoint_runtimes, has_checkpoint_runtimes + + if has_checkpoint_runtimes(loop): + + async def drain_then_stop() -> None: + await drain_checkpoint_runtimes() + loop.call_soon(loop.stop) + + cleanup = asyncio.run_coroutine_threadsafe(drain_then_stop(), loop) + if thread is threading.current_thread(): + return + try: + cleanup.result(timeout=1) + except FuturesTimeoutError: + logger.warning("Retaining isolated subagent loop ownership until checkpoint resources finish draining") + return + except Exception: + logger.exception("Retaining isolated subagent loop ownership because checkpoint resource cleanup failed") + return + else: + loop.call_soon_threadsafe(loop.stop) if thread is not None and thread.is_alive() and thread is not threading.current_thread(): thread.join(timeout=1) @@ -808,10 +828,10 @@ def _run_on_isolated_loop_if_live(callback: Callable[[], None]) -> None: callback() -def _submit_to_isolated_loop_in_context( +def _submit_to_isolated_loop_in_context[T]( context: Context, - coro_factory: Callable[[], Coroutine[Any, Any, SubagentResult]], -) -> Future[SubagentResult]: + coro_factory: Callable[[], Coroutine[Any, Any, T]], +) -> Future[T]: """Submit a coroutine to the isolated loop while preserving ContextVar state. The loop must be resolved before the coroutine is created: as direct @@ -826,7 +846,7 @@ def _submit_to_isolated_loop_in_context( before the error propagates. """ - def _submit() -> Future[SubagentResult]: + def _submit() -> Future[T]: loop = _get_isolated_subagent_loop() coroutine = coro_factory() try: @@ -867,6 +887,9 @@ def _copy_isolated_subagent_context() -> Context: continue to reach the parent stream. """ context = copy_context() + from deerflow.subagents.checkpoints import capture_checkpoint_runtime + + capture_checkpoint_runtime(context) inherited_config = context.get(var_child_runnable_config) if inherited_config is None or "callbacks" not in inherited_config: return context @@ -1251,7 +1274,6 @@ class SubagentExecutor: authorization_provider=authz_provider, ) middlewares = apply_declared_tool_view(middlewares, authorized_names=declared_authorized) - verify_declared_tool_view(middlewares, authorized_names=declared_authorized) elif authz_provider is not None: # Defense in depth: the provider is configured but the seeded # decision state is missing, so the declaration pass is skipped. @@ -1281,6 +1303,17 @@ class SubagentExecutor: policy_names = ToolPolicyNameResolver() middlewares = apply_declared_tool_policy(middlewares, allowed=self.config.tools, disallowed=self.config.disallowed_tools, name_resolver=policy_names) + + from deerflow.subagents.checkpoints import checkpoint_runtime + + history_runtime = checkpoint_runtime() + state_schema = ThreadState + if history_runtime is not None and history_runtime.mode == "delta": + from deerflow.agents.thread_state import get_thread_state_schema, normalize_middleware_state_schemas + + state_schema = get_thread_state_schema("delta") + middlewares = normalize_middleware_state_schemas(middlewares, "delta") + # Collect every guard middleware that exposes ``consume_stop_reason`` # (TokenBudgetMiddleware, LoopDetectionMiddleware, # ModelLengthFinishReasonMiddleware) so _aexecute can read @@ -1293,6 +1326,10 @@ class SubagentExecutor: # system_prompt is included in initial state messages (see _build_initial_state) # to avoid multiple SystemMessages which some LLM APIs don't support. + # Delta normalization can copy middleware, so audit both policies on + # the final instances that create_agent will bind. + if declared_authorized is not None: + verify_declared_tool_view(middlewares, authorized_names=declared_authorized) verify_subagent_tool_policy( bound_tools, middlewares, @@ -1306,8 +1343,11 @@ class SubagentExecutor: tools=bound_tools, middleware=middlewares, system_prompt=None, - state_schema=ThreadState, - checkpointer=False, + state_schema=state_schema, + # The opted-in scope replaces the inherited saver with its + # loop-owned peer before graph construction. All other entrypoints + # keep isolation from a possibly synchronous parent saver. + checkpointer=None if history_runtime is not None else False, ) # Use the compiled registry: it includes middleware tools, normalized # callables, and LangChain's last-definition-wins name precedence. @@ -1732,17 +1772,25 @@ class SubagentExecutor: trace_id=self.trace_id, status=SubagentStatus.PENDING, ) + from deerflow.subagents.checkpoints import SubagentCheckpointError, subagent_checkpoint_scope + with ensure_trace_context(self.deerflow_trace_id): try: capacity = self.execution_capacity or get_subagent_execution_capacity() slot = capacity.slot(wait=False) if self.input_gate is not None and not self.input_gate.can_queue else capacity.slot() - async with slot: + async with ( + subagent_checkpoint_scope(execution_id=result.task_id, run_id=self.run_id, tool_call_id=result.external_task_id, agent_name=self.config.name), + slot, + ): with result._state_lock: if result.status.is_terminal: return result result.status = SubagentStatus.RUNNING result.started_at = _utcnow() return await self._aexecute_admitted(task, result) + except SubagentCheckpointError as exc: + result.try_set_terminal(SubagentStatus.FAILED, error=str(exc), admission_failure=True) + return result except SubagentCapacityError as exc: result.try_set_terminal( SubagentStatus.FAILED, diff --git a/backend/tests/blocking_io/test_subagent_checkpoints_offloop.py b/backend/tests/blocking_io/test_subagent_checkpoints_offloop.py new file mode 100644 index 0000000000..343d9e0169 --- /dev/null +++ b/backend/tests/blocking_io/test_subagent_checkpoints_offloop.py @@ -0,0 +1,40 @@ +"""Opening the isolated saver must leave the Gateway loop responsive.""" + +from __future__ import annotations + +import asyncio +import importlib +import sys + +import pytest + + +@pytest.mark.asyncio +async def test_checkpoint_peer_lifetime_does_not_block_gateway_loop(monkeypatch, tmp_path): + import deerflow.agents.thread_state # noqa: F401 + import deerflow.subagents as package + from deerflow.config.app_config import AppConfig + from deerflow.runtime.checkpointer.async_provider import make_checkpointer + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + + monkeypatch.delitem(sys.modules, "deerflow.subagents.executor", raising=False) + monkeypatch.delattr(package, "executor", raising=False) + executor = importlib.import_module("deerflow.subagents.executor") + config = AppConfig.model_validate( + { + "database": {"backend": "sqlite", "sqlite_dir": str(tmp_path)}, + "subagent_runtime": {"checkpoint_enabled": True}, + "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, + } + ) + try: + async with make_checkpointer(config) as saver: + async with subagent_checkpoint_runtime(config, saver) as runtime: + assert runtime.loop is not asyncio.get_running_loop() + assert runtime.saver is not saver + await saver.setup() + assert runtime.closed + finally: + await asyncio.to_thread(executor._shutdown_isolated_subagent_loop) + if getattr(package, "executor", None) is executor: + delattr(package, "executor") diff --git a/backend/tests/test_subagent_checkpoint_persistence.py b/backend/tests/test_subagent_checkpoint_persistence.py new file mode 100644 index 0000000000..95e6e941f4 --- /dev/null +++ b/backend/tests/test_subagent_checkpoint_persistence.py @@ -0,0 +1,692 @@ +"""Native subagent checkpoint history across the real isolated-loop boundary. + +The model is deterministic and local. The executor, graph builder, middleware, +SQLite saver, context propagation and background execution are production code. +""" + +from __future__ import annotations + +import asyncio +import importlib +import importlib.util +import sys +import threading +from collections import defaultdict +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest +from langchain.agents.middleware import AgentMiddleware +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.runnables import RunnableConfig +from langchain_core.runnables.config import var_child_runnable_config +from langchain_core.tools import tool +from langgraph.graph import END, START, MessagesState, StateGraph + +from deerflow.config.app_config import AppConfig +from deerflow.config.paths import Paths +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.runtime.checkpointer.async_provider import make_checkpointer + + +class _OfflineModel(BaseChatModel): + @property + def _llm_type(self): + return "offline-subagent-checkpoints" + + def bind_tools(self, tools, **kwargs): + return self + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): + if any(isinstance(message, ToolMessage) for message in messages): + answer = AIMessage(content="CHILD_FINAL") + else: + answer = AIMessage(content="", tool_calls=[{"name": "echo", "args": {"text": "child evidence"}, "id": "child-call", "type": "tool_call"}]) + return ChatResult(generations=[ChatGeneration(message=answer)]) + + +@tool +async def echo(text: str) -> str: + """Return deterministic evidence without external side effects.""" + return text + + +@tool +async def forbidden() -> str: + """A declaration that the test policy must remove before model binding.""" + pytest.fail("A denied middleware tool executed") + + +class _CopyingToolMiddleware(AgentMiddleware): + """Exercise state-preserving and constructor-rebuilding extension copies.""" + + def __init__(self, *, rebuild_on_copy: bool = False): + self.rebuild_on_copy = rebuild_on_copy + self.tools = [echo, forbidden] + + def __copy__(self): + clone = type(self)(rebuild_on_copy=self.rebuild_on_copy) + if not self.rebuild_on_copy: + clone.__dict__.update(self.__dict__) + return clone + + +@pytest.fixture +def native_executor(monkeypatch, tmp_path): + # conftest supplies an executor stub for collection-time cycles. Import + # its real dependencies before loading the actual executor for this test. + import deerflow.agents.thread_state # noqa: F401 + import deerflow.subagents as package + + monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "home")) + monkeypatch.setattr("deerflow.config.paths._paths", Paths(str(tmp_path / "home"))) + monkeypatch.delitem(sys.modules, "deerflow.subagents.executor", raising=False) + monkeypatch.delattr(package, "executor", raising=False) + module = importlib.import_module("deerflow.subagents.executor") + monkeypatch.setattr(module, "create_chat_model", lambda **kwargs: _OfflineModel()) + monkeypatch.setattr(module, "build_tracing_callbacks", lambda: []) + yield module + module._shutdown_isolated_subagent_loop() + # importlib installed this attribute outside monkeypatch's tracked writes. + if getattr(package, "executor", None) is module: + delattr(package, "executor") + + +@pytest.fixture +def native_task_tool(native_executor, monkeypatch): + import deerflow.subagents as package + + # The package export also comes from conftest's executor stub. Restore the + # real class before loading task_tool with its real polling/cleanup helpers. + # A separate module avoids changing cached tool references in other suites. + monkeypatch.setattr(package, "SubagentExecutor", native_executor.SubagentExecutor) + path = Path(native_executor.__file__).parents[1] / "tools/builtins/task_tool.py" + spec = importlib.util.spec_from_file_location("_checkpoint_task_tool", path) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module.task_tool + + +def _config(tmp_path, *, enabled=True, mode="full"): + config = AppConfig.model_validate( + { + "models": [{"name": "offline", "use": "langchain_openai:ChatOpenAI", "model": "offline", "api_key": "unused"}], + "database": {"backend": "sqlite", "sqlite_dir": str(tmp_path / "db"), "checkpoint_channel_mode": mode}, + "subagent_runtime": {"checkpoint_enabled": enabled}, + "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, + "memory": {"enabled": False}, + "summarization": {"enabled": False}, + "title": {"enabled": False}, + } + ) + return config + + +def test_checkpoint_persistence_is_opt_in(): + assert SubagentRuntimeConfig().checkpoint_enabled is False + assert SubagentRuntimeConfig(checkpoint_enabled=True).checkpoint_enabled is True + + +@pytest.mark.asyncio +async def test_peer_factory_recovers_after_import_under_temporary_gateway_patch(tmp_path, monkeypatch): + from deerflow.runtime.checkpointer import async_provider + + @asynccontextmanager + async def temporary_factory(_config): + yield object() + + # Load an isolated copy of the production module while the Gateway's + # resource factory is patched, without replacing its live registry. + path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/checkpoints.py" + spec = importlib.util.spec_from_file_location("_checkpoint_factory_import_probe", path) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + with monkeypatch.context() as importing: + importing.setattr(async_provider, "make_checkpointer", temporary_factory) + spec.loader.exec_module(module) + + config = _config(tmp_path) + invocation = {"configurable": {"thread_id": "restored-factory-thread"}} + async with make_checkpointer(config) as parent_saver: + runtime = module.SubagentCheckpointRuntime(config, parent_saver) + try: + await runtime.start() + builder = StateGraph(MessagesState) + builder.add_node("reply", lambda _state: {"messages": [AIMessage(content="RESTORED_FACTORY")]}) + builder.add_edge(START, "reply") + builder.add_edge("reply", END) + graph = builder.compile(checkpointer=runtime.saver) + state = await graph.ainvoke({"messages": [HumanMessage(content="Record evidence")]}, invocation) + assert state["messages"][-1].content == "RESTORED_FACTORY" + finally: + await runtime.stop() + + # A fresh reader must find real SQLite history after both writers close. + async with make_checkpointer(config) as reader: + saved = await reader.aget_tuple(invocation) + assert saved is not None + assert saved.checkpoint["channel_values"]["messages"][-1].content == "RESTORED_FACTORY" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", [False, True], ids=["disabled", "enabled"]) +@pytest.mark.parametrize("mode", ["full", "delta"]) +async def test_task_tool_checkpoint_history_reopens_with_delegation_identity(native_task_tool, tmp_path, monkeypatch, enabled, mode): + from langchain.agents import create_agent + + from deerflow.agents.thread_state import get_thread_state_schema + from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + + app_config = _config(tmp_path, enabled=enabled, mode=mode) + config_path = tmp_path / "config.yaml" + config_path.write_text(app_config.model_dump_json(), encoding="utf-8") + extensions_path = tmp_path / "extensions_config.json" + extensions_path.write_text("{}", encoding="utf-8") + monkeypatch.setenv("DEER_FLOW_CONFIG_PATH", str(config_path)) + monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path)) + monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [echo]) + freeze_checkpoint_channel_mode(mode) + + class ParentModel(_OfflineModel): + def _generate(self, messages, stop=None, run_manager=None, **kwargs): + if any(isinstance(message, ToolMessage) for message in messages): + answer = AIMessage(content="PARENT_FINAL") + else: + answer = AIMessage( + content="", + tool_calls=[{"id": "outer-call", "name": "task", "args": {"subagent_type": "general-purpose", "prompt": "Use echo once, then report the result", "description": "Offline checkpoint integration"}}], + ) + return ChatResult(generations=[ChatGeneration(message=answer)]) + + async with make_checkpointer(app_config) as saver: + async with subagent_checkpoint_runtime(app_config, saver): + parent = create_agent(model=ParentModel(), tools=[native_task_tool], checkpointer=saver, context_schema=dict, state_schema=get_thread_state_schema(mode)) + state = await asyncio.wait_for( + parent.ainvoke( + {"messages": [HumanMessage(content="Delegate the offline work")]}, + config={"configurable": {"thread_id": "task-history-thread"}, "metadata": {"model_name": "offline", "run_id": "task-parent-run"}}, + context={"app_config": app_config, "thread_id": "task-history-thread", "run_id": "task-parent-run", "user_id": "task-user", "user_role": "admin"}, + ), + timeout=25, + ) + outer_result = next(message for message in state["messages"] if isinstance(message, ToolMessage) and message.tool_call_id == "outer-call") + assert outer_result.additional_kwargs["subagent_status"] == "completed" + assert "CHILD_FINAL" in outer_result.content + assert state["messages"][-1].content == "PARENT_FINAL" + assert not any(isinstance(message, ToolMessage) and message.tool_call_id == "child-call" for message in state["messages"]) + + # Both writer lifetimes have ended: read from a new connection, just as + # a history consumer does after a Gateway restart. + async with make_checkpointer(app_config) as reader: + history = [entry async for entry in reader.alist({"configurable": {"thread_id": "task-history-thread"}})] + roots = [entry for entry in history if not entry.config["configurable"].get("checkpoint_ns")] + children = [entry for entry in history if entry.config["configurable"].get("checkpoint_ns")] + assert roots + if not enabled: + assert not children + return + assert children + namespaces = {entry.config["configurable"]["checkpoint_ns"] for entry in children} + execution_ids = {entry.metadata["deerflow_subagent_execution_id"] for entry in children} + assert len(namespaces) == len(execution_ids) == 1 + execution_id = next(iter(execution_ids)) + namespace = next(iter(namespaces)) + assert execution_id != "outer-call" + assert namespace.endswith(f"|deerflow_subagent:{execution_id}") + assert all(entry.metadata["deerflow_subagent_name"] == "general-purpose" for entry in children) + assert all(entry.metadata["deerflow_parent_run_id"] == "task-parent-run" for entry in children) + assert all(entry.metadata["deerflow_subagent_tool_call_id"] == "outer-call" for entry in children) + root_ids = {entry.checkpoint["id"] for entry in roots} + assert all(entry.metadata["parents"][""] in root_ids for entry in children) + + child = create_agent(model=_OfflineModel(), tools=[echo], state_schema=get_thread_state_schema(mode), checkpointer=reader) + restored = await child.aget_state({"configurable": {"thread_id": "task-history-thread", "checkpoint_ns": namespace, "__pregel_checkpointer": reader}}) + messages = restored.values["messages"] + calls = [call for message in messages if isinstance(message, AIMessage) for call in message.tool_calls] + assert any(call["id"] == "child-call" and call["name"] == "echo" and call["args"] == {"text": "child evidence"} for call in calls) + assert any(isinstance(message, ToolMessage) and message.tool_call_id == "child-call" and message.content == "child evidence" for message in messages) + assert messages[-1].content == "CHILD_FINAL" + assert all(message.content not in {"Delegate the offline work", "PARENT_FINAL"} for message in messages if isinstance(message.content, str)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("restriction", ["global", "subagent"]) +@pytest.mark.parametrize("rebuild_on_copy", [False, True], ids=["preserving-copy", "rebuilding-copy"]) +async def test_delta_middleware_copies_preserve_tool_restrictions(native_executor, tmp_path, monkeypatch, restriction, rebuild_on_copy): + from deerflow.config.authorization_config import AuthorizationConfig + from deerflow.config.extensions_config import ConfiguredMiddlewareSpec + from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + from deerflow.subagents.config import SubagentConfig + + app_config = _config(tmp_path, mode="delta") + app_config.extensions.middlewares = [ConfiguredMiddlewareSpec(class_path=f"{__name__}:_CopyingToolMiddleware", kwargs={"rebuild_on_copy": rebuild_on_copy}, apply_to="subagent")] + if restriction == "global": + app_config.authorization = AuthorizationConfig.model_validate({"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"user": {"tools": {"allow": "*", "deny": ["forbidden"]}}}}}}) + freeze_checkpoint_channel_mode("delta") + bound_names = [] + + class RecordingModel(_OfflineModel): + def bind_tools(self, tools, **kwargs): + bound_names.append([tool.name for tool in tools]) + return self + + monkeypatch.setattr(native_executor, "create_chat_model", lambda **kwargs: RecordingModel()) + + async def delegate(_state): + executor = native_executor.SubagentExecutor( + config=SubagentConfig(name="copy-policy", description="Check extension copies", skills=[], disallowed_tools=["forbidden"] if restriction == "subagent" else []), + tools=[echo], + parent_model="offline", + thread_id="copy-thread", + user_id="copy-user", + user_role="user", + app_config=app_config, + ) + execution_id = executor.execute_async("Use echo to return child evidence") + result = native_executor.get_background_task_result(execution_id) + try: + assert await asyncio.to_thread(result.execution_done_event.wait, 15) + if rebuild_on_copy: + assert result.status is native_executor.SubagentStatus.FAILED + assert "forbidden" in result.error + assert ("not authorized" if restriction == "global" else "final tool set") in result.error + assert not bound_names + else: + assert result.status is native_executor.SubagentStatus.COMPLETED, result.error + assert bound_names + assert all("echo" in names and "forbidden" not in names for names in bound_names) + finally: + native_executor.cleanup_background_task(result.task_id) + return {} + + builder = StateGraph(MessagesState) + builder.add_node("delegate", delegate) + builder.add_edge(START, "delegate") + builder.add_edge("delegate", END) + async with make_checkpointer(app_config) as saver: + async with subagent_checkpoint_runtime(app_config, saver): + parent = builder.compile(checkpointer=saver) + await parent.ainvoke({"messages": [HumanMessage(content="Check tool restrictions")]}, {"configurable": {"thread_id": "copy-thread"}}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", [False, True]) +@pytest.mark.parametrize("mode", ["full", "delta"]) +@pytest.mark.parametrize("clone_parent", [False, True]) +async def test_parallel_children_reopen_with_execution_identity(native_executor, tmp_path, enabled, mode, clone_parent): + from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + from deerflow.subagents.config import SubagentConfig + + app_config = _config(tmp_path, enabled=enabled, mode=mode) + freeze_checkpoint_channel_mode(mode) + completed = [] + parent_loop = asyncio.get_running_loop() + + async with make_checkpointer(app_config) as parent_saver: + async with subagent_checkpoint_runtime(app_config, parent_saver) as runtime: + + async def delegate(_state, config: RunnableConfig): + # Bind the real SQLite lock to the parent loop through actual + # contention, and keep it held until both children complete. + # This stabilizes the original cross-loop failure without + # injecting an exception or holding a database transaction. + raw = getattr(parent_saver, "source_saver", parent_saver) + await raw.lock.acquire() + waiter = asyncio.create_task(raw.setup()) + await asyncio.sleep(0) + assert raw.lock._loop is parent_loop + parent_context = var_child_runnable_config.get() + parent_namespace = parent_context["configurable"]["checkpoint_ns"] + inherited_saver = parent_context["configurable"]["__pregel_checkpointer"] + try: + results = [] + for _ in range(2): + executor = native_executor.SubagentExecutor( + config=SubagentConfig(name="checkpoint-test", description="Offline checkpoint regression", system_prompt="Use echo once, then answer.", timeout_seconds=30), + tools=[echo], + parent_model="offline", + thread_id="history-thread", + run_id=config["metadata"]["run_id"], + app_config=app_config, + ) + # Intentionally reused across children and parent runs. + execution_id = executor.execute_async("Return evidence", task_id="reused-provider-call") + results.append(native_executor.get_background_task_result(execution_id)) + await asyncio.wait_for(asyncio.gather(*(asyncio.to_thread(result.execution_done_event.wait) for result in results)), timeout=20) + for result in results: + assert result.status.value == "completed", result.error + completed.append((config["metadata"]["run_id"], result.task_id)) + finally: + raw.lock.release() + await waiter + for result in results: + native_executor.cleanup_background_task(result.task_id) + assert var_child_runnable_config.get() is parent_context + assert parent_context["configurable"]["checkpoint_ns"] == parent_namespace + assert parent_context["configurable"]["__pregel_checkpointer"] is inherited_saver + assert "deerflow_subagent_execution_id" not in parent_context["metadata"] + return {"messages": [AIMessage(content="PARENT_FINAL")]} + + builder = StateGraph(MessagesState) + builder.add_node("delegate", delegate) + builder.add_edge(START, "delegate") + builder.add_edge("delegate", END) + # Same connection, different saver identity: mirrors the strict + # serializer allowlist clones used by newer LangGraph releases. + graph_saver = parent_saver.with_allowlist({("builtins", "dict")}) if clone_parent else parent_saver + parent = builder.compile(checkpointer=graph_saver) + for run_id in ("parent-run-1", "parent-run-2"): + streamed = [ + message + async for message, _ in parent.astream( + {"messages": [HumanMessage(content="Delegate")]}, + config={"configurable": {"thread_id": "history-thread"}, "metadata": {"run_id": run_id}}, + stream_mode="messages", + ) + ] + assert any(message.content == "PARENT_FINAL" for message in streamed) + assert all(message.content != "CHILD_FINAL" and not isinstance(message, ToolMessage) for message in streamed) + if enabled: + assert runtime.saver is not parent_saver + assert runtime.loop is not parent_loop + if enabled: + assert runtime.closed + + # Fresh resources materialize evidence after both writer lifetimes end. + async with make_checkpointer(app_config) as reader: + history = [entry async for entry in reader.alist({"configurable": {"thread_id": "history-thread"}})] + root = [entry for entry in history if not entry.config["configurable"].get("checkpoint_ns")] + assert root + root_ids = {entry.checkpoint["id"] for entry in root} + assert all(message.content != "CHILD_FINAL" for entry in root for message in entry.checkpoint["channel_values"].get("messages", [])) + children = [entry for entry in history if entry.config["configurable"].get("checkpoint_ns")] + if not enabled: + assert not children + return + groups = defaultdict(list) + for entry in children: + groups[entry.metadata["deerflow_subagent_execution_id"]].append(entry) + assert set(groups) == {execution_id for _, execution_id in completed} + namespaces = set() + for parent_run_id, execution_id in completed: + entries = groups[execution_id] + own_namespaces = {entry.config["configurable"]["checkpoint_ns"] for entry in entries} + assert len(own_namespaces) == 1 + assert not own_namespaces & namespaces + namespaces.update(own_namespaces) + assert all(entry.metadata["deerflow_parent_run_id"] == parent_run_id for entry in entries) + assert all(entry.metadata["deerflow_subagent_tool_call_id"] == "reused-provider-call" for entry in entries) + assert all(entry.metadata["parents"][""] in root_ids for entry in entries) + # Use the actual child graph schema to reconstruct full and delta + # states, rather than expecting raw delta channel blobs to be text. + from deerflow.agents.thread_state import get_thread_state_schema + + accessor_builder = StateGraph(get_thread_state_schema(mode)) + accessor_builder.add_node("unused", lambda state: {}) + accessor_builder.add_edge(START, "unused") + accessor_builder.add_edge("unused", END) + accessor = accessor_builder.compile(checkpointer=reader) + snapshot = await accessor.aget_state({"configurable": {"thread_id": "history-thread", "checkpoint_ns": next(iter(own_namespaces)), "__pregel_checkpointer": reader}}) + messages = snapshot.values["messages"] + assert any(isinstance(message, ToolMessage) and "child evidence" in message.content for message in messages) + assert any(message.content == "CHILD_FINAL" for message in messages) + assert all(message.content not in {"Delegate", "PARENT_FINAL"} for message in messages if isinstance(message.content, str)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize("enabled", [False, True]) +async def test_checkpoint_history_only_accepts_sqlite(native_executor, tmp_path, legacy, enabled): + from deerflow.config.checkpointer_config import CheckpointerConfig + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + + config = _config(tmp_path, enabled=enabled) + async with make_checkpointer(config) as saver: + # Only exercise admission here: no PostgreSQL driver or server is used. + # A legacy section must take precedence over the unified SQLite config. + if legacy: + config.checkpointer = CheckpointerConfig(type="postgres") + else: + config.database.backend = "postgres" + if enabled: + with pytest.raises(ValueError, match="only supports file-backed SQLite"): + async with subagent_checkpoint_runtime(config, saver): + pytest.fail("Unsupported checkpoint backend was accepted") + else: + async with subagent_checkpoint_runtime(config, saver) as runtime: + assert runtime is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("legacy", [False, True]) +async def test_memory_checkpoint_history_is_rejected(native_executor, tmp_path, legacy): + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + + config = _config(tmp_path) + if legacy: + from deerflow.config.checkpointer_config import CheckpointerConfig + + config.checkpointer = CheckpointerConfig(type="sqlite", connection_string=":memory:") + else: + config.database.backend = "memory" + async with make_checkpointer(config) as saver: + with pytest.raises(ValueError, match="persistent|file-backed"): + async with subagent_checkpoint_runtime(config, saver): + pytest.fail("Non-persistent checkpoint storage was accepted") + + +@pytest.mark.asyncio +async def test_partial_host_startup_closes_peer_before_parent(native_executor, tmp_path): + from deerflow.subagents.checkpoints import _runtimes, subagent_checkpoint_runtime + + config = _config(tmp_path) + async with make_checkpointer(config) as saver: + with pytest.raises(RuntimeError, match="later startup failure"): + async with subagent_checkpoint_runtime(config, saver) as runtime: + raise RuntimeError("later startup failure") + assert runtime.closed + assert id(saver) not in _runtimes + # Parent resource remains usable after the peer's teardown. + await saver.setup() + + +@pytest.mark.asyncio +async def test_gateway_runtime_installs_and_closes_peer(native_executor, tmp_path): + from fastapi import FastAPI + + from app.gateway.deps import langgraph_runtime + from deerflow.subagents.checkpoints import _runtimes + + config = _config(tmp_path) + app = FastAPI() + async with langgraph_runtime(app, config): + saver = app.state.checkpointer + runtime = _runtimes[id(saver)] + assert runtime.saver is not saver + assert runtime.loop is not asyncio.get_running_loop() + assert config.run_events.backend == "memory" + await saver.setup() + assert runtime.closed + assert id(saver) not in _runtimes + + +@pytest.mark.asyncio +async def test_startup_cancellation_closes_created_peer(native_executor, tmp_path, monkeypatch): + from deerflow.subagents.checkpoints import SubagentCheckpointRuntime, _runtimes, subagent_checkpoint_runtime + + config = _config(tmp_path) + started = threading.Event() + release = threading.Event() + opened = [] + original_start = SubagentCheckpointRuntime.start + + async def paused_start(runtime): + await original_start(runtime) + opened.append(runtime) + started.set() + await asyncio.to_thread(release.wait) + + monkeypatch.setattr(SubagentCheckpointRuntime, "start", paused_start) + async with make_checkpointer(config) as saver: + manager = subagent_checkpoint_runtime(config, saver) + startup = asyncio.create_task(manager.__aenter__()) + try: + assert await asyncio.to_thread(started.wait, 10) + startup.cancel() + await asyncio.sleep(0) + startup.cancel() + await asyncio.sleep(0) + assert not startup.done() + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(startup, 10) + assert opened[0].closed + assert id(saver) not in _runtimes + await saver.setup() + + +@pytest.mark.asyncio +async def test_loop_shutdown_closes_peer_on_owner_loop(native_executor, tmp_path): + from deerflow.subagents.checkpoints import subagent_checkpoint_runtime + + config = _config(tmp_path) + async with make_checkpointer(config) as saver: + async with subagent_checkpoint_runtime(config, saver) as runtime: + owner = runtime.loop + await asyncio.to_thread(native_executor._shutdown_isolated_subagent_loop) + assert runtime.closed + assert owner.is_closed() + await saver.setup() + + +def test_default_sync_parent_keeps_async_child_isolated(native_executor, tmp_path): + from langgraph.checkpoint.sqlite import SqliteSaver + + from deerflow.subagents.config import SubagentConfig + + config = _config(tmp_path, enabled=False) + + def delegate(_state): + executor = native_executor.SubagentExecutor( + config=SubagentConfig(name="checkpoint-test", description="Sync parent regression", system_prompt="Use echo.", timeout_seconds=30), + tools=[echo], + parent_model="offline", + thread_id="sync-thread", + app_config=config, + ) + result = executor.execute("Return evidence") + assert result.status.value == "completed", result.error + return {"messages": [AIMessage(content="PARENT_DONE")]} + + with SqliteSaver.from_conn_string(str(tmp_path / "sync.sqlite")) as saver: + builder = StateGraph(MessagesState) + builder.add_node("delegate", delegate) + builder.add_edge(START, "delegate") + builder.add_edge("delegate", END) + parent = builder.compile(checkpointer=saver) + state = parent.invoke({"messages": [HumanMessage(content="Delegate")]}, {"configurable": {"thread_id": "sync-thread"}}) + assert state["messages"][-1].content == "PARENT_DONE" + + +@pytest.mark.asyncio +async def test_shutdown_drains_cancelled_child_and_fences_late_dispatch(native_executor, tmp_path): + from deerflow.subagents.checkpoints import _runtimes, subagent_checkpoint_runtime + from deerflow.subagents.config import SubagentConfig + + config = _config(tmp_path) + entered = threading.Event() + cleanup_entered = threading.Event() + release_cleanup = threading.Event() + results = [] + captured_contexts = [] + + @tool("echo") + async def blocking_echo(text: str) -> str: + """Pause a deterministic child until shutdown cancels it.""" + entered.set() + try: + await asyncio.Event().wait() + finally: + cleanup_entered.set() + await asyncio.to_thread(release_cleanup.wait) + return text + + def executor(): + return native_executor.SubagentExecutor( + config=SubagentConfig(name="checkpoint-test", description="Shutdown regression", system_prompt="Use echo.", timeout_seconds=30), + tools=[blocking_echo], + parent_model="offline", + thread_id="shutdown-thread", + run_id="shutdown-parent-run", + app_config=config, + ) + + async def delegate(_state): + captured_contexts.append(native_executor._copy_isolated_subagent_context()) + execution_id = executor().execute_async("Pause") + result = native_executor.get_background_task_result(execution_id) + results.append(result) + await asyncio.to_thread(result.execution_done_event.wait) + return {"messages": [AIMessage(content="PARENT_DONE")]} + + async with make_checkpointer(config) as saver: + manager = subagent_checkpoint_runtime(config, saver) + runtime = await manager.__aenter__() + builder = StateGraph(MessagesState) + builder.add_node("delegate", delegate) + builder.add_edge(START, "delegate") + builder.add_edge("delegate", END) + parent = builder.compile(checkpointer=saver) + parent_task = asyncio.create_task(parent.ainvoke({"messages": [HumanMessage(content="Pause")]}, {"configurable": {"thread_id": "shutdown-thread"}})) + stop = None + late_result = None + try: + assert await asyncio.to_thread(entered.wait, 10) + stop = asyncio.create_task(manager.__aexit__(None, None, None)) + assert await asyncio.to_thread(cleanup_entered.wait, 10) + stop.cancel() + await asyncio.sleep(0) + stop.cancel() + await asyncio.sleep(0) + assert not stop.done() + assert not runtime.closed + assert runtime.closing + await saver.setup() + + late_id = captured_contexts[0].run(executor().execute_async, "Late task") + late_result = native_executor.get_background_task_result(late_id) + assert await asyncio.to_thread(late_result.execution_done_event.wait, 10) + assert late_result.status.value == "failed" + assert "shutting down" in late_result.error + finally: + release_cleanup.set() + if stop is not None: + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(stop, 10) + else: + await manager.__aexit__(None, None, None) + await asyncio.wait_for(parent_task, 10) + for result in [*results, *([late_result] if late_result is not None else [])]: + native_executor.cleanup_background_task(result.task_id) + assert runtime.closed + assert id(saver) not in _runtimes + assert results[0].status.value == "cancelled" + assert results[0].execution_done_event.is_set() + await saver.setup() + # A dispatch context captured before shutdown must remain fenced even + # after the runtime has been removed from the process registry. + after_close_id = captured_contexts[0].run(executor().execute_async, "After close") + after_close = native_executor.get_background_task_result(after_close_id) + try: + assert await asyncio.to_thread(after_close.execution_done_event.wait, 10) + assert after_close.status.value == "failed" + assert "shutting down" in after_close.error + finally: + native_executor.cleanup_background_task(after_close_id) diff --git a/backend/tests/test_subagent_checkpointer_isolation.py b/backend/tests/test_subagent_checkpointer_isolation.py index 45eb4745b0..f1d87e99a4 100644 --- a/backend/tests/test_subagent_checkpointer_isolation.py +++ b/backend/tests/test_subagent_checkpointer_isolation.py @@ -6,9 +6,9 @@ DeerFlowClient), the subagent's ``agent.astream()`` inherits it through at compile time, LangGraph's resolution prioritizes the inherited value and calls the sync checkpointer's async methods, raising NotImplementedError. -The subagent is a one-shot delegation — it rebuilds state, calls astream -once, and extracts the last AIMessage. It never resumes, so persistence -is unnecessary and inheriting the parent checkpointer is harmful. +The default subagent is a one-shot delegation and remains checkpoint-disabled. +Opt-in Gateway history replaces the inherited saver with a loop-owned peer; +that separate path is covered by test_subagent_checkpoint_persistence.py. """ import asyncio @@ -83,14 +83,14 @@ def _setup_executor_module(): class TestSubagentCheckpointerIsolation: - """Verify _create_agent() unconditionally passes checkpointer=False to create_agent().""" + """Verify _create_agent() passes checkpointer=False without an opted-in scope.""" def test_create_agent_receives_checkpointer_false( self, _setup_executor_module, monkeypatch: pytest.MonkeyPatch, ): - """Assert checkpointer=False is always passed to create_agent().""" + """Assert checkpointer=False is passed by default to create_agent().""" SubagentConfig = _setup_executor_module["SubagentConfig"] SubagentExecutor = _setup_executor_module["SubagentExecutor"] executor_module = _setup_executor_module["executor_module"] diff --git a/config.example.yaml b/config.example.yaml index 5b787da857..25ca877c70 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -2030,6 +2030,10 @@ sandbox: # shared by ordinary `task` calls and durable batches. Waiting work is async; # queued subagents do not occupy a scheduler thread. subagent_runtime: + # Optional history for ordinary task delegates under a persistent parent + # graph. Currently supports only file-backed SQLite checkpoint storage. + # Restart Gateway after changing; independent of run_events.backend. + checkpoint_enabled: false max_running: 3 max_queued: 64 admission_policy: queue # queue or reject when all slots are occupied diff --git a/docs/guide/subagents.md b/docs/guide/subagents.md index 884dadd5b0..1f30286cb9 100644 --- a/docs/guide/subagents.md +++ b/docs/guide/subagents.md @@ -1,5 +1,84 @@ # Sub-Agents +## Optional checkpoint history + +The Gateway can save checkpoint history for ordinary subagent executions. This +is disabled by default. + +The default configuration already sets `database.backend` to `sqlite`. +Keep the `database:` settings unchanged. In `config.yaml`, set +`subagent_runtime.checkpoint_enabled` to `true`: + +```yaml +subagent_runtime: + checkpoint_enabled: true +``` + +Restart the Gateway for the change to take effect. Parent and child checkpoints are saved in the same SQLite database selected by the host. When the host uses the default `database:` configuration, this is `deerflow.db` under `database.sqlite_dir`. + +This option currently supports SQLite; PostgreSQL and memory storage are not +supported. Both `full` and `delta` checkpoint channel modes are supported. Keep +the database on persistent storage to retain history across Gateway or +container restarts. This option is independent of `run_events.backend`, which +can remain `memory`. + +The Gateway owns a second saver on the persistent subagent event loop. Subagents +using this checkpoint runtime share it, while the parent keeps its own saver and +both connect to the same checkpoint database. Each child execution gets a +distinct non-root namespace under its parent node. Child messages remain outside +the parent transcript and its root message stream. + +The normal `task` delegation flow supplies the required parent graph context. +Calling the executor directly from a parent graph node can also save history +when it inherits that context and uses the installed checkpoint runtime. +This option covers direct delegation from the parent graph. If a subagent +dispatches another subagent, the nested execution runs with checkpoint history +disabled. Tool calls and results within a recorded subagent can still be +retained in its checkpoint history. Checkpoint history for items executed through `batch_task` is not included. + +Child checkpoint metadata includes `deerflow_subagent_execution_id` and +`deerflow_subagent_name`, plus `deerflow_parent_run_id` and +`deerflow_subagent_tool_call_id` when those identities are available. Use the +server execution ID to distinguish attempts; provider tool-call IDs can repeat. +The inherited checkpoint parent map retains the parent checkpoint reference. +Readers can use these fields to attribute retained history without inferring +ownership from timestamps. Concurrent children do not have a single global +execution order. + +This option stores historical evidence and adds checkpoint writes and storage +usage. It does not add a history UI or automatic subagent recovery. Standalone +embedded clients and direct factories retain their existing checkpoint-disabled +behavior unless they explicitly install the matching checkpoint resource +lifetime around a persistent parent graph. + +Child checkpoint history is not automatically pruned by the existing checkpoint +retention policies. Each child execution has its own namespace, whose latest +checkpoint and ancestor chain are protected. Deleting the entire thread is +currently the only supported way to reclaim this history. Storage usage grows +with retained child executions. + +The Gateway closes admission and drains child execution, stream cleanup and +checkpoint writes before closing the child saver. The parent saver remains open +through that drain. + +### Checkpoint resource ownership + +All native subagents share one persistent event loop. The Gateway installs the +optional child saver inside the parent saver's lifetime. Capture that runtime +before dispatch and replace the inherited saver in a copied runnable config; +LangGraph's inherited reference takes precedence over a compile-time saver. +Identify strict-msgpack serializer clones by their underlying connection, +because a clone can share its original saver's connection and lock. + +Extend the ambient checkpoint namespace with the server execution ID and retain +its parent map. Keep checkpoint coordinates out of the explicit child invocation +config, which can reset the namespace on newer LangGraph versions. Drain complete +child execution and cleanup before closing the saver or stopping its owner loop. +Delta schema normalization can copy middleware; audit both global authorization +and subagent tool restrictions on the final instances before creating the graph. + +## Delegation behavior + Web chats accept direct model choices such as “ask MODEL to handle this”, “use MODEL for this turn”, and “from now on use MODEL”; Chinese equivalents are supported too. A child choice leaves the main model unchanged, a one-turn choice leaves defaults unchanged, and a conversation choice uses the existing browser-local thread override without changing account preferences. Choices resolve against available configurations and optional `models[].aliases`; ambiguous names or scopes show a selection card before execution, while delegation still requires enabled subagents (Ultra mode). When a sub-agent ends with `return_direct=True` tools, including tools contributed by extension middleware, their outputs are returned in tool-call order. A failed tool marks the task as failed while preserving the batch outputs.