mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 08:20:34 +00:00
feat(subagents): add optional SQLite checkpoint history (#8265)
* feat(subagents): add optional SQLite checkpoint history * docs(subagents): clarify checkpoint history limits * fix(subagents): resolve checkpoint factory at startup --------- Co-authored-by: Nan Gao <thunderggnn@gmail.com>
This commit is contained in:
parent
5690221272
commit
c5d072af91
@ -113,7 +113,7 @@ Each guide is the full reference for its topic. Open the one you need.
|
||||
| Sandbox modes, file system, uploads and outputs | [Sandbox](docs/guide/sandbox.md) |
|
||||
| Skills, built-in tools, script workflows, Claude Code integration | [Skills & Tools](docs/guide/skills-and-tools.md) |
|
||||
| MCP servers | [MCP Server](docs/guide/mcp.md) |
|
||||
| Sub-agents and task delegation | [Sub-Agents](docs/guide/subagents.md), [Tool restrictions](docs/guide/subagents.md#tool-restrictions) |
|
||||
| Sub-agents, task delegation and optional SQLite checkpoint history | [Sub-Agents](docs/guide/subagents.md), [Tool restrictions](docs/guide/subagents.md#tool-restrictions) |
|
||||
| Long-term memory | [Long-Term Memory](docs/guide/memory.md) |
|
||||
| Session goals, context compaction, task notes, evidence-based decisions | [Context, Goals and Agent Decisions](docs/guide/context.md) |
|
||||
| Private knowledge retrieval (RAGFlow) | [Private Knowledge Retrieval](docs/guide/knowledge-retrieval.md) |
|
||||
|
||||
@ -873,6 +873,11 @@ async def langgraph_runtime(app: FastAPI, startup_config: AppConfig) -> AsyncGen
|
||||
await init_engine_from_config(config.database)
|
||||
|
||||
app.state.checkpointer = await stack.enter_async_context(make_checkpointer(config))
|
||||
# The peer is closed before the parent saver, including partial
|
||||
# startup failure. Its own shutdown fences and drains isolated tasks.
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
|
||||
await stack.enter_async_context(subagent_checkpoint_runtime(config, app.state.checkpointer))
|
||||
app.state.store = await stack.enter_async_context(make_store(config))
|
||||
|
||||
# Record the checkpointer/Store backend and the provisioner endpoint
|
||||
|
||||
@ -85,7 +85,10 @@ STARTUP_ONLY_FIELDS: dict[str, str] = {
|
||||
"McpTaskService is constructed and started once during Gateway lifespan startup; enabled, poll_interval_seconds, lease_seconds, "
|
||||
"and max_concurrent_polls are captured into the service instance and the background poller task is not rebuilt on config.yaml edits."
|
||||
),
|
||||
"subagent_runtime": ("the shared native-subagent admission controller and isolated execution loop are configured once during Gateway lifespan startup; changing process slots, queue policy, or queue bounds requires a restart."),
|
||||
"subagent_runtime": (
|
||||
"the shared native-subagent admission controller, isolated execution loop, and optional checkpoint saver are configured once during Gateway lifespan startup; "
|
||||
"changing checkpoint persistence, process slots, queue policy, or queue bounds requires a restart."
|
||||
),
|
||||
"subagent_batches": ("the durable subagent batch service is constructed and started once during Gateway lifespan startup; scheduler limits, leases, and recovery behavior are captured by that service instance."),
|
||||
"deployment": (
|
||||
"langgraph_runtime() evaluates deployment.multi_instance once at startup inside the multi-process safety gate (_enforce_postgres_for_multi_worker); "
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""Startup-only process capacity for native subagent execution."""
|
||||
"""Startup-only capacity and checkpoint settings for native subagents."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
@ -6,7 +6,12 @@ from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SubagentRuntimeConfig(BaseModel):
|
||||
"""Process-local admission and execution limits shared by all subagents."""
|
||||
"""Process-local execution capacity and optional native task history."""
|
||||
|
||||
checkpoint_enabled: bool = Field(
|
||||
default=False,
|
||||
description="Persist native task checkpoint history through a loop-owned saver. Requires file-backed SQLite checkpoint storage and a Gateway restart.",
|
||||
)
|
||||
|
||||
max_running: int = Field(
|
||||
default=3,
|
||||
|
||||
@ -78,9 +78,8 @@ from `deerflow.utils.messages`. Keep delayed Steer reports as sanitized history.
|
||||
- **Sandbox use is leased.** An admitted subagent carries a stable lease owner ID
|
||||
and command scope ID derived from its task, so the shared sandbox can stop only
|
||||
that subagent's processes.
|
||||
- **All native subagents share one persistent event loop.** Keep loop and thread
|
||||
ownership until the worker has stopped. A bounded join does not prove that it
|
||||
stopped.
|
||||
- **Retain loop/thread ownership until subagents stop; a bounded join is insufficient.**
|
||||
See [checkpoint lifetimes](../../../../../docs/guide/subagents.md#checkpoint-resource-ownership).
|
||||
- **Delegation is not the default.** The routing policy permits `task` only when
|
||||
parallel latency, a specialist skill or context isolation clearly outweighs the
|
||||
startup cost and the duplicated discovery.
|
||||
|
||||
228
backend/packages/harness/deerflow/subagents/checkpoints.py
Normal file
228
backend/packages/harness/deerflow/subagents/checkpoints.py
Normal file
@ -0,0 +1,228 @@
|
||||
"""Loop-owned SQLite checkpoint resources for opt-in native task history.
|
||||
|
||||
The registry associates a parent storage owner with its isolated-loop peer. It
|
||||
does not infer a database from a request or hot-reload infrastructure settings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from contextvars import Context, ContextVar
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.runnables.config import var_child_runnable_config
|
||||
from langgraph._internal._constants import CONFIG_KEY_CHECKPOINTER, NS_SEP
|
||||
from langgraph.checkpoint.base import BaseCheckpointSaver
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.runtime.checkpointer import async_provider
|
||||
from deerflow.utils.file_io import await_drained
|
||||
|
||||
_runtimes: dict[int, SubagentCheckpointRuntime] = {}
|
||||
_registry_lock = threading.Lock()
|
||||
_captured_runtime: ContextVar[SubagentCheckpointRuntime | None] = ContextVar("subagent_checkpoint_runtime", default=None)
|
||||
_active_runtime: ContextVar[SubagentCheckpointRuntime | None] = ContextVar("active_subagent_checkpoint_runtime", default=None)
|
||||
|
||||
|
||||
class SubagentCheckpointError(RuntimeError):
|
||||
"""Checkpoint admission failed before a native graph could start."""
|
||||
|
||||
|
||||
async def _on_isolated_loop[T](factory: Callable[[], Coroutine[Any, Any, T]], *, loop: asyncio.AbstractEventLoop | None = None) -> T:
|
||||
# Infrastructure startup/teardown must not inherit a run's graph context.
|
||||
from deerflow.subagents.executor import _submit_to_isolated_loop_in_context
|
||||
|
||||
if loop is None:
|
||||
# First use may start a thread and wait for its loop-ready handshake.
|
||||
future = await asyncio.to_thread(_submit_to_isolated_loop_in_context, Context(), factory)
|
||||
else:
|
||||
if loop.is_closed() or not loop.is_running():
|
||||
raise RuntimeError("Subagent checkpoint owner loop stopped before its resources were drained")
|
||||
|
||||
def submit():
|
||||
coroutine = factory()
|
||||
try:
|
||||
return asyncio.run_coroutine_threadsafe(coroutine, loop)
|
||||
except BaseException:
|
||||
coroutine.close()
|
||||
raise
|
||||
|
||||
future = Context().run(submit)
|
||||
return await asyncio.wrap_future(future)
|
||||
|
||||
|
||||
class SubagentCheckpointRuntime:
|
||||
"""One saver shared by native children on their persistent execution loop."""
|
||||
|
||||
def __init__(self, config: AppConfig, parent_saver: BaseCheckpointSaver) -> None:
|
||||
self.config = config.model_copy(deep=True)
|
||||
self.parent_saver = parent_saver
|
||||
self.mode = config.database.checkpoint_channel_mode
|
||||
self.loop: asyncio.AbstractEventLoop | None = None
|
||||
self.saver: BaseCheckpointSaver | None = None
|
||||
self.closed = False
|
||||
self.closing = False
|
||||
self._stack: AsyncExitStack | None = None
|
||||
self._executions: set[asyncio.Task] = set()
|
||||
self._stop_task: asyncio.Task | None = None
|
||||
|
||||
def owns_parent_saver(self, saver: BaseCheckpointSaver | None) -> bool:
|
||||
# Strict msgpack support makes shallow saver clones with a derived
|
||||
# serializer. They still share the parent's connection and lock.
|
||||
source = getattr(saver, "source_saver", saver)
|
||||
owner = getattr(self.parent_saver, "source_saver", self.parent_saver)
|
||||
if source is owner:
|
||||
return True
|
||||
connection = getattr(source, "conn", None)
|
||||
return connection is not None and connection is getattr(owner, "conn", None)
|
||||
|
||||
async def start(self) -> None:
|
||||
self.loop = asyncio.get_running_loop()
|
||||
self._stack = AsyncExitStack()
|
||||
self.saver = await self._stack.enter_async_context(async_provider.make_checkpointer(self.config))
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self.closed:
|
||||
return
|
||||
if asyncio.get_running_loop() is not self.loop:
|
||||
raise RuntimeError("Subagent checkpoint resources must close on their owning event loop")
|
||||
self.closing = True
|
||||
if self._stop_task is None:
|
||||
self._stop_task = asyncio.create_task(self._stop())
|
||||
await asyncio.shield(self._stop_task)
|
||||
|
||||
async def _stop(self) -> None:
|
||||
# These tasks own their complete stream/checkpoint cleanup tails, not
|
||||
# just their public terminal status. Do not close storage until they exit.
|
||||
tasks = tuple(self._executions)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
if self._stack is not None:
|
||||
await self._stack.aclose()
|
||||
self.closed = True
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def subagent_checkpoint_runtime(config: AppConfig, parent_saver: BaseCheckpointSaver) -> AsyncIterator[SubagentCheckpointRuntime | None]:
|
||||
"""Own the isolated saver inside the parent saver's application lifetime.
|
||||
|
||||
Admission is captured at dispatch; shutdown fences new leases, cancels and
|
||||
drains existing executions, and closes the peer on its owning loop. The
|
||||
parent connection must remain open until this context exits.
|
||||
"""
|
||||
if getattr(getattr(config, "subagent_runtime", None), "checkpoint_enabled", False) is not True:
|
||||
yield None
|
||||
return
|
||||
source = getattr(parent_saver, "source_saver", parent_saver)
|
||||
legacy = config.checkpointer
|
||||
backend = legacy.type if legacy is not None else config.database.backend
|
||||
if backend != "sqlite" or isinstance(source, InMemorySaver):
|
||||
raise ValueError("subagent_runtime.checkpoint_enabled only supports file-backed SQLite checkpoint storage")
|
||||
if legacy is not None and legacy.type == "sqlite" and legacy.connection_string == ":memory:":
|
||||
raise ValueError("subagent_runtime.checkpoint_enabled requires file-backed SQLite, not :memory:")
|
||||
|
||||
runtime = SubagentCheckpointRuntime(config, parent_saver)
|
||||
registered = False
|
||||
try:
|
||||
await await_drained(_on_isolated_loop(runtime.start))
|
||||
with _registry_lock:
|
||||
if id(parent_saver) in _runtimes:
|
||||
raise RuntimeError("Subagent checkpoint runtime is already installed for this parent saver")
|
||||
_runtimes[id(parent_saver)] = runtime
|
||||
registered = True
|
||||
yield runtime
|
||||
finally:
|
||||
try:
|
||||
if runtime.loop is not None and not runtime.closed:
|
||||
await await_drained(_on_isolated_loop(runtime.stop, loop=runtime.loop))
|
||||
finally:
|
||||
if registered and runtime.closed:
|
||||
with _registry_lock:
|
||||
_runtimes.pop(id(parent_saver), None)
|
||||
|
||||
|
||||
def has_checkpoint_runtimes(loop: asyncio.AbstractEventLoop) -> bool:
|
||||
with _registry_lock:
|
||||
return any(runtime.loop is loop and not runtime.closed for runtime in _runtimes.values())
|
||||
|
||||
|
||||
async def drain_checkpoint_runtimes() -> None:
|
||||
"""Last-resort loop shutdown still drains its registered storage owners."""
|
||||
loop = asyncio.get_running_loop()
|
||||
with _registry_lock:
|
||||
runtimes = [runtime for runtime in _runtimes.values() if runtime.loop is loop]
|
||||
for runtime in runtimes:
|
||||
await runtime.stop()
|
||||
|
||||
|
||||
def capture_checkpoint_runtime(context: Context) -> None:
|
||||
"""Pin an installed runtime before a copied context crosses to the worker."""
|
||||
inherited = context.get(var_child_runnable_config) or {}
|
||||
parent = inherited.get("configurable", {}).get(CONFIG_KEY_CHECKPOINTER)
|
||||
with _registry_lock:
|
||||
runtime = _runtimes.get(id(parent))
|
||||
if runtime is None:
|
||||
runtime = next((candidate for candidate in _runtimes.values() if candidate.owns_parent_saver(parent)), None)
|
||||
captured = context.get(_captured_runtime)
|
||||
if runtime is None and captured is not None and captured.owns_parent_saver(parent):
|
||||
runtime = captured
|
||||
# A captured runtime remains fenced after its registration is removed.
|
||||
context.run(_captured_runtime.set, runtime)
|
||||
|
||||
|
||||
def checkpoint_runtime() -> SubagentCheckpointRuntime | None:
|
||||
"""Return only a runtime leased by the current native execution."""
|
||||
return _active_runtime.get()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def subagent_checkpoint_scope(*, execution_id: str, run_id: str | None, tool_call_id: str | None, agent_name: str) -> AsyncIterator[None]:
|
||||
runtime = _captured_runtime.get()
|
||||
if runtime is None:
|
||||
yield
|
||||
return
|
||||
if runtime.closing or runtime.closed:
|
||||
raise SubagentCheckpointError("Subagent checkpoint runtime is shutting down")
|
||||
if asyncio.get_running_loop() is not runtime.loop:
|
||||
raise SubagentCheckpointError("Subagent checkpoint runtime must be used on its owning event loop")
|
||||
inherited = var_child_runnable_config.get() or {}
|
||||
configurable = inherited.get("configurable", {})
|
||||
if not runtime.owns_parent_saver(configurable.get(CONFIG_KEY_CHECKPOINTER)):
|
||||
raise SubagentCheckpointError("Subagent checkpoint runtime does not match the inherited parent saver")
|
||||
if not configurable.get("thread_id") or not configurable.get("checkpoint_ns"):
|
||||
raise SubagentCheckpointError("Subagent checkpoint history requires an inherited parent graph namespace")
|
||||
|
||||
# Replace the inherited saver, not just the compile-time argument:
|
||||
# LangGraph gives CONFIG_KEY_CHECKPOINTER precedence over that argument.
|
||||
# A server execution ID also gives each independent attempt a fresh history,
|
||||
# even if a provider reuses a tool ID or a parent node is replayed.
|
||||
child_configurable = {
|
||||
**configurable,
|
||||
CONFIG_KEY_CHECKPOINTER: runtime.saver,
|
||||
"checkpoint_ns": NS_SEP.join((configurable["checkpoint_ns"], f"deerflow_subagent:{execution_id}")),
|
||||
}
|
||||
metadata = {
|
||||
**inherited.get("metadata", {}),
|
||||
"deerflow_subagent_execution_id": execution_id,
|
||||
"deerflow_subagent_name": agent_name,
|
||||
}
|
||||
if run_id:
|
||||
metadata["deerflow_parent_run_id"] = run_id
|
||||
if tool_call_id:
|
||||
metadata["deerflow_subagent_tool_call_id"] = tool_call_id
|
||||
config_token = var_child_runnable_config.set({**inherited, "configurable": child_configurable, "metadata": metadata})
|
||||
runtime_token = _active_runtime.set(runtime)
|
||||
task = asyncio.current_task()
|
||||
runtime._executions.add(task)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
runtime._executions.discard(task)
|
||||
_active_runtime.reset(runtime_token)
|
||||
var_child_runnable_config.reset(config_token)
|
||||
@ -704,7 +704,27 @@ def _shutdown_isolated_subagent_loop(*, only_if_pending: bool = False) -> None:
|
||||
return
|
||||
|
||||
if loop.is_running():
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
from deerflow.subagents.checkpoints import drain_checkpoint_runtimes, has_checkpoint_runtimes
|
||||
|
||||
if has_checkpoint_runtimes(loop):
|
||||
|
||||
async def drain_then_stop() -> None:
|
||||
await drain_checkpoint_runtimes()
|
||||
loop.call_soon(loop.stop)
|
||||
|
||||
cleanup = asyncio.run_coroutine_threadsafe(drain_then_stop(), loop)
|
||||
if thread is threading.current_thread():
|
||||
return
|
||||
try:
|
||||
cleanup.result(timeout=1)
|
||||
except FuturesTimeoutError:
|
||||
logger.warning("Retaining isolated subagent loop ownership until checkpoint resources finish draining")
|
||||
return
|
||||
except Exception:
|
||||
logger.exception("Retaining isolated subagent loop ownership because checkpoint resource cleanup failed")
|
||||
return
|
||||
else:
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
|
||||
if thread is not None and thread.is_alive() and thread is not threading.current_thread():
|
||||
thread.join(timeout=1)
|
||||
@ -808,10 +828,10 @@ def _run_on_isolated_loop_if_live(callback: Callable[[], None]) -> None:
|
||||
callback()
|
||||
|
||||
|
||||
def _submit_to_isolated_loop_in_context(
|
||||
def _submit_to_isolated_loop_in_context[T](
|
||||
context: Context,
|
||||
coro_factory: Callable[[], Coroutine[Any, Any, SubagentResult]],
|
||||
) -> Future[SubagentResult]:
|
||||
coro_factory: Callable[[], Coroutine[Any, Any, T]],
|
||||
) -> Future[T]:
|
||||
"""Submit a coroutine to the isolated loop while preserving ContextVar state.
|
||||
|
||||
The loop must be resolved before the coroutine is created: as direct
|
||||
@ -826,7 +846,7 @@ def _submit_to_isolated_loop_in_context(
|
||||
before the error propagates.
|
||||
"""
|
||||
|
||||
def _submit() -> Future[SubagentResult]:
|
||||
def _submit() -> Future[T]:
|
||||
loop = _get_isolated_subagent_loop()
|
||||
coroutine = coro_factory()
|
||||
try:
|
||||
@ -867,6 +887,9 @@ def _copy_isolated_subagent_context() -> Context:
|
||||
continue to reach the parent stream.
|
||||
"""
|
||||
context = copy_context()
|
||||
from deerflow.subagents.checkpoints import capture_checkpoint_runtime
|
||||
|
||||
capture_checkpoint_runtime(context)
|
||||
inherited_config = context.get(var_child_runnable_config)
|
||||
if inherited_config is None or "callbacks" not in inherited_config:
|
||||
return context
|
||||
@ -1251,7 +1274,6 @@ class SubagentExecutor:
|
||||
authorization_provider=authz_provider,
|
||||
)
|
||||
middlewares = apply_declared_tool_view(middlewares, authorized_names=declared_authorized)
|
||||
verify_declared_tool_view(middlewares, authorized_names=declared_authorized)
|
||||
elif authz_provider is not None:
|
||||
# Defense in depth: the provider is configured but the seeded
|
||||
# decision state is missing, so the declaration pass is skipped.
|
||||
@ -1281,6 +1303,17 @@ class SubagentExecutor:
|
||||
|
||||
policy_names = ToolPolicyNameResolver()
|
||||
middlewares = apply_declared_tool_policy(middlewares, allowed=self.config.tools, disallowed=self.config.disallowed_tools, name_resolver=policy_names)
|
||||
|
||||
from deerflow.subagents.checkpoints import checkpoint_runtime
|
||||
|
||||
history_runtime = checkpoint_runtime()
|
||||
state_schema = ThreadState
|
||||
if history_runtime is not None and history_runtime.mode == "delta":
|
||||
from deerflow.agents.thread_state import get_thread_state_schema, normalize_middleware_state_schemas
|
||||
|
||||
state_schema = get_thread_state_schema("delta")
|
||||
middlewares = normalize_middleware_state_schemas(middlewares, "delta")
|
||||
|
||||
# Collect every guard middleware that exposes ``consume_stop_reason``
|
||||
# (TokenBudgetMiddleware, LoopDetectionMiddleware,
|
||||
# ModelLengthFinishReasonMiddleware) so _aexecute can read
|
||||
@ -1293,6 +1326,10 @@ class SubagentExecutor:
|
||||
|
||||
# system_prompt is included in initial state messages (see _build_initial_state)
|
||||
# to avoid multiple SystemMessages which some LLM APIs don't support.
|
||||
# Delta normalization can copy middleware, so audit both policies on
|
||||
# the final instances that create_agent will bind.
|
||||
if declared_authorized is not None:
|
||||
verify_declared_tool_view(middlewares, authorized_names=declared_authorized)
|
||||
verify_subagent_tool_policy(
|
||||
bound_tools,
|
||||
middlewares,
|
||||
@ -1306,8 +1343,11 @@ class SubagentExecutor:
|
||||
tools=bound_tools,
|
||||
middleware=middlewares,
|
||||
system_prompt=None,
|
||||
state_schema=ThreadState,
|
||||
checkpointer=False,
|
||||
state_schema=state_schema,
|
||||
# The opted-in scope replaces the inherited saver with its
|
||||
# loop-owned peer before graph construction. All other entrypoints
|
||||
# keep isolation from a possibly synchronous parent saver.
|
||||
checkpointer=None if history_runtime is not None else False,
|
||||
)
|
||||
# Use the compiled registry: it includes middleware tools, normalized
|
||||
# callables, and LangChain's last-definition-wins name precedence.
|
||||
@ -1732,17 +1772,25 @@ class SubagentExecutor:
|
||||
trace_id=self.trace_id,
|
||||
status=SubagentStatus.PENDING,
|
||||
)
|
||||
from deerflow.subagents.checkpoints import SubagentCheckpointError, subagent_checkpoint_scope
|
||||
|
||||
with ensure_trace_context(self.deerflow_trace_id):
|
||||
try:
|
||||
capacity = self.execution_capacity or get_subagent_execution_capacity()
|
||||
slot = capacity.slot(wait=False) if self.input_gate is not None and not self.input_gate.can_queue else capacity.slot()
|
||||
async with slot:
|
||||
async with (
|
||||
subagent_checkpoint_scope(execution_id=result.task_id, run_id=self.run_id, tool_call_id=result.external_task_id, agent_name=self.config.name),
|
||||
slot,
|
||||
):
|
||||
with result._state_lock:
|
||||
if result.status.is_terminal:
|
||||
return result
|
||||
result.status = SubagentStatus.RUNNING
|
||||
result.started_at = _utcnow()
|
||||
return await self._aexecute_admitted(task, result)
|
||||
except SubagentCheckpointError as exc:
|
||||
result.try_set_terminal(SubagentStatus.FAILED, error=str(exc), admission_failure=True)
|
||||
return result
|
||||
except SubagentCapacityError as exc:
|
||||
result.try_set_terminal(
|
||||
SubagentStatus.FAILED,
|
||||
|
||||
@ -0,0 +1,40 @@
|
||||
"""Opening the isolated saver must leave the Gateway loop responsive."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checkpoint_peer_lifetime_does_not_block_gateway_loop(monkeypatch, tmp_path):
|
||||
import deerflow.agents.thread_state # noqa: F401
|
||||
import deerflow.subagents as package
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
|
||||
monkeypatch.delitem(sys.modules, "deerflow.subagents.executor", raising=False)
|
||||
monkeypatch.delattr(package, "executor", raising=False)
|
||||
executor = importlib.import_module("deerflow.subagents.executor")
|
||||
config = AppConfig.model_validate(
|
||||
{
|
||||
"database": {"backend": "sqlite", "sqlite_dir": str(tmp_path)},
|
||||
"subagent_runtime": {"checkpoint_enabled": True},
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
}
|
||||
)
|
||||
try:
|
||||
async with make_checkpointer(config) as saver:
|
||||
async with subagent_checkpoint_runtime(config, saver) as runtime:
|
||||
assert runtime.loop is not asyncio.get_running_loop()
|
||||
assert runtime.saver is not saver
|
||||
await saver.setup()
|
||||
assert runtime.closed
|
||||
finally:
|
||||
await asyncio.to_thread(executor._shutdown_isolated_subagent_loop)
|
||||
if getattr(package, "executor", None) is executor:
|
||||
delattr(package, "executor")
|
||||
692
backend/tests/test_subagent_checkpoint_persistence.py
Normal file
692
backend/tests/test_subagent_checkpoint_persistence.py
Normal file
@ -0,0 +1,692 @@
|
||||
"""Native subagent checkpoint history across the real isolated-loop boundary.
|
||||
|
||||
The model is deterministic and local. The executor, graph builder, middleware,
|
||||
SQLite saver, context propagation and background execution are production code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import importlib.util
|
||||
import sys
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langchain_core.outputs import ChatGeneration, ChatResult
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
from langchain_core.runnables.config import var_child_runnable_config
|
||||
from langchain_core.tools import tool
|
||||
from langgraph.graph import END, START, MessagesState, StateGraph
|
||||
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.paths import Paths
|
||||
from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig
|
||||
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
|
||||
|
||||
|
||||
class _OfflineModel(BaseChatModel):
|
||||
@property
|
||||
def _llm_type(self):
|
||||
return "offline-subagent-checkpoints"
|
||||
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
return self
|
||||
|
||||
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
if any(isinstance(message, ToolMessage) for message in messages):
|
||||
answer = AIMessage(content="CHILD_FINAL")
|
||||
else:
|
||||
answer = AIMessage(content="", tool_calls=[{"name": "echo", "args": {"text": "child evidence"}, "id": "child-call", "type": "tool_call"}])
|
||||
return ChatResult(generations=[ChatGeneration(message=answer)])
|
||||
|
||||
|
||||
@tool
|
||||
async def echo(text: str) -> str:
|
||||
"""Return deterministic evidence without external side effects."""
|
||||
return text
|
||||
|
||||
|
||||
@tool
|
||||
async def forbidden() -> str:
|
||||
"""A declaration that the test policy must remove before model binding."""
|
||||
pytest.fail("A denied middleware tool executed")
|
||||
|
||||
|
||||
class _CopyingToolMiddleware(AgentMiddleware):
|
||||
"""Exercise state-preserving and constructor-rebuilding extension copies."""
|
||||
|
||||
def __init__(self, *, rebuild_on_copy: bool = False):
|
||||
self.rebuild_on_copy = rebuild_on_copy
|
||||
self.tools = [echo, forbidden]
|
||||
|
||||
def __copy__(self):
|
||||
clone = type(self)(rebuild_on_copy=self.rebuild_on_copy)
|
||||
if not self.rebuild_on_copy:
|
||||
clone.__dict__.update(self.__dict__)
|
||||
return clone
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_executor(monkeypatch, tmp_path):
|
||||
# conftest supplies an executor stub for collection-time cycles. Import
|
||||
# its real dependencies before loading the actual executor for this test.
|
||||
import deerflow.agents.thread_state # noqa: F401
|
||||
import deerflow.subagents as package
|
||||
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "home"))
|
||||
monkeypatch.setattr("deerflow.config.paths._paths", Paths(str(tmp_path / "home")))
|
||||
monkeypatch.delitem(sys.modules, "deerflow.subagents.executor", raising=False)
|
||||
monkeypatch.delattr(package, "executor", raising=False)
|
||||
module = importlib.import_module("deerflow.subagents.executor")
|
||||
monkeypatch.setattr(module, "create_chat_model", lambda **kwargs: _OfflineModel())
|
||||
monkeypatch.setattr(module, "build_tracing_callbacks", lambda: [])
|
||||
yield module
|
||||
module._shutdown_isolated_subagent_loop()
|
||||
# importlib installed this attribute outside monkeypatch's tracked writes.
|
||||
if getattr(package, "executor", None) is module:
|
||||
delattr(package, "executor")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_task_tool(native_executor, monkeypatch):
|
||||
import deerflow.subagents as package
|
||||
|
||||
# The package export also comes from conftest's executor stub. Restore the
|
||||
# real class before loading task_tool with its real polling/cleanup helpers.
|
||||
# A separate module avoids changing cached tool references in other suites.
|
||||
monkeypatch.setattr(package, "SubagentExecutor", native_executor.SubagentExecutor)
|
||||
path = Path(native_executor.__file__).parents[1] / "tools/builtins/task_tool.py"
|
||||
spec = importlib.util.spec_from_file_location("_checkpoint_task_tool", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module.task_tool
|
||||
|
||||
|
||||
def _config(tmp_path, *, enabled=True, mode="full"):
|
||||
config = AppConfig.model_validate(
|
||||
{
|
||||
"models": [{"name": "offline", "use": "langchain_openai:ChatOpenAI", "model": "offline", "api_key": "unused"}],
|
||||
"database": {"backend": "sqlite", "sqlite_dir": str(tmp_path / "db"), "checkpoint_channel_mode": mode},
|
||||
"subagent_runtime": {"checkpoint_enabled": enabled},
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
"memory": {"enabled": False},
|
||||
"summarization": {"enabled": False},
|
||||
"title": {"enabled": False},
|
||||
}
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def test_checkpoint_persistence_is_opt_in():
|
||||
assert SubagentRuntimeConfig().checkpoint_enabled is False
|
||||
assert SubagentRuntimeConfig(checkpoint_enabled=True).checkpoint_enabled is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_peer_factory_recovers_after_import_under_temporary_gateway_patch(tmp_path, monkeypatch):
|
||||
from deerflow.runtime.checkpointer import async_provider
|
||||
|
||||
@asynccontextmanager
|
||||
async def temporary_factory(_config):
|
||||
yield object()
|
||||
|
||||
# Load an isolated copy of the production module while the Gateway's
|
||||
# resource factory is patched, without replacing its live registry.
|
||||
path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/checkpoints.py"
|
||||
spec = importlib.util.spec_from_file_location("_checkpoint_factory_import_probe", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
with monkeypatch.context() as importing:
|
||||
importing.setattr(async_provider, "make_checkpointer", temporary_factory)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
config = _config(tmp_path)
|
||||
invocation = {"configurable": {"thread_id": "restored-factory-thread"}}
|
||||
async with make_checkpointer(config) as parent_saver:
|
||||
runtime = module.SubagentCheckpointRuntime(config, parent_saver)
|
||||
try:
|
||||
await runtime.start()
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("reply", lambda _state: {"messages": [AIMessage(content="RESTORED_FACTORY")]})
|
||||
builder.add_edge(START, "reply")
|
||||
builder.add_edge("reply", END)
|
||||
graph = builder.compile(checkpointer=runtime.saver)
|
||||
state = await graph.ainvoke({"messages": [HumanMessage(content="Record evidence")]}, invocation)
|
||||
assert state["messages"][-1].content == "RESTORED_FACTORY"
|
||||
finally:
|
||||
await runtime.stop()
|
||||
|
||||
# A fresh reader must find real SQLite history after both writers close.
|
||||
async with make_checkpointer(config) as reader:
|
||||
saved = await reader.aget_tuple(invocation)
|
||||
assert saved is not None
|
||||
assert saved.checkpoint["channel_values"]["messages"][-1].content == "RESTORED_FACTORY"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enabled", [False, True], ids=["disabled", "enabled"])
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
async def test_task_tool_checkpoint_history_reopens_with_delegation_identity(native_task_tool, tmp_path, monkeypatch, enabled, mode):
|
||||
from langchain.agents import create_agent
|
||||
|
||||
from deerflow.agents.thread_state import get_thread_state_schema
|
||||
from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
|
||||
app_config = _config(tmp_path, enabled=enabled, mode=mode)
|
||||
config_path = tmp_path / "config.yaml"
|
||||
config_path.write_text(app_config.model_dump_json(), encoding="utf-8")
|
||||
extensions_path = tmp_path / "extensions_config.json"
|
||||
extensions_path.write_text("{}", encoding="utf-8")
|
||||
monkeypatch.setenv("DEER_FLOW_CONFIG_PATH", str(config_path))
|
||||
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
|
||||
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [echo])
|
||||
freeze_checkpoint_channel_mode(mode)
|
||||
|
||||
class ParentModel(_OfflineModel):
|
||||
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
if any(isinstance(message, ToolMessage) for message in messages):
|
||||
answer = AIMessage(content="PARENT_FINAL")
|
||||
else:
|
||||
answer = AIMessage(
|
||||
content="",
|
||||
tool_calls=[{"id": "outer-call", "name": "task", "args": {"subagent_type": "general-purpose", "prompt": "Use echo once, then report the result", "description": "Offline checkpoint integration"}}],
|
||||
)
|
||||
return ChatResult(generations=[ChatGeneration(message=answer)])
|
||||
|
||||
async with make_checkpointer(app_config) as saver:
|
||||
async with subagent_checkpoint_runtime(app_config, saver):
|
||||
parent = create_agent(model=ParentModel(), tools=[native_task_tool], checkpointer=saver, context_schema=dict, state_schema=get_thread_state_schema(mode))
|
||||
state = await asyncio.wait_for(
|
||||
parent.ainvoke(
|
||||
{"messages": [HumanMessage(content="Delegate the offline work")]},
|
||||
config={"configurable": {"thread_id": "task-history-thread"}, "metadata": {"model_name": "offline", "run_id": "task-parent-run"}},
|
||||
context={"app_config": app_config, "thread_id": "task-history-thread", "run_id": "task-parent-run", "user_id": "task-user", "user_role": "admin"},
|
||||
),
|
||||
timeout=25,
|
||||
)
|
||||
outer_result = next(message for message in state["messages"] if isinstance(message, ToolMessage) and message.tool_call_id == "outer-call")
|
||||
assert outer_result.additional_kwargs["subagent_status"] == "completed"
|
||||
assert "CHILD_FINAL" in outer_result.content
|
||||
assert state["messages"][-1].content == "PARENT_FINAL"
|
||||
assert not any(isinstance(message, ToolMessage) and message.tool_call_id == "child-call" for message in state["messages"])
|
||||
|
||||
# Both writer lifetimes have ended: read from a new connection, just as
|
||||
# a history consumer does after a Gateway restart.
|
||||
async with make_checkpointer(app_config) as reader:
|
||||
history = [entry async for entry in reader.alist({"configurable": {"thread_id": "task-history-thread"}})]
|
||||
roots = [entry for entry in history if not entry.config["configurable"].get("checkpoint_ns")]
|
||||
children = [entry for entry in history if entry.config["configurable"].get("checkpoint_ns")]
|
||||
assert roots
|
||||
if not enabled:
|
||||
assert not children
|
||||
return
|
||||
assert children
|
||||
namespaces = {entry.config["configurable"]["checkpoint_ns"] for entry in children}
|
||||
execution_ids = {entry.metadata["deerflow_subagent_execution_id"] for entry in children}
|
||||
assert len(namespaces) == len(execution_ids) == 1
|
||||
execution_id = next(iter(execution_ids))
|
||||
namespace = next(iter(namespaces))
|
||||
assert execution_id != "outer-call"
|
||||
assert namespace.endswith(f"|deerflow_subagent:{execution_id}")
|
||||
assert all(entry.metadata["deerflow_subagent_name"] == "general-purpose" for entry in children)
|
||||
assert all(entry.metadata["deerflow_parent_run_id"] == "task-parent-run" for entry in children)
|
||||
assert all(entry.metadata["deerflow_subagent_tool_call_id"] == "outer-call" for entry in children)
|
||||
root_ids = {entry.checkpoint["id"] for entry in roots}
|
||||
assert all(entry.metadata["parents"][""] in root_ids for entry in children)
|
||||
|
||||
child = create_agent(model=_OfflineModel(), tools=[echo], state_schema=get_thread_state_schema(mode), checkpointer=reader)
|
||||
restored = await child.aget_state({"configurable": {"thread_id": "task-history-thread", "checkpoint_ns": namespace, "__pregel_checkpointer": reader}})
|
||||
messages = restored.values["messages"]
|
||||
calls = [call for message in messages if isinstance(message, AIMessage) for call in message.tool_calls]
|
||||
assert any(call["id"] == "child-call" and call["name"] == "echo" and call["args"] == {"text": "child evidence"} for call in calls)
|
||||
assert any(isinstance(message, ToolMessage) and message.tool_call_id == "child-call" and message.content == "child evidence" for message in messages)
|
||||
assert messages[-1].content == "CHILD_FINAL"
|
||||
assert all(message.content not in {"Delegate the offline work", "PARENT_FINAL"} for message in messages if isinstance(message.content, str))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("restriction", ["global", "subagent"])
|
||||
@pytest.mark.parametrize("rebuild_on_copy", [False, True], ids=["preserving-copy", "rebuilding-copy"])
|
||||
async def test_delta_middleware_copies_preserve_tool_restrictions(native_executor, tmp_path, monkeypatch, restriction, rebuild_on_copy):
|
||||
from deerflow.config.authorization_config import AuthorizationConfig
|
||||
from deerflow.config.extensions_config import ConfiguredMiddlewareSpec
|
||||
from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
from deerflow.subagents.config import SubagentConfig
|
||||
|
||||
app_config = _config(tmp_path, mode="delta")
|
||||
app_config.extensions.middlewares = [ConfiguredMiddlewareSpec(class_path=f"{__name__}:_CopyingToolMiddleware", kwargs={"rebuild_on_copy": rebuild_on_copy}, apply_to="subagent")]
|
||||
if restriction == "global":
|
||||
app_config.authorization = AuthorizationConfig.model_validate({"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"user": {"tools": {"allow": "*", "deny": ["forbidden"]}}}}}})
|
||||
freeze_checkpoint_channel_mode("delta")
|
||||
bound_names = []
|
||||
|
||||
class RecordingModel(_OfflineModel):
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
bound_names.append([tool.name for tool in tools])
|
||||
return self
|
||||
|
||||
monkeypatch.setattr(native_executor, "create_chat_model", lambda **kwargs: RecordingModel())
|
||||
|
||||
async def delegate(_state):
|
||||
executor = native_executor.SubagentExecutor(
|
||||
config=SubagentConfig(name="copy-policy", description="Check extension copies", skills=[], disallowed_tools=["forbidden"] if restriction == "subagent" else []),
|
||||
tools=[echo],
|
||||
parent_model="offline",
|
||||
thread_id="copy-thread",
|
||||
user_id="copy-user",
|
||||
user_role="user",
|
||||
app_config=app_config,
|
||||
)
|
||||
execution_id = executor.execute_async("Use echo to return child evidence")
|
||||
result = native_executor.get_background_task_result(execution_id)
|
||||
try:
|
||||
assert await asyncio.to_thread(result.execution_done_event.wait, 15)
|
||||
if rebuild_on_copy:
|
||||
assert result.status is native_executor.SubagentStatus.FAILED
|
||||
assert "forbidden" in result.error
|
||||
assert ("not authorized" if restriction == "global" else "final tool set") in result.error
|
||||
assert not bound_names
|
||||
else:
|
||||
assert result.status is native_executor.SubagentStatus.COMPLETED, result.error
|
||||
assert bound_names
|
||||
assert all("echo" in names and "forbidden" not in names for names in bound_names)
|
||||
finally:
|
||||
native_executor.cleanup_background_task(result.task_id)
|
||||
return {}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("delegate", delegate)
|
||||
builder.add_edge(START, "delegate")
|
||||
builder.add_edge("delegate", END)
|
||||
async with make_checkpointer(app_config) as saver:
|
||||
async with subagent_checkpoint_runtime(app_config, saver):
|
||||
parent = builder.compile(checkpointer=saver)
|
||||
await parent.ainvoke({"messages": [HumanMessage(content="Check tool restrictions")]}, {"configurable": {"thread_id": "copy-thread"}})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
@pytest.mark.parametrize("clone_parent", [False, True])
|
||||
async def test_parallel_children_reopen_with_execution_identity(native_executor, tmp_path, enabled, mode, clone_parent):
|
||||
from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
from deerflow.subagents.config import SubagentConfig
|
||||
|
||||
app_config = _config(tmp_path, enabled=enabled, mode=mode)
|
||||
freeze_checkpoint_channel_mode(mode)
|
||||
completed = []
|
||||
parent_loop = asyncio.get_running_loop()
|
||||
|
||||
async with make_checkpointer(app_config) as parent_saver:
|
||||
async with subagent_checkpoint_runtime(app_config, parent_saver) as runtime:
|
||||
|
||||
async def delegate(_state, config: RunnableConfig):
|
||||
# Bind the real SQLite lock to the parent loop through actual
|
||||
# contention, and keep it held until both children complete.
|
||||
# This stabilizes the original cross-loop failure without
|
||||
# injecting an exception or holding a database transaction.
|
||||
raw = getattr(parent_saver, "source_saver", parent_saver)
|
||||
await raw.lock.acquire()
|
||||
waiter = asyncio.create_task(raw.setup())
|
||||
await asyncio.sleep(0)
|
||||
assert raw.lock._loop is parent_loop
|
||||
parent_context = var_child_runnable_config.get()
|
||||
parent_namespace = parent_context["configurable"]["checkpoint_ns"]
|
||||
inherited_saver = parent_context["configurable"]["__pregel_checkpointer"]
|
||||
try:
|
||||
results = []
|
||||
for _ in range(2):
|
||||
executor = native_executor.SubagentExecutor(
|
||||
config=SubagentConfig(name="checkpoint-test", description="Offline checkpoint regression", system_prompt="Use echo once, then answer.", timeout_seconds=30),
|
||||
tools=[echo],
|
||||
parent_model="offline",
|
||||
thread_id="history-thread",
|
||||
run_id=config["metadata"]["run_id"],
|
||||
app_config=app_config,
|
||||
)
|
||||
# Intentionally reused across children and parent runs.
|
||||
execution_id = executor.execute_async("Return evidence", task_id="reused-provider-call")
|
||||
results.append(native_executor.get_background_task_result(execution_id))
|
||||
await asyncio.wait_for(asyncio.gather(*(asyncio.to_thread(result.execution_done_event.wait) for result in results)), timeout=20)
|
||||
for result in results:
|
||||
assert result.status.value == "completed", result.error
|
||||
completed.append((config["metadata"]["run_id"], result.task_id))
|
||||
finally:
|
||||
raw.lock.release()
|
||||
await waiter
|
||||
for result in results:
|
||||
native_executor.cleanup_background_task(result.task_id)
|
||||
assert var_child_runnable_config.get() is parent_context
|
||||
assert parent_context["configurable"]["checkpoint_ns"] == parent_namespace
|
||||
assert parent_context["configurable"]["__pregel_checkpointer"] is inherited_saver
|
||||
assert "deerflow_subagent_execution_id" not in parent_context["metadata"]
|
||||
return {"messages": [AIMessage(content="PARENT_FINAL")]}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("delegate", delegate)
|
||||
builder.add_edge(START, "delegate")
|
||||
builder.add_edge("delegate", END)
|
||||
# Same connection, different saver identity: mirrors the strict
|
||||
# serializer allowlist clones used by newer LangGraph releases.
|
||||
graph_saver = parent_saver.with_allowlist({("builtins", "dict")}) if clone_parent else parent_saver
|
||||
parent = builder.compile(checkpointer=graph_saver)
|
||||
for run_id in ("parent-run-1", "parent-run-2"):
|
||||
streamed = [
|
||||
message
|
||||
async for message, _ in parent.astream(
|
||||
{"messages": [HumanMessage(content="Delegate")]},
|
||||
config={"configurable": {"thread_id": "history-thread"}, "metadata": {"run_id": run_id}},
|
||||
stream_mode="messages",
|
||||
)
|
||||
]
|
||||
assert any(message.content == "PARENT_FINAL" for message in streamed)
|
||||
assert all(message.content != "CHILD_FINAL" and not isinstance(message, ToolMessage) for message in streamed)
|
||||
if enabled:
|
||||
assert runtime.saver is not parent_saver
|
||||
assert runtime.loop is not parent_loop
|
||||
if enabled:
|
||||
assert runtime.closed
|
||||
|
||||
# Fresh resources materialize evidence after both writer lifetimes end.
|
||||
async with make_checkpointer(app_config) as reader:
|
||||
history = [entry async for entry in reader.alist({"configurable": {"thread_id": "history-thread"}})]
|
||||
root = [entry for entry in history if not entry.config["configurable"].get("checkpoint_ns")]
|
||||
assert root
|
||||
root_ids = {entry.checkpoint["id"] for entry in root}
|
||||
assert all(message.content != "CHILD_FINAL" for entry in root for message in entry.checkpoint["channel_values"].get("messages", []))
|
||||
children = [entry for entry in history if entry.config["configurable"].get("checkpoint_ns")]
|
||||
if not enabled:
|
||||
assert not children
|
||||
return
|
||||
groups = defaultdict(list)
|
||||
for entry in children:
|
||||
groups[entry.metadata["deerflow_subagent_execution_id"]].append(entry)
|
||||
assert set(groups) == {execution_id for _, execution_id in completed}
|
||||
namespaces = set()
|
||||
for parent_run_id, execution_id in completed:
|
||||
entries = groups[execution_id]
|
||||
own_namespaces = {entry.config["configurable"]["checkpoint_ns"] for entry in entries}
|
||||
assert len(own_namespaces) == 1
|
||||
assert not own_namespaces & namespaces
|
||||
namespaces.update(own_namespaces)
|
||||
assert all(entry.metadata["deerflow_parent_run_id"] == parent_run_id for entry in entries)
|
||||
assert all(entry.metadata["deerflow_subagent_tool_call_id"] == "reused-provider-call" for entry in entries)
|
||||
assert all(entry.metadata["parents"][""] in root_ids for entry in entries)
|
||||
# Use the actual child graph schema to reconstruct full and delta
|
||||
# states, rather than expecting raw delta channel blobs to be text.
|
||||
from deerflow.agents.thread_state import get_thread_state_schema
|
||||
|
||||
accessor_builder = StateGraph(get_thread_state_schema(mode))
|
||||
accessor_builder.add_node("unused", lambda state: {})
|
||||
accessor_builder.add_edge(START, "unused")
|
||||
accessor_builder.add_edge("unused", END)
|
||||
accessor = accessor_builder.compile(checkpointer=reader)
|
||||
snapshot = await accessor.aget_state({"configurable": {"thread_id": "history-thread", "checkpoint_ns": next(iter(own_namespaces)), "__pregel_checkpointer": reader}})
|
||||
messages = snapshot.values["messages"]
|
||||
assert any(isinstance(message, ToolMessage) and "child evidence" in message.content for message in messages)
|
||||
assert any(message.content == "CHILD_FINAL" for message in messages)
|
||||
assert all(message.content not in {"Delegate", "PARENT_FINAL"} for message in messages if isinstance(message.content, str))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("legacy", [False, True])
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
async def test_checkpoint_history_only_accepts_sqlite(native_executor, tmp_path, legacy, enabled):
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
|
||||
config = _config(tmp_path, enabled=enabled)
|
||||
async with make_checkpointer(config) as saver:
|
||||
# Only exercise admission here: no PostgreSQL driver or server is used.
|
||||
# A legacy section must take precedence over the unified SQLite config.
|
||||
if legacy:
|
||||
config.checkpointer = CheckpointerConfig(type="postgres")
|
||||
else:
|
||||
config.database.backend = "postgres"
|
||||
if enabled:
|
||||
with pytest.raises(ValueError, match="only supports file-backed SQLite"):
|
||||
async with subagent_checkpoint_runtime(config, saver):
|
||||
pytest.fail("Unsupported checkpoint backend was accepted")
|
||||
else:
|
||||
async with subagent_checkpoint_runtime(config, saver) as runtime:
|
||||
assert runtime is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("legacy", [False, True])
|
||||
async def test_memory_checkpoint_history_is_rejected(native_executor, tmp_path, legacy):
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
|
||||
config = _config(tmp_path)
|
||||
if legacy:
|
||||
from deerflow.config.checkpointer_config import CheckpointerConfig
|
||||
|
||||
config.checkpointer = CheckpointerConfig(type="sqlite", connection_string=":memory:")
|
||||
else:
|
||||
config.database.backend = "memory"
|
||||
async with make_checkpointer(config) as saver:
|
||||
with pytest.raises(ValueError, match="persistent|file-backed"):
|
||||
async with subagent_checkpoint_runtime(config, saver):
|
||||
pytest.fail("Non-persistent checkpoint storage was accepted")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_host_startup_closes_peer_before_parent(native_executor, tmp_path):
|
||||
from deerflow.subagents.checkpoints import _runtimes, subagent_checkpoint_runtime
|
||||
|
||||
config = _config(tmp_path)
|
||||
async with make_checkpointer(config) as saver:
|
||||
with pytest.raises(RuntimeError, match="later startup failure"):
|
||||
async with subagent_checkpoint_runtime(config, saver) as runtime:
|
||||
raise RuntimeError("later startup failure")
|
||||
assert runtime.closed
|
||||
assert id(saver) not in _runtimes
|
||||
# Parent resource remains usable after the peer's teardown.
|
||||
await saver.setup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gateway_runtime_installs_and_closes_peer(native_executor, tmp_path):
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.gateway.deps import langgraph_runtime
|
||||
from deerflow.subagents.checkpoints import _runtimes
|
||||
|
||||
config = _config(tmp_path)
|
||||
app = FastAPI()
|
||||
async with langgraph_runtime(app, config):
|
||||
saver = app.state.checkpointer
|
||||
runtime = _runtimes[id(saver)]
|
||||
assert runtime.saver is not saver
|
||||
assert runtime.loop is not asyncio.get_running_loop()
|
||||
assert config.run_events.backend == "memory"
|
||||
await saver.setup()
|
||||
assert runtime.closed
|
||||
assert id(saver) not in _runtimes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_cancellation_closes_created_peer(native_executor, tmp_path, monkeypatch):
|
||||
from deerflow.subagents.checkpoints import SubagentCheckpointRuntime, _runtimes, subagent_checkpoint_runtime
|
||||
|
||||
config = _config(tmp_path)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
opened = []
|
||||
original_start = SubagentCheckpointRuntime.start
|
||||
|
||||
async def paused_start(runtime):
|
||||
await original_start(runtime)
|
||||
opened.append(runtime)
|
||||
started.set()
|
||||
await asyncio.to_thread(release.wait)
|
||||
|
||||
monkeypatch.setattr(SubagentCheckpointRuntime, "start", paused_start)
|
||||
async with make_checkpointer(config) as saver:
|
||||
manager = subagent_checkpoint_runtime(config, saver)
|
||||
startup = asyncio.create_task(manager.__aenter__())
|
||||
try:
|
||||
assert await asyncio.to_thread(started.wait, 10)
|
||||
startup.cancel()
|
||||
await asyncio.sleep(0)
|
||||
startup.cancel()
|
||||
await asyncio.sleep(0)
|
||||
assert not startup.done()
|
||||
finally:
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(startup, 10)
|
||||
assert opened[0].closed
|
||||
assert id(saver) not in _runtimes
|
||||
await saver.setup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loop_shutdown_closes_peer_on_owner_loop(native_executor, tmp_path):
|
||||
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
||||
|
||||
config = _config(tmp_path)
|
||||
async with make_checkpointer(config) as saver:
|
||||
async with subagent_checkpoint_runtime(config, saver) as runtime:
|
||||
owner = runtime.loop
|
||||
await asyncio.to_thread(native_executor._shutdown_isolated_subagent_loop)
|
||||
assert runtime.closed
|
||||
assert owner.is_closed()
|
||||
await saver.setup()
|
||||
|
||||
|
||||
def test_default_sync_parent_keeps_async_child_isolated(native_executor, tmp_path):
|
||||
from langgraph.checkpoint.sqlite import SqliteSaver
|
||||
|
||||
from deerflow.subagents.config import SubagentConfig
|
||||
|
||||
config = _config(tmp_path, enabled=False)
|
||||
|
||||
def delegate(_state):
|
||||
executor = native_executor.SubagentExecutor(
|
||||
config=SubagentConfig(name="checkpoint-test", description="Sync parent regression", system_prompt="Use echo.", timeout_seconds=30),
|
||||
tools=[echo],
|
||||
parent_model="offline",
|
||||
thread_id="sync-thread",
|
||||
app_config=config,
|
||||
)
|
||||
result = executor.execute("Return evidence")
|
||||
assert result.status.value == "completed", result.error
|
||||
return {"messages": [AIMessage(content="PARENT_DONE")]}
|
||||
|
||||
with SqliteSaver.from_conn_string(str(tmp_path / "sync.sqlite")) as saver:
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("delegate", delegate)
|
||||
builder.add_edge(START, "delegate")
|
||||
builder.add_edge("delegate", END)
|
||||
parent = builder.compile(checkpointer=saver)
|
||||
state = parent.invoke({"messages": [HumanMessage(content="Delegate")]}, {"configurable": {"thread_id": "sync-thread"}})
|
||||
assert state["messages"][-1].content == "PARENT_DONE"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_drains_cancelled_child_and_fences_late_dispatch(native_executor, tmp_path):
|
||||
from deerflow.subagents.checkpoints import _runtimes, subagent_checkpoint_runtime
|
||||
from deerflow.subagents.config import SubagentConfig
|
||||
|
||||
config = _config(tmp_path)
|
||||
entered = threading.Event()
|
||||
cleanup_entered = threading.Event()
|
||||
release_cleanup = threading.Event()
|
||||
results = []
|
||||
captured_contexts = []
|
||||
|
||||
@tool("echo")
|
||||
async def blocking_echo(text: str) -> str:
|
||||
"""Pause a deterministic child until shutdown cancels it."""
|
||||
entered.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
cleanup_entered.set()
|
||||
await asyncio.to_thread(release_cleanup.wait)
|
||||
return text
|
||||
|
||||
def executor():
|
||||
return native_executor.SubagentExecutor(
|
||||
config=SubagentConfig(name="checkpoint-test", description="Shutdown regression", system_prompt="Use echo.", timeout_seconds=30),
|
||||
tools=[blocking_echo],
|
||||
parent_model="offline",
|
||||
thread_id="shutdown-thread",
|
||||
run_id="shutdown-parent-run",
|
||||
app_config=config,
|
||||
)
|
||||
|
||||
async def delegate(_state):
|
||||
captured_contexts.append(native_executor._copy_isolated_subagent_context())
|
||||
execution_id = executor().execute_async("Pause")
|
||||
result = native_executor.get_background_task_result(execution_id)
|
||||
results.append(result)
|
||||
await asyncio.to_thread(result.execution_done_event.wait)
|
||||
return {"messages": [AIMessage(content="PARENT_DONE")]}
|
||||
|
||||
async with make_checkpointer(config) as saver:
|
||||
manager = subagent_checkpoint_runtime(config, saver)
|
||||
runtime = await manager.__aenter__()
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("delegate", delegate)
|
||||
builder.add_edge(START, "delegate")
|
||||
builder.add_edge("delegate", END)
|
||||
parent = builder.compile(checkpointer=saver)
|
||||
parent_task = asyncio.create_task(parent.ainvoke({"messages": [HumanMessage(content="Pause")]}, {"configurable": {"thread_id": "shutdown-thread"}}))
|
||||
stop = None
|
||||
late_result = None
|
||||
try:
|
||||
assert await asyncio.to_thread(entered.wait, 10)
|
||||
stop = asyncio.create_task(manager.__aexit__(None, None, None))
|
||||
assert await asyncio.to_thread(cleanup_entered.wait, 10)
|
||||
stop.cancel()
|
||||
await asyncio.sleep(0)
|
||||
stop.cancel()
|
||||
await asyncio.sleep(0)
|
||||
assert not stop.done()
|
||||
assert not runtime.closed
|
||||
assert runtime.closing
|
||||
await saver.setup()
|
||||
|
||||
late_id = captured_contexts[0].run(executor().execute_async, "Late task")
|
||||
late_result = native_executor.get_background_task_result(late_id)
|
||||
assert await asyncio.to_thread(late_result.execution_done_event.wait, 10)
|
||||
assert late_result.status.value == "failed"
|
||||
assert "shutting down" in late_result.error
|
||||
finally:
|
||||
release_cleanup.set()
|
||||
if stop is not None:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(stop, 10)
|
||||
else:
|
||||
await manager.__aexit__(None, None, None)
|
||||
await asyncio.wait_for(parent_task, 10)
|
||||
for result in [*results, *([late_result] if late_result is not None else [])]:
|
||||
native_executor.cleanup_background_task(result.task_id)
|
||||
assert runtime.closed
|
||||
assert id(saver) not in _runtimes
|
||||
assert results[0].status.value == "cancelled"
|
||||
assert results[0].execution_done_event.is_set()
|
||||
await saver.setup()
|
||||
# A dispatch context captured before shutdown must remain fenced even
|
||||
# after the runtime has been removed from the process registry.
|
||||
after_close_id = captured_contexts[0].run(executor().execute_async, "After close")
|
||||
after_close = native_executor.get_background_task_result(after_close_id)
|
||||
try:
|
||||
assert await asyncio.to_thread(after_close.execution_done_event.wait, 10)
|
||||
assert after_close.status.value == "failed"
|
||||
assert "shutting down" in after_close.error
|
||||
finally:
|
||||
native_executor.cleanup_background_task(after_close_id)
|
||||
@ -6,9 +6,9 @@ DeerFlowClient), the subagent's ``agent.astream()`` inherits it through
|
||||
at compile time, LangGraph's resolution prioritizes the inherited value
|
||||
and calls the sync checkpointer's async methods, raising NotImplementedError.
|
||||
|
||||
The subagent is a one-shot delegation — it rebuilds state, calls astream
|
||||
once, and extracts the last AIMessage. It never resumes, so persistence
|
||||
is unnecessary and inheriting the parent checkpointer is harmful.
|
||||
The default subagent is a one-shot delegation and remains checkpoint-disabled.
|
||||
Opt-in Gateway history replaces the inherited saver with a loop-owned peer;
|
||||
that separate path is covered by test_subagent_checkpoint_persistence.py.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@ -83,14 +83,14 @@ def _setup_executor_module():
|
||||
|
||||
|
||||
class TestSubagentCheckpointerIsolation:
|
||||
"""Verify _create_agent() unconditionally passes checkpointer=False to create_agent()."""
|
||||
"""Verify _create_agent() passes checkpointer=False without an opted-in scope."""
|
||||
|
||||
def test_create_agent_receives_checkpointer_false(
|
||||
self,
|
||||
_setup_executor_module,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Assert checkpointer=False is always passed to create_agent()."""
|
||||
"""Assert checkpointer=False is passed by default to create_agent()."""
|
||||
SubagentConfig = _setup_executor_module["SubagentConfig"]
|
||||
SubagentExecutor = _setup_executor_module["SubagentExecutor"]
|
||||
executor_module = _setup_executor_module["executor_module"]
|
||||
|
||||
@ -2030,6 +2030,10 @@ sandbox:
|
||||
# shared by ordinary `task` calls and durable batches. Waiting work is async;
|
||||
# queued subagents do not occupy a scheduler thread.
|
||||
subagent_runtime:
|
||||
# Optional history for ordinary task delegates under a persistent parent
|
||||
# graph. Currently supports only file-backed SQLite checkpoint storage.
|
||||
# Restart Gateway after changing; independent of run_events.backend.
|
||||
checkpoint_enabled: false
|
||||
max_running: 3
|
||||
max_queued: 64
|
||||
admission_policy: queue # queue or reject when all slots are occupied
|
||||
|
||||
@ -1,5 +1,84 @@
|
||||
# Sub-Agents
|
||||
|
||||
## Optional checkpoint history
|
||||
|
||||
The Gateway can save checkpoint history for ordinary subagent executions. This
|
||||
is disabled by default.
|
||||
|
||||
The default configuration already sets `database.backend` to `sqlite`.
|
||||
Keep the `database:` settings unchanged. In `config.yaml`, set
|
||||
`subagent_runtime.checkpoint_enabled` to `true`:
|
||||
|
||||
```yaml
|
||||
subagent_runtime:
|
||||
checkpoint_enabled: true
|
||||
```
|
||||
|
||||
Restart the Gateway for the change to take effect. Parent and child checkpoints are saved in the same SQLite database selected by the host. When the host uses the default `database:` configuration, this is `deerflow.db` under `database.sqlite_dir`.
|
||||
|
||||
This option currently supports SQLite; PostgreSQL and memory storage are not
|
||||
supported. Both `full` and `delta` checkpoint channel modes are supported. Keep
|
||||
the database on persistent storage to retain history across Gateway or
|
||||
container restarts. This option is independent of `run_events.backend`, which
|
||||
can remain `memory`.
|
||||
|
||||
The Gateway owns a second saver on the persistent subagent event loop. Subagents
|
||||
using this checkpoint runtime share it, while the parent keeps its own saver and
|
||||
both connect to the same checkpoint database. Each child execution gets a
|
||||
distinct non-root namespace under its parent node. Child messages remain outside
|
||||
the parent transcript and its root message stream.
|
||||
|
||||
The normal `task` delegation flow supplies the required parent graph context.
|
||||
Calling the executor directly from a parent graph node can also save history
|
||||
when it inherits that context and uses the installed checkpoint runtime.
|
||||
This option covers direct delegation from the parent graph. If a subagent
|
||||
dispatches another subagent, the nested execution runs with checkpoint history
|
||||
disabled. Tool calls and results within a recorded subagent can still be
|
||||
retained in its checkpoint history. Checkpoint history for items executed through `batch_task` is not included.
|
||||
|
||||
Child checkpoint metadata includes `deerflow_subagent_execution_id` and
|
||||
`deerflow_subagent_name`, plus `deerflow_parent_run_id` and
|
||||
`deerflow_subagent_tool_call_id` when those identities are available. Use the
|
||||
server execution ID to distinguish attempts; provider tool-call IDs can repeat.
|
||||
The inherited checkpoint parent map retains the parent checkpoint reference.
|
||||
Readers can use these fields to attribute retained history without inferring
|
||||
ownership from timestamps. Concurrent children do not have a single global
|
||||
execution order.
|
||||
|
||||
This option stores historical evidence and adds checkpoint writes and storage
|
||||
usage. It does not add a history UI or automatic subagent recovery. Standalone
|
||||
embedded clients and direct factories retain their existing checkpoint-disabled
|
||||
behavior unless they explicitly install the matching checkpoint resource
|
||||
lifetime around a persistent parent graph.
|
||||
|
||||
Child checkpoint history is not automatically pruned by the existing checkpoint
|
||||
retention policies. Each child execution has its own namespace, whose latest
|
||||
checkpoint and ancestor chain are protected. Deleting the entire thread is
|
||||
currently the only supported way to reclaim this history. Storage usage grows
|
||||
with retained child executions.
|
||||
|
||||
The Gateway closes admission and drains child execution, stream cleanup and
|
||||
checkpoint writes before closing the child saver. The parent saver remains open
|
||||
through that drain.
|
||||
|
||||
### Checkpoint resource ownership
|
||||
|
||||
All native subagents share one persistent event loop. The Gateway installs the
|
||||
optional child saver inside the parent saver's lifetime. Capture that runtime
|
||||
before dispatch and replace the inherited saver in a copied runnable config;
|
||||
LangGraph's inherited reference takes precedence over a compile-time saver.
|
||||
Identify strict-msgpack serializer clones by their underlying connection,
|
||||
because a clone can share its original saver's connection and lock.
|
||||
|
||||
Extend the ambient checkpoint namespace with the server execution ID and retain
|
||||
its parent map. Keep checkpoint coordinates out of the explicit child invocation
|
||||
config, which can reset the namespace on newer LangGraph versions. Drain complete
|
||||
child execution and cleanup before closing the saver or stopping its owner loop.
|
||||
Delta schema normalization can copy middleware; audit both global authorization
|
||||
and subagent tool restrictions on the final instances before creating the graph.
|
||||
|
||||
## Delegation behavior
|
||||
|
||||
Web chats accept direct model choices such as “ask MODEL to handle this”, “use MODEL for this turn”, and “from now on use MODEL”; Chinese equivalents are supported too. A child choice leaves the main model unchanged, a one-turn choice leaves defaults unchanged, and a conversation choice uses the existing browser-local thread override without changing account preferences. Choices resolve against available configurations and optional `models[].aliases`; ambiguous names or scopes show a selection card before execution, while delegation still requires enabled subagents (Ultra mode).
|
||||
|
||||
When a sub-agent ends with `return_direct=True` tools, including tools contributed by extension middleware, their outputs are returned in tool-call order. A failed tool marks the task as failed while preserving the batch outputs.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user