mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 08:20:34 +00:00
* feat(subagents): add optional SQLite checkpoint history * docs(subagents): clarify checkpoint history limits * fix(subagents): resolve checkpoint factory at startup --------- Co-authored-by: Nan Gao <thunderggnn@gmail.com>
693 lines
34 KiB
Python
693 lines
34 KiB
Python
"""Native subagent checkpoint history across the real isolated-loop boundary.
|
|
|
|
The model is deterministic and local. The executor, graph builder, middleware,
|
|
SQLite saver, context propagation and background execution are production code.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import importlib
|
|
import importlib.util
|
|
import sys
|
|
import threading
|
|
from collections import defaultdict
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from langchain.agents.middleware import AgentMiddleware
|
|
from langchain_core.language_models import BaseChatModel
|
|
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
|
from langchain_core.outputs import ChatGeneration, ChatResult
|
|
from langchain_core.runnables import RunnableConfig
|
|
from langchain_core.runnables.config import var_child_runnable_config
|
|
from langchain_core.tools import tool
|
|
from langgraph.graph import END, START, MessagesState, StateGraph
|
|
|
|
from deerflow.config.app_config import AppConfig
|
|
from deerflow.config.paths import Paths
|
|
from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig
|
|
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
|
|
|
|
|
|
class _OfflineModel(BaseChatModel):
|
|
@property
|
|
def _llm_type(self):
|
|
return "offline-subagent-checkpoints"
|
|
|
|
def bind_tools(self, tools, **kwargs):
|
|
return self
|
|
|
|
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
|
|
if any(isinstance(message, ToolMessage) for message in messages):
|
|
answer = AIMessage(content="CHILD_FINAL")
|
|
else:
|
|
answer = AIMessage(content="", tool_calls=[{"name": "echo", "args": {"text": "child evidence"}, "id": "child-call", "type": "tool_call"}])
|
|
return ChatResult(generations=[ChatGeneration(message=answer)])
|
|
|
|
|
|
@tool
|
|
async def echo(text: str) -> str:
|
|
"""Return deterministic evidence without external side effects."""
|
|
return text
|
|
|
|
|
|
@tool
|
|
async def forbidden() -> str:
|
|
"""A declaration that the test policy must remove before model binding."""
|
|
pytest.fail("A denied middleware tool executed")
|
|
|
|
|
|
class _CopyingToolMiddleware(AgentMiddleware):
|
|
"""Exercise state-preserving and constructor-rebuilding extension copies."""
|
|
|
|
def __init__(self, *, rebuild_on_copy: bool = False):
|
|
self.rebuild_on_copy = rebuild_on_copy
|
|
self.tools = [echo, forbidden]
|
|
|
|
def __copy__(self):
|
|
clone = type(self)(rebuild_on_copy=self.rebuild_on_copy)
|
|
if not self.rebuild_on_copy:
|
|
clone.__dict__.update(self.__dict__)
|
|
return clone
|
|
|
|
|
|
@pytest.fixture
|
|
def native_executor(monkeypatch, tmp_path):
|
|
# conftest supplies an executor stub for collection-time cycles. Import
|
|
# its real dependencies before loading the actual executor for this test.
|
|
import deerflow.agents.thread_state # noqa: F401
|
|
import deerflow.subagents as package
|
|
|
|
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "home"))
|
|
monkeypatch.setattr("deerflow.config.paths._paths", Paths(str(tmp_path / "home")))
|
|
monkeypatch.delitem(sys.modules, "deerflow.subagents.executor", raising=False)
|
|
monkeypatch.delattr(package, "executor", raising=False)
|
|
module = importlib.import_module("deerflow.subagents.executor")
|
|
monkeypatch.setattr(module, "create_chat_model", lambda **kwargs: _OfflineModel())
|
|
monkeypatch.setattr(module, "build_tracing_callbacks", lambda: [])
|
|
yield module
|
|
module._shutdown_isolated_subagent_loop()
|
|
# importlib installed this attribute outside monkeypatch's tracked writes.
|
|
if getattr(package, "executor", None) is module:
|
|
delattr(package, "executor")
|
|
|
|
|
|
@pytest.fixture
|
|
def native_task_tool(native_executor, monkeypatch):
|
|
import deerflow.subagents as package
|
|
|
|
# The package export also comes from conftest's executor stub. Restore the
|
|
# real class before loading task_tool with its real polling/cleanup helpers.
|
|
# A separate module avoids changing cached tool references in other suites.
|
|
monkeypatch.setattr(package, "SubagentExecutor", native_executor.SubagentExecutor)
|
|
path = Path(native_executor.__file__).parents[1] / "tools/builtins/task_tool.py"
|
|
spec = importlib.util.spec_from_file_location("_checkpoint_task_tool", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
monkeypatch.setitem(sys.modules, spec.name, module)
|
|
spec.loader.exec_module(module)
|
|
return module.task_tool
|
|
|
|
|
|
def _config(tmp_path, *, enabled=True, mode="full"):
|
|
config = AppConfig.model_validate(
|
|
{
|
|
"models": [{"name": "offline", "use": "langchain_openai:ChatOpenAI", "model": "offline", "api_key": "unused"}],
|
|
"database": {"backend": "sqlite", "sqlite_dir": str(tmp_path / "db"), "checkpoint_channel_mode": mode},
|
|
"subagent_runtime": {"checkpoint_enabled": enabled},
|
|
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
|
"memory": {"enabled": False},
|
|
"summarization": {"enabled": False},
|
|
"title": {"enabled": False},
|
|
}
|
|
)
|
|
return config
|
|
|
|
|
|
def test_checkpoint_persistence_is_opt_in():
|
|
assert SubagentRuntimeConfig().checkpoint_enabled is False
|
|
assert SubagentRuntimeConfig(checkpoint_enabled=True).checkpoint_enabled is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_peer_factory_recovers_after_import_under_temporary_gateway_patch(tmp_path, monkeypatch):
|
|
from deerflow.runtime.checkpointer import async_provider
|
|
|
|
@asynccontextmanager
|
|
async def temporary_factory(_config):
|
|
yield object()
|
|
|
|
# Load an isolated copy of the production module while the Gateway's
|
|
# resource factory is patched, without replacing its live registry.
|
|
path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/checkpoints.py"
|
|
spec = importlib.util.spec_from_file_location("_checkpoint_factory_import_probe", path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
monkeypatch.setitem(sys.modules, spec.name, module)
|
|
with monkeypatch.context() as importing:
|
|
importing.setattr(async_provider, "make_checkpointer", temporary_factory)
|
|
spec.loader.exec_module(module)
|
|
|
|
config = _config(tmp_path)
|
|
invocation = {"configurable": {"thread_id": "restored-factory-thread"}}
|
|
async with make_checkpointer(config) as parent_saver:
|
|
runtime = module.SubagentCheckpointRuntime(config, parent_saver)
|
|
try:
|
|
await runtime.start()
|
|
builder = StateGraph(MessagesState)
|
|
builder.add_node("reply", lambda _state: {"messages": [AIMessage(content="RESTORED_FACTORY")]})
|
|
builder.add_edge(START, "reply")
|
|
builder.add_edge("reply", END)
|
|
graph = builder.compile(checkpointer=runtime.saver)
|
|
state = await graph.ainvoke({"messages": [HumanMessage(content="Record evidence")]}, invocation)
|
|
assert state["messages"][-1].content == "RESTORED_FACTORY"
|
|
finally:
|
|
await runtime.stop()
|
|
|
|
# A fresh reader must find real SQLite history after both writers close.
|
|
async with make_checkpointer(config) as reader:
|
|
saved = await reader.aget_tuple(invocation)
|
|
assert saved is not None
|
|
assert saved.checkpoint["channel_values"]["messages"][-1].content == "RESTORED_FACTORY"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("enabled", [False, True], ids=["disabled", "enabled"])
|
|
@pytest.mark.parametrize("mode", ["full", "delta"])
|
|
async def test_task_tool_checkpoint_history_reopens_with_delegation_identity(native_task_tool, tmp_path, monkeypatch, enabled, mode):
|
|
from langchain.agents import create_agent
|
|
|
|
from deerflow.agents.thread_state import get_thread_state_schema
|
|
from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
|
|
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
|
|
|
app_config = _config(tmp_path, enabled=enabled, mode=mode)
|
|
config_path = tmp_path / "config.yaml"
|
|
config_path.write_text(app_config.model_dump_json(), encoding="utf-8")
|
|
extensions_path = tmp_path / "extensions_config.json"
|
|
extensions_path.write_text("{}", encoding="utf-8")
|
|
monkeypatch.setenv("DEER_FLOW_CONFIG_PATH", str(config_path))
|
|
monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(extensions_path))
|
|
monkeypatch.setattr("deerflow.tools.get_available_tools", lambda **kwargs: [echo])
|
|
freeze_checkpoint_channel_mode(mode)
|
|
|
|
class ParentModel(_OfflineModel):
|
|
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
|
|
if any(isinstance(message, ToolMessage) for message in messages):
|
|
answer = AIMessage(content="PARENT_FINAL")
|
|
else:
|
|
answer = AIMessage(
|
|
content="",
|
|
tool_calls=[{"id": "outer-call", "name": "task", "args": {"subagent_type": "general-purpose", "prompt": "Use echo once, then report the result", "description": "Offline checkpoint integration"}}],
|
|
)
|
|
return ChatResult(generations=[ChatGeneration(message=answer)])
|
|
|
|
async with make_checkpointer(app_config) as saver:
|
|
async with subagent_checkpoint_runtime(app_config, saver):
|
|
parent = create_agent(model=ParentModel(), tools=[native_task_tool], checkpointer=saver, context_schema=dict, state_schema=get_thread_state_schema(mode))
|
|
state = await asyncio.wait_for(
|
|
parent.ainvoke(
|
|
{"messages": [HumanMessage(content="Delegate the offline work")]},
|
|
config={"configurable": {"thread_id": "task-history-thread"}, "metadata": {"model_name": "offline", "run_id": "task-parent-run"}},
|
|
context={"app_config": app_config, "thread_id": "task-history-thread", "run_id": "task-parent-run", "user_id": "task-user", "user_role": "admin"},
|
|
),
|
|
timeout=25,
|
|
)
|
|
outer_result = next(message for message in state["messages"] if isinstance(message, ToolMessage) and message.tool_call_id == "outer-call")
|
|
assert outer_result.additional_kwargs["subagent_status"] == "completed"
|
|
assert "CHILD_FINAL" in outer_result.content
|
|
assert state["messages"][-1].content == "PARENT_FINAL"
|
|
assert not any(isinstance(message, ToolMessage) and message.tool_call_id == "child-call" for message in state["messages"])
|
|
|
|
# Both writer lifetimes have ended: read from a new connection, just as
|
|
# a history consumer does after a Gateway restart.
|
|
async with make_checkpointer(app_config) as reader:
|
|
history = [entry async for entry in reader.alist({"configurable": {"thread_id": "task-history-thread"}})]
|
|
roots = [entry for entry in history if not entry.config["configurable"].get("checkpoint_ns")]
|
|
children = [entry for entry in history if entry.config["configurable"].get("checkpoint_ns")]
|
|
assert roots
|
|
if not enabled:
|
|
assert not children
|
|
return
|
|
assert children
|
|
namespaces = {entry.config["configurable"]["checkpoint_ns"] for entry in children}
|
|
execution_ids = {entry.metadata["deerflow_subagent_execution_id"] for entry in children}
|
|
assert len(namespaces) == len(execution_ids) == 1
|
|
execution_id = next(iter(execution_ids))
|
|
namespace = next(iter(namespaces))
|
|
assert execution_id != "outer-call"
|
|
assert namespace.endswith(f"|deerflow_subagent:{execution_id}")
|
|
assert all(entry.metadata["deerflow_subagent_name"] == "general-purpose" for entry in children)
|
|
assert all(entry.metadata["deerflow_parent_run_id"] == "task-parent-run" for entry in children)
|
|
assert all(entry.metadata["deerflow_subagent_tool_call_id"] == "outer-call" for entry in children)
|
|
root_ids = {entry.checkpoint["id"] for entry in roots}
|
|
assert all(entry.metadata["parents"][""] in root_ids for entry in children)
|
|
|
|
child = create_agent(model=_OfflineModel(), tools=[echo], state_schema=get_thread_state_schema(mode), checkpointer=reader)
|
|
restored = await child.aget_state({"configurable": {"thread_id": "task-history-thread", "checkpoint_ns": namespace, "__pregel_checkpointer": reader}})
|
|
messages = restored.values["messages"]
|
|
calls = [call for message in messages if isinstance(message, AIMessage) for call in message.tool_calls]
|
|
assert any(call["id"] == "child-call" and call["name"] == "echo" and call["args"] == {"text": "child evidence"} for call in calls)
|
|
assert any(isinstance(message, ToolMessage) and message.tool_call_id == "child-call" and message.content == "child evidence" for message in messages)
|
|
assert messages[-1].content == "CHILD_FINAL"
|
|
assert all(message.content not in {"Delegate the offline work", "PARENT_FINAL"} for message in messages if isinstance(message.content, str))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("restriction", ["global", "subagent"])
|
|
@pytest.mark.parametrize("rebuild_on_copy", [False, True], ids=["preserving-copy", "rebuilding-copy"])
|
|
async def test_delta_middleware_copies_preserve_tool_restrictions(native_executor, tmp_path, monkeypatch, restriction, rebuild_on_copy):
|
|
from deerflow.config.authorization_config import AuthorizationConfig
|
|
from deerflow.config.extensions_config import ConfiguredMiddlewareSpec
|
|
from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
|
|
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
|
from deerflow.subagents.config import SubagentConfig
|
|
|
|
app_config = _config(tmp_path, mode="delta")
|
|
app_config.extensions.middlewares = [ConfiguredMiddlewareSpec(class_path=f"{__name__}:_CopyingToolMiddleware", kwargs={"rebuild_on_copy": rebuild_on_copy}, apply_to="subagent")]
|
|
if restriction == "global":
|
|
app_config.authorization = AuthorizationConfig.model_validate({"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"user": {"tools": {"allow": "*", "deny": ["forbidden"]}}}}}})
|
|
freeze_checkpoint_channel_mode("delta")
|
|
bound_names = []
|
|
|
|
class RecordingModel(_OfflineModel):
|
|
def bind_tools(self, tools, **kwargs):
|
|
bound_names.append([tool.name for tool in tools])
|
|
return self
|
|
|
|
monkeypatch.setattr(native_executor, "create_chat_model", lambda **kwargs: RecordingModel())
|
|
|
|
async def delegate(_state):
|
|
executor = native_executor.SubagentExecutor(
|
|
config=SubagentConfig(name="copy-policy", description="Check extension copies", skills=[], disallowed_tools=["forbidden"] if restriction == "subagent" else []),
|
|
tools=[echo],
|
|
parent_model="offline",
|
|
thread_id="copy-thread",
|
|
user_id="copy-user",
|
|
user_role="user",
|
|
app_config=app_config,
|
|
)
|
|
execution_id = executor.execute_async("Use echo to return child evidence")
|
|
result = native_executor.get_background_task_result(execution_id)
|
|
try:
|
|
assert await asyncio.to_thread(result.execution_done_event.wait, 15)
|
|
if rebuild_on_copy:
|
|
assert result.status is native_executor.SubagentStatus.FAILED
|
|
assert "forbidden" in result.error
|
|
assert ("not authorized" if restriction == "global" else "final tool set") in result.error
|
|
assert not bound_names
|
|
else:
|
|
assert result.status is native_executor.SubagentStatus.COMPLETED, result.error
|
|
assert bound_names
|
|
assert all("echo" in names and "forbidden" not in names for names in bound_names)
|
|
finally:
|
|
native_executor.cleanup_background_task(result.task_id)
|
|
return {}
|
|
|
|
builder = StateGraph(MessagesState)
|
|
builder.add_node("delegate", delegate)
|
|
builder.add_edge(START, "delegate")
|
|
builder.add_edge("delegate", END)
|
|
async with make_checkpointer(app_config) as saver:
|
|
async with subagent_checkpoint_runtime(app_config, saver):
|
|
parent = builder.compile(checkpointer=saver)
|
|
await parent.ainvoke({"messages": [HumanMessage(content="Check tool restrictions")]}, {"configurable": {"thread_id": "copy-thread"}})
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("enabled", [False, True])
|
|
@pytest.mark.parametrize("mode", ["full", "delta"])
|
|
@pytest.mark.parametrize("clone_parent", [False, True])
|
|
async def test_parallel_children_reopen_with_execution_identity(native_executor, tmp_path, enabled, mode, clone_parent):
|
|
from deerflow.runtime.checkpoint_mode import freeze_checkpoint_channel_mode
|
|
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
|
from deerflow.subagents.config import SubagentConfig
|
|
|
|
app_config = _config(tmp_path, enabled=enabled, mode=mode)
|
|
freeze_checkpoint_channel_mode(mode)
|
|
completed = []
|
|
parent_loop = asyncio.get_running_loop()
|
|
|
|
async with make_checkpointer(app_config) as parent_saver:
|
|
async with subagent_checkpoint_runtime(app_config, parent_saver) as runtime:
|
|
|
|
async def delegate(_state, config: RunnableConfig):
|
|
# Bind the real SQLite lock to the parent loop through actual
|
|
# contention, and keep it held until both children complete.
|
|
# This stabilizes the original cross-loop failure without
|
|
# injecting an exception or holding a database transaction.
|
|
raw = getattr(parent_saver, "source_saver", parent_saver)
|
|
await raw.lock.acquire()
|
|
waiter = asyncio.create_task(raw.setup())
|
|
await asyncio.sleep(0)
|
|
assert raw.lock._loop is parent_loop
|
|
parent_context = var_child_runnable_config.get()
|
|
parent_namespace = parent_context["configurable"]["checkpoint_ns"]
|
|
inherited_saver = parent_context["configurable"]["__pregel_checkpointer"]
|
|
try:
|
|
results = []
|
|
for _ in range(2):
|
|
executor = native_executor.SubagentExecutor(
|
|
config=SubagentConfig(name="checkpoint-test", description="Offline checkpoint regression", system_prompt="Use echo once, then answer.", timeout_seconds=30),
|
|
tools=[echo],
|
|
parent_model="offline",
|
|
thread_id="history-thread",
|
|
run_id=config["metadata"]["run_id"],
|
|
app_config=app_config,
|
|
)
|
|
# Intentionally reused across children and parent runs.
|
|
execution_id = executor.execute_async("Return evidence", task_id="reused-provider-call")
|
|
results.append(native_executor.get_background_task_result(execution_id))
|
|
await asyncio.wait_for(asyncio.gather(*(asyncio.to_thread(result.execution_done_event.wait) for result in results)), timeout=20)
|
|
for result in results:
|
|
assert result.status.value == "completed", result.error
|
|
completed.append((config["metadata"]["run_id"], result.task_id))
|
|
finally:
|
|
raw.lock.release()
|
|
await waiter
|
|
for result in results:
|
|
native_executor.cleanup_background_task(result.task_id)
|
|
assert var_child_runnable_config.get() is parent_context
|
|
assert parent_context["configurable"]["checkpoint_ns"] == parent_namespace
|
|
assert parent_context["configurable"]["__pregel_checkpointer"] is inherited_saver
|
|
assert "deerflow_subagent_execution_id" not in parent_context["metadata"]
|
|
return {"messages": [AIMessage(content="PARENT_FINAL")]}
|
|
|
|
builder = StateGraph(MessagesState)
|
|
builder.add_node("delegate", delegate)
|
|
builder.add_edge(START, "delegate")
|
|
builder.add_edge("delegate", END)
|
|
# Same connection, different saver identity: mirrors the strict
|
|
# serializer allowlist clones used by newer LangGraph releases.
|
|
graph_saver = parent_saver.with_allowlist({("builtins", "dict")}) if clone_parent else parent_saver
|
|
parent = builder.compile(checkpointer=graph_saver)
|
|
for run_id in ("parent-run-1", "parent-run-2"):
|
|
streamed = [
|
|
message
|
|
async for message, _ in parent.astream(
|
|
{"messages": [HumanMessage(content="Delegate")]},
|
|
config={"configurable": {"thread_id": "history-thread"}, "metadata": {"run_id": run_id}},
|
|
stream_mode="messages",
|
|
)
|
|
]
|
|
assert any(message.content == "PARENT_FINAL" for message in streamed)
|
|
assert all(message.content != "CHILD_FINAL" and not isinstance(message, ToolMessage) for message in streamed)
|
|
if enabled:
|
|
assert runtime.saver is not parent_saver
|
|
assert runtime.loop is not parent_loop
|
|
if enabled:
|
|
assert runtime.closed
|
|
|
|
# Fresh resources materialize evidence after both writer lifetimes end.
|
|
async with make_checkpointer(app_config) as reader:
|
|
history = [entry async for entry in reader.alist({"configurable": {"thread_id": "history-thread"}})]
|
|
root = [entry for entry in history if not entry.config["configurable"].get("checkpoint_ns")]
|
|
assert root
|
|
root_ids = {entry.checkpoint["id"] for entry in root}
|
|
assert all(message.content != "CHILD_FINAL" for entry in root for message in entry.checkpoint["channel_values"].get("messages", []))
|
|
children = [entry for entry in history if entry.config["configurable"].get("checkpoint_ns")]
|
|
if not enabled:
|
|
assert not children
|
|
return
|
|
groups = defaultdict(list)
|
|
for entry in children:
|
|
groups[entry.metadata["deerflow_subagent_execution_id"]].append(entry)
|
|
assert set(groups) == {execution_id for _, execution_id in completed}
|
|
namespaces = set()
|
|
for parent_run_id, execution_id in completed:
|
|
entries = groups[execution_id]
|
|
own_namespaces = {entry.config["configurable"]["checkpoint_ns"] for entry in entries}
|
|
assert len(own_namespaces) == 1
|
|
assert not own_namespaces & namespaces
|
|
namespaces.update(own_namespaces)
|
|
assert all(entry.metadata["deerflow_parent_run_id"] == parent_run_id for entry in entries)
|
|
assert all(entry.metadata["deerflow_subagent_tool_call_id"] == "reused-provider-call" for entry in entries)
|
|
assert all(entry.metadata["parents"][""] in root_ids for entry in entries)
|
|
# Use the actual child graph schema to reconstruct full and delta
|
|
# states, rather than expecting raw delta channel blobs to be text.
|
|
from deerflow.agents.thread_state import get_thread_state_schema
|
|
|
|
accessor_builder = StateGraph(get_thread_state_schema(mode))
|
|
accessor_builder.add_node("unused", lambda state: {})
|
|
accessor_builder.add_edge(START, "unused")
|
|
accessor_builder.add_edge("unused", END)
|
|
accessor = accessor_builder.compile(checkpointer=reader)
|
|
snapshot = await accessor.aget_state({"configurable": {"thread_id": "history-thread", "checkpoint_ns": next(iter(own_namespaces)), "__pregel_checkpointer": reader}})
|
|
messages = snapshot.values["messages"]
|
|
assert any(isinstance(message, ToolMessage) and "child evidence" in message.content for message in messages)
|
|
assert any(message.content == "CHILD_FINAL" for message in messages)
|
|
assert all(message.content not in {"Delegate", "PARENT_FINAL"} for message in messages if isinstance(message.content, str))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("legacy", [False, True])
|
|
@pytest.mark.parametrize("enabled", [False, True])
|
|
async def test_checkpoint_history_only_accepts_sqlite(native_executor, tmp_path, legacy, enabled):
|
|
from deerflow.config.checkpointer_config import CheckpointerConfig
|
|
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
|
|
|
config = _config(tmp_path, enabled=enabled)
|
|
async with make_checkpointer(config) as saver:
|
|
# Only exercise admission here: no PostgreSQL driver or server is used.
|
|
# A legacy section must take precedence over the unified SQLite config.
|
|
if legacy:
|
|
config.checkpointer = CheckpointerConfig(type="postgres")
|
|
else:
|
|
config.database.backend = "postgres"
|
|
if enabled:
|
|
with pytest.raises(ValueError, match="only supports file-backed SQLite"):
|
|
async with subagent_checkpoint_runtime(config, saver):
|
|
pytest.fail("Unsupported checkpoint backend was accepted")
|
|
else:
|
|
async with subagent_checkpoint_runtime(config, saver) as runtime:
|
|
assert runtime is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("legacy", [False, True])
|
|
async def test_memory_checkpoint_history_is_rejected(native_executor, tmp_path, legacy):
|
|
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
|
|
|
config = _config(tmp_path)
|
|
if legacy:
|
|
from deerflow.config.checkpointer_config import CheckpointerConfig
|
|
|
|
config.checkpointer = CheckpointerConfig(type="sqlite", connection_string=":memory:")
|
|
else:
|
|
config.database.backend = "memory"
|
|
async with make_checkpointer(config) as saver:
|
|
with pytest.raises(ValueError, match="persistent|file-backed"):
|
|
async with subagent_checkpoint_runtime(config, saver):
|
|
pytest.fail("Non-persistent checkpoint storage was accepted")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_partial_host_startup_closes_peer_before_parent(native_executor, tmp_path):
|
|
from deerflow.subagents.checkpoints import _runtimes, subagent_checkpoint_runtime
|
|
|
|
config = _config(tmp_path)
|
|
async with make_checkpointer(config) as saver:
|
|
with pytest.raises(RuntimeError, match="later startup failure"):
|
|
async with subagent_checkpoint_runtime(config, saver) as runtime:
|
|
raise RuntimeError("later startup failure")
|
|
assert runtime.closed
|
|
assert id(saver) not in _runtimes
|
|
# Parent resource remains usable after the peer's teardown.
|
|
await saver.setup()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_gateway_runtime_installs_and_closes_peer(native_executor, tmp_path):
|
|
from fastapi import FastAPI
|
|
|
|
from app.gateway.deps import langgraph_runtime
|
|
from deerflow.subagents.checkpoints import _runtimes
|
|
|
|
config = _config(tmp_path)
|
|
app = FastAPI()
|
|
async with langgraph_runtime(app, config):
|
|
saver = app.state.checkpointer
|
|
runtime = _runtimes[id(saver)]
|
|
assert runtime.saver is not saver
|
|
assert runtime.loop is not asyncio.get_running_loop()
|
|
assert config.run_events.backend == "memory"
|
|
await saver.setup()
|
|
assert runtime.closed
|
|
assert id(saver) not in _runtimes
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_startup_cancellation_closes_created_peer(native_executor, tmp_path, monkeypatch):
|
|
from deerflow.subagents.checkpoints import SubagentCheckpointRuntime, _runtimes, subagent_checkpoint_runtime
|
|
|
|
config = _config(tmp_path)
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
opened = []
|
|
original_start = SubagentCheckpointRuntime.start
|
|
|
|
async def paused_start(runtime):
|
|
await original_start(runtime)
|
|
opened.append(runtime)
|
|
started.set()
|
|
await asyncio.to_thread(release.wait)
|
|
|
|
monkeypatch.setattr(SubagentCheckpointRuntime, "start", paused_start)
|
|
async with make_checkpointer(config) as saver:
|
|
manager = subagent_checkpoint_runtime(config, saver)
|
|
startup = asyncio.create_task(manager.__aenter__())
|
|
try:
|
|
assert await asyncio.to_thread(started.wait, 10)
|
|
startup.cancel()
|
|
await asyncio.sleep(0)
|
|
startup.cancel()
|
|
await asyncio.sleep(0)
|
|
assert not startup.done()
|
|
finally:
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await asyncio.wait_for(startup, 10)
|
|
assert opened[0].closed
|
|
assert id(saver) not in _runtimes
|
|
await saver.setup()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_loop_shutdown_closes_peer_on_owner_loop(native_executor, tmp_path):
|
|
from deerflow.subagents.checkpoints import subagent_checkpoint_runtime
|
|
|
|
config = _config(tmp_path)
|
|
async with make_checkpointer(config) as saver:
|
|
async with subagent_checkpoint_runtime(config, saver) as runtime:
|
|
owner = runtime.loop
|
|
await asyncio.to_thread(native_executor._shutdown_isolated_subagent_loop)
|
|
assert runtime.closed
|
|
assert owner.is_closed()
|
|
await saver.setup()
|
|
|
|
|
|
def test_default_sync_parent_keeps_async_child_isolated(native_executor, tmp_path):
|
|
from langgraph.checkpoint.sqlite import SqliteSaver
|
|
|
|
from deerflow.subagents.config import SubagentConfig
|
|
|
|
config = _config(tmp_path, enabled=False)
|
|
|
|
def delegate(_state):
|
|
executor = native_executor.SubagentExecutor(
|
|
config=SubagentConfig(name="checkpoint-test", description="Sync parent regression", system_prompt="Use echo.", timeout_seconds=30),
|
|
tools=[echo],
|
|
parent_model="offline",
|
|
thread_id="sync-thread",
|
|
app_config=config,
|
|
)
|
|
result = executor.execute("Return evidence")
|
|
assert result.status.value == "completed", result.error
|
|
return {"messages": [AIMessage(content="PARENT_DONE")]}
|
|
|
|
with SqliteSaver.from_conn_string(str(tmp_path / "sync.sqlite")) as saver:
|
|
builder = StateGraph(MessagesState)
|
|
builder.add_node("delegate", delegate)
|
|
builder.add_edge(START, "delegate")
|
|
builder.add_edge("delegate", END)
|
|
parent = builder.compile(checkpointer=saver)
|
|
state = parent.invoke({"messages": [HumanMessage(content="Delegate")]}, {"configurable": {"thread_id": "sync-thread"}})
|
|
assert state["messages"][-1].content == "PARENT_DONE"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_shutdown_drains_cancelled_child_and_fences_late_dispatch(native_executor, tmp_path):
|
|
from deerflow.subagents.checkpoints import _runtimes, subagent_checkpoint_runtime
|
|
from deerflow.subagents.config import SubagentConfig
|
|
|
|
config = _config(tmp_path)
|
|
entered = threading.Event()
|
|
cleanup_entered = threading.Event()
|
|
release_cleanup = threading.Event()
|
|
results = []
|
|
captured_contexts = []
|
|
|
|
@tool("echo")
|
|
async def blocking_echo(text: str) -> str:
|
|
"""Pause a deterministic child until shutdown cancels it."""
|
|
entered.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
finally:
|
|
cleanup_entered.set()
|
|
await asyncio.to_thread(release_cleanup.wait)
|
|
return text
|
|
|
|
def executor():
|
|
return native_executor.SubagentExecutor(
|
|
config=SubagentConfig(name="checkpoint-test", description="Shutdown regression", system_prompt="Use echo.", timeout_seconds=30),
|
|
tools=[blocking_echo],
|
|
parent_model="offline",
|
|
thread_id="shutdown-thread",
|
|
run_id="shutdown-parent-run",
|
|
app_config=config,
|
|
)
|
|
|
|
async def delegate(_state):
|
|
captured_contexts.append(native_executor._copy_isolated_subagent_context())
|
|
execution_id = executor().execute_async("Pause")
|
|
result = native_executor.get_background_task_result(execution_id)
|
|
results.append(result)
|
|
await asyncio.to_thread(result.execution_done_event.wait)
|
|
return {"messages": [AIMessage(content="PARENT_DONE")]}
|
|
|
|
async with make_checkpointer(config) as saver:
|
|
manager = subagent_checkpoint_runtime(config, saver)
|
|
runtime = await manager.__aenter__()
|
|
builder = StateGraph(MessagesState)
|
|
builder.add_node("delegate", delegate)
|
|
builder.add_edge(START, "delegate")
|
|
builder.add_edge("delegate", END)
|
|
parent = builder.compile(checkpointer=saver)
|
|
parent_task = asyncio.create_task(parent.ainvoke({"messages": [HumanMessage(content="Pause")]}, {"configurable": {"thread_id": "shutdown-thread"}}))
|
|
stop = None
|
|
late_result = None
|
|
try:
|
|
assert await asyncio.to_thread(entered.wait, 10)
|
|
stop = asyncio.create_task(manager.__aexit__(None, None, None))
|
|
assert await asyncio.to_thread(cleanup_entered.wait, 10)
|
|
stop.cancel()
|
|
await asyncio.sleep(0)
|
|
stop.cancel()
|
|
await asyncio.sleep(0)
|
|
assert not stop.done()
|
|
assert not runtime.closed
|
|
assert runtime.closing
|
|
await saver.setup()
|
|
|
|
late_id = captured_contexts[0].run(executor().execute_async, "Late task")
|
|
late_result = native_executor.get_background_task_result(late_id)
|
|
assert await asyncio.to_thread(late_result.execution_done_event.wait, 10)
|
|
assert late_result.status.value == "failed"
|
|
assert "shutting down" in late_result.error
|
|
finally:
|
|
release_cleanup.set()
|
|
if stop is not None:
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await asyncio.wait_for(stop, 10)
|
|
else:
|
|
await manager.__aexit__(None, None, None)
|
|
await asyncio.wait_for(parent_task, 10)
|
|
for result in [*results, *([late_result] if late_result is not None else [])]:
|
|
native_executor.cleanup_background_task(result.task_id)
|
|
assert runtime.closed
|
|
assert id(saver) not in _runtimes
|
|
assert results[0].status.value == "cancelled"
|
|
assert results[0].execution_done_event.is_set()
|
|
await saver.setup()
|
|
# A dispatch context captured before shutdown must remain fenced even
|
|
# after the runtime has been removed from the process registry.
|
|
after_close_id = captured_contexts[0].run(executor().execute_async, "After close")
|
|
after_close = native_executor.get_background_task_result(after_close_id)
|
|
try:
|
|
assert await asyncio.to_thread(after_close.execution_done_event.wait, 10)
|
|
assert after_close.status.value == "failed"
|
|
assert "shutting down" in after_close.error
|
|
finally:
|
|
native_executor.cleanup_background_task(after_close_id)
|