mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-13 14:50:36 +00:00
8850f8eb7e
* fix(runtime): keep assembly evidence from reviving a run a peer reclaimed In heartbeat mode every run-row write is conditional on this worker's live lease except assembly evidence, which went through RunStore.put(). That upsert rewrote status, owner, lease, error and stop_reason from the local record. When a peer had already reclaimed the run, it put the row back to running and erased the takeover error. If the owner's local lease was still live, its renewal then succeeded, the peer never reclaimed it again, and the run finished as success after the peer had published its terminal recovery event. If the write landed late, the row came back with an expired lease and was reclaimed a second time. Add RunStore.merge_owned_metadata(), which merges metadata keys only for the owner holding a live lease on a pending/running row and writes no other field. SQL follows the existing clock -> row lock -> conditional UPDATE order; MemoryRunStore mirrors it. With heartbeat enabled, record_assembly_evidence() uses it, refuses a locally fenced run before any write, fences the run when the store rejects the merge, and fails closed on stores without the method instead of falling back to put(). Single-worker mode keeps the full snapshot. * docs(changelog): reference #8046 for the assembly evidence takeover fix * test(runtime): scope the simulated lease expiry to the run's thread --------- Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
670 lines
34 KiB
Python
670 lines
34 KiB
Python
"""Host-bound run control: authorization, lifecycle and ordinary admission."""
|
|
|
|
import os
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
from deerflow_extension_api.agent_runs import AgentRunError, require_agent_runs, resolve_agent_runs
|
|
from fastapi import FastAPI, Request
|
|
|
|
from app.gateway.auth.models import User
|
|
from app.gateway.authz import AuthContext
|
|
from app.gateway.extension_agent_runs import GatewayAgentRunsHost
|
|
|
|
|
|
@pytest.fixture
|
|
def host(monkeypatch):
|
|
user = User(email="alice@example.com")
|
|
app = FastAPI()
|
|
load_user = AsyncMock(return_value=user)
|
|
control = GatewayAgentRunsHost(app, load_user=load_user)
|
|
request = Request({"type": "http", "app": app, "headers": [], "state": {}})
|
|
request.state.user = user
|
|
request.state.auth_source = "session"
|
|
request.state.auth = AuthContext(user, ["threads:write", "threads:read", "runs:create", "runs:read", "runs:cancel"])
|
|
monkeypatch.setattr("app.gateway.extension_agent_runs.resolve_route_permissions", AsyncMock(return_value=list(request.state.auth.permissions)))
|
|
return control, request, load_user
|
|
|
|
|
|
def test_unsupported_host_and_credentials_fail_closed(host):
|
|
control, request, _ = host
|
|
assert resolve_agent_runs(request) is None
|
|
with pytest.raises(NotImplementedError):
|
|
require_agent_runs(request)
|
|
request.state.auth_source = "pat"
|
|
assert control.bind(request) is None
|
|
request.state.auth_source = "internal"
|
|
assert control.bind(request) is None
|
|
|
|
|
|
def test_unstamped_requests_receive_no_delegated_capability(host):
|
|
control, request, _ = host
|
|
assert control.bind(SimpleNamespace(state=SimpleNamespace())) is None
|
|
assert control.bind(SimpleNamespace(state=SimpleNamespace(user=request.state.user, auth_source="internal"))) is None
|
|
request.state.auth = SimpleNamespace(is_authenticated=False)
|
|
assert control.bind(request) is None
|
|
|
|
|
|
def test_binding_rejects_inconsistent_authenticated_identity(host):
|
|
control, request, _ = host
|
|
request.state.user = User(email="other@example.com")
|
|
with pytest.raises(PermissionError):
|
|
control.bind(request)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_handle_rechecks_user_and_shutdown_before_dispatch(host, monkeypatch):
|
|
control, request, load_user = host
|
|
runs = control.bind(request)
|
|
handler = AsyncMock(return_value=SimpleNamespace(thread_id="thread-a"))
|
|
monkeypatch.setattr("app.gateway.routers.threads.create_thread", handler)
|
|
assert await runs.create_thread() == "thread-a"
|
|
load_user.return_value = request.state.user.model_copy(update={"token_version": 1})
|
|
with pytest.raises(AgentRunError, match="revoked"):
|
|
await runs.create_thread()
|
|
load_user.return_value = request.state.user
|
|
control.close()
|
|
with pytest.raises(AgentRunError, match="unavailable"):
|
|
await runs.create_thread()
|
|
assert handler.await_count == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_permissions_are_intersected_with_original_grant_and_current_policy(host, monkeypatch):
|
|
control, request, _ = host
|
|
request.state.auth.permissions = ["runs:read"]
|
|
runs = control.bind(request)
|
|
request.state.auth.permissions.append("threads:write")
|
|
monkeypatch.setattr("app.gateway.extension_agent_runs.resolve_route_permissions", AsyncMock(return_value=["threads:write", "runs:read"]))
|
|
with pytest.raises(AgentRunError) as denied:
|
|
await runs.create_thread()
|
|
assert denied.value.status_code == 403
|
|
monkeypatch.setattr("app.gateway.extension_agent_runs.resolve_route_permissions", AsyncMock(return_value=[]))
|
|
with pytest.raises(AgentRunError) as revoked:
|
|
await runs.get(thread_id="thread-a", run_id="run-a")
|
|
assert revoked.value.status_code == 403
|
|
|
|
|
|
def test_runtime_handle_is_host_owned_redacted_and_released():
|
|
from deerflow_extension_api.agent_runs import AGENT_RUNS_CONTEXT_KEY
|
|
|
|
from deerflow.runtime.runs.worker import _build_runtime_context, _install_runtime_context, _release_run_scoped_references
|
|
from deerflow.runtime.secret_context import redact_config_secrets
|
|
|
|
key = AGENT_RUNS_CONTEXT_KEY
|
|
assert key not in _build_runtime_context("thread", "run", {key: "forged"})
|
|
handle = object()
|
|
runtime = _build_runtime_context("thread", "run", {key: "forged"}, agent_runs=handle)
|
|
config = {"context": {key: "forged"}, "configurable": {key: "forged"}}
|
|
_install_runtime_context(config, runtime)
|
|
assert config["context"][key] is handle
|
|
assert key not in config["configurable"]
|
|
assert key not in redact_config_secrets(config)["context"]
|
|
_release_run_scoped_references([config], runtime, None)
|
|
assert key not in runtime
|
|
assert key not in config["context"]
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def runtime(host, monkeypatch, tmp_path):
|
|
import asyncio
|
|
|
|
from deerflow_extension_api.agent_runs import AGENT_RUNS_CONTEXT_KEY
|
|
from langchain_core.messages import AIMessage
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
from langgraph.graph import MessagesState, StateGraph
|
|
from langgraph.runtime import Runtime
|
|
from langgraph.store.memory import InMemoryStore
|
|
from langgraph.types import interrupt
|
|
|
|
from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config
|
|
from deerflow.persistence.thread_meta.memory import MemoryThreadMetaStore
|
|
from deerflow.runtime import RunManager
|
|
from deerflow.runtime.events.store.memory import MemoryRunEventStore
|
|
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
|
from deerflow.runtime.stream_bridge.memory import MemoryStreamBridge
|
|
|
|
control, request, _ = host
|
|
state = request.app.state
|
|
state.agent_runs_host = control
|
|
state.checkpointer = InMemorySaver()
|
|
state.store = InMemoryStore()
|
|
state.thread_store = MemoryThreadMetaStore(state.store)
|
|
state.run_store = MemoryRunStore()
|
|
state.run_manager = RunManager(store=state.run_store)
|
|
state.run_event_store = MemoryRunEventStore()
|
|
state.stream_bridge = MemoryStreamBridge()
|
|
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path))
|
|
set_app_config(AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}}))
|
|
entered = asyncio.Event()
|
|
handles = []
|
|
|
|
async def answer(state: MessagesState, runtime: Runtime):
|
|
handles.append(runtime.context.get(AGENT_RUNS_CONTEXT_KEY))
|
|
prompt = state["messages"][-1].content
|
|
if prompt == "wait":
|
|
entered.set()
|
|
await asyncio.Event().wait()
|
|
if prompt == "approve":
|
|
prompt = interrupt({"question": "Continue?"})
|
|
return {"messages": [AIMessage(content=f"answer:{prompt}")]}
|
|
|
|
def factory(*, config):
|
|
builder = StateGraph(MessagesState)
|
|
builder.add_node("answer", answer)
|
|
builder.set_entry_point("answer")
|
|
builder.set_finish_point("answer")
|
|
from deerflow_extension_api import AgentAssemblyDescriptor
|
|
|
|
from deerflow.agents.lead_agent.agent import LeadAgentAssembly
|
|
|
|
descriptor = AgentAssemblyDescriptor(
|
|
namespace="fixture",
|
|
agent_name="lead-agent",
|
|
requested_model=None,
|
|
effective_model="fixture-model",
|
|
model_parameters={"temperature": 0},
|
|
thinking_enabled=False,
|
|
reasoning_effort=None,
|
|
base_prompt_hash="fixture-prompt",
|
|
tools=(),
|
|
middlewares=(),
|
|
deferred_tool_names=(),
|
|
enabled_skills=(),
|
|
effective_policies={"skill_catalog_hash": "fixture-skills"},
|
|
)
|
|
return LeadAgentAssembly(graph=builder.compile(), descriptor=descriptor, effective_model="fixture-model")
|
|
|
|
monkeypatch.setattr("app.gateway.services.resolve_agent_factory", lambda _: factory)
|
|
try:
|
|
yield control.bind(request), state, request, entered, handles
|
|
finally:
|
|
control.close()
|
|
await state.run_manager.shutdown()
|
|
reset_app_config()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_real_run_lifecycle_and_automatic_followup(runtime):
|
|
runs, state, request, entered, handles = runtime
|
|
thread_a = await runs.create_thread(thread_id="extension-a")
|
|
first = await runs.start(thread_id=thread_a, input={"messages": [{"role": "user", "content": "first"}]}, idempotency_key="team:first")
|
|
assert (await runs.wait(thread_id=thread_a, run_id=first.run_id, timeout=5)).status == "success"
|
|
retry = await runs.start(thread_id=thread_a, input={"messages": [{"role": "user", "content": "first"}]}, idempotency_key="team:first")
|
|
assert retry.run_id == first.run_id
|
|
output_a = await runs.get_state(thread_id=thread_a)
|
|
assert output_a["values"]["messages"][-1]["content"] == "answer:first"
|
|
assert handles and all(handle is not None for handle in handles)
|
|
# A service holding only the original capability can hand A's result to B.
|
|
thread_b = await runs.create_thread(thread_id="extension-b")
|
|
second = await runs.start(thread_id=thread_b, input={"messages": [{"role": "user", "content": output_a["values"]["messages"][-1]["content"]}]})
|
|
assert (await runs.wait(thread_id=thread_b, run_id=second.run_id, timeout=5)).status == "success"
|
|
continued = await runs.start(thread_id=thread_a, input={"messages": [{"role": "user", "content": "next"}]})
|
|
assert (await runs.wait(thread_id=thread_a, run_id=continued.run_id, timeout=5)).status == "success"
|
|
snapshot = await runs.get_state(thread_id=thread_a)
|
|
assert [message["content"] for message in snapshot["values"]["messages"]] == ["first", "answer:first", "next", "answer:next"]
|
|
stored = await state.thread_store.get(thread_a, user_id=str(request.state.user.id))
|
|
assert stored["user_id"] == str(request.state.user.id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.skipif(os.name != "posix", reason="Descriptor-confined artifact imports require POSIX")
|
|
async def test_team_artifact_import_is_authorized_idempotent_and_confined(runtime):
|
|
import asyncio
|
|
import hashlib
|
|
|
|
from deerflow.config.paths import get_paths
|
|
|
|
runs, state, request, entered, _ = runtime
|
|
runs = runs.for_plugin("artifact-fixture")
|
|
thread = await runs.create_thread(thread_id="artifact-recipient")
|
|
content = b"%PDF-1.7\n\x00\xff immutable source"
|
|
imported = await runs.import_artifact(thread_id=thread, filename="report.pdf", content=content)
|
|
assert imported["sha256"] == hashlib.sha256(content).hexdigest()
|
|
assert imported["size"] == len(content)
|
|
target = get_paths().sandbox_uploads_dir(thread, user_id=str(request.state.user.id)) / imported["filename"]
|
|
assert await asyncio.to_thread(target.read_bytes) == content
|
|
assert await runs.import_artifact(thread_id=thread, filename="report.pdf", content=content) == imported
|
|
assert len(await asyncio.to_thread(lambda: list(target.parent.iterdir()))) == 1
|
|
await asyncio.to_thread(target.write_bytes, b"recipient edited its copy")
|
|
with pytest.raises(AgentRunError) as changed:
|
|
await runs.import_artifact(thread_id=thread, filename="report.pdf", content=content)
|
|
assert changed.value.status_code == 422
|
|
assert await asyncio.to_thread(target.read_bytes) == b"recipient edited its copy"
|
|
run = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "wait"}]})
|
|
await asyncio.wait_for(entered.wait(), 5)
|
|
with pytest.raises(AgentRunError) as busy:
|
|
await runs.import_artifact(thread_id=thread, filename="other.pdf", content=content)
|
|
assert busy.value.status_code == 409
|
|
with pytest.raises(AgentRunError) as capture_busy:
|
|
await runs.read_artifact(thread_id=thread, path="/mnt/user-data/outputs/report.pdf")
|
|
assert capture_busy.value.status_code == 409
|
|
await runs.cancel(thread_id=thread, run_id=run.run_id)
|
|
await runs.wait(thread_id=thread, run_id=run.run_id, timeout=5)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.skipif(os.name != "posix", reason="Descriptor-confined artifact imports require POSIX")
|
|
async def test_team_artifact_import_rechecks_permission_owner_and_payload(runtime, monkeypatch):
|
|
runs, state, request, _, _ = runtime
|
|
thread = await runs.create_thread(thread_id="artifact-recipient")
|
|
runs = runs.for_plugin("artifact-fixture")
|
|
for filename, content in (("../escape.txt", b"x"), ("/etc/passwd", b"x"), ("bad\nname.txt", b"x"), ("report.pdf", b"x" * (2 * 1024 * 1024 + 1))):
|
|
with pytest.raises(AgentRunError) as invalid:
|
|
await runs.import_artifact(thread_id=thread, filename=filename, content=content)
|
|
assert invalid.value.status_code == 422
|
|
with pytest.raises(AgentRunError) as missing:
|
|
await runs.import_artifact(thread_id="unknown-thread", filename="report.txt", content=b"x")
|
|
assert missing.value.status_code == 404
|
|
monkeypatch.setattr("app.gateway.extension_agent_runs.resolve_route_permissions", AsyncMock(return_value=["threads:read"]))
|
|
with pytest.raises(AgentRunError) as denied:
|
|
await runs.import_artifact(thread_id=thread, filename="report.txt", content=b"x")
|
|
assert denied.value.status_code == 403
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("launcher", ["scheduled", "mcp"])
|
|
async def test_internal_launchers_work_with_installed_host_without_delegation(runtime, launcher, monkeypatch):
|
|
from app.gateway.services import launch_mcp_task_notification_run, launch_scheduled_thread_run
|
|
|
|
runs, state, request, _, handles = runtime
|
|
monkeypatch.setattr("app.gateway.services.get_local_provider", lambda: SimpleNamespace(get_user=AsyncMock(return_value=request.state.user)))
|
|
thread = await runs.create_thread(thread_id="internal-launch")
|
|
kwargs = dict(app=request.app, thread_id=thread, assistant_id="lead_agent", owner_user_id=str(request.state.user.id))
|
|
if launcher == "scheduled":
|
|
result = await launch_scheduled_thread_run(**kwargs, prompt="scheduled", metadata={"scheduled_task_run_id": "occurrence-1"})
|
|
else:
|
|
result = await launch_mcp_task_notification_run(**kwargs, task_id="task-1", dispatch_version=1, dispatch_attempt=1, event={"status": "completed"})
|
|
assert (await runs.wait(thread_id=thread, run_id=result["run_id"], timeout=5)).status == "success"
|
|
assert handles == [None]
|
|
assert (await state.thread_store.get(thread, user_id=str(request.state.user.id)))["user_id"] == str(request.state.user.id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_plugin_actions_isolate_same_local_idempotency_key(runtime):
|
|
import json
|
|
|
|
from deerflow_extension_api.agent_runs import AGENT_RUNS_RESOLVER_KEY
|
|
from deerflow_extension_api.auth import EXTENSION_PRINCIPAL_RESOLVER_KEY, ExtensionPrincipal
|
|
from deerflow_extension_api.plugins import BackendAction, PluginContribution
|
|
|
|
from app.gateway.routers.plugins import invoke_plugin_action
|
|
from deerflow.extensions.registry import ExtensionRegistry
|
|
|
|
runs, state, request, _, _ = runtime
|
|
thread = await runs.create_thread(thread_id="plugin-keys")
|
|
|
|
async def start(payload, context):
|
|
run = await context.agent_runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "same input"}]}, idempotency_key=payload["key"])
|
|
return {"run_id": run.run_id}
|
|
|
|
registry = ExtensionRegistry()
|
|
for namespace in ("community.first", "community.second"):
|
|
with registry.attributed_to(namespace):
|
|
registry.plugin(PluginContribution(namespace=namespace, title=namespace, enabled=True, backend=(BackendAction("start", start),)))
|
|
state.extensions = registry.build()
|
|
setattr(state, AGENT_RUNS_RESOLVER_KEY, state.agent_runs_host.bind)
|
|
setattr(state, EXTENSION_PRINCIPAL_RESOLVER_KEY, lambda _: ExtensionPrincipal(str(request.state.user.id)))
|
|
|
|
async def invoke(namespace):
|
|
async def receive():
|
|
return {"type": "http.request", "body": json.dumps({"key": "shared:operation"}).encode(), "more_body": False}
|
|
|
|
admitted = Request({"type": "http", "app": request.app, "headers": [], "state": {"user": request.state.user, "auth": request.state.auth, "auth_source": "session"}}, receive)
|
|
return await invoke_plugin_action(admitted, namespace, "start")
|
|
|
|
first = await invoke("community.first")
|
|
await runs.wait(thread_id=thread, run_id=first["run_id"], timeout=5)
|
|
second = await invoke("community.second")
|
|
assert second["run_id"] != first["run_id"]
|
|
await runs.wait(thread_id=thread, run_id=second["run_id"], timeout=5)
|
|
assert (await invoke("community.first"))["run_id"] == first["run_id"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_plugin_scoping_preserves_key_validation_and_authorization(host):
|
|
control, request, load_user = host
|
|
scoped = control.bind(request).for_plugin("community.test")
|
|
assert scoped.for_plugin("community.test") is scoped
|
|
with pytest.raises(AgentRunError):
|
|
scoped.for_plugin("community.other")
|
|
for key in ("", " ", "x" * 201, 1):
|
|
with pytest.raises(AgentRunError) as invalid:
|
|
await scoped.start(thread_id="thread", input={}, idempotency_key=key)
|
|
assert invalid.value.status_code == 422
|
|
load_user.return_value = None
|
|
with pytest.raises(AgentRunError) as revoked:
|
|
await scoped.get(thread_id="thread", run_id="run")
|
|
assert revoked.value.status_code == 403
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_wait_uses_capped_backoff_and_returns_terminal_status(host, monkeypatch):
|
|
from deerflow_extension_api.agent_runs import AgentRun
|
|
|
|
control, request, _ = host
|
|
runs = control.bind(request)
|
|
runs.get = AsyncMock(side_effect=[AgentRun("thread", "run", "running")] * 6 + [AgentRun("thread", "run", "interrupted")])
|
|
sleep = AsyncMock()
|
|
monkeypatch.setattr("app.gateway.extension_agent_runs.asyncio.sleep", sleep)
|
|
assert (await runs.wait(thread_id="thread", run_id="run")).status == "interrupted"
|
|
assert [call.args[0] for call in sleep.await_args_list] == [0.25, 0.5, 1, 2, 4, 4]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_real_interrupt_resume_conflict_and_cancel(runtime):
|
|
import asyncio
|
|
|
|
runs, _, _, entered, _ = runtime
|
|
thread = await runs.create_thread(thread_id="interrupt-test")
|
|
first = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "approve"}]})
|
|
assert (await runs.wait(thread_id=thread, run_id=first.run_id, timeout=5)).status == "waiting_for_input"
|
|
assert (await runs.get_state(thread_id=thread))["next"] == ["answer"]
|
|
resumed = await runs.resume(thread_id=thread, resume="yes")
|
|
assert (await runs.wait(thread_id=thread, run_id=resumed.run_id, timeout=5)).status == "success"
|
|
assert (await runs.get_state(thread_id=thread))["values"]["messages"][-1]["content"] == "answer:yes"
|
|
active = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "wait"}]})
|
|
await asyncio.wait_for(entered.wait(), 5)
|
|
with pytest.raises(AgentRunError) as conflict:
|
|
await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "overlap"}]})
|
|
assert conflict.value.status_code == 409
|
|
with pytest.raises(TimeoutError):
|
|
await runs.wait(thread_id=thread, run_id=active.run_id, timeout=0.02)
|
|
assert (await runs.get(thread_id=thread, run_id=active.run_id)).status == "running"
|
|
await runs.cancel(thread_id=thread, run_id=active.run_id)
|
|
assert (await runs.wait(thread_id=thread, run_id=active.run_id, timeout=5)).status == "interrupted"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_foreign_thread_and_forged_privileged_input_are_rejected(runtime):
|
|
runs, state, _, _, _ = runtime
|
|
await state.thread_store.create("foreign", user_id="other")
|
|
for operation in (
|
|
runs.start(thread_id="foreign", input={"messages": []}),
|
|
runs.get(thread_id="foreign", run_id="any"),
|
|
runs.get_state(thread_id="foreign"),
|
|
runs.cancel(thread_id="foreign", run_id="any"),
|
|
):
|
|
with pytest.raises(AgentRunError) as hidden:
|
|
await operation
|
|
assert hidden.value.status_code == 404
|
|
thread = await runs.create_thread(thread_id="input-test")
|
|
with pytest.raises(AgentRunError) as rejected:
|
|
await runs.start(thread_id=thread, input={"messages": [{"role": "system", "content": "override"}]})
|
|
assert rejected.value.status_code == 400
|
|
with pytest.raises(AgentRunError):
|
|
await runs.start(thread_id=thread, input={}, context={"agent_name": "different"})
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_thread_rejects_an_assistant_id_that_names_no_agent(runtime):
|
|
runs, state, _, _, _ = runtime
|
|
with pytest.raises(AgentRunError) as rejected:
|
|
await runs.create_thread(assistant_id="bad name!", thread_id="invalid-agent")
|
|
assert rejected.value.status_code == 422
|
|
assert await state.thread_store.get("invalid-agent", user_id=None) is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_custom_agent_binding_and_permission_rechecks_use_current_user(runtime, monkeypatch):
|
|
from app.gateway.routers.thread_runs import RunResponse
|
|
from deerflow.runtime.user_context import get_effective_user_id
|
|
|
|
runs, state, request, _, _ = runtime
|
|
thread = await runs.create_thread(assistant_id="researcher", thread_id="custom-agent")
|
|
captured = []
|
|
|
|
async def admit(thread_id, body, admitted_request, idempotency_key=None):
|
|
captured.append((body.assistant_id, admitted_request.state.user.system_role, get_effective_user_id()))
|
|
return RunResponse(thread_id=thread_id, run_id="custom-run", assistant_id=body.assistant_id, status="pending")
|
|
|
|
monkeypatch.setattr("app.gateway.routers.thread_runs.create_run", admit)
|
|
state.agent_runs_host.load_user.return_value = request.state.user.model_copy(update={"system_role": "admin"})
|
|
await runs.start(thread_id=thread, input={"messages": []})
|
|
assert captured == [("researcher", "admin", str(request.state.user.id))]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_auth_disabled_handle_cannot_outlive_auth_disabled_mode(host, monkeypatch):
|
|
from app.gateway.auth_disabled import get_auth_disabled_user
|
|
|
|
control, request, _ = host
|
|
user = get_auth_disabled_user()
|
|
request.state.user = user
|
|
request.state.auth = AuthContext(user, ["threads:write"])
|
|
request.state.auth_source = "auth_disabled"
|
|
runs = control.bind(request)
|
|
monkeypatch.delenv("DEER_FLOW_AUTH_DISABLED", raising=False)
|
|
with pytest.raises(AgentRunError) as denied:
|
|
await runs.create_thread()
|
|
assert denied.value.status_code == 403
|
|
|
|
|
|
def test_action_and_tool_context_keep_positional_compatibility():
|
|
from deerflow_extension_api import ActionContext, ExtensionPrincipal, ToolContext
|
|
|
|
principal = ExtensionPrincipal("owner")
|
|
assert ActionContext(principal, {}).agent_runs is None
|
|
assert ToolContext(principal, {}, "thread").agent_runs is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_configuration_revision_is_private_scoped_and_detects_changes(host, monkeypatch):
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock
|
|
|
|
control, request, load_user = host
|
|
app_config = {"models": [{"api_key": "secret", "name": "a"}]}
|
|
agent_config = {"name": "reviewer", "model": "a"}
|
|
soul = Mock(return_value="review carefully")
|
|
monkeypatch.setattr("deerflow.config.app_config.get_app_config", lambda: SimpleNamespace(model_dump=lambda **kw: app_config))
|
|
monkeypatch.setattr("deerflow.config.extensions_config.get_extensions_config", lambda: SimpleNamespace(model_dump=lambda **kw: {}))
|
|
monkeypatch.setattr("deerflow.config.agents_config.load_agent_config", lambda *a, **kw: SimpleNamespace(model_dump=lambda **kw: agent_config))
|
|
monkeypatch.setattr("deerflow.config.agents_config.load_agent_soul", soul)
|
|
runs = control.bind(request)
|
|
before = await runs.configuration_revision(assistant_ids=["reviewer", "lead_agent"])
|
|
assert len(before) == 64 and "secret" not in before
|
|
assert before == await runs.configuration_revision(assistant_ids=["lead_agent", "reviewer"])
|
|
soul.return_value = "changed instructions"
|
|
assert before != await runs.configuration_revision(assistant_ids=["reviewer", "lead_agent"])
|
|
soul.return_value = "review carefully"
|
|
agent_config["model"] = "b"
|
|
assert before != await runs.configuration_revision(assistant_ids=["reviewer", "lead_agent"])
|
|
load_user.return_value = None
|
|
with pytest.raises(AgentRunError, match="revoked"):
|
|
await runs.configuration_revision(assistant_ids=["lead_agent"])
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_actual_assembly_evidence_is_retained_and_bound(runtime):
|
|
runs, state, request, _, _ = runtime
|
|
thread = await runs.create_thread(thread_id="proof-thread")
|
|
run = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "proof"}]})
|
|
assert (await runs.wait(thread_id=thread, run_id=run.run_id, timeout=5)).status == "success"
|
|
evidence = await runs.configuration_evidence(thread_id=thread, run_id=run.run_id)
|
|
assert evidence["model"] == "fixture-model"
|
|
assert evidence["run_id"] == run.run_id
|
|
evidence["model"] = "tampered"
|
|
retained = await runs.configuration_evidence(thread_id=thread, run_id=run.run_id)
|
|
assert retained["model"] == "fixture-model"
|
|
with pytest.raises(ValueError, match="immutable"):
|
|
await state.run_manager.record_assembly_evidence(run.run_id, evidence)
|
|
other = await runs.create_thread(thread_id="other-proof-thread")
|
|
with pytest.raises(AgentRunError):
|
|
await runs.configuration_evidence(thread_id=other, run_id=run.run_id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_evidence_persistence_failure_prevents_execution(runtime, monkeypatch):
|
|
runs, state, _, _, handles = runtime
|
|
|
|
async def fail(*args, **kwargs):
|
|
raise RuntimeError("evidence persistence unavailable")
|
|
|
|
monkeypatch.setattr(state.run_manager, "record_assembly_evidence", fail)
|
|
thread = await runs.create_thread(thread_id="proof-failure")
|
|
run = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "must not run"}]})
|
|
assert (await runs.wait(thread_id=thread, run_id=run.run_id, timeout=5)).status == "error"
|
|
assert not handles
|
|
with pytest.raises(AgentRunError, match="unavailable"):
|
|
await runs.configuration_evidence(thread_id=thread, run_id=run.run_id)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_peer_takeover_during_assembly_is_not_undone_by_evidence(runtime, monkeypatch, tmp_path):
|
|
"""A peer reclaims the run while this worker assembles; evidence must not revive the row."""
|
|
import asyncio
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy import update
|
|
from test_run_repository import _cleanup, _make_repo
|
|
|
|
from deerflow.config.run_ownership_config import RunOwnershipConfig
|
|
from deerflow.persistence.run.model import RunRow
|
|
from deerflow.runtime import RunManager
|
|
from deerflow.runtime.assembly_evidence import EVIDENCE_KEY
|
|
|
|
runs, state, _, _, handles = runtime
|
|
repository = await _make_repo(tmp_path)
|
|
state.run_store = repository
|
|
await state.run_manager.shutdown()
|
|
ownership = RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=30, grace_seconds=1)
|
|
state.run_manager = RunManager(store=repository, run_ownership_config=ownership, worker_id="worker-a")
|
|
peer = RunManager(store=repository, run_ownership_config=ownership, worker_id="worker-b")
|
|
loop = asyncio.get_running_loop()
|
|
taken_over = []
|
|
|
|
async def peer_takeover():
|
|
# The peer sees an expired lease (clock skew or a stalled owner) that
|
|
# the owning worker still believes is live.
|
|
async with repository._sf() as session:
|
|
await session.execute(update(RunRow).where(RunRow.thread_id == thread).values(lease_expires_at=datetime.now(UTC) - timedelta(minutes=1)))
|
|
await session.commit()
|
|
taken_over.extend(await peer.reconcile_orphaned_inflight_runs(error="reclaimed by worker-b", stop_reason="orphan_recovered"))
|
|
|
|
import app.gateway.services as services
|
|
|
|
assemble = services.resolve_agent_factory("lead_agent")
|
|
|
|
def factory_with_takeover(*, config):
|
|
asyncio.run_coroutine_threadsafe(peer_takeover(), loop).result(timeout=5)
|
|
return assemble(config=config)
|
|
|
|
monkeypatch.setattr("app.gateway.services.resolve_agent_factory", lambda _: factory_with_takeover)
|
|
try:
|
|
thread = await runs.create_thread(thread_id="takeover-during-assembly")
|
|
run = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "must not run"}]})
|
|
record = await state.run_manager.get(run.run_id)
|
|
await asyncio.wait({record.task}, timeout=5)
|
|
assert [record.run_id for record in taken_over] == [run.run_id]
|
|
row = await repository.get(run.run_id, user_id=None)
|
|
assert row["status"] == "error"
|
|
assert row["error"] == "reclaimed by worker-b"
|
|
assert row["stop_reason"] == "orphan_recovered"
|
|
assert EVIDENCE_KEY not in row["metadata"]
|
|
assert record.ownership_lost
|
|
assert not handles
|
|
# The owner's renewal stays rejected, so the peer's outcome is final.
|
|
await state.run_manager._renew_leases()
|
|
assert (await repository.get(run.run_id, user_id=None))["status"] == "error"
|
|
finally:
|
|
await state.run_manager.shutdown()
|
|
await _cleanup()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_durable_configuration_revision_survives_host_replacement_and_rotation(host, monkeypatch):
|
|
control, request, load_user = host
|
|
monkeypatch.setattr("deerflow.config.app_config.get_app_config", lambda: SimpleNamespace(model_dump=lambda **kw: {"model": "test"}))
|
|
monkeypatch.setattr("deerflow.config.extensions_config.get_extensions_config", lambda: SimpleNamespace(model_dump=lambda **kw: {}))
|
|
secret = SimpleNamespace(jwt_secret="stable-test-signing-key")
|
|
monkeypatch.setattr("app.gateway.auth.config.get_auth_config", lambda: secret)
|
|
first = control.bind(request)
|
|
durable = await first.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
|
ephemeral = await first.configuration_revision(assistant_ids=["lead_agent"])
|
|
control.close()
|
|
second = GatewayAgentRunsHost(request.app, load_user=load_user).bind(request)
|
|
assert durable == await second.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
|
assert ephemeral != await second.configuration_revision(assistant_ids=["lead_agent"])
|
|
secret.jwt_secret = "rotated"
|
|
assert durable != await second.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
|
load_user.return_value = None
|
|
with pytest.raises(AgentRunError):
|
|
await second.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execution_fence_namespace_and_host_prerequisites(runtime):
|
|
from deerflow_extension_api import ExecutionFence
|
|
|
|
runs, state, request, _, _ = runtime
|
|
runs = runs.for_plugin("test.fenced")
|
|
thread_id = await runs.create_thread(thread_id="fenced-thread")
|
|
with pytest.raises(AgentRunError, match="belong to this plugin"):
|
|
await runs.start(thread_id=thread_id, input={}, execution_fence=ExecutionFence("other", "workflow", "owner", 1))
|
|
with pytest.raises(AgentRunError, match="heartbeats"):
|
|
await runs.start(thread_id=thread_id, input={}, execution_fence=ExecutionFence("test.fenced", "workflow", "owner", 1))
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_host_forwards_fence_to_atomic_admission(runtime, monkeypatch, tmp_path):
|
|
from deerflow_extension_api import ExecutionFence
|
|
from sqlalchemy import insert
|
|
from test_run_repository import _cleanup, _make_repo
|
|
|
|
from deerflow.config.run_ownership_config import RunOwnershipConfig
|
|
from deerflow.persistence.extension_leases.model import ExtensionExecutionLeaseRow
|
|
from deerflow.runtime import RunManager
|
|
|
|
runs, state, request, _, _ = runtime
|
|
repository = await _make_repo(tmp_path)
|
|
state.run_store = repository
|
|
await state.run_manager.shutdown()
|
|
state.run_manager = RunManager(store=repository, run_ownership_config=RunOwnershipConfig(heartbeat_enabled=True))
|
|
# Transport is irrelevant to this admission test; advertise its prerequisite
|
|
# without starting Redis. The actual host/manager/store admission path is real.
|
|
monkeypatch.setattr(type(state.stream_bridge), "supports_cross_process", True, raising=False)
|
|
monkeypatch.setattr(type(state.stream_bridge), "start_producer_lease", AsyncMock(return_value=True), raising=False)
|
|
runs = runs.for_plugin("test.fenced")
|
|
thread_id = await runs.create_thread(thread_id="fenced-host-thread")
|
|
async with repository._sf() as session:
|
|
await session.execute(insert(ExtensionExecutionLeaseRow).values(namespace="test.fenced", resource_id="workflow", user_id=str(request.state.user.id), holder_id="new", epoch=2, expires_at=9999999999))
|
|
await session.commit()
|
|
try:
|
|
with pytest.raises(AgentRunError) as failure:
|
|
await runs.start(thread_id=thread_id, input={"messages": [{"role": "user", "content": "must not execute"}]}, execution_fence=ExecutionFence("test.fenced", "workflow", "old", 1))
|
|
assert failure.value.status_code == 409
|
|
assert "execution fence" in str(failure.value)
|
|
finally:
|
|
await state.run_manager.shutdown()
|
|
await _cleanup()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_configuration_revision_canonicalizes_sets_but_preserves_list_order(host, monkeypatch):
|
|
class OrderedSet(set):
|
|
def __init__(self, values):
|
|
super().__init__(values)
|
|
self.values = values
|
|
|
|
def __iter__(self):
|
|
return iter(self.values)
|
|
|
|
control, request, _ = host
|
|
data = {"tools": ["first", "second"], "exempt_tools": OrderedSet(["task", "present_files"])}
|
|
|
|
# Pydantic JSON mode loses the distinction between sets and ordered lists.
|
|
def dump(**kwargs):
|
|
return {**data, "exempt_tools": list(data["exempt_tools"])} if kwargs.get("mode") == "json" else data
|
|
|
|
monkeypatch.setattr("deerflow.config.app_config.get_app_config", lambda: SimpleNamespace(model_dump=dump))
|
|
monkeypatch.setattr("deerflow.config.extensions_config.get_extensions_config", lambda: SimpleNamespace(model_dump=lambda **kw: {}))
|
|
runs = control.bind(request)
|
|
before = await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
|
data["exempt_tools"] = OrderedSet(["present_files", "task"])
|
|
assert before == await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|
|
data["tools"].reverse()
|
|
assert before != await runs.configuration_revision(assistant_ids=["lead_agent"], durable=True)
|