fix(subagents): revalidate durable batch owners before execution (#8272)

* fix(subagents): revalidate durable batch owners before execution

* fix(subagents): log batch owner lookup failures safely

---------

Co-authored-by: Totoro-qaq <279883115+Totoro-qaq@users.noreply.github.com>
This commit is contained in:
Totoro 2026-11-16 15:59:09 +08:00 committed by GitHub
parent 54dc5024a6
commit 4b98e46388
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 702 additions and 8 deletions

View File

@ -926,6 +926,7 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
raise RuntimeError("subagent_batches.enabled requires database.backend sqlite or postgres")
if batch_repo is not None:
from app.subagent_batches.continuation import BatchContinuations
from app.subagent_batches.identity import resolve_batch_principal
continuations = BatchContinuations(app)
app.state.batch_continuations = continuations
@ -937,6 +938,7 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
prepare_continuation=continuations.prepare,
dispatch_continuation=continuations.dispatch,
cancel_continuation=continuations.cancel,
resolve_principal=lambda batch: resolve_batch_principal(batch, load_user=app.state.agent_runs_host.load_user),
)
app.state.subagent_batch_service = batch_service
if subagent_batches_config.enabled:

View File

@ -0,0 +1,47 @@
"""Resolve durable item owners through the Gateway's current account store."""
import logging
from app.gateway.auth_disabled import AUTH_DISABLED_USER_ID, get_auth_disabled_user, is_auth_disabled
from deerflow.authz.provider import Principal
logger = logging.getLogger(__name__)
async def resolve_batch_principal(batch, *, load_user) -> Principal:
owner = batch["user_id"]
spec = batch["execution_spec"]
internal = spec.get("is_internal") is True
if owner == AUTH_DISABLED_USER_ID and is_auth_disabled():
user = get_auth_disabled_user()
else:
try:
if load_user is None:
raise RuntimeError("Account repository unavailable")
user = await load_user(owner)
except Exception as exc:
# Database exception text can contain SQL parameters or credentials.
logger.warning("Batch owner lookup failed (batch_id=%s, error_type=%s)", batch.get("id"), type(exc).__name__)
raise PermissionError("Batch owner could not be verified") from None
if user is None:
# Gateway stamps no role on an unmapped, trusted internal identity.
# A missing formerly mapped account must not take this fallback.
if internal and spec.get("user_role") is None:
return Principal(user_id=owner, is_internal=True, channel_user_id=spec.get("channel_user_id"))
raise PermissionError("Batch owner is no longer available")
if str(user.id) != owner or getattr(user, "disabled", False) or user.needs_setup:
raise PermissionError("Batch owner is no longer active")
if user.system_role != spec.get("user_role"):
# Custom roles are not ordered. Refuse changed grants rather than
# retaining a revoked role or widening old work after a promotion.
raise PermissionError("Batch owner role changed; submit a new batch with current permissions")
return Principal(
user_id=owner,
role=user.system_role,
oauth_provider=user.oauth_provider,
oauth_id=getattr(user, "oauth_id", None),
channel_user_id=spec.get("channel_user_id"),
is_internal=internal,
# Gateway admission does not mint ABAC attributes. Stored attributes
# and OAuth claims are not a renewable source of authority.
)

View File

@ -39,6 +39,20 @@ suppress pending work; a failed, cancelled or hard-stopped source Run cannot res
## Authority, cancellation and recovery
Each Gateway batch item rechecks its owner before tool discovery and again after
acquiring native execution capacity. This applies to the first attempt and to
recovery after a restart. Missing, disabled or reset accounts cannot start work.
A changed role rejects the old item; submit a new batch with current permissions.
Custom roles have no general ordering, so this rule prevents both stale privileges
after demotion and additional privileges after promotion. OAuth identity comes from
the current account; serialized authorization attributes are not restored.
Explicit local auth-disabled mode and unmapped trusted internal owners retain their
existing identity semantics. A deleted local account cannot become an unmapped
internal owner. These checks govern admission, not cancellation of already admitted
work. Standalone harness hosts with mutable accounts can supply the same
`resolve_principal` callback without importing the Gateway.
Follow-up needs a session-authorized Gateway AgentRuns capability or local auth-disabled
mode. Internal/PAT callers and unsupported embedded runtimes retain ordinary batches.
Persist only the original permission ceiling, account token version and auth source;

View File

@ -4,10 +4,12 @@ import asyncio
import logging
import socket
import uuid
from collections.abc import Awaitable, Callable
from concurrent.futures import Future
from datetime import UTC, datetime
from typing import Any
from deerflow.authz.provider import Principal
from deerflow.community.ragflow.sources import durable_source_artifact
from deerflow.config.app_config import AppConfig, get_app_config
from deerflow.config.subagent_batches_config import SubagentBatchesConfig
@ -160,6 +162,7 @@ class SubagentBatchService:
prepare_continuation=None,
dispatch_continuation=None,
cancel_continuation=None,
resolve_principal: Callable[[dict[str, Any]], Awaitable[Principal]] | None = None,
) -> None:
self._repository = repository
self._config = config
@ -181,6 +184,7 @@ class SubagentBatchService:
self._prepare_continuation = prepare_continuation
self._dispatch_continuation = dispatch_continuation
self._cancel_continuation = cancel_continuation
self._resolve_principal = resolve_principal
self._continuations: dict[str, asyncio.Task] = {}
async def start(self) -> None:
@ -468,6 +472,32 @@ class SubagentBatchService:
if lease.valid:
await self._execute_leased_item(item, lease)
def _bind_principal_resolver(self, batch: dict[str, Any]) -> Callable[[], Awaitable[Principal]] | None:
if self._resolve_principal is None:
return None
owner_loop = asyncio.get_running_loop()
async def resolve() -> Principal:
principal = await self._resolve_principal(batch)
if not isinstance(principal, Principal) or principal.user_id != batch["user_id"]:
raise PermissionError("Batch principal does not match its owner")
return principal
async def resolve_on_owner_loop() -> Principal:
# Native executors use a separate loop; the host's async database
# pool must stay on the Gateway loop, including after capacity waits.
if asyncio.get_running_loop() is owner_loop:
return await resolve()
coro = resolve()
try:
pending = asyncio.run_coroutine_threadsafe(coro, owner_loop)
except BaseException:
coro.close()
raise
return await asyncio.wrap_future(pending)
return resolve_on_owner_loop
async def _execute_leased_item(self, item: dict[str, Any], lease: BatchItemLease) -> None:
item_id = item["id"]
execution_id: str | None = None
@ -475,13 +505,15 @@ class SubagentBatchService:
batch = item["batch"]
self._item_batches[item_id] = batch["id"]
spec = batch["execution_spec"]
principal_resolver = self._bind_principal_resolver(batch)
principal = await principal_resolver() if principal_resolver is not None else None
config_data = spec["subagent_config"]
if "prompt_overlay" in config_data:
from deerflow.config.prompt_overlay import PromptOverlay
config_data = {**config_data, "prompt_overlay": PromptOverlay.model_validate(config_data["prompt_overlay"])}
config = SubagentConfig(**config_data)
app_config = self._app_config or get_app_config()
app_config = self._app_config or await run_assembly(get_app_config)
from deerflow.tools import get_available_tools
effective_model = resolve_subagent_model_name(
@ -513,6 +545,8 @@ class SubagentBatchService:
)
return
executor_kwargs = {}
if principal_resolver is not None:
executor_kwargs["principal_resolver"] = principal_resolver
if "mcp_agent_auth" in spec:
from deerflow.mcp.agent_auth import restore_agent_authority
@ -526,13 +560,13 @@ class SubagentBatchService:
parent_model=spec.get("parent_model"),
thread_id=batch["thread_id"],
user_id=batch["user_id"],
user_role=spec.get("user_role"),
oauth_provider=spec.get("oauth_provider"),
oauth_id=spec.get("oauth_id"),
user_role=principal.role if principal is not None else spec.get("user_role"),
oauth_provider=principal.oauth_provider if principal is not None else spec.get("oauth_provider"),
oauth_id=principal.oauth_id if principal is not None else spec.get("oauth_id"),
run_id=batch.get("run_id"),
channel_user_id=spec.get("channel_user_id"),
is_internal=spec.get("is_internal") is True,
authz_attributes=spec.get("authz_attributes"),
channel_user_id=principal.channel_user_id if principal is not None else spec.get("channel_user_id"),
is_internal=principal.is_internal if principal is not None else spec.get("is_internal") is True,
authz_attributes=principal.attributes if principal is not None else spec.get("authz_attributes"),
knowledge_scope=spec.get("knowledge_scope"),
execution_capacity=self._execution_capacity,
extensions=self._extensions,

View File

@ -8,7 +8,7 @@ import os
import sys
import threading
import uuid
from collections.abc import Callable, Coroutine, Mapping
from collections.abc import Awaitable, Callable, Coroutine, Mapping
from concurrent.futures import Future
from concurrent.futures import TimeoutError as FuturesTimeoutError
from contextvars import Context, copy_context
@ -33,6 +33,7 @@ from deerflow.agents.middlewares.audit_context import (
)
from deerflow.agents.thread_state import SandboxState, ThreadDataState, ThreadState
from deerflow.authz.principal import normalize_authz_attributes
from deerflow.authz.provider import Principal
from deerflow.config import get_app_config
from deerflow.config.app_config import AppConfig
from deerflow.knowledge_scope import KNOWLEDGE_SCOPE_RUNTIME_KEY, execution_scope
@ -983,6 +984,7 @@ class SubagentExecutor:
thread_incarnation: str | None | object = _THREAD_INCARNATION_UNSET,
input_gate: Any | None = None,
agent_mcp_authority: Any | None = None,
principal_resolver: Callable[[], Awaitable[Principal]] | None = None,
):
"""Initialize the executor.
@ -1071,6 +1073,7 @@ class SubagentExecutor:
self.user_id = user_id
# Guardrail attribution propagated from the parent runtime context.
self.user_role = user_role
self._principal_resolver = principal_resolver
self.oauth_provider = oauth_provider
self.oauth_id = oauth_id
self.run_id = run_id
@ -1782,6 +1785,24 @@ class SubagentExecutor:
subagent_checkpoint_scope(execution_id=result.task_id, run_id=self.run_id, tool_call_id=result.external_task_id, agent_name=self.config.name),
slot,
):
if self._principal_resolver is not None:
# Durable work can wait behind ordinary subagents after
# its worker-side check. Revalidate before marking it
# running or entering any task lifecycle hooks.
try:
principal = await self._principal_resolver()
if not isinstance(principal, Principal) or principal.user_id != self.user_id:
raise PermissionError("Subagent principal does not match its owner")
self.user_role = principal.role
self.oauth_provider = principal.oauth_provider
self.oauth_id = principal.oauth_id
self.channel_user_id = principal.channel_user_id
self.is_internal = principal.is_internal
self.authz_attributes = normalize_authz_attributes(principal.attributes)
self._skill_authorization = None
except Exception as exc:
result.try_set_terminal(SubagentStatus.FAILED, error=str(exc))
return result
with result._state_lock:
if result.status.is_terminal:
return result

View File

@ -0,0 +1,36 @@
"""Dispatch revalidation reads the live account without blocking the Gateway."""
import pytest
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.auth.models import User
from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
from app.subagent_batches.identity import resolve_batch_principal
from deerflow.config.subagent_batches_config import SubagentBatchesConfig
from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig
from deerflow.persistence.user.model import UserRow
from deerflow.subagents.batch_service import SubagentBatchService
@pytest.mark.asyncio
async def test_batch_owner_lookup_does_not_block_the_host_loop(tmp_path, monkeypatch):
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0")
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/users.db")
try:
async with engine.begin() as connection:
await connection.run_sync(UserRow.__table__.create)
repo = SQLiteUserRepository(async_sessionmaker(engine, expire_on_commit=False))
user = await repo.create_user(User(email="owner@example.com"))
service = SubagentBatchService(
repository=None,
config=SubagentBatchesConfig(),
runtime_config=SubagentRuntimeConfig(),
resolve_principal=lambda batch: resolve_batch_principal(batch, load_user=repo.get_user_by_id),
)
resolver = service._bind_principal_resolver({"user_id": str(user.id), "execution_spec": {"user_role": "user"}})
assert (await resolver()).role == "user"
await repo.set_disabled(str(user.id), True)
with pytest.raises(PermissionError):
await resolver()
finally:
await engine.dispose()

View File

@ -0,0 +1,67 @@
"""Model/tool boundaries for the real Gateway batch-owner recovery test."""
import json
import os
import time
from pathlib import Path
from langchain.tools import ToolRuntime, tool
from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
def _record(name, item, runtime):
record = {"tool": name, "item": item, "role": runtime.context.get("user_role"), "user_id": runtime.context.get("user_id"), "pid": os.getpid()}
path = Path(os.environ["BATCH_OWNER_PROBE_DIR"]) / "calls.jsonl"
with path.open("a", encoding="utf-8") as log:
log.write(json.dumps(record) + "\n")
return "Completed " + name
@tool
def owner_normal(item: str, runtime: ToolRuntime) -> str:
"""Record an operation available to every configured role."""
return _record("normal", item, runtime)
@tool
def owner_admin(item: str, runtime: ToolRuntime) -> str:
"""Record an operation restricted by the host RBAC policy to administrators."""
return _record("admin", item, runtime)
class OwnerModel(BaseChatModel):
@property
def _llm_type(self):
return "offline-batch-owner"
def bind_tools(self, tools, **kwargs):
return self
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
if any(isinstance(message, ToolMessage) for message in messages):
answer = AIMessage(content="Done")
elif any(isinstance(message, HumanMessage) and "OWNER_CHILD" in str(message.content) for message in messages):
root = Path(os.environ["BATCH_OWNER_PROBE_DIR"])
item = "first" if any("OWNER_CHILD:first" in str(message.content) for message in messages) else "second"
(root / f"reached-{item}").write_text(str(os.getpid()), encoding="utf-8")
deadline = time.monotonic() + 60
while item == "first" and (root / "hold").exists():
if time.monotonic() >= deadline:
raise TimeoutError("Batch owner test barrier not released")
time.sleep(0.02)
answer = AIMessage(content="", tool_calls=[{"name": name, "args": {"item": item}, "id": name, "type": "tool_call"} for name in ("owner_admin", "owner_normal")])
else:
answer = AIMessage(
content="",
tool_calls=[
{
"name": "batch_task",
"args": {"title": "Owner recovery", "subagent_type": "general-purpose", "items": [{"key": key, "prompt": f"OWNER_CHILD:{key}: run both configured observations"} for key in ("first", "second")]},
"id": "batch",
"type": "tool_call",
}
],
)
return ChatResult(generations=[ChatGeneration(message=answer)])

View File

@ -0,0 +1,123 @@
"""Native capacity waits revalidate identity on the host's database loop."""
import asyncio
import importlib.util
import json
import sys
from pathlib import Path
import pytest
from support.batch_owner import owner_admin, owner_normal
from deerflow.authz.provider import Principal
from deerflow.config.app_config import AppConfig
from deerflow.config.subagent_batches_config import SubagentBatchesConfig
from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig
from deerflow.subagents.batch_service import SubagentBatchService
from deerflow.subagents.capacity import SubagentExecutionCapacity
from deerflow.subagents.config import SubagentConfig
@pytest.fixture
def executor_module(monkeypatch, tmp_path):
# conftest installs an executor stub; load the real graph and worker loop.
path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/executor.py"
spec = importlib.util.spec_from_file_location("_batch_owner_executor", path)
module = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, spec.name, module)
spec.loader.exec_module(module)
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "home"))
monkeypatch.setenv("BATCH_OWNER_PROBE_DIR", str(tmp_path))
yield module
module._shutdown_isolated_subagent_loop()
def _executor(module, resolver, capacity=None):
config = AppConfig.model_validate(
{
"models": [{"name": "owner-probe", "use": "support.batch_owner:OwnerModel", "model": "offline"}],
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
"summarization": {"enabled": False},
"tool_search": {"enabled": False},
"authorization": {"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"admin": {}, "user": {"tools": {"deny": ["owner_admin"]}}}}}},
}
)
return module.SubagentExecutor(
config=SubagentConfig(name="general-purpose", description="Owner probe", skills=[]),
tools=[owner_admin, owner_normal],
app_config=config,
parent_model="owner-probe",
thread_id="batch-owner-test",
user_id="owner",
user_role="admin",
oauth_provider="old",
oauth_id="old",
authz_attributes={"old_grant": True},
execution_capacity=capacity,
principal_resolver=resolver,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["disabled", "different_owner", "fresh_profile", "unchanged"])
async def test_identity_is_rechecked_after_capacity_wait(executor_module, tmp_path, change):
capacity = SubagentExecutionCapacity(SubagentRuntimeConfig(max_running=1))
state = "queued"
calls = []
async def resolve():
calls.append(state)
if state == "disabled":
raise PermissionError("Owner disabled")
return Principal(user_id="different" if state == "different_owner" else "owner", role="user" if state == "fresh_profile" else "admin", oauth_provider="oidc", oauth_id="current")
executor = _executor(executor_module, resolve, capacity)
task = None
try:
async with capacity.slot():
task = asyncio.create_task(executor._aexecute("OWNER_CHILD:second"))
async with asyncio.timeout(5):
while capacity.snapshot().queued != 1:
await asyncio.sleep(0)
assert not calls and not (tmp_path / "reached-second").exists()
state = change
result = await asyncio.wait_for(task, 20)
assert calls == [change]
assert result.execution_done_event.is_set()
if change in ("disabled", "different_owner"):
assert result.status == executor_module.SubagentStatus.FAILED
assert result.started_at is None
assert not (tmp_path / "calls.jsonl").exists()
else:
assert result.status == executor_module.SubagentStatus.COMPLETED, result.error
observed = [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text(encoding="utf-8").splitlines()]
assert {call["tool"] for call in observed} == ({"normal"} if change == "fresh_profile" else {"normal", "admin"})
assert executor.oauth_provider == "oidc" and executor.oauth_id == "current"
assert executor.authz_attributes == {}
assert capacity.snapshot().running == capacity.snapshot().queued == 0
finally:
if task is not None and not task.done():
task.cancel()
await asyncio.gather(task, return_exceptions=True)
@pytest.mark.asyncio
async def test_native_worker_resolves_on_gateway_loop(executor_module, tmp_path):
host_loop = asyncio.get_running_loop()
lookups = []
async def resolve(batch):
assert asyncio.get_running_loop() is host_loop
lookups.append(batch["user_id"])
return Principal(user_id=batch["user_id"], role="user")
service = SubagentBatchService(repository=None, config=SubagentBatchesConfig(), runtime_config=SubagentRuntimeConfig(), resolve_principal=resolve)
resolver = service._bind_principal_resolver({"user_id": "owner"})
await resolver() # Pre-discovery check on the host loop.
executor = _executor(executor_module, resolver)
pending = executor_module.run_on_isolated_subagent_loop(executor._aexecute("OWNER_CHILD:second"))
result = await asyncio.wait_for(asyncio.wrap_future(pending), 20)
assert result.status == executor_module.SubagentStatus.COMPLETED, result.error
assert lookups == ["owner", "owner"]
observed = [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text(encoding="utf-8").splitlines()]
assert {call["tool"] for call in observed} == {"normal"}

View File

@ -0,0 +1,131 @@
"""Gateway batch identity resolves against the current account repository."""
from traceback import format_exception
import pytest
import pytest_asyncio
from sqlalchemy import delete
from sqlalchemy.exc import OperationalError
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.auth.models import User
from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
from app.subagent_batches.identity import resolve_batch_principal
from deerflow.persistence.user.model import UserRow
@pytest_asyncio.fixture
async def accounts(tmp_path, monkeypatch):
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/users.db")
async with engine.begin() as connection:
await connection.run_sync(UserRow.__table__.create)
sessions = async_sessionmaker(engine, expire_on_commit=False)
repo = SQLiteUserRepository(sessions)
owner = await repo.create_user(User(email="owner@example.com", system_role="admin", oauth_provider="oidc", oauth_id="current"))
await repo.create_user(User(email="root@example.com", system_role="admin"))
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0")
try:
yield repo, owner, sessions
finally:
await engine.dispose()
def _batch(owner, **spec):
return {"id": "batch-1", "user_id": owner, "execution_spec": spec}
@pytest.mark.asyncio
async def test_current_profile_replaces_stored_oauth_and_attributes(accounts):
repo, owner, _ = accounts
principal = await resolve_batch_principal(_batch(str(owner.id), user_role="admin", oauth_provider="old", oauth_id="old", authz_attributes={"admin": True}, is_internal=True, channel_user_id="im-user"), load_user=repo.get_user_by_id)
assert (principal.user_id, principal.role, principal.oauth_provider, principal.oauth_id) == (str(owner.id), "admin", "oidc", "current")
assert principal.is_internal and principal.channel_user_id == "im-user"
assert principal.attributes == {}
@pytest.mark.asyncio
@pytest.mark.parametrize("change", ["disabled", "demoted", "reset", "deleted"])
@pytest.mark.parametrize("internal", [False, True])
async def test_revoked_local_owner_cannot_fall_back_to_internal(accounts, change, internal):
repo, owner, sessions = accounts
batch = _batch(str(owner.id), user_role="admin", is_internal=internal)
if change == "disabled":
await repo.set_disabled(str(owner.id), True)
elif change == "demoted":
await repo.update_system_role(str(owner.id), "user")
elif change == "reset":
await repo.update_user(owner.model_copy(update={"needs_setup": True}))
else:
async with sessions() as session:
await session.execute(delete(UserRow).where(UserRow.id == str(owner.id)))
await session.commit()
with pytest.raises(PermissionError):
await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
@pytest.mark.asyncio
@pytest.mark.parametrize("original,current", [("user", "admin"), ("reviewer", "operator"), (None, "admin")])
async def test_role_changes_do_not_invent_a_role_hierarchy(accounts, original, current):
repo, owner, _ = accounts
await repo.update_system_role(str(owner.id), current)
with pytest.raises(PermissionError):
await resolve_batch_principal(_batch(str(owner.id), user_role=original), load_user=repo.get_user_by_id)
@pytest.mark.asyncio
async def test_unmapped_internal_owner_keeps_default_role_without_stale_claims(accounts):
repo, _, _ = accounts
batch = _batch("external-owner", is_internal=True, oauth_provider="old", oauth_id="old", authz_attributes={"admin": True})
principal = await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
assert principal.user_id == "external-owner" and principal.is_internal
assert principal.role is None and principal.oauth_provider is None and principal.oauth_id is None
assert principal.attributes == {}
batch["execution_spec"]["is_internal"] = False
with pytest.raises(PermissionError):
await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
@pytest.mark.asyncio
async def test_auth_disabled_only_preserves_explicit_synthetic_owner(accounts, monkeypatch):
repo, owner, _ = accounts
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
monkeypatch.delenv("DEER_FLOW_ENV", raising=False)
monkeypatch.delenv("ENVIRONMENT", raising=False)
batch = _batch("default", user_role="admin")
principal = await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
assert principal.user_id == "default" and principal.role == "admin"
await repo.set_disabled(str(owner.id), True)
with pytest.raises(PermissionError):
await resolve_batch_principal(_batch(str(owner.id), user_role="admin"), load_user=repo.get_user_by_id)
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0")
with pytest.raises(PermissionError):
await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["missing_loader", "exception", "mismatch"])
async def test_unavailable_identity_fails_closed(accounts, failure, caplog):
_, owner, _ = accounts
batch = _batch(str(owner.id), user_role="admin", is_internal=True)
fake_password = "fixture-db-password"
statement = "SELECT id FROM users WHERE id = :id"
async def load_user(user_id):
if failure == "exception":
raise OperationalError(statement, {"id": user_id}, RuntimeError(f"postgresql://test:{fake_password}@test-db/deerflow"))
return User(email="different@example.com", system_role="admin")
with pytest.raises(PermissionError) as exc:
await resolve_batch_principal(batch, load_user=None if failure == "missing_loader" else load_user)
assert fake_password not in str(exc.value)
if failure == "exception":
records = [record for record in caplog.records if record.name == "app.subagent_batches.identity"]
assert len(records) == 1
record = records[0]
assert batch["id"] in record.getMessage()
assert "OperationalError" in record.getMessage()
assert record.exc_info is None
diagnostics = caplog.text + "".join(format_exception(exc.value))
assert fake_password not in diagnostics
assert str(owner.id) not in diagnostics
assert statement not in diagnostics

View File

@ -0,0 +1,219 @@
"""Queued and recovered batches must honor current account state and role."""
import json
import os
import signal
import socket
import subprocess
import sys
import time
import uuid
from contextlib import contextmanager
from pathlib import Path
import httpx
import pytest
from sqlalchemy import create_engine, text
from sqlalchemy.engine import make_url
from sqlalchemy.schema import CreateSchema, DropSchema
from deerflow.persistence.postgres_schema import dsn_with_search_path
ROOT = Path(__file__).resolve().parents[2]
pytestmark = pytest.mark.skipif(os.name != "posix", reason="Tests SIGKILL recovery in separate Gateway processes")
def _until(check, timeout=40):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = check()
if result:
return result
time.sleep(0.05)
raise AssertionError(f"Timed out waiting for {check}")
@pytest.fixture(params=["sqlite", "postgres"])
def database(request, tmp_path):
schema = None
if request.param == "postgres":
uri = os.environ.get("TEST_POSTGRES_URI")
if not uri:
pytest.skip("requires TEST_POSTGRES_URI (real Gateway restart)")
pytest.importorskip("psycopg")
pytest.importorskip("asyncpg")
schema = "test_batch_owner_" + uuid.uuid4().hex
config = {"backend": "postgres", "postgres_url": uri, "postgres_schema": schema}
engine = create_engine(make_url(dsn_with_search_path(uri, schema)).set(drivername="postgresql+psycopg"))
else:
config = {"backend": "sqlite", "sqlite_dir": str(tmp_path / "db")}
engine = create_engine(f"sqlite:///{tmp_path}/db/deerflow.db")
def read(statement, **params):
with engine.connect() as connection:
return connection.execute(text(statement), params).all()
try:
if schema:
with engine.begin() as connection:
connection.execute(CreateSchema(schema))
yield config, read
finally:
try:
if schema:
with engine.begin() as connection:
connection.execute(DropSchema(schema, cascade=True, if_exists=True))
finally:
engine.dispose()
@contextmanager
def _gateway(root, client, port, generation):
env = {key: os.environ[key] for key in ("PATH", "HOME", "TMPDIR", "LANG") if key in os.environ}
env.update(
PYTHONPATH=os.pathsep.join(str(ROOT / path) for path in ("backend/packages/harness", "backend/packages/extension-api", "backend", "backend/tests")),
DEER_FLOW_HOME=str(root / "home"),
DEER_FLOW_PROJECT_ROOT=str(ROOT),
DEER_FLOW_CONFIG_PATH=str(root / "config.yaml"),
DEER_FLOW_EXTENSIONS_CONFIG_PATH=str(root / "extensions.json"),
DEER_FLOW_ENV_FILE=str(root / "empty.env"),
BATCH_OWNER_PROBE_DIR=str(root),
AUTH_JWT_SECRET="local-batch-owner-test-secret-1234567890",
DEER_FLOW_AUTH_DISABLED="0",
NO_PROXY="127.0.0.1,localhost",
GATEWAY_WORKERS="1",
WEB_CONCURRENCY="1",
PYTHONIOENCODING="utf-8",
)
log_path = root / f"gateway-{generation}.log"
with log_path.open("w", encoding="utf-8") as log:
process = subprocess.Popen([sys.executable, "-m", "uvicorn", "app.gateway.app:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=root, env=env, stdout=log, stderr=subprocess.STDOUT)
try:
def healthy():
assert process.poll() is None, log_path.read_text(encoding="utf-8")[-8000:]
try:
return client.get("/health").status_code == 200
except httpx.TransportError:
return False
_until(healthy)
yield process
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=10)
@pytest.mark.parametrize("dispatch", ["batch_queue", "restart"])
@pytest.mark.parametrize("change", ["disabled", "demoted", "promoted", "unchanged_admin", "unchanged_user"])
def test_batch_revalidates_owner_at_execution(tmp_path, database, dispatch, change):
database_config, read = database
config = {
"models": [{"name": "owner-probe", "use": "support.batch_owner:OwnerModel", "model": "offline", "supports_tool_calling": True}],
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
"database": database_config,
"tools": [{"name": name, "group": "probe", "use": "support.batch_owner:" + name} for name in ("owner_normal", "owner_admin")],
"tool_groups": [{"name": "probe"}],
"title": {"enabled": False},
"memory": {"enabled": False},
"summarization": {"enabled": False},
"tool_search": {"enabled": False},
"hitl_approval": {"enabled": False},
"subagent_batches": {"enabled": True, "poll_interval_seconds": 0.1, "lease_seconds": 10, "default_max_running_items": 1, "max_attempts": 3 if dispatch == "restart" else 1},
"subagent_runtime": {"max_running": 1},
"authorization": {"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"admin": {}, "user": {"tools": {"deny": ["owner_admin"]}}}}}},
}
(tmp_path / "config.yaml").write_text(json.dumps(config), encoding="utf-8")
(tmp_path / "extensions.json").write_text("{}", encoding="utf-8")
(tmp_path / "empty.env").touch()
(tmp_path / "hold").touch()
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
with httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=30) as admin, httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=30) as actor:
def request(client, method, path, *, expected=200, **kwargs):
response = client.request(method, path, **kwargs)
assert response.status_code == expected, response.text
return response.json()
with _gateway(tmp_path, admin, port, 1) as first:
for client, route, email in ((admin, "initialize", "admin@example.com"), (actor, "register", "actor@example.com")):
account = request(client, "POST", "/api/v1/auth/" + route, expected=201, json={"email": email, "password": "local-batch-owner-password-123"})
client.headers["X-CSRF-Token"] = client.cookies["csrf_token"]
owner = account["id"]
original_role = "user" if change in ("unchanged_user", "promoted") else "admin"
if original_role == "admin":
request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"system_role": "admin"})
thread = request(actor, "POST", "/api/threads", json={})["thread_id"]
request(
actor,
"POST",
f"/api/threads/{thread}/runs/wait",
json={"assistant_id": "lead_agent", "input": {"messages": [{"role": "user", "content": "Submit the owner recovery batch"}]}, "context": {"mode": "pro", "subagent_enabled": True}},
)
_until(lambda: (tmp_path / "reached-first").exists())
assert not (tmp_path / "reached-second").exists()
assert not (tmp_path / "calls.jsonl").exists()
batch_id, stored_spec = read("SELECT id, execution_spec FROM subagent_batches WHERE user_id = :owner", owner=owner)[0]
spec = json.loads(stored_spec) if isinstance(stored_spec, str) else stored_spec
assert spec["user_role"] == original_role
def items():
return read("SELECT item_key, status, attempt, error FROM subagent_batch_items WHERE batch_id = :batch_id ORDER BY position", batch_id=batch_id)
_until(lambda: len(rows := items()) == 2 and rows[1][2] == 0)
if change == "disabled":
request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"disabled": True})
request(actor, "GET", "/api/v1/auth/me", expected=401)
elif change in ("demoted", "promoted"):
role = "user" if change == "demoted" else "admin"
request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"system_role": role})
assert request(actor, "GET", "/api/v1/auth/me")["system_role"] == role
def verify(pid):
def terminal():
rows = items()
return rows if len(rows) == 2 and all(row[1] in ("succeeded", "failed", "cancelled") for row in rows) else None
rows = _until(terminal)
path = tmp_path / "calls.jsonl"
calls = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
# Preserve observable results before the PostgreSQL schema is
# dropped; never manufacture task state for recovery.
(tmp_path / "evidence.json").write_text(
json.dumps(
{"backend": database_config["backend"], "change": change, "dispatch": dispatch, "old_pid": first.pid, "new_pid": pid, "old_exit": first.returncode, "items": [dict(row._mapping) for row in rows], "calls": calls},
indent=2,
),
encoding="utf-8",
)
assert all(call["pid"] == pid and call["user_id"] == owner for call in calls)
if dispatch == "restart":
assert rows[0][2] >= 2
else:
assert rows[1][2] == 1
assert rows[0][1] == "succeeded" # Already admitted, outside the revocation gate.
observed = calls if dispatch == "restart" else [call for call in calls if call["item"] == "second"]
if change in ("disabled", "demoted", "promoted"):
assert rows[1][1] == "failed", (rows, calls)
assert not observed
assert not (tmp_path / "reached-second").exists()
else:
assert all(row[1] == "succeeded" for row in rows), (rows, calls)
assert all(call["role"] == original_role for call in observed)
assert {call["tool"] for call in observed} == ({"admin", "normal"} if original_role == "admin" else {"normal"})
if dispatch != "restart":
(tmp_path / "hold").unlink()
verify(first.pid)
return
first.kill()
first.wait(timeout=10)
assert first.returncode == -signal.SIGKILL
(tmp_path / "hold").unlink()
with _gateway(tmp_path, admin, port, 2) as second:
assert second.pid != first.pid
verify(second.pid)