mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 08:20:34 +00:00
fix(subagents): revalidate durable batch owners before execution (#8272)
* fix(subagents): revalidate durable batch owners before execution * fix(subagents): log batch owner lookup failures safely --------- Co-authored-by: Totoro-qaq <279883115+Totoro-qaq@users.noreply.github.com>
This commit is contained in:
parent
54dc5024a6
commit
4b98e46388
@ -926,6 +926,7 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
raise RuntimeError("subagent_batches.enabled requires database.backend sqlite or postgres")
|
||||
if batch_repo is not None:
|
||||
from app.subagent_batches.continuation import BatchContinuations
|
||||
from app.subagent_batches.identity import resolve_batch_principal
|
||||
|
||||
continuations = BatchContinuations(app)
|
||||
app.state.batch_continuations = continuations
|
||||
@ -937,6 +938,7 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
prepare_continuation=continuations.prepare,
|
||||
dispatch_continuation=continuations.dispatch,
|
||||
cancel_continuation=continuations.cancel,
|
||||
resolve_principal=lambda batch: resolve_batch_principal(batch, load_user=app.state.agent_runs_host.load_user),
|
||||
)
|
||||
app.state.subagent_batch_service = batch_service
|
||||
if subagent_batches_config.enabled:
|
||||
|
||||
47
backend/app/subagent_batches/identity.py
Normal file
47
backend/app/subagent_batches/identity.py
Normal file
@ -0,0 +1,47 @@
|
||||
"""Resolve durable item owners through the Gateway's current account store."""
|
||||
|
||||
import logging
|
||||
|
||||
from app.gateway.auth_disabled import AUTH_DISABLED_USER_ID, get_auth_disabled_user, is_auth_disabled
|
||||
from deerflow.authz.provider import Principal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def resolve_batch_principal(batch, *, load_user) -> Principal:
|
||||
owner = batch["user_id"]
|
||||
spec = batch["execution_spec"]
|
||||
internal = spec.get("is_internal") is True
|
||||
if owner == AUTH_DISABLED_USER_ID and is_auth_disabled():
|
||||
user = get_auth_disabled_user()
|
||||
else:
|
||||
try:
|
||||
if load_user is None:
|
||||
raise RuntimeError("Account repository unavailable")
|
||||
user = await load_user(owner)
|
||||
except Exception as exc:
|
||||
# Database exception text can contain SQL parameters or credentials.
|
||||
logger.warning("Batch owner lookup failed (batch_id=%s, error_type=%s)", batch.get("id"), type(exc).__name__)
|
||||
raise PermissionError("Batch owner could not be verified") from None
|
||||
if user is None:
|
||||
# Gateway stamps no role on an unmapped, trusted internal identity.
|
||||
# A missing formerly mapped account must not take this fallback.
|
||||
if internal and spec.get("user_role") is None:
|
||||
return Principal(user_id=owner, is_internal=True, channel_user_id=spec.get("channel_user_id"))
|
||||
raise PermissionError("Batch owner is no longer available")
|
||||
if str(user.id) != owner or getattr(user, "disabled", False) or user.needs_setup:
|
||||
raise PermissionError("Batch owner is no longer active")
|
||||
if user.system_role != spec.get("user_role"):
|
||||
# Custom roles are not ordered. Refuse changed grants rather than
|
||||
# retaining a revoked role or widening old work after a promotion.
|
||||
raise PermissionError("Batch owner role changed; submit a new batch with current permissions")
|
||||
return Principal(
|
||||
user_id=owner,
|
||||
role=user.system_role,
|
||||
oauth_provider=user.oauth_provider,
|
||||
oauth_id=getattr(user, "oauth_id", None),
|
||||
channel_user_id=spec.get("channel_user_id"),
|
||||
is_internal=internal,
|
||||
# Gateway admission does not mint ABAC attributes. Stored attributes
|
||||
# and OAuth claims are not a renewable source of authority.
|
||||
)
|
||||
@ -39,6 +39,20 @@ suppress pending work; a failed, cancelled or hard-stopped source Run cannot res
|
||||
|
||||
## Authority, cancellation and recovery
|
||||
|
||||
Each Gateway batch item rechecks its owner before tool discovery and again after
|
||||
acquiring native execution capacity. This applies to the first attempt and to
|
||||
recovery after a restart. Missing, disabled or reset accounts cannot start work.
|
||||
A changed role rejects the old item; submit a new batch with current permissions.
|
||||
Custom roles have no general ordering, so this rule prevents both stale privileges
|
||||
after demotion and additional privileges after promotion. OAuth identity comes from
|
||||
the current account; serialized authorization attributes are not restored.
|
||||
|
||||
Explicit local auth-disabled mode and unmapped trusted internal owners retain their
|
||||
existing identity semantics. A deleted local account cannot become an unmapped
|
||||
internal owner. These checks govern admission, not cancellation of already admitted
|
||||
work. Standalone harness hosts with mutable accounts can supply the same
|
||||
`resolve_principal` callback without importing the Gateway.
|
||||
|
||||
Follow-up needs a session-authorized Gateway AgentRuns capability or local auth-disabled
|
||||
mode. Internal/PAT callers and unsupported embedded runtimes retain ordinary batches.
|
||||
Persist only the original permission ceiling, account token version and auth source;
|
||||
|
||||
@ -4,10 +4,12 @@ import asyncio
|
||||
import logging
|
||||
import socket
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from concurrent.futures import Future
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.authz.provider import Principal
|
||||
from deerflow.community.ragflow.sources import durable_source_artifact
|
||||
from deerflow.config.app_config import AppConfig, get_app_config
|
||||
from deerflow.config.subagent_batches_config import SubagentBatchesConfig
|
||||
@ -160,6 +162,7 @@ class SubagentBatchService:
|
||||
prepare_continuation=None,
|
||||
dispatch_continuation=None,
|
||||
cancel_continuation=None,
|
||||
resolve_principal: Callable[[dict[str, Any]], Awaitable[Principal]] | None = None,
|
||||
) -> None:
|
||||
self._repository = repository
|
||||
self._config = config
|
||||
@ -181,6 +184,7 @@ class SubagentBatchService:
|
||||
self._prepare_continuation = prepare_continuation
|
||||
self._dispatch_continuation = dispatch_continuation
|
||||
self._cancel_continuation = cancel_continuation
|
||||
self._resolve_principal = resolve_principal
|
||||
self._continuations: dict[str, asyncio.Task] = {}
|
||||
|
||||
async def start(self) -> None:
|
||||
@ -468,6 +472,32 @@ class SubagentBatchService:
|
||||
if lease.valid:
|
||||
await self._execute_leased_item(item, lease)
|
||||
|
||||
def _bind_principal_resolver(self, batch: dict[str, Any]) -> Callable[[], Awaitable[Principal]] | None:
|
||||
if self._resolve_principal is None:
|
||||
return None
|
||||
owner_loop = asyncio.get_running_loop()
|
||||
|
||||
async def resolve() -> Principal:
|
||||
principal = await self._resolve_principal(batch)
|
||||
if not isinstance(principal, Principal) or principal.user_id != batch["user_id"]:
|
||||
raise PermissionError("Batch principal does not match its owner")
|
||||
return principal
|
||||
|
||||
async def resolve_on_owner_loop() -> Principal:
|
||||
# Native executors use a separate loop; the host's async database
|
||||
# pool must stay on the Gateway loop, including after capacity waits.
|
||||
if asyncio.get_running_loop() is owner_loop:
|
||||
return await resolve()
|
||||
coro = resolve()
|
||||
try:
|
||||
pending = asyncio.run_coroutine_threadsafe(coro, owner_loop)
|
||||
except BaseException:
|
||||
coro.close()
|
||||
raise
|
||||
return await asyncio.wrap_future(pending)
|
||||
|
||||
return resolve_on_owner_loop
|
||||
|
||||
async def _execute_leased_item(self, item: dict[str, Any], lease: BatchItemLease) -> None:
|
||||
item_id = item["id"]
|
||||
execution_id: str | None = None
|
||||
@ -475,13 +505,15 @@ class SubagentBatchService:
|
||||
batch = item["batch"]
|
||||
self._item_batches[item_id] = batch["id"]
|
||||
spec = batch["execution_spec"]
|
||||
principal_resolver = self._bind_principal_resolver(batch)
|
||||
principal = await principal_resolver() if principal_resolver is not None else None
|
||||
config_data = spec["subagent_config"]
|
||||
if "prompt_overlay" in config_data:
|
||||
from deerflow.config.prompt_overlay import PromptOverlay
|
||||
|
||||
config_data = {**config_data, "prompt_overlay": PromptOverlay.model_validate(config_data["prompt_overlay"])}
|
||||
config = SubagentConfig(**config_data)
|
||||
app_config = self._app_config or get_app_config()
|
||||
app_config = self._app_config or await run_assembly(get_app_config)
|
||||
from deerflow.tools import get_available_tools
|
||||
|
||||
effective_model = resolve_subagent_model_name(
|
||||
@ -513,6 +545,8 @@ class SubagentBatchService:
|
||||
)
|
||||
return
|
||||
executor_kwargs = {}
|
||||
if principal_resolver is not None:
|
||||
executor_kwargs["principal_resolver"] = principal_resolver
|
||||
if "mcp_agent_auth" in spec:
|
||||
from deerflow.mcp.agent_auth import restore_agent_authority
|
||||
|
||||
@ -526,13 +560,13 @@ class SubagentBatchService:
|
||||
parent_model=spec.get("parent_model"),
|
||||
thread_id=batch["thread_id"],
|
||||
user_id=batch["user_id"],
|
||||
user_role=spec.get("user_role"),
|
||||
oauth_provider=spec.get("oauth_provider"),
|
||||
oauth_id=spec.get("oauth_id"),
|
||||
user_role=principal.role if principal is not None else spec.get("user_role"),
|
||||
oauth_provider=principal.oauth_provider if principal is not None else spec.get("oauth_provider"),
|
||||
oauth_id=principal.oauth_id if principal is not None else spec.get("oauth_id"),
|
||||
run_id=batch.get("run_id"),
|
||||
channel_user_id=spec.get("channel_user_id"),
|
||||
is_internal=spec.get("is_internal") is True,
|
||||
authz_attributes=spec.get("authz_attributes"),
|
||||
channel_user_id=principal.channel_user_id if principal is not None else spec.get("channel_user_id"),
|
||||
is_internal=principal.is_internal if principal is not None else spec.get("is_internal") is True,
|
||||
authz_attributes=principal.attributes if principal is not None else spec.get("authz_attributes"),
|
||||
knowledge_scope=spec.get("knowledge_scope"),
|
||||
execution_capacity=self._execution_capacity,
|
||||
extensions=self._extensions,
|
||||
|
||||
@ -8,7 +8,7 @@ import os
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Mapping
|
||||
from concurrent.futures import Future
|
||||
from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||||
from contextvars import Context, copy_context
|
||||
@ -33,6 +33,7 @@ from deerflow.agents.middlewares.audit_context import (
|
||||
)
|
||||
from deerflow.agents.thread_state import SandboxState, ThreadDataState, ThreadState
|
||||
from deerflow.authz.principal import normalize_authz_attributes
|
||||
from deerflow.authz.provider import Principal
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.knowledge_scope import KNOWLEDGE_SCOPE_RUNTIME_KEY, execution_scope
|
||||
@ -983,6 +984,7 @@ class SubagentExecutor:
|
||||
thread_incarnation: str | None | object = _THREAD_INCARNATION_UNSET,
|
||||
input_gate: Any | None = None,
|
||||
agent_mcp_authority: Any | None = None,
|
||||
principal_resolver: Callable[[], Awaitable[Principal]] | None = None,
|
||||
):
|
||||
"""Initialize the executor.
|
||||
|
||||
@ -1071,6 +1073,7 @@ class SubagentExecutor:
|
||||
self.user_id = user_id
|
||||
# Guardrail attribution propagated from the parent runtime context.
|
||||
self.user_role = user_role
|
||||
self._principal_resolver = principal_resolver
|
||||
self.oauth_provider = oauth_provider
|
||||
self.oauth_id = oauth_id
|
||||
self.run_id = run_id
|
||||
@ -1782,6 +1785,24 @@ class SubagentExecutor:
|
||||
subagent_checkpoint_scope(execution_id=result.task_id, run_id=self.run_id, tool_call_id=result.external_task_id, agent_name=self.config.name),
|
||||
slot,
|
||||
):
|
||||
if self._principal_resolver is not None:
|
||||
# Durable work can wait behind ordinary subagents after
|
||||
# its worker-side check. Revalidate before marking it
|
||||
# running or entering any task lifecycle hooks.
|
||||
try:
|
||||
principal = await self._principal_resolver()
|
||||
if not isinstance(principal, Principal) or principal.user_id != self.user_id:
|
||||
raise PermissionError("Subagent principal does not match its owner")
|
||||
self.user_role = principal.role
|
||||
self.oauth_provider = principal.oauth_provider
|
||||
self.oauth_id = principal.oauth_id
|
||||
self.channel_user_id = principal.channel_user_id
|
||||
self.is_internal = principal.is_internal
|
||||
self.authz_attributes = normalize_authz_attributes(principal.attributes)
|
||||
self._skill_authorization = None
|
||||
except Exception as exc:
|
||||
result.try_set_terminal(SubagentStatus.FAILED, error=str(exc))
|
||||
return result
|
||||
with result._state_lock:
|
||||
if result.status.is_terminal:
|
||||
return result
|
||||
|
||||
36
backend/tests/blocking_io/test_batch_owner_identity.py
Normal file
36
backend/tests/blocking_io/test_batch_owner_identity.py
Normal file
@ -0,0 +1,36 @@
|
||||
"""Dispatch revalidation reads the live account without blocking the Gateway."""
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.auth.models import User
|
||||
from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
|
||||
from app.subagent_batches.identity import resolve_batch_principal
|
||||
from deerflow.config.subagent_batches_config import SubagentBatchesConfig
|
||||
from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.subagents.batch_service import SubagentBatchService
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_owner_lookup_does_not_block_the_host_loop(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0")
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/users.db")
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(UserRow.__table__.create)
|
||||
repo = SQLiteUserRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
user = await repo.create_user(User(email="owner@example.com"))
|
||||
service = SubagentBatchService(
|
||||
repository=None,
|
||||
config=SubagentBatchesConfig(),
|
||||
runtime_config=SubagentRuntimeConfig(),
|
||||
resolve_principal=lambda batch: resolve_batch_principal(batch, load_user=repo.get_user_by_id),
|
||||
)
|
||||
resolver = service._bind_principal_resolver({"user_id": str(user.id), "execution_spec": {"user_role": "user"}})
|
||||
assert (await resolver()).role == "user"
|
||||
await repo.set_disabled(str(user.id), True)
|
||||
with pytest.raises(PermissionError):
|
||||
await resolver()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
67
backend/tests/support/batch_owner.py
Normal file
67
backend/tests/support/batch_owner.py
Normal file
@ -0,0 +1,67 @@
|
||||
"""Model/tool boundaries for the real Gateway batch-owner recovery test."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from langchain.tools import ToolRuntime, tool
|
||||
from langchain_core.language_models import BaseChatModel
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langchain_core.outputs import ChatGeneration, ChatResult
|
||||
|
||||
|
||||
def _record(name, item, runtime):
|
||||
record = {"tool": name, "item": item, "role": runtime.context.get("user_role"), "user_id": runtime.context.get("user_id"), "pid": os.getpid()}
|
||||
path = Path(os.environ["BATCH_OWNER_PROBE_DIR"]) / "calls.jsonl"
|
||||
with path.open("a", encoding="utf-8") as log:
|
||||
log.write(json.dumps(record) + "\n")
|
||||
return "Completed " + name
|
||||
|
||||
|
||||
@tool
|
||||
def owner_normal(item: str, runtime: ToolRuntime) -> str:
|
||||
"""Record an operation available to every configured role."""
|
||||
return _record("normal", item, runtime)
|
||||
|
||||
|
||||
@tool
|
||||
def owner_admin(item: str, runtime: ToolRuntime) -> str:
|
||||
"""Record an operation restricted by the host RBAC policy to administrators."""
|
||||
return _record("admin", item, runtime)
|
||||
|
||||
|
||||
class OwnerModel(BaseChatModel):
|
||||
@property
|
||||
def _llm_type(self):
|
||||
return "offline-batch-owner"
|
||||
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
return self
|
||||
|
||||
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
if any(isinstance(message, ToolMessage) for message in messages):
|
||||
answer = AIMessage(content="Done")
|
||||
elif any(isinstance(message, HumanMessage) and "OWNER_CHILD" in str(message.content) for message in messages):
|
||||
root = Path(os.environ["BATCH_OWNER_PROBE_DIR"])
|
||||
item = "first" if any("OWNER_CHILD:first" in str(message.content) for message in messages) else "second"
|
||||
(root / f"reached-{item}").write_text(str(os.getpid()), encoding="utf-8")
|
||||
deadline = time.monotonic() + 60
|
||||
while item == "first" and (root / "hold").exists():
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("Batch owner test barrier not released")
|
||||
time.sleep(0.02)
|
||||
answer = AIMessage(content="", tool_calls=[{"name": name, "args": {"item": item}, "id": name, "type": "tool_call"} for name in ("owner_admin", "owner_normal")])
|
||||
else:
|
||||
answer = AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "batch_task",
|
||||
"args": {"title": "Owner recovery", "subagent_type": "general-purpose", "items": [{"key": key, "prompt": f"OWNER_CHILD:{key}: run both configured observations"} for key in ("first", "second")]},
|
||||
"id": "batch",
|
||||
"type": "tool_call",
|
||||
}
|
||||
],
|
||||
)
|
||||
return ChatResult(generations=[ChatGeneration(message=answer)])
|
||||
123
backend/tests/test_batch_owner_admission.py
Normal file
123
backend/tests/test_batch_owner_admission.py
Normal file
@ -0,0 +1,123 @@
|
||||
"""Native capacity waits revalidate identity on the host's database loop."""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from support.batch_owner import owner_admin, owner_normal
|
||||
|
||||
from deerflow.authz.provider import Principal
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.subagent_batches_config import SubagentBatchesConfig
|
||||
from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig
|
||||
from deerflow.subagents.batch_service import SubagentBatchService
|
||||
from deerflow.subagents.capacity import SubagentExecutionCapacity
|
||||
from deerflow.subagents.config import SubagentConfig
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def executor_module(monkeypatch, tmp_path):
|
||||
# conftest installs an executor stub; load the real graph and worker loop.
|
||||
path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/executor.py"
|
||||
spec = importlib.util.spec_from_file_location("_batch_owner_executor", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
spec.loader.exec_module(module)
|
||||
monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "home"))
|
||||
monkeypatch.setenv("BATCH_OWNER_PROBE_DIR", str(tmp_path))
|
||||
yield module
|
||||
module._shutdown_isolated_subagent_loop()
|
||||
|
||||
|
||||
def _executor(module, resolver, capacity=None):
|
||||
config = AppConfig.model_validate(
|
||||
{
|
||||
"models": [{"name": "owner-probe", "use": "support.batch_owner:OwnerModel", "model": "offline"}],
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
"summarization": {"enabled": False},
|
||||
"tool_search": {"enabled": False},
|
||||
"authorization": {"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"admin": {}, "user": {"tools": {"deny": ["owner_admin"]}}}}}},
|
||||
}
|
||||
)
|
||||
return module.SubagentExecutor(
|
||||
config=SubagentConfig(name="general-purpose", description="Owner probe", skills=[]),
|
||||
tools=[owner_admin, owner_normal],
|
||||
app_config=config,
|
||||
parent_model="owner-probe",
|
||||
thread_id="batch-owner-test",
|
||||
user_id="owner",
|
||||
user_role="admin",
|
||||
oauth_provider="old",
|
||||
oauth_id="old",
|
||||
authz_attributes={"old_grant": True},
|
||||
execution_capacity=capacity,
|
||||
principal_resolver=resolver,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("change", ["disabled", "different_owner", "fresh_profile", "unchanged"])
|
||||
async def test_identity_is_rechecked_after_capacity_wait(executor_module, tmp_path, change):
|
||||
capacity = SubagentExecutionCapacity(SubagentRuntimeConfig(max_running=1))
|
||||
state = "queued"
|
||||
calls = []
|
||||
|
||||
async def resolve():
|
||||
calls.append(state)
|
||||
if state == "disabled":
|
||||
raise PermissionError("Owner disabled")
|
||||
return Principal(user_id="different" if state == "different_owner" else "owner", role="user" if state == "fresh_profile" else "admin", oauth_provider="oidc", oauth_id="current")
|
||||
|
||||
executor = _executor(executor_module, resolve, capacity)
|
||||
task = None
|
||||
try:
|
||||
async with capacity.slot():
|
||||
task = asyncio.create_task(executor._aexecute("OWNER_CHILD:second"))
|
||||
async with asyncio.timeout(5):
|
||||
while capacity.snapshot().queued != 1:
|
||||
await asyncio.sleep(0)
|
||||
assert not calls and not (tmp_path / "reached-second").exists()
|
||||
state = change
|
||||
result = await asyncio.wait_for(task, 20)
|
||||
assert calls == [change]
|
||||
assert result.execution_done_event.is_set()
|
||||
if change in ("disabled", "different_owner"):
|
||||
assert result.status == executor_module.SubagentStatus.FAILED
|
||||
assert result.started_at is None
|
||||
assert not (tmp_path / "calls.jsonl").exists()
|
||||
else:
|
||||
assert result.status == executor_module.SubagentStatus.COMPLETED, result.error
|
||||
observed = [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
assert {call["tool"] for call in observed} == ({"normal"} if change == "fresh_profile" else {"normal", "admin"})
|
||||
assert executor.oauth_provider == "oidc" and executor.oauth_id == "current"
|
||||
assert executor.authz_attributes == {}
|
||||
assert capacity.snapshot().running == capacity.snapshot().queued == 0
|
||||
finally:
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_worker_resolves_on_gateway_loop(executor_module, tmp_path):
|
||||
host_loop = asyncio.get_running_loop()
|
||||
lookups = []
|
||||
|
||||
async def resolve(batch):
|
||||
assert asyncio.get_running_loop() is host_loop
|
||||
lookups.append(batch["user_id"])
|
||||
return Principal(user_id=batch["user_id"], role="user")
|
||||
|
||||
service = SubagentBatchService(repository=None, config=SubagentBatchesConfig(), runtime_config=SubagentRuntimeConfig(), resolve_principal=resolve)
|
||||
resolver = service._bind_principal_resolver({"user_id": "owner"})
|
||||
await resolver() # Pre-discovery check on the host loop.
|
||||
executor = _executor(executor_module, resolver)
|
||||
pending = executor_module.run_on_isolated_subagent_loop(executor._aexecute("OWNER_CHILD:second"))
|
||||
result = await asyncio.wait_for(asyncio.wrap_future(pending), 20)
|
||||
assert result.status == executor_module.SubagentStatus.COMPLETED, result.error
|
||||
assert lookups == ["owner", "owner"]
|
||||
observed = [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text(encoding="utf-8").splitlines()]
|
||||
assert {call["tool"] for call in observed} == {"normal"}
|
||||
131
backend/tests/test_batch_owner_identity.py
Normal file
131
backend/tests/test_batch_owner_identity.py
Normal file
@ -0,0 +1,131 @@
|
||||
"""Gateway batch identity resolves against the current account repository."""
|
||||
|
||||
from traceback import format_exception
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway.auth.models import User
|
||||
from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
|
||||
from app.subagent_batches.identity import resolve_batch_principal
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def accounts(tmp_path, monkeypatch):
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/users.db")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(UserRow.__table__.create)
|
||||
sessions = async_sessionmaker(engine, expire_on_commit=False)
|
||||
repo = SQLiteUserRepository(sessions)
|
||||
owner = await repo.create_user(User(email="owner@example.com", system_role="admin", oauth_provider="oidc", oauth_id="current"))
|
||||
await repo.create_user(User(email="root@example.com", system_role="admin"))
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0")
|
||||
try:
|
||||
yield repo, owner, sessions
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _batch(owner, **spec):
|
||||
return {"id": "batch-1", "user_id": owner, "execution_spec": spec}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_current_profile_replaces_stored_oauth_and_attributes(accounts):
|
||||
repo, owner, _ = accounts
|
||||
principal = await resolve_batch_principal(_batch(str(owner.id), user_role="admin", oauth_provider="old", oauth_id="old", authz_attributes={"admin": True}, is_internal=True, channel_user_id="im-user"), load_user=repo.get_user_by_id)
|
||||
assert (principal.user_id, principal.role, principal.oauth_provider, principal.oauth_id) == (str(owner.id), "admin", "oidc", "current")
|
||||
assert principal.is_internal and principal.channel_user_id == "im-user"
|
||||
assert principal.attributes == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("change", ["disabled", "demoted", "reset", "deleted"])
|
||||
@pytest.mark.parametrize("internal", [False, True])
|
||||
async def test_revoked_local_owner_cannot_fall_back_to_internal(accounts, change, internal):
|
||||
repo, owner, sessions = accounts
|
||||
batch = _batch(str(owner.id), user_role="admin", is_internal=internal)
|
||||
if change == "disabled":
|
||||
await repo.set_disabled(str(owner.id), True)
|
||||
elif change == "demoted":
|
||||
await repo.update_system_role(str(owner.id), "user")
|
||||
elif change == "reset":
|
||||
await repo.update_user(owner.model_copy(update={"needs_setup": True}))
|
||||
else:
|
||||
async with sessions() as session:
|
||||
await session.execute(delete(UserRow).where(UserRow.id == str(owner.id)))
|
||||
await session.commit()
|
||||
with pytest.raises(PermissionError):
|
||||
await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("original,current", [("user", "admin"), ("reviewer", "operator"), (None, "admin")])
|
||||
async def test_role_changes_do_not_invent_a_role_hierarchy(accounts, original, current):
|
||||
repo, owner, _ = accounts
|
||||
await repo.update_system_role(str(owner.id), current)
|
||||
with pytest.raises(PermissionError):
|
||||
await resolve_batch_principal(_batch(str(owner.id), user_role=original), load_user=repo.get_user_by_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unmapped_internal_owner_keeps_default_role_without_stale_claims(accounts):
|
||||
repo, _, _ = accounts
|
||||
batch = _batch("external-owner", is_internal=True, oauth_provider="old", oauth_id="old", authz_attributes={"admin": True})
|
||||
principal = await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
|
||||
assert principal.user_id == "external-owner" and principal.is_internal
|
||||
assert principal.role is None and principal.oauth_provider is None and principal.oauth_id is None
|
||||
assert principal.attributes == {}
|
||||
batch["execution_spec"]["is_internal"] = False
|
||||
with pytest.raises(PermissionError):
|
||||
await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auth_disabled_only_preserves_explicit_synthetic_owner(accounts, monkeypatch):
|
||||
repo, owner, _ = accounts
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1")
|
||||
monkeypatch.delenv("DEER_FLOW_ENV", raising=False)
|
||||
monkeypatch.delenv("ENVIRONMENT", raising=False)
|
||||
batch = _batch("default", user_role="admin")
|
||||
principal = await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
|
||||
assert principal.user_id == "default" and principal.role == "admin"
|
||||
await repo.set_disabled(str(owner.id), True)
|
||||
with pytest.raises(PermissionError):
|
||||
await resolve_batch_principal(_batch(str(owner.id), user_role="admin"), load_user=repo.get_user_by_id)
|
||||
monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0")
|
||||
with pytest.raises(PermissionError):
|
||||
await resolve_batch_principal(batch, load_user=repo.get_user_by_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["missing_loader", "exception", "mismatch"])
|
||||
async def test_unavailable_identity_fails_closed(accounts, failure, caplog):
|
||||
_, owner, _ = accounts
|
||||
batch = _batch(str(owner.id), user_role="admin", is_internal=True)
|
||||
fake_password = "fixture-db-password"
|
||||
statement = "SELECT id FROM users WHERE id = :id"
|
||||
|
||||
async def load_user(user_id):
|
||||
if failure == "exception":
|
||||
raise OperationalError(statement, {"id": user_id}, RuntimeError(f"postgresql://test:{fake_password}@test-db/deerflow"))
|
||||
return User(email="different@example.com", system_role="admin")
|
||||
|
||||
with pytest.raises(PermissionError) as exc:
|
||||
await resolve_batch_principal(batch, load_user=None if failure == "missing_loader" else load_user)
|
||||
assert fake_password not in str(exc.value)
|
||||
if failure == "exception":
|
||||
records = [record for record in caplog.records if record.name == "app.subagent_batches.identity"]
|
||||
assert len(records) == 1
|
||||
record = records[0]
|
||||
assert batch["id"] in record.getMessage()
|
||||
assert "OperationalError" in record.getMessage()
|
||||
assert record.exc_info is None
|
||||
diagnostics = caplog.text + "".join(format_exception(exc.value))
|
||||
assert fake_password not in diagnostics
|
||||
assert str(owner.id) not in diagnostics
|
||||
assert statement not in diagnostics
|
||||
219
backend/tests/test_batch_owner_restart.py
Normal file
219
backend/tests/test_batch_owner_restart.py
Normal file
@ -0,0 +1,219 @@
|
||||
"""Queued and recovered batches must honor current account state and role."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.schema import CreateSchema, DropSchema
|
||||
|
||||
from deerflow.persistence.postgres_schema import dsn_with_search_path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
pytestmark = pytest.mark.skipif(os.name != "posix", reason="Tests SIGKILL recovery in separate Gateway processes")
|
||||
|
||||
|
||||
def _until(check, timeout=40):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = check()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(0.05)
|
||||
raise AssertionError(f"Timed out waiting for {check}")
|
||||
|
||||
|
||||
@pytest.fixture(params=["sqlite", "postgres"])
|
||||
def database(request, tmp_path):
|
||||
schema = None
|
||||
if request.param == "postgres":
|
||||
uri = os.environ.get("TEST_POSTGRES_URI")
|
||||
if not uri:
|
||||
pytest.skip("requires TEST_POSTGRES_URI (real Gateway restart)")
|
||||
pytest.importorskip("psycopg")
|
||||
pytest.importorskip("asyncpg")
|
||||
schema = "test_batch_owner_" + uuid.uuid4().hex
|
||||
config = {"backend": "postgres", "postgres_url": uri, "postgres_schema": schema}
|
||||
engine = create_engine(make_url(dsn_with_search_path(uri, schema)).set(drivername="postgresql+psycopg"))
|
||||
else:
|
||||
config = {"backend": "sqlite", "sqlite_dir": str(tmp_path / "db")}
|
||||
engine = create_engine(f"sqlite:///{tmp_path}/db/deerflow.db")
|
||||
|
||||
def read(statement, **params):
|
||||
with engine.connect() as connection:
|
||||
return connection.execute(text(statement), params).all()
|
||||
|
||||
try:
|
||||
if schema:
|
||||
with engine.begin() as connection:
|
||||
connection.execute(CreateSchema(schema))
|
||||
yield config, read
|
||||
finally:
|
||||
try:
|
||||
if schema:
|
||||
with engine.begin() as connection:
|
||||
connection.execute(DropSchema(schema, cascade=True, if_exists=True))
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _gateway(root, client, port, generation):
|
||||
env = {key: os.environ[key] for key in ("PATH", "HOME", "TMPDIR", "LANG") if key in os.environ}
|
||||
env.update(
|
||||
PYTHONPATH=os.pathsep.join(str(ROOT / path) for path in ("backend/packages/harness", "backend/packages/extension-api", "backend", "backend/tests")),
|
||||
DEER_FLOW_HOME=str(root / "home"),
|
||||
DEER_FLOW_PROJECT_ROOT=str(ROOT),
|
||||
DEER_FLOW_CONFIG_PATH=str(root / "config.yaml"),
|
||||
DEER_FLOW_EXTENSIONS_CONFIG_PATH=str(root / "extensions.json"),
|
||||
DEER_FLOW_ENV_FILE=str(root / "empty.env"),
|
||||
BATCH_OWNER_PROBE_DIR=str(root),
|
||||
AUTH_JWT_SECRET="local-batch-owner-test-secret-1234567890",
|
||||
DEER_FLOW_AUTH_DISABLED="0",
|
||||
NO_PROXY="127.0.0.1,localhost",
|
||||
GATEWAY_WORKERS="1",
|
||||
WEB_CONCURRENCY="1",
|
||||
PYTHONIOENCODING="utf-8",
|
||||
)
|
||||
log_path = root / f"gateway-{generation}.log"
|
||||
with log_path.open("w", encoding="utf-8") as log:
|
||||
process = subprocess.Popen([sys.executable, "-m", "uvicorn", "app.gateway.app:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=root, env=env, stdout=log, stderr=subprocess.STDOUT)
|
||||
try:
|
||||
|
||||
def healthy():
|
||||
assert process.poll() is None, log_path.read_text(encoding="utf-8")[-8000:]
|
||||
try:
|
||||
return client.get("/health").status_code == 200
|
||||
except httpx.TransportError:
|
||||
return False
|
||||
|
||||
_until(healthy)
|
||||
yield process
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=10)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dispatch", ["batch_queue", "restart"])
|
||||
@pytest.mark.parametrize("change", ["disabled", "demoted", "promoted", "unchanged_admin", "unchanged_user"])
|
||||
def test_batch_revalidates_owner_at_execution(tmp_path, database, dispatch, change):
|
||||
database_config, read = database
|
||||
config = {
|
||||
"models": [{"name": "owner-probe", "use": "support.batch_owner:OwnerModel", "model": "offline", "supports_tool_calling": True}],
|
||||
"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"},
|
||||
"database": database_config,
|
||||
"tools": [{"name": name, "group": "probe", "use": "support.batch_owner:" + name} for name in ("owner_normal", "owner_admin")],
|
||||
"tool_groups": [{"name": "probe"}],
|
||||
"title": {"enabled": False},
|
||||
"memory": {"enabled": False},
|
||||
"summarization": {"enabled": False},
|
||||
"tool_search": {"enabled": False},
|
||||
"hitl_approval": {"enabled": False},
|
||||
"subagent_batches": {"enabled": True, "poll_interval_seconds": 0.1, "lease_seconds": 10, "default_max_running_items": 1, "max_attempts": 3 if dispatch == "restart" else 1},
|
||||
"subagent_runtime": {"max_running": 1},
|
||||
"authorization": {"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"admin": {}, "user": {"tools": {"deny": ["owner_admin"]}}}}}},
|
||||
}
|
||||
(tmp_path / "config.yaml").write_text(json.dumps(config), encoding="utf-8")
|
||||
(tmp_path / "extensions.json").write_text("{}", encoding="utf-8")
|
||||
(tmp_path / "empty.env").touch()
|
||||
(tmp_path / "hold").touch()
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
with httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=30) as admin, httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=30) as actor:
|
||||
|
||||
def request(client, method, path, *, expected=200, **kwargs):
|
||||
response = client.request(method, path, **kwargs)
|
||||
assert response.status_code == expected, response.text
|
||||
return response.json()
|
||||
|
||||
with _gateway(tmp_path, admin, port, 1) as first:
|
||||
for client, route, email in ((admin, "initialize", "admin@example.com"), (actor, "register", "actor@example.com")):
|
||||
account = request(client, "POST", "/api/v1/auth/" + route, expected=201, json={"email": email, "password": "local-batch-owner-password-123"})
|
||||
client.headers["X-CSRF-Token"] = client.cookies["csrf_token"]
|
||||
owner = account["id"]
|
||||
original_role = "user" if change in ("unchanged_user", "promoted") else "admin"
|
||||
if original_role == "admin":
|
||||
request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"system_role": "admin"})
|
||||
thread = request(actor, "POST", "/api/threads", json={})["thread_id"]
|
||||
request(
|
||||
actor,
|
||||
"POST",
|
||||
f"/api/threads/{thread}/runs/wait",
|
||||
json={"assistant_id": "lead_agent", "input": {"messages": [{"role": "user", "content": "Submit the owner recovery batch"}]}, "context": {"mode": "pro", "subagent_enabled": True}},
|
||||
)
|
||||
_until(lambda: (tmp_path / "reached-first").exists())
|
||||
assert not (tmp_path / "reached-second").exists()
|
||||
assert not (tmp_path / "calls.jsonl").exists()
|
||||
batch_id, stored_spec = read("SELECT id, execution_spec FROM subagent_batches WHERE user_id = :owner", owner=owner)[0]
|
||||
spec = json.loads(stored_spec) if isinstance(stored_spec, str) else stored_spec
|
||||
assert spec["user_role"] == original_role
|
||||
|
||||
def items():
|
||||
return read("SELECT item_key, status, attempt, error FROM subagent_batch_items WHERE batch_id = :batch_id ORDER BY position", batch_id=batch_id)
|
||||
|
||||
_until(lambda: len(rows := items()) == 2 and rows[1][2] == 0)
|
||||
if change == "disabled":
|
||||
request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"disabled": True})
|
||||
request(actor, "GET", "/api/v1/auth/me", expected=401)
|
||||
elif change in ("demoted", "promoted"):
|
||||
role = "user" if change == "demoted" else "admin"
|
||||
request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"system_role": role})
|
||||
assert request(actor, "GET", "/api/v1/auth/me")["system_role"] == role
|
||||
|
||||
def verify(pid):
|
||||
def terminal():
|
||||
rows = items()
|
||||
return rows if len(rows) == 2 and all(row[1] in ("succeeded", "failed", "cancelled") for row in rows) else None
|
||||
|
||||
rows = _until(terminal)
|
||||
path = tmp_path / "calls.jsonl"
|
||||
calls = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else []
|
||||
# Preserve observable results before the PostgreSQL schema is
|
||||
# dropped; never manufacture task state for recovery.
|
||||
(tmp_path / "evidence.json").write_text(
|
||||
json.dumps(
|
||||
{"backend": database_config["backend"], "change": change, "dispatch": dispatch, "old_pid": first.pid, "new_pid": pid, "old_exit": first.returncode, "items": [dict(row._mapping) for row in rows], "calls": calls},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert all(call["pid"] == pid and call["user_id"] == owner for call in calls)
|
||||
if dispatch == "restart":
|
||||
assert rows[0][2] >= 2
|
||||
else:
|
||||
assert rows[1][2] == 1
|
||||
assert rows[0][1] == "succeeded" # Already admitted, outside the revocation gate.
|
||||
observed = calls if dispatch == "restart" else [call for call in calls if call["item"] == "second"]
|
||||
if change in ("disabled", "demoted", "promoted"):
|
||||
assert rows[1][1] == "failed", (rows, calls)
|
||||
assert not observed
|
||||
assert not (tmp_path / "reached-second").exists()
|
||||
else:
|
||||
assert all(row[1] == "succeeded" for row in rows), (rows, calls)
|
||||
assert all(call["role"] == original_role for call in observed)
|
||||
assert {call["tool"] for call in observed} == ({"admin", "normal"} if original_role == "admin" else {"normal"})
|
||||
|
||||
if dispatch != "restart":
|
||||
(tmp_path / "hold").unlink()
|
||||
verify(first.pid)
|
||||
return
|
||||
first.kill()
|
||||
first.wait(timeout=10)
|
||||
assert first.returncode == -signal.SIGKILL
|
||||
|
||||
(tmp_path / "hold").unlink()
|
||||
with _gateway(tmp_path, admin, port, 2) as second:
|
||||
assert second.pid != first.pid
|
||||
verify(second.pid)
|
||||
Loading…
x
Reference in New Issue
Block a user