diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 835ebb0c67..1320b62ef1 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -926,6 +926,7 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]: raise RuntimeError("subagent_batches.enabled requires database.backend sqlite or postgres") if batch_repo is not None: from app.subagent_batches.continuation import BatchContinuations + from app.subagent_batches.identity import resolve_batch_principal continuations = BatchContinuations(app) app.state.batch_continuations = continuations @@ -937,6 +938,7 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]: prepare_continuation=continuations.prepare, dispatch_continuation=continuations.dispatch, cancel_continuation=continuations.cancel, + resolve_principal=lambda batch: resolve_batch_principal(batch, load_user=app.state.agent_runs_host.load_user), ) app.state.subagent_batch_service = batch_service if subagent_batches_config.enabled: diff --git a/backend/app/subagent_batches/identity.py b/backend/app/subagent_batches/identity.py new file mode 100644 index 0000000000..ad6766cf60 --- /dev/null +++ b/backend/app/subagent_batches/identity.py @@ -0,0 +1,47 @@ +"""Resolve durable item owners through the Gateway's current account store.""" + +import logging + +from app.gateway.auth_disabled import AUTH_DISABLED_USER_ID, get_auth_disabled_user, is_auth_disabled +from deerflow.authz.provider import Principal + +logger = logging.getLogger(__name__) + + +async def resolve_batch_principal(batch, *, load_user) -> Principal: + owner = batch["user_id"] + spec = batch["execution_spec"] + internal = spec.get("is_internal") is True + if owner == AUTH_DISABLED_USER_ID and is_auth_disabled(): + user = get_auth_disabled_user() + else: + try: + if load_user is None: + raise RuntimeError("Account repository unavailable") + user = await load_user(owner) + except Exception as exc: + # Database exception text can contain SQL parameters or credentials. + logger.warning("Batch owner lookup failed (batch_id=%s, error_type=%s)", batch.get("id"), type(exc).__name__) + raise PermissionError("Batch owner could not be verified") from None + if user is None: + # Gateway stamps no role on an unmapped, trusted internal identity. + # A missing formerly mapped account must not take this fallback. + if internal and spec.get("user_role") is None: + return Principal(user_id=owner, is_internal=True, channel_user_id=spec.get("channel_user_id")) + raise PermissionError("Batch owner is no longer available") + if str(user.id) != owner or getattr(user, "disabled", False) or user.needs_setup: + raise PermissionError("Batch owner is no longer active") + if user.system_role != spec.get("user_role"): + # Custom roles are not ordered. Refuse changed grants rather than + # retaining a revoked role or widening old work after a promotion. + raise PermissionError("Batch owner role changed; submit a new batch with current permissions") + return Principal( + user_id=owner, + role=user.system_role, + oauth_provider=user.oauth_provider, + oauth_id=getattr(user, "oauth_id", None), + channel_user_id=spec.get("channel_user_id"), + is_internal=internal, + # Gateway admission does not mint ABAC attributes. Stored attributes + # and OAuth claims are not a renewable source of authority. + ) diff --git a/backend/docs/BATCH_CONTINUATION.md b/backend/docs/BATCH_CONTINUATION.md index a7fc8d696a..eaf7af99a5 100644 --- a/backend/docs/BATCH_CONTINUATION.md +++ b/backend/docs/BATCH_CONTINUATION.md @@ -39,6 +39,20 @@ suppress pending work; a failed, cancelled or hard-stopped source Run cannot res ## Authority, cancellation and recovery +Each Gateway batch item rechecks its owner before tool discovery and again after +acquiring native execution capacity. This applies to the first attempt and to +recovery after a restart. Missing, disabled or reset accounts cannot start work. +A changed role rejects the old item; submit a new batch with current permissions. +Custom roles have no general ordering, so this rule prevents both stale privileges +after demotion and additional privileges after promotion. OAuth identity comes from +the current account; serialized authorization attributes are not restored. + +Explicit local auth-disabled mode and unmapped trusted internal owners retain their +existing identity semantics. A deleted local account cannot become an unmapped +internal owner. These checks govern admission, not cancellation of already admitted +work. Standalone harness hosts with mutable accounts can supply the same +`resolve_principal` callback without importing the Gateway. + Follow-up needs a session-authorized Gateway AgentRuns capability or local auth-disabled mode. Internal/PAT callers and unsupported embedded runtimes retain ordinary batches. Persist only the original permission ceiling, account token version and auth source; diff --git a/backend/packages/harness/deerflow/subagents/batch_service.py b/backend/packages/harness/deerflow/subagents/batch_service.py index 554ccbef76..a71735f984 100644 --- a/backend/packages/harness/deerflow/subagents/batch_service.py +++ b/backend/packages/harness/deerflow/subagents/batch_service.py @@ -4,10 +4,12 @@ import asyncio import logging import socket import uuid +from collections.abc import Awaitable, Callable from concurrent.futures import Future from datetime import UTC, datetime from typing import Any +from deerflow.authz.provider import Principal from deerflow.community.ragflow.sources import durable_source_artifact from deerflow.config.app_config import AppConfig, get_app_config from deerflow.config.subagent_batches_config import SubagentBatchesConfig @@ -160,6 +162,7 @@ class SubagentBatchService: prepare_continuation=None, dispatch_continuation=None, cancel_continuation=None, + resolve_principal: Callable[[dict[str, Any]], Awaitable[Principal]] | None = None, ) -> None: self._repository = repository self._config = config @@ -181,6 +184,7 @@ class SubagentBatchService: self._prepare_continuation = prepare_continuation self._dispatch_continuation = dispatch_continuation self._cancel_continuation = cancel_continuation + self._resolve_principal = resolve_principal self._continuations: dict[str, asyncio.Task] = {} async def start(self) -> None: @@ -468,6 +472,32 @@ class SubagentBatchService: if lease.valid: await self._execute_leased_item(item, lease) + def _bind_principal_resolver(self, batch: dict[str, Any]) -> Callable[[], Awaitable[Principal]] | None: + if self._resolve_principal is None: + return None + owner_loop = asyncio.get_running_loop() + + async def resolve() -> Principal: + principal = await self._resolve_principal(batch) + if not isinstance(principal, Principal) or principal.user_id != batch["user_id"]: + raise PermissionError("Batch principal does not match its owner") + return principal + + async def resolve_on_owner_loop() -> Principal: + # Native executors use a separate loop; the host's async database + # pool must stay on the Gateway loop, including after capacity waits. + if asyncio.get_running_loop() is owner_loop: + return await resolve() + coro = resolve() + try: + pending = asyncio.run_coroutine_threadsafe(coro, owner_loop) + except BaseException: + coro.close() + raise + return await asyncio.wrap_future(pending) + + return resolve_on_owner_loop + async def _execute_leased_item(self, item: dict[str, Any], lease: BatchItemLease) -> None: item_id = item["id"] execution_id: str | None = None @@ -475,13 +505,15 @@ class SubagentBatchService: batch = item["batch"] self._item_batches[item_id] = batch["id"] spec = batch["execution_spec"] + principal_resolver = self._bind_principal_resolver(batch) + principal = await principal_resolver() if principal_resolver is not None else None config_data = spec["subagent_config"] if "prompt_overlay" in config_data: from deerflow.config.prompt_overlay import PromptOverlay config_data = {**config_data, "prompt_overlay": PromptOverlay.model_validate(config_data["prompt_overlay"])} config = SubagentConfig(**config_data) - app_config = self._app_config or get_app_config() + app_config = self._app_config or await run_assembly(get_app_config) from deerflow.tools import get_available_tools effective_model = resolve_subagent_model_name( @@ -513,6 +545,8 @@ class SubagentBatchService: ) return executor_kwargs = {} + if principal_resolver is not None: + executor_kwargs["principal_resolver"] = principal_resolver if "mcp_agent_auth" in spec: from deerflow.mcp.agent_auth import restore_agent_authority @@ -526,13 +560,13 @@ class SubagentBatchService: parent_model=spec.get("parent_model"), thread_id=batch["thread_id"], user_id=batch["user_id"], - user_role=spec.get("user_role"), - oauth_provider=spec.get("oauth_provider"), - oauth_id=spec.get("oauth_id"), + user_role=principal.role if principal is not None else spec.get("user_role"), + oauth_provider=principal.oauth_provider if principal is not None else spec.get("oauth_provider"), + oauth_id=principal.oauth_id if principal is not None else spec.get("oauth_id"), run_id=batch.get("run_id"), - channel_user_id=spec.get("channel_user_id"), - is_internal=spec.get("is_internal") is True, - authz_attributes=spec.get("authz_attributes"), + channel_user_id=principal.channel_user_id if principal is not None else spec.get("channel_user_id"), + is_internal=principal.is_internal if principal is not None else spec.get("is_internal") is True, + authz_attributes=principal.attributes if principal is not None else spec.get("authz_attributes"), knowledge_scope=spec.get("knowledge_scope"), execution_capacity=self._execution_capacity, extensions=self._extensions, diff --git a/backend/packages/harness/deerflow/subagents/executor.py b/backend/packages/harness/deerflow/subagents/executor.py index f7c56e832b..59d50cf570 100644 --- a/backend/packages/harness/deerflow/subagents/executor.py +++ b/backend/packages/harness/deerflow/subagents/executor.py @@ -8,7 +8,7 @@ import os import sys import threading import uuid -from collections.abc import Callable, Coroutine, Mapping +from collections.abc import Awaitable, Callable, Coroutine, Mapping from concurrent.futures import Future from concurrent.futures import TimeoutError as FuturesTimeoutError from contextvars import Context, copy_context @@ -33,6 +33,7 @@ from deerflow.agents.middlewares.audit_context import ( ) from deerflow.agents.thread_state import SandboxState, ThreadDataState, ThreadState from deerflow.authz.principal import normalize_authz_attributes +from deerflow.authz.provider import Principal from deerflow.config import get_app_config from deerflow.config.app_config import AppConfig from deerflow.knowledge_scope import KNOWLEDGE_SCOPE_RUNTIME_KEY, execution_scope @@ -983,6 +984,7 @@ class SubagentExecutor: thread_incarnation: str | None | object = _THREAD_INCARNATION_UNSET, input_gate: Any | None = None, agent_mcp_authority: Any | None = None, + principal_resolver: Callable[[], Awaitable[Principal]] | None = None, ): """Initialize the executor. @@ -1071,6 +1073,7 @@ class SubagentExecutor: self.user_id = user_id # Guardrail attribution propagated from the parent runtime context. self.user_role = user_role + self._principal_resolver = principal_resolver self.oauth_provider = oauth_provider self.oauth_id = oauth_id self.run_id = run_id @@ -1782,6 +1785,24 @@ class SubagentExecutor: subagent_checkpoint_scope(execution_id=result.task_id, run_id=self.run_id, tool_call_id=result.external_task_id, agent_name=self.config.name), slot, ): + if self._principal_resolver is not None: + # Durable work can wait behind ordinary subagents after + # its worker-side check. Revalidate before marking it + # running or entering any task lifecycle hooks. + try: + principal = await self._principal_resolver() + if not isinstance(principal, Principal) or principal.user_id != self.user_id: + raise PermissionError("Subagent principal does not match its owner") + self.user_role = principal.role + self.oauth_provider = principal.oauth_provider + self.oauth_id = principal.oauth_id + self.channel_user_id = principal.channel_user_id + self.is_internal = principal.is_internal + self.authz_attributes = normalize_authz_attributes(principal.attributes) + self._skill_authorization = None + except Exception as exc: + result.try_set_terminal(SubagentStatus.FAILED, error=str(exc)) + return result with result._state_lock: if result.status.is_terminal: return result diff --git a/backend/tests/blocking_io/test_batch_owner_identity.py b/backend/tests/blocking_io/test_batch_owner_identity.py new file mode 100644 index 0000000000..70412c4fbf --- /dev/null +++ b/backend/tests/blocking_io/test_batch_owner_identity.py @@ -0,0 +1,36 @@ +"""Dispatch revalidation reads the live account without blocking the Gateway.""" + +import pytest +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.gateway.auth.models import User +from app.gateway.auth.repositories.sqlite import SQLiteUserRepository +from app.subagent_batches.identity import resolve_batch_principal +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.persistence.user.model import UserRow +from deerflow.subagents.batch_service import SubagentBatchService + + +@pytest.mark.asyncio +async def test_batch_owner_lookup_does_not_block_the_host_loop(tmp_path, monkeypatch): + monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0") + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/users.db") + try: + async with engine.begin() as connection: + await connection.run_sync(UserRow.__table__.create) + repo = SQLiteUserRepository(async_sessionmaker(engine, expire_on_commit=False)) + user = await repo.create_user(User(email="owner@example.com")) + service = SubagentBatchService( + repository=None, + config=SubagentBatchesConfig(), + runtime_config=SubagentRuntimeConfig(), + resolve_principal=lambda batch: resolve_batch_principal(batch, load_user=repo.get_user_by_id), + ) + resolver = service._bind_principal_resolver({"user_id": str(user.id), "execution_spec": {"user_role": "user"}}) + assert (await resolver()).role == "user" + await repo.set_disabled(str(user.id), True) + with pytest.raises(PermissionError): + await resolver() + finally: + await engine.dispose() diff --git a/backend/tests/support/batch_owner.py b/backend/tests/support/batch_owner.py new file mode 100644 index 0000000000..3c39c7bc33 --- /dev/null +++ b/backend/tests/support/batch_owner.py @@ -0,0 +1,67 @@ +"""Model/tool boundaries for the real Gateway batch-owner recovery test.""" + +import json +import os +import time +from pathlib import Path + +from langchain.tools import ToolRuntime, tool +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult + + +def _record(name, item, runtime): + record = {"tool": name, "item": item, "role": runtime.context.get("user_role"), "user_id": runtime.context.get("user_id"), "pid": os.getpid()} + path = Path(os.environ["BATCH_OWNER_PROBE_DIR"]) / "calls.jsonl" + with path.open("a", encoding="utf-8") as log: + log.write(json.dumps(record) + "\n") + return "Completed " + name + + +@tool +def owner_normal(item: str, runtime: ToolRuntime) -> str: + """Record an operation available to every configured role.""" + return _record("normal", item, runtime) + + +@tool +def owner_admin(item: str, runtime: ToolRuntime) -> str: + """Record an operation restricted by the host RBAC policy to administrators.""" + return _record("admin", item, runtime) + + +class OwnerModel(BaseChatModel): + @property + def _llm_type(self): + return "offline-batch-owner" + + def bind_tools(self, tools, **kwargs): + return self + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): + if any(isinstance(message, ToolMessage) for message in messages): + answer = AIMessage(content="Done") + elif any(isinstance(message, HumanMessage) and "OWNER_CHILD" in str(message.content) for message in messages): + root = Path(os.environ["BATCH_OWNER_PROBE_DIR"]) + item = "first" if any("OWNER_CHILD:first" in str(message.content) for message in messages) else "second" + (root / f"reached-{item}").write_text(str(os.getpid()), encoding="utf-8") + deadline = time.monotonic() + 60 + while item == "first" and (root / "hold").exists(): + if time.monotonic() >= deadline: + raise TimeoutError("Batch owner test barrier not released") + time.sleep(0.02) + answer = AIMessage(content="", tool_calls=[{"name": name, "args": {"item": item}, "id": name, "type": "tool_call"} for name in ("owner_admin", "owner_normal")]) + else: + answer = AIMessage( + content="", + tool_calls=[ + { + "name": "batch_task", + "args": {"title": "Owner recovery", "subagent_type": "general-purpose", "items": [{"key": key, "prompt": f"OWNER_CHILD:{key}: run both configured observations"} for key in ("first", "second")]}, + "id": "batch", + "type": "tool_call", + } + ], + ) + return ChatResult(generations=[ChatGeneration(message=answer)]) diff --git a/backend/tests/test_batch_owner_admission.py b/backend/tests/test_batch_owner_admission.py new file mode 100644 index 0000000000..0781b3af67 --- /dev/null +++ b/backend/tests/test_batch_owner_admission.py @@ -0,0 +1,123 @@ +"""Native capacity waits revalidate identity on the host's database loop.""" + +import asyncio +import importlib.util +import json +import sys +from pathlib import Path + +import pytest +from support.batch_owner import owner_admin, owner_normal + +from deerflow.authz.provider import Principal +from deerflow.config.app_config import AppConfig +from deerflow.config.subagent_batches_config import SubagentBatchesConfig +from deerflow.config.subagent_runtime_config import SubagentRuntimeConfig +from deerflow.subagents.batch_service import SubagentBatchService +from deerflow.subagents.capacity import SubagentExecutionCapacity +from deerflow.subagents.config import SubagentConfig + + +@pytest.fixture +def executor_module(monkeypatch, tmp_path): + # conftest installs an executor stub; load the real graph and worker loop. + path = Path(__file__).parents[1] / "packages/harness/deerflow/subagents/executor.py" + spec = importlib.util.spec_from_file_location("_batch_owner_executor", path) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + monkeypatch.setenv("DEER_FLOW_HOME", str(tmp_path / "home")) + monkeypatch.setenv("BATCH_OWNER_PROBE_DIR", str(tmp_path)) + yield module + module._shutdown_isolated_subagent_loop() + + +def _executor(module, resolver, capacity=None): + config = AppConfig.model_validate( + { + "models": [{"name": "owner-probe", "use": "support.batch_owner:OwnerModel", "model": "offline"}], + "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, + "summarization": {"enabled": False}, + "tool_search": {"enabled": False}, + "authorization": {"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"admin": {}, "user": {"tools": {"deny": ["owner_admin"]}}}}}}, + } + ) + return module.SubagentExecutor( + config=SubagentConfig(name="general-purpose", description="Owner probe", skills=[]), + tools=[owner_admin, owner_normal], + app_config=config, + parent_model="owner-probe", + thread_id="batch-owner-test", + user_id="owner", + user_role="admin", + oauth_provider="old", + oauth_id="old", + authz_attributes={"old_grant": True}, + execution_capacity=capacity, + principal_resolver=resolver, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["disabled", "different_owner", "fresh_profile", "unchanged"]) +async def test_identity_is_rechecked_after_capacity_wait(executor_module, tmp_path, change): + capacity = SubagentExecutionCapacity(SubagentRuntimeConfig(max_running=1)) + state = "queued" + calls = [] + + async def resolve(): + calls.append(state) + if state == "disabled": + raise PermissionError("Owner disabled") + return Principal(user_id="different" if state == "different_owner" else "owner", role="user" if state == "fresh_profile" else "admin", oauth_provider="oidc", oauth_id="current") + + executor = _executor(executor_module, resolve, capacity) + task = None + try: + async with capacity.slot(): + task = asyncio.create_task(executor._aexecute("OWNER_CHILD:second")) + async with asyncio.timeout(5): + while capacity.snapshot().queued != 1: + await asyncio.sleep(0) + assert not calls and not (tmp_path / "reached-second").exists() + state = change + result = await asyncio.wait_for(task, 20) + assert calls == [change] + assert result.execution_done_event.is_set() + if change in ("disabled", "different_owner"): + assert result.status == executor_module.SubagentStatus.FAILED + assert result.started_at is None + assert not (tmp_path / "calls.jsonl").exists() + else: + assert result.status == executor_module.SubagentStatus.COMPLETED, result.error + observed = [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text(encoding="utf-8").splitlines()] + assert {call["tool"] for call in observed} == ({"normal"} if change == "fresh_profile" else {"normal", "admin"}) + assert executor.oauth_provider == "oidc" and executor.oauth_id == "current" + assert executor.authz_attributes == {} + assert capacity.snapshot().running == capacity.snapshot().queued == 0 + finally: + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_native_worker_resolves_on_gateway_loop(executor_module, tmp_path): + host_loop = asyncio.get_running_loop() + lookups = [] + + async def resolve(batch): + assert asyncio.get_running_loop() is host_loop + lookups.append(batch["user_id"]) + return Principal(user_id=batch["user_id"], role="user") + + service = SubagentBatchService(repository=None, config=SubagentBatchesConfig(), runtime_config=SubagentRuntimeConfig(), resolve_principal=resolve) + resolver = service._bind_principal_resolver({"user_id": "owner"}) + await resolver() # Pre-discovery check on the host loop. + executor = _executor(executor_module, resolver) + pending = executor_module.run_on_isolated_subagent_loop(executor._aexecute("OWNER_CHILD:second")) + result = await asyncio.wait_for(asyncio.wrap_future(pending), 20) + assert result.status == executor_module.SubagentStatus.COMPLETED, result.error + assert lookups == ["owner", "owner"] + observed = [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text(encoding="utf-8").splitlines()] + assert {call["tool"] for call in observed} == {"normal"} diff --git a/backend/tests/test_batch_owner_identity.py b/backend/tests/test_batch_owner_identity.py new file mode 100644 index 0000000000..2512da78ce --- /dev/null +++ b/backend/tests/test_batch_owner_identity.py @@ -0,0 +1,131 @@ +"""Gateway batch identity resolves against the current account repository.""" + +from traceback import format_exception + +import pytest +import pytest_asyncio +from sqlalchemy import delete +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.gateway.auth.models import User +from app.gateway.auth.repositories.sqlite import SQLiteUserRepository +from app.subagent_batches.identity import resolve_batch_principal +from deerflow.persistence.user.model import UserRow + + +@pytest_asyncio.fixture +async def accounts(tmp_path, monkeypatch): + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/users.db") + async with engine.begin() as connection: + await connection.run_sync(UserRow.__table__.create) + sessions = async_sessionmaker(engine, expire_on_commit=False) + repo = SQLiteUserRepository(sessions) + owner = await repo.create_user(User(email="owner@example.com", system_role="admin", oauth_provider="oidc", oauth_id="current")) + await repo.create_user(User(email="root@example.com", system_role="admin")) + monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0") + try: + yield repo, owner, sessions + finally: + await engine.dispose() + + +def _batch(owner, **spec): + return {"id": "batch-1", "user_id": owner, "execution_spec": spec} + + +@pytest.mark.asyncio +async def test_current_profile_replaces_stored_oauth_and_attributes(accounts): + repo, owner, _ = accounts + principal = await resolve_batch_principal(_batch(str(owner.id), user_role="admin", oauth_provider="old", oauth_id="old", authz_attributes={"admin": True}, is_internal=True, channel_user_id="im-user"), load_user=repo.get_user_by_id) + assert (principal.user_id, principal.role, principal.oauth_provider, principal.oauth_id) == (str(owner.id), "admin", "oidc", "current") + assert principal.is_internal and principal.channel_user_id == "im-user" + assert principal.attributes == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["disabled", "demoted", "reset", "deleted"]) +@pytest.mark.parametrize("internal", [False, True]) +async def test_revoked_local_owner_cannot_fall_back_to_internal(accounts, change, internal): + repo, owner, sessions = accounts + batch = _batch(str(owner.id), user_role="admin", is_internal=internal) + if change == "disabled": + await repo.set_disabled(str(owner.id), True) + elif change == "demoted": + await repo.update_system_role(str(owner.id), "user") + elif change == "reset": + await repo.update_user(owner.model_copy(update={"needs_setup": True})) + else: + async with sessions() as session: + await session.execute(delete(UserRow).where(UserRow.id == str(owner.id))) + await session.commit() + with pytest.raises(PermissionError): + await resolve_batch_principal(batch, load_user=repo.get_user_by_id) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("original,current", [("user", "admin"), ("reviewer", "operator"), (None, "admin")]) +async def test_role_changes_do_not_invent_a_role_hierarchy(accounts, original, current): + repo, owner, _ = accounts + await repo.update_system_role(str(owner.id), current) + with pytest.raises(PermissionError): + await resolve_batch_principal(_batch(str(owner.id), user_role=original), load_user=repo.get_user_by_id) + + +@pytest.mark.asyncio +async def test_unmapped_internal_owner_keeps_default_role_without_stale_claims(accounts): + repo, _, _ = accounts + batch = _batch("external-owner", is_internal=True, oauth_provider="old", oauth_id="old", authz_attributes={"admin": True}) + principal = await resolve_batch_principal(batch, load_user=repo.get_user_by_id) + assert principal.user_id == "external-owner" and principal.is_internal + assert principal.role is None and principal.oauth_provider is None and principal.oauth_id is None + assert principal.attributes == {} + batch["execution_spec"]["is_internal"] = False + with pytest.raises(PermissionError): + await resolve_batch_principal(batch, load_user=repo.get_user_by_id) + + +@pytest.mark.asyncio +async def test_auth_disabled_only_preserves_explicit_synthetic_owner(accounts, monkeypatch): + repo, owner, _ = accounts + monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "1") + monkeypatch.delenv("DEER_FLOW_ENV", raising=False) + monkeypatch.delenv("ENVIRONMENT", raising=False) + batch = _batch("default", user_role="admin") + principal = await resolve_batch_principal(batch, load_user=repo.get_user_by_id) + assert principal.user_id == "default" and principal.role == "admin" + await repo.set_disabled(str(owner.id), True) + with pytest.raises(PermissionError): + await resolve_batch_principal(_batch(str(owner.id), user_role="admin"), load_user=repo.get_user_by_id) + monkeypatch.setenv("DEER_FLOW_AUTH_DISABLED", "0") + with pytest.raises(PermissionError): + await resolve_batch_principal(batch, load_user=repo.get_user_by_id) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["missing_loader", "exception", "mismatch"]) +async def test_unavailable_identity_fails_closed(accounts, failure, caplog): + _, owner, _ = accounts + batch = _batch(str(owner.id), user_role="admin", is_internal=True) + fake_password = "fixture-db-password" + statement = "SELECT id FROM users WHERE id = :id" + + async def load_user(user_id): + if failure == "exception": + raise OperationalError(statement, {"id": user_id}, RuntimeError(f"postgresql://test:{fake_password}@test-db/deerflow")) + return User(email="different@example.com", system_role="admin") + + with pytest.raises(PermissionError) as exc: + await resolve_batch_principal(batch, load_user=None if failure == "missing_loader" else load_user) + assert fake_password not in str(exc.value) + if failure == "exception": + records = [record for record in caplog.records if record.name == "app.subagent_batches.identity"] + assert len(records) == 1 + record = records[0] + assert batch["id"] in record.getMessage() + assert "OperationalError" in record.getMessage() + assert record.exc_info is None + diagnostics = caplog.text + "".join(format_exception(exc.value)) + assert fake_password not in diagnostics + assert str(owner.id) not in diagnostics + assert statement not in diagnostics diff --git a/backend/tests/test_batch_owner_restart.py b/backend/tests/test_batch_owner_restart.py new file mode 100644 index 0000000000..3ad94cac42 --- /dev/null +++ b/backend/tests/test_batch_owner_restart.py @@ -0,0 +1,219 @@ +"""Queued and recovered batches must honor current account state and role.""" + +import json +import os +import signal +import socket +import subprocess +import sys +import time +import uuid +from contextlib import contextmanager +from pathlib import Path + +import httpx +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url +from sqlalchemy.schema import CreateSchema, DropSchema + +from deerflow.persistence.postgres_schema import dsn_with_search_path + +ROOT = Path(__file__).resolve().parents[2] +pytestmark = pytest.mark.skipif(os.name != "posix", reason="Tests SIGKILL recovery in separate Gateway processes") + + +def _until(check, timeout=40): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = check() + if result: + return result + time.sleep(0.05) + raise AssertionError(f"Timed out waiting for {check}") + + +@pytest.fixture(params=["sqlite", "postgres"]) +def database(request, tmp_path): + schema = None + if request.param == "postgres": + uri = os.environ.get("TEST_POSTGRES_URI") + if not uri: + pytest.skip("requires TEST_POSTGRES_URI (real Gateway restart)") + pytest.importorskip("psycopg") + pytest.importorskip("asyncpg") + schema = "test_batch_owner_" + uuid.uuid4().hex + config = {"backend": "postgres", "postgres_url": uri, "postgres_schema": schema} + engine = create_engine(make_url(dsn_with_search_path(uri, schema)).set(drivername="postgresql+psycopg")) + else: + config = {"backend": "sqlite", "sqlite_dir": str(tmp_path / "db")} + engine = create_engine(f"sqlite:///{tmp_path}/db/deerflow.db") + + def read(statement, **params): + with engine.connect() as connection: + return connection.execute(text(statement), params).all() + + try: + if schema: + with engine.begin() as connection: + connection.execute(CreateSchema(schema)) + yield config, read + finally: + try: + if schema: + with engine.begin() as connection: + connection.execute(DropSchema(schema, cascade=True, if_exists=True)) + finally: + engine.dispose() + + +@contextmanager +def _gateway(root, client, port, generation): + env = {key: os.environ[key] for key in ("PATH", "HOME", "TMPDIR", "LANG") if key in os.environ} + env.update( + PYTHONPATH=os.pathsep.join(str(ROOT / path) for path in ("backend/packages/harness", "backend/packages/extension-api", "backend", "backend/tests")), + DEER_FLOW_HOME=str(root / "home"), + DEER_FLOW_PROJECT_ROOT=str(ROOT), + DEER_FLOW_CONFIG_PATH=str(root / "config.yaml"), + DEER_FLOW_EXTENSIONS_CONFIG_PATH=str(root / "extensions.json"), + DEER_FLOW_ENV_FILE=str(root / "empty.env"), + BATCH_OWNER_PROBE_DIR=str(root), + AUTH_JWT_SECRET="local-batch-owner-test-secret-1234567890", + DEER_FLOW_AUTH_DISABLED="0", + NO_PROXY="127.0.0.1,localhost", + GATEWAY_WORKERS="1", + WEB_CONCURRENCY="1", + PYTHONIOENCODING="utf-8", + ) + log_path = root / f"gateway-{generation}.log" + with log_path.open("w", encoding="utf-8") as log: + process = subprocess.Popen([sys.executable, "-m", "uvicorn", "app.gateway.app:app", "--host", "127.0.0.1", "--port", str(port), "--log-level", "warning"], cwd=root, env=env, stdout=log, stderr=subprocess.STDOUT) + try: + + def healthy(): + assert process.poll() is None, log_path.read_text(encoding="utf-8")[-8000:] + try: + return client.get("/health").status_code == 200 + except httpx.TransportError: + return False + + _until(healthy) + yield process + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + +@pytest.mark.parametrize("dispatch", ["batch_queue", "restart"]) +@pytest.mark.parametrize("change", ["disabled", "demoted", "promoted", "unchanged_admin", "unchanged_user"]) +def test_batch_revalidates_owner_at_execution(tmp_path, database, dispatch, change): + database_config, read = database + config = { + "models": [{"name": "owner-probe", "use": "support.batch_owner:OwnerModel", "model": "offline", "supports_tool_calling": True}], + "sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}, + "database": database_config, + "tools": [{"name": name, "group": "probe", "use": "support.batch_owner:" + name} for name in ("owner_normal", "owner_admin")], + "tool_groups": [{"name": "probe"}], + "title": {"enabled": False}, + "memory": {"enabled": False}, + "summarization": {"enabled": False}, + "tool_search": {"enabled": False}, + "hitl_approval": {"enabled": False}, + "subagent_batches": {"enabled": True, "poll_interval_seconds": 0.1, "lease_seconds": 10, "default_max_running_items": 1, "max_attempts": 3 if dispatch == "restart" else 1}, + "subagent_runtime": {"max_running": 1}, + "authorization": {"enabled": True, "provider": {"use": "deerflow.authz.rbac:RbacAuthorizationProvider", "config": {"roles": {"admin": {}, "user": {"tools": {"deny": ["owner_admin"]}}}}}}, + } + (tmp_path / "config.yaml").write_text(json.dumps(config), encoding="utf-8") + (tmp_path / "extensions.json").write_text("{}", encoding="utf-8") + (tmp_path / "empty.env").touch() + (tmp_path / "hold").touch() + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + with httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=30) as admin, httpx.Client(base_url=f"http://127.0.0.1:{port}", trust_env=False, timeout=30) as actor: + + def request(client, method, path, *, expected=200, **kwargs): + response = client.request(method, path, **kwargs) + assert response.status_code == expected, response.text + return response.json() + + with _gateway(tmp_path, admin, port, 1) as first: + for client, route, email in ((admin, "initialize", "admin@example.com"), (actor, "register", "actor@example.com")): + account = request(client, "POST", "/api/v1/auth/" + route, expected=201, json={"email": email, "password": "local-batch-owner-password-123"}) + client.headers["X-CSRF-Token"] = client.cookies["csrf_token"] + owner = account["id"] + original_role = "user" if change in ("unchanged_user", "promoted") else "admin" + if original_role == "admin": + request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"system_role": "admin"}) + thread = request(actor, "POST", "/api/threads", json={})["thread_id"] + request( + actor, + "POST", + f"/api/threads/{thread}/runs/wait", + json={"assistant_id": "lead_agent", "input": {"messages": [{"role": "user", "content": "Submit the owner recovery batch"}]}, "context": {"mode": "pro", "subagent_enabled": True}}, + ) + _until(lambda: (tmp_path / "reached-first").exists()) + assert not (tmp_path / "reached-second").exists() + assert not (tmp_path / "calls.jsonl").exists() + batch_id, stored_spec = read("SELECT id, execution_spec FROM subagent_batches WHERE user_id = :owner", owner=owner)[0] + spec = json.loads(stored_spec) if isinstance(stored_spec, str) else stored_spec + assert spec["user_role"] == original_role + + def items(): + return read("SELECT item_key, status, attempt, error FROM subagent_batch_items WHERE batch_id = :batch_id ORDER BY position", batch_id=batch_id) + + _until(lambda: len(rows := items()) == 2 and rows[1][2] == 0) + if change == "disabled": + request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"disabled": True}) + request(actor, "GET", "/api/v1/auth/me", expected=401) + elif change in ("demoted", "promoted"): + role = "user" if change == "demoted" else "admin" + request(admin, "PATCH", f"/api/v1/admin/users/{owner}", json={"system_role": role}) + assert request(actor, "GET", "/api/v1/auth/me")["system_role"] == role + + def verify(pid): + def terminal(): + rows = items() + return rows if len(rows) == 2 and all(row[1] in ("succeeded", "failed", "cancelled") for row in rows) else None + + rows = _until(terminal) + path = tmp_path / "calls.jsonl" + calls = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] if path.exists() else [] + # Preserve observable results before the PostgreSQL schema is + # dropped; never manufacture task state for recovery. + (tmp_path / "evidence.json").write_text( + json.dumps( + {"backend": database_config["backend"], "change": change, "dispatch": dispatch, "old_pid": first.pid, "new_pid": pid, "old_exit": first.returncode, "items": [dict(row._mapping) for row in rows], "calls": calls}, + indent=2, + ), + encoding="utf-8", + ) + assert all(call["pid"] == pid and call["user_id"] == owner for call in calls) + if dispatch == "restart": + assert rows[0][2] >= 2 + else: + assert rows[1][2] == 1 + assert rows[0][1] == "succeeded" # Already admitted, outside the revocation gate. + observed = calls if dispatch == "restart" else [call for call in calls if call["item"] == "second"] + if change in ("disabled", "demoted", "promoted"): + assert rows[1][1] == "failed", (rows, calls) + assert not observed + assert not (tmp_path / "reached-second").exists() + else: + assert all(row[1] == "succeeded" for row in rows), (rows, calls) + assert all(call["role"] == original_role for call in observed) + assert {call["tool"] for call in observed} == ({"admin", "normal"} if original_role == "admin" else {"normal"}) + + if dispatch != "restart": + (tmp_path / "hold").unlink() + verify(first.pid) + return + first.kill() + first.wait(timeout=10) + assert first.returncode == -signal.SIGKILL + + (tmp_path / "hold").unlink() + with _gateway(tmp_path, admin, port, 2) as second: + assert second.pid != first.pid + verify(second.pid)