deer-flow/backend/tests/test_skill_mutation_host_lifecycle.py

"""The host owns recovery even when every extension is disabled."""

import asyncio
import os
import threading
from contextlib import contextmanager
from types import SimpleNamespace

import pytest
import pytest_asyncio
from sqlalchemy import create_engine
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.orm import Session

from deerflow.config.database_config import DatabaseConfig
from deerflow.config.run_events_config import RunEventsConfig
from deerflow.extensions.host_access import HostAccess
from deerflow.extensions.loader import ExtensionSpec
from deerflow.persistence.base import Base
from deerflow.persistence.run import RunRepository
from deerflow.persistence.skill_mutations.model import SkillOwnerRow, SkillProposalRow
from deerflow.persistence.user.model import UserRow
from deerflow.runtime.events.store.db import DbRunEventStore

pytestmark = pytest.mark.skipif(os.name != "posix", reason="host skill mutation tests require POSIX filesystem primitives")


@pytest_asyncio.fixture
async def durable(tmp_path):
    database = DatabaseConfig(backend="sqlite", sqlite_dir=str(tmp_path))
    sync = create_engine(database.app_sync_sqlalchemy_url)
    Base.metadata.create_all(sync)
    with Session(sync) as session, session.begin():
        session.add(UserRow(id="owner", email="owner@example.test"))
    sync.dispose()
    engine = create_async_engine(database.app_sqlalchemy_url)
    sf = async_sessionmaker(engine, expire_on_commit=False)
    yield database, sf
    await engine.dispose()


def config(database, plugins=()):
    return SimpleNamespace(database=database, run_events=RunEventsConfig(backend="db"), plugins=plugins)


def granted():
    return ExtensionSpec(
        use="example:install",
        name="evolution",
        host_access=HostAccess.model_validate(
            {
                "evidence": {"owners": ["owner"]},
                "skill_mutations": {"owners": ["owner"], "operations": ["stage", "check", "commit", "revert"], "trigger_agents": ["lead_agent"], "target_skills": ["example"], "topology": "single_host_local"},
            }
        ),
    )


@pytest.mark.asyncio
async def test_default_has_no_mutation_runtime_or_worker_pool(durable, tmp_path):
    from deerflow.extensions.host_capabilities import HostCapabilities

    database, sf = durable
    host = HostCapabilities(config(database), RunRepository(sf), DbRunEventStore(sf), root=tmp_path)
    await host.start()
    try:
        assert host.bindings == {}
        assert host.runtime is None
        assert host.recovery is None
    finally:
        await host.close()


@pytest.mark.asyncio
async def test_disabled_plugin_still_recovers_enrolled_owners(durable, tmp_path):
    from deerflow.extensions.host_capabilities import HostCapabilities

    database, sf = durable
    async with sf.begin() as session:
        session.add(SkillOwnerRow(owner_id="owner", generation=4, deleting=False))
    events = []

    class Recovery:
        def __init__(self, runtime, storage_factory):
            assert runtime.owners == frozenset({"owner"})
            events.append("constructed")

        def recover_all(self):
            events.append("recovered")

        def collect_garbage(self):
            events.append("collected")

    host = HostCapabilities(config(database), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, recovery_factory=Recovery)
    await host.start()
    try:
        assert host.bindings == {}
        assert events == ["constructed", "recovered", "collected"]
        assert host.runtime.owners == frozenset({"owner"})
    finally:
        await host.close()
    assert host.workers._closed


@pytest.mark.asyncio
async def test_source_binding_and_independent_evidence_scope(durable, tmp_path):
    from deerflow.extensions.host_capabilities import HostCapabilities

    database, sf = durable
    seen = []

    class Recovery:
        def __init__(self, *args):
            pass

        def recover_all(self):
            pass

        def collect_garbage(self):
            pass

    def service(binding, **kwargs):
        seen.append((binding, kwargs))
        return object()

    host = HostCapabilities(config(database, [granted()]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, recovery_factory=Recovery, service_factory=service)
    await host.start()
    try:
        reader, mutations = host.bindings["example:install"]
        assert reader is seen[0][1]["evidence"]
        assert reader._owners == frozenset({"owner"})
        assert mutations is not None
        assert seen[0][0].plugin_id != "evolution"
    finally:
        await host.close()


@pytest.mark.asyncio
async def test_missing_topology_refuses_mutation_startup(durable, tmp_path):
    from deerflow_extension_api import HostCapabilityError

    from deerflow.extensions.host_capabilities import HostCapabilities

    database, sf = durable
    spec = granted()
    spec = spec.model_copy(update={"host_access": spec.host_access.model_copy(update={"skill_mutations": spec.host_access.skill_mutations.model_copy(update={"topology": None})})})
    host = HostCapabilities(config(database, [spec]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path)
    try:
        with pytest.raises(HostCapabilityError, match="UNSUPPORTED_TOPOLOGY"):
            await host.start()
    finally:
        await host.close()


@pytest.mark.asyncio
async def test_cancelled_startup_drains_worker_before_releasing_topology(durable, tmp_path):
    from deerflow.extensions.host_capabilities import HostCapabilities
    from deerflow.skills.mutations.topology import PublicationTopology

    database, sf = durable
    entered, release = threading.Event(), threading.Event()

    class Recovery:
        def __init__(self, *args):
            pass

        def recover_all(self):
            entered.set()
            assert release.wait(5)

        def collect_garbage(self):
            pass

    host = HostCapabilities(config(database, [granted()]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, recovery_factory=Recovery)
    task = asyncio.create_task(host.start())
    assert await asyncio.to_thread(entered.wait, 5)
    task.cancel()
    with pytest.raises(asyncio.CancelledError):
        await task
    closing = asyncio.create_task(host.close())
    await asyncio.sleep(0)
    assert not closing.done()
    assert host._topology._fd is not None
    release.set()
    await closing
    next_host = PublicationTopology("sqlite", tmp_path)
    next_host.acquire()
    next_host.close()


@pytest.mark.asyncio
async def test_read_only_evidence_reports_unsupported_without_mutation_pool(tmp_path):
    from deerflow_extension_api import HostCapabilityError

    from deerflow.extensions.host_capabilities import HostCapabilities
    from deerflow.runtime.events.store.memory import MemoryRunEventStore
    from deerflow.runtime.runs.store.memory import MemoryRunStore

    spec = ExtensionSpec(use="example:install", name="reader", host_access=HostAccess.model_validate({"evidence": {"owners": ["owner"]}}))
    host = HostCapabilities(config(DatabaseConfig(), [spec]), MemoryRunStore(), MemoryRunEventStore(), root=tmp_path)
    await host.start()
    try:
        reader, mutations = host.bindings["example:install"]
        assert mutations is None
        assert host.workers is None
        with pytest.raises(HostCapabilityError, match="UNSUPPORTED"):
            await reader.list_changed_runs()
    finally:
        await host.close()


@pytest.mark.asyncio
async def test_mutation_owner_without_evidence_grant_is_rejected(durable, tmp_path):
    from deerflow_extension_api import HostCapabilityError

    from deerflow.extensions.host_capabilities import HostCapabilities

    database, sf = durable
    spec = granted()
    spec.host_access = spec.host_access.model_copy(update={"evidence": HostAccess().evidence})
    host = HostCapabilities(config(database, [spec]), RunRepository(sf), DbRunEventStore(sf), root=tmp_path)
    try:
        with pytest.raises(HostCapabilityError, match="INVALID_GRANT"):
            await host.start()
        assert host.workers is None
    finally:
        await host.close()


@pytest.mark.asyncio
@pytest.mark.parametrize("phase", ["startup", "maintenance"])
@pytest.mark.parametrize("failure", ["lock-directory", "lock-timeout"])
async def test_owner_gc_failure_does_not_block_host_or_healthy_owner(durable, tmp_path, monkeypatch, phase, failure):
    from deerflow.config.paths import Paths
    from deerflow.extensions.host_capabilities import HostCapabilities
    from deerflow.skills.mutations import recovery as recovery_module
    from deerflow.skills.projection import get_skill_projection_paths
    from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage

    database, sf = durable
    monkeypatch.setattr("deerflow.config.paths._paths", Paths(base_dir=tmp_path / "home"))
    owners = ("a-broken", "z-healthy")
    async with sf.begin() as session:
        for owner_id in owners:
            session.add(UserRow(id=owner_id, email=f"{owner_id}@example.test"))
            session.add(SkillOwnerRow(owner_id=owner_id, generation=0, deleting=False))

    def storage_factory(owner_id):
        return UserScopedSkillStorage(owner_id, host_path=str(tmp_path / "skills"))

    async def expire_proposals():
        async with sf.begin() as session:
            for owner_id in owners:
                session.add(
                    SkillProposalRow(
                        proposal_id=owner_id,
                        plugin_id="removed-plugin",
                        owner_id=owner_id,
                        target_ref=owner_id,
                        name="example",
                        base_revision={},
                        sources=[],
                        candidate_hash="unused",
                        package_blob=b"baseline",
                        candidate_blob=b"candidate",
                        state="PENDING",
                        expires_at=0,
                        idempotency_key=owner_id,
                        request_hash="unused",
                    )
                )

    def break_owner_lock():
        if failure == "lock-directory":
            root = get_skill_projection_paths(storage_factory(owners[0])).custom.parent
            lock_path = root.parent / f".{root.name}.projection.lock"
            # During maintenance an earlier successful acquire made a file.
            if lock_path.is_file():
                lock_path.unlink()
            lock_path.mkdir(parents=True)
        else:
            original_lock = recovery_module.skill_projection_read_lock

            @contextmanager
            def blocked_lock(storage, **kwargs):
                if storage.user_id == owners[0]:
                    raise TimeoutError("owner lock unavailable")
                with original_lock(storage, **kwargs):
                    yield

            monkeypatch.setattr(recovery_module, "skill_projection_read_lock", blocked_lock)

    # No enabled plugin: persisted host state alone must remain recoverable.
    host = HostCapabilities(config(database), RunRepository(sf), DbRunEventStore(sf), root=tmp_path, storage_factory=storage_factory)
    try:
        if phase == "startup":
            await expire_proposals()
            break_owner_lock()
        await host.start()
        if phase == "maintenance":
            await expire_proposals()
            break_owner_lock()
            await host.workers.run(lambda _: (host.recovery.recover_all(), host.recovery.collect_garbage()))
        assert host.bindings == {}
        assert host.recovery is not None
        assert not host._maintenance.done()
        with host.runtime.repository.sessions() as session:
            broken = session.get(SkillProposalRow, owners[0])
            healthy = session.get(SkillProposalRow, owners[1])
            assert broken.state == "PENDING"
            assert broken.package_blob == b"baseline" and broken.candidate_blob == b"candidate"
            assert healthy.state == "EXPIRED"
            assert healthy.package_blob is None and healthy.candidate_blob is None
    finally:
        await host.close()