deer-flow/backend/tests/test_lark_auth_lifecycle_failures.py

"""Offline failure injection at the login process and filesystem boundaries."""

from __future__ import annotations

import errno
import os
import subprocess
import sys
import threading
from pathlib import Path
from types import SimpleNamespace

import pytest

from deerflow.config import paths as paths_module
from deerflow.config.paths import Paths
from deerflow.integrations import lark_cli

pytestmark = pytest.mark.filterwarnings("error::pytest.PytestUnhandledThreadExceptionWarning")


class _LoginProcess:
    pid = None

    def __init__(self):
        self.released = threading.Event()
        self.killed = False

    def communicate(self, timeout=None):
        assert self.released.wait(10), "test did not release the login process"
        return "{}", ""

    @property
    def returncode(self):
        return -9 if self.killed else 0

    def poll(self):
        return self.returncode if self.released.is_set() else None

    def kill(self):
        self.killed = True
        self.released.set()


@pytest.fixture
def login_flow(monkeypatch, tmp_path):
    monkeypatch.setattr(paths_module, "_paths", Paths(base_dir=tmp_path / "home"))
    monkeypatch.setattr(lark_cli, "_require_lark_cli_path", lambda: "synthetic-lark-cli")
    monkeypatch.setattr(lark_cli, "LARK_AUTH_COMPLETE_MIN_WAIT_SECONDS", 0)
    spawned = []

    def spawn(*_args, **_kwargs):
        process = _LoginProcess()
        spawned.append(process)
        return process

    monkeypatch.setattr(lark_cli, "_spawn_lark_login_process", spawn)
    with lark_cli._lark_credential_lock("alice"):
        generation = lark_cli._advance_lark_flow_generation_locked("alice")
    yield generation, spawned
    for process in spawned:
        process.kill()
    lark_cli._clear_lark_login_runs_for_tests()


@pytest.mark.parametrize("operation", ["mkstemp", "write", "chmod", "publish"])
def test_result_io_failure_is_terminal_for_local_and_foreign_polls(monkeypatch, login_flow, operation):
    generation, spawned = login_flow
    with pytest.raises(TimeoutError):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    injected = threading.Event()

    def is_result_temp(path):
        return Path(path).name.startswith("..login-result-")

    def fail():
        injected.set()
        raise OSError(errno.EIO, "synthetic result I/O failure")

    if operation == "mkstemp":
        original = lark_cli.tempfile.mkstemp

        def mkstemp(*args, **kwargs):
            if kwargs.get("prefix", "").startswith("..login-result-"):
                fail()
            return original(*args, **kwargs)

        monkeypatch.setattr(lark_cli.tempfile, "mkstemp", mkstemp)
    elif operation == "publish":

        def fail_publication(original):
            def publish(source, destination, *args, **kwargs):
                if Path(destination).name.startswith(".login-result-"):
                    fail()
                return original(source, destination, *args, **kwargs)

            return publish

        monkeypatch.setattr(os, "link", fail_publication(os.link))
        monkeypatch.setattr(os, "replace", fail_publication(os.replace))
    else:
        name = "write_text" if operation == "write" else "chmod"
        original = getattr(Path, name)

        def file_operation(path, *args, **kwargs):
            if is_result_temp(path):
                fail()
            return original(path, *args, **kwargs)

        monkeypatch.setattr(Path, name, file_operation)

    spawned[0].released.set()
    with pytest.raises(ValueError, match="Restart the authorization flow"):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=5)
    assert injected.is_set(), "failure injection was not reached"

    # A fresh worker has no registry entry; the shared terminal receipt must
    # reject this code instead of starting a second login.
    with lark_cli._LARK_LOGIN_RUNS_LOCK:
        run = lark_cli._LARK_LOGIN_RUNS.pop(("alice", generation))
    run.thread.join(5)
    assert not run.thread.is_alive()
    with pytest.raises(ValueError, match="Restart the authorization flow"):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    assert len(spawned) == 1


def test_lost_result_and_fallback_are_bounded_without_redeeming_again(monkeypatch, login_flow):
    generation, spawned = login_flow
    with pytest.raises(TimeoutError):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    run = lark_cli._LARK_LOGIN_RUNS[("alice", generation)]
    failed_destinations = set()
    original_replace = os.replace

    def replace(source, destination, *args, **kwargs):
        name = Path(destination).name
        if name.startswith((".login-result-", ".login-run-")):
            failed_destinations.add(name.split("-")[1])
            raise OSError(errno.ENOSPC, "synthetic full credential volume")
        return original_replace(source, destination, *args, **kwargs)

    with monkeypatch.context() as storage_failure:
        storage_failure.setattr(os, "replace", replace)
        spawned[0].released.set()
        with pytest.raises(ValueError, match="Restart the authorization flow"):
            lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=5)
        run.thread.join(5)
        assert not run.thread.is_alive()
    assert failed_destinations == {"result", "run"}
    with lark_cli._LARK_LOGIN_RUNS_LOCK:
        lark_cli._LARK_LOGIN_RUNS.pop(("alice", generation))

    # Storage recovers, but neither result nor failure receipt made it to disk.
    # A new worker may wait one publication grace, then must reject the old code.
    clock = [lark_cli.time.time()]
    monkeypatch.setattr(lark_cli, "time", SimpleNamespace(time=lambda: clock[0], monotonic=lark_cli.time.monotonic, sleep=lark_cli.time.sleep))
    with pytest.raises(TimeoutError):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    clock[0] += lark_cli._LARK_LOGIN_OWNER_ALIVE_GRACE_SECONDS + 1
    with pytest.raises(ValueError, match="Restart the authorization flow"):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    assert len(spawned) == 1
    digest = lark_cli._lark_login_code_digest("code-A")
    assert lark_cli._lark_login_marker_path("alice", generation, digest).is_file()
    assert not lark_cli._lark_login_result_path("alice", generation, digest).exists()


def test_foreign_poll_rereads_result_after_marker_disappears(monkeypatch, login_flow):
    generation, spawned = login_flow
    with pytest.raises(TimeoutError):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    run = lark_cli._LARK_LOGIN_RUNS[("alice", generation)]
    digest = lark_cli._lark_login_code_digest("code-A")
    marker = lark_cli._lark_login_marker_path("alice", generation, digest)
    result = lark_cli._lark_login_result_path("alice", generation, digest)
    original_is_file = Path.is_file
    injected = threading.Event()

    def is_file(path):
        if path == marker and not injected.is_set():
            # The foreign poll has already read a missing result. Let the
            # real owner finish publishing before its next marker probe.
            assert not result.exists()
            injected.set()
            spawned[0].released.set()
            run.thread.join(10)
            assert not run.thread.is_alive()
            assert result.is_file() and not marker.exists()
        return original_is_file(path)

    monkeypatch.setattr(Path, "is_file", is_file)
    assert lark_cli._await_lark_login_result("alice", generation, "code-A", 0) == {"ok": True}
    assert injected.is_set(), "publication must straddle the foreign poll's two reads"
    assert len(spawned) == 1


@pytest.mark.parametrize("failure_step", ["reserve_marker", "marker_pid", "spawn", "thread_start"])
def test_failed_startup_reaps_cli_and_wakes_an_attached_poll(monkeypatch, tmp_path, failure_step):
    monkeypatch.setattr(paths_module, "_paths", Paths(base_dir=tmp_path / "home"))
    monkeypatch.setattr(lark_cli, "_require_lark_cli_path", lambda: "synthetic-lark-cli")
    monkeypatch.setattr(lark_cli, "LARK_AUTH_COMPLETE_MIN_WAIT_SECONDS", 0)
    with lark_cli._lark_credential_lock("alice"):
        generation = lark_cli._advance_lark_flow_generation_locked("alice")
    spawned, attached = [], []
    injected = threading.Event()

    def spawn(*_args, **_kwargs):
        if failure_step == "spawn":
            injected.set()
            raise OSError(errno.EAGAIN, "synthetic process launch failure")
        process = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=os.name != "nt")
        spawned.append(process)
        return process

    monkeypatch.setattr(lark_cli, "_spawn_lark_login_process", spawn)
    original_start = threading.Thread.start

    def fail_start(thread):
        if thread.name.startswith("lark-login-"):
            attached.append(lark_cli._LARK_LOGIN_RUNS[("alice", generation)])
            injected.set()
            raise RuntimeError("can't start new thread")
        return original_start(thread)

    if failure_step in {"reserve_marker", "marker_pid"}:
        original_write = lark_cli._write_json_file_atomic

        def fail_marker(path, payload):
            should_fail = failure_step == "reserve_marker" or payload.get("cli_pid") is not None
            if path.name.startswith(".login-run-") and "outcome" not in payload and should_fail:
                injected.set()
                raise OSError(errno.ENOSPC, "synthetic marker write failure")
            return original_write(path, payload)

        monkeypatch.setattr(lark_cli, "_write_json_file_atomic", fail_marker)
        error = OSError
    elif failure_step == "thread_start":
        monkeypatch.setattr(threading.Thread, "start", fail_start)
        error = RuntimeError
    else:
        error = OSError
    try:
        with pytest.raises(error):
            lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
        assert injected.is_set()
        assert ("alice", generation) not in lark_cli._LARK_LOGIN_RUNS
        assert all(process.poll() is not None for process in spawned), "startup failure leaked a real CLI"
        for run in attached:
            assert run.done.is_set(), "a concurrent waiter would remain blocked"
            assert isinstance(run.outcome, BaseException)
        with pytest.raises(ValueError):
            lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
        assert len(spawned) <= 1, "a failed startup must leave a terminal receipt"
    finally:
        with lark_cli._LARK_LOGIN_RUNS_LOCK:
            lark_cli._LARK_LOGIN_RUNS.pop(("alice", generation), None)
        for process in spawned:
            if process.poll() is None:
                process.kill()
            process.communicate(timeout=5)


@pytest.mark.parametrize("cleanup", ["temporary_result", "marker"])
def test_cleanup_failure_preserves_a_published_result(monkeypatch, login_flow, cleanup):
    generation, spawned = login_flow
    with pytest.raises(TimeoutError):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    original_unlink = Path.unlink
    injected = threading.Event()

    def unlink(path, *args, **kwargs):
        match = path.name.startswith("..login-result-") if cleanup == "temporary_result" else path.name.startswith(".login-run-")
        if match and threading.current_thread().name.startswith("lark-login-"):
            injected.set()
            raise PermissionError(errno.EACCES, "synthetic cleanup failure")
        return original_unlink(path, *args, **kwargs)

    monkeypatch.setattr(Path, "unlink", unlink)
    spawned[0].released.set()
    first = lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=5)
    assert injected.is_set()
    with lark_cli._LARK_LOGIN_RUNS_LOCK:
        run = lark_cli._LARK_LOGIN_RUNS.pop(("alice", generation))
    run.thread.join(5)
    assert run.outcome == {}
    second = lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    assert second.success == first.success
    assert len(spawned) == 1


@pytest.mark.parametrize("publication_fails", [False, True])
def test_supersession_receipt_survives_recovered_publication(monkeypatch, login_flow, publication_fails):
    generation, spawned = login_flow
    with pytest.raises(TimeoutError):
        lark_cli.complete_lark_auth("alice", None, generation=generation, device_code="code-A", wait_timeout_seconds=0)
    run = lark_cli._LARK_LOGIN_RUNS[("alice", generation)]
    injected = threading.Event()

    def fail_once(original):
        def publish(source, destination, *args, **kwargs):
            if publication_fails and Path(destination).name.startswith(".login-result-") and not injected.is_set():
                injected.set()
                raise OSError(errno.EIO, "synthetic one-shot publication failure")
            return original(source, destination, *args, **kwargs)

        return publish

    monkeypatch.setattr(os, "link", fail_once(os.link))
    monkeypatch.setattr(os, "replace", fail_once(os.replace))
    monkeypatch.setattr(lark_cli, "_run_lark_cli_json", lambda *_args, **_kwargs: {"verification_url": "https://example.com/verify", "device_code": "code-B"})
    lark_cli.start_lark_auth("alice", generation=generation)
    run.thread.join(5)
    assert not run.thread.is_alive()
    assert injected.is_set() is publication_fails
    assert isinstance(run.outcome, lark_cli.LarkFlowSupersededError)
    digest = lark_cli._lark_login_code_digest("code-A")
    payload = lark_cli._read_json_object_file(lark_cli._lark_login_result_path("alice", generation, digest))
    assert payload["kind"] == "superseded", "a recovered owner must adopt the fallback receipt"
    assert spawned[0].killed