deer-flow/backend/tests/test_remote_sandbox_backend.py

from __future__ import annotations

import importlib
import threading
import time
from unittest.mock import MagicMock

import pytest
import requests

import deerflow.skills.storage as storage_mod
from deerflow.community.aio_sandbox import remote_backend as remote_backend_mod
from deerflow.community.aio_sandbox.remote_backend import RemoteSandboxBackend
from deerflow.community.aio_sandbox.sandbox_info import SandboxInfo
from deerflow.skills.types import SkillCategory


class _StubResponse:
    def __init__(
        self,
        *,
        status_code: int = 200,
        payload: object | None = None,
        json_exc: Exception | None = None,
    ):
        self.status_code = status_code
        self._payload = {} if payload is None else payload
        self._json_exc = json_exc
        self.ok = 200 <= status_code < 400
        self.text = ""

    def raise_for_status(self) -> None:
        if self.status_code >= 400:
            raise requests.HTTPError(f"HTTP {self.status_code}")

    def json(self) -> object:
        if self._json_exc is not None:
            raise self._json_exc
        return self._payload


def test_list_running_delegates_to_provisioner_list(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    sandbox_info = SandboxInfo(sandbox_id="test-id", sandbox_url="http://localhost:8080")

    def mock_list():
        return [sandbox_info]

    monkeypatch.setattr(backend, "_provisioner_list", mock_list)

    assert backend.list_running() == [sandbox_info]


def test_reconciliation_discovery_does_not_trust_legacy_route_404(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(requests, "get", lambda *_args, **_kwargs: _StubResponse(status_code=404))

    result = backend.discover_for_reconciliation("old-id")

    assert result.status == "unknown"
    assert result.info is None


def test_reconciliation_discovery_accepts_pod_authoritative_absence(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(
        requests,
        "get",
        lambda *_args, **_kwargs: _StubResponse(
            payload={
                "sandbox_id": "old-id",
                "status": "absent",
                "backend_namespace": "namespace-uid-1",
            }
        ),
    )

    result = backend.discover_for_reconciliation("old-id")

    assert result.status == "absent"
    assert result.backend_namespace == "namespace-uid-1"


def test_reconciliation_discovery_preserves_transport_uncertainty(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def fail(*_args, **_kwargs):
        raise requests.ConnectionError("offline")

    monkeypatch.setattr(requests, "get", fail)

    result = backend.discover_for_reconciliation("old-id")

    assert result.status == "unknown"
    assert result.info is None


def test_reconciliation_discovery_accepts_exact_old_mount_contract(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(
        requests,
        "get",
        lambda *_args, **_kwargs: _StubResponse(
            payload={
                "sandbox_id": "old-id",
                "status": "found",
                "backend_namespace": "namespace-uid-1",
                "incarnation_id": "pod-uid-1",
                "sandbox_url": "http://sandbox-old",
                "thread_id": "thread-1",
                "user_id": "alice",
                "mount_contract_version": 1,
            }
        ),
    )

    result = backend.discover_for_reconciliation("old-id")

    assert result.status == "found"
    assert result.info is not None
    assert result.info.sandbox_id == "old-id"
    assert result.info.mount_contract_version == 1
    assert result.backend_namespace == "namespace-uid-1"
    assert result.incarnation_id == "pod-uid-1"


def test_provisioner_list_returns_sandbox_infos_and_filters_invalid_entries(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        assert url == "http://provisioner:8002/api/sandboxes"
        assert timeout == 10
        assert headers == {}
        return _StubResponse(
            payload={
                "sandboxes": [
                    {"sandbox_id": "abc123", "sandbox_url": "http://k3s:31001"},
                    {"sandbox_id": "missing-url"},
                    {"sandbox_url": "http://k3s:31002"},
                ]
            }
        )

    monkeypatch.setattr(requests, "get", mock_get)

    infos = backend._provisioner_list()
    assert len(infos) == 1
    assert infos[0].sandbox_id == "abc123"
    assert infos[0].sandbox_url == "http://k3s:31001"


def test_provisioner_list_sends_auth_header_when_api_key_set(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002", api_key="secret")
    captured: list[dict] = []

    def mock_get(url: str, timeout: int, headers=None):
        captured.append({"headers": headers})
        return _StubResponse(payload={"sandboxes": []})

    monkeypatch.setattr(requests, "get", mock_get)

    backend._provisioner_list()
    assert captured[0]["headers"] == {"X-API-Key": "secret"}


def test_provisioner_list_returns_empty_on_request_exception(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        raise requests.RequestException("network down")

    monkeypatch.setattr(requests, "get", mock_get)

    assert backend._provisioner_list() == []


def test_provisioner_list_returns_empty_when_payload_is_not_dict(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        return _StubResponse(payload=[{"sandbox_id": "abc", "sandbox_url": "http://k3s:31001"}])

    monkeypatch.setattr(requests, "get", mock_get)

    assert backend._provisioner_list() == []


def test_provisioner_list_returns_empty_when_sandboxes_is_not_list(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        return _StubResponse(payload={"sandboxes": {"sandbox_id": "abc"}})

    monkeypatch.setattr(requests, "get", mock_get)

    assert backend._provisioner_list() == []


def test_provisioner_list_skips_non_dict_sandbox_entries(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        return _StubResponse(
            payload={
                "sandboxes": [
                    {"sandbox_id": "abc123", "sandbox_url": "http://k3s:31001"},
                    "bad-entry",
                    123,
                    None,
                ]
            }
        )

    monkeypatch.setattr(requests, "get", mock_get)

    infos = backend._provisioner_list()
    assert len(infos) == 1
    assert infos[0].sandbox_id == "abc123"
    assert infos[0].sandbox_url == "http://k3s:31001"


@pytest.mark.parametrize(
    ("categories", "expected"),
    [
        ([SkillCategory.LEGACY], True),
        (["legacy"], True),
        ([SkillCategory.CUSTOM], False),
    ],
)
def test_user_should_see_legacy_skills_follows_storage_visibility_rule(monkeypatch, categories, expected):
    class _Storage:
        def load_skills(self, *, enabled_only: bool = False):
            assert enabled_only is False
            return [type("SkillStub", (), {"category": category})() for category in categories]

    monkeypatch.setattr(storage_mod, "get_or_new_user_skill_storage", lambda user_id: _Storage())

    assert storage_mod.user_should_see_legacy_skills("user-1") is expected


@pytest.mark.parametrize("expected_user_id", [None, "owner-1"])
def test_create_delegates_to_provisioner_create(monkeypatch, expected_user_id):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    expected = SandboxInfo(sandbox_id="abc123", sandbox_url="http://k3s:31001")

    def mock_create(
        thread_id: str,
        sandbox_id: str,
        extra_mounts=None,
        *,
        user_id=None,
        provision_lark_cli_runtime=False,
        provision_lark_cli_broker=False,
        required_mount_contract_version=None,
    ):
        assert thread_id == "thread-1"
        assert sandbox_id == "abc123"
        assert extra_mounts == [("/host", "/container", False)]
        assert user_id == expected_user_id
        assert provision_lark_cli_runtime is True
        assert provision_lark_cli_broker is False
        assert required_mount_contract_version == 2
        return expected

    monkeypatch.setattr(backend, "_provisioner_create", mock_create)

    result = backend.create(
        "thread-1",
        "abc123",
        extra_mounts=[("/host", "/container", False)],
        user_id=expected_user_id,
        provision_lark_cli_runtime=True,
    )
    assert result == expected


def test_create_rejects_nondefault_user_on_unverified_legacy_provisioner(monkeypatch):
    backend = RemoteSandboxBackend("http://legacy-provisioner:8002")
    called = False

    def unexpected_create(*_args, **_kwargs):
        nonlocal called
        called = True
        raise AssertionError("unsafe legacy create should not be attempted")

    monkeypatch.setattr(backend, "_provisioner_create", unexpected_create)

    with pytest.raises(RuntimeError, match="cannot isolate user 'alice'"):
        backend.create("shared-thread", "sandbox-1", user_id="alice")

    assert called is False


@pytest.mark.parametrize(
    "response_payload",
    [
        {
            "sandbox_id": "abc123",
            "sandbox_url": "http://k3s:31001",
            "user_id": "mallory",
            "thread_id": "thread-1",
            "mount_contract_version": 2,
        },
        {
            "sandbox_id": "abc123",
            "sandbox_url": "http://k3s:31001",
            "user_id": "alice",
            "thread_id": "other-thread",
            "mount_contract_version": 2,
        },
    ],
)
def test_current_contract_create_rejects_mismatched_identity(monkeypatch, response_payload):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    backend._mount_contract_capability_known = True
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    monkeypatch.setattr(
        requests,
        "post",
        lambda *_args, **_kwargs: _StubResponse(payload=response_payload),
    )

    with pytest.raises(RuntimeError, match="mount contract response"):
        backend.create("thread-1", "abc123", user_id="alice")


@pytest.mark.parametrize(
    "response_payload",
    [
        {"sandbox_id": "abc123", "sandbox_url": "http://k3s:31001"},
        {
            "sandbox_id": "abc123",
            "sandbox_url": "http://k3s:31001",
            "user_id": "alice",
            "thread_id": "thread-1",
            "mount_contract_version": 1,
        },
    ],
)
def test_current_contract_create_restarts_after_response_contract_change(monkeypatch, response_payload):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = time.monotonic() + 30
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    monkeypatch.setattr(
        requests,
        "post",
        lambda *_args, **_kwargs: _StubResponse(payload=response_payload),
    )

    with pytest.raises(remote_backend_mod.MountContractChangedError):
        backend.create("thread-1", "abc123", user_id="alice")

    assert backend._capability_next_probe_at == 0.0


def test_current_contract_create_returns_verified_identity(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    backend._mount_contract_capability_known = True
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    monkeypatch.setattr(
        requests,
        "post",
        lambda *_args, **_kwargs: _StubResponse(
            payload={
                "sandbox_id": "abc123",
                "sandbox_url": "http://k3s:31001",
                "user_id": "alice",
                "thread_id": "thread-1",
                "mount_contract_version": 2,
            }
        ),
    )

    info = backend.create("thread-1", "abc123", user_id="alice")

    assert (info.sandbox_id, info.user_id, info.thread_id, info.mount_contract_version) == (
        "abc123",
        "alice",
        "thread-1",
        2,
    )


def test_create_keeps_v2_response_validation_when_capability_changes_during_post(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    backend._mount_contract_capability_known = True
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)

    def old_peer_response(*_args, **_kwargs):
        backend._mount_contract_version = 0
        backend._mount_contract_capability_known = False
        return _StubResponse(
            payload={
                "sandbox_id": "abc123",
                "sandbox_url": "http://legacy.local",
            }
        )

    monkeypatch.setattr(requests, "post", old_peer_response)

    with pytest.raises(remote_backend_mod.MountContractChangedError):
        backend.create("thread-1", "abc123", user_id="alice")

    assert backend._capability_next_probe_at == 0.0


def test_legacy_create_restarts_when_response_advertises_another_version(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 0
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = time.monotonic() + 30
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    monkeypatch.setattr(
        requests,
        "post",
        lambda *_args, **_kwargs: _StubResponse(
            payload={
                "sandbox_id": "abc123",
                "sandbox_url": "http://legacy.local",
                "mount_contract_version": 1,
            }
        ),
    )

    with pytest.raises(remote_backend_mod.MountContractChangedError):
        backend.create("thread-1", "abc123", user_id="default")

    assert backend._capability_next_probe_at == 0.0


def test_capability_retry_backoff_does_not_overflow_after_long_outage(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._capability_probe_failures = 1024
    backend._capability_next_probe_at = 0.0
    now = [100.0]
    monkeypatch.setattr(remote_backend_mod.time, "monotonic", lambda: now[0])
    monkeypatch.setattr(
        requests,
        "get",
        lambda *_args, **_kwargs: (_ for _ in ()).throw(requests.ConnectionError("still down")),
    )

    assert backend.refresh_capabilities_if_stale() is True
    assert backend._capability_probe_failures == 1025
    assert backend._capability_next_probe_at == 130.0


def test_mount_mode_waits_for_an_inflight_stale_probe(monkeypatch):
    provider_mod = importlib.import_module("deerflow.sandbox.sandbox_provider")
    aio_mod = importlib.import_module("deerflow.community.aio_sandbox.aio_sandbox_provider")
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = 0.0
    provider = aio_mod.AioSandboxProvider.__new__(aio_mod.AioSandboxProvider)
    provider._config = {"thread_data_mounts": True}
    provider._backend = backend
    result: list[bool] = []
    completed = threading.Event()

    backend._capability_probe_lock.acquire()
    try:
        worker = threading.Thread(
            target=lambda: (
                result.append(provider_mod.sandbox_provider_uses_thread_data_mounts(provider)),
                completed.set(),
            ),
            daemon=True,
        )
        worker.start()
        assert not completed.wait(0.05)
        backend._mount_contract_version = 0
        backend._mount_contract_capability_known = True
        backend._capability_next_probe_at = time.monotonic() + 30
    finally:
        backend._capability_probe_lock.release()

    assert completed.wait(1)
    worker.join(timeout=1)
    assert result == [False]


def test_create_restarts_when_required_mount_contract_changed(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 0
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = time.monotonic() + 30
    post = MagicMock()
    monkeypatch.setattr(requests, "post", post)

    with pytest.raises(remote_backend_mod.MountContractChangedError):
        backend.create(
            "thread-1",
            "sandbox-v2",
            user_id="default",
            required_mount_contract_version=2,
        )

    post.assert_not_called()


def test_create_accepts_and_validates_future_compatible_contract(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 3
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = time.monotonic() + 30
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    monkeypatch.setattr(
        requests,
        "post",
        lambda *_args, **_kwargs: _StubResponse(
            payload={
                "sandbox_id": "sandbox-v3",
                "sandbox_url": "http://v3.local",
                "user_id": "alice",
                "thread_id": "thread-1",
                "mount_contract_version": 3,
            }
        ),
    )

    info = backend.create("thread-1", "sandbox-v3", user_id="alice")

    assert info.mount_contract_version == 3


def test_create_sends_frozen_mount_contract_as_a_provisioner_precondition(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = time.monotonic() + 30
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    captured: dict = {}

    def post(_url, *, json, **_kwargs):
        captured.update(json)
        return _StubResponse(
            payload={
                "sandbox_id": "sandbox-v2",
                "sandbox_url": "http://v2.local",
                "user_id": "alice",
                "thread_id": "thread-1",
                "mount_contract_version": 2,
            }
        )

    monkeypatch.setattr(requests, "post", post)

    backend.create("thread-1", "sandbox-v2", user_id="alice")

    assert captured["required_mount_contract_version"] == 2


def test_create_retries_structured_provisioner_contract_precondition_failure(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 2
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = time.monotonic() + 30
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    monkeypatch.setattr(
        requests,
        "post",
        lambda *_args, **_kwargs: _StubResponse(
            status_code=409,
            payload={
                "detail": {
                    "code": "mount_contract_changed",
                    "expected": 2,
                    "actual": 3,
                }
            },
        ),
    )

    with pytest.raises(remote_backend_mod.MountContractChangedError) as exc_info:
        backend.create("thread-1", "sandbox-v2", user_id="alice")

    assert (exc_info.value.expected, exc_info.value.actual) == (2, 3)
    assert backend._capability_next_probe_at == 0.0


@pytest.mark.parametrize("invalid_version", ["2", True])
def test_legacy_create_rejects_non_integer_response_contract(monkeypatch, invalid_version):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    backend._mount_contract_version = 0
    backend._mount_contract_capability_known = True
    backend._capability_next_probe_at = time.monotonic() + 30
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda _user_id: False)
    monkeypatch.setattr(
        requests,
        "post",
        lambda *_args, **_kwargs: _StubResponse(
            payload={
                "sandbox_id": "sandbox-v0",
                "sandbox_url": "http://legacy.local",
                "mount_contract_version": invalid_version,
            }
        ),
    )

    with pytest.raises(RuntimeError, match="mount contract response"):
        backend.create("thread-1", "sandbox-v0", user_id="default")


def test_provisioner_create_returns_sandbox_info(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda user_id: True)

    def mock_post(url: str, json: dict, timeout: int, headers=None):
        assert url == "http://provisioner:8002/api/sandboxes"
        assert json == {
            "sandbox_id": "abc123",
            "thread_id": "thread-1",
            "user_id": "test-user-autouse",
            "required_mount_contract_version": 0,
            "include_legacy_skills": True,
            "provision_lark_cli_runtime": False,
            "provision_lark_cli_broker": False,
        }
        assert timeout == 30
        return _StubResponse(payload={"sandbox_id": "abc123", "sandbox_url": "http://k3s:31001"})

    monkeypatch.setattr(requests, "post", mock_post)

    info = backend._provisioner_create("thread-1", "abc123")
    assert info.sandbox_id == "abc123"
    assert info.sandbox_url == "http://k3s:31001"


def test_provisioner_create_forwards_supported_extra_mounts(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda user_id: False)

    def mock_post(url: str, json: dict, timeout: int, headers=None):
        assert url == "http://provisioner:8002/api/sandboxes"
        assert json["include_legacy_skills"] is False
        assert json["extra_mounts"] == [
            {
                "host_path": "/state/users/alice/skills/integrations",
                "container_path": "/mnt/skills/integrations",
                "read_only": True,
            },
            {
                "host_path": "/state/users/alice/integrations/lark-cli/config",
                "container_path": "/mnt/integrations/lark-cli/config",
                "read_only": False,
            },
        ]
        assert timeout == 30
        return _StubResponse(payload={"sandbox_id": "abc123", "sandbox_url": "http://k3s:31001"})

    monkeypatch.setattr(requests, "post", mock_post)

    backend._provisioner_create(
        "thread-1",
        "abc123",
        extra_mounts=[
            ("/state/users/alice/threads/thread-1/user-data/workspace", "/mnt/user-data/workspace", False),
            ("/skills", "/mnt/skills", True),
            ("/state/users/alice/skills/integrations", "/mnt/skills/integrations", True),
            ("/state/users/alice/integrations/lark-cli/config", "/mnt/integrations/lark-cli/config", False),
        ],
        user_id="alice",
    )


def test_provisioner_create_strips_runtime_mount_when_init_container_enabled(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda user_id: False)

    captured: dict = {}

    def mock_post(url: str, json: dict, timeout: int, headers=None):
        captured.update(json)
        return _StubResponse(payload={"sandbox_id": "abc123", "sandbox_url": "http://k3s:31001"})

    monkeypatch.setattr(requests, "post", mock_post)

    backend._provisioner_create(
        "thread-1",
        "abc123",
        extra_mounts=[
            ("/state/users/alice/integrations/lark-cli/config", "/mnt/integrations/lark-cli/config", False),
            ("/state/users/alice/integrations/lark-cli/data", "/mnt/integrations/lark-cli/data", False),
            ("/state/integrations/lark-cli/sandbox-cli", "/mnt/integrations/lark-cli/runtime", True),
        ],
        user_id="alice",
        provision_lark_cli_runtime=True,
    )

    assert captured["provision_lark_cli_runtime"] is True
    container_paths = {mount["container_path"] for mount in captured["extra_mounts"]}
    # The init container supplies the runtime, so its mount is dropped, but the
    # per-user credential mounts are still forwarded.
    assert "/mnt/integrations/lark-cli/runtime" not in container_paths
    assert "/mnt/integrations/lark-cli/config" in container_paths
    assert "/mnt/integrations/lark-cli/data" in container_paths


def test_provisioner_create_keeps_runtime_mount_when_init_container_disabled(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda user_id: False)

    captured: dict = {}

    def mock_post(url: str, json: dict, timeout: int, headers=None):
        captured.update(json)
        return _StubResponse(payload={"sandbox_id": "abc123", "sandbox_url": "http://k3s:31001"})

    monkeypatch.setattr(requests, "post", mock_post)

    backend._provisioner_create(
        "thread-1",
        "abc123",
        extra_mounts=[
            ("/state/integrations/lark-cli/sandbox-cli", "/mnt/integrations/lark-cli/runtime", True),
        ],
        user_id="alice",
        provision_lark_cli_runtime=False,
    )

    assert captured["provision_lark_cli_runtime"] is False
    container_paths = {mount["container_path"] for mount in captured["extra_mounts"]}
    assert "/mnt/integrations/lark-cli/runtime" in container_paths


def test_provisioner_create_accepts_anonymous_thread_id(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda user_id: False)

    def mock_post(url: str, json: dict, timeout: int, headers=None):
        assert url == "http://provisioner:8002/api/sandboxes"
        assert json == {
            "sandbox_id": "anon123",
            "thread_id": None,
            "user_id": "test-user-autouse",
            "required_mount_contract_version": 0,
            "include_legacy_skills": False,
            "provision_lark_cli_runtime": False,
            "provision_lark_cli_broker": False,
        }
        assert timeout == 30
        return _StubResponse(payload={"sandbox_id": "anon123", "sandbox_url": "http://k3s:31002"})

    monkeypatch.setattr(requests, "post", mock_post)

    info = backend.create(None, "anon123")
    assert info.sandbox_id == "anon123"
    assert info.sandbox_url == "http://k3s:31002"


def test_provisioner_create_raises_runtime_error_on_request_exception(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    monkeypatch.setattr(remote_backend_mod, "user_should_see_legacy_skills", lambda user_id: False)

    def mock_post(url: str, json: dict, timeout: int, headers=None):
        raise requests.RequestException("boom")

    monkeypatch.setattr(requests, "post", mock_post)

    with pytest.raises(RuntimeError, match="Provisioner create failed"):
        backend._provisioner_create("thread-1", "abc123")


def test_destroy_delegates_to_provisioner_destroy(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    called: list[str] = []

    def mock_destroy(sandbox_id: str):
        called.append(sandbox_id)

    monkeypatch.setattr(backend, "_provisioner_destroy", mock_destroy)

    backend.destroy(SandboxInfo(sandbox_id="abc123", sandbox_url="http://k3s:31001"))
    assert called == ["abc123"]


def test_provisioner_destroy_calls_delete(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_delete(url: str, timeout: int, headers=None):
        assert url == "http://provisioner:8002/api/sandboxes/abc123"
        assert timeout == 15
        return _StubResponse(status_code=200)

    monkeypatch.setattr(requests, "delete", mock_delete)

    backend._provisioner_destroy("abc123")


def test_provisioner_destroy_swallows_request_exception(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_delete(url: str, timeout: int, headers=None):
        raise requests.RequestException("network down")

    monkeypatch.setattr(requests, "delete", mock_delete)

    backend._provisioner_destroy("abc123")


def test_is_alive_delegates_to_provisioner_is_alive(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_is_alive(sandbox_id: str):
        assert sandbox_id == "abc123"
        return True

    monkeypatch.setattr(backend, "_provisioner_is_alive", mock_is_alive)

    alive = backend.is_alive(SandboxInfo(sandbox_id="abc123", sandbox_url="http://k3s:31001"))
    assert alive is True


def test_provisioner_is_alive_true_only_when_status_running(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get_running(url: str, timeout: int, headers=None):
        return _StubResponse(payload={"status": "Running"})

    monkeypatch.setattr(requests, "get", mock_get_running)
    assert backend._provisioner_is_alive("abc123") is True

    def mock_get_pending(url: str, timeout: int, headers=None):
        return _StubResponse(payload={"status": "Pending"})

    monkeypatch.setattr(requests, "get", mock_get_pending)
    assert backend._provisioner_is_alive("abc123") is False


def test_provisioner_is_alive_returns_false_on_404(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        return _StubResponse(status_code=404)

    monkeypatch.setattr(requests, "get", mock_get)
    assert backend._provisioner_is_alive("abc123") is False


def test_provisioner_is_alive_raises_on_request_exception(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        raise requests.RequestException("boom")

    monkeypatch.setattr(requests, "get", mock_get)
    with pytest.raises(RuntimeError, match="Provisioner health check failed for abc123"):
        backend._provisioner_is_alive("abc123")


def test_provisioner_is_alive_raises_on_server_error(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        response = _StubResponse(status_code=503)
        response.text = "unavailable"
        return response

    monkeypatch.setattr(requests, "get", mock_get)
    with pytest.raises(RuntimeError, match="HTTP 503 unavailable"):
        backend._provisioner_is_alive("abc123")


def test_discover_delegates_to_provisioner_discover(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")
    expected = SandboxInfo(sandbox_id="abc123", sandbox_url="http://k3s:31001")

    def mock_discover(sandbox_id: str):
        assert sandbox_id == "abc123"
        return expected

    monkeypatch.setattr(backend, "_provisioner_discover", mock_discover)

    result = backend.discover("abc123")
    assert result == expected


def test_provisioner_discover_returns_none_on_404(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        return _StubResponse(status_code=404)

    monkeypatch.setattr(requests, "get", mock_get)

    assert backend._provisioner_discover("abc123") is None


def test_provisioner_discover_returns_info_on_success(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        return _StubResponse(
            payload={
                "sandbox_id": "abc123",
                "sandbox_url": "http://k3s:31001",
                "user_id": "alice",
                "thread_id": "thread-1",
                "mount_contract_version": 2,
            }
        )

    monkeypatch.setattr(requests, "get", mock_get)

    info = backend._provisioner_discover("abc123")
    assert info is not None
    assert info.sandbox_id == "abc123"
    assert info.sandbox_url == "http://k3s:31001"
    assert (info.user_id, info.thread_id, info.mount_contract_version) == (
        "alice",
        "thread-1",
        2,
    )


def test_provisioner_discover_returns_none_on_request_exception(monkeypatch):
    backend = RemoteSandboxBackend("http://provisioner:8002")

    def mock_get(url: str, timeout: int, headers=None):
        raise requests.RequestException("boom")

    monkeypatch.setattr(requests, "get", mock_get)

    assert backend._provisioner_discover("abc123") is None