deer-flow/backend/tests/test_extension_api_contracts.py
Xinmin Zeng df544f126a
test(backend): prune 32 zero-unique redundant suites and rewrite 9 source-inspection tests (#8362)
* test(backend): prune 32 zero-unique redundant suites and rewrite 9 source-inspection tests

- Remove 32 superseded test files proven lossless across statement/branch coverage (per-test dynamic contexts), hotspot mutation testing (mutmut), and 20 historical bug replays.
- Replace all inspect.getsource and brittle source-string assertions across 9 test files with behavioral runtime assertions.

* test(backend): consolidate hidden-answer and compaction-hash guards and assert protocol defaults

- Fold hidden human-input turn-boundary regressions into tests/test_channels.py while keeping test_channel_hidden_human_input.py removed.
- Fold multimodal source content hashing and unobserved-skip regressions into tests/test_compaction_event_identity.py.
- Assert contract-specific default return values/types (False for bool registrations, empty sequence for middleware contributions, explicit return None for void hooks) in test_extension_api_contracts.py and add stub rejection tests.
2026-11-18 21:13:19 +08:00

458 lines
16 KiB
Python

"""Tests for the extension contract surface.
The contracts carry two compatibility promises that are easy to break by
accident and impossible to catch at runtime later: every Protocol method has a
default implementation, and every optional dataclass field has a default. Both
are asserted here.
"""
from __future__ import annotations
import asyncio
import dataclasses
import dis
import importlib.resources
import inspect
from collections.abc import Sequence
from typing import Any, Protocol
import pytest
from deerflow_extension_api import (
API_VERSION,
AgentBuildContext,
AgentScope,
ExtensionData,
ExtensionInstall,
ExtensionRegistry,
ExtensionRuntimeDeps,
ExtensionService,
HostPolicySnapshot,
InvalidRunEvidenceCursor,
MiddlewareContributor,
MiddlewarePlacement,
Placement,
RunEventPage,
RunEventView,
RunEvidenceReader,
RunPage,
RunStatusView,
SystemModelCallObserver,
SystemModelRequest,
SystemModelResult,
SystemOperationKind,
TaskInfo,
TaskLifecycleContributor,
TaskOutcome,
extension,
)
from deerflow_extension_api.runtime_bridge import (
EXTENSION_TASK_STORE_KEY,
task_store_from_runtime,
)
def test_placement_members_cover_both_axes():
assert Placement.MODEL_LOGICAL.value == "model_logical"
assert Placement.MODEL_PHYSICAL.value == "model_physical"
assert Placement.TOOL_VISIBLE.value == "tool_visible"
assert Placement.TOOL_RAW.value == "tool_raw"
assert Placement.STANDARD.value == "standard"
def test_agent_scope_both_is_union():
assert AgentScope.BOTH == AgentScope.LEAD | AgentScope.SUBAGENT
assert AgentScope.LEAD in AgentScope.BOTH
def test_middleware_placement_defaults():
p = MiddlewarePlacement(middleware=object(), placement=Placement.STANDARD)
assert p.scope is AgentScope.BOTH
assert p.order == 0
@pytest.mark.parametrize(
"cls",
[
HostPolicySnapshot,
ExtensionRuntimeDeps,
AgentBuildContext,
TaskInfo,
SystemModelRequest,
SystemModelResult,
MiddlewarePlacement,
RunEventPage,
RunEventView,
RunPage,
RunStatusView,
],
)
def test_every_dataclass_is_frozen(cls):
assert dataclasses.is_dataclass(cls)
assert cls.__dataclass_params__.frozen, f"{cls.__name__} must be frozen"
@pytest.mark.parametrize(
"cls",
[HostPolicySnapshot, ExtensionRuntimeDeps, TaskInfo, SystemModelRequest, SystemModelResult],
)
def test_additive_dataclasses_are_constructible_with_required_fields_only(cls):
"""Fields added later must carry defaults, or old extensions break on upgrade.
HostPolicySnapshot and the two system-call snapshots are host-constructed
and fully optional. TaskInfo has a required identity core and optional
remainder. AgentBuildContext gets its own dedicated test below because its
scope is legitimately required.
"""
if cls is TaskInfo:
info = cls(task_id="t", run_id="r", thread_id="th", kind="lead")
assert info.parent_task_id is None
assert info.agent_name is None
assert info.resumed is False
else:
assert cls() is not None
def test_agent_build_context_optional_fields_keep_their_defaults():
"""AgentBuildContext has one required field (scope); the rest must default.
Unlike the fully-optional dataclasses above, scope is legitimately
required, so this is not folded into the parametrized test above — it
would misrepresent the required/optional split this suite is meant to
document.
"""
ctx = AgentBuildContext(scope=AgentScope.LEAD)
assert ctx.agent_name is None
assert ctx.model_name is None
assert isinstance(ctx.policy, HostPolicySnapshot)
def _dummy_args_for_protocol_method(func):
sig = inspect.signature(func)
kwargs = {}
positional = []
sample_by_name = {
"app_store": ExtensionData("app"),
"task_store": ExtensionData("task-1"),
"ctx": AgentBuildContext(scope=AgentScope.LEAD),
"info": TaskInfo(task_id="t", run_id="r", thread_id="th", kind="lead"),
"outcome": TaskOutcome.COMPLETED,
"kind": SystemOperationKind.GOAL,
"request": SystemModelRequest(messages=("prompt",)),
"result": SystemModelResult(response="ok", duration_ms=1.0),
"deps": ExtensionRuntimeDeps(),
"cursor": None,
"limit": 10,
"thread_id": "th",
"run_id": "r",
"after_seq": None,
"name": "demo",
"routers": (),
}
for idx, (pname, param) in enumerate(sig.parameters.items()):
if idx == 0:
continue # self
val = sample_by_name.get(pname, object())
if param.kind == inspect.Parameter.KEYWORD_ONLY:
kwargs[pname] = val
elif param.kind in (
inspect.Parameter.POSITIONAL_ONLY,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
):
positional.append(val)
return positional, kwargs
def _assert_protocol_has_defaults(protocol: type, *, expects_not_implemented: bool = False) -> None:
"""Invoke every public Protocol method on a bare instance and verify its documented default."""
class _Bare:
pass
bare = _Bare()
checked = 0
for name, member in vars(protocol).items():
if name.startswith("_") or not inspect.isfunction(member):
continue
checked += 1
sig = inspect.signature(member)
ret_ann = sig.return_annotation
pos_args, kw_args = _dummy_args_for_protocol_method(member)
if expects_not_implemented:
with pytest.raises(NotImplementedError):
if inspect.iscoroutinefunction(member):
asyncio.run(member(bare, *pos_args, **kw_args))
else:
member(bare, *pos_args, **kw_args)
continue
result = asyncio.run(member(bare, *pos_args, **kw_args)) if inspect.iscoroutinefunction(member) else member(bare, *pos_args, **kw_args)
if ret_ann in ("bool", bool):
assert result is False, f"{protocol.__name__}.{name} must default to False (got {result!r}); an unimplemented `...` stub implicitly returns None"
elif (isinstance(ret_ann, str) and ret_ann.startswith("Sequence[")) or ret_ann is Sequence:
assert isinstance(result, Sequence) and not isinstance(result, str | bytes) and tuple(result) == (), f"{protocol.__name__}.{name} must default to an empty sequence (got {result!r})"
elif ret_ann in ("None", None):
assert result is None, f"{protocol.__name__}.{name} must return None (got {result!r})"
ret_spans = [(ins.positions.end_col_offset or 0) - (ins.positions.col_offset or 0) for ins in dis.get_instructions(member) if ins.opname in ("RETURN_VALUE", "RETURN_CONST") and ins.positions is not None]
assert ret_spans and all(span not in (0, 3) for span in ret_spans), f"{protocol.__name__}.{name} uses an unimplemented `...` stub instead of an explicit `return None` default"
else:
raise AssertionError(f"Unhandled return annotation {ret_ann!r} on {protocol.__name__}.{name}; assert its contract-specific default explicitly")
assert checked > 0, f"{protocol.__name__} declared no methods to check"
@pytest.mark.parametrize(
"protocol",
[
ExtensionRegistry,
ExtensionService,
MiddlewareContributor,
RunEvidenceReader,
TaskLifecycleContributor,
SystemModelCallObserver,
],
)
def test_every_protocol_method_has_a_default_implementation(protocol):
"""Adding a method to a Protocol is only additive when it carries a concrete default."""
_assert_protocol_has_defaults(protocol, expects_not_implemented=(protocol is RunEvidenceReader))
def test_protocol_default_guard_rejects_unimplemented_ellipsis_stubs():
"""Verify the default-implementation guard catches `...` stubs across bool, sequence, void, and reader contracts."""
class _StubBoolProtocol(Protocol):
def knowledge_provider(self, name: str, provider: Any) -> bool: ...
class _StubSequenceProtocol(Protocol):
def contribute_middlewares(self) -> Sequence[Any]: ...
class _StubInlineVoidProtocol(Protocol):
def on_event(self) -> None: ...
class _StubMultilineVoidProtocol(Protocol):
async def start(self) -> None:
"""Unimplemented async hook stub."""
...
class _StubReaderProtocol(Protocol):
def get_run(self, run_id: str) -> Any: ...
for broken_proto in (
_StubBoolProtocol,
_StubSequenceProtocol,
_StubInlineVoidProtocol,
_StubMultilineVoidProtocol,
):
with pytest.raises(AssertionError):
_assert_protocol_has_defaults(broken_proto)
with pytest.raises(pytest.fail.Exception):
_assert_protocol_has_defaults(_StubReaderProtocol, expects_not_implemented=True)
def test_contributor_defaults_return_empty():
class _Bare:
pass
bare = _Bare()
assert MiddlewareContributor.contribute_middlewares(bare, ExtensionData("app"), AgentBuildContext(scope=AgentScope.LEAD)) == ()
def test_task_lifecycle_contract_is_public_and_defaults_to_noop():
class _Bare:
pass
app_store = ExtensionData("app")
task_store = ExtensionData("task-1")
info = TaskInfo(
task_id="task-1",
run_id="run-1",
thread_id="thread-1",
kind="lead",
)
assert TaskOutcome.COMPLETED.value == "completed"
assert asyncio.run(TaskLifecycleContributor.on_task_start(_Bare(), app_store, task_store, info)) is None
assert asyncio.run(TaskLifecycleContributor.on_task_stop(_Bare(), app_store, task_store, info, TaskOutcome.COMPLETED)) is None
def test_system_model_observer_contract_reports_success_and_failure_shapes():
class _Bare:
pass
app_store = ExtensionData("app")
task_store = ExtensionData("task-1")
request = SystemModelRequest(messages=("prompt",), model_name="system-model")
success = SystemModelResult(response="answer", duration_ms=1.5)
failure = SystemModelResult(error=RuntimeError("provider failed"), duration_ms=2.0)
assert SystemOperationKind.GOAL.value == "goal"
assert asyncio.run(SystemModelCallObserver.on_system_model_call(_Bare(), app_store, task_store, SystemOperationKind.GOAL, request, success)) is None
assert asyncio.run(SystemModelCallObserver.on_system_model_call(_Bare(), app_store, task_store, SystemOperationKind.GOAL, request, failure)) is None
def test_system_model_request_normalizes_messages_into_an_immutable_sequence():
"""``messages`` is a snapshot of a message sequence, never a per-character view.
Title and summarization pass a single prompt string, so a bare ``str`` must not
reach observers as a ``Sequence`` whose items are characters. A live ``list`` from
a call site must also be copied: the snapshot is documented as read-only, and the
caller keeps mutating its own list after the observation is dispatched.
"""
assert SystemModelRequest(messages="one prompt").messages == ("one prompt",)
live: list[str] = ["first"]
request = SystemModelRequest(messages=live)
live.append("second")
assert request.messages == ("first",)
assert SystemModelRequest().messages == ()
assert SystemModelRequest(messages=("already", "a", "tuple")).messages == ("already", "a", "tuple")
def test_gateway_contribution_points_are_part_of_the_public_surface():
import deerflow_extension_api
for name in (
"ExtensionRuntimeDeps",
"ExtensionService",
"InvalidRunEvidenceCursor",
"RunEvidenceReader",
"RunEventPage",
"RunPage",
):
assert name in deerflow_extension_api.__all__
assert hasattr(deerflow_extension_api, name)
assert callable(ExtensionRegistry.service)
assert callable(ExtensionRegistry.routers)
assert not hasattr(deerflow_extension_api, "RouterContributor")
def test_run_evidence_pages_are_immutable_and_empty_by_default():
assert issubclass(InvalidRunEvidenceCursor, ValueError)
assert RunPage().items == ()
assert RunPage().next_cursor is None
assert RunPage().has_more is False
assert RunEventPage().items == ()
assert RunEventPage().next_after_seq is None
def test_bare_run_evidence_reader_fails_explicitly_instead_of_looking_caught_up():
class _Bare:
pass
async def invoke():
with pytest.raises(NotImplementedError):
await RunEvidenceReader.list_changed_runs(_Bare(), cursor=None, limit=10)
asyncio.run(invoke())
def test_task_store_from_runtime_reads_the_host_key():
class _Runtime:
def __init__(self, context):
self.context = context
store = ExtensionData("task-1")
assert task_store_from_runtime(_Runtime({EXTENSION_TASK_STORE_KEY: store})) is store
def test_task_store_from_runtime_returns_none_on_missing_or_wrong_shape():
class _Runtime:
def __init__(self, context):
self.context = context
assert task_store_from_runtime(None) is None
assert task_store_from_runtime(_Runtime({})) is None
assert task_store_from_runtime(_Runtime("not-a-mapping")) is None
assert task_store_from_runtime(_Runtime({EXTENSION_TASK_STORE_KEY: "wrong type"})) is None
def test_extension_decorator_stamps_api_requirement():
@extension(api="0.1", name="demo")
def install(registry, config):
return None
assert install.__deerflow_api__ == "0.1"
assert install.__deerflow_name__ == "demo"
def test_task_outcome_members():
assert {outcome.value for outcome in TaskOutcome} == {"completed", "aborted", "failed"}
def test_system_operation_kind_members():
assert {kind.value for kind in SystemOperationKind} == {"goal", "memory", "title", "summarization"}
def test_registry_and_install_alias_are_part_of_the_public_surface():
"""Independent extensions annotate install(registry, config) against the
contract package alone — importing the host's concrete registry would pin
them to the harness release cadence and advertise host-only machinery."""
import typing
import deerflow_extension_api
assert "ExtensionRegistry" in deerflow_extension_api.__all__
assert "ExtensionInstall" in deerflow_extension_api.__all__
parameters, return_type = typing.get_args(ExtensionInstall)
assert parameters[0] is ExtensionRegistry, "install()'s first argument must be the public registry contract"
def test_distribution_marks_the_contract_package_as_typed():
marker = importlib.resources.files("deerflow_extension_api").joinpath("py.typed")
assert marker.is_file()
def test_contract_package_keeps_runtime_dependencies_empty():
import tomllib
from pathlib import Path
pyproject = Path(__file__).parent.parent / "packages" / "extension-api" / "pyproject.toml"
assert tomllib.loads(pyproject.read_text())["project"]["dependencies"] == []
def test_harness_pins_the_contract_package_exactly():
"""The version contract (extension-system design): the host pins the
contract package exactly, extensions use ranges. A range here would let an
older harness resolve a newer 1.x contract package — API_VERSION would
then come from the upgraded package and newer extensions would look
supported against a host whose registry/placements/hook pipeline still
implements the older contract. The pin makes pip reject that skew at
install time."""
import tomllib
from importlib.metadata import version
from pathlib import Path
from packaging.requirements import Requirement
pyproject = Path(__file__).parent.parent / "packages" / "harness" / "pyproject.toml"
dependencies = tomllib.loads(pyproject.read_text())["project"]["dependencies"]
requirement = next(Requirement(dep) for dep in dependencies if Requirement(dep).name == "deerflow-extension-api")
expected = f"=={version('deerflow-extension-api')}"
assert str(requirement.specifier) == expected, f"the host must pin deerflow-extension-api exactly ({expected}); a range lets pip resolve a contract newer than the host implements"
def test_runtime_api_version_matches_the_installed_contract_package():
"""Every additive contract slice bumps both gates together."""
from importlib.metadata import version
assert API_VERSION == "0.2.13"
assert API_VERSION == version("deerflow-extension-api")
def test_extension_service_contract_is_public_and_defaults_to_noop():
class _Bare:
pass
deps = ExtensionRuntimeDeps()
assert deps.app_store is None
assert deps.session_factory is None
assert deps.run_evidence_reader is None
assert asyncio.run(ExtensionService.start(_Bare(), deps)) is None
assert asyncio.run(ExtensionService.stop(_Bare())) is None