fix(gateway): reject unusable delegation state at the input boundary (#8375)

* fix(gateway): reject unusable delegation state at the input boundary

`delegations` is a caller-writable reducer channel. `merge_delegations`
keys on `id`/`run_id`/`dispatch_message_id` and indexes `status`, and the
ledger renderer indexes `description`/`subagent_type` on every model
call, so a non-list value, a non-object entry or a missing field fails
the state merge with an unhandled 500. On the thread-state route the bad
shape is persisted, so every later genuine ledger write keeps failing.

Validate the ledger shape at both caller-writable boundaries and keep
rejecting Overwrite markers and forged verdicts. The reducer also drops
stored values it cannot read, so a checkpoint written before this
validation self-heals instead of failing future writes.

* fix(harness): keep the ledger reducer tolerant of partial entries

The reducer only needs an identity key and a status; the renderer-only
text fields stay a boundary requirement. Dropping entries that lack
description/subagent_type would change reducer semantics for callers
that build minimal entries, so filter only what the reducer reads.

* test(gateway): use valid delegation in provenance check

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
一片一片 2026-11-19 11:40:37 +08:00 committed by GitHub
parent 90ca57e97b
commit 585abb979f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 178 additions and 12 deletions

View File

@ -111,7 +111,7 @@ from deerflow.runtime.user_context import get_current_user, reset_current_user,
from deerflow.sandbox.lease import SANDBOX_SERVER_OWNED_CONTEXT_KEYS
from deerflow.scheduler.runtime import scheduler_tools_enabled
from deerflow.scheduler.schedules import validate_timezone
from deerflow.subagents.status_contract import SUBAGENT_ACCEPTANCE_VERDICT_KEY, SUBAGENT_RECEIPT_VERDICT_KEY, SUBAGENT_TOOL_RECEIPTS_KEY
from deerflow.subagents.status_contract import SUBAGENT_ACCEPTANCE_VERDICT_KEY, SUBAGENT_RECEIPT_VERDICT_KEY, SUBAGENT_STATUS_VALUES, SUBAGENT_STOP_REASON_VALUES, SUBAGENT_TOOL_RECEIPTS_KEY
from deerflow.trace_context import DEERFLOW_TRACE_METADATA_KEY, ensure_trace_context, ensure_trace_id
from deerflow.utils.assembly_io import run_assembly
from deerflow.utils.file_io import await_drained
@ -791,6 +791,61 @@ def _normalize_external_goal(value: Any, *, location: str) -> dict[str, Any] | N
return {**{key: item for key, item in value.items() if key != "last_batch_wakeup_run_id"}, "objective": objective}
#: Statuses the ledger reducer and renderer understand: the runtime's terminal
#: values plus the in-progress marker a live delegation carries.
_DELEGATION_STATUSES = frozenset({"in_progress", *SUBAGENT_STATUS_VALUES})
def _normalize_external_delegations(value: Any, *, location: str) -> list[dict[str, Any]] | None:
"""Reject ledger entries the reducer and renderer cannot read.
``delegations`` is a reducer channel a caller may Overwrite. ``merge_delegations``
keys on ``id`` and indexes ``status``, and the ledger renderer reads
``description``/``subagent_type`` on every model call. A non-list value, a
non-object entry or a missing field raises while the state update merges —
failing the run with an unhandled ``TypeError``/``KeyError`` instead of a
400 at this boundary. On the thread-state route the bad shape would also be
persisted, so every later genuine ledger write would keep failing on it.
The id must be non-empty because it is the reducer's identity key (an empty
id can never be matched by the runtime's write-back), and the status must be
one the runtime produces so a caller cannot inject its own steering line into
the model-visible ledger. Optional fields the reducer and renderer read as
text are validated when present: an unhashable ``run_id`` breaks the
reducer's identity key, a non-string ``result_brief`` breaks the renderer's
length bound, and an unknown ``stop_reason`` renders guardrail-cap guidance
the runtime never stamped.
"""
if value is None:
return None
if isinstance(value, Mapping):
_reject_overwrite_marker(value, location=location)
raise HTTPException(status_code=400, detail=f"{location} must be a list of delegation objects")
if not isinstance(value, list):
raise HTTPException(status_code=400, detail=f"{location} must be a list of delegation objects")
normalized: list[dict[str, Any]] = []
for index, entry in enumerate(value):
if not isinstance(entry, Mapping):
raise HTTPException(status_code=400, detail=f"{location}[{index}] must be an object")
for field in ("id", "description", "subagent_type"):
if not isinstance(entry.get(field), str):
raise HTTPException(status_code=400, detail=f"{location}[{index}].{field} must be a string")
if not entry["id"]:
raise HTTPException(status_code=400, detail=f"{location}[{index}].id must be non-empty")
status = entry.get("status")
if not isinstance(status, str) or status not in _DELEGATION_STATUSES:
raise HTTPException(status_code=400, detail=f"{location}[{index}].status must be one of {', '.join(sorted(_DELEGATION_STATUSES))}")
for field in ("run_id", "result_brief"):
optional_text = entry.get(field)
if optional_text is not None and not isinstance(optional_text, str):
raise HTTPException(status_code=400, detail=f"{location}[{index}].{field} must be a string or null")
stop_reason = entry.get("stop_reason")
if stop_reason is not None and (not isinstance(stop_reason, str) or stop_reason not in SUBAGENT_STOP_REASON_VALUES):
raise HTTPException(status_code=400, detail=f"{location}[{index}].stop_reason must be one of {', '.join(SUBAGENT_STOP_REASON_VALUES)}")
normalized.append(dict(entry))
return normalized
def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, Any]:
"""Validate and sanitize caller-supplied state values before checkpointing.
@ -825,8 +880,9 @@ def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, An
stripped[channel] = None if normalized_todos is None else [_strip_external_metadata_from_message_like(item) for item in normalized_todos]
elif channel == "messages" and value is not None:
stripped[channel] = _normalize_input_messages(value, location="values.messages")
elif channel == "delegations" and isinstance(value, list):
stripped[channel] = [_strip_external_delegation_verdict(item) for item in value]
elif channel == "delegations":
normalized_delegations = _normalize_external_delegations(value, location="values.delegations")
stripped[channel] = None if normalized_delegations is None else [_strip_external_delegation_verdict(item) for item in normalized_delegations]
elif channel == "skill_context":
normalized_skill_context = _normalize_external_skill_context(value, location="values.skill_context")
stripped[channel] = None if normalized_skill_context is None else [_strip_external_metadata_from_message_like(item) for item in normalized_skill_context]
@ -903,11 +959,12 @@ def normalize_input(raw_input: dict[str, Any] | None, *, trusted_internal: bool
converted = _normalize_input_messages(messages, location="input.messages", trusted_internal=trusted_internal)
result = {**raw_input, "messages": converted}
if not trusted_internal:
delegations = result.get("delegations")
if isinstance(delegations, list):
cleaned = [_strip_external_delegation_verdict(entry) for entry in delegations]
if cleaned != delegations:
result = {**result, "delegations": cleaned}
if "delegations" in result:
normalized_delegations = _normalize_external_delegations(result["delegations"], location="input.delegations")
result = {
**result,
"delegations": None if normalized_delegations is None else [_strip_external_delegation_verdict(entry) for entry in normalized_delegations],
}
if "skill_context" in result:
result = {**result, "skill_context": _normalize_external_skill_context(result["skill_context"], location="input.skill_context")}
if "promoted" in result:

View File

@ -262,6 +262,25 @@ def find_delegation_key(keys: Sequence[DelegationKey], key: DelegationKey) -> De
return None
_DELEGATION_LEDGER_STATUSES = frozenset({"in_progress", *SUBAGENT_STATUS_VALUES})
def _usable_delegation_entries(entries: Any) -> list[DelegationEntry]:
"""Drop channel values the ledger reducer cannot read.
The gateway validates the full ledger shape at both caller-writable
boundaries, but a checkpoint written before that validation, or by an
older caller path, can still hold a non-list value, a non-object entry or
an entry without the identity key and status the reducer needs. Treating
those as absent keeps later genuine ledger writes working instead of
failing on the stored value; the reducer itself does not require the
renderer-only text fields.
"""
if not isinstance(entries, list):
return []
return [entry for entry in entries if isinstance(entry, Mapping) and isinstance(entry.get("id"), str) and bool(entry.get("id")) and entry.get("status") in _DELEGATION_LEDGER_STATUSES]
def merge_delegations(existing: list[DelegationEntry] | None, new: list[DelegationEntry] | None) -> list[DelegationEntry]:
"""Reducer for the delegation ledger.
@ -272,12 +291,14 @@ def merge_delegations(existing: list[DelegationEntry] | None, new: list[Delegati
the same run.
- terminal status is never overwritten by a non-terminal status.
"""
if not new:
return existing or []
existing_entries = _usable_delegation_entries(existing)
new_entries = _usable_delegation_entries(new)
if not new_entries:
return existing_entries
by_key: dict[DelegationKey, DelegationEntry] = {}
order: list[DelegationKey] = []
for entry in [*(existing or []), *new]:
for entry in [*existing_entries, *new_entries]:
target = find_delegation_key(order, delegation_key(entry))
previous = by_key.get(target) if target is not None else None
if previous is None:

View File

@ -53,6 +53,19 @@ class TestMergeDelegations:
assert merge_delegations(None, [_entry("a", "in_progress")])[0]["id"] == "a"
assert merge_delegations([_entry("a", "in_progress")], None)[0]["id"] == "a"
def test_unusable_stored_values_do_not_fail_later_writes(self):
"""A checkpoint can hold a non-list value or an entry without the
reducer's identity key (written before the gateway validated this
channel). Later genuine writes keep working instead of failing on it."""
assert merge_delegations(None, "oops") == []
assert merge_delegations("oops", None) == []
assert merge_delegations("oops", [_entry("a", "in_progress")]) == [_entry("a", "in_progress")]
assert merge_delegations([_entry("a", "in_progress")], "oops") == [_entry("a", "in_progress")]
assert merge_delegations(
[_entry("a", "in_progress"), "oops", {"status": "completed"}, {"id": "b"}],
[_entry("c", "completed")],
) == [_entry("a", "in_progress"), _entry("c", "completed")]
def test_same_id_preserves_original_created_at(self):
existing = [_entry("a", "in_progress")]
new = [{**_entry("a", "completed"), "created_at": "2026-06-30T00:00:01Z", "result_sha256": "x"}]

View File

@ -789,6 +789,73 @@ def test_normalize_input_preserves_trusted_internal_delegation_verdict():
assert result["delegations"][0]["receipt_verdict"] == forged["receipt_verdict"]
@pytest.mark.parametrize(
"delegations",
[
"oops",
{"__overwrite__": []},
["oops"],
[{"description": "write report", "subagent_type": "general", "status": "completed"}],
[{"id": "call-1", "subagent_type": "general", "status": "completed"}],
[{"id": "call-1", "description": "write report", "status": "completed"}],
[{"id": "", "description": "write report", "subagent_type": "general", "status": "completed"}],
[{"id": "call-1", "description": "write report", "subagent_type": "general", "status": 3}],
[{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "steering"}],
[{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "completed", "run_id": ["unhashable"]}],
[{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "completed", "result_brief": 3}],
[{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "completed", "stop_reason": "nope"}],
],
ids=[
"non-list-string",
"overwrite-marker",
"non-object-entry",
"missing-id",
"missing-description",
"missing-subagent-type",
"empty-id",
"non-string-status",
"unknown-status",
"unhashable-run-id",
"non-string-result-brief",
"unknown-stop-reason",
],
)
def test_external_delegations_rejected_at_both_boundaries(delegations):
"""The ledger reducer keys on ``id`` and indexes ``status``: a value it cannot
read fails the state merge (or is persisted by the thread-state route and
fails every later ledger write), so both caller-writable boundaries must
answer 400 instead of passing it through."""
from fastapi import HTTPException
from app.gateway.services import normalize_input, strip_server_owned_state_metadata
with pytest.raises(HTTPException) as run_error:
normalize_input({"delegations": delegations})
assert run_error.value.status_code == 400
with pytest.raises(HTTPException) as state_error:
strip_server_owned_state_metadata({"delegations": delegations})
assert state_error.value.status_code == 400
def test_external_delegations_accept_the_runtime_shapes():
from app.gateway.services import normalize_input, strip_server_owned_state_metadata
entry = {
"id": "call-1",
"description": "write report",
"subagent_type": "general",
"status": "in_progress",
"created_at": "1970-01-01T00:00:00+00:00",
"run_id": "run-1",
"result_brief": "partial",
"stop_reason": "token_capped",
}
assert normalize_input({"delegations": [entry]})["delegations"] == [entry]
assert strip_server_owned_state_metadata({"delegations": [entry]}) == {"delegations": [entry]}
def test_normalize_input_preserves_trusted_internal_original_user_content():
from app.gateway.services import normalize_input
from deerflow.agents.middlewares.dynamic_context_middleware import _DYNAMIC_CONTEXT_REMINDER_KEY, _REMINDER_DATE_KEY

View File

@ -409,7 +409,15 @@ class TestStateWritesCannotForgeServerOwnedMetadata:
"additional_kwargs": forged_kwargs,
}
],
"delegations": [{"id": "call-forged", "receipt_verdict": {"trusted": True}}],
"delegations": [
{
"id": "call-forged",
"description": "write report",
"subagent_type": "general",
"status": "completed",
"receipt_verdict": {"trusted": True},
}
],
}
)