From 585abb979fbc483485c0fc53ed0e571ab7eebadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E7=89=87=E4=B8=80=E7=89=87?= <3037678834@qq.com> Date: Thu, 19 Nov 2026 11:40:37 +0800 Subject: [PATCH] fix(gateway): reject unusable delegation state at the input boundary (#8375) * fix(gateway): reject unusable delegation state at the input boundary `delegations` is a caller-writable reducer channel. `merge_delegations` keys on `id`/`run_id`/`dispatch_message_id` and indexes `status`, and the ledger renderer indexes `description`/`subagent_type` on every model call, so a non-list value, a non-object entry or a missing field fails the state merge with an unhandled 500. On the thread-state route the bad shape is persisted, so every later genuine ledger write keeps failing. Validate the ledger shape at both caller-writable boundaries and keep rejecting Overwrite markers and forged verdicts. The reducer also drops stored values it cannot read, so a checkpoint written before this validation self-heals instead of failing future writes. * fix(harness): keep the ledger reducer tolerant of partial entries The reducer only needs an identity key and a status; the renderer-only text fields stay a boundary requirement. Dropping entries that lack description/subagent_type would change reducer semantics for callers that build minimal entries, so filter only what the reducer reads. * test(gateway): use valid delegation in provenance check --------- Co-authored-by: Willem Jiang --- backend/app/gateway/services.py | 73 +++++++++++++++++-- .../harness/deerflow/agents/thread_state.py | 27 ++++++- backend/tests/test_delegation_ledger.py | 13 ++++ backend/tests/test_gateway_services.py | 67 +++++++++++++++++ backend/tests/test_message_provenance.py | 10 ++- 5 files changed, 178 insertions(+), 12 deletions(-) diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index 3fee652886..541fa1a6de 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -111,7 +111,7 @@ from deerflow.runtime.user_context import get_current_user, reset_current_user, from deerflow.sandbox.lease import SANDBOX_SERVER_OWNED_CONTEXT_KEYS from deerflow.scheduler.runtime import scheduler_tools_enabled from deerflow.scheduler.schedules import validate_timezone -from deerflow.subagents.status_contract import SUBAGENT_ACCEPTANCE_VERDICT_KEY, SUBAGENT_RECEIPT_VERDICT_KEY, SUBAGENT_TOOL_RECEIPTS_KEY +from deerflow.subagents.status_contract import SUBAGENT_ACCEPTANCE_VERDICT_KEY, SUBAGENT_RECEIPT_VERDICT_KEY, SUBAGENT_STATUS_VALUES, SUBAGENT_STOP_REASON_VALUES, SUBAGENT_TOOL_RECEIPTS_KEY from deerflow.trace_context import DEERFLOW_TRACE_METADATA_KEY, ensure_trace_context, ensure_trace_id from deerflow.utils.assembly_io import run_assembly from deerflow.utils.file_io import await_drained @@ -791,6 +791,61 @@ def _normalize_external_goal(value: Any, *, location: str) -> dict[str, Any] | N return {**{key: item for key, item in value.items() if key != "last_batch_wakeup_run_id"}, "objective": objective} +#: Statuses the ledger reducer and renderer understand: the runtime's terminal +#: values plus the in-progress marker a live delegation carries. +_DELEGATION_STATUSES = frozenset({"in_progress", *SUBAGENT_STATUS_VALUES}) + + +def _normalize_external_delegations(value: Any, *, location: str) -> list[dict[str, Any]] | None: + """Reject ledger entries the reducer and renderer cannot read. + + ``delegations`` is a reducer channel a caller may Overwrite. ``merge_delegations`` + keys on ``id`` and indexes ``status``, and the ledger renderer reads + ``description``/``subagent_type`` on every model call. A non-list value, a + non-object entry or a missing field raises while the state update merges — + failing the run with an unhandled ``TypeError``/``KeyError`` instead of a + 400 at this boundary. On the thread-state route the bad shape would also be + persisted, so every later genuine ledger write would keep failing on it. + + The id must be non-empty because it is the reducer's identity key (an empty + id can never be matched by the runtime's write-back), and the status must be + one the runtime produces so a caller cannot inject its own steering line into + the model-visible ledger. Optional fields the reducer and renderer read as + text are validated when present: an unhashable ``run_id`` breaks the + reducer's identity key, a non-string ``result_brief`` breaks the renderer's + length bound, and an unknown ``stop_reason`` renders guardrail-cap guidance + the runtime never stamped. + """ + if value is None: + return None + if isinstance(value, Mapping): + _reject_overwrite_marker(value, location=location) + raise HTTPException(status_code=400, detail=f"{location} must be a list of delegation objects") + if not isinstance(value, list): + raise HTTPException(status_code=400, detail=f"{location} must be a list of delegation objects") + normalized: list[dict[str, Any]] = [] + for index, entry in enumerate(value): + if not isinstance(entry, Mapping): + raise HTTPException(status_code=400, detail=f"{location}[{index}] must be an object") + for field in ("id", "description", "subagent_type"): + if not isinstance(entry.get(field), str): + raise HTTPException(status_code=400, detail=f"{location}[{index}].{field} must be a string") + if not entry["id"]: + raise HTTPException(status_code=400, detail=f"{location}[{index}].id must be non-empty") + status = entry.get("status") + if not isinstance(status, str) or status not in _DELEGATION_STATUSES: + raise HTTPException(status_code=400, detail=f"{location}[{index}].status must be one of {', '.join(sorted(_DELEGATION_STATUSES))}") + for field in ("run_id", "result_brief"): + optional_text = entry.get(field) + if optional_text is not None and not isinstance(optional_text, str): + raise HTTPException(status_code=400, detail=f"{location}[{index}].{field} must be a string or null") + stop_reason = entry.get("stop_reason") + if stop_reason is not None and (not isinstance(stop_reason, str) or stop_reason not in SUBAGENT_STOP_REASON_VALUES): + raise HTTPException(status_code=400, detail=f"{location}[{index}].stop_reason must be one of {', '.join(SUBAGENT_STOP_REASON_VALUES)}") + normalized.append(dict(entry)) + return normalized + + def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, Any]: """Validate and sanitize caller-supplied state values before checkpointing. @@ -825,8 +880,9 @@ def strip_server_owned_state_metadata(values: Mapping[str, Any]) -> dict[str, An stripped[channel] = None if normalized_todos is None else [_strip_external_metadata_from_message_like(item) for item in normalized_todos] elif channel == "messages" and value is not None: stripped[channel] = _normalize_input_messages(value, location="values.messages") - elif channel == "delegations" and isinstance(value, list): - stripped[channel] = [_strip_external_delegation_verdict(item) for item in value] + elif channel == "delegations": + normalized_delegations = _normalize_external_delegations(value, location="values.delegations") + stripped[channel] = None if normalized_delegations is None else [_strip_external_delegation_verdict(item) for item in normalized_delegations] elif channel == "skill_context": normalized_skill_context = _normalize_external_skill_context(value, location="values.skill_context") stripped[channel] = None if normalized_skill_context is None else [_strip_external_metadata_from_message_like(item) for item in normalized_skill_context] @@ -903,11 +959,12 @@ def normalize_input(raw_input: dict[str, Any] | None, *, trusted_internal: bool converted = _normalize_input_messages(messages, location="input.messages", trusted_internal=trusted_internal) result = {**raw_input, "messages": converted} if not trusted_internal: - delegations = result.get("delegations") - if isinstance(delegations, list): - cleaned = [_strip_external_delegation_verdict(entry) for entry in delegations] - if cleaned != delegations: - result = {**result, "delegations": cleaned} + if "delegations" in result: + normalized_delegations = _normalize_external_delegations(result["delegations"], location="input.delegations") + result = { + **result, + "delegations": None if normalized_delegations is None else [_strip_external_delegation_verdict(entry) for entry in normalized_delegations], + } if "skill_context" in result: result = {**result, "skill_context": _normalize_external_skill_context(result["skill_context"], location="input.skill_context")} if "promoted" in result: diff --git a/backend/packages/harness/deerflow/agents/thread_state.py b/backend/packages/harness/deerflow/agents/thread_state.py index dd08454b72..8521d9022d 100644 --- a/backend/packages/harness/deerflow/agents/thread_state.py +++ b/backend/packages/harness/deerflow/agents/thread_state.py @@ -262,6 +262,25 @@ def find_delegation_key(keys: Sequence[DelegationKey], key: DelegationKey) -> De return None +_DELEGATION_LEDGER_STATUSES = frozenset({"in_progress", *SUBAGENT_STATUS_VALUES}) + + +def _usable_delegation_entries(entries: Any) -> list[DelegationEntry]: + """Drop channel values the ledger reducer cannot read. + + The gateway validates the full ledger shape at both caller-writable + boundaries, but a checkpoint written before that validation, or by an + older caller path, can still hold a non-list value, a non-object entry or + an entry without the identity key and status the reducer needs. Treating + those as absent keeps later genuine ledger writes working instead of + failing on the stored value; the reducer itself does not require the + renderer-only text fields. + """ + if not isinstance(entries, list): + return [] + return [entry for entry in entries if isinstance(entry, Mapping) and isinstance(entry.get("id"), str) and bool(entry.get("id")) and entry.get("status") in _DELEGATION_LEDGER_STATUSES] + + def merge_delegations(existing: list[DelegationEntry] | None, new: list[DelegationEntry] | None) -> list[DelegationEntry]: """Reducer for the delegation ledger. @@ -272,12 +291,14 @@ def merge_delegations(existing: list[DelegationEntry] | None, new: list[Delegati the same run. - terminal status is never overwritten by a non-terminal status. """ - if not new: - return existing or [] + existing_entries = _usable_delegation_entries(existing) + new_entries = _usable_delegation_entries(new) + if not new_entries: + return existing_entries by_key: dict[DelegationKey, DelegationEntry] = {} order: list[DelegationKey] = [] - for entry in [*(existing or []), *new]: + for entry in [*existing_entries, *new_entries]: target = find_delegation_key(order, delegation_key(entry)) previous = by_key.get(target) if target is not None else None if previous is None: diff --git a/backend/tests/test_delegation_ledger.py b/backend/tests/test_delegation_ledger.py index c91487f700..79b01e3068 100644 --- a/backend/tests/test_delegation_ledger.py +++ b/backend/tests/test_delegation_ledger.py @@ -53,6 +53,19 @@ class TestMergeDelegations: assert merge_delegations(None, [_entry("a", "in_progress")])[0]["id"] == "a" assert merge_delegations([_entry("a", "in_progress")], None)[0]["id"] == "a" + def test_unusable_stored_values_do_not_fail_later_writes(self): + """A checkpoint can hold a non-list value or an entry without the + reducer's identity key (written before the gateway validated this + channel). Later genuine writes keep working instead of failing on it.""" + assert merge_delegations(None, "oops") == [] + assert merge_delegations("oops", None) == [] + assert merge_delegations("oops", [_entry("a", "in_progress")]) == [_entry("a", "in_progress")] + assert merge_delegations([_entry("a", "in_progress")], "oops") == [_entry("a", "in_progress")] + assert merge_delegations( + [_entry("a", "in_progress"), "oops", {"status": "completed"}, {"id": "b"}], + [_entry("c", "completed")], + ) == [_entry("a", "in_progress"), _entry("c", "completed")] + def test_same_id_preserves_original_created_at(self): existing = [_entry("a", "in_progress")] new = [{**_entry("a", "completed"), "created_at": "2026-06-30T00:00:01Z", "result_sha256": "x"}] diff --git a/backend/tests/test_gateway_services.py b/backend/tests/test_gateway_services.py index c8c5199ea6..efe1dd3b64 100644 --- a/backend/tests/test_gateway_services.py +++ b/backend/tests/test_gateway_services.py @@ -789,6 +789,73 @@ def test_normalize_input_preserves_trusted_internal_delegation_verdict(): assert result["delegations"][0]["receipt_verdict"] == forged["receipt_verdict"] +@pytest.mark.parametrize( + "delegations", + [ + "oops", + {"__overwrite__": []}, + ["oops"], + [{"description": "write report", "subagent_type": "general", "status": "completed"}], + [{"id": "call-1", "subagent_type": "general", "status": "completed"}], + [{"id": "call-1", "description": "write report", "status": "completed"}], + [{"id": "", "description": "write report", "subagent_type": "general", "status": "completed"}], + [{"id": "call-1", "description": "write report", "subagent_type": "general", "status": 3}], + [{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "steering"}], + [{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "completed", "run_id": ["unhashable"]}], + [{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "completed", "result_brief": 3}], + [{"id": "call-1", "description": "write report", "subagent_type": "general", "status": "completed", "stop_reason": "nope"}], + ], + ids=[ + "non-list-string", + "overwrite-marker", + "non-object-entry", + "missing-id", + "missing-description", + "missing-subagent-type", + "empty-id", + "non-string-status", + "unknown-status", + "unhashable-run-id", + "non-string-result-brief", + "unknown-stop-reason", + ], +) +def test_external_delegations_rejected_at_both_boundaries(delegations): + """The ledger reducer keys on ``id`` and indexes ``status``: a value it cannot + read fails the state merge (or is persisted by the thread-state route and + fails every later ledger write), so both caller-writable boundaries must + answer 400 instead of passing it through.""" + from fastapi import HTTPException + + from app.gateway.services import normalize_input, strip_server_owned_state_metadata + + with pytest.raises(HTTPException) as run_error: + normalize_input({"delegations": delegations}) + assert run_error.value.status_code == 400 + + with pytest.raises(HTTPException) as state_error: + strip_server_owned_state_metadata({"delegations": delegations}) + assert state_error.value.status_code == 400 + + +def test_external_delegations_accept_the_runtime_shapes(): + from app.gateway.services import normalize_input, strip_server_owned_state_metadata + + entry = { + "id": "call-1", + "description": "write report", + "subagent_type": "general", + "status": "in_progress", + "created_at": "1970-01-01T00:00:00+00:00", + "run_id": "run-1", + "result_brief": "partial", + "stop_reason": "token_capped", + } + + assert normalize_input({"delegations": [entry]})["delegations"] == [entry] + assert strip_server_owned_state_metadata({"delegations": [entry]}) == {"delegations": [entry]} + + def test_normalize_input_preserves_trusted_internal_original_user_content(): from app.gateway.services import normalize_input from deerflow.agents.middlewares.dynamic_context_middleware import _DYNAMIC_CONTEXT_REMINDER_KEY, _REMINDER_DATE_KEY diff --git a/backend/tests/test_message_provenance.py b/backend/tests/test_message_provenance.py index 9444646004..5168f38283 100644 --- a/backend/tests/test_message_provenance.py +++ b/backend/tests/test_message_provenance.py @@ -409,7 +409,15 @@ class TestStateWritesCannotForgeServerOwnedMetadata: "additional_kwargs": forged_kwargs, } ], - "delegations": [{"id": "call-forged", "receipt_verdict": {"trusted": True}}], + "delegations": [ + { + "id": "call-forged", + "description": "write report", + "subagent_type": "general", + "status": "completed", + "receipt_verdict": {"trusted": True}, + } + ], } )