mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-12 14:00:35 +00:00
feat(hitl): add human-in-the-loop approval for high-risk tool calls (#7979)
* feat(hitl): add human-in-the-loop approval for high-risk tool calls * fix(hitl): address approval review findings * docs(hitl): keep agent guidance under size limit * fix(hitl): guard approval pause recovery * docs(hitl): shrink inherited guidance * fix(hitl): harden approval admission and recovery
This commit is contained in:
parent
3b05ab7dc3
commit
0d04322d41
10
CHANGELOG.md
10
CHANGELOG.md
@ -11,6 +11,16 @@ This section accumulates work toward the **2.3.0** milestone
|
||||
([2.3.0](https://github.com/bytedance/deer-flow/milestone/4)).
|
||||
|
||||
### Added
|
||||
- **hitl:** Optional human-in-the-loop approval for high-risk tool calls.
|
||||
`hitl_approval.tools` (host-owned) pauses matching calls via LangGraph
|
||||
`interrupt()`; the run enters `waiting_for_input` and the Web UI shows an
|
||||
approval card with per-field argument summaries. Approve executes the call,
|
||||
reject returns the reason to the model without executing. IM channels,
|
||||
GitHub runs, and scheduled tasks follow `hitl_approval.non_interactive`
|
||||
(`deny` by default, `allow` opt-in). Subagents and runtimes without resume
|
||||
support always reject matching calls regardless of that setting.
|
||||
`hitl_approval.max_actions` configures the approval-batch count limit
|
||||
(default 32; 0 disables it). HITL is disabled by default; `config_version` 65.
|
||||
|
||||
- **subagents:** Durable batch items can wait for explicitly named prerequisites.
|
||||
An optional, once-only lead-Agent follow-up completes a previously requested
|
||||
|
||||
@ -342,6 +342,13 @@
|
||||
选择器缺失、是目录或不可读时会在调用 Docker 之前直接失败,而文件删除后
|
||||
`make down` 仍能停止整个栈;不设置或为空的选择器保持可选的仓库根 `.env`
|
||||
默认值。([#6522])
|
||||
- **HITL:** 可选的高风险工具调用人工审批。`hitl_approval.tools`(由宿主管理)通过
|
||||
LangGraph `interrupt()` 暂停匹配的调用,运行进入 `waiting_for_input`,Web UI 展示
|
||||
带逐字段参数摘要的审批卡片。批准后执行调用;拒绝则不执行,并将原因返回给模型。
|
||||
IM 渠道、GitHub 运行和定时任务遵循 `hitl_approval.non_interactive`(默认 `deny`,
|
||||
可选 `allow`)。子代理及不支持 resume 的运行环境无论该设置如何,始终拒绝匹配的
|
||||
调用。`hitl_approval.max_actions` 配置审批批次的调用数量上限(默认 32,0 表示关闭)。
|
||||
默认关闭;`config_version` 65。
|
||||
|
||||
- **MCP:** 按未变更的服务器缓存完成包装的工具,仅在绑定对账后重新发现缺失或配置
|
||||
发生变化的服务器。将成功但未发现工具的结果与发现失败分开保存,并保留代际、共享
|
||||
|
||||
14
README.md
14
README.md
@ -1775,6 +1775,20 @@ actions remain visible beside the compact sidebar on narrow screens.
|
||||
|
||||
### Skills & Tools
|
||||
|
||||
**Human approval for high-risk tools**: Set `hitl_approval.enabled: true` and
|
||||
list tools such as `write_file`, `str_replace`, or `bash` in `config.yaml` to
|
||||
pause matching calls before execution. The Web UI shows an approval card for
|
||||
each action; approval attempts execution under the existing guardrails and
|
||||
sandbox authorization, while rejection returns the reason to the agent without
|
||||
running the tool. Non-interactive runs follow `hitl_approval.non_interactive`;
|
||||
subagents and runtimes without resume support always fail closed.
|
||||
|
||||
Pending approvals do not expire and block new runs on the same thread until
|
||||
resolved. Long argument values may be truncated in the display, but approval
|
||||
executes the original complete arguments. The feature is disabled by default.
|
||||
See [HITL Tool Approval](backend/docs/CONFIGURATION.md#hitl-tool-approval) for
|
||||
configuration and safety semantics.
|
||||
|
||||
Agents can call the read-only `inspect_run_status` tool to inspect visible run
|
||||
token usage, message and tool-call counts, an approximate context-window fill,
|
||||
and whether a summary or unfinished to-dos exist. It returns counts only, not
|
||||
|
||||
@ -299,7 +299,7 @@ multi-step tasks: one `in_progress` task, real-time updates. See
|
||||
|
||||
### Run Interaction Policy
|
||||
|
||||
Interaction-sensitive changes must follow [policy](docs/RUN_INTERACTION_POLICY.md).
|
||||
Interaction/HITL rules: [policy](docs/RUN_INTERACTION_POLICY.md).
|
||||
|
||||
### Context Summarization
|
||||
|
||||
|
||||
@ -6,6 +6,26 @@ from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
|
||||
def task_interrupts_by_id(snapshot: Any) -> dict[str, list[Any]]:
|
||||
"""Group non-empty task interrupts of one snapshot by task id."""
|
||||
grouped: dict[str, list[Any]] = {}
|
||||
for task in getattr(snapshot, "tasks", None) or ():
|
||||
interrupts = list(getattr(task, "interrupts", ()) or ())
|
||||
if interrupts:
|
||||
grouped[task.id] = interrupts
|
||||
return grouped
|
||||
|
||||
|
||||
def is_hitl_approval_payload(value: Any) -> bool:
|
||||
"""Whether an interrupt payload is a HITL tool-approval request."""
|
||||
return isinstance(value, dict) and value.get("kind") == "hitl_approval"
|
||||
|
||||
|
||||
def hitl_approval_payloads(grouped: dict[str, list[Any]]) -> list[dict[str, Any]]:
|
||||
"""Flatten approval payloads from task_interrupts_by_id output."""
|
||||
return [interrupt.value for interrupts in grouped.values() for interrupt in interrupts if is_hitl_approval_payload(interrupt.value)]
|
||||
|
||||
|
||||
class CheckpointLineageError(RuntimeError):
|
||||
"""Raised when a requested checkpoint ancestor cannot be resolved safely."""
|
||||
|
||||
|
||||
302
backend/app/gateway/hitl_approval.py
Normal file
302
backend/app/gateway/hitl_approval.py
Normal file
@ -0,0 +1,302 @@
|
||||
"""Server-side validation for HITL tool-approval resume and admission.
|
||||
|
||||
All checks live here (called from ``services.start_run``) so every entry point
|
||||
— HTTP routes, extension host, internal launchers — shares one choke point.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.gateway.checkpoint_lineage import hitl_approval_payloads
|
||||
from deerflow.runtime.checkpoint_history import recent_checkpoint_history
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.runtime.runs.manager import ApprovalAnswerHandle
|
||||
|
||||
|
||||
def _error(status: int, code: str, message: str) -> HTTPException:
|
||||
return HTTPException(status_code=status, detail={"code": code, "message": message})
|
||||
|
||||
|
||||
def _responses_of(command: Any) -> list[Any] | None:
|
||||
if not isinstance(command, Mapping):
|
||||
return None
|
||||
resume = command.get("resume")
|
||||
if not isinstance(resume, Mapping):
|
||||
return None
|
||||
responses = resume.get("responses")
|
||||
return responses if isinstance(responses, list) else None
|
||||
|
||||
|
||||
def _all_reject(responses: list[Any] | None) -> bool:
|
||||
return bool(responses) and all(isinstance(r, Mapping) and r.get("option_id") == "reject" for r in responses)
|
||||
|
||||
|
||||
async def has_any_pending_interrupt(accessor: Any, read_config: dict[str, Any]) -> bool:
|
||||
"""Whether a recent checkpoint has an unconsumed interrupt write.
|
||||
Used for generic (non-hitl) resume admission where snapshot.next may be
|
||||
empty after a head-moving write. The newest resume/interrupt marker wins,
|
||||
so an answered pause cannot be resurrected by its older interrupt write.
|
||||
"""
|
||||
async for checkpoint in recent_checkpoint_history(accessor.checkpointer, read_config):
|
||||
writes = list(getattr(checkpoint, "pending_writes", ()) or ())
|
||||
if any(len(write) >= 2 and write[1] == "__resume__" for write in writes):
|
||||
return False
|
||||
if any(len(write) >= 2 and write[1] == "__interrupt__" for write in writes):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def _pending_interrupts_from_checkpoint_history(checkpointer: Any, read_config: dict[str, Any]) -> dict[str, list[Any]]:
|
||||
"""Read pending interrupts without materializing a graph state snapshot."""
|
||||
async for checkpoint in recent_checkpoint_history(checkpointer, read_config):
|
||||
writes = list(getattr(checkpoint, "pending_writes", ()) or ())
|
||||
if any(len(write) >= 2 and write[1] == "__resume__" for write in writes):
|
||||
return {}
|
||||
interrupt_writes = [write for write in writes if len(write) >= 3 and write[1] == "__interrupt__"]
|
||||
if not interrupt_writes:
|
||||
continue
|
||||
grouped: dict[str, list[Any]] = {}
|
||||
for write in interrupt_writes:
|
||||
interrupts = write[2] if isinstance(write[2], (list, tuple)) else (write[2],)
|
||||
grouped.setdefault(str(write[0]), []).extend(interrupts)
|
||||
return grouped
|
||||
return {}
|
||||
|
||||
|
||||
async def has_waiting_for_input_run(request: Any, thread_id: str) -> bool:
|
||||
"""Whether any run in the thread is paused for input (any interrupt kind)."""
|
||||
from app.gateway.deps import get_run_store
|
||||
|
||||
try:
|
||||
run_store = get_run_store(request)
|
||||
except (HTTPException, AttributeError) as exc:
|
||||
if isinstance(exc, AttributeError) or exc.status_code == 503:
|
||||
return False
|
||||
raise
|
||||
runs = await run_store.list_by_thread(thread_id, limit=5)
|
||||
return any(run.get("status") == "waiting_for_input" for run in runs)
|
||||
|
||||
|
||||
async def mark_approval_answered(request: Any, thread_id: str, *, expected_request_ids: dict[str, str] | None = None) -> ApprovalAnswerHandle | None:
|
||||
"""Mark the validated paused run as answered and return its recovery handle."""
|
||||
from app.gateway.deps import get_run_manager, get_run_store
|
||||
|
||||
try:
|
||||
run_manager = get_run_manager(request)
|
||||
except (HTTPException, AttributeError) as exc:
|
||||
if isinstance(exc, HTTPException) and exc.status_code != 503:
|
||||
raise
|
||||
# Lightweight compositions may provide only a store. Keep their
|
||||
# persistence on RunManager's retry and in-memory mutation path too.
|
||||
from deerflow.runtime import RunManager
|
||||
|
||||
try:
|
||||
run_manager = RunManager(store=get_run_store(request))
|
||||
except (HTTPException, AttributeError) as store_exc:
|
||||
if isinstance(store_exc, AttributeError) or store_exc.status_code == 503:
|
||||
return None
|
||||
raise
|
||||
return await run_manager.mark_approval_answered(thread_id, expected_request_ids=expected_request_ids)
|
||||
|
||||
|
||||
async def _interrupt_from_checkpoint_history(accessor: Any, read_config: dict[str, Any], payload: dict[str, Any]) -> tuple[str, Any] | None:
|
||||
"""Recover the original LangGraph task identity after the head has moved."""
|
||||
if accessor is None:
|
||||
return None
|
||||
if getattr(accessor, "checkpointer", None) is None:
|
||||
return None
|
||||
async for checkpoint in recent_checkpoint_history(accessor.checkpointer, read_config):
|
||||
writes = list(getattr(checkpoint, "pending_writes", ()) or ())
|
||||
if any(len(write) >= 2 and write[1] == "__resume__" for write in writes):
|
||||
return None
|
||||
for write in writes:
|
||||
if len(write) < 3 or write[1] != "__interrupt__":
|
||||
continue
|
||||
interrupts = write[2] if isinstance(write[2], (list, tuple)) else (write[2],)
|
||||
for pending_interrupt in interrupts:
|
||||
if getattr(pending_interrupt, "value", None) == payload:
|
||||
return str(write[0]), pending_interrupt
|
||||
return None
|
||||
|
||||
|
||||
async def find_pending_interrupts(
|
||||
request: Any,
|
||||
thread_id: str,
|
||||
*,
|
||||
accessor: Any = None,
|
||||
read_config: dict[str, Any] | None = None,
|
||||
snapshot: Any = None,
|
||||
) -> dict[str, list[Any]]:
|
||||
"""Find pending interrupts from an existing snapshot or raw checkpoints.
|
||||
|
||||
The live snapshot catches a pause before the worker has finalized the run;
|
||||
raw pending writes cover head-moving writes and restricted mutation graphs;
|
||||
the durable run record remains the final fallback.
|
||||
"""
|
||||
from langgraph.types import Interrupt
|
||||
|
||||
from app.gateway.checkpoint_lineage import task_interrupts_by_id
|
||||
from app.gateway.deps import get_checkpointer, get_run_store
|
||||
|
||||
if snapshot is None and accessor is not None and read_config is not None:
|
||||
snapshot = await accessor.aget(read_config)
|
||||
if snapshot is not None:
|
||||
grouped = task_interrupts_by_id(snapshot)
|
||||
if grouped:
|
||||
return grouped
|
||||
|
||||
checkpointer = getattr(accessor, "checkpointer", None)
|
||||
if checkpointer is None:
|
||||
try:
|
||||
checkpointer = get_checkpointer(request)
|
||||
except (HTTPException, AttributeError) as exc:
|
||||
if not isinstance(exc, AttributeError) and exc.status_code != 503:
|
||||
raise
|
||||
if checkpointer is not None:
|
||||
grouped = await _pending_interrupts_from_checkpoint_history(
|
||||
checkpointer,
|
||||
read_config or {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}},
|
||||
)
|
||||
if grouped:
|
||||
return grouped
|
||||
|
||||
try:
|
||||
run_store = get_run_store(request)
|
||||
except (HTTPException, AttributeError) as exc:
|
||||
if isinstance(exc, AttributeError) or exc.status_code == 503:
|
||||
return {}
|
||||
raise
|
||||
runs = await run_store.list_by_thread(thread_id, limit=10)
|
||||
for run in runs:
|
||||
if run.get("status") != "waiting_for_input":
|
||||
continue
|
||||
payload = (run.get("metadata") or {}).get("hitl_approval")
|
||||
if payload:
|
||||
recovered = await _interrupt_from_checkpoint_history(accessor, read_config, payload) if read_config is not None else None
|
||||
if recovered is not None:
|
||||
task_id, pending_interrupt = recovered
|
||||
return {task_id: [pending_interrupt]}
|
||||
task_id = run.get("run_id") or "approval"
|
||||
return {task_id: [Interrupt(value=payload, id=payload.get("request_id", task_id))]}
|
||||
return {}
|
||||
|
||||
|
||||
def _approval_request_ids(payload: dict[str, Any]) -> dict[str, str]:
|
||||
"""Extract and validate the action identity binding for one HITL pause."""
|
||||
expected_ids = payload.get("pending_tool_call_ids")
|
||||
if not isinstance(expected_ids, list) or any(not isinstance(call_id, str) for call_id in expected_ids) or len(set(expected_ids)) != len(expected_ids):
|
||||
raise _error(422, "invalid_approval_response", "pending approval has invalid tool_call_ids")
|
||||
expected_set = set(expected_ids)
|
||||
actions = payload.get("actions")
|
||||
if not isinstance(actions, list):
|
||||
raise _error(422, "invalid_approval_response", "pending approval has invalid actions")
|
||||
expected_request_ids: dict[str, str] = {}
|
||||
for action in actions:
|
||||
if not isinstance(action, Mapping):
|
||||
raise _error(422, "invalid_approval_response", "pending approval has invalid actions")
|
||||
call_id = action.get("tool_call_id")
|
||||
request_id = action.get("request_id")
|
||||
if not isinstance(call_id, str) or call_id not in expected_set or call_id in expected_request_ids or not isinstance(request_id, str) or not request_id:
|
||||
raise _error(422, "invalid_approval_response", "pending approval has invalid actions")
|
||||
expected_request_ids[call_id] = request_id
|
||||
if set(expected_request_ids) != expected_set:
|
||||
raise _error(422, "invalid_approval_response", "pending approval has invalid actions")
|
||||
return expected_request_ids
|
||||
|
||||
|
||||
async def current_approval_request_ids(
|
||||
request: Any,
|
||||
thread_id: str,
|
||||
*,
|
||||
accessor: Any,
|
||||
read_config: dict[str, Any],
|
||||
) -> dict[str, str] | None:
|
||||
"""Read the current live pause's identity for checkpoint-only recovery."""
|
||||
pending = await find_pending_interrupts(request, thread_id, accessor=accessor, read_config=read_config)
|
||||
payloads = hitl_approval_payloads(pending)
|
||||
return _approval_request_ids(payloads[0]) if payloads else None
|
||||
|
||||
|
||||
async def validate_resume_request(*, request: Any, thread_id: str, accessor: Any, read_config: dict[str, Any], enabled: bool, body: Any) -> dict[str, str] | None:
|
||||
"""Validate a command.resume against the current pause, or raise 409/422.
|
||||
|
||||
Static breakpoints (next non-empty, no approval payload) pass through
|
||||
unchanged. All-reject remains the escape hatch for the validated pause when
|
||||
the feature is disabled or the payload was lost; attachment later verifies
|
||||
that checkpoint-only recovery still targets that same pause.
|
||||
"""
|
||||
snapshot = await accessor.aget(read_config)
|
||||
responses = _responses_of(getattr(body, "command", None))
|
||||
all_reject = _all_reject(responses)
|
||||
pending = await find_pending_interrupts(request, thread_id, accessor=accessor, read_config=read_config, snapshot=snapshot)
|
||||
payloads = hitl_approval_payloads(pending)
|
||||
has_waiting_run = await has_waiting_for_input_run(request, thread_id)
|
||||
has_pending_interrupt = await has_any_pending_interrupt(accessor, read_config)
|
||||
head_checkpoint_id = (getattr(snapshot, "config", None) or {}).get("configurable", {}).get("checkpoint_id")
|
||||
if not pending and not has_waiting_run and not (getattr(snapshot, "next", None) or ()) and not has_pending_interrupt:
|
||||
raise _error(409, "no_pending_approval", "No resumable pause exists for this thread")
|
||||
|
||||
if not payloads:
|
||||
return None
|
||||
|
||||
if responses is None:
|
||||
raise _error(422, "invalid_approval_response", "resume must carry a 'responses' list for hitl_approval")
|
||||
expected_request_ids = _approval_request_ids(payloads[0])
|
||||
expected_ids = list(expected_request_ids)
|
||||
expected_set = set(expected_ids)
|
||||
if len(responses) != len(expected_ids):
|
||||
raise _error(422, "invalid_approval_response", f"expected {len(expected_ids)} responses, got {len(responses)}")
|
||||
seen_ids: set[str] = set()
|
||||
for response in responses:
|
||||
if not isinstance(response, Mapping):
|
||||
raise _error(422, "invalid_approval_response", "each response must be an object")
|
||||
response_id = response.get("tool_call_id")
|
||||
if not isinstance(response_id, str) or response_id not in expected_set:
|
||||
raise _error(422, "invalid_approval_response", "unknown tool_call_id")
|
||||
if response.get("request_id") != expected_request_ids[response_id]:
|
||||
raise _error(422, "invalid_approval_response", "stale or mismatched request_id")
|
||||
if response_id in seen_ids:
|
||||
raise _error(422, "invalid_approval_response", "duplicate tool_call_id")
|
||||
seen_ids.add(response_id)
|
||||
if response.get("option_id") not in ("approve", "reject"):
|
||||
raise _error(422, "invalid_approval_response", "unsupported option_id; expected 'approve' or 'reject'")
|
||||
|
||||
if all_reject:
|
||||
return expected_request_ids
|
||||
|
||||
if not enabled:
|
||||
raise _error(409, "hitl_approval_disabled", "hitl_approval is currently disabled; resume with all-reject to clear the pause")
|
||||
|
||||
requested_checkpoint_id = getattr(body, "checkpoint_id", None)
|
||||
body_checkpoint = getattr(body, "checkpoint", None)
|
||||
if requested_checkpoint_id is None and isinstance(body_checkpoint, Mapping):
|
||||
requested_checkpoint_id = body_checkpoint.get("checkpoint_id")
|
||||
if requested_checkpoint_id and head_checkpoint_id and str(requested_checkpoint_id) != str(head_checkpoint_id):
|
||||
raise _error(422, "checkpoint_not_head", "A resume must target the current head checkpoint, never an older pause checkpoint")
|
||||
return expected_request_ids
|
||||
|
||||
|
||||
async def reject_new_run_when_approval_pending(
|
||||
request: Any,
|
||||
thread_id: str,
|
||||
*,
|
||||
accessor: Any = None,
|
||||
read_config: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Reject a brand-new (non-resume) run while an approval is pending.
|
||||
|
||||
Without this guard a fresh input silently discards the pending approval
|
||||
(measured behavior of LangGraph). Internal launchers catch this specific
|
||||
error code to implement skip-and-retry semantics.
|
||||
"""
|
||||
pending = await find_pending_interrupts(request, thread_id, accessor=accessor, read_config=read_config)
|
||||
if hitl_approval_payloads(pending):
|
||||
raise _error(409, "approval_pending", "Thread has a pending tool approval; resume or reject it before starting a new run")
|
||||
@ -39,6 +39,7 @@ from app.gateway.checkpoint_lineage import (
|
||||
)
|
||||
from app.gateway.deps import get_checkpointer, get_run_event_store, get_run_manager, get_run_store
|
||||
from app.gateway.history_seed import reserve_history_seed, write_history_seed
|
||||
from app.gateway.hitl_approval import find_pending_interrupts
|
||||
from app.gateway.internal_auth import get_trusted_internal_owner_user_id
|
||||
from app.gateway.services import (
|
||||
abuild_checkpoint_state_accessor,
|
||||
@ -81,6 +82,7 @@ from deerflow.runtime.run_origin import DEERFLOW_ORIGIN_KEY, admit_origin
|
||||
from deerflow.runtime.runs.manager import ConflictError
|
||||
from deerflow.runtime.runs.worker import RUN_MESSAGE_IDS_METADATA_KEY, valid_duration_entry, valid_run_message_id_entry
|
||||
from deerflow.runtime.secret_context import redact_metadata_secrets
|
||||
from deerflow.runtime.serialization import serialize_lc_object
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.uploads.companions import register_companion, resolve_companion
|
||||
from deerflow.utils.file_io import await_drained, run_file_io
|
||||
@ -570,7 +572,7 @@ class ThreadPatchRequest(BaseModel):
|
||||
|
||||
|
||||
class ThreadStateUpdateRequest(BaseModel):
|
||||
"""Request body for updating thread state (human-in-the-loop resume)."""
|
||||
"""Request body for updating thread state (checkpoint mutation, not a resume API)."""
|
||||
|
||||
values: dict[str, Any] | None = Field(default=None, description="Channel values to merge")
|
||||
checkpoint_id: str | None = Field(default=None, description="Checkpoint to branch from")
|
||||
@ -623,10 +625,13 @@ class HistoryEntry(_MetadataRedactingResponse):
|
||||
|
||||
checkpoint_id: str
|
||||
parent_checkpoint_id: str | None = None
|
||||
created_at: str | None = Field(default=None, description="Checkpoint timestamp")
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
values: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: str | None = None
|
||||
next: list[str] = Field(default_factory=list)
|
||||
# Only the head entry carries pending interrupts; older entries are history,
|
||||
# never a pending-approval signal. Shape: {id, name, interrupts} only.
|
||||
tasks: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ThreadHistoryRequest(BaseModel):
|
||||
@ -1485,6 +1490,7 @@ async def get_thread(thread_id: ThreadId, request: Request) -> ThreadResponse:
|
||||
updated_at=coerce_iso(record.get("updated_at", "")),
|
||||
metadata=record.get("metadata", {}),
|
||||
values=serialize_channel_values_for_api(snapshot.values),
|
||||
interrupts={task_id: [serialize_lc_object(interrupt) for interrupt in interrupts] for task_id, interrupts in (await find_pending_interrupts(request, thread_id, accessor=accessor, read_config=config, snapshot=snapshot)).items()},
|
||||
)
|
||||
|
||||
|
||||
@ -1510,6 +1516,13 @@ async def set_thread_goal(thread_id: ThreadId, body: ThreadGoalRequest, request:
|
||||
this endpoint creates the missing thread checkpoint on demand.
|
||||
"""
|
||||
checkpointer = get_checkpointer(request)
|
||||
try:
|
||||
accessor, read_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
from app.gateway.hitl_approval import reject_new_run_when_approval_pending
|
||||
|
||||
await reject_new_run_when_approval_pending(request, thread_id, accessor=accessor, read_config=read_config)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
raise _checkpoint_mode_http_error(exc, thread_id) from exc
|
||||
try:
|
||||
goal = build_goal_state(body.objective, max_continuations=body.max_continuations)
|
||||
async with reserve_checkpoint_write(request, thread_id, user_id=get_effective_user_id()):
|
||||
@ -1535,6 +1548,13 @@ async def set_thread_goal(thread_id: ThreadId, body: ThreadGoalRequest, request:
|
||||
async def clear_thread_goal(thread_id: ThreadId, request: Request) -> ThreadGoalResponse:
|
||||
"""Clear the active goal for a thread."""
|
||||
checkpointer = get_checkpointer(request)
|
||||
try:
|
||||
accessor, read_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id)
|
||||
from app.gateway.hitl_approval import reject_new_run_when_approval_pending
|
||||
|
||||
await reject_new_run_when_approval_pending(request, thread_id, accessor=accessor, read_config=read_config)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
raise _checkpoint_mode_http_error(exc, thread_id) from exc
|
||||
try:
|
||||
async with reserve_checkpoint_write(request, thread_id, user_id=get_effective_user_id()):
|
||||
await write_thread_goal(checkpointer, thread_id, None, as_node="goal")
|
||||
@ -1573,11 +1593,14 @@ async def compact_thread(thread_id: ThreadId, body: ThreadCompactRequest, reques
|
||||
# forward by checkpoint fork inheritance, so the base-schema mutation
|
||||
# graph is sufficient (and avoids building the full lead graph per call).
|
||||
try:
|
||||
accessor, _ = build_checkpoint_state_mutation_accessor(
|
||||
accessor, read_config = build_checkpoint_state_mutation_accessor(
|
||||
request,
|
||||
thread_id=thread_id,
|
||||
as_node="manual_compaction",
|
||||
)
|
||||
from app.gateway.hitl_approval import reject_new_run_when_approval_pending
|
||||
|
||||
await reject_new_run_when_approval_pending(request, thread_id)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
raise _checkpoint_mode_http_error(exc, thread_id) from exc
|
||||
keep = body.keep.to_tuple() if body.keep is not None else None
|
||||
@ -1612,6 +1635,32 @@ async def compact_thread(thread_id: ThreadId, body: ThreadCompactRequest, reques
|
||||
return _thread_compact_response(result)
|
||||
|
||||
|
||||
def _pending_interrupt_tasks(tasks_raw: Any, pending_interrupts: dict[str, list[Any]]) -> list[dict[str, Any]]:
|
||||
"""Project pending interrupts without dropping run-store-only identities."""
|
||||
tasks: list[dict[str, Any]] = []
|
||||
matched_task_ids: set[str] = set()
|
||||
for task in tasks_raw:
|
||||
task_id = getattr(task, "id", "")
|
||||
matched_task_ids.add(task_id)
|
||||
tasks.append(
|
||||
{
|
||||
"id": task_id,
|
||||
"name": getattr(task, "name", ""),
|
||||
"interrupts": [serialize_lc_object(interrupt) for interrupt in pending_interrupts.get(task_id, [])],
|
||||
}
|
||||
)
|
||||
for task_id, interrupts in pending_interrupts.items():
|
||||
if task_id not in matched_task_ids:
|
||||
tasks.append(
|
||||
{
|
||||
"id": task_id,
|
||||
"name": "",
|
||||
"interrupts": [serialize_lc_object(interrupt) for interrupt in interrupts],
|
||||
}
|
||||
)
|
||||
return tasks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@router.get("/{thread_id}/state", response_model=ThreadStateResponse)
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
@ -1635,13 +1684,12 @@ async def get_thread_state(thread_id: ThreadId, request: Request) -> ThreadState
|
||||
checkpoint_id = snapshot_config.get("configurable", {}).get("checkpoint_id")
|
||||
if not checkpoint_id:
|
||||
raise HTTPException(status_code=404, detail=f"Thread {thread_id} not found")
|
||||
|
||||
parent_config = snapshot.parent_config or {}
|
||||
pending_interrupts = await find_pending_interrupts(request, thread_id, accessor=accessor, read_config=config, snapshot=snapshot)
|
||||
parent_checkpoint_id = parent_config.get("configurable", {}).get("checkpoint_id")
|
||||
metadata = snapshot.metadata or {}
|
||||
created_at = snapshot.created_at or metadata.get("created_at", "")
|
||||
tasks_raw = snapshot.tasks or ()
|
||||
tasks = [{"id": getattr(task, "id", ""), "name": getattr(task, "name", "")} for task in tasks_raw]
|
||||
tasks = _pending_interrupt_tasks(snapshot.tasks or (), pending_interrupts)
|
||||
|
||||
values = serialize_channel_values_for_api(snapshot.values)
|
||||
messages = values.get("messages")
|
||||
@ -1667,7 +1715,9 @@ async def get_thread_state(thread_id: ThreadId, request: Request) -> ThreadState
|
||||
async def update_thread_state(thread_id: ThreadId, body: ThreadStateUpdateRequest, request: Request) -> ThreadStateResponse:
|
||||
"""Replace selected thread-state fields through the materialized graph."""
|
||||
from app.gateway.deps import get_thread_store
|
||||
from app.gateway.hitl_approval import reject_new_run_when_approval_pending
|
||||
|
||||
await reject_new_run_when_approval_pending(request, thread_id)
|
||||
thread_store = get_thread_store(request)
|
||||
# Validate external roles before materializing a graph or reserving a write.
|
||||
values = strip_server_owned_state_metadata(dict(body.values or {}))
|
||||
@ -2082,10 +2132,14 @@ async def get_thread_history(
|
||||
serialized_msgs,
|
||||
)
|
||||
|
||||
is_latest_checkpoint = False
|
||||
|
||||
next_tasks = list(snapshot.next or ())
|
||||
if is_latest_checkpoint:
|
||||
pending_interrupts = await find_pending_interrupts(request, thread_id, accessor=accessor, read_config=config, snapshot=snapshot)
|
||||
head_tasks = _pending_interrupt_tasks(getattr(snapshot, "tasks", None) or (), pending_interrupts)
|
||||
else:
|
||||
head_tasks = [{"id": getattr(task, "id", ""), "name": getattr(task, "name", "")} for task in (getattr(snapshot, "tasks", None) or ())]
|
||||
|
||||
is_latest_checkpoint = False
|
||||
# Strip LangGraph internal keys from metadata
|
||||
user_meta = {k: v for k, v in metadata.items() if k not in ("created_at", "updated_at", "step", "source", "writes", "parents", "run_durations", RUN_MESSAGE_IDS_METADATA_KEY)}
|
||||
# Keep step for ordering context
|
||||
@ -2100,6 +2154,7 @@ async def get_thread_history(
|
||||
values=values,
|
||||
created_at=coerce_iso(snapshot.created_at or metadata.get("created_at", "")),
|
||||
next=next_tasks,
|
||||
tasks=head_tasks,
|
||||
)
|
||||
)
|
||||
except _CHECKPOINT_MODE_ERRORS as exc:
|
||||
|
||||
@ -30,7 +30,7 @@ from langgraph.types import Command
|
||||
|
||||
from app.gateway.auth_disabled import AUTH_DISABLED_USER_ID, AUTH_SOURCE_INTERNAL, get_auth_disabled_user, is_auth_disabled
|
||||
from app.gateway.authz import require_cancel_permission_if, resolve_route_permissions
|
||||
from app.gateway.deps import get_checkpointer, get_local_provider, get_run_context, get_run_manager, get_stream_bridge
|
||||
from app.gateway.deps import get_checkpointer, get_local_provider, get_run_context, get_run_manager, get_run_store, get_stream_bridge
|
||||
from app.gateway.history_seed import reserve_history_seed, write_history_seed
|
||||
from app.gateway.internal_auth import (
|
||||
INTERNAL_OWNER_USER_ID_HEADER_NAME,
|
||||
@ -106,7 +106,7 @@ from deerflow.runtime.secret_context import (
|
||||
validate_run_metadata_secrets,
|
||||
)
|
||||
from deerflow.runtime.stream_modes import normalize_stream_modes
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
from deerflow.runtime.user_context import get_current_user, reset_current_user, set_current_user
|
||||
from deerflow.sandbox.lease import SANDBOX_SERVER_OWNED_CONTEXT_KEYS
|
||||
from deerflow.scheduler.runtime import scheduler_tools_enabled
|
||||
from deerflow.scheduler.schedules import validate_timezone
|
||||
@ -158,6 +158,7 @@ async def reserve_checkpoint_write(
|
||||
|
||||
_TERMINAL_RUN_STATUSES = {
|
||||
RunStatus.success,
|
||||
RunStatus.waiting_for_input,
|
||||
RunStatus.error,
|
||||
RunStatus.timeout,
|
||||
RunStatus.interrupted,
|
||||
@ -939,6 +940,45 @@ def _legacy_stored_idempotency_request(kwargs: dict[str, Any]) -> dict[str, Any]
|
||||
return identity
|
||||
|
||||
|
||||
async def _find_admitted_run_for_idempotency_key(
|
||||
run_mgr: RunManager,
|
||||
thread_id: str,
|
||||
idempotency_key: str,
|
||||
*,
|
||||
user_id: str | None,
|
||||
) -> RunRecord | None:
|
||||
"""Resolve an existing key on this thread and owner before pause checks."""
|
||||
before_created_at: str | None = None
|
||||
before_run_id: str | None = None
|
||||
while True:
|
||||
records = await run_mgr.list_by_thread(
|
||||
thread_id,
|
||||
user_id=user_id,
|
||||
limit=100,
|
||||
before_created_at=before_created_at,
|
||||
before_run_id=before_run_id,
|
||||
)
|
||||
for record in records:
|
||||
if record.idempotency_key == idempotency_key and record.user_id == user_id:
|
||||
return record
|
||||
if len(records) < 100:
|
||||
return None
|
||||
cursor = records[-1]
|
||||
before_created_at = cursor.created_at
|
||||
before_run_id = cursor.run_id
|
||||
|
||||
|
||||
def _is_retryable_resume_start_failure(record: RunRecord | None) -> bool:
|
||||
"""Whether admission failed before a resume worker could attach."""
|
||||
return record is not None and record.status == RunStatus.error and isinstance(record.error, str) and record.error.startswith("Failed to attach run worker:")
|
||||
|
||||
|
||||
async def _discard_failed_resume_admission(request: Request, run_mgr: RunManager, record: RunRecord) -> None:
|
||||
"""Release a failed admission's key so an identical retry can start work."""
|
||||
await get_run_store(request).delete(record.run_id, user_id=record.user_id)
|
||||
await run_mgr.cleanup(record.run_id, delay=0)
|
||||
|
||||
|
||||
_DEFAULT_ASSISTANT_ID = "lead_agent"
|
||||
|
||||
|
||||
@ -2263,6 +2303,7 @@ async def start_run(
|
||||
admission_run_id: str | None = None
|
||||
producer_lease_started = False
|
||||
producer_lease_handed_off = False
|
||||
validated_approval_request_ids: dict[str, str] | None = None
|
||||
|
||||
disconnect = DisconnectMode.cancel if body.on_disconnect == "cancel" else DisconnectMode.continue_
|
||||
|
||||
@ -2335,10 +2376,40 @@ async def start_run(
|
||||
if admitted_origin not in EVIDENCE_ORIGINS:
|
||||
raise ValueError("Unsupported host evidence origin")
|
||||
command = getattr(body, "command", None)
|
||||
from app.gateway.hitl_approval import reject_new_run_when_approval_pending, validate_resume_request
|
||||
|
||||
current_user = get_current_user()
|
||||
run_user_id = str(current_user.id) if current_user is not None else None
|
||||
admitted_run = (
|
||||
await _find_admitted_run_for_idempotency_key(
|
||||
run_mgr,
|
||||
thread_id,
|
||||
idempotency_key,
|
||||
user_id=run_user_id,
|
||||
)
|
||||
if idempotency_key is not None
|
||||
else None
|
||||
)
|
||||
if command and command.get("resume") is not None:
|
||||
graph_input = Command(resume=command["resume"])
|
||||
stored_idempotency_request = (admitted_run.idempotency_request or _legacy_stored_idempotency_request(admitted_run.kwargs or {})) if admitted_run is not None else None
|
||||
retry_failed_resume = _is_retryable_resume_start_failure(admitted_run) and stored_idempotency_request == idempotency_request
|
||||
if admitted_run is None or retry_failed_resume:
|
||||
accessor, read_config = await build_thread_checkpoint_state_accessor(
|
||||
request,
|
||||
thread_id=thread_id,
|
||||
)
|
||||
validated_approval_request_ids = await validate_resume_request(
|
||||
request=request,
|
||||
thread_id=thread_id,
|
||||
accessor=accessor,
|
||||
read_config=read_config,
|
||||
enabled=get_app_config().hitl_approval.enabled,
|
||||
body=body,
|
||||
)
|
||||
else:
|
||||
# LangGraph continues pending work with None; {} starts new input.
|
||||
if admitted_run is None:
|
||||
await reject_new_run_when_approval_pending(request, thread_id)
|
||||
graph_input = normalized_input if body.input is not None else None
|
||||
# Keep the idempotency identity in the caller's pre-stamp shape. A
|
||||
# keyed retry can then reuse a run even though its stored run input has
|
||||
@ -2745,55 +2816,64 @@ async def start_run(
|
||||
)
|
||||
if start_producer_lease is not None:
|
||||
producer_lease_started = await start_producer_lease(admission_run_id)
|
||||
if not producer_lease_started and getattr(bridge, "supports_cross_process", False) is True:
|
||||
# Best-effort: a cross-process bridge returns False only
|
||||
# after a transient backend error whose partial lease
|
||||
# registration was safely rolled back (nothing is left
|
||||
# tracked). An untracked producer falls back to the
|
||||
# historical conservative recovery path -- wait for the
|
||||
# real END or the retained-stream TTL rather than
|
||||
# synthesizing one -- so this run is no less safe than it
|
||||
# was before producer leases existed. Rejecting admission
|
||||
# here would instead turn a transient blip into a 500.
|
||||
logger.warning(
|
||||
"Stream producer tracking unavailable for run %s; admitting it with untracked (conservative) stream recovery",
|
||||
sanitize_log_param(admission_run_id),
|
||||
)
|
||||
try:
|
||||
record = await run_mgr.create_or_reject(
|
||||
thread_id,
|
||||
body.assistant_id,
|
||||
on_disconnect=disconnect,
|
||||
metadata=run_metadata,
|
||||
# Persist a redacted copy of the config: the run record is
|
||||
# written to runs.kwargs_json and echoed by the run API, so a
|
||||
# request-scoped secret (#3861) or client hint must not ride along.
|
||||
# The live config built above keeps them for the actual run.
|
||||
kwargs={
|
||||
"input": run_record_input,
|
||||
**({"knowledge_default_request_hash": knowledge_default_request_hash} if accepts_knowledge_default else {}),
|
||||
"config": redact_config_secrets(body.config),
|
||||
**({"conversation_references": conversation_references} if conversation_references else {}),
|
||||
},
|
||||
idempotency_request=idempotency_request,
|
||||
multitask_strategy=body.multitask_strategy,
|
||||
model_name=model_name,
|
||||
user_id=owner_user_id,
|
||||
idempotency_key=idempotency_key,
|
||||
evidence_origin=admitted_origin,
|
||||
# Canonical agent keys, not routing aliases or display names.
|
||||
# Bootstrap and unresolved legacy identities stay read-only.
|
||||
evidence_agent_id=(None if scope_runtime_config.get("is_bootstrap") else (_DEFAULT_ASSISTANT_ID if scope_assistant_id == _DEFAULT_ASSISTANT_ID else getattr(agent_config, "name", None))),
|
||||
run_id=admission_run_id,
|
||||
**({"execution_fence": request.state.extension_execution_fence} if getattr(request.state, "extension_execution_fence", None) is not None else {}),
|
||||
if not producer_lease_started and getattr(bridge, "supports_cross_process", False) is True:
|
||||
# Best-effort: a cross-process bridge returns False only
|
||||
# after a transient backend error whose partial lease
|
||||
# registration was safely rolled back (nothing is left
|
||||
# tracked). An untracked producer falls back to the
|
||||
# historical conservative recovery path -- wait for the
|
||||
# real END or the retained-stream TTL rather than
|
||||
# synthesizing one -- so this run is no less safe than it
|
||||
# was before producer leases existed. Rejecting admission
|
||||
# here would instead turn a transient blip into a 500.
|
||||
logger.warning(
|
||||
"Stream producer tracking unavailable for run %s; admitting it with untracked (conservative) stream recovery",
|
||||
sanitize_log_param(admission_run_id),
|
||||
)
|
||||
except RunIdempotencyUnsupported as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc)) from exc
|
||||
except RunIdempotencyKeyMismatch as exc:
|
||||
# A trusted internal key is scoped by owner, not thread: the
|
||||
# same message requested on another thread (IM ``/new``) is
|
||||
# a different request for the run already admitted.
|
||||
raise HTTPException(status_code=409, detail="Idempotency-Key already used with a different request") from exc
|
||||
|
||||
async def admit_run() -> RunRecord:
|
||||
try:
|
||||
if admitted_run is None and not (command and command.get("resume") is not None):
|
||||
# Validation above is intentionally early, but a pause can
|
||||
# appear while this request prepares its admission. Recheck
|
||||
# under the thread lock so a fresh input cannot replace it.
|
||||
await reject_new_run_when_approval_pending(request, thread_id)
|
||||
return await run_mgr.create_or_reject(
|
||||
thread_id,
|
||||
body.assistant_id,
|
||||
on_disconnect=disconnect,
|
||||
metadata=run_metadata,
|
||||
# Persist a redacted copy of the config: the run record is
|
||||
# written to runs.kwargs_json and echoed by the run API, so a
|
||||
# request-scoped secret (#3861) or client hint must not ride along.
|
||||
# The live config built above keeps them for the actual run.
|
||||
kwargs={
|
||||
"input": run_record_input,
|
||||
**({"knowledge_default_request_hash": knowledge_default_request_hash} if accepts_knowledge_default else {}),
|
||||
"config": redact_config_secrets(body.config),
|
||||
**({"conversation_references": conversation_references} if conversation_references else {}),
|
||||
},
|
||||
idempotency_request=idempotency_request,
|
||||
multitask_strategy=body.multitask_strategy,
|
||||
model_name=model_name,
|
||||
user_id=owner_user_id,
|
||||
idempotency_key=idempotency_key,
|
||||
evidence_origin=admitted_origin,
|
||||
# Canonical agent keys, not routing aliases or display names.
|
||||
# Bootstrap and unresolved legacy identities stay read-only.
|
||||
evidence_agent_id=(None if scope_runtime_config.get("is_bootstrap") else (_DEFAULT_ASSISTANT_ID if scope_assistant_id == _DEFAULT_ASSISTANT_ID else getattr(agent_config, "name", None))),
|
||||
run_id=admission_run_id,
|
||||
**({"execution_fence": request.state.extension_execution_fence} if getattr(request.state, "extension_execution_fence", None) is not None else {}),
|
||||
)
|
||||
except RunIdempotencyUnsupported as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc)) from exc
|
||||
except RunIdempotencyKeyMismatch as exc:
|
||||
# A trusted internal key is scoped by owner, not thread: the
|
||||
# same message requested on another thread (IM ``/new``) is
|
||||
# a different request for the run already admitted.
|
||||
raise HTTPException(status_code=409, detail="Idempotency-Key already used with a different request") from exc
|
||||
|
||||
record = await admit_run()
|
||||
|
||||
if record.idempotency_reused:
|
||||
stored = record.kwargs or {}
|
||||
@ -2821,13 +2901,48 @@ async def start_run(
|
||||
status_code=409,
|
||||
detail="Idempotency-Key already used with a different request",
|
||||
)
|
||||
return record
|
||||
|
||||
if request_identity["kind"] == "resume" and _is_retryable_resume_start_failure(record):
|
||||
# The original request never attached a worker. Preserve
|
||||
# idempotency for real execution outcomes, but let this
|
||||
# identical retry replace only that failed admission.
|
||||
await _discard_failed_resume_admission(request, run_mgr, record)
|
||||
record = await admit_run()
|
||||
if record.idempotency_reused:
|
||||
# A concurrent identical retry won the replacement.
|
||||
return record
|
||||
else:
|
||||
return record
|
||||
worker = run_after_metadata(record)
|
||||
answered_approval = None
|
||||
try:
|
||||
if command and command.get("resume") is not None:
|
||||
from app.gateway.hitl_approval import _all_reject, _responses_of, current_approval_request_ids, mark_approval_answered
|
||||
|
||||
responses = _responses_of(command)
|
||||
answered_approval = await mark_approval_answered(
|
||||
request,
|
||||
thread_id,
|
||||
expected_request_ids=(validated_approval_request_ids if responses is not None else None),
|
||||
)
|
||||
if answered_approval is None and responses is not None:
|
||||
if not _all_reject(responses):
|
||||
# Competing or stale approvals fail closed.
|
||||
raise HTTPException(status_code=409, detail={"code": "no_pending_approval", "message": "No resumable pause exists for this thread"})
|
||||
# A crashed attempt may have consumed the run row while
|
||||
# leaving the same checkpoint pause live. Only that exact
|
||||
# pause may use checkpoint-only recovery; a stale reject
|
||||
# must never dismiss a newer approval.
|
||||
current_request_ids = await current_approval_request_ids(
|
||||
request,
|
||||
thread_id,
|
||||
accessor=accessor,
|
||||
read_config=read_config,
|
||||
)
|
||||
if current_request_ids != validated_approval_request_ids:
|
||||
raise HTTPException(status_code=409, detail={"code": "no_pending_approval", "message": "No resumable pause exists for this thread"})
|
||||
# The stream lease was established before durable admission.
|
||||
# Keep the admission-to-task-attachment window await-free so
|
||||
# cancellation cannot strand a pending row.
|
||||
# Once resume persistence succeeds, keep task attachment
|
||||
# await-free so cancellation cannot strand a pending row.
|
||||
record.task = asyncio.create_task(worker)
|
||||
if producer_lease_started:
|
||||
transfer_producer_lease = _declared_bridge_method(
|
||||
@ -2840,12 +2955,28 @@ async def start_run(
|
||||
record.task,
|
||||
)
|
||||
producer_lease_handed_off = True
|
||||
except Exception as exc:
|
||||
worker.close()
|
||||
await run_mgr.fail_start_if_pending(
|
||||
record.run_id,
|
||||
error=f"Failed to attach run worker: {exc}",
|
||||
)
|
||||
except BaseException as exc:
|
||||
attachment_error = f"Failed to attach run worker: {exc}"
|
||||
task = record.task
|
||||
if task is None:
|
||||
worker.close()
|
||||
|
||||
async def cleanup_failed_attachment() -> None:
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
if record.task is task:
|
||||
record.task = None
|
||||
await run_mgr.fail_start_if_pending(
|
||||
record.run_id,
|
||||
error=attachment_error,
|
||||
)
|
||||
if answered_approval is not None:
|
||||
restored = await run_mgr.restore_approval_answer(answered_approval)
|
||||
if not restored:
|
||||
raise RuntimeError(f"Failed to restore approval run {answered_approval.run_id} after worker attachment failed")
|
||||
|
||||
await asyncio.shield(cleanup_failed_attachment())
|
||||
raise
|
||||
except ConflictError as exc:
|
||||
raise BusyThreadConflict(str(exc)) from exc
|
||||
@ -2957,14 +3088,23 @@ async def launch_scheduled_thread_run(
|
||||
# scheduler service's own per-occurrence scope -- ensure_trace_context
|
||||
# keeps that trace instead of minting a competing one.
|
||||
with ensure_trace_context():
|
||||
record = await start_run(
|
||||
body,
|
||||
thread_id,
|
||||
request,
|
||||
idempotency_key=idempotency_key,
|
||||
evidence_origin="scheduled",
|
||||
scheduled_task_runtime=scheduled_task_runtime,
|
||||
)
|
||||
try:
|
||||
record = await start_run(
|
||||
body,
|
||||
thread_id,
|
||||
request,
|
||||
idempotency_key=idempotency_key,
|
||||
evidence_origin="scheduled",
|
||||
scheduled_task_runtime=scheduled_task_runtime,
|
||||
)
|
||||
except HTTPException as exc:
|
||||
if isinstance(exc.detail, dict) and exc.detail.get("code") == "approval_pending":
|
||||
# Skip-and-retry: an approval pause on the thread is temporary,
|
||||
# not a permanent launch failure. The scheduler treats this as
|
||||
# an overlap conflict and retries on its normal cadence.
|
||||
logger.info("Scheduled launch skipped: thread %s has a pending approval", thread_id)
|
||||
raise ConflictError(str(exc.detail)) from exc
|
||||
raise
|
||||
return {"run_id": record.run_id, "thread_id": record.thread_id}
|
||||
|
||||
|
||||
@ -3049,6 +3189,9 @@ async def launch_mcp_task_notification_run(
|
||||
require_existing_thread=True,
|
||||
)
|
||||
except HTTPException as exc:
|
||||
if isinstance(exc.detail, dict) and exc.detail.get("code") == "approval_pending":
|
||||
# Temporary condition, not a permanent notification failure.
|
||||
raise ConflictError(str(exc.detail)) from exc
|
||||
if isinstance(exc, BusyThreadConflict):
|
||||
raise ConflictError(str(exc.detail)) from exc
|
||||
if exc.status_code in {400, 401, 403, 404, 409, 422, 501}:
|
||||
|
||||
@ -2254,6 +2254,52 @@ only when the agent is expected to edit files or run commands for a trusted
|
||||
task.
|
||||
|
||||
|
||||
## HITL Tool Approval
|
||||
|
||||
`hitl_approval` pauses high-risk tool calls for human approval before execution.
|
||||
It is disabled by default; enabling it without a tool list changes nothing.
|
||||
|
||||
```yaml
|
||||
hitl_approval:
|
||||
enabled: true
|
||||
tools: [write_file, str_replace, bash]
|
||||
non_interactive: deny # deny | allow
|
||||
max_actions: 32 # 0 disables the approval-batch count limit
|
||||
```
|
||||
|
||||
- `enabled`: master switch. `false` leaves all behavior unchanged.
|
||||
- `tools`: host-owned high-risk list edited by the operator. Unknown names log
|
||||
a warning at assembly (a typo would otherwise silently disable approval).
|
||||
- `non_interactive`: behavior for IM channels, GitHub runs, scheduled tasks,
|
||||
and other non-interactive entry points. `deny` rejects the call with an
|
||||
error ToolMessage; `allow` executes it directly. Subagents and runtimes
|
||||
assembled without resume support always reject matching calls regardless of
|
||||
this setting.
|
||||
- `max_actions`: maximum matching tool calls in one approval batch. Batches
|
||||
above the limit are rejected without executing any call; `0` disables this
|
||||
count limit. The approval payload size limit still applies.
|
||||
|
||||
Approval is **not** authorization. `read_before_write`, guardrails, and sandbox
|
||||
authorization still apply after approval — "approve to try executing", not
|
||||
"approve to guarantee execution". Approval requires a resume-capable entry:
|
||||
the Web UI and SDK support it. IM channels, GitHub runs, and scheduled tasks
|
||||
follow `non_interactive`; subagents and runtimes without resume support always
|
||||
fail closed.
|
||||
|
||||
Argument summaries are display-only. Fields other than `path`, `command`, `url`,
|
||||
`old_str`, and `new_str` are truncated when they exceed the display budget; the
|
||||
summary includes a `chars omitted` marker. Approving executes the original,
|
||||
complete arguments, including content hidden by that marker. The payload-size
|
||||
limit bounds the approval summary, not the tool arguments.
|
||||
|
||||
Pending approvals do not expire. While the thread remains paused at the current
|
||||
checkpoint, its approval stays resumable and blocks new runs on that thread,
|
||||
including scheduled runs, until every action is approved or rejected. Operators
|
||||
must resolve stale approvals explicitly. Disabling HITL while paused still
|
||||
allows the all-reject response as a safe escape hatch; other configuration
|
||||
changes fail closed.
|
||||
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Place `config.yaml` in project root** - Set `DEER_FLOW_PROJECT_ROOT` if the runtime starts elsewhere
|
||||
|
||||
@ -36,3 +36,26 @@ remain runtime-only and are never copied into checkpoint-persisted
|
||||
`configurable`. Public run requests cannot use these keys to disable
|
||||
clarification, impersonate a webhook channel, or override a scheduled run back
|
||||
to interactive behavior.
|
||||
|
||||
## HITL tool approval
|
||||
|
||||
`hitl_approval` is an operator-controlled, opt-in gate around selected tools,
|
||||
not a way for a non-interactive run to wait for clarification. Interactive
|
||||
Web/SDK runs pause through LangGraph `interrupt()` and enter
|
||||
`waiting_for_input`; only a matching approval response can resume the current
|
||||
checkpoint. Scheduled, webhook, autonomous, IM, extension, and MCP notification
|
||||
runs follow `hitl_approval.non_interactive`. Subagents and graphs without resume
|
||||
support always fail closed, including when that setting is `allow`.
|
||||
|
||||
Resume consumption is a store-level compare-and-swap bound to the validated
|
||||
action `request_id` mapping; the middleware recomputes those IDs from the live
|
||||
AI message before applying any decision. If worker attachment fails after
|
||||
consumption, the Gateway conditionally restores that exact answered run to
|
||||
`waiting_for_input` so a valid response or all-reject can retry safely.
|
||||
|
||||
`waiting_for_input` is not a terminal success or failure. Goal checks, final
|
||||
artifact delivery checks, and batch continuation cancellation must not treat a
|
||||
pending approval as a completed source run. Pending approvals do not expire and
|
||||
block new runs on the same thread until resolved. See
|
||||
[HITL Tool Approval](CONFIGURATION.md#hitl-tool-approval) for configuration and
|
||||
argument-summary visibility limits.
|
||||
|
||||
@ -828,6 +828,34 @@ def _available_skill_names(agent_config, is_bootstrap: bool) -> set[str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _insert_hitl_approval_middleware(middlewares: list, final_tools: list, app_config: AppConfig, *, supports_resume: bool) -> None:
|
||||
hitl_config = getattr(app_config, "hitl_approval", None)
|
||||
if hitl_config is None or not hitl_config.enabled:
|
||||
return
|
||||
from deerflow.agents.middlewares.hitl_approval_middleware import HitlApprovalMiddleware
|
||||
|
||||
known_tool_names = {tool.name for tool in final_tools}
|
||||
unknown_names = set(hitl_config.tools) - known_tool_names
|
||||
if unknown_names:
|
||||
logger.warning("hitl_approval.tools contains names not present in this build's tool set: %s", sorted(unknown_names))
|
||||
logger.info(
|
||||
"hitl_approval enabled: tools=%s non_interactive=%s max_actions=%s",
|
||||
hitl_config.tools,
|
||||
hitl_config.non_interactive,
|
||||
hitl_config.max_actions,
|
||||
)
|
||||
middlewares.insert(
|
||||
0,
|
||||
HitlApprovalMiddleware(
|
||||
tool_names=set(hitl_config.tools),
|
||||
non_interactive=hitl_config.non_interactive,
|
||||
max_actions=hitl_config.max_actions,
|
||||
supports_resume=supports_resume,
|
||||
pii_config=getattr(app_config, "pii_redaction", None),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _load_enabled_available_skills(available_skills: set[str] | None, *, app_config: AppConfig, user_id: str | None = None) -> list[Skill]:
|
||||
try:
|
||||
from deerflow.agents.lead_agent.prompt import get_enabled_skills_for_config
|
||||
@ -1231,6 +1259,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le
|
||||
app_config=resolved_app_config,
|
||||
authorization_provider=_authz_provider,
|
||||
)
|
||||
_insert_hitl_approval_middleware(middlewares, final_tools, resolved_app_config, supports_resume=False)
|
||||
system_prompt = apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
@ -1394,6 +1423,7 @@ def _assemble_lead_agent(config: RunnableConfig, *, app_config: AppConfig) -> Le
|
||||
app_config=resolved_app_config,
|
||||
authorization_provider=_authz_provider,
|
||||
)
|
||||
_insert_hitl_approval_middleware(middlewares, final_tools, resolved_app_config, supports_resume=not (config.get("context") or {}).get("_hitl_disabled_resume_escape", False))
|
||||
system_prompt = apply_prompt_template(
|
||||
subagent_enabled=subagent_enabled,
|
||||
max_concurrent_subagents=max_concurrent_subagents,
|
||||
|
||||
@ -0,0 +1,251 @@
|
||||
"""Human-in-the-loop approval for high-risk tool calls (Phase 1).
|
||||
|
||||
Uses the after_model pattern (same shape as LangChain's HumanInTheLoopMiddleware):
|
||||
dangerous calls pause via interrupt(); rejections become artificial ToolMessages.
|
||||
Never rejects inside wrap_tool_call — that path is bypassed by the DeerFlow
|
||||
execution chain (validated by the design spike).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langgraph.config import get_config
|
||||
from langgraph.types import interrupt
|
||||
|
||||
from deerflow.agents.interaction_policy import resolve_run_interaction_policy
|
||||
from deerflow.agents.middlewares.pii_redaction_middleware import redact_text
|
||||
from deerflow.runtime.run_origin import origin_kind_of
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KIND = "hitl_approval"
|
||||
MAX_REASON_CHARS = 2000
|
||||
MAX_SUMMARY_FIELD_CHARS = 2000
|
||||
MAX_PAYLOAD_BYTES = 32_000
|
||||
HEAD_TAIL_CHARS = 400
|
||||
|
||||
# Fields whose meaning must never be hidden from the approver.
|
||||
_UNTRUNCATED_FIELDS = frozenset({"path", "command", "url", "old_str", "new_str"})
|
||||
|
||||
|
||||
def _shorten_middle(text: str) -> str:
|
||||
if len(text) <= MAX_SUMMARY_FIELD_CHARS:
|
||||
return text
|
||||
omitted = len(text) - 2 * HEAD_TAIL_CHARS
|
||||
return f"{text[:HEAD_TAIL_CHARS]}…[{omitted} chars omitted]…{text[-HEAD_TAIL_CHARS:]}"
|
||||
|
||||
|
||||
def summarize_args(args: Any, *, pii_config: Any = None) -> Any:
|
||||
"""Display-only argument summary. High-semantic fields are never truncated."""
|
||||
if not isinstance(args, Mapping):
|
||||
return _shorten_middle(str(args))
|
||||
summary: dict[str, Any] = {}
|
||||
for key, value in args.items():
|
||||
text = value if isinstance(value, str) else str(value)
|
||||
if key in _UNTRUNCATED_FIELDS:
|
||||
summary[key] = text
|
||||
else:
|
||||
summary[key] = _shorten_middle(text)
|
||||
return redact_text(str(summary), pii_config) if pii_config is not None else summary
|
||||
|
||||
|
||||
def _bounded_text(value: Any) -> str:
|
||||
"""Render untrusted values without allowing malformed objects to escape or grow output."""
|
||||
if isinstance(value, str):
|
||||
return value[:MAX_REASON_CHARS]
|
||||
try:
|
||||
return repr(value)[:MAX_REASON_CHARS]
|
||||
except Exception:
|
||||
return f"<unprintable {type(value).__name__}>"[:MAX_REASON_CHARS]
|
||||
|
||||
|
||||
def _deny_message(tool_call: Mapping[str, Any], reason: str) -> ToolMessage:
|
||||
return ToolMessage(
|
||||
content=reason,
|
||||
tool_call_id=tool_call["id"],
|
||||
name=tool_call["name"],
|
||||
status="error",
|
||||
)
|
||||
|
||||
|
||||
class HitlApprovalMiddleware(AgentMiddleware):
|
||||
"""Pause high-risk tool calls for human approval.
|
||||
|
||||
``supports_resume`` is decided at assembly time (only the normal Gateway
|
||||
branch passes True); runtime cannot detect a checkpointer. ``is_subagent``
|
||||
is checked at runtime as a second defensive gate.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
tool_names: set[str],
|
||||
non_interactive: str = "deny",
|
||||
max_actions: int = 32,
|
||||
supports_resume: bool = True,
|
||||
pii_config: Any = None,
|
||||
) -> None:
|
||||
self.tool_names = set(tool_names)
|
||||
self.non_interactive = non_interactive
|
||||
self.supports_resume = supports_resume
|
||||
self.pii_config = pii_config
|
||||
self.max_actions = max_actions
|
||||
self._warned_unknown: set[str] = set()
|
||||
|
||||
# -- helpers ---------------------------------------------------------
|
||||
|
||||
def _target_calls(self, last: AIMessage) -> list[dict[str, Any]]:
|
||||
return [tc for tc in last.tool_calls if tc["name"] in self.tool_names]
|
||||
|
||||
def _deny_all(self, calls: list[dict[str, Any]], reason: str) -> dict[str, Any]:
|
||||
return {"messages": [_deny_message(tc, reason) for tc in calls]}
|
||||
|
||||
def _is_subagent(self, runtime: Any) -> bool:
|
||||
context = getattr(runtime, "context", None) or {}
|
||||
return isinstance(context, Mapping) and bool(context.get("is_subagent"))
|
||||
|
||||
def _can_resume(self, runtime: Any) -> bool:
|
||||
if not self.supports_resume or self._is_subagent(runtime):
|
||||
return False
|
||||
policy = resolve_run_interaction_policy({"context": getattr(runtime, "context", None)})
|
||||
if not policy.allows_clarification:
|
||||
return False
|
||||
metadata = get_config().get("metadata") or {}
|
||||
kind = origin_kind_of(metadata)
|
||||
# Only Web/SDK runs (no origin) can submit command.resume via the run API.
|
||||
# With non_interactive="allow", all other origin kinds auto-execute except
|
||||
# subagents and runtimes assembled without resume support.
|
||||
return kind is None
|
||||
|
||||
# -- main hook -------------------------------------------------------
|
||||
|
||||
def after_model(self, state: Any, runtime: Any) -> dict[str, Any] | None:
|
||||
messages = state.get("messages") if isinstance(state, Mapping) else state.messages
|
||||
if not messages:
|
||||
return None
|
||||
last = messages[-1]
|
||||
if not isinstance(last, AIMessage) or not last.tool_calls:
|
||||
return None
|
||||
|
||||
context = getattr(runtime, "context", None)
|
||||
resume_tool_names = context.get("_hitl_resume_tool_names", set()) if isinstance(context, Mapping) else set()
|
||||
if not isinstance(resume_tool_names, (set, frozenset)) or any(not isinstance(name, str) for name in resume_tool_names):
|
||||
resume_tool_names = set()
|
||||
live_tool_names = set(self.tool_names)
|
||||
if resume_tool_names:
|
||||
# The graph is assembled again on every resume. If approval tools were
|
||||
# changed or cleared while paused, the worker recovers their names from
|
||||
# the durable interrupt payload so replay still reaches interrupt().
|
||||
# Keep the recovered gate for subsequent model steps in this graph.
|
||||
self.tool_names.update(resume_tool_names)
|
||||
targets = self._target_calls(last)
|
||||
if not targets:
|
||||
return None
|
||||
if self.max_actions and len(targets) > self.max_actions:
|
||||
return self._deny_all(targets, _too_many_reason(len(targets), self.max_actions))
|
||||
|
||||
if not self._can_resume(runtime):
|
||||
if self.non_interactive == "allow":
|
||||
# allow is meaningful for genuine non-interactive entries (e.g. IM).
|
||||
# Missing resume support or subagent is always a hard fail-closed.
|
||||
if not self.supports_resume or self._is_subagent(runtime):
|
||||
return self._deny_all(targets, _no_resume_reason())
|
||||
return None
|
||||
return self._deny_all(targets, _no_resume_reason())
|
||||
|
||||
ai_message_id = last.id or ""
|
||||
actions = [
|
||||
{
|
||||
"tool_call_id": tc["id"],
|
||||
"tool_name": tc["name"],
|
||||
"args_summary": summarize_args(tc.get("args", {}), pii_config=self.pii_config),
|
||||
"request_id": f"hitl_approval:{ai_message_id}:{tc['id']}",
|
||||
}
|
||||
for tc in targets
|
||||
]
|
||||
payload = {
|
||||
"kind": KIND,
|
||||
"version": 1,
|
||||
"request_id": f"hitl_approval:{ai_message_id}:{targets[0]['id']}",
|
||||
"actions": actions,
|
||||
"allowed_decisions": ["approve", "reject"],
|
||||
"pending_tool_call_ids": [tc["id"] for tc in targets],
|
||||
}
|
||||
if len(str(payload)) > MAX_PAYLOAD_BYTES:
|
||||
return self._deny_all(targets, _too_large_reason())
|
||||
responses = interrupt(payload)
|
||||
if any(tc["name"] in resume_tool_names and tc["name"] not in live_tool_names for tc in targets):
|
||||
return self._deny_all(targets, _invalid_response_reason())
|
||||
return self._process_responses(last, targets, responses)
|
||||
|
||||
async def aafter_model(self, state: Any, runtime: Any) -> dict[str, Any] | None:
|
||||
return self.after_model(state, runtime)
|
||||
|
||||
# -- resume handling -------------------------------------------------
|
||||
|
||||
def _process_responses(
|
||||
self,
|
||||
last: AIMessage,
|
||||
targets: list[dict[str, Any]],
|
||||
resume_value: Any,
|
||||
) -> dict[str, Any]:
|
||||
target_ids = [tc["id"] for tc in targets]
|
||||
if any(not isinstance(call_id, str) for call_id in target_ids) or len(set(target_ids)) != len(target_ids):
|
||||
return self._deny_all(targets, _invalid_response_reason())
|
||||
ai_message_id = last.id or ""
|
||||
expected_request_ids = {call_id: f"hitl_approval:{ai_message_id}:{call_id}" for call_id in target_ids}
|
||||
responses = resume_value.get("responses") if isinstance(resume_value, Mapping) else None
|
||||
if not isinstance(responses, list) or len(responses) != len(target_ids):
|
||||
return self._deny_all(targets, _invalid_response_reason())
|
||||
by_id = {tc["id"]: tc for tc in targets}
|
||||
seen_ids: set[str] = set()
|
||||
artificial: list[ToolMessage] = []
|
||||
for response in responses:
|
||||
if not isinstance(response, Mapping):
|
||||
return self._deny_all(targets, _invalid_response_reason())
|
||||
response_id = response.get("tool_call_id")
|
||||
if not isinstance(response_id, str) or response_id in seen_ids or response_id not in by_id:
|
||||
return self._deny_all(targets, _invalid_response_reason())
|
||||
if response.get("request_id") != expected_request_ids[response_id]:
|
||||
return self._deny_all(targets, _invalid_response_reason())
|
||||
seen_ids.add(response_id)
|
||||
call = by_id[response_id]
|
||||
decision = response.get("option_id")
|
||||
if decision == "approve":
|
||||
continue
|
||||
reason_value = response.get("value")
|
||||
reason = _bounded_text(reason_value) if reason_value is not None else ""
|
||||
if decision != "reject":
|
||||
reason = _unknown_decision_reason(decision)
|
||||
artificial.append(
|
||||
_deny_message(
|
||||
call,
|
||||
"The tool was not executed. Do not retry unless the user explicitly asks." + (f" User reason: {reason}" if reason else ""),
|
||||
)
|
||||
)
|
||||
return {"messages": artificial}
|
||||
|
||||
|
||||
def _no_resume_reason() -> str:
|
||||
return "Approval is required for this tool, but this run cannot collect approval (non-interactive entry). The tool was not executed."
|
||||
|
||||
|
||||
def _invalid_response_reason() -> str:
|
||||
return "The approval response was invalid, mismatched, or the approval configuration changed while paused. The tool calls were rejected for safety. The tool was not executed. Do not retry unless the user explicitly asks."
|
||||
|
||||
|
||||
def _unknown_decision_reason(decision: Any) -> str:
|
||||
return f"Unsupported approval decision {_bounded_text(decision)}; treated as rejection."[:MAX_REASON_CHARS]
|
||||
|
||||
|
||||
def _too_many_reason(count: int, max_actions: int) -> str:
|
||||
return f"Too many tool calls require approval ({count} > {max_actions}); rejected for safety."
|
||||
|
||||
|
||||
def _too_large_reason() -> str:
|
||||
return "Approval payload exceeded the size budget; rejected for safety."
|
||||
@ -28,6 +28,7 @@ from deerflow.config.file_signature import get_config_signature as _get_config_s
|
||||
from deerflow.config.file_signature import read_config_with_signature as _read_config_with_signature
|
||||
from deerflow.config.guardrails_config import GuardrailsConfig, load_guardrails_config_from_dict
|
||||
from deerflow.config.harness_guard_config import HarnessGuardConfig
|
||||
from deerflow.config.hitl_approval_config import HitlApprovalConfig
|
||||
from deerflow.config.input_polish_config import InputPolishConfig
|
||||
from deerflow.config.knowledge_base_config import KnowledgeBaseConfig
|
||||
from deerflow.config.loop_detection_config import LoopDetectionConfig
|
||||
@ -320,6 +321,7 @@ class AppConfig(BaseModel):
|
||||
loop_detection: LoopDetectionConfig = Field(default_factory=LoopDetectionConfig, description="Loop detection middleware configuration")
|
||||
tool_progress: ToolProgressConfig = Field(default_factory=ToolProgressConfig, description="Tool progress state machine middleware configuration")
|
||||
verification: VerificationConfig = Field(default_factory=VerificationConfig, description="Subagent result verification (receipts, checklist, judge)")
|
||||
hitl_approval: HitlApprovalConfig = Field(default_factory=HitlApprovalConfig, description="Human-in-the-loop approval for high-risk tool calls")
|
||||
read_before_write: ReadBeforeWriteConfig = Field(default_factory=ReadBeforeWriteConfig, description="Read-before-write file gate middleware configuration")
|
||||
harness_guard: HarnessGuardConfig = Field(default_factory=HarnessGuardConfig, description="HarnessGuard tool-loop interceptor middleware configuration")
|
||||
projects: ProjectsConfig = Field(default_factory=ProjectsConfig, description="User projects configuration (instructions injection, shelf index, trash retention)")
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
"""Configuration for the HITL tool-approval middleware."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class HitlApprovalConfig(BaseModel):
|
||||
"""Human-in-the-loop approval for high-risk tool calls.
|
||||
|
||||
The high-risk tool list is a host-owned security policy. Extensions may only
|
||||
add entries (never remove them) in later phases; there is no merge logic yet.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Enable approval pauses for tools listed in `tools`",
|
||||
)
|
||||
tools: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Tool names that require human approval before execution",
|
||||
)
|
||||
non_interactive: Literal["deny", "allow"] = Field(
|
||||
default="deny",
|
||||
description="Behavior for runs that cannot collect approval (scheduled, IM channels, subagents)",
|
||||
)
|
||||
max_actions: int = Field(
|
||||
default=32,
|
||||
ge=0,
|
||||
description="Maximum matching tool calls included in one approval batch; 0 disables the count limit",
|
||||
)
|
||||
|
||||
@field_validator("tools")
|
||||
@classmethod
|
||||
def _validate_tool_names(cls, value: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
for name in value:
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise ValueError("hitl_approval.tools entries must be non-empty strings")
|
||||
if name in seen:
|
||||
raise ValueError(f"hitl_approval.tools contains duplicate entry {name!r}")
|
||||
seen.add(name)
|
||||
return value
|
||||
@ -259,7 +259,7 @@ class RunRepository(RunStore):
|
||||
RunRow.owner_worker_id == owner_worker_id,
|
||||
or_(RunRow.lease_expires_at.is_(None), RunRow.lease_expires_at > datetime.now(UTC)),
|
||||
RunRow.evidence_retention_revision == receipt.retention_revision,
|
||||
RunRow.status.in_(("success", "error", "timeout", "interrupted")),
|
||||
RunRow.status.in_(("success", "error", "timeout", "interrupted", "waiting_for_input")),
|
||||
RunRow.evidence_seal_state == "pending",
|
||||
)
|
||||
.values(evidence_seal_state="sealed", evidence_revision=uuid4().hex, evidence_upper_seq=receipt.upper_event_seq, evidence_event_count=receipt.event_count, change_seq=change_seq)
|
||||
@ -480,6 +480,40 @@ class RunRepository(RunStore):
|
||||
await session.commit()
|
||||
return result.rowcount != 0
|
||||
|
||||
async def answer_approval(self, run_id, *, metadata, answered_updated_at, expected_request_id=None) -> bool:
|
||||
async with self._sf() as session:
|
||||
values = {
|
||||
"status": "success",
|
||||
"metadata_json": self._safe_json(metadata) or {},
|
||||
"updated_at": datetime.fromisoformat(answered_updated_at),
|
||||
"change_seq": await self._next_change_seq(session),
|
||||
}
|
||||
conditions = [RunRow.run_id == run_id, RunRow.status == "waiting_for_input"]
|
||||
if expected_request_id is not None:
|
||||
conditions.append(RunRow.metadata_json["hitl_approval"]["request_id"].as_string() == expected_request_id)
|
||||
result = await session.execute(update(RunRow).where(*conditions).values(**values))
|
||||
await session.commit()
|
||||
return result.rowcount != 0
|
||||
|
||||
async def restore_approval(self, run_id, *, metadata, expected_updated_at) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(RunRow)
|
||||
.where(
|
||||
RunRow.run_id == run_id,
|
||||
RunRow.status == "success",
|
||||
RunRow.updated_at == datetime.fromisoformat(expected_updated_at),
|
||||
)
|
||||
.values(
|
||||
status="waiting_for_input",
|
||||
metadata_json=self._safe_json(metadata) or {},
|
||||
updated_at=datetime.now(UTC),
|
||||
change_seq=await self._next_change_seq(session),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount != 0
|
||||
|
||||
async def start_run(self, run_id: str) -> bool:
|
||||
"""Start only a still-pending run; cancelled rows must not be resurrected."""
|
||||
async with self._sf() as session:
|
||||
@ -1008,7 +1042,7 @@ class RunRepository(RunStore):
|
||||
cutoff = datetime.now(UTC) - timedelta(seconds=grace_seconds)
|
||||
predicates = (
|
||||
RunRow.operation_kind == "run",
|
||||
RunRow.status.in_(("success", "error", "timeout", "interrupted")),
|
||||
RunRow.status.in_(("success", "error", "timeout", "interrupted", "waiting_for_input")),
|
||||
RunRow.evidence_seal_state == "pending",
|
||||
RunRow.created_at <= before_dt,
|
||||
_lease_expired_or_null(RunRow.lease_expires_at, cutoff),
|
||||
|
||||
@ -0,0 +1,59 @@
|
||||
"""Shared checkpoint-history traversal for partially asynchronous savers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def recent_checkpoint_history(checkpointer: Any, read_config: dict[str, Any], *, limit: int = 20) -> AsyncIterator[Any]:
|
||||
"""Yield checkpoints newest-first, degrading to the current tuple when needed.
|
||||
|
||||
``BaseCheckpointSaver`` exposes asynchronous methods even when a saver only
|
||||
implements their synchronous counterparts. Each stage therefore treats
|
||||
``NotImplementedError`` as a capability signal and moves to the next one.
|
||||
Synchronous iteration and reads are kept off the event loop.
|
||||
"""
|
||||
if checkpointer is None:
|
||||
return
|
||||
|
||||
async_history = getattr(checkpointer, "alist", None)
|
||||
if callable(async_history):
|
||||
try:
|
||||
async for checkpoint_tuple in async_history(read_config, limit=limit):
|
||||
yield checkpoint_tuple
|
||||
return
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
sync_history = getattr(checkpointer, "list", None)
|
||||
if callable(sync_history):
|
||||
try:
|
||||
checkpoints = await asyncio.to_thread(lambda: list(sync_history(read_config, limit=limit)))
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
for checkpoint_tuple in checkpoints:
|
||||
yield checkpoint_tuple
|
||||
return
|
||||
|
||||
async_get_tuple = getattr(checkpointer, "aget_tuple", None)
|
||||
if callable(async_get_tuple):
|
||||
try:
|
||||
checkpoint_tuple = await async_get_tuple(read_config)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
else:
|
||||
if checkpoint_tuple is not None:
|
||||
yield checkpoint_tuple
|
||||
return
|
||||
|
||||
sync_get_tuple = getattr(checkpointer, "get_tuple", None)
|
||||
if callable(sync_get_tuple):
|
||||
try:
|
||||
checkpoint_tuple = await asyncio.to_thread(sync_get_tuple, read_config)
|
||||
except NotImplementedError:
|
||||
return
|
||||
if checkpoint_tuple is not None:
|
||||
yield checkpoint_tuple
|
||||
@ -13,8 +13,10 @@ from langgraph.types import Overwrite
|
||||
from deerflow.agents.middlewares.summarization_middleware import DeerFlowSummarizationMiddleware, SummaryGenerationError, create_summarization_middleware
|
||||
from deerflow.config.agents_config import validate_agent_name
|
||||
from deerflow.config.app_config import AppConfig, get_app_config
|
||||
from deerflow.runtime.checkpoint_history import recent_checkpoint_history
|
||||
from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
|
||||
from deerflow.runtime.context_keys import CHECKPOINT_AGENT_NAME_METADATA_KEY, DEFAULT_AGENT_NAME_METADATA_VALUE
|
||||
from deerflow.runtime.goal import _call_checkpointer_method
|
||||
from deerflow.utils.assembly_io import run_assembly
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@ -165,12 +167,34 @@ async def compact_thread_context(
|
||||
checkpoint_id = snapshot_config.get("configurable", {}).get("checkpoint_id")
|
||||
if not checkpoint_id:
|
||||
raise LookupError(f"Thread {thread_id} checkpoint not found")
|
||||
checkpoint_tuple = await accessor.aget_tuple(read_config)
|
||||
checkpoint_tuple = await _call_checkpointer_method(accessor, "aget_tuple", "get_tuple", read_config)
|
||||
if checkpoint_tuple is None:
|
||||
raise ContextCompactionPendingWork("Thread checkpoint is unavailable. Cannot verify pending execution before compacting.")
|
||||
pending_sends = (checkpoint_tuple.checkpoint.get("channel_values") or {}).get("__pregel_tasks")
|
||||
if pending_sends:
|
||||
raise ContextCompactionPendingWork("Thread has pending execution. Resume the interrupted run before compacting.")
|
||||
# Static breakpoints retain pending sends. A dynamic/HITL pause is instead
|
||||
# durable as a task's __interrupt__ write; the production state-mutation
|
||||
# accessor does not know the agent nodes and therefore can return empty
|
||||
# snapshot.next/tasks even while that write is genuinely resumable. Walk
|
||||
# recent history because a head-moving state write can hide the pause. The
|
||||
# newest checkpoint carrying either marker wins, so an answered pause is
|
||||
# not revived by its older interrupt write.
|
||||
pending_writes = getattr(checkpoint_tuple, "pending_writes", ()) or ()
|
||||
interrupted_task_ids = {write[0] for write in pending_writes if isinstance(write, (list, tuple)) and len(write) >= 2 and write[1] == "__interrupt__"}
|
||||
resumed_task_ids = {write[0] for write in pending_writes if isinstance(write, (list, tuple)) and len(write) >= 2 and write[1] == "__resume__"}
|
||||
history_checked = False
|
||||
history_pending = False
|
||||
async for historical_tuple in recent_checkpoint_history(getattr(accessor, "checkpointer", None), read_config):
|
||||
history_checked = True
|
||||
writes = list(getattr(historical_tuple, "pending_writes", ()) or ())
|
||||
if any(isinstance(write, (list, tuple)) and len(write) >= 2 and write[1] == "__resume__" for write in writes):
|
||||
break
|
||||
if any(isinstance(write, (list, tuple)) and len(write) >= 2 and write[1] == "__interrupt__" for write in writes):
|
||||
history_pending = True
|
||||
break
|
||||
pending_next = getattr(snapshot, "next", None) or ()
|
||||
snapshot_interrupts = bool(pending_next) and any(getattr(task, "interrupts", None) or () for task in (getattr(snapshot, "tasks", None) or ()))
|
||||
if pending_sends or snapshot_interrupts or history_pending or (not history_checked and interrupted_task_ids - resumed_task_ids):
|
||||
raise ContextCompactionPendingWork("Thread has resumable pending execution. Resume the paused run before compacting.")
|
||||
|
||||
binding_known, checkpoint_agent_name = _checkpoint_agent_binding(getattr(snapshot, "metadata", None))
|
||||
# The body hint may still select a compatible summarization model for
|
||||
|
||||
@ -14,7 +14,7 @@ from deerflow.runtime.events.store.base import RunEventLifecycleRejected, RunEve
|
||||
from deerflow.runtime.runs.store.base import RunStore
|
||||
|
||||
TERMINAL_PROTOCOL_VERSION = 1
|
||||
TERMINAL_STATUSES = frozenset({"success", "error", "interrupted", "timeout"})
|
||||
TERMINAL_STATUSES = frozenset({"success", "error", "interrupted", "timeout", "waiting_for_input"})
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ import logging
|
||||
import socket
|
||||
import sqlite3
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping
|
||||
from contextlib import AsyncExitStack, asynccontextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
@ -248,6 +248,30 @@ class TerminalCommitProof:
|
||||
goal_verdict: dict[str, Any] | None
|
||||
|
||||
|
||||
def _approval_request_ids(metadata: Mapping[str, Any]) -> dict[str, str] | None:
|
||||
payload = metadata.get("hitl_approval")
|
||||
if not isinstance(payload, Mapping) or not isinstance(payload.get("actions"), list):
|
||||
return None
|
||||
request_ids: dict[str, str] = {}
|
||||
for action in payload["actions"]:
|
||||
if not isinstance(action, Mapping):
|
||||
return None
|
||||
call_id = action.get("tool_call_id")
|
||||
request_id = action.get("request_id")
|
||||
if not isinstance(call_id, str) or not isinstance(request_id, str) or call_id in request_ids:
|
||||
return None
|
||||
request_ids[call_id] = request_id
|
||||
return request_ids
|
||||
|
||||
|
||||
def _approval_request_id(metadata: Mapping[str, Any]) -> str | None:
|
||||
payload = metadata.get("hitl_approval")
|
||||
if not isinstance(payload, Mapping):
|
||||
return None
|
||||
request_id = payload.get("request_id")
|
||||
return request_id if isinstance(request_id, str) else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunRecord:
|
||||
"""Mutable record for a single run."""
|
||||
@ -307,6 +331,10 @@ class RunRecord:
|
||||
# An interrupted ACK does not authorize completion of an unfinished rollback.
|
||||
rollback_finalization_pending: bool = False
|
||||
terminal_commit_proof: TerminalCommitProof | None = field(default=None, repr=False)
|
||||
# A paused worker may still be confirming its waiting_for_input event when
|
||||
# an answered ACK supersedes that outcome. Only that active worker may use
|
||||
# this prior proof to finish its already-started finalization.
|
||||
paused_worker_terminal_proof: TerminalCommitProof | None = field(default=None, repr=False)
|
||||
terminal_status_revision: int = field(default=0, repr=False)
|
||||
idempotency_key: str | None = None
|
||||
idempotency_request: dict[str, Any] | None = None
|
||||
@ -317,6 +345,24 @@ class RunRecord:
|
||||
evidence_agent_id: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApprovalAnswerHandle:
|
||||
"""The exact answered pause needed to roll back a failed worker attachment."""
|
||||
|
||||
run_id: str
|
||||
original_metadata: dict[str, Any]
|
||||
answered_metadata: dict[str, Any]
|
||||
answered_updated_at: str
|
||||
record: RunRecord = field(repr=False, compare=False)
|
||||
previous_updated_at: str = field(repr=False, compare=False)
|
||||
previous_terminal_proof: TerminalCommitProof | None = field(repr=False, compare=False)
|
||||
previous_terminal_revision: int = field(repr=False, compare=False)
|
||||
previous_terminal_persisted: bool = field(repr=False, compare=False)
|
||||
previous_terminal_commit_pending: bool = field(repr=False, compare=False)
|
||||
previous_paused_worker_proof: TerminalCommitProof | None = field(repr=False, compare=False)
|
||||
answered_revision: int = field(repr=False, compare=False)
|
||||
|
||||
|
||||
class RunStartOutcome(StrEnum):
|
||||
"""Result of the pending-to-running startup barrier."""
|
||||
|
||||
@ -523,21 +569,49 @@ class RunManager:
|
||||
delay = min(policy.max_delay, delay * policy.backoff_factor if delay else policy.initial_delay)
|
||||
attempt += 1
|
||||
|
||||
async def _persist_snapshot_to_store(self, run_id: str, payload: dict[str, Any]) -> bool:
|
||||
async def _persist_snapshot_to_store(
|
||||
self,
|
||||
run_id: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
on_persisted: Callable[[dict[str, Any] | None], None] | None = None,
|
||||
) -> bool:
|
||||
"""Best-effort persist a previously captured run snapshot."""
|
||||
if self._store is None:
|
||||
return True
|
||||
try:
|
||||
await self._call_store_with_retry(
|
||||
admitted = await self._call_store_with_retry(
|
||||
"put",
|
||||
run_id,
|
||||
lambda: self._store.put(run_id, **payload),
|
||||
)
|
||||
if on_persisted is not None:
|
||||
on_persisted(admitted)
|
||||
return True
|
||||
except Exception:
|
||||
logger.warning("Failed to persist run %s to store", run_id, exc_info=True)
|
||||
return False
|
||||
|
||||
async def _persist_snapshot_with_reconciliation(
|
||||
self,
|
||||
run_id: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
on_persisted: Callable[[dict[str, Any] | None], None] | None = None,
|
||||
) -> bool:
|
||||
"""Persist a snapshot, repairing it if the live record advances during store I/O."""
|
||||
while True:
|
||||
if not await self._persist_snapshot_to_store(run_id, payload, on_persisted=on_persisted):
|
||||
return False
|
||||
async with self._lock:
|
||||
record = self._runs.get(run_id)
|
||||
if record is None or record.ownership_lost or self._store is None:
|
||||
return True
|
||||
latest_payload = self._store_put_payload(record)
|
||||
if latest_payload == payload:
|
||||
return True
|
||||
payload = latest_payload
|
||||
|
||||
async def _persist_new_run_to_store(self, record: RunRecord) -> None:
|
||||
"""Persist a newly created run record to the backing store.
|
||||
|
||||
@ -589,11 +663,18 @@ class RunManager:
|
||||
def _has_final_terminal_ack(self, record: RunRecord) -> bool:
|
||||
return not record.rollback_finalization_pending and self._has_terminal_ack(record)
|
||||
|
||||
@staticmethod
|
||||
def _active_paused_worker_status(record: RunRecord) -> str | None:
|
||||
proof = record.paused_worker_terminal_proof
|
||||
if proof is None or proof.status != RunStatus.waiting_for_input or record.status != RunStatus.success or record.task is None or record.task.done():
|
||||
return None
|
||||
return proof.status.value
|
||||
|
||||
def terminal_event_status(self, record: RunRecord) -> str | None:
|
||||
"""Return an attributable final outcome for this local worker only."""
|
||||
if self._runs.get(record.run_id) is not record or record.store_only or record.owner_worker_id != self._worker_id or record.status in (RunStatus.pending, RunStatus.running) or not self._has_final_terminal_ack(record):
|
||||
return None
|
||||
return record.status.value
|
||||
return self._active_paused_worker_status(record) or record.status.value
|
||||
|
||||
def can_release_terminal_barrier(self, record: RunRecord) -> bool:
|
||||
"""Keep durable ownership barriers through rollback's final error ACK."""
|
||||
@ -615,10 +696,12 @@ class RunManager:
|
||||
def _acknowledge_terminal(self, record: RunRecord, outcome: TerminalCommitProof, staged_revision: int) -> None:
|
||||
if self._runs.get(record.run_id) is not record or record.ownership_lost:
|
||||
return
|
||||
# Publish the acknowledged outcome without an await. With a backing
|
||||
# store, an older interrupted ACK cannot supersede a newer error.
|
||||
# Storeless transitions are synchronous and retain local mutability.
|
||||
# Publish the acknowledged outcome without an await. A stale durable
|
||||
# ACK must not replace a newer answered/error outcome or its persistence
|
||||
# barrier. Storeless transitions are synchronous and retain mutability.
|
||||
previous = record.terminal_commit_proof
|
||||
if self._store is not None and previous is not None and previous.status != RunStatus.interrupted and staged_revision < record.terminal_status_revision:
|
||||
return
|
||||
if self._store is None or previous is None or previous.status == RunStatus.interrupted or outcome.status != RunStatus.interrupted:
|
||||
record.terminal_commit_proof = outcome
|
||||
authoritative = record.terminal_commit_proof
|
||||
@ -857,13 +940,20 @@ class RunManager:
|
||||
if record is not None and record.ownership_lost:
|
||||
logger.warning("Skipped completion persistence for run %s after lease ownership was lost", run_id)
|
||||
return
|
||||
paused_worker_status = self._active_paused_worker_status(record) if record is not None else None
|
||||
stale_paused_completion = paused_worker_status is not None and kwargs.get("status") == paused_worker_status
|
||||
if self.heartbeat_enabled and self._store is not None:
|
||||
if record is None or record.store_only or not self._has_final_terminal_ack(record) or kwargs.get("status") != record.status.value:
|
||||
if record is None or record.store_only or not self._has_final_terminal_ack(record) or (kwargs.get("status") != record.status.value and not stale_paused_completion):
|
||||
logger.warning("Skipped completion data without an attributable terminal ack for run %s", run_id)
|
||||
return
|
||||
if any(kwargs.get(key) is not None and kwargs[key] != getattr(record, key) for key in ("error", "stop_reason", "goal_verdict")):
|
||||
logger.warning("Skipped completion data that changes the acknowledged outcome for run %s", run_id)
|
||||
return
|
||||
if stale_paused_completion:
|
||||
# The answered row is already successful. The paused worker's
|
||||
# stale status only attributes its finalization; completion
|
||||
# metadata must target the acknowledged row, not resurrect it.
|
||||
kwargs["status"] = record.status.value
|
||||
if record is not None:
|
||||
if record.goal_verdict is not None:
|
||||
# This fallback can be the first successful terminal write
|
||||
@ -1488,6 +1578,175 @@ class RunManager:
|
||||
)
|
||||
return persisted
|
||||
|
||||
async def mark_approval_answered(self, thread_id: str, *, expected_request_ids: dict[str, str] | None = None) -> ApprovalAnswerHandle | None:
|
||||
"""Atomically answer the validated pause; return ``None`` for a stale response or another winner."""
|
||||
records = await self.list_by_thread(thread_id, limit=5)
|
||||
record = next((item for item in records if item.status == RunStatus.waiting_for_input), None)
|
||||
if record is None:
|
||||
return None
|
||||
if expected_request_ids is not None and _approval_request_ids(record.metadata) != expected_request_ids:
|
||||
return None
|
||||
|
||||
previous_status = record.status
|
||||
previous_metadata = copy.deepcopy(record.metadata or {})
|
||||
expected_request_id = _approval_request_id(previous_metadata) if expected_request_ids is not None else None
|
||||
if expected_request_ids is not None and expected_request_id is None:
|
||||
return None
|
||||
previous_updated_at = record.updated_at
|
||||
previous_terminal_proof = record.terminal_commit_proof
|
||||
previous_terminal_revision = record.terminal_status_revision
|
||||
previous_terminal_persisted = record.terminal_status_persisted
|
||||
previous_terminal_commit_pending = record.terminal_commit_pending
|
||||
previous_paused_worker_proof = record.paused_worker_terminal_proof
|
||||
staged_revision = previous_terminal_revision + 1
|
||||
outcome = self._terminal_outcome(record, RunStatus.success)
|
||||
async with self._lock:
|
||||
if not record.store_only:
|
||||
current = self._runs.get(record.run_id)
|
||||
if current is not record or current.status != RunStatus.waiting_for_input:
|
||||
return None
|
||||
if expected_request_ids is not None and (_approval_request_ids(record.metadata) != expected_request_ids or _approval_request_id(record.metadata) != expected_request_id):
|
||||
return None
|
||||
metadata = dict(previous_metadata)
|
||||
metadata.pop("hitl_approval", None)
|
||||
record.metadata = metadata
|
||||
record.status = RunStatus.success
|
||||
record.updated_at = _now_iso()
|
||||
record.terminal_status_revision = staged_revision
|
||||
record.terminal_status_persisted = self._store is None
|
||||
if not record.store_only:
|
||||
record.paused_worker_terminal_proof = previous_terminal_proof if previous_terminal_proof is not None and previous_terminal_proof.status == RunStatus.waiting_for_input else None
|
||||
# The paused worker may already be finalizing its terminal event,
|
||||
# evidence and usage. ACK this concrete outcome before store I/O so
|
||||
# those consumers do not mistake the answered run for an unfinished
|
||||
# terminal transition. Durability remains gated separately below.
|
||||
self._acknowledge_terminal(record, outcome, staged_revision)
|
||||
record.terminal_status_persisted = self._store is None
|
||||
payload = self._store_put_payload(record)
|
||||
|
||||
async def restore_local_record() -> None:
|
||||
if record.store_only:
|
||||
return
|
||||
async with self._lock:
|
||||
if self._runs.get(record.run_id) is record and record.metadata is metadata and record.status == RunStatus.success and record.terminal_status_revision == staged_revision:
|
||||
record.status = previous_status
|
||||
record.metadata = previous_metadata
|
||||
record.updated_at = previous_updated_at
|
||||
record.terminal_commit_proof = previous_terminal_proof
|
||||
record.paused_worker_terminal_proof = previous_paused_worker_proof
|
||||
record.terminal_status_revision = previous_terminal_revision
|
||||
record.terminal_status_persisted = previous_terminal_persisted
|
||||
record.terminal_commit_pending = previous_terminal_commit_pending
|
||||
|
||||
persisted_snapshots: list[dict[str, Any] | None] = []
|
||||
answered_updated_at = record.updated_at
|
||||
try:
|
||||
if self._store is not None:
|
||||
answered = await self._call_store_with_retry(
|
||||
"answer_approval",
|
||||
record.run_id,
|
||||
lambda: self._store.answer_approval(
|
||||
record.run_id,
|
||||
metadata=metadata,
|
||||
answered_updated_at=record.updated_at,
|
||||
expected_request_id=expected_request_id,
|
||||
),
|
||||
)
|
||||
if not answered:
|
||||
await restore_local_record()
|
||||
return None
|
||||
persisted = (
|
||||
True
|
||||
if record.store_only
|
||||
else await self._persist_snapshot_with_reconciliation(
|
||||
record.run_id,
|
||||
payload,
|
||||
on_persisted=persisted_snapshots.append,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
await restore_local_record()
|
||||
raise RunStartupError(f"Failed to mark approval run {record.run_id} as answered") from exc
|
||||
except BaseException:
|
||||
await restore_local_record()
|
||||
raise
|
||||
if not persisted:
|
||||
await restore_local_record()
|
||||
raise RunStartupError(f"Failed to mark approval run {record.run_id} as answered")
|
||||
if not record.store_only:
|
||||
async with self._lock:
|
||||
if self._runs.get(record.run_id) is record and record.terminal_status_revision == staged_revision and record.terminal_commit_proof == outcome:
|
||||
record.terminal_status_persisted = self._has_final_terminal_ack(record) if self.heartbeat_enabled and self._store is not None else persisted
|
||||
if persisted_snapshots and persisted_snapshots[-1] is not None:
|
||||
answered_updated_at = persisted_snapshots[-1]["updated_at"]
|
||||
return ApprovalAnswerHandle(
|
||||
run_id=record.run_id,
|
||||
original_metadata=previous_metadata,
|
||||
answered_metadata=copy.deepcopy(metadata),
|
||||
answered_updated_at=answered_updated_at,
|
||||
record=record,
|
||||
previous_updated_at=previous_updated_at,
|
||||
previous_terminal_proof=previous_terminal_proof,
|
||||
previous_terminal_revision=previous_terminal_revision,
|
||||
previous_terminal_persisted=previous_terminal_persisted,
|
||||
previous_terminal_commit_pending=previous_terminal_commit_pending,
|
||||
previous_paused_worker_proof=previous_paused_worker_proof,
|
||||
answered_revision=staged_revision,
|
||||
)
|
||||
|
||||
async def restore_approval_answer(self, answer: ApprovalAnswerHandle) -> bool:
|
||||
"""Restore only the exact success transition created by *answer*."""
|
||||
record = answer.record
|
||||
if not record.store_only:
|
||||
async with self._lock:
|
||||
current = self._runs.get(record.run_id)
|
||||
if current is not None and current is not record:
|
||||
return False
|
||||
if current is record and (record.status != RunStatus.success or record.metadata != answer.answered_metadata or record.terminal_status_revision != answer.answered_revision):
|
||||
return False
|
||||
if current is None and self._store is None:
|
||||
return False
|
||||
|
||||
if self._store is not None:
|
||||
try:
|
||||
restored = await self._call_store_with_retry(
|
||||
"restore_approval",
|
||||
record.run_id,
|
||||
lambda: self._store.restore_approval(
|
||||
record.run_id,
|
||||
metadata=answer.original_metadata,
|
||||
expected_updated_at=answer.answered_updated_at,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RunStartupError(f"Failed to restore approval run {record.run_id} after worker attachment failed") from exc
|
||||
if not restored:
|
||||
return False
|
||||
|
||||
if not record.store_only:
|
||||
async with self._lock:
|
||||
if self._runs.get(record.run_id) is record and record.status == RunStatus.success and record.metadata == answer.answered_metadata and record.terminal_status_revision == answer.answered_revision:
|
||||
record.status = RunStatus.waiting_for_input
|
||||
record.metadata = copy.deepcopy(answer.original_metadata)
|
||||
record.updated_at = answer.previous_updated_at
|
||||
record.terminal_commit_proof = answer.previous_terminal_proof
|
||||
record.paused_worker_terminal_proof = answer.previous_paused_worker_proof
|
||||
record.terminal_status_revision = answer.previous_terminal_revision
|
||||
record.terminal_status_persisted = answer.previous_terminal_persisted
|
||||
record.terminal_commit_pending = answer.previous_terminal_commit_pending
|
||||
return True
|
||||
|
||||
async def persist_metadata(self, run_id: str) -> bool:
|
||||
"""Upsert the full run record so metadata (e.g. HITL payload) is durable."""
|
||||
async with self._lock:
|
||||
record = self._runs.get(run_id)
|
||||
if record is None or record.ownership_lost:
|
||||
return False
|
||||
if self._store is None:
|
||||
return True
|
||||
payload = self._store_put_payload(record)
|
||||
return await self._persist_snapshot_with_reconciliation(run_id, payload)
|
||||
|
||||
async def set_status_if_not_cancelled(
|
||||
self,
|
||||
run_id: str,
|
||||
|
||||
@ -24,6 +24,7 @@ class RunStatus(StrEnum):
|
||||
pending = "pending"
|
||||
running = "running"
|
||||
success = "success"
|
||||
waiting_for_input = "waiting_for_input"
|
||||
error = "error"
|
||||
timeout = "timeout"
|
||||
interrupted = "interrupted"
|
||||
|
||||
@ -267,6 +267,27 @@ class RunStore(abc.ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
async def answer_approval(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
metadata: dict[str, Any],
|
||||
answered_updated_at: str,
|
||||
expected_request_id: str | None = None,
|
||||
) -> bool:
|
||||
"""Atomically answer one ``waiting_for_input`` run.
|
||||
|
||||
Implementations must condition the write on the row still being paused,
|
||||
remove the consumed approval through *metadata*, and return ``True`` only
|
||||
for the single winning transition. When supplied, *expected_request_id*
|
||||
fences the transition to the exact validated pause. Legacy stores fail closed.
|
||||
"""
|
||||
raise NotImplementedError("Store must implement atomic approval answering")
|
||||
|
||||
async def restore_approval(self, run_id: str, *, metadata: dict[str, Any], expected_updated_at: str) -> bool:
|
||||
"""Restore only the answered transition stamped with *expected_updated_at*."""
|
||||
raise NotImplementedError("Store must implement conditional approval restoration")
|
||||
|
||||
@abc.abstractmethod
|
||||
async def start_run(self, run_id: str) -> bool:
|
||||
"""Atomically transition a pending run to running.
|
||||
|
||||
@ -137,6 +137,7 @@ class MemoryRunStore(RunStore):
|
||||
self._unindex_run(run_id, existing["thread_id"])
|
||||
self._mark_changed(self._runs[run_id])
|
||||
self._index_run(run_id, thread_id)
|
||||
return self._runs[run_id]
|
||||
|
||||
async def get(self, run_id, *, user_id=None):
|
||||
run = self._runs.get(run_id)
|
||||
@ -263,6 +264,30 @@ class MemoryRunStore(RunStore):
|
||||
self._mark_changed(run)
|
||||
return True
|
||||
|
||||
async def answer_approval(self, run_id, *, metadata, answered_updated_at, expected_request_id=None):
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "waiting_for_input":
|
||||
return False
|
||||
if expected_request_id is not None:
|
||||
pause = (run.get("metadata") or {}).get("hitl_approval")
|
||||
if not isinstance(pause, dict) or pause.get("request_id") != expected_request_id:
|
||||
return False
|
||||
run["status"] = "success"
|
||||
run["metadata"] = copy.deepcopy(metadata)
|
||||
run["updated_at"] = answered_updated_at
|
||||
self._mark_changed(run)
|
||||
return True
|
||||
|
||||
async def restore_approval(self, run_id, *, metadata, expected_updated_at):
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "success" or run.get("updated_at") != expected_updated_at:
|
||||
return False
|
||||
run["status"] = "waiting_for_input"
|
||||
run["metadata"] = copy.deepcopy(metadata)
|
||||
run["updated_at"] = datetime.now(UTC).isoformat()
|
||||
self._mark_changed(run)
|
||||
return True
|
||||
|
||||
async def start_run(self, run_id) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "pending":
|
||||
|
||||
@ -42,7 +42,7 @@ from langgraph.types import Command, Overwrite
|
||||
from deerflow.agents.goal_state import GoalEvaluation, GoalState
|
||||
from deerflow.agents.interaction_policy import RunInteractionPolicy, resolve_run_interaction_policy
|
||||
from deerflow.agents.middlewares.input_sanitization_middleware import neutralize_untrusted_tags
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.app_config import AppConfig, get_app_config
|
||||
from deerflow.config.database_config import CheckpointChannelMode
|
||||
from deerflow.constants import CONVERSATION_READER_CONTEXT_KEY, DELIVERY_PRESENTATION_TOOL_NAMES, TOOL_RESULTS_DIRNAME
|
||||
from deerflow.knowledge_scope import KNOWLEDGE_SCOPE_RUNTIME_KEY, execution_scope
|
||||
@ -50,6 +50,7 @@ from deerflow.mcp_scope import (
|
||||
THREAD_INCARNATION_CONTEXT_KEY,
|
||||
THREAD_INCARNATION_METADATA_GUARD_KEY,
|
||||
)
|
||||
from deerflow.runtime.checkpoint_history import recent_checkpoint_history
|
||||
from deerflow.runtime.checkpoint_mode import (
|
||||
aensure_checkpoint_mode_compatible,
|
||||
inject_checkpoint_mode,
|
||||
@ -624,6 +625,8 @@ _SERVER_OWNED_RUNTIME_CONTEXT_KEYS: Final[frozenset[str]] = (
|
||||
SCHEDULER_CAPABILITY_CONTEXT_KEY,
|
||||
AGENT_RUNS_CONTEXT_KEY,
|
||||
"__run_input_control",
|
||||
"_hitl_disabled_resume_escape",
|
||||
"_hitl_resume_tool_names",
|
||||
THREAD_INCARNATION_CONTEXT_KEY,
|
||||
THREAD_INCARNATION_METADATA_GUARD_KEY,
|
||||
"is_subagent",
|
||||
@ -1077,6 +1080,8 @@ async def run_agent(
|
||||
goal_evaluator_model: Any | None = None
|
||||
delivery_content: dict[str, Any] | None = None
|
||||
goal_completion: _GoalCompletionCandidate | None = None
|
||||
pending_pause = False
|
||||
streamed_interrupts: list[Any] = []
|
||||
input_control = None
|
||||
scheduled_goal: GoalState | None = None
|
||||
final_goal_verdict: dict[str, Any] | None = None
|
||||
@ -1344,11 +1349,22 @@ async def run_agent(
|
||||
# access thread-level data. langgraph-cli does this automatically; we must do it
|
||||
# manually here because we drive the graph through ``agent.astream(config=...)``
|
||||
# without passing the official ``context=`` parameter.
|
||||
hitl_app_config = ctx.app_config
|
||||
all_reject_hitl_resume = _is_all_reject_hitl_resume(graph_input)
|
||||
if hitl_app_config is None and all_reject_hitl_resume:
|
||||
hitl_app_config = get_app_config()
|
||||
hitl_config = getattr(hitl_app_config, "hitl_approval", None)
|
||||
hitl_resume_escape = hitl_config is not None and not hitl_config.enabled and all_reject_hitl_resume
|
||||
hitl_resume_tool_names = await _pending_hitl_resume_tool_names(checkpointer, checkpoint_config, graph_input)
|
||||
agent_app_config = ctx.app_config
|
||||
if hitl_resume_escape:
|
||||
agent_app_config = hitl_app_config.model_copy(deep=True)
|
||||
agent_app_config.hitl_approval.enabled = True
|
||||
runtime_ctx = _build_runtime_context(
|
||||
thread_id,
|
||||
run_id,
|
||||
config.get("context"),
|
||||
ctx.app_config,
|
||||
agent_app_config,
|
||||
task_store,
|
||||
extensions,
|
||||
ctx.conversation_reader,
|
||||
@ -1356,6 +1372,15 @@ async def run_agent(
|
||||
thread_incarnation=thread_incarnation,
|
||||
scheduler_capability=ctx.scheduler_capability,
|
||||
)
|
||||
# A disabled all-reject escape still needs the HITL hook to consume the
|
||||
# pending interrupt. The assembly marker makes that hook deny rather
|
||||
# than pause again; all other runs cannot inject this internal mode.
|
||||
runtime_ctx.pop("_hitl_disabled_resume_escape", None)
|
||||
runtime_ctx.pop("_hitl_resume_tool_names", None)
|
||||
if hitl_resume_tool_names:
|
||||
runtime_ctx["_hitl_resume_tool_names"] = hitl_resume_tool_names
|
||||
if hitl_resume_escape:
|
||||
runtime_ctx["_hitl_disabled_resume_escape"] = True
|
||||
register_sandbox_prewarm_cleanup(runtime_ctx)
|
||||
# Bind every checkpoint produced by this run to the effective agent
|
||||
# identity that produced its state. Manual compaction uses only this
|
||||
@ -1430,8 +1455,8 @@ async def run_agent(
|
||||
return continuation
|
||||
|
||||
agent_factory_kwargs: dict[str, Any] = {"config": initial_runnable_config}
|
||||
if ctx.app_config is not None and _agent_factory_supports_app_config(agent_factory):
|
||||
agent_factory_kwargs["app_config"] = ctx.app_config
|
||||
if agent_app_config is not None and _agent_factory_supports_app_config(agent_factory):
|
||||
agent_factory_kwargs["app_config"] = agent_app_config
|
||||
from deerflow.extensions import bind_agent_build_extensions
|
||||
from deerflow.runtime.assembly_evidence import capture_assembly, project_assembly_evidence
|
||||
|
||||
@ -1596,6 +1621,47 @@ async def run_agent(
|
||||
# re-enter the stream and would otherwise discard the resolved seqs.
|
||||
seq_stamper = _build_seq_stamper(event_store, thread_id, journal) if "values" in requested_modes else None
|
||||
|
||||
def observe_stream_pause(mode: str, chunk: Any, namespace: tuple[str, ...]) -> None:
|
||||
if namespace or mode not in {"values", "updates"} or not isinstance(chunk, Mapping):
|
||||
return
|
||||
if mode == "values":
|
||||
streamed_interrupts.clear()
|
||||
interrupts = chunk.get("__interrupt__")
|
||||
if not interrupts:
|
||||
return
|
||||
streamed_interrupts.extend(interrupts if isinstance(interrupts, (list, tuple)) else [interrupts])
|
||||
|
||||
def remember_approval(interrupts: Any) -> None:
|
||||
for pending_interrupt in interrupts:
|
||||
value = pending_interrupt.get("value") if isinstance(pending_interrupt, Mapping) else getattr(pending_interrupt, "value", None)
|
||||
if isinstance(value, dict) and value.get("kind") == "hitl_approval":
|
||||
record.metadata["hitl_approval"] = value
|
||||
return
|
||||
|
||||
async def detect_pending_pause() -> None:
|
||||
nonlocal pending_pause
|
||||
if pending_pause or record.abort_event.is_set():
|
||||
return
|
||||
if streamed_interrupts:
|
||||
pending_pause = True
|
||||
remember_approval(streamed_interrupts)
|
||||
return
|
||||
# Values/update frames expose interrupts directly, so disabled HITL
|
||||
# keeps the old hot path there. Message-only streams need the snapshot
|
||||
# fallback for generic (non-HITL) interrupts as well.
|
||||
hitl_enabled = bool(hitl_config is not None and hitl_config.enabled)
|
||||
if not hitl_enabled and {"values", "updates"} & requested_modes:
|
||||
return
|
||||
snapshot_capable = accessor is not None and accessor.checkpointer is not None and callable(getattr(getattr(accessor, "graph", None), "aget_state", None))
|
||||
if not snapshot_capable:
|
||||
return
|
||||
snapshot = await accessor.aget({"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}})
|
||||
snapshot_tasks = getattr(snapshot, "tasks", None) or ()
|
||||
pending_pause = any(getattr(task, "interrupts", ()) for task in snapshot_tasks)
|
||||
if pending_pause:
|
||||
for task in snapshot_tasks:
|
||||
remember_approval(getattr(task, "interrupts", ()) or ())
|
||||
|
||||
async def _stream_once(input_payload: Any, stream_config: RunnableConfig) -> None:
|
||||
nonlocal llm_error_fallback_message
|
||||
if input_control is not None:
|
||||
@ -1625,6 +1691,7 @@ async def run_agent(
|
||||
await bridge.publish(run_id, sse_event, single_payload)
|
||||
if single_mode == "messages" and partial_assistant_response is not None:
|
||||
partial_assistant_response.observe(chunk)
|
||||
observe_stream_pause(single_mode, chunk, ())
|
||||
if single_mode == "custom":
|
||||
await subagent_events.add(chunk)
|
||||
finally:
|
||||
@ -1673,6 +1740,7 @@ async def run_agent(
|
||||
# includes the child messages task_running custom events
|
||||
# carry, which are root frames too.
|
||||
llm_error_fallback_message = llm_error_fallback_message or _extract_llm_error_fallback_message(chunk, pre_existing_message_ids)
|
||||
observe_stream_pause(mode, chunk, namespace)
|
||||
await _publish_stream_item(
|
||||
bridge=bridge,
|
||||
run_id=run_id,
|
||||
@ -1740,7 +1808,8 @@ async def run_agent(
|
||||
await _stream_once(graph_input, initial_runnable_config)
|
||||
if input_control is not None:
|
||||
await input_control.confirm_checkpoint()
|
||||
while not record.abort_event.is_set() and not llm_error_fallback_message and (journal is None or not journal.had_llm_error_fallback):
|
||||
await detect_pending_pause()
|
||||
while not pending_pause and not record.abort_event.is_set() and not llm_error_fallback_message and (journal is None or not journal.had_llm_error_fallback):
|
||||
continuation_input = await _prepare_goal_continuation_input(
|
||||
bridge=bridge,
|
||||
accessor=accessor,
|
||||
@ -1767,6 +1836,7 @@ async def run_agent(
|
||||
if continuation_input is None or record.abort_event.is_set():
|
||||
break
|
||||
await _stream_once(continuation_input, _continuation_runnable_config())
|
||||
await detect_pending_pause()
|
||||
|
||||
record.goal_verdict = final_goal_verdict
|
||||
|
||||
@ -1809,21 +1879,23 @@ async def run_agent(
|
||||
from deerflow.sandbox.lease import cancel_sandbox_execution_lease_async
|
||||
|
||||
await cancel_sandbox_execution_lease_async(runtime_ctx)
|
||||
produced_output_paths = await _produced_output_paths(
|
||||
pre_run_workspace_snapshot,
|
||||
thread_id=thread_id,
|
||||
user_id=workspace_changes_user_id,
|
||||
extra_excluded_dir_names=workspace_excluded_dir_names,
|
||||
)
|
||||
delivery_content = _delivery_content_with_outputs(
|
||||
journal.get_delivery_content() if journal is not None else _empty_delivery_content(),
|
||||
produced_output_paths,
|
||||
pre_run_snapshot=pre_run_workspace_snapshot,
|
||||
)
|
||||
delivery_error = _delivery_error(delivery_content)
|
||||
delivery_error = None
|
||||
if not pending_pause:
|
||||
produced_output_paths = await _produced_output_paths(
|
||||
pre_run_workspace_snapshot,
|
||||
thread_id=thread_id,
|
||||
user_id=workspace_changes_user_id,
|
||||
extra_excluded_dir_names=workspace_excluded_dir_names,
|
||||
)
|
||||
delivery_content = _delivery_content_with_outputs(
|
||||
journal.get_delivery_content() if journal is not None else _empty_delivery_content(),
|
||||
produced_output_paths,
|
||||
pre_run_snapshot=pre_run_workspace_snapshot,
|
||||
)
|
||||
delivery_error = _delivery_error(delivery_content)
|
||||
cancel_action = await run_manager.set_status_if_not_cancelled(
|
||||
run_id,
|
||||
RunStatus.error if delivery_error else RunStatus.success,
|
||||
RunStatus.error if delivery_error else (RunStatus.waiting_for_input if pending_pause else RunStatus.success),
|
||||
error=delivery_error,
|
||||
stop_reason=stop_reason,
|
||||
goal_verdict=record.goal_verdict,
|
||||
@ -2072,6 +2144,11 @@ async def run_agent(
|
||||
await run_manager.persist_current_status(run_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to persist terminal status for run %s after delivery receipt attempts", run_id, exc_info=True)
|
||||
if record.status == RunStatus.waiting_for_input and not record.ownership_lost:
|
||||
# A full metadata snapshot also carries the status. Write it
|
||||
# only after the explicit terminal transition above, otherwise
|
||||
# the same-status row short-circuits terminal event attribution.
|
||||
await run_manager.persist_metadata(run_id)
|
||||
# Only a confirmed terminal write or ownership loss releases the
|
||||
# renewal barrier in multi-worker mode. A returned/aborted call is
|
||||
# not evidence that this worker committed its terminal outcome.
|
||||
@ -2138,7 +2215,7 @@ async def run_agent(
|
||||
# Update threads_meta status based on run outcome
|
||||
if started and not record.ownership_lost and thread_store is not None:
|
||||
try:
|
||||
final_status = "idle" if record.status == RunStatus.success else record.status.value
|
||||
final_status = "idle" if record.status == RunStatus.success else ("interrupted" if record.status == RunStatus.waiting_for_input else record.status.value)
|
||||
await thread_store.update_status(thread_id, final_status)
|
||||
except Exception:
|
||||
logger.debug("Failed to update thread_meta status for %s (non-fatal)", thread_id)
|
||||
@ -2157,7 +2234,7 @@ async def run_agent(
|
||||
except Exception:
|
||||
logger.warning("Failed to confirm terminal status for run %s before completion hook", run_id, exc_info=True)
|
||||
|
||||
if not record.ownership_lost and getattr(record, "terminal_status_persisted", True) and ctx.on_run_completed is not None:
|
||||
if not record.ownership_lost and record.status != RunStatus.waiting_for_input and getattr(record, "terminal_status_persisted", True) and ctx.on_run_completed is not None:
|
||||
try:
|
||||
# Preserve hook liveness without forwarding cancellation
|
||||
# to the finalizer's durable writes or resource cleanup.
|
||||
@ -3104,6 +3181,54 @@ def _is_resume_command_input(graph_input: Any) -> bool:
|
||||
return isinstance(graph_input, Command) and graph_input.resume is not None
|
||||
|
||||
|
||||
async def _pending_hitl_resume_tool_names(checkpointer: Any, checkpoint_config: dict[str, Any], graph_input: Any) -> set[str]:
|
||||
"""Recover gated names from the durable pause matched by this resume."""
|
||||
if checkpointer is None or not _is_resume_command_input(graph_input) or not isinstance(graph_input.resume, Mapping):
|
||||
return set()
|
||||
responses = graph_input.resume.get("responses")
|
||||
if not isinstance(responses, list):
|
||||
return set()
|
||||
response_ids = {response.get("tool_call_id") for response in responses if isinstance(response, Mapping) and isinstance(response.get("tool_call_id"), str)}
|
||||
if not response_ids:
|
||||
return set()
|
||||
async for checkpoint_tuple in recent_checkpoint_history(checkpointer, checkpoint_config):
|
||||
writes = list(getattr(checkpoint_tuple, "pending_writes", ()) or ())
|
||||
# The newest checkpoint carrying either marker decides whether the pause
|
||||
# is still live. A newer head-moving write must not hide an interrupt;
|
||||
# a newer resume must not resurrect an older one.
|
||||
if any(isinstance(write, (list, tuple)) and len(write) >= 2 and write[1] == "__resume__" for write in writes):
|
||||
return set()
|
||||
interrupt_writes = [write for write in writes if isinstance(write, (list, tuple)) and len(write) >= 3 and write[1] == "__interrupt__"]
|
||||
if not interrupt_writes:
|
||||
continue
|
||||
names: set[str] = set()
|
||||
for write in interrupt_writes:
|
||||
interrupts = write[2] if isinstance(write[2], (list, tuple)) else (write[2],)
|
||||
for pending_interrupt in interrupts:
|
||||
payload = getattr(pending_interrupt, "value", None)
|
||||
if not isinstance(payload, Mapping) or payload.get("kind") != "hitl_approval":
|
||||
continue
|
||||
pending_ids = payload.get("pending_tool_call_ids")
|
||||
actions = payload.get("actions")
|
||||
if not isinstance(pending_ids, list) or set(pending_ids) != response_ids or not isinstance(actions, list):
|
||||
continue
|
||||
names.update(action.get("tool_name") for action in actions if isinstance(action, Mapping) and isinstance(action.get("tool_name"), str))
|
||||
return names
|
||||
return set()
|
||||
|
||||
|
||||
def _is_all_reject_hitl_resume(graph_input: Any) -> bool:
|
||||
"""Return whether this is a well-formed reject-only HITL resume payload."""
|
||||
if not isinstance(graph_input, Command) or not isinstance(graph_input.resume, dict):
|
||||
return False
|
||||
responses = graph_input.resume.get("responses")
|
||||
return (
|
||||
isinstance(responses, list)
|
||||
and bool(responses)
|
||||
and all(isinstance(response, dict) and response.get("option_id") == "reject" and isinstance(response.get("tool_call_id"), str) and bool(response["tool_call_id"]) for response in responses)
|
||||
)
|
||||
|
||||
|
||||
class CheckpointResumeRejected(RuntimeError):
|
||||
"""Checkpoint preparation was rejected before any state write."""
|
||||
|
||||
|
||||
@ -1245,6 +1245,13 @@ class SubagentExecutor:
|
||||
"is" if len(missing) == 1 else "are",
|
||||
self.config.name,
|
||||
)
|
||||
bound_tools = list(tools if tools is not None else self.tools)
|
||||
# Native subagents have no resumable checkpoint/runtime. Reuse the lead
|
||||
# builder's configured policy so matching calls fail closed instead of
|
||||
# bypassing approval; disabled configuration remains a no-op.
|
||||
from deerflow.agents.lead_agent.agent import _insert_hitl_approval_middleware
|
||||
|
||||
_insert_hitl_approval_middleware(middlewares, bound_tools, app_config, supports_resume=False)
|
||||
# Collect every guard middleware that exposes ``consume_stop_reason``
|
||||
# (TokenBudgetMiddleware, LoopDetectionMiddleware,
|
||||
# ModelLengthFinishReasonMiddleware) so _aexecute can read
|
||||
@ -1257,7 +1264,6 @@ class SubagentExecutor:
|
||||
|
||||
# system_prompt is included in initial state messages (see _build_initial_state)
|
||||
# to avoid multiple SystemMessages which some LLM APIs don't support.
|
||||
bound_tools = list(tools if tools is not None else self.tools)
|
||||
agent = create_agent(
|
||||
model=model,
|
||||
tools=bound_tools,
|
||||
|
||||
@ -232,7 +232,7 @@ async def test_resumed_native_batch_preserves_admitted_scope_in_followup(batch_h
|
||||
monkeypatch.setattr(services, "resolve_agent_factory", lambda _: graph_factory(factory, answer))
|
||||
thread = await runs.create_thread(thread_id="scoped-resume")
|
||||
first = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "scoped work", "additional_kwargs": {KNOWLEDGE_SCOPE_KEY: scope}}]})
|
||||
assert (await runs.wait(thread_id=thread, run_id=first.run_id, timeout=5)).status == "success"
|
||||
assert (await runs.wait(thread_id=thread, run_id=first.run_id, timeout=5)).status == "waiting_for_input"
|
||||
resumed = await runs.resume(thread_id=thread, resume="yes")
|
||||
assert (await runs.wait(thread_id=thread, run_id=resumed.run_id, timeout=5)).status == "success"
|
||||
assert observed == [scope]
|
||||
|
||||
@ -15,6 +15,7 @@ state accessor are all real; only the external model service is faked.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
@ -28,8 +29,9 @@ from langchain_core.outputs import ChatGeneration, ChatResult
|
||||
from langchain_core.runnables import Runnable
|
||||
from langchain_core.tools import tool
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.graph import StateGraph
|
||||
from langgraph.store.memory import InMemoryStore
|
||||
from langgraph.types import Command
|
||||
from langgraph.types import Command, interrupt
|
||||
|
||||
from app.gateway.auth.models import User
|
||||
from app.gateway.routers import threads
|
||||
@ -37,7 +39,7 @@ from deerflow.agents.thread_state import get_thread_state_schema
|
||||
from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config
|
||||
from deerflow.runtime import RunManager, context_compaction
|
||||
from deerflow.runtime.checkpoint_mode import inject_checkpoint_mode
|
||||
from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
|
||||
from deerflow.runtime.checkpoint_state import CheckpointStateAccessor, build_state_mutation_graph
|
||||
|
||||
THREAD_ID = "thread-compact-interrupted"
|
||||
TOOL_CALL_ID = "call_probe_1"
|
||||
@ -157,7 +159,7 @@ def test_compact_rejects_thread_paused_before_tools(mode: str, _summarization_ap
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/api/threads/{THREAD_ID}/compact", json={"force": True, "keep": {"type": "messages", "value": 1}})
|
||||
assert response.status_code == 409, response.text
|
||||
assert response.json()["detail"] == "Thread has pending execution. Resume the interrupted run before compacting."
|
||||
assert response.json()["detail"] == "Thread has resumable pending execution. Resume the paused run before compacting."
|
||||
|
||||
# The rejected compaction wrote nothing: head checkpoint, pending position
|
||||
# and pending task identity are exactly as before the attempt.
|
||||
@ -175,6 +177,130 @@ def test_compact_rejects_thread_paused_before_tools(mode: str, _summarization_ap
|
||||
assert any(isinstance(m, AIMessage) and m.content == FINAL_ANSWER for m in final_messages)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
def test_compact_rejects_dynamic_interrupt_through_mutation_accessor(mode, _summarization_app_config, _fake_summary_model):
|
||||
"""The node-restricted mutation graph cannot expose the agent's pending
|
||||
task, so compaction must use the raw interrupt write as its pause signal."""
|
||||
checkpointer = InMemorySaver()
|
||||
|
||||
def pause(state):
|
||||
answer = interrupt({"question": "Continue?"})
|
||||
return {"messages": [AIMessage(content=f"resumed:{answer}", id="resumed")]}
|
||||
|
||||
builder = StateGraph(get_thread_state_schema(mode))
|
||||
builder.add_node("pause", pause)
|
||||
builder.set_entry_point("pause")
|
||||
builder.set_finish_point("pause")
|
||||
graph = builder.compile(checkpointer=checkpointer)
|
||||
config = _config(mode)
|
||||
graph.invoke({"messages": [HumanMessage(content="hello", id="h1")]}, config)
|
||||
|
||||
paused = graph.get_state(config)
|
||||
assert list(paused.next) == ["pause"]
|
||||
paused_checkpoint_id = paused.config["configurable"]["checkpoint_id"]
|
||||
raw_tuple = checkpointer.get_tuple(config)
|
||||
assert raw_tuple is not None
|
||||
assert any(write[1] == "__interrupt__" for write in raw_tuple.pending_writes or ())
|
||||
|
||||
mutation_graph = build_state_mutation_graph("manual_compaction", mode, get_thread_state_schema(mode))
|
||||
mutation_accessor = CheckpointStateAccessor.bind(mutation_graph, checkpointer, mode=mode, direct_namespace=True)
|
||||
mutation_snapshot = mutation_accessor.get(config)
|
||||
assert not mutation_snapshot.next
|
||||
assert not mutation_snapshot.tasks
|
||||
|
||||
app = _make_compact_app(mode, checkpointer)
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/api/threads/{THREAD_ID}/compact", json={"force": True, "keep": {"type": "messages", "value": 1}})
|
||||
assert response.status_code == 409, response.text
|
||||
assert response.json()["detail"] == "Thread has resumable pending execution. Resume the paused run before compacting."
|
||||
assert graph.get_state(config).config["configurable"]["checkpoint_id"] == paused_checkpoint_id
|
||||
|
||||
graph.invoke(Command(resume="yes"), config)
|
||||
assert any(isinstance(message, AIMessage) and message.content == "resumed:yes" for message in graph.get_state(config).values["messages"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
def test_compact_rejects_off_head_dynamic_interrupt_after_state_update(mode, _summarization_app_config, _fake_summary_model):
|
||||
checkpointer = InMemorySaver()
|
||||
|
||||
def pause(state):
|
||||
answer = interrupt({"question": "Continue?"})
|
||||
return {"messages": [AIMessage(content=f"resumed:{answer}", id="resumed")]}
|
||||
|
||||
builder = StateGraph(get_thread_state_schema(mode))
|
||||
builder.add_node("pause", pause)
|
||||
builder.set_entry_point("pause")
|
||||
builder.set_finish_point("pause")
|
||||
graph = builder.compile(checkpointer=checkpointer)
|
||||
config = _config(mode)
|
||||
graph.invoke({"messages": [HumanMessage(content="hello", id="h1")]}, config)
|
||||
|
||||
mutation_graph = build_state_mutation_graph("manual_compaction", mode, get_thread_state_schema(mode))
|
||||
mutation_accessor = CheckpointStateAccessor.bind(mutation_graph, checkpointer, mode=mode, direct_namespace=True)
|
||||
mutation_accessor.update(config, {"messages": [AIMessage(content="head move", id="head-move")]}, as_node="manual_compaction")
|
||||
|
||||
moved = graph.get_state(config)
|
||||
assert list(moved.next) == ["pause"]
|
||||
moved_checkpoint_id = moved.config["configurable"]["checkpoint_id"]
|
||||
raw_head = checkpointer.get_tuple(config)
|
||||
assert raw_head is not None
|
||||
assert not any(write[1] == "__interrupt__" for write in raw_head.pending_writes or ())
|
||||
|
||||
app = _make_compact_app(mode, checkpointer)
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/api/threads/{THREAD_ID}/compact", json={"force": True, "keep": {"type": "messages", "value": 1}})
|
||||
|
||||
assert response.status_code == 409, response.text
|
||||
assert response.json()["detail"] == "Thread has resumable pending execution. Resume the paused run before compacting."
|
||||
assert graph.get_state(config).config["configurable"]["checkpoint_id"] == moved_checkpoint_id
|
||||
|
||||
graph.invoke(Command(resume="yes"), config)
|
||||
assert any(isinstance(message, AIMessage) and message.content == "resumed:yes" for message in graph.get_state(config).values["messages"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
def test_compact_allows_answered_off_head_interrupt_after_state_update(mode, _summarization_app_config, _fake_summary_model):
|
||||
from app.gateway.hitl_approval import has_any_pending_interrupt
|
||||
|
||||
checkpointer = InMemorySaver()
|
||||
|
||||
def pause(state):
|
||||
answer = interrupt({"question": "Continue?"})
|
||||
return {"messages": [AIMessage(content=f"resumed:{answer}", id="resumed")]}
|
||||
|
||||
builder = StateGraph(get_thread_state_schema(mode))
|
||||
builder.add_node("pause", pause)
|
||||
builder.set_entry_point("pause")
|
||||
builder.set_finish_point("pause")
|
||||
graph = builder.compile(checkpointer=checkpointer)
|
||||
config = _config(mode)
|
||||
graph.invoke({"messages": [HumanMessage(content="hello", id="h1")]}, config)
|
||||
|
||||
mutation_graph = build_state_mutation_graph("manual_compaction", mode, get_thread_state_schema(mode))
|
||||
mutation_accessor = CheckpointStateAccessor.bind(mutation_graph, checkpointer, mode=mode, direct_namespace=True)
|
||||
assert asyncio.run(has_any_pending_interrupt(mutation_accessor, config)) is True
|
||||
|
||||
graph.invoke(Command(resume="yes"), config)
|
||||
mutation_accessor.update(config, {"messages": [AIMessage(content="head move", id="head-move")]}, as_node="manual_compaction")
|
||||
|
||||
moved = graph.get_state(config)
|
||||
assert not moved.next
|
||||
raw_head = checkpointer.get_tuple(config)
|
||||
assert raw_head is not None
|
||||
assert not any(write[1] in {"__interrupt__", "__resume__"} for write in raw_head.pending_writes or ())
|
||||
# The current HITL middleware makes one interrupt() call per after_model
|
||||
# task, so [__interrupt__, __resume__, __interrupt__] cannot occur in one
|
||||
# saver checkpoint; the newest checkpoint carrying either marker is decisive.
|
||||
assert asyncio.run(has_any_pending_interrupt(mutation_accessor, config)) is False
|
||||
|
||||
app = _make_compact_app(mode, checkpointer)
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/api/threads/{THREAD_ID}/compact", json={"force": True, "keep": {"type": "messages", "value": 1}})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["compacted"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
def test_compact_rejects_unavailable_checkpoint(mode: str, monkeypatch, _summarization_app_config, _fake_summary_model):
|
||||
"""A valid snapshot alone cannot prove the raw checkpoint has no pending work."""
|
||||
|
||||
@ -380,6 +380,108 @@ async def test_seal_requires_drain_and_current_owner(evidence):
|
||||
await journal.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_waiting_for_input_run_seals_drained_evidence(evidence):
|
||||
_sf, events, runs, reader = evidence
|
||||
await put_run(
|
||||
runs,
|
||||
status="waiting_for_input",
|
||||
owner_worker_id="worker",
|
||||
lease_expires_at=(datetime.now(UTC) + timedelta(minutes=5)).isoformat(),
|
||||
)
|
||||
journal = RunJournal("r", "t-r", events, user_id="alice")
|
||||
journal.record_middleware("hitl", name="hitl", hook="after_model", action="pause", changes={})
|
||||
receipt = await journal.finish_evidence()
|
||||
|
||||
assert await runs.seal_evidence("r", owner_worker_id="worker", receipt=receipt) is True
|
||||
snapshot = await reader.get_snapshot(thread_id="t-r", run_id="r")
|
||||
assert snapshot.status == "waiting_for_input"
|
||||
assert snapshot.seal_state == "sealed"
|
||||
assert snapshot.event_count == 1
|
||||
await journal.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_answered_ack_lets_paused_worker_finish_committed_terminal(evidence):
|
||||
import asyncio
|
||||
from contextlib import suppress
|
||||
|
||||
from deerflow.config.run_ownership_config import RunOwnershipConfig
|
||||
from deerflow.runtime import RunManager, RunStatus
|
||||
from deerflow.runtime.events.terminal import confirm_terminal_event
|
||||
|
||||
_sf, events, runs, reader = evidence
|
||||
manager = RunManager(
|
||||
store=runs,
|
||||
worker_id="owner",
|
||||
run_ownership_config=RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=30, grace_seconds=0),
|
||||
)
|
||||
paused = await manager.create_or_reject(
|
||||
"t-answered",
|
||||
user_id="alice",
|
||||
metadata={"hitl_approval": {"request_id": "approval-1"}},
|
||||
evidence_origin="interactive",
|
||||
evidence_agent_id="lead_agent",
|
||||
)
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
journal = RunJournal(paused.run_id, paused.thread_id, events, user_id="alice")
|
||||
journal.record_middleware("hitl", name="hitl", hook="after_model", action="pause", changes={})
|
||||
original_put_if_absent = events.put_if_absent
|
||||
terminal_committed, release_terminal = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def commit_then_hold_terminal(*args, **kwargs):
|
||||
result = await original_put_if_absent(*args, **kwargs)
|
||||
terminal_committed.set()
|
||||
await release_terminal.wait()
|
||||
return result
|
||||
|
||||
async def finalize_paused_worker():
|
||||
status = manager.terminal_event_status(paused)
|
||||
assert status == RunStatus.waiting_for_input.value
|
||||
|
||||
async def confirm_terminal():
|
||||
confirmation = await confirm_terminal_event(
|
||||
events,
|
||||
thread_id=paused.thread_id,
|
||||
run_id=paused.run_id,
|
||||
status=status,
|
||||
user_id=paused.user_id,
|
||||
authorized=lambda: manager.terminal_event_status(paused) == status,
|
||||
)
|
||||
assert confirmation.confirmed
|
||||
|
||||
receipt = await journal.finish_evidence(before_receipt=confirm_terminal)
|
||||
assert receipt is not None
|
||||
assert await manager.seal_completed_evidence(paused.run_id, receipt) is True
|
||||
await manager.update_run_completion(paused.run_id, status=status, total_tokens=17, message_count=2)
|
||||
|
||||
events.put_if_absent = commit_then_hold_terminal
|
||||
worker = asyncio.create_task(finalize_paused_worker())
|
||||
paused.task = worker
|
||||
try:
|
||||
await asyncio.wait_for(terminal_committed.wait(), 1)
|
||||
assert await manager.mark_approval_answered(paused.thread_id) is not None
|
||||
assert paused.status == RunStatus.success
|
||||
release_terminal.set()
|
||||
await asyncio.wait_for(worker, 1)
|
||||
finally:
|
||||
release_terminal.set()
|
||||
if not worker.done():
|
||||
worker.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await worker
|
||||
events.put_if_absent = original_put_if_absent
|
||||
await journal.close()
|
||||
|
||||
assert manager.terminal_event_status(paused) == RunStatus.success.value
|
||||
snapshot = await reader.get_snapshot(thread_id=paused.thread_id, run_id=paused.run_id)
|
||||
assert snapshot.status == RunStatus.success.value
|
||||
assert snapshot.seal_state == "sealed"
|
||||
answered = await runs.get(paused.run_id, user_id="alice")
|
||||
assert answered["total_tokens"] == 17
|
||||
assert answered["message_count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proven_not_committed_flush_never_issues_seal_and_close_can_retry():
|
||||
from deerflow.runtime.events.store.base import RunEventStoreNotCommittedError
|
||||
@ -445,8 +547,8 @@ async def test_expired_lease_cannot_seal(evidence):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["success", "error", "interrupted", "timeout"])
|
||||
async def test_terminal_unsealed_crash_recovery_advances_discovery(evidence, status):
|
||||
@pytest.mark.parametrize("status", ["success", "error", "interrupted", "timeout", "waiting_for_input"])
|
||||
async def test_unsealed_crash_recovery_advances_discovery(evidence, status):
|
||||
from deerflow.runtime.runs.manager import RunManager
|
||||
|
||||
sf, events, runs, reader = evidence
|
||||
|
||||
@ -130,6 +130,36 @@ async def test_compact_thread_context_reads_materialized_state_and_overwrites_me
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_thread_context_rejects_genuine_interrupt_pause(monkeypatch):
|
||||
accessor = _FakeAccessor({"messages": [HumanMessage(content="question"), AIMessage(content="answer")]})
|
||||
accessor.snapshot.next = ("agent",)
|
||||
accessor.snapshot.tasks = (SimpleNamespace(interrupts=(SimpleNamespace(value={"kind": "hitl_approval"}),)),)
|
||||
create_middleware = MagicMock(return_value=_FakeCompactionMiddleware())
|
||||
monkeypatch.setattr(context_compaction, "_create_compaction_middleware", create_middleware)
|
||||
|
||||
with pytest.raises(ContextCompactionPendingWork, match="resumable pending execution"):
|
||||
await compact_thread_context(accessor, "thread-1", app_config=SimpleNamespace())
|
||||
|
||||
create_middleware.assert_not_called()
|
||||
assert accessor.update_args is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_thread_context_allows_terminal_mid_graph_snapshot(monkeypatch):
|
||||
messages = [HumanMessage(content="old question"), AIMessage(content="old answer"), HumanMessage(content="latest question")]
|
||||
accessor = _FakeAccessor({"messages": messages})
|
||||
# Errored/cancelled graphs can retain next nodes without resumable tasks.
|
||||
accessor.snapshot.next = ("tools",)
|
||||
middleware = _FakeCompactionMiddleware()
|
||||
monkeypatch.setattr(context_compaction, "_create_compaction_middleware", lambda **_kwargs: middleware)
|
||||
|
||||
result = await compact_thread_context(accessor, "thread-1", app_config=SimpleNamespace())
|
||||
|
||||
assert result.compacted is True
|
||||
assert accessor.update_args is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_thread_context_rejects_unavailable_checkpoint_before_summarizing(monkeypatch):
|
||||
accessor = _FakeAccessor({"messages": [HumanMessage(content="question"), AIMessage(content="answer")]})
|
||||
@ -149,6 +179,91 @@ async def test_compact_thread_context_rejects_unavailable_checkpoint_before_summ
|
||||
accessor.aupdate.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_thread_context_falls_back_to_sync_head_and_history(monkeypatch):
|
||||
messages = [HumanMessage(content="question"), AIMessage(content="answer")]
|
||||
head = SimpleNamespace(checkpoint={"channel_values": {"messages": messages}}, pending_writes=())
|
||||
paused = SimpleNamespace(
|
||||
checkpoint={"channel_values": {"messages": messages}},
|
||||
pending_writes=(("task-1", "__interrupt__", SimpleNamespace(value={"kind": "hitl_approval"})),),
|
||||
)
|
||||
|
||||
class SyncOnlySaver:
|
||||
list_called = False
|
||||
|
||||
async def alist(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
yield # pragma: no cover
|
||||
|
||||
def list(self, *args, **kwargs):
|
||||
self.list_called = True
|
||||
return iter((head, paused))
|
||||
|
||||
def get_tuple(self, *args, **kwargs):
|
||||
return head
|
||||
|
||||
saver = SyncOnlySaver()
|
||||
accessor = _FakeAccessor({"messages": messages})
|
||||
accessor.checkpointer = saver
|
||||
accessor.aget_tuple = AsyncMock(side_effect=NotImplementedError)
|
||||
accessor.get_tuple = saver.get_tuple
|
||||
create_middleware = MagicMock(return_value=_FakeCompactionMiddleware())
|
||||
monkeypatch.setattr(context_compaction, "_create_compaction_middleware", create_middleware)
|
||||
|
||||
with pytest.raises(ContextCompactionPendingWork, match="resumable pending execution"):
|
||||
await compact_thread_context(accessor, "thread-1", app_config=SimpleNamespace())
|
||||
|
||||
accessor.aget_tuple.assert_awaited_once()
|
||||
assert saver.list_called
|
||||
create_middleware.assert_not_called()
|
||||
assert accessor.update_args is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_thread_context_falls_back_to_sync_current_tuple_without_history(monkeypatch):
|
||||
messages = [HumanMessage(content="question"), AIMessage(content="answer")]
|
||||
paused = SimpleNamespace(
|
||||
checkpoint={"channel_values": {"messages": messages}},
|
||||
pending_writes=(("task-1", "__interrupt__", SimpleNamespace(value={"kind": "hitl_approval"})),),
|
||||
)
|
||||
|
||||
class TupleOnlySaver:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
async def alist(self, *args, **kwargs):
|
||||
self.calls.append("alist")
|
||||
raise NotImplementedError
|
||||
yield # pragma: no cover
|
||||
|
||||
def list(self, *args, **kwargs):
|
||||
self.calls.append("list")
|
||||
raise NotImplementedError
|
||||
|
||||
async def aget_tuple(self, *args, **kwargs):
|
||||
self.calls.append("aget_tuple")
|
||||
raise NotImplementedError
|
||||
|
||||
def get_tuple(self, *args, **kwargs):
|
||||
self.calls.append("get_tuple")
|
||||
return paused
|
||||
|
||||
saver = TupleOnlySaver()
|
||||
accessor = _FakeAccessor({"messages": messages})
|
||||
accessor.checkpointer = saver
|
||||
accessor.aget_tuple = AsyncMock(side_effect=NotImplementedError)
|
||||
accessor.get_tuple = saver.get_tuple
|
||||
create_middleware = MagicMock(return_value=_FakeCompactionMiddleware())
|
||||
monkeypatch.setattr(context_compaction, "_create_compaction_middleware", create_middleware)
|
||||
|
||||
with pytest.raises(ContextCompactionPendingWork, match="resumable pending execution"):
|
||||
await compact_thread_context(accessor, "thread-1", app_config=SimpleNamespace())
|
||||
|
||||
assert saver.calls == ["get_tuple", "alist", "list", "aget_tuple", "get_tuple"]
|
||||
create_middleware.assert_not_called()
|
||||
assert accessor.update_args is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_thread_context_real_mutation_graph_finishes_without_scheduling(monkeypatch):
|
||||
messages = [
|
||||
|
||||
@ -215,7 +215,7 @@ def test_missing_transcript_is_reported_without_checkpoint_reconstruction():
|
||||
|
||||
|
||||
def test_start_run_installs_fresh_capability_without_persisting_it(monkeypatch):
|
||||
from test_gateway_services import _make_start_run_persistence_context
|
||||
from test_gateway_services import _make_start_run_persistence_context, _seed_generic_resume_checkpoint
|
||||
|
||||
from app.gateway import services
|
||||
from deerflow.config.app_config import reset_app_config, set_app_config
|
||||
@ -245,6 +245,7 @@ def test_start_run_installs_fresh_capability_without_persisting_it(monkeypatch):
|
||||
second = await services.start_run(RunCreateRequest(input={"messages": [{"role": "user", "content": 'Quoted text: "read source"'}]}), "later", request)
|
||||
await second.task
|
||||
assert captured[1]["ctx"].conversation_reader is None
|
||||
await _seed_generic_resume_checkpoint(request.app.state.checkpointer, "resume")
|
||||
resumed = await services.start_run(RunCreateRequest(command={"resume": "source"}), "resume", request)
|
||||
await resumed.task
|
||||
assert captured[2]["ctx"].conversation_reader is None
|
||||
|
||||
@ -360,7 +360,7 @@ async def test_real_interrupt_resume_conflict_and_cancel(runtime):
|
||||
runs, _, _, entered, _ = runtime
|
||||
thread = await runs.create_thread(thread_id="interrupt-test")
|
||||
first = await runs.start(thread_id=thread, input={"messages": [{"role": "user", "content": "approve"}]})
|
||||
assert (await runs.wait(thread_id=thread, run_id=first.run_id, timeout=5)).status == "success"
|
||||
assert (await runs.wait(thread_id=thread, run_id=first.run_id, timeout=5)).status == "waiting_for_input"
|
||||
assert (await runs.get_state(thread_id=thread))["next"] == ["answer"]
|
||||
resumed = await runs.resume(thread_id=thread, resume="yes")
|
||||
assert (await runs.wait(thread_id=thread, run_id=resumed.run_id, timeout=5)).status == "success"
|
||||
|
||||
@ -2971,6 +2971,26 @@ def test_inject_authenticated_user_context_strips_internal_spoofed_attribution()
|
||||
assert "oauth_id" not in config["context"]
|
||||
|
||||
|
||||
async def _seed_generic_resume_checkpoint(checkpointer, thread_id):
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.graph import MessagesState, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
def answer(_state):
|
||||
interrupt({"question": "Continue?"})
|
||||
return {}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("answer", answer)
|
||||
builder.set_entry_point("answer")
|
||||
builder.set_finish_point("answer")
|
||||
graph = builder.compile(checkpointer=checkpointer)
|
||||
await graph.ainvoke(
|
||||
{"messages": [HumanMessage(content="start")]},
|
||||
{"configurable": {"thread_id": thread_id}},
|
||||
)
|
||||
|
||||
|
||||
async def _capture_start_run_graph_input(body, *, auth_source=None):
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
@ -2998,6 +3018,8 @@ async def _capture_start_run_graph_input(body, *, auth_source=None):
|
||||
state=SimpleNamespace(auth_source=auth_source),
|
||||
app=SimpleNamespace(state=state),
|
||||
)
|
||||
if getattr(body, "command", None) and body.command.get("resume") is not None:
|
||||
await _seed_generic_resume_checkpoint(state.checkpointer, "thread-command-test")
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_run_agent(*args, **kwargs):
|
||||
|
||||
740
backend/tests/test_hitl_approval_api.py
Normal file
740
backend/tests/test_hitl_approval_api.py
Normal file
@ -0,0 +1,740 @@
|
||||
"""Gateway E2E for HITL tool approval (real app, scripted model)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langchain_core.outputs import ChatGeneration, ChatResult
|
||||
from starlette.testclient import TestClient
|
||||
from test_runtime_lifecycle_e2e import ( # noqa: F401
|
||||
_create_thread,
|
||||
_preserve_process_config_singletons,
|
||||
_register_user,
|
||||
_reset_process_singletons,
|
||||
_run_body,
|
||||
isolated_deer_flow_home,
|
||||
)
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
pytestmark = pytest.mark.no_auto_user
|
||||
|
||||
_MODEL_PATCH = "deerflow.agents.lead_agent.agent.create_chat_model"
|
||||
|
||||
|
||||
def _enable_approval():
|
||||
from deerflow.config.tool_config import ToolConfig
|
||||
|
||||
cfg = get_app_config()
|
||||
cfg.hitl_approval.enabled = True
|
||||
cfg.hitl_approval.tools = ["write_file"]
|
||||
if not any(tool.name == "write_file" for tool in cfg.tools):
|
||||
cfg.tools.append(ToolConfig(name="write_file", use="deerflow.sandbox.tools:write_file_tool", group="file:write"))
|
||||
|
||||
|
||||
class _StateAwareModel(FakeMessagesListChatModel):
|
||||
call_count: int = 1
|
||||
|
||||
def bind_tools(self, tools: Any, **kwargs: Any) -> Any:
|
||||
return self
|
||||
|
||||
def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult:
|
||||
if any(isinstance(m, ToolMessage) for m in messages):
|
||||
message = AIMessage(content="final answer")
|
||||
else:
|
||||
call_count = self.call_count
|
||||
message = AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "write_file",
|
||||
"args": {"path": f"/mnt/user-data/workspace/hitl-test-{index}.txt", "content": "ok"},
|
||||
"id": f"call_hitl_{index}",
|
||||
}
|
||||
for index in range(1, call_count + 1)
|
||||
],
|
||||
)
|
||||
return ChatResult(generations=[ChatGeneration(message=message)])
|
||||
|
||||
|
||||
class _TwoStepStateAwareModel(_StateAwareModel):
|
||||
reuse_tool_call_id: bool = False
|
||||
|
||||
def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult:
|
||||
tool_message_count = sum(isinstance(message, ToolMessage) for message in messages)
|
||||
if tool_message_count >= 2:
|
||||
message = AIMessage(content="final answer")
|
||||
else:
|
||||
message = AIMessage(
|
||||
content="",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "write_file",
|
||||
"args": {"path": f"/mnt/user-data/workspace/hitl-test-{tool_message_count + 1}.txt", "content": "ok"},
|
||||
"id": "call_hitl_1" if self.reuse_tool_call_id else f"call_hitl_{tool_message_count + 1}",
|
||||
}
|
||||
],
|
||||
)
|
||||
return ChatResult(generations=[ChatGeneration(message=message)])
|
||||
|
||||
|
||||
def _fake_two_step_model(*args, **kwargs):
|
||||
return _TwoStepStateAwareModel(responses=[])
|
||||
|
||||
|
||||
def _fake_reused_call_id_model(*args, **kwargs):
|
||||
return _TwoStepStateAwareModel(responses=[], reuse_tool_call_id=True)
|
||||
|
||||
|
||||
def _fake_model(*args, **kwargs):
|
||||
return _StateAwareModel(responses=[])
|
||||
|
||||
|
||||
def _fake_two_target_model(*args, **kwargs):
|
||||
return _StateAwareModel(responses=[], call_count=2)
|
||||
|
||||
|
||||
@pytest.fixture(params=["full", "delta"])
|
||||
def isolated_app(request, isolated_deer_flow_home, monkeypatch): # noqa: F811
|
||||
if request.param == "delta":
|
||||
config_path = isolated_deer_flow_home.parent / "config.yaml"
|
||||
config_path.write_text(
|
||||
config_path.read_text(encoding="utf-8").replace(
|
||||
"database:\n backend: sqlite\n",
|
||||
"database:\n backend: sqlite\n checkpoint_channel_mode: delta\n",
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
_preserve_process_config_singletons(monkeypatch)
|
||||
_reset_process_singletons(monkeypatch)
|
||||
from deerflow.config import app_config as app_config_module
|
||||
|
||||
cfg = app_config_module.get_app_config()
|
||||
cfg.database.sqlite_dir = str(isolated_deer_flow_home / "db")
|
||||
from app.gateway.app import create_app
|
||||
|
||||
return create_app()
|
||||
|
||||
|
||||
def _run(client, csrf, tid, body):
|
||||
return client.post(f"/api/threads/{tid}/runs", json=body, headers={"X-CSRF-Token": csrf})
|
||||
|
||||
|
||||
def _wait_for_status(client, tid, wanted, timeout=20.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
state = {}
|
||||
while time.monotonic() < deadline:
|
||||
state = client.get(f"/api/threads/{tid}").json()
|
||||
if state.get("status") == wanted:
|
||||
return state
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f"thread did not reach {wanted}: {state}")
|
||||
|
||||
|
||||
def _wait_for_approval(client, tid, timeout=20.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
state = {}
|
||||
while time.monotonic() < deadline:
|
||||
state = client.get(f"/api/threads/{tid}").json()
|
||||
payloads = [item for group in state.get("interrupts", {}).values() for item in group]
|
||||
if payloads and payloads[0].get("value", {}).get("kind") == "hitl_approval":
|
||||
return state
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f"thread did not expose a HITL approval: {state}")
|
||||
|
||||
|
||||
def _wait_run_status(client, tid, rid, wanted, timeout=20.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
run = {}
|
||||
while time.monotonic() < deadline and run.get("status") != wanted:
|
||||
run = client.get(f"/api/threads/{tid}/runs/{rid}").json()
|
||||
time.sleep(0.2)
|
||||
return run
|
||||
|
||||
|
||||
def _wait_run_terminal(client, tid, rid, timeout=20.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
run = {}
|
||||
while time.monotonic() < deadline:
|
||||
run = client.get(f"/api/threads/{tid}/runs/{rid}").json()
|
||||
if run.get("status") not in ("running", "pending", "queued"):
|
||||
return run
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f"run did not finish: {run}")
|
||||
|
||||
|
||||
def _approval_request_id(client, tid, tool_call_id):
|
||||
state = _wait_for_approval(client, tid)
|
||||
payload = next(item for group in state["interrupts"].values() for item in group)["value"]
|
||||
return next(action["request_id"] for action in payload["actions"] if action["tool_call_id"] == tool_call_id)
|
||||
|
||||
|
||||
def _resume(client, csrf, tid, option_id, value=None, *, tool_call_id="call_hitl_1", request_id=None):
|
||||
response = {
|
||||
"option_id": option_id,
|
||||
"tool_call_id": tool_call_id,
|
||||
"request_id": request_id or _approval_request_id(client, tid, tool_call_id),
|
||||
}
|
||||
if value is not None:
|
||||
response["value"] = value
|
||||
return _run(client, csrf, tid, {"command": {"resume": {"responses": [response]}}})
|
||||
|
||||
|
||||
def test_approval_pause_then_approve_executes(isolated_app):
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
rid = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write"}]})).json()["run_id"]
|
||||
state = _wait_for_approval(client, tid)
|
||||
deadline = time.monotonic() + 20.0
|
||||
run = {}
|
||||
while time.monotonic() < deadline and run.get("status") != "waiting_for_input":
|
||||
run = client.get(f"/api/threads/{tid}/runs/{rid}").json()
|
||||
time.sleep(0.2)
|
||||
assert run["status"] == "waiting_for_input"
|
||||
payloads = [i for group in state["interrupts"].values() for i in group]
|
||||
assert payloads and payloads[0]["value"]["kind"] == "hitl_approval"
|
||||
assert payloads[0]["value"]["pending_tool_call_ids"] == ["call_hitl_1"]
|
||||
|
||||
conflict = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "other"}]}))
|
||||
assert conflict.status_code == 409
|
||||
assert conflict.json()["detail"]["code"] == "approval_pending"
|
||||
|
||||
approved = _resume(client, csrf, tid, "approve")
|
||||
assert approved.status_code == 200, approved.text
|
||||
_ = _wait_run_terminal(client, tid, approved.json()["run_id"])
|
||||
state = client.get(f"/api/threads/{tid}").json()
|
||||
contents = [m.get("content") for m in state["values"]["messages"] if m.get("type") == "tool"]
|
||||
assert "OK" in contents
|
||||
assert state["status"] == "idle"
|
||||
|
||||
duplicate = _resume(client, csrf, tid, "approve", request_id=payloads[0]["value"]["request_id"])
|
||||
assert duplicate.status_code == 409
|
||||
assert duplicate.json()["detail"]["code"] == "no_pending_approval"
|
||||
|
||||
|
||||
def test_fresh_run_rechecks_pending_approval_at_durable_admission(isolated_app):
|
||||
"""A pause created after the early guard still blocks fresh admission."""
|
||||
_enable_approval()
|
||||
model_started = threading.Event()
|
||||
release_model = threading.Event()
|
||||
admission_started = threading.Event()
|
||||
release_admission = threading.Event()
|
||||
|
||||
class _GatedModel(_StateAwareModel):
|
||||
def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult:
|
||||
model_started.set()
|
||||
if not release_model.wait(20):
|
||||
raise TimeoutError("test did not release the model")
|
||||
return super()._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
|
||||
|
||||
def _gated_model(*args, **kwargs):
|
||||
return _GatedModel(responses=[])
|
||||
|
||||
from app.gateway import services
|
||||
|
||||
original_resolve = services.resolve_project_context
|
||||
|
||||
async def _gate_fresh_admission(*args, **kwargs):
|
||||
if model_started.is_set() and not admission_started.is_set():
|
||||
admission_started.set()
|
||||
if not await asyncio.to_thread(release_admission.wait, 20):
|
||||
raise TimeoutError("test did not release fresh admission")
|
||||
return await original_resolve(*args, **kwargs)
|
||||
|
||||
with (
|
||||
patch(_MODEL_PATCH, new=_gated_model),
|
||||
patch("app.gateway.services.resolve_project_context", new=_gate_fresh_admission),
|
||||
TestClient(isolated_app) as client,
|
||||
):
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
initial = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write"}]}))
|
||||
assert initial.status_code == 200, initial.text
|
||||
assert model_started.wait(20)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
fresh = pool.submit(_run, client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "replace"}]}))
|
||||
try:
|
||||
# The fresh request has passed its early check but not durable
|
||||
# admission. Only now let the other run expose its pause.
|
||||
assert admission_started.wait(20)
|
||||
release_model.set()
|
||||
_wait_for_approval(client, tid)
|
||||
release_admission.set()
|
||||
conflict = fresh.result(timeout=20)
|
||||
finally:
|
||||
release_model.set()
|
||||
release_admission.set()
|
||||
|
||||
assert conflict.status_code == 409, conflict.text
|
||||
assert conflict.json()["detail"]["code"] == "approval_pending"
|
||||
assert _wait_run_status(client, tid, initial.json()["run_id"], "waiting_for_input")["status"] == "waiting_for_input"
|
||||
|
||||
cleanup = _resume(client, csrf, tid, "reject")
|
||||
assert cleanup.status_code == 200, cleanup.text
|
||||
_wait_run_terminal(client, tid, cleanup.json()["run_id"])
|
||||
|
||||
|
||||
def test_approval_reject_returns_reason_without_execution(isolated_app):
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
_run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write"}]}))
|
||||
_wait_for_approval(client, tid)
|
||||
rejected = _resume(client, csrf, tid, "reject", value="do not write")
|
||||
assert rejected.status_code == 200, rejected.text
|
||||
_wait_run_terminal(client, tid, rejected.json()["run_id"])
|
||||
state = client.get(f"/api/threads/{tid}").json()
|
||||
contents = [m.get("content") for m in state["values"]["messages"] if m.get("type") == "tool"]
|
||||
assert any("do not write" in str(c) for c in contents)
|
||||
assert "OK" not in contents
|
||||
|
||||
|
||||
def test_invalid_resume_is_422_and_does_not_create_run(isolated_app):
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
rid = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write"}]})).json()["run_id"]
|
||||
_wait_run_status(client, tid, rid, "waiting_for_input")
|
||||
# Wrong id → 422 without admitting a run; a valid approval still works.
|
||||
bad = _run(client, csrf, tid, {"command": {"resume": {"responses": [{"option_id": "approve", "tool_call_id": "wrong", "request_id": "unused"}]}}})
|
||||
assert bad.status_code == 422, bad.text
|
||||
assert bad.json()["detail"]["code"] == "invalid_approval_response"
|
||||
assert {run["run_id"] for run in client.get(f"/api/threads/{tid}/runs").json()} == {rid}
|
||||
good = _resume(client, csrf, tid, "approve")
|
||||
assert good.status_code == 200, good.text
|
||||
|
||||
|
||||
def test_stale_request_id_cannot_approve_reused_tool_call(isolated_app):
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_reused_call_id_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
first_run_id = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write twice"}]})).json()["run_id"]
|
||||
first_pause = _wait_for_approval(client, tid)
|
||||
first_payload = next(item for group in first_pause["interrupts"].values() for item in group)["value"]
|
||||
first_request_id = first_payload["request_id"]
|
||||
_wait_run_status(client, tid, first_run_id, "waiting_for_input")
|
||||
|
||||
rejected = _resume(client, csrf, tid, "reject", value="reject the first arguments")
|
||||
assert rejected.status_code == 200, rejected.text
|
||||
second_run_id = rejected.json()["run_id"]
|
||||
_wait_run_status(client, tid, second_run_id, "waiting_for_input")
|
||||
second_pause = _wait_for_approval(client, tid)
|
||||
second_payload = next(item for group in second_pause["interrupts"].values() for item in group)["value"]
|
||||
assert second_payload["pending_tool_call_ids"] == first_payload["pending_tool_call_ids"] == ["call_hitl_1"]
|
||||
assert second_payload["request_id"] != first_request_id
|
||||
|
||||
stale = _run(
|
||||
client,
|
||||
csrf,
|
||||
tid,
|
||||
{"command": {"resume": {"responses": [{"option_id": "approve", "tool_call_id": "call_hitl_1", "request_id": first_request_id}]}}},
|
||||
)
|
||||
assert stale.status_code == 422, stale.text
|
||||
assert stale.json()["detail"]["code"] == "invalid_approval_response"
|
||||
assert {run["run_id"] for run in client.get(f"/api/threads/{tid}/runs").json()} == {first_run_id, second_run_id}
|
||||
state = client.get(f"/api/threads/{tid}").json()
|
||||
tool_messages = [message for message in state["values"]["messages"] if message.get("type") == "tool"]
|
||||
assert len(tool_messages) == 1
|
||||
assert "OK" not in [message.get("content") for message in tool_messages]
|
||||
|
||||
approved = _resume(client, csrf, tid, "approve")
|
||||
assert approved.status_code == 200, approved.text
|
||||
terminal = _wait_run_terminal(client, tid, approved.json()["run_id"])
|
||||
assert terminal["status"] == "success"
|
||||
state = client.get(f"/api/threads/{tid}").json()
|
||||
tool_messages = [message for message in state["values"]["messages"] if message.get("type") == "tool"]
|
||||
assert len(tool_messages) == 2
|
||||
assert sum(message.get("content") == "OK" for message in tool_messages) == 1
|
||||
|
||||
|
||||
def test_duplicate_resume_ids_are_422_without_creating_a_run(isolated_app):
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_two_target_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
rid = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write twice"}]})).json()["run_id"]
|
||||
_wait_run_status(client, tid, rid, "waiting_for_input")
|
||||
duplicate = _run(
|
||||
client,
|
||||
csrf,
|
||||
tid,
|
||||
{
|
||||
"command": {
|
||||
"resume": {
|
||||
"responses": [
|
||||
{"option_id": "reject", "tool_call_id": "call_hitl_1", "request_id": _approval_request_id(client, tid, "call_hitl_1")},
|
||||
{"option_id": "reject", "tool_call_id": "call_hitl_1", "request_id": _approval_request_id(client, tid, "call_hitl_1")},
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
assert duplicate.status_code == 422, duplicate.text
|
||||
assert duplicate.json()["detail"]["code"] == "invalid_approval_response"
|
||||
assert {run["run_id"] for run in client.get(f"/api/threads/{tid}/runs").json()} == {rid}
|
||||
|
||||
mixed = _run(
|
||||
client,
|
||||
csrf,
|
||||
tid,
|
||||
{
|
||||
"command": {
|
||||
"resume": {
|
||||
"responses": [
|
||||
{"option_id": "approve", "tool_call_id": "call_hitl_1", "request_id": _approval_request_id(client, tid, "call_hitl_1")},
|
||||
{"option_id": "reject", "tool_call_id": "call_hitl_2", "value": "second rejected", "request_id": _approval_request_id(client, tid, "call_hitl_2")},
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
assert mixed.status_code == 200, mixed.text
|
||||
terminal = _wait_run_terminal(client, tid, mixed.json()["run_id"])
|
||||
assert terminal["status"] == "success"
|
||||
state = client.get(f"/api/threads/{tid}").json()
|
||||
tool_messages = [message for message in state["values"]["messages"] if message.get("type") == "tool"]
|
||||
assert sum(message.get("content") == "OK" for message in tool_messages) == 1
|
||||
assert any("second rejected" in str(message.get("content")) for message in tool_messages)
|
||||
|
||||
|
||||
def test_disabled_resume_409_and_all_reject_escape(isolated_app):
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_two_step_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
_run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write"}]}))
|
||||
_wait_for_approval(client, tid)
|
||||
get_app_config().hitl_approval.enabled = False
|
||||
disabled = _resume(client, csrf, tid, "approve")
|
||||
assert disabled.status_code == 409
|
||||
assert disabled.json()["detail"]["code"] == "hitl_approval_disabled"
|
||||
escape = _resume(client, csrf, tid, "reject")
|
||||
assert escape.status_code == 200, escape.text
|
||||
terminal = _wait_run_terminal(client, tid, escape.json()["run_id"])
|
||||
assert terminal["status"] == "success"
|
||||
assert _wait_for_status(client, tid, "idle")["status"] == "idle"
|
||||
contents = [m.get("content") for m in client.get(f"/api/threads/{tid}").json()["values"]["messages"] if m.get("type") == "tool"]
|
||||
assert "OK" not in contents
|
||||
assert len(contents) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("current_tools", [["bash"], []], ids=["changed", "cleared"])
|
||||
def test_changed_tools_all_reject_escape_preserves_later_gate(isolated_app, current_tools):
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_two_step_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
_run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write"}]}))
|
||||
_wait_for_approval(client, tid)
|
||||
get_app_config().hitl_approval.tools = current_tools
|
||||
|
||||
escape = _resume(client, csrf, tid, "reject")
|
||||
assert escape.status_code == 200, escape.text
|
||||
second_run_id = escape.json()["run_id"]
|
||||
second_pause = _wait_run_status(client, tid, second_run_id, "waiting_for_input")
|
||||
assert second_pause["status"] == "waiting_for_input"
|
||||
state = _wait_for_approval(client, tid)
|
||||
payloads = [item for group in state["interrupts"].values() for item in group]
|
||||
assert payloads[0]["value"]["pending_tool_call_ids"] == ["call_hitl_2"]
|
||||
tool_messages = [message for message in state["values"]["messages"] if message.get("type") == "tool"]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0]["tool_call_id"] == "call_hitl_1"
|
||||
assert tool_messages[0]["status"] == "error"
|
||||
assert "OK" not in [message.get("content") for message in tool_messages]
|
||||
|
||||
second_escape = _resume(client, csrf, tid, "reject", tool_call_id="call_hitl_2")
|
||||
assert second_escape.status_code == 200, second_escape.text
|
||||
terminal = _wait_run_terminal(client, tid, second_escape.json()["run_id"])
|
||||
assert terminal["status"] == "success"
|
||||
assert _wait_for_status(client, tid, "idle")["status"] == "idle"
|
||||
tool_messages = [message for message in client.get(f"/api/threads/{tid}").json()["values"]["messages"] if message.get("type") == "tool"]
|
||||
assert len(tool_messages) == 2
|
||||
assert all(message.get("status") == "error" for message in tool_messages)
|
||||
assert "OK" not in [message.get("content") for message in tool_messages]
|
||||
|
||||
|
||||
def test_stale_all_reject_cannot_dismiss_newer_pause(isolated_app):
|
||||
"""Checkpoint-only recovery is limited to the pause that was validated."""
|
||||
_enable_approval()
|
||||
with patch(_MODEL_PATCH, new=_fake_two_step_model), TestClient(isolated_app) as client:
|
||||
csrf = _register_user(client)
|
||||
tid = _create_thread(client, csrf)
|
||||
initial = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "write twice"}]}))
|
||||
assert initial.status_code == 200, initial.text
|
||||
_wait_for_approval(client, tid)
|
||||
first_request_id = _approval_request_id(client, tid, "call_hitl_1")
|
||||
|
||||
reject_validated = threading.Event()
|
||||
release_reject = threading.Event()
|
||||
from app.gateway import services
|
||||
|
||||
original_resolve = services.resolve_project_context
|
||||
|
||||
async def _queue_reject_before_admission(*args, **kwargs):
|
||||
if not reject_validated.is_set():
|
||||
# resolve_project_context runs after resume validation but before
|
||||
# the per-thread admission lock, so P1 can still be approved.
|
||||
reject_validated.set()
|
||||
if not await asyncio.to_thread(release_reject.wait, 20):
|
||||
raise TimeoutError("test did not release stale rejection")
|
||||
return await original_resolve(*args, **kwargs)
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_project_context", new=_queue_reject_before_admission),
|
||||
ThreadPoolExecutor(max_workers=1) as pool,
|
||||
):
|
||||
stale_reject = pool.submit(_resume, client, csrf, tid, "reject", tool_call_id="call_hitl_1", request_id=first_request_id)
|
||||
try:
|
||||
assert reject_validated.wait(20)
|
||||
approved = _resume(client, csrf, tid, "approve", request_id=first_request_id)
|
||||
assert approved.status_code == 200, approved.text
|
||||
second_run_id = approved.json()["run_id"]
|
||||
assert _wait_run_status(client, tid, second_run_id, "waiting_for_input")["status"] == "waiting_for_input"
|
||||
second_request_id = _approval_request_id(client, tid, "call_hitl_2")
|
||||
assert second_request_id != first_request_id
|
||||
|
||||
release_reject.set()
|
||||
conflict = stale_reject.result(timeout=20)
|
||||
finally:
|
||||
release_reject.set()
|
||||
|
||||
assert conflict.status_code == 409, conflict.text
|
||||
assert conflict.json()["detail"]["code"] == "no_pending_approval"
|
||||
assert client.get(f"/api/threads/{tid}/runs/{second_run_id}").json()["status"] == "waiting_for_input"
|
||||
state = _wait_for_approval(client, tid)
|
||||
payloads = [item for group in state["interrupts"].values() for item in group]
|
||||
assert payloads[0]["value"]["pending_tool_call_ids"] == ["call_hitl_2"]
|
||||
|
||||
cleanup = _resume(client, csrf, tid, "reject", tool_call_id="call_hitl_2", request_id=second_request_id)
|
||||
assert cleanup.status_code == 200, cleanup.text
|
||||
_wait_run_terminal(client, tid, cleanup.json()["run_id"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_interrupt_resume_admission_survives_head_moving_state_update(monkeypatch):
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.graph import MessagesState, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
from app.gateway.hitl_approval import validate_resume_request
|
||||
|
||||
class EmptyRunStore:
|
||||
async def list_by_thread(self, thread_id, limit):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr("app.gateway.deps.get_run_store", lambda request: EmptyRunStore())
|
||||
|
||||
def answer(state):
|
||||
value = interrupt({"question": "Continue?"})
|
||||
return {"messages": [AIMessage(content=f"answer:{value}")]}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("answer", answer)
|
||||
builder.set_entry_point("answer")
|
||||
builder.set_finish_point("answer")
|
||||
saver = InMemorySaver()
|
||||
graph = builder.compile(checkpointer=saver)
|
||||
config = {"configurable": {"thread_id": "generic-head-move"}}
|
||||
await graph.ainvoke({"messages": [HumanMessage(content="go")]}, config)
|
||||
await graph.aupdate_state(config, {"messages": [AIMessage(content="head move")]}, as_node="answer")
|
||||
accessor = SimpleNamespace(aget=graph.aget_state, checkpointer=saver)
|
||||
assert not (await graph.aget_state(config)).next
|
||||
|
||||
body = SimpleNamespace(command={"resume": "yes"})
|
||||
await validate_resume_request(
|
||||
request=SimpleNamespace(),
|
||||
thread_id="generic-head-move",
|
||||
accessor=accessor,
|
||||
read_config=config,
|
||||
enabled=False,
|
||||
body=body,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mark_approval_answered_preserves_idempotency_fields(monkeypatch):
|
||||
from app.gateway.hitl_approval import mark_approval_answered
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
|
||||
identity = {"version": 1, "kind": "resume", "sha256": "a" * 64}
|
||||
store = MemoryRunStore()
|
||||
await store.put(
|
||||
"paused-run",
|
||||
thread_id="thread-1",
|
||||
status="waiting_for_input",
|
||||
metadata={"hitl_approval": {"kind": "hitl_approval", "request_id": "approval-1"}, "retained": True},
|
||||
idempotency_key="http-run:key",
|
||||
idempotency_request=identity,
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.deps.get_run_store", lambda request: store)
|
||||
|
||||
await mark_approval_answered(SimpleNamespace(), "thread-1")
|
||||
|
||||
answered = await store.get("paused-run")
|
||||
assert answered is not None
|
||||
assert answered["status"] == "success"
|
||||
assert answered["idempotency_key"] == "http-run:key"
|
||||
assert answered["idempotency_request"] == identity
|
||||
assert answered["metadata"] == {"retained": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_store_fallback_uses_original_langgraph_task_id(monkeypatch):
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.graph import MessagesState, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
from app.gateway.hitl_approval import find_pending_interrupts
|
||||
|
||||
payload = {
|
||||
"kind": "hitl_approval",
|
||||
"version": 1,
|
||||
"request_id": "approval-1",
|
||||
"actions": [],
|
||||
"allowed_decisions": ["approve", "reject"],
|
||||
"pending_tool_call_ids": ["call-1"],
|
||||
}
|
||||
|
||||
class WaitingRunStore:
|
||||
async def list_by_thread(self, thread_id, limit):
|
||||
return [{"run_id": "paused-run", "status": "waiting_for_input", "metadata": {"hitl_approval": payload}}]
|
||||
|
||||
monkeypatch.setattr("app.gateway.deps.get_run_store", lambda request: WaitingRunStore())
|
||||
|
||||
def approval(state):
|
||||
value = interrupt(payload)
|
||||
return {"messages": [AIMessage(content=f"answer:{value}")]}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("approval", approval)
|
||||
builder.set_entry_point("approval")
|
||||
builder.set_finish_point("approval")
|
||||
saver = InMemorySaver()
|
||||
graph = builder.compile(checkpointer=saver)
|
||||
config = {"configurable": {"thread_id": "approval-head-move"}}
|
||||
await graph.ainvoke({"messages": [HumanMessage(content="go")]}, config)
|
||||
paused = await graph.aget_state(config)
|
||||
paused_task = next(task for task in paused.tasks if task.interrupts)
|
||||
original_interrupt = paused_task.interrupts[0]
|
||||
await graph.aupdate_state(config, {"messages": [AIMessage(content="head move")]}, as_node="approval")
|
||||
accessor = SimpleNamespace(aget=graph.aget_state, checkpointer=saver)
|
||||
assert not (await graph.aget_state(config)).next
|
||||
|
||||
pending = await find_pending_interrupts(SimpleNamespace(), "approval-head-move", accessor=accessor, read_config=config)
|
||||
|
||||
assert set(pending) == {paused_task.id}
|
||||
assert pending[paused_task.id][0].value == payload
|
||||
assert pending[paused_task.id][0].id == original_interrupt.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_store_fallback_reconstructs_interrupt_without_checkpoint(monkeypatch):
|
||||
from app.gateway.hitl_approval import find_pending_interrupts
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
|
||||
payload = {
|
||||
"kind": "hitl_approval",
|
||||
"request_id": "approval-store-only",
|
||||
"pending_tool_call_ids": ["call-1"],
|
||||
}
|
||||
store = MemoryRunStore()
|
||||
await store.put(
|
||||
"paused-run",
|
||||
thread_id="thread-store-only",
|
||||
status="waiting_for_input",
|
||||
metadata={"hitl_approval": payload},
|
||||
)
|
||||
monkeypatch.setattr("app.gateway.deps.get_run_store", lambda request: store)
|
||||
|
||||
pending = await find_pending_interrupts(SimpleNamespace(), "thread-store-only")
|
||||
|
||||
assert set(pending) == {"paused-run"}
|
||||
assert len(pending["paused-run"]) == 1
|
||||
assert pending["paused-run"][0].value == payload
|
||||
assert pending["paused-run"][0].id == "approval-store-only"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_newest_resume_write_subtracts_answered_checkpoint_pause():
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.graph import MessagesState, StateGraph
|
||||
from langgraph.types import Command, interrupt
|
||||
|
||||
from app.gateway.hitl_approval import has_any_pending_interrupt
|
||||
|
||||
def answer(state):
|
||||
value = interrupt({"question": "Continue?"})
|
||||
return {"messages": [AIMessage(content=f"answer:{value}")]}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("answer", answer)
|
||||
builder.set_entry_point("answer")
|
||||
builder.set_finish_point("answer")
|
||||
saver = InMemorySaver()
|
||||
graph = builder.compile(checkpointer=saver)
|
||||
config = {"configurable": {"thread_id": "answered-pause"}}
|
||||
await graph.ainvoke({"messages": [HumanMessage(content="go")]}, config)
|
||||
accessor = SimpleNamespace(checkpointer=saver)
|
||||
|
||||
assert await has_any_pending_interrupt(accessor, config) is True
|
||||
await graph.ainvoke(Command(resume="yes"), config)
|
||||
assert not (await graph.aget_state(config)).next
|
||||
assert await has_any_pending_interrupt(accessor, config) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_interrupts_fall_back_to_sync_saver_history():
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.graph import MessagesState, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
from app.gateway.hitl_approval import _interrupt_from_checkpoint_history, _pending_interrupts_from_checkpoint_history, has_any_pending_interrupt
|
||||
|
||||
class SyncOnlySaver(InMemorySaver):
|
||||
def alist(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
payload = {"question": "Continue?"}
|
||||
|
||||
def answer(state):
|
||||
value = interrupt(payload)
|
||||
return {"messages": [AIMessage(content=f"answer:{value}")]}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("answer", answer)
|
||||
builder.set_entry_point("answer")
|
||||
builder.set_finish_point("answer")
|
||||
saver = SyncOnlySaver()
|
||||
graph = builder.compile(checkpointer=saver)
|
||||
config = {"configurable": {"thread_id": "sync-saver-pause"}}
|
||||
await graph.ainvoke({"messages": [HumanMessage(content="go")]}, config)
|
||||
|
||||
pending = await _pending_interrupts_from_checkpoint_history(saver, config)
|
||||
recovered = await _interrupt_from_checkpoint_history(SimpleNamespace(checkpointer=saver), config, payload)
|
||||
|
||||
assert len(pending) == 1
|
||||
assert next(iter(pending.values()))[0].value["question"] == "Continue?"
|
||||
assert recovered is not None
|
||||
recovered_task_id, recovered_interrupt = recovered
|
||||
assert recovered_task_id in pending
|
||||
assert recovered_interrupt.value == payload
|
||||
assert await has_any_pending_interrupt(SimpleNamespace(checkpointer=saver), config) is True
|
||||
435
backend/tests/test_hitl_approval_middleware.py
Normal file
435
backend/tests/test_hitl_approval_middleware.py
Normal file
@ -0,0 +1,435 @@
|
||||
"""Unit tests for HitlApprovalMiddleware (after_model pattern)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from _agent_e2e_helpers import build_single_tool_call_model
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langchain_core.outputs import ChatGeneration, ChatResult
|
||||
from langchain_core.tools import tool
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.types import Command
|
||||
|
||||
from deerflow.agents.middlewares.hitl_approval_middleware import (
|
||||
MAX_PAYLOAD_BYTES,
|
||||
MAX_REASON_CHARS,
|
||||
HitlApprovalMiddleware,
|
||||
summarize_args,
|
||||
)
|
||||
from deerflow.config.hitl_approval_config import HitlApprovalConfig
|
||||
|
||||
|
||||
@tool
|
||||
def write_file(path: str, content: str) -> str:
|
||||
"""Write a file."""
|
||||
return "OK"
|
||||
|
||||
|
||||
class _StateAwareToolModel(FakeMessagesListChatModel):
|
||||
calls: list[dict[str, Any]]
|
||||
|
||||
def bind_tools(self, tools: Any, **kwargs: Any) -> Any:
|
||||
return self
|
||||
|
||||
def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult:
|
||||
if any(isinstance(message, ToolMessage) for message in messages):
|
||||
message = AIMessage(content="final answer")
|
||||
else:
|
||||
message = AIMessage(content="", tool_calls=self.calls)
|
||||
return ChatResult(generations=[ChatGeneration(message=message)])
|
||||
|
||||
|
||||
def build_agent(middleware, *, tool_calls=None, tool_args=None, checkpointer=None):
|
||||
if tool_calls is not None:
|
||||
model = _StateAwareToolModel(responses=[], calls=tool_calls)
|
||||
else:
|
||||
model = build_single_tool_call_model(
|
||||
tool_name="write_file",
|
||||
tool_args=tool_args or {"path": "/tmp/a.txt", "content": "hello"},
|
||||
tool_call_id="call_0",
|
||||
)
|
||||
return create_agent(model, tools=[write_file], middleware=[middleware], checkpointer=checkpointer or InMemorySaver())
|
||||
|
||||
|
||||
def messages_of(out):
|
||||
return out["messages"]
|
||||
|
||||
|
||||
def run_pause(middleware, *, metadata=None, context=None, thread="t", tool_calls=None, tool_args=None, checkpointer=None):
|
||||
agent = build_agent(middleware, tool_calls=tool_calls, tool_args=tool_args, checkpointer=checkpointer)
|
||||
config = {"configurable": {"thread_id": thread}}
|
||||
if metadata is not None:
|
||||
config["metadata"] = metadata
|
||||
out = agent.invoke({"messages": [{"role": "user", "content": "write"}]}, config, context=context)
|
||||
return agent, config, out
|
||||
|
||||
|
||||
def approval_response(out, option_id, tool_call_id="call_0", **extra):
|
||||
payload = out["__interrupt__"][0].value
|
||||
action = next(action for action in payload["actions"] if action["tool_call_id"] == tool_call_id)
|
||||
return {"option_id": option_id, "tool_call_id": tool_call_id, "request_id": action["request_id"], **extra}
|
||||
|
||||
|
||||
def test_outside_tool_list_passes_through():
|
||||
middleware = HitlApprovalMiddleware(tool_names={"bash"})
|
||||
_, _, out = run_pause(middleware)
|
||||
assert "__interrupt__" not in out
|
||||
assert any(isinstance(m, ToolMessage) and m.content == "OK" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_approve_executes_tool():
|
||||
agent, config, out = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
interrupts = out["__interrupt__"]
|
||||
payload = interrupts[0].value
|
||||
assert payload["kind"] == "hitl_approval"
|
||||
assert payload["pending_tool_call_ids"] == ["call_0"]
|
||||
out = agent.invoke(Command(resume={"responses": [approval_response(out, "approve")]}), config)
|
||||
assert any(isinstance(m, ToolMessage) and m.content == "OK" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_stale_request_id_denies_all_and_never_executes():
|
||||
calls = [
|
||||
{"name": "write_file", "args": {"path": "/tmp/a.txt", "content": "a"}, "id": "call_0"},
|
||||
{"name": "write_file", "args": {"path": "/tmp/b.txt", "content": "b"}, "id": "call_1"},
|
||||
]
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}), tool_calls=calls)
|
||||
stale = approval_response(paused, "approve", tool_call_id="call_1")
|
||||
stale["request_id"] = "hitl_approval:stale-message:call_1"
|
||||
|
||||
out = agent.invoke(Command(resume={"responses": [approval_response(paused, "approve"), stale]}), config)
|
||||
|
||||
tool_messages = [message for message in messages_of(out) if isinstance(message, ToolMessage)]
|
||||
assert len(tool_messages) == 2
|
||||
assert {message.tool_call_id for message in tool_messages} == {"call_0", "call_1"}
|
||||
assert all(message.status == "error" for message in tool_messages)
|
||||
assert not any(message.content == "OK" for message in tool_messages)
|
||||
|
||||
|
||||
def test_reject_produces_named_error_tool_message_and_no_execution():
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
out = agent.invoke(
|
||||
Command(resume={"responses": [approval_response(paused, "reject", value="too risky")]}),
|
||||
config,
|
||||
)
|
||||
tool_messages = [m for m in messages_of(out) if isinstance(m, ToolMessage)]
|
||||
assert len(tool_messages) == 1
|
||||
message = tool_messages[0]
|
||||
assert message.name == "write_file"
|
||||
assert message.status == "error"
|
||||
assert "too risky" in message.content
|
||||
assert "The tool was not executed" in message.content
|
||||
assert not any(m.content == "OK" for m in tool_messages)
|
||||
|
||||
|
||||
def test_approve_after_tool_removal_denies_resumed_call():
|
||||
middleware = HitlApprovalMiddleware(tool_names={"write_file"})
|
||||
agent, config, paused = run_pause(middleware)
|
||||
middleware.tool_names.clear()
|
||||
|
||||
out = agent.invoke(
|
||||
Command(resume={"responses": [approval_response(paused, "approve")]}),
|
||||
config,
|
||||
context={"_hitl_resume_tool_names": {"write_file"}},
|
||||
)
|
||||
|
||||
tool_messages = [message for message in messages_of(out) if isinstance(message, ToolMessage)]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].status == "error"
|
||||
assert "approval configuration changed" in tool_messages[0].content
|
||||
assert not any(message.content == "OK" for message in tool_messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_removed_tool_approval_recovers_gate_after_head_moving_write():
|
||||
from deerflow.runtime.runs.worker import _pending_hitl_resume_tool_names
|
||||
|
||||
middleware = HitlApprovalMiddleware(tool_names={"write_file"})
|
||||
agent, config, paused = run_pause(middleware)
|
||||
agent.update_state(config, {"messages": [AIMessage(content="head move", id="head-move")]}, as_node="model")
|
||||
command = Command(resume={"responses": [approval_response(paused, "approve")]})
|
||||
|
||||
names = await _pending_hitl_resume_tool_names(agent.checkpointer, config, command)
|
||||
|
||||
middleware.tool_names.clear()
|
||||
assert names == {"write_file"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_reject_escape_recovers_gate_after_head_moving_write():
|
||||
from deerflow.runtime.runs.worker import _is_all_reject_hitl_resume, _pending_hitl_resume_tool_names
|
||||
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
agent.update_state(config, {"messages": [AIMessage(content="head move", id="head-move")]}, as_node="model")
|
||||
command = Command(resume={"responses": [approval_response(paused, "reject")]})
|
||||
|
||||
names = await _pending_hitl_resume_tool_names(agent.checkpointer, config, command)
|
||||
|
||||
assert _is_all_reject_hitl_resume(command)
|
||||
assert names == {"write_file"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_resume_names_fall_back_to_sync_saver_history():
|
||||
from deerflow.runtime.runs.worker import _pending_hitl_resume_tool_names
|
||||
|
||||
class SyncOnlySaver(InMemorySaver):
|
||||
async def alist(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
yield # pragma: no cover
|
||||
|
||||
async def aget_tuple(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
saver = SyncOnlySaver()
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}), checkpointer=saver)
|
||||
agent.update_state(config, {"messages": [AIMessage(content="head move", id="head-move")]}, as_node="model")
|
||||
command = Command(resume={"responses": [approval_response(paused, "approve")]})
|
||||
|
||||
assert await _pending_hitl_resume_tool_names(saver, config, command) == {"write_file"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_resume_names_tuple_only_saver_handles_empty_and_paused_heads():
|
||||
from deerflow.runtime.runs.worker import _pending_hitl_resume_tool_names
|
||||
|
||||
class TupleOnlySaver(InMemorySaver):
|
||||
async def alist(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
yield # pragma: no cover
|
||||
|
||||
def list(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
async def aget_tuple(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
saver = TupleOnlySaver()
|
||||
command = Command(resume={"responses": [{"option_id": "approve", "tool_call_id": "call_0"}]})
|
||||
config = {"configurable": {"thread_id": "t"}}
|
||||
|
||||
assert await _pending_hitl_resume_tool_names(saver, config, command) == set()
|
||||
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}), checkpointer=saver)
|
||||
command = Command(resume={"responses": [approval_response(paused, "approve")]})
|
||||
|
||||
assert await _pending_hitl_resume_tool_names(saver, config, command) == {"write_file"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_resume_names_newer_resume_wins_over_old_interrupt():
|
||||
from deerflow.runtime.runs.worker import _pending_hitl_resume_tool_names
|
||||
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
command = Command(resume={"responses": [approval_response(paused, "reject")]})
|
||||
agent.invoke(command, config)
|
||||
|
||||
assert await _pending_hitl_resume_tool_names(agent.checkpointer, config, command) == set()
|
||||
|
||||
|
||||
def test_request_id_stable_across_replay_and_contains_message_id():
|
||||
agent, config, out = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
first = out["__interrupt__"][0].value["request_id"]
|
||||
agent.invoke(Command(resume={"responses": [approval_response(out, "approve")]}), config)
|
||||
# The payload is rebuilt identically on replay; id contains the AI message id.
|
||||
ai_id = next(m.id for m in messages_of(out) if isinstance(m, AIMessage))
|
||||
assert first == f"hitl_approval:{ai_id}:call_0"
|
||||
|
||||
|
||||
def test_non_interactive_denies_without_interrupt():
|
||||
_, _, out = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}), context={"non_interactive": True})
|
||||
assert "__interrupt__" not in out
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages_of(out))
|
||||
assert not any(isinstance(m, ToolMessage) and m.content == "OK" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_im_origin_allow_executes_without_pause():
|
||||
_, _, out = run_pause(
|
||||
HitlApprovalMiddleware(tool_names={"write_file"}, non_interactive="allow"),
|
||||
metadata={"deerflow_origin": {"kind": "im_channel"}},
|
||||
)
|
||||
assert "__interrupt__" not in out
|
||||
assert any(isinstance(m, ToolMessage) and m.content == "OK" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_im_origin_deny_rejects_without_pause():
|
||||
_, _, out = run_pause(
|
||||
HitlApprovalMiddleware(tool_names={"write_file"}),
|
||||
metadata={"deerflow_origin": {"kind": "im_channel"}},
|
||||
)
|
||||
assert "__interrupt__" not in out
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_subagent_fails_closed_even_with_allow():
|
||||
_, _, out = run_pause(
|
||||
HitlApprovalMiddleware(tool_names={"write_file"}, non_interactive="allow"),
|
||||
context={"is_subagent": True},
|
||||
)
|
||||
assert "__interrupt__" not in out
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_supports_resume_false_fails_closed():
|
||||
_, _, out = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}, supports_resume=False))
|
||||
assert "__interrupt__" not in out
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_response_count_mismatch_denies_all_and_can_resume_again():
|
||||
agent, config, _ = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
out = agent.invoke(Command(resume={"responses": []}), config)
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages_of(out))
|
||||
assert not any(isinstance(m, ToolMessage) and m.content == "OK" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_unknown_tool_call_id_denies_all():
|
||||
agent, config, _ = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
out = agent.invoke(Command(resume={"responses": [{"option_id": "approve", "tool_call_id": "nope", "request_id": "hitl_approval:unknown:nope"}]}), config)
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages_of(out))
|
||||
assert not any(isinstance(m, ToolMessage) and m.content == "OK" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_unknown_option_rejects_item():
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
out = agent.invoke(Command(resume={"responses": [approval_response(paused, "edit")]}), config)
|
||||
assert any(isinstance(m, ToolMessage) and m.status == "error" for m in messages_of(out))
|
||||
assert not any(isinstance(m, ToolMessage) and m.content == "OK" for m in messages_of(out))
|
||||
|
||||
|
||||
def test_duplicate_resume_ids_deny_all_targets():
|
||||
calls = [
|
||||
{"name": "write_file", "args": {"path": "/tmp/a.txt", "content": "a"}, "id": "call_0"},
|
||||
{"name": "write_file", "args": {"path": "/tmp/b.txt", "content": "b"}, "id": "call_1"},
|
||||
]
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}), tool_calls=calls)
|
||||
out = agent.invoke(
|
||||
Command(
|
||||
resume={
|
||||
"responses": [
|
||||
approval_response(paused, "approve"),
|
||||
approval_response(paused, "reject"),
|
||||
]
|
||||
}
|
||||
),
|
||||
config,
|
||||
)
|
||||
tool_messages = [message for message in messages_of(out) if isinstance(message, ToolMessage)]
|
||||
assert len(tool_messages) == 2
|
||||
assert {message.tool_call_id for message in tool_messages} == {"call_0", "call_1"}
|
||||
assert all(message.status == "error" for message in tool_messages)
|
||||
assert not any(message.content == "OK" for message in tool_messages)
|
||||
|
||||
|
||||
def test_mixed_approve_and_reject_requires_and_covers_each_target_once():
|
||||
calls = [
|
||||
{"name": "write_file", "args": {"path": "/tmp/a.txt", "content": "a"}, "id": "call_0"},
|
||||
{"name": "write_file", "args": {"path": "/tmp/b.txt", "content": "b"}, "id": "call_1"},
|
||||
]
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}), tool_calls=calls)
|
||||
out = agent.invoke(
|
||||
Command(
|
||||
resume={
|
||||
"responses": [
|
||||
approval_response(paused, "approve"),
|
||||
approval_response(paused, "reject", tool_call_id="call_1", value="not this one"),
|
||||
]
|
||||
}
|
||||
),
|
||||
config,
|
||||
)
|
||||
tool_messages = [message for message in messages_of(out) if isinstance(message, ToolMessage)]
|
||||
assert len(tool_messages) == 2
|
||||
by_id = {message.tool_call_id: message for message in tool_messages}
|
||||
assert by_id["call_0"].content == "OK"
|
||||
assert by_id["call_1"].status == "error"
|
||||
assert "not this one" in by_id["call_1"].content
|
||||
|
||||
|
||||
def test_non_string_rejection_reason_is_rendered_as_bounded_text():
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
out = agent.invoke(
|
||||
Command(resume={"responses": [approval_response(paused, "reject", value={"why": "unsafe"})]}),
|
||||
config,
|
||||
)
|
||||
tool_messages = [message for message in messages_of(out) if isinstance(message, ToolMessage)]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].status == "error"
|
||||
assert "unsafe" in tool_messages[0].content
|
||||
|
||||
|
||||
def test_rejection_reason_is_truncated():
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
out = agent.invoke(
|
||||
Command(resume={"responses": [approval_response(paused, "reject", value="r" * (MAX_REASON_CHARS + 100))]}),
|
||||
config,
|
||||
)
|
||||
message = next(message for message in messages_of(out) if isinstance(message, ToolMessage))
|
||||
assert message.content.endswith("r" * MAX_REASON_CHARS)
|
||||
assert "r" * (MAX_REASON_CHARS + 1) not in message.content
|
||||
|
||||
|
||||
def test_unknown_option_reason_is_bounded():
|
||||
agent, config, paused = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}))
|
||||
out = agent.invoke(
|
||||
Command(resume={"responses": [approval_response(paused, "x" * (MAX_REASON_CHARS * 2))]}),
|
||||
config,
|
||||
)
|
||||
message = next(message for message in messages_of(out) if isinstance(message, ToolMessage))
|
||||
prefix = "The tool was not executed. Do not retry unless the user explicitly asks. User reason: "
|
||||
assert message.status == "error"
|
||||
assert len(message.content) <= len(prefix) + MAX_REASON_CHARS
|
||||
|
||||
|
||||
def _approval_calls(count: int) -> list[dict[str, Any]]:
|
||||
return [{"name": "write_file", "args": {"path": f"/tmp/{index}.txt", "content": "x"}, "id": f"call_{index}"} for index in range(count)]
|
||||
|
||||
|
||||
def test_more_than_configured_max_actions_denies_every_call_without_pausing():
|
||||
_, _, out = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}, max_actions=2), tool_calls=_approval_calls(3))
|
||||
tool_messages = [message for message in messages_of(out) if isinstance(message, ToolMessage)]
|
||||
assert "__interrupt__" not in out
|
||||
assert len(tool_messages) == 3
|
||||
assert all(message.status == "error" for message in tool_messages)
|
||||
assert all("3 > 2" in message.content for message in tool_messages)
|
||||
|
||||
|
||||
def test_max_actions_boundary_still_pauses_for_approval():
|
||||
_, _, out = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}, max_actions=2), tool_calls=_approval_calls(2))
|
||||
assert len(out["__interrupt__"][0].value["actions"]) == 2
|
||||
|
||||
|
||||
def test_zero_max_actions_disables_count_limit():
|
||||
_, _, out = run_pause(HitlApprovalMiddleware(tool_names={"write_file"}, max_actions=0), tool_calls=_approval_calls(33))
|
||||
assert len(out["__interrupt__"][0].value["actions"]) == 33
|
||||
|
||||
|
||||
def test_max_actions_config_rejects_negative_values():
|
||||
assert HitlApprovalConfig(max_actions=0).max_actions == 0
|
||||
with pytest.raises(ValueError, match="max_actions"):
|
||||
HitlApprovalConfig(max_actions=-1)
|
||||
|
||||
|
||||
def test_oversized_approval_payload_denies_without_pausing():
|
||||
_, _, out = run_pause(
|
||||
HitlApprovalMiddleware(tool_names={"write_file"}),
|
||||
tool_args={"path": "/tmp/" + "x" * MAX_PAYLOAD_BYTES, "content": "hello"},
|
||||
)
|
||||
tool_messages = [message for message in messages_of(out) if isinstance(message, ToolMessage)]
|
||||
assert "__interrupt__" not in out
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].status == "error"
|
||||
|
||||
|
||||
def test_summarize_args_preserves_high_semantic_fields():
|
||||
long_content = "x" * 5000
|
||||
summary = summarize_args({"path": "/tmp/a.txt", "content": long_content, "old_str": "a", "new_str": "b"})
|
||||
assert summary["path"] == "/tmp/a.txt"
|
||||
assert summary["old_str"] == "a"
|
||||
assert summary["new_str"] == "b"
|
||||
assert "chars omitted" in summary["content"]
|
||||
assert summary["content"].startswith("xxxx")
|
||||
assert summary["content"].endswith("xxxx")
|
||||
@ -9,7 +9,7 @@ from unittest.mock import ANY, AsyncMock, MagicMock, create_autospec
|
||||
|
||||
import pytest
|
||||
from langchain.agents.middleware import AgentMiddleware
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
|
||||
from deerflow.agents.lead_agent import agent as lead_agent_module
|
||||
from deerflow.agents.middlewares import summarization_middleware as summarization_middleware_module
|
||||
@ -292,6 +292,27 @@ def test_make_lead_agent_scopes_bootstrap_middlewares_to_custom_agent(monkeypatc
|
||||
assert len(prompt_calls) == 1
|
||||
|
||||
|
||||
def test_bootstrap_hitl_tools_fail_closed_without_resume_support(model_control_assembly):
|
||||
from deerflow.agents.middlewares.hitl_approval_middleware import HitlApprovalMiddleware
|
||||
|
||||
app_config, _strict, _fallback, _model_factory = model_control_assembly
|
||||
app_config.hitl_approval.enabled = True
|
||||
app_config.hitl_approval.tools = ["setup_agent"]
|
||||
app_config.hitl_approval.max_actions = 7
|
||||
|
||||
assembly = lead_agent_module._make_lead_agent({"configurable": {"is_bootstrap": True}}, app_config=app_config)
|
||||
hitl = next(middleware for middleware in assembly["middleware"] if isinstance(middleware, HitlApprovalMiddleware))
|
||||
assert hitl.supports_resume is False
|
||||
assert hitl.max_actions == 7
|
||||
|
||||
message = AIMessage(content="", tool_calls=[{"id": "call-setup", "name": "setup_agent", "args": {}}])
|
||||
result = hitl.after_model({"messages": [message]}, SimpleNamespace(context={"is_bootstrap": True}))
|
||||
|
||||
assert result is not None
|
||||
assert "__interrupt__" not in result
|
||||
assert result["messages"][0].status == "error"
|
||||
|
||||
|
||||
def test_make_lead_agent_attaches_tracing_callbacks_at_graph_root(monkeypatch):
|
||||
"""Regression guard: tracing handlers must be appended to
|
||||
``config["callbacks"]`` (graph invocation root), and every in-graph
|
||||
|
||||
@ -532,6 +532,7 @@ class TestModeGating:
|
||||
memory=MemoryConfig(enabled=True, mode="tool"),
|
||||
skills=SimpleNamespace(deferred_discovery=False, container_path="/tmp/skills"),
|
||||
tool_search=SimpleNamespace(enabled=False, auto_promote_top_k=0),
|
||||
hitl_approval=SimpleNamespace(enabled=False),
|
||||
database=SimpleNamespace(checkpoint_channel_mode="full"),
|
||||
authorization=AuthorizationConfig(enabled=False),
|
||||
)
|
||||
@ -568,6 +569,7 @@ class TestModeGating:
|
||||
skills=SimpleNamespace(deferred_discovery=False, container_path="/tmp/skills"),
|
||||
tool_search=SimpleNamespace(enabled=False, auto_promote_top_k=0),
|
||||
database=SimpleNamespace(checkpoint_channel_mode="full"),
|
||||
hitl_approval=SimpleNamespace(enabled=False),
|
||||
authorization=AuthorizationConfig(enabled=False),
|
||||
)
|
||||
|
||||
|
||||
@ -580,7 +580,7 @@ def test_known_gaps_do_not_reclassify_current_events_as_missing():
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("status", ["success", "error", "interrupted", "timeout"])
|
||||
@pytest.mark.parametrize("status", ["success", "error", "interrupted", "timeout", "waiting_for_input"])
|
||||
async def test_authoritative_terminal_producer_matches_contract(status):
|
||||
from deerflow.runtime.events.terminal import confirm_terminal_event
|
||||
|
||||
|
||||
@ -3,10 +3,14 @@
|
||||
Uses a temp SQLite DB to test ORM-backed CRUD operations.
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.dialects import postgresql
|
||||
from sqlalchemy.schema import DropSchema
|
||||
|
||||
from deerflow.persistence.run import RunRepository
|
||||
from deerflow.runtime import CancelOutcome, RunManager, RunStatus, ThreadOperationKind
|
||||
@ -243,6 +247,107 @@ class TestRunRepository:
|
||||
assert row["status"] == "running"
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_answer_approval_is_single_winner_and_restores_exact_pause(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
original_metadata = {"hitl_approval": {"request_id": "approval-1"}, "retained": True}
|
||||
answered_metadata = {"retained": True}
|
||||
answered_updated_at = "2026-01-02T03:04:05.123456+00:00"
|
||||
await repo.put(
|
||||
"paused-run",
|
||||
thread_id="t1",
|
||||
status="waiting_for_input",
|
||||
metadata=original_metadata,
|
||||
)
|
||||
|
||||
assert await repo.answer_approval("paused-run", metadata=answered_metadata, answered_updated_at=answered_updated_at, expected_request_id="approval-0") is False
|
||||
assert await repo.answer_approval("paused-run", metadata=answered_metadata, answered_updated_at=answered_updated_at, expected_request_id="approval-1") is True
|
||||
assert await repo.answer_approval("paused-run", metadata=answered_metadata, answered_updated_at=answered_updated_at, expected_request_id="approval-1") is False
|
||||
row = await repo.get("paused-run")
|
||||
assert row["status"] == "success"
|
||||
assert row["metadata"] == answered_metadata
|
||||
assert row["updated_at"] == answered_updated_at
|
||||
|
||||
assert await repo.restore_approval("paused-run", metadata=original_metadata, expected_updated_at="2026-01-02T03:04:05.123455+00:00") is False
|
||||
assert await repo.restore_approval("paused-run", metadata=original_metadata, expected_updated_at=answered_updated_at) is True
|
||||
assert await repo.restore_approval("paused-run", metadata=original_metadata, expected_updated_at=answered_updated_at) is False
|
||||
restored = await repo.get("paused-run")
|
||||
assert restored["status"] == "waiting_for_input"
|
||||
assert restored["metadata"] == original_metadata
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_approval_cas_compiles_without_postgres_json_equality(self):
|
||||
captured = []
|
||||
|
||||
class FakeResult:
|
||||
rowcount = 1
|
||||
|
||||
class FakeSession:
|
||||
async def execute(self, stmt):
|
||||
captured.append(stmt)
|
||||
return FakeResult()
|
||||
|
||||
async def commit(self):
|
||||
return None
|
||||
|
||||
class FakeSessionContext:
|
||||
async def __aenter__(self):
|
||||
return FakeSession()
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
repo = RunRepository(lambda: FakeSessionContext())
|
||||
repo._next_change_seq = AsyncMock(return_value=1)
|
||||
await repo.answer_approval(
|
||||
"paused-run",
|
||||
metadata={"retained": True},
|
||||
answered_updated_at="2026-01-02T03:04:05.123456+00:00",
|
||||
expected_request_id="approval-1",
|
||||
)
|
||||
await repo.restore_approval(
|
||||
"paused-run",
|
||||
metadata={"hitl_approval": {"request_id": "approval-1"}},
|
||||
expected_updated_at="2026-01-02T03:04:05.123456+00:00",
|
||||
)
|
||||
|
||||
answer_where = str(captured[0].compile(dialect=postgresql.dialect())).split(" WHERE ", 1)[1]
|
||||
restore_where = str(captured[1].compile(dialect=postgresql.dialect())).split(" WHERE ", 1)[1]
|
||||
assert "metadata_json" in answer_where
|
||||
assert "->>" in answer_where
|
||||
assert "metadata_json =" not in answer_where
|
||||
assert "updated_at =" in restore_where
|
||||
assert "metadata_json =" not in restore_where
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_answer_and_restore_approval_on_postgres(self):
|
||||
uri = os.environ.get("TEST_POSTGRES_URI") or os.environ.get("DEERFLOW_TEST_POSTGRES_URL")
|
||||
if not uri:
|
||||
pytest.skip("requires TEST_POSTGRES_URI or DEERFLOW_TEST_POSTGRES_URL")
|
||||
|
||||
from support.postgres import asyncpg_test_url
|
||||
|
||||
from deerflow.config.database_config import DatabaseConfig
|
||||
from deerflow.persistence.engine import get_engine, get_session_factory, init_engine_from_config
|
||||
|
||||
schema = f"approval_cas_{uuid4().hex}"
|
||||
config = DatabaseConfig(backend="postgres", postgres_url=asyncpg_test_url(uri), postgres_schema=schema)
|
||||
await init_engine_from_config(config)
|
||||
try:
|
||||
repo = RunRepository(get_session_factory())
|
||||
original_metadata = {"hitl_approval": {"request_id": "approval-1"}, "retained": True}
|
||||
answered_updated_at = "2026-01-02T03:04:05.123456+00:00"
|
||||
await repo.put("paused-run", thread_id="t1", status="waiting_for_input", metadata=original_metadata)
|
||||
assert await repo.answer_approval("paused-run", metadata={"retained": True}, answered_updated_at=answered_updated_at, expected_request_id="approval-0") is False
|
||||
assert await repo.answer_approval("paused-run", metadata={"retained": True}, answered_updated_at=answered_updated_at, expected_request_id="approval-1") is True
|
||||
assert await repo.restore_approval("paused-run", metadata=original_metadata, expected_updated_at="2026-01-02T03:04:05.123455+00:00") is False
|
||||
assert await repo.restore_approval("paused-run", metadata=original_metadata, expected_updated_at=answered_updated_at) is True
|
||||
finally:
|
||||
async with get_engine().begin() as connection:
|
||||
await connection.execute(DropSchema(schema, cascade=True))
|
||||
await _cleanup()
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_update_status_returns_false_for_missing_row(self, tmp_path):
|
||||
repo = await _make_repo(tmp_path)
|
||||
|
||||
@ -6,11 +6,13 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from langgraph.types import Command
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langgraph.graph import END, START, StateGraph
|
||||
from langgraph.types import Command, Interrupt, interrupt
|
||||
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.paths import Paths
|
||||
from deerflow.config.run_ownership_config import RunOwnershipConfig
|
||||
from deerflow.config.sandbox_config import SandboxConfig
|
||||
from deerflow.config.tool_output_config import ToolOutputConfig
|
||||
from deerflow.runtime.events.store.memory import MemoryRunEventStore
|
||||
@ -965,3 +967,308 @@ async def test_delivery_event_emitted_when_cancelled_waiting_for_prior_finalizat
|
||||
fetched = await run_manager.get(record.run_id)
|
||||
assert fetched.status == RunStatus.interrupted
|
||||
run_manager.update_run_completion.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pending_snapshot_failure_cannot_complete_run_successfully():
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
run_store = MemoryRunStore()
|
||||
run_manager = RunManager(store=run_store)
|
||||
record = await run_manager.create("thread-snapshot-failure")
|
||||
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
||||
app_config.hitl_approval.enabled = True
|
||||
|
||||
class SnapshotFailureAgent:
|
||||
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
||||
yield {"messages": []}
|
||||
|
||||
async def aget_state(self, _config):
|
||||
raise RuntimeError("checkpoint snapshot unavailable")
|
||||
|
||||
await run_agent(
|
||||
_make_bridge(),
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(checkpointer=InMemorySaver(), app_config=app_config),
|
||||
agent_factory=lambda *, config: SnapshotFailureAgent(),
|
||||
graph_input={},
|
||||
config={},
|
||||
)
|
||||
|
||||
assert record.status == RunStatus.error
|
||||
assert record.error == "checkpoint snapshot unavailable"
|
||||
assert (await run_store.get(record.run_id))["status"] == "error"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_paused_metadata_is_not_written_after_ownership_loss():
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
run_store = MemoryRunStore()
|
||||
run_manager = RunManager(
|
||||
store=run_store,
|
||||
worker_id="owner",
|
||||
run_ownership_config=RunOwnershipConfig(heartbeat_enabled=True, lease_seconds=30, grace_seconds=0),
|
||||
)
|
||||
record = await run_manager.create_or_reject("thread-paused-ownership")
|
||||
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
||||
app_config.hitl_approval.enabled = True
|
||||
original_finalize = run_store.finalize_if_not_cancelled
|
||||
|
||||
async def finalize_then_lose_ownership(*args, **kwargs):
|
||||
result = await original_finalize(*args, **kwargs)
|
||||
run_store._runs[record.run_id].update(status="error", error="peer takeover", owner_worker_id="peer", metadata={"owner": "peer"})
|
||||
await run_manager._mark_ownership_lost(record, reason="Test peer takeover", require_active=False)
|
||||
return result
|
||||
|
||||
run_store.finalize_if_not_cancelled = finalize_then_lose_ownership
|
||||
run_store.put = AsyncMock(wraps=run_store.put)
|
||||
run_manager.persist_metadata = AsyncMock(wraps=run_manager.persist_metadata)
|
||||
payload = {"kind": "hitl_approval", "request_id": "approval-1"}
|
||||
|
||||
class PausedAgent:
|
||||
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
||||
yield {"messages": []}
|
||||
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(
|
||||
values={"messages": []},
|
||||
config={},
|
||||
metadata={},
|
||||
next=("agent",),
|
||||
tasks=(SimpleNamespace(interrupts=(SimpleNamespace(value=payload),)),),
|
||||
)
|
||||
|
||||
await run_agent(
|
||||
_make_bridge(),
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(checkpointer=InMemorySaver(), app_config=app_config),
|
||||
agent_factory=lambda *, config: PausedAgent(),
|
||||
graph_input={},
|
||||
config={},
|
||||
)
|
||||
|
||||
run_manager.persist_metadata.assert_not_awaited()
|
||||
run_store.put.assert_not_awaited()
|
||||
row = await run_store.get(record.run_id)
|
||||
assert row["status"] == "error"
|
||||
assert row["error"] == "peer takeover"
|
||||
assert row["metadata"] == {"owner": "peer"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("stream_modes", [None, ["messages-tuple"]], ids=["values", "messages-only"])
|
||||
async def test_generic_interrupt_is_waiting_even_when_hitl_is_disabled(stream_modes):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
run_store = MemoryRunStore()
|
||||
run_manager = RunManager(store=run_store)
|
||||
record = await run_manager.create("thread-generic-pause" if stream_modes is None else "thread-generic-pause-messages")
|
||||
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
||||
|
||||
class GenericPausedAgent:
|
||||
async def astream(self, graph_input, config=None, stream_mode=None, subgraphs=False):
|
||||
yield {"messages": [], "__interrupt__": (Interrupt(value={"question": "Continue?"}, id="generic-pause"),)}
|
||||
|
||||
async def aget_state(self, _config):
|
||||
return SimpleNamespace(
|
||||
values={"messages": []},
|
||||
config={},
|
||||
metadata={},
|
||||
next=("answer",),
|
||||
tasks=(SimpleNamespace(interrupts=(SimpleNamespace(value={"question": "Continue?"}),)),),
|
||||
)
|
||||
|
||||
await run_agent(
|
||||
_make_bridge(),
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(checkpointer=InMemorySaver(), app_config=app_config),
|
||||
agent_factory=lambda *, config: GenericPausedAgent(),
|
||||
graph_input={},
|
||||
config={},
|
||||
stream_modes=stream_modes,
|
||||
)
|
||||
|
||||
row = await run_store.get(record.run_id)
|
||||
assert row["status"] == "waiting_for_input"
|
||||
assert "hitl_approval" not in row["metadata"]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
async def test_approval_pause_bypasses_terminal_goal_delivery_and_batch_completion(tmp_path, monkeypatch, mode):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.subagent_batches.continuation import BatchContinuations
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.extensions.registry import ExtensionRegistry
|
||||
from deerflow.runtime.goal import build_goal_state, read_thread_goal, write_thread_goal
|
||||
|
||||
paths = Paths(base_dir=tmp_path)
|
||||
monkeypatch.setattr("deerflow.workspace_changes.recorder.get_paths", lambda: paths)
|
||||
user_id = get_effective_user_id()
|
||||
thread_id = f"thread-approval-pause-{mode}"
|
||||
checkpointer = InMemorySaver()
|
||||
event_store = MemoryRunEventStore()
|
||||
run_store = MemoryRunStore()
|
||||
run_manager = RunManager(store=run_store)
|
||||
record = await run_manager.create(thread_id, user_id=user_id)
|
||||
await write_thread_goal(checkpointer, thread_id, build_goal_state("Finish the report"), create_if_missing=True)
|
||||
|
||||
output_dir = paths.sandbox_outputs_dir(thread_id, user_id=user_id)
|
||||
payload = {"kind": "hitl_approval", "request_id": f"approval-{mode}", "pending_tool_call_ids": ["call-write"]}
|
||||
|
||||
def produce_output(_state):
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "report.md").write_text("Ready for review\n", encoding="utf-8")
|
||||
return {}
|
||||
|
||||
def request_approval(_state):
|
||||
interrupt(payload)
|
||||
|
||||
builder = StateGraph(ThreadState)
|
||||
builder.add_node("produce_output", produce_output)
|
||||
builder.add_node("request_approval", request_approval)
|
||||
builder.add_edge(START, "produce_output")
|
||||
builder.add_edge("produce_output", "request_approval")
|
||||
builder.add_edge("request_approval", END)
|
||||
graph = builder.compile(checkpointer=checkpointer)
|
||||
|
||||
requested_followups = {"batch-1": "requested"}
|
||||
|
||||
class BatchRepo:
|
||||
async def list_continuation_batches_for_run(self, run_id):
|
||||
assert run_id == record.run_id
|
||||
return [{"id": "batch-1", "user_id": user_id}]
|
||||
|
||||
async def suppress_thread_continuations(self, _thread_id, *, goal_created_at):
|
||||
requested_followups["batch-1"] = f"suppressed:{goal_created_at}"
|
||||
|
||||
class BatchService:
|
||||
async def cancel_batch(self, *, batch_id, user_id):
|
||||
assert batch_id == "batch-1"
|
||||
assert user_id == record.user_id
|
||||
requested_followups.pop(batch_id, None)
|
||||
|
||||
batch_app = SimpleNamespace(
|
||||
state=SimpleNamespace(
|
||||
subagent_batch_repo=BatchRepo(),
|
||||
subagent_batch_service=BatchService(),
|
||||
checkpointer=checkpointer,
|
||||
)
|
||||
)
|
||||
batch_continuations = BatchContinuations(batch_app)
|
||||
|
||||
class LifecycleRecorder:
|
||||
def __init__(self):
|
||||
self.events = []
|
||||
|
||||
async def on_task_start(self, _app_store, _task_store, info):
|
||||
self.events.append(("start", info.run_id))
|
||||
|
||||
async def on_task_stop(self, _app_store, _task_store, info, outcome):
|
||||
self.events.append(("stop", info.run_id, outcome.value))
|
||||
|
||||
recorder = LifecycleRecorder()
|
||||
registry = ExtensionRegistry()
|
||||
with registry.attributed_to("test:hitl-pause"):
|
||||
registry.task_lifecycle(recorder)
|
||||
|
||||
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
||||
app_config.hitl_approval.enabled = True
|
||||
app_config.title.enabled = app_config.memory.enabled = app_config.memory.injection_enabled = False
|
||||
|
||||
await run_agent(
|
||||
_make_bridge(),
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(
|
||||
checkpointer=checkpointer,
|
||||
event_store=event_store,
|
||||
app_config=app_config,
|
||||
extensions=registry.build(),
|
||||
checkpoint_channel_mode=mode,
|
||||
on_run_completed=batch_continuations.source_finished,
|
||||
),
|
||||
agent_factory=lambda *, config: graph,
|
||||
graph_input={"messages": [HumanMessage(content="Write the report")]},
|
||||
config={"configurable": {"thread_id": thread_id}},
|
||||
)
|
||||
|
||||
row = await run_store.get(record.run_id)
|
||||
assert row["status"] == "waiting_for_input"
|
||||
assert row["error"] is None
|
||||
assert row["metadata"]["hitl_approval"] == payload
|
||||
paused_goal = await read_thread_goal(checkpointer, thread_id)
|
||||
assert paused_goal["status"] == "active"
|
||||
assert paused_goal["continuation_count"] == paused_goal["no_progress_count"] == 0
|
||||
assert "last_evaluation" not in paused_goal
|
||||
assert requested_followups == {"batch-1": "requested"}
|
||||
assert recorder.events == [("start", record.run_id), ("stop", record.run_id, "failed")]
|
||||
|
||||
delivery = await _delivery_events(event_store, thread_id, record.run_id)
|
||||
assert len(delivery) == 1
|
||||
assert delivery[0]["content"]["produced_paths"] == ["/mnt/user-data/outputs/report.md"]
|
||||
assert delivery[0]["content"]["satisfied"] is False
|
||||
terminal = await event_store.list_events(thread_id, record.run_id, event_types=["run.terminal"])
|
||||
assert [event["content"] for event in terminal] == [{"status": "waiting_for_input"}]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize("mode", ["full", "delta"])
|
||||
async def test_disabled_hitl_does_not_materialize_pending_approval_snapshot(mode):
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
|
||||
checkpointer = InMemorySaver()
|
||||
run_manager = RunManager()
|
||||
record = await run_manager.create(f"thread-disabled-hitl-{mode}")
|
||||
|
||||
def finish(_state):
|
||||
return {"messages": [AIMessage(content="Done")]}
|
||||
|
||||
builder = StateGraph(ThreadState)
|
||||
builder.add_node("finish", finish)
|
||||
builder.add_edge(START, "finish")
|
||||
builder.add_edge("finish", END)
|
||||
compiled = builder.compile(checkpointer=checkpointer)
|
||||
|
||||
class SnapshotTrackingGraph:
|
||||
def __init__(self):
|
||||
self.stream_finished = False
|
||||
self.post_stream_running_snapshots = 0
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(compiled, name)
|
||||
|
||||
async def astream(self, *args, **kwargs):
|
||||
async for item in compiled.astream(*args, **kwargs):
|
||||
yield item
|
||||
self.stream_finished = True
|
||||
|
||||
async def aget_state(self, config):
|
||||
if self.stream_finished and record.status == RunStatus.running:
|
||||
self.post_stream_running_snapshots += 1
|
||||
return await compiled.aget_state(config)
|
||||
|
||||
graph = SnapshotTrackingGraph()
|
||||
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
||||
app_config.hitl_approval.enabled = False
|
||||
app_config.title.enabled = app_config.memory.enabled = app_config.memory.injection_enabled = False
|
||||
|
||||
await run_agent(
|
||||
_make_bridge(),
|
||||
run_manager,
|
||||
record,
|
||||
ctx=RunContext(checkpointer=checkpointer, app_config=app_config, checkpoint_channel_mode=mode),
|
||||
agent_factory=lambda *, config: graph,
|
||||
graph_input={"messages": [HumanMessage(content="Finish normally")]},
|
||||
config={"configurable": {"thread_id": record.thread_id}},
|
||||
)
|
||||
|
||||
assert record.status == RunStatus.success
|
||||
assert graph.post_stream_running_snapshots == 0
|
||||
|
||||
145
backend/tests/test_subagents_hitl_approval.py
Normal file
145
backend/tests/test_subagents_hitl_approval.py
Normal file
@ -0,0 +1,145 @@
|
||||
"""Native subagents enforce configured HITL approval without resume support."""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from types import ModuleType
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langchain_core.tools import tool
|
||||
|
||||
from deerflow.config.app_config import AppConfig
|
||||
|
||||
_MOCKED_MODULE_NAMES = [
|
||||
"deerflow.agents",
|
||||
"deerflow.agents.thread_state",
|
||||
"deerflow.agents.middlewares",
|
||||
"deerflow.agents.middlewares.thread_data_middleware",
|
||||
"deerflow.sandbox",
|
||||
"deerflow.sandbox.middleware",
|
||||
"deerflow.sandbox.security",
|
||||
"deerflow.models",
|
||||
"deerflow.skills.storage",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def executor_module():
|
||||
"""Import the real executor despite the suite's cycle-breaking doubles."""
|
||||
originals = {name: sys.modules.get(name) for name in _MOCKED_MODULE_NAMES}
|
||||
original_executor = sys.modules.get("deerflow.subagents.executor")
|
||||
preloaded_names = (
|
||||
"deerflow.agents.lead_agent.agent",
|
||||
"deerflow.agents.thread_state",
|
||||
"deerflow.agents.middlewares.audit_context",
|
||||
"deerflow.agents.middlewares.hitl_approval_middleware",
|
||||
"deerflow.agents.middlewares.tool_declarations",
|
||||
"deerflow.agents.middlewares.tool_error_handling_middleware",
|
||||
"deerflow.models.factory",
|
||||
"deerflow.models.openai_codex_provider",
|
||||
"deerflow.agents.assembly_descriptor",
|
||||
"deerflow.sandbox.sandbox_provider",
|
||||
"deerflow.sandbox.overwrite",
|
||||
"deerflow.sandbox.prewarm",
|
||||
)
|
||||
original_preloaded = {name: sys.modules.get(name) for name in preloaded_names}
|
||||
preloaded = {name: importlib.import_module(name) for name in preloaded_names}
|
||||
|
||||
sys.modules.pop("deerflow.subagents.executor", None)
|
||||
subagents_package = sys.modules.get("deerflow.subagents")
|
||||
if subagents_package is not None and hasattr(subagents_package, "executor"):
|
||||
delattr(subagents_package, "executor")
|
||||
for name in _MOCKED_MODULE_NAMES:
|
||||
sys.modules[name] = MagicMock()
|
||||
storage = ModuleType("deerflow.skills.storage")
|
||||
storage.get_or_new_skill_storage = lambda **kwargs: None
|
||||
sys.modules["deerflow.skills.storage"] = storage
|
||||
sys.modules.update(preloaded)
|
||||
|
||||
module = importlib.import_module("deerflow.subagents.executor")
|
||||
try:
|
||||
yield module
|
||||
finally:
|
||||
for name, original in originals.items():
|
||||
if original is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = original
|
||||
for name, original in original_preloaded.items():
|
||||
if original is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = original
|
||||
if original_executor is None:
|
||||
sys.modules.pop("deerflow.subagents.executor", None)
|
||||
else:
|
||||
sys.modules["deerflow.subagents.executor"] = original_executor
|
||||
|
||||
|
||||
class _BindableFakeChatModel(GenericFakeChatModel):
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
return self
|
||||
|
||||
|
||||
async def _run_native_subagent(executor_module, monkeypatch, *, approval_enabled: bool):
|
||||
executions = []
|
||||
|
||||
@tool
|
||||
def bash(command: str) -> str:
|
||||
"""Record a harmless command instead of executing a shell."""
|
||||
executions.append(command)
|
||||
return "executed"
|
||||
|
||||
app_config = AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}})
|
||||
app_config.hitl_approval.enabled = approval_enabled
|
||||
app_config.hitl_approval.tools = ["bash"]
|
||||
app_config.hitl_approval.non_interactive = "deny"
|
||||
model = _BindableFakeChatModel(
|
||||
messages=iter(
|
||||
[
|
||||
AIMessage(content="", tool_calls=[{"name": "bash", "args": {"command": "record"}, "id": "call-1"}]),
|
||||
AIMessage(content="done"),
|
||||
]
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(executor_module, "create_chat_model", lambda **kwargs: model)
|
||||
runtime_middlewares = sys.modules["deerflow.agents.middlewares.tool_error_handling_middleware"]
|
||||
monkeypatch.setattr(runtime_middlewares, "build_subagent_runtime_middlewares", lambda **kwargs: [])
|
||||
|
||||
executor = executor_module.SubagentExecutor(
|
||||
config=executor_module.SubagentConfig(name="worker", description="test", skills=[]),
|
||||
tools=[bash],
|
||||
app_config=app_config,
|
||||
parent_model="test-model",
|
||||
)
|
||||
agent = await executor._create_agent([bash])
|
||||
result = await agent.ainvoke(
|
||||
{"messages": [HumanMessage(content="run the command")]},
|
||||
context={"is_subagent": True, "app_config": app_config},
|
||||
)
|
||||
return executions, result
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_native_subagent_denies_configured_tool_without_resume(executor_module, monkeypatch):
|
||||
executions, result = await _run_native_subagent(executor_module, monkeypatch, approval_enabled=True)
|
||||
|
||||
assert executions == []
|
||||
assert "__interrupt__" not in result
|
||||
tool_messages = [message for message in result["messages"] if isinstance(message, ToolMessage)]
|
||||
assert len(tool_messages) == 1
|
||||
assert tool_messages[0].name == "bash"
|
||||
assert tool_messages[0].status == "error"
|
||||
assert "Approval is required" in tool_messages[0].content
|
||||
assert "cannot collect approval" in tool_messages[0].content
|
||||
assert "tool was not executed" in tool_messages[0].content
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_native_subagent_default_keeps_tool_execution_enabled(executor_module, monkeypatch):
|
||||
executions, result = await _run_native_subagent(executor_module, monkeypatch, approval_enabled=False)
|
||||
|
||||
assert executions == ["record"]
|
||||
assert any(isinstance(message, ToolMessage) and message.content == "executed" for message in result["messages"])
|
||||
@ -74,6 +74,317 @@ async def test_unacknowledged_completion_cannot_terminalize_or_recreate():
|
||||
assert await store.get(record.run_id) is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_metadata_persistence_retains_idempotency_without_store_io_under_lock():
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store)
|
||||
record = await manager.create_or_reject("thread", idempotency_key="admission-key", idempotency_request={"input": "same"})
|
||||
record.metadata["hitl_approval"] = {"request_id": "approval-1"}
|
||||
original_put = store.put
|
||||
|
||||
async def checked_put(*args, **kwargs):
|
||||
assert not manager._lock.locked()
|
||||
return await original_put(*args, **kwargs)
|
||||
|
||||
store.put = checked_put
|
||||
|
||||
assert await manager.persist_metadata(record.run_id) is True
|
||||
row = await store.get(record.run_id)
|
||||
assert row["idempotency_key"] == "admission-key"
|
||||
assert row["idempotency_request"] == {"input": "same"}
|
||||
assert row["metadata"]["hitl_approval"] == {"request_id": "approval-1"}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mark_approval_answered_retries_and_updates_memory_and_store():
|
||||
import sqlite3
|
||||
|
||||
from deerflow.runtime.runs.manager import PersistenceRetryPolicy
|
||||
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(
|
||||
store=store,
|
||||
persistence_retry_policy=PersistenceRetryPolicy(max_attempts=2, initial_delay=0, max_delay=0),
|
||||
)
|
||||
identity = {"version": 1, "kind": "resume", "sha256": "a" * 64}
|
||||
paused = await manager.create_or_reject(
|
||||
"thread",
|
||||
metadata={"hitl_approval": {"request_id": "approval-1"}, "retained": True},
|
||||
idempotency_key="admission-key",
|
||||
idempotency_request=identity,
|
||||
)
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
original_put = store.put
|
||||
attempts = 0
|
||||
|
||||
async def fail_once(run_id, **kwargs):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
return await original_put(run_id, **kwargs)
|
||||
|
||||
store.put = fail_once
|
||||
assert await manager.mark_approval_answered("thread") is not None
|
||||
|
||||
assert attempts == 2
|
||||
assert paused.status == RunStatus.success
|
||||
assert paused.metadata == {"retained": True}
|
||||
answered = await store.get(paused.run_id)
|
||||
assert answered["status"] == "success"
|
||||
assert answered["metadata"] == {"retained": True}
|
||||
assert answered["idempotency_key"] == "admission-key"
|
||||
assert answered["idempotency_request"] == identity
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mark_approval_answered_has_single_cross_manager_winner():
|
||||
store = MemoryRunStore()
|
||||
first = RunManager(store=store, worker_id="worker-a")
|
||||
second = RunManager(store=store, worker_id="worker-b")
|
||||
paused = await first.create_or_reject(
|
||||
"thread",
|
||||
metadata={"hitl_approval": {"request_id": "approval-1"}, "retained": True},
|
||||
)
|
||||
await first.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
original_answer = store.answer_approval
|
||||
both_answer_attempts = asyncio.Event()
|
||||
attempts = 0
|
||||
|
||||
async def synchronized_answer(run_id, *, metadata, answered_updated_at, expected_request_id=None):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 2:
|
||||
both_answer_attempts.set()
|
||||
await asyncio.wait_for(both_answer_attempts.wait(), 1)
|
||||
return await original_answer(
|
||||
run_id,
|
||||
metadata=metadata,
|
||||
answered_updated_at=answered_updated_at,
|
||||
expected_request_id=expected_request_id,
|
||||
)
|
||||
|
||||
store.answer_approval = synchronized_answer
|
||||
|
||||
results = await asyncio.gather(
|
||||
first.mark_approval_answered("thread"),
|
||||
second.mark_approval_answered("thread"),
|
||||
)
|
||||
|
||||
assert sorted(result is not None for result in results) == [False, True]
|
||||
answered = await store.get(paused.run_id)
|
||||
assert answered["status"] == "success"
|
||||
assert answered["metadata"] == {"retained": True}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_stale_response_cannot_consume_newer_pause_across_managers():
|
||||
store = MemoryRunStore()
|
||||
owner = RunManager(store=store, worker_id="worker-a")
|
||||
peer = RunManager(store=store, worker_id="worker-b")
|
||||
|
||||
def metadata(request_id):
|
||||
return {
|
||||
"hitl_approval": {
|
||||
"kind": "hitl_approval",
|
||||
"request_id": request_id,
|
||||
"pending_tool_call_ids": ["call-1"],
|
||||
"actions": [{"tool_call_id": "call-1", "request_id": request_id}],
|
||||
},
|
||||
"retained": True,
|
||||
}
|
||||
|
||||
first = await owner.create_or_reject("thread", metadata=metadata("approval-1"))
|
||||
await owner.set_status(first.run_id, RunStatus.waiting_for_input)
|
||||
first_answer = await owner.mark_approval_answered("thread", expected_request_ids={"call-1": "approval-1"})
|
||||
assert first_answer is not None
|
||||
|
||||
second = await owner.create_or_reject("thread", metadata=metadata("approval-2"))
|
||||
await owner.set_status(second.run_id, RunStatus.waiting_for_input)
|
||||
|
||||
# The peer has no process-local record or lock in common with the owner.
|
||||
# Its response was validated against P1, but the newest durable pause is P2.
|
||||
assert await peer.mark_approval_answered("thread", expected_request_ids={"call-1": "approval-1"}) is None
|
||||
assert (await store.get(second.run_id))["status"] == "waiting_for_input"
|
||||
assert (await store.get(second.run_id))["metadata"] == metadata("approval-2")
|
||||
|
||||
second_answer = await peer.mark_approval_answered("thread", expected_request_ids={"call-1": "approval-2"})
|
||||
assert second_answer is not None
|
||||
assert second_answer.record.store_only
|
||||
assert (await store.get(second.run_id))["status"] == "success"
|
||||
|
||||
assert await peer.restore_approval_answer(second_answer) is True
|
||||
restored = await store.get(second.run_id)
|
||||
assert restored["metadata"] == metadata("approval-2")
|
||||
assert await peer.restore_approval_answer(second_answer) is False
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_restore_approval_answer_rejects_stale_committed_version():
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store)
|
||||
paused = await manager.create_or_reject("thread", metadata={"hitl_approval": {"request_id": "approval-1"}, "retained": True})
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
|
||||
answer = await manager.mark_approval_answered("thread")
|
||||
assert answer is not None
|
||||
committed = await store.get(paused.run_id)
|
||||
assert answer.answered_updated_at == committed["updated_at"]
|
||||
|
||||
await store.put(paused.run_id, thread_id="thread", status="success", metadata={"retained": True})
|
||||
assert await manager.restore_approval_answer(answer) is False
|
||||
assert (await store.get(paused.run_id))["status"] == "success"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mark_approval_answered_acknowledges_worker_finalization_before_store_io():
|
||||
store = MemoryRunStore()
|
||||
manager = manager_for(store)
|
||||
paused = await manager.create_or_reject("thread", metadata={"hitl_approval": {"request_id": "approval-1"}})
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
previous_revision = paused.terminal_status_revision
|
||||
original_put = store.put
|
||||
answered_put_started, release_answered_put = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def hold_answered_put(run_id, **kwargs):
|
||||
answered_put_started.set()
|
||||
await release_answered_put.wait()
|
||||
return await original_put(run_id, **kwargs)
|
||||
|
||||
store.put = hold_answered_put
|
||||
answered = asyncio.create_task(manager.mark_approval_answered("thread"))
|
||||
try:
|
||||
await asyncio.wait_for(answered_put_started.wait(), 1)
|
||||
assert paused.terminal_status_revision == previous_revision + 1
|
||||
assert paused.terminal_commit_proof is not None
|
||||
assert paused.terminal_commit_proof.status == RunStatus.success
|
||||
assert manager.terminal_event_status(paused) == "success"
|
||||
assert not paused.terminal_status_persisted
|
||||
release_answered_put.set()
|
||||
assert await asyncio.wait_for(answered, 1) is not None
|
||||
finally:
|
||||
release_answered_put.set()
|
||||
await finish(answered)
|
||||
|
||||
receipt = object()
|
||||
seal_evidence = AsyncMock(return_value=True)
|
||||
store.seal_evidence = seal_evidence
|
||||
assert await manager.seal_completed_evidence(paused.run_id, receipt) is True
|
||||
seal_evidence.assert_awaited_once_with(paused.run_id, owner_worker_id="owner", receipt=receipt)
|
||||
await manager.update_run_completion(paused.run_id, status="success", total_tokens=17, message_count=2)
|
||||
completed = await store.get(paused.run_id)
|
||||
assert completed["total_tokens"] == 17
|
||||
assert completed["message_count"] == 2
|
||||
assert paused.terminal_status_persisted
|
||||
assert manager.terminal_event_status(paused) == "success"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_stale_paused_status_ack_cannot_replace_answered_proof():
|
||||
store = MemoryRunStore()
|
||||
manager = manager_for(store)
|
||||
paused = await manager.create_or_reject("thread", metadata={"hitl_approval": {"request_id": "approval-1"}})
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input, persist=False)
|
||||
paused_revision = paused.terminal_status_revision
|
||||
original_finalize = store.finalize_if_not_cancelled
|
||||
committed, release_ack = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def commit_then_hold_ack(*args, **kwargs):
|
||||
result = await original_finalize(*args, **kwargs)
|
||||
committed.set()
|
||||
await release_ack.wait()
|
||||
return result
|
||||
|
||||
store.finalize_if_not_cancelled = commit_then_hold_ack
|
||||
worker_commit = asyncio.create_task(manager.set_status_if_not_cancelled(paused.run_id, RunStatus.waiting_for_input))
|
||||
try:
|
||||
await asyncio.wait_for(committed.wait(), 1)
|
||||
assert await manager.mark_approval_answered("thread") is not None
|
||||
assert paused.terminal_status_revision == paused_revision + 1
|
||||
assert paused.terminal_commit_proof is not None
|
||||
assert paused.terminal_commit_proof.status == RunStatus.success
|
||||
assert paused.terminal_status_persisted
|
||||
release_ack.set()
|
||||
await asyncio.wait_for(worker_commit, 1)
|
||||
finally:
|
||||
release_ack.set()
|
||||
await finish(worker_commit)
|
||||
|
||||
answered = await store.get(paused.run_id)
|
||||
assert answered["status"] == RunStatus.success.value
|
||||
assert paused.status == RunStatus.success
|
||||
assert paused.terminal_commit_proof.status == RunStatus.success
|
||||
assert paused.terminal_status_revision == paused_revision + 1
|
||||
assert manager.terminal_event_status(paused) == "success"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_worker_waiting_metadata_put_cannot_resurrect_answered_run():
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store)
|
||||
paused = await manager.create_or_reject("thread", metadata={"hitl_approval": {"request_id": "approval-1"}, "retained": True})
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
original_put = store.put
|
||||
stale_put_started, release_stale_put = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def hold_worker_waiting_put(run_id, **kwargs):
|
||||
if kwargs["status"] == RunStatus.waiting_for_input.value:
|
||||
stale_put_started.set()
|
||||
await release_stale_put.wait()
|
||||
await original_put(run_id, **kwargs)
|
||||
|
||||
store.put = hold_worker_waiting_put
|
||||
worker_persist = asyncio.create_task(manager.persist_metadata(paused.run_id))
|
||||
try:
|
||||
await asyncio.wait_for(stale_put_started.wait(), 1)
|
||||
assert await manager.mark_approval_answered("thread") is not None
|
||||
assert (await store.get(paused.run_id))["status"] == RunStatus.success.value
|
||||
release_stale_put.set()
|
||||
assert await asyncio.wait_for(worker_persist, 1) is True
|
||||
finally:
|
||||
release_stale_put.set()
|
||||
await finish(worker_persist)
|
||||
|
||||
assert paused.status == RunStatus.success
|
||||
assert paused.metadata == {"retained": True}
|
||||
answered = await store.get(paused.run_id)
|
||||
assert answered["status"] == RunStatus.success.value
|
||||
assert answered["metadata"] == {"retained": True}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mark_approval_answered_reconciles_newer_local_state():
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store)
|
||||
paused = await manager.create_or_reject("thread", metadata={"hitl_approval": {"request_id": "approval-1"}})
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
original_put = store.put
|
||||
answered_put_started, release_answered_put = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def hold_answered_put(run_id, **kwargs):
|
||||
if kwargs["status"] == RunStatus.success.value:
|
||||
answered_put_started.set()
|
||||
await release_answered_put.wait()
|
||||
await original_put(run_id, **kwargs)
|
||||
|
||||
store.put = hold_answered_put
|
||||
answered = asyncio.create_task(manager.mark_approval_answered("thread"))
|
||||
try:
|
||||
await asyncio.wait_for(answered_put_started.wait(), 1)
|
||||
await manager.set_status(paused.run_id, RunStatus.error, error="newer terminal state", persist=False)
|
||||
release_answered_put.set()
|
||||
assert await asyncio.wait_for(answered, 1) is not None
|
||||
finally:
|
||||
release_answered_put.set()
|
||||
await finish(answered)
|
||||
|
||||
assert paused.status == RunStatus.error
|
||||
assert paused.error == "newer terminal state"
|
||||
latest = await store.get(paused.run_id)
|
||||
assert latest["status"] == RunStatus.error.value
|
||||
assert latest["error"] == "newer terminal state"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_committed_cas_waits_for_its_ack_after_renewal_rejection():
|
||||
store = MemoryRunStore()
|
||||
|
||||
@ -665,6 +665,27 @@ def _make_start_run_request(run_manager):
|
||||
)
|
||||
|
||||
|
||||
async def _pause_checkpoint(thread_id: str = "thread-1", payload: dict | None = None):
|
||||
"""Create a real checkpoint whose only pause signal is a raw interrupt write."""
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from langgraph.graph import MessagesState, StateGraph
|
||||
from langgraph.types import interrupt
|
||||
|
||||
def approval(_state):
|
||||
interrupt(payload or {"kind": "hitl_approval", "request_id": "approval-1", "pending_tool_call_ids": ["call-1"]})
|
||||
return {}
|
||||
|
||||
builder = StateGraph(MessagesState)
|
||||
builder.add_node("approval", approval)
|
||||
builder.set_entry_point("approval")
|
||||
builder.set_finish_point("approval")
|
||||
saver = InMemorySaver()
|
||||
graph = builder.compile(checkpointer=saver)
|
||||
await graph.ainvoke({"messages": [HumanMessage(content="start")]}, {"configurable": {"thread_id": thread_id}})
|
||||
return saver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _stub_app_config():
|
||||
set_app_config(AppConfig.model_validate({"sandbox": {"use": "deerflow.sandbox.local:LocalSandboxProvider"}}))
|
||||
@ -1034,12 +1055,73 @@ async def test_start_run_idempotency_distinguishes_static_resume_from_new_input(
|
||||
assert "different request" in str(excinfo.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_start_run_reuses_admitted_resume_before_current_pause_validation(_stub_app_config):
|
||||
from app.gateway.services import _run_idempotency_request, start_run
|
||||
|
||||
approved = RunCreateRequest(command={"resume": {"answer": "approve"}})
|
||||
store = MemoryRunStore()
|
||||
owner = RunManager(store=store, worker_id="worker-a")
|
||||
peer = RunManager(store=store, worker_id="worker-b")
|
||||
first = await owner.create_or_reject(
|
||||
"thread-1",
|
||||
user_id=None,
|
||||
idempotency_key="http-run:same",
|
||||
idempotency_request=_run_idempotency_request(approved),
|
||||
kwargs={"input": None, "config": None},
|
||||
)
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new=AsyncMock(side_effect=AssertionError("current pause must not be revalidated"))) as validate,
|
||||
patch("app.gateway.hitl_approval.mark_approval_answered", new=AsyncMock(side_effect=AssertionError("reused resume must not mutate the pause"))) as mark_answered,
|
||||
patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
|
||||
):
|
||||
reused = await start_run(
|
||||
approved,
|
||||
"thread-1",
|
||||
_make_start_run_request(peer),
|
||||
idempotency_key="http-run:same",
|
||||
)
|
||||
|
||||
assert reused.run_id == first.run_id
|
||||
assert reused.idempotency_reused is True
|
||||
validate.assert_not_awaited()
|
||||
mark_answered.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_start_run_rejects_checkpointless_resume(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
manager = RunManager(store=MemoryRunStore(), worker_id="worker-a")
|
||||
request = _make_start_run_request(manager)
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.services.run_agent", new_callable=AsyncMock) as worker,
|
||||
pytest.raises(HTTPException) as excinfo,
|
||||
):
|
||||
await start_run(
|
||||
RunCreateRequest(command={"resume": {"answer": "approve"}}),
|
||||
"thread-1",
|
||||
request,
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 409
|
||||
assert excinfo.value.detail["code"] == "no_pending_approval"
|
||||
worker.assert_not_called()
|
||||
assert await manager.list_by_thread("thread-1") == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_start_run_persists_resume_fingerprint_without_payload(_stub_app_config):
|
||||
from app.gateway.services import _run_idempotency_request, start_run
|
||||
|
||||
request_body = RunCreateRequest(command={"resume": {"answer": "approve"}})
|
||||
manager = RunManager(store=MemoryRunStore(), worker_id="worker-a")
|
||||
request = _make_start_run_request(manager)
|
||||
request.app.state.checkpointer = await _pause_checkpoint(payload={"question": "Continue?"})
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
@ -1048,7 +1130,7 @@ async def test_start_run_persists_resume_fingerprint_without_payload(_stub_app_c
|
||||
record = await start_run(
|
||||
request_body,
|
||||
"thread-1",
|
||||
_make_start_run_request(manager),
|
||||
request,
|
||||
idempotency_key="http-run:same",
|
||||
)
|
||||
assert record.task is not None
|
||||
@ -1061,6 +1143,464 @@ async def test_start_run_persists_resume_fingerprint_without_payload(_stub_app_c
|
||||
assert "idempotency_request" not in record.kwargs
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_start_run_rejects_checkpoint_only_pending_approval(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
manager = RunManager(store=MemoryRunStore(), worker_id="worker-a")
|
||||
request = _make_start_run_request(manager)
|
||||
request.app.state.checkpointer = await _pause_checkpoint()
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.services.build_thread_checkpoint_state_accessor", new=AsyncMock(side_effect=AssertionError("ordinary admission must use raw checkpoints"))),
|
||||
patch("app.gateway.services.run_agent", new_callable=AsyncMock) as worker,
|
||||
pytest.raises(HTTPException) as excinfo,
|
||||
):
|
||||
await start_run(
|
||||
RunCreateRequest(input={"messages": [{"role": "user", "content": "new work"}]}),
|
||||
"thread-1",
|
||||
request,
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 409
|
||||
assert excinfo.value.detail["code"] == "approval_pending"
|
||||
worker.assert_not_called()
|
||||
assert await manager.list_by_thread("thread-1") == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_paused_non_resume_keyed_retry_defers_to_reuse_and_fingerprint(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
input_payload = {"messages": [{"role": "user", "content": "original"}]}
|
||||
store = MemoryRunStore()
|
||||
owner = RunManager(store=store, worker_id="worker-a")
|
||||
peer = RunManager(store=store, worker_id="worker-b")
|
||||
first = await owner.create_or_reject(
|
||||
"thread-1",
|
||||
user_id=None,
|
||||
idempotency_key="http-run:same",
|
||||
kwargs={"input": input_payload, "config": None},
|
||||
)
|
||||
request = _make_start_run_request(peer)
|
||||
request.app.state.checkpointer = await _pause_checkpoint()
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.services.build_thread_checkpoint_state_accessor", new=AsyncMock(side_effect=AssertionError("a keyed retry must not revalidate the pause"))),
|
||||
patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
|
||||
):
|
||||
reused = await start_run(RunCreateRequest(input=input_payload), "thread-1", request, idempotency_key="http-run:same")
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await start_run(
|
||||
RunCreateRequest(input={"messages": [{"role": "user", "content": "different"}]}),
|
||||
"thread-1",
|
||||
request,
|
||||
idempotency_key="http-run:same",
|
||||
)
|
||||
|
||||
assert reused.run_id == first.run_id
|
||||
assert reused.idempotency_reused is True
|
||||
assert excinfo.value.status_code == 409
|
||||
assert "different request" in str(excinfo.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_rejected_resume_admission_leaves_paused_run_untouched(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store, worker_id="worker-a")
|
||||
paused = await manager.create_or_reject(
|
||||
"thread-1",
|
||||
user_id=None,
|
||||
metadata={"hitl_approval": {"kind": "hitl_approval", "pending_tool_call_ids": ["call-1"]}},
|
||||
kwargs={"input": None, "config": None},
|
||||
)
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
paused_before = await store.get(paused.run_id, user_id=None)
|
||||
await manager.create_or_reject("thread-1", user_id=None, kwargs={"input": None, "config": None})
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new_callable=AsyncMock) as validate,
|
||||
patch("app.gateway.services.run_agent", side_effect=AssertionError("worker must not attach")),
|
||||
pytest.raises(HTTPException) as excinfo,
|
||||
):
|
||||
await start_run(
|
||||
RunCreateRequest(command={"resume": {"answer": "approve"}}),
|
||||
"thread-1",
|
||||
_make_start_run_request(manager),
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 409
|
||||
validate.assert_awaited_once()
|
||||
assert await store.get(paused.run_id, user_id=None) == paused_before
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_cancelled_resume_marks_admitted_run_failed_without_attaching_worker(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store, worker_id="worker-a")
|
||||
request = _make_start_run_request(manager)
|
||||
mark_started = asyncio.Event()
|
||||
cleanup_started = asyncio.Event()
|
||||
release_cleanup = asyncio.Event()
|
||||
cleanup_finished = asyncio.Event()
|
||||
original_fail_start = manager.fail_start_if_pending
|
||||
|
||||
async def block_mark_answered(*args, **kwargs):
|
||||
del args, kwargs
|
||||
mark_started.set()
|
||||
await asyncio.Future()
|
||||
|
||||
async def controlled_fail_start(*args, **kwargs):
|
||||
cleanup_started.set()
|
||||
await release_cleanup.wait()
|
||||
await original_fail_start(*args, **kwargs)
|
||||
cleanup_finished.set()
|
||||
|
||||
manager.fail_start_if_pending = controlled_fail_start
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new_callable=AsyncMock),
|
||||
patch("app.gateway.hitl_approval.mark_approval_answered", side_effect=block_mark_answered),
|
||||
patch("app.gateway.services.run_agent", new_callable=AsyncMock) as run_agent,
|
||||
):
|
||||
start_task = asyncio.create_task(
|
||||
start_run(
|
||||
RunCreateRequest(command={"resume": {"answer": "approve"}}),
|
||||
"thread-1",
|
||||
request,
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(mark_started.wait(), 1)
|
||||
start_task.cancel()
|
||||
await asyncio.wait_for(cleanup_started.wait(), 1)
|
||||
|
||||
# A repeated cancellation must not cancel the shielded terminal write.
|
||||
start_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await start_task
|
||||
assert not cleanup_finished.is_set()
|
||||
|
||||
release_cleanup.set()
|
||||
await asyncio.wait_for(cleanup_finished.wait(), 1)
|
||||
|
||||
run_agent.assert_not_called()
|
||||
admitted = await manager.list_by_thread("thread-1")
|
||||
assert len(admitted) == 1
|
||||
assert admitted[0].status == RunStatus.error
|
||||
assert admitted[0].task is None
|
||||
assert (await store.get(admitted[0].run_id))["status"] == "error"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_concurrent_resume_only_attaches_transition_winner(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store, worker_id="worker-a")
|
||||
request = _make_start_run_request(manager)
|
||||
paused = await manager.create_or_reject(
|
||||
"thread-1",
|
||||
user_id=None,
|
||||
metadata={
|
||||
"hitl_approval": {
|
||||
"kind": "hitl_approval",
|
||||
"request_id": "approval-1",
|
||||
"pending_tool_call_ids": ["call-1"],
|
||||
"actions": [{"tool_call_id": "call-1", "request_id": "approval-1"}],
|
||||
}
|
||||
},
|
||||
kwargs={"input": None, "config": None},
|
||||
)
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
|
||||
body = RunCreateRequest(command={"resume": {"responses": [{"option_id": "approve", "tool_call_id": "call-1", "request_id": "approval-1"}]}})
|
||||
both_validated = asyncio.Event()
|
||||
worker_finished = asyncio.Event()
|
||||
validated = 0
|
||||
mark_results = []
|
||||
executions = []
|
||||
|
||||
async def validate_after_both_arrive(*args, **kwargs):
|
||||
nonlocal validated
|
||||
validated += 1
|
||||
arrival = validated
|
||||
if arrival == 2:
|
||||
both_validated.set()
|
||||
await asyncio.wait_for(both_validated.wait(), 1)
|
||||
if arrival == 2:
|
||||
# Both requests validated against the stale pause. Let the first
|
||||
# finish before the second enters the per-thread admission lock.
|
||||
await asyncio.wait_for(worker_finished.wait(), 5)
|
||||
return {"call-1": "approval-1"}
|
||||
|
||||
original_mark = manager.mark_approval_answered
|
||||
|
||||
async def record_mark_result(thread_id, *, expected_request_ids=None):
|
||||
result = await original_mark(thread_id, expected_request_ids=expected_request_ids)
|
||||
mark_results.append(result)
|
||||
return result
|
||||
|
||||
manager.mark_approval_answered = record_mark_result
|
||||
|
||||
async def fake_run_agent(bridge, run_manager, record, **kwargs):
|
||||
executions.append(record.run_id)
|
||||
await run_manager.set_status(record.run_id, RunStatus.success)
|
||||
worker_finished.set()
|
||||
|
||||
async def invoke(key):
|
||||
try:
|
||||
outcome = await start_run(body, "thread-1", request, idempotency_key=key)
|
||||
except HTTPException as exc:
|
||||
outcome = exc
|
||||
return key, outcome
|
||||
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", side_effect=validate_after_both_arrive),
|
||||
patch("app.gateway.services.run_agent", new=fake_run_agent),
|
||||
):
|
||||
results = dict(await asyncio.gather(invoke("http-run:race-a"), invoke("http-run:race-b")))
|
||||
winners = [(key, outcome) for key, outcome in results.items() if isinstance(outcome, RunRecord)]
|
||||
conflicts = [outcome for outcome in results.values() if isinstance(outcome, HTTPException)]
|
||||
|
||||
assert len(winners) == 1
|
||||
assert len(conflicts) == 1
|
||||
assert conflicts[0].status_code == 409
|
||||
assert conflicts[0].detail["code"] == "no_pending_approval"
|
||||
winner_key, winner = winners[0]
|
||||
assert winner.task is not None
|
||||
await winner.task
|
||||
|
||||
# The accepted request remains reusable under its original key.
|
||||
reused = await start_run(body, "thread-1", request, idempotency_key=winner_key)
|
||||
assert reused.run_id == winner.run_id
|
||||
assert reused.idempotency_reused is True
|
||||
|
||||
assert sorted(result is not None for result in mark_results) == [False, True]
|
||||
assert executions == [winner.run_id]
|
||||
assert paused.status == RunStatus.success
|
||||
failed = next(record for record in await manager.list_by_thread("thread-1") if record.run_id not in {paused.run_id, winner.run_id})
|
||||
assert failed.status == RunStatus.error
|
||||
assert failed.task is None
|
||||
assert (await store.get(failed.run_id))["status"] == "error"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_resume_attach_failure_restores_pause_and_allows_all_reject_retry(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store, worker_id="worker-a")
|
||||
original_metadata = {
|
||||
"hitl_approval": {
|
||||
"kind": "hitl_approval",
|
||||
"request_id": "approval-1",
|
||||
"pending_tool_call_ids": ["call-1"],
|
||||
"actions": [{"tool_call_id": "call-1", "request_id": "approval-1"}],
|
||||
},
|
||||
"retained": True,
|
||||
}
|
||||
paused = await manager.create_or_reject(
|
||||
"thread-1",
|
||||
user_id=None,
|
||||
metadata=original_metadata,
|
||||
kwargs={"input": None, "config": None},
|
||||
)
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
request = _make_start_run_request(manager)
|
||||
request.app.state.run_store = store
|
||||
|
||||
class FailingTransferBridge:
|
||||
supports_cross_process = True
|
||||
owner_task = None
|
||||
|
||||
async def start_producer_lease(self, run_id):
|
||||
return True
|
||||
|
||||
def transfer_producer_lease(self, run_id, owner_task):
|
||||
self.owner_task = owner_task
|
||||
raise RuntimeError("injected attachment failure")
|
||||
|
||||
async def abort_producer_lease(self, run_id):
|
||||
return None
|
||||
|
||||
failing_bridge = FailingTransferBridge()
|
||||
request.app.state.stream_bridge = failing_bridge
|
||||
executions = []
|
||||
|
||||
async def fake_run_agent(bridge, run_manager, record, **kwargs):
|
||||
executions.append(record.run_id)
|
||||
await run_manager.set_status(record.run_id, RunStatus.success)
|
||||
|
||||
all_reject = RunCreateRequest(command={"resume": {"responses": [{"option_id": "reject", "tool_call_id": "call-1", "request_id": "approval-1"}]}})
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new=AsyncMock(return_value={"call-1": "approval-1"})),
|
||||
patch("app.gateway.services.run_agent", new=fake_run_agent),
|
||||
pytest.raises(RuntimeError, match="injected attachment failure"),
|
||||
):
|
||||
await start_run(all_reject, "thread-1", request, idempotency_key="http-run:attach-retry")
|
||||
|
||||
assert failing_bridge.owner_task is not None
|
||||
assert failing_bridge.owner_task.cancelled()
|
||||
assert executions == []
|
||||
assert paused.status == RunStatus.waiting_for_input
|
||||
assert paused.metadata == original_metadata
|
||||
restored = await store.get(paused.run_id)
|
||||
assert restored["status"] == "waiting_for_input"
|
||||
assert restored["metadata"] == original_metadata
|
||||
failed = next(record for record in await manager.list_by_thread("thread-1") if record.run_id != paused.run_id)
|
||||
assert failed.status == RunStatus.error
|
||||
assert failed.task is None
|
||||
|
||||
request.app.state.stream_bridge = SimpleNamespace()
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new=AsyncMock(return_value={"call-1": "approval-1"})) as validate,
|
||||
patch("app.gateway.services.run_agent", new=fake_run_agent),
|
||||
):
|
||||
retried = await start_run(all_reject, "thread-1", request, idempotency_key="http-run:attach-retry")
|
||||
assert retried.task is not None
|
||||
await retried.task
|
||||
|
||||
validate.assert_awaited_once()
|
||||
assert executions == [retried.run_id]
|
||||
assert retried.run_id != failed.run_id
|
||||
assert await store.get(failed.run_id) is None
|
||||
assert paused.status == RunStatus.success
|
||||
assert paused.metadata == {"retained": True}
|
||||
assert (await store.get(paused.run_id))["status"] == "success"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_all_reject_recovers_checkpoint_pause_after_resume_crash(_stub_app_config):
|
||||
from app.gateway.services import start_run
|
||||
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(store=store, worker_id="worker-a")
|
||||
request = _make_start_run_request(manager)
|
||||
request.app.state.run_store = store
|
||||
metadata = {
|
||||
"hitl_approval": {
|
||||
"kind": "hitl_approval",
|
||||
"request_id": "approval-1",
|
||||
"pending_tool_call_ids": ["call-1"],
|
||||
"actions": [{"tool_call_id": "call-1", "request_id": "approval-1"}],
|
||||
}
|
||||
}
|
||||
paused = await manager.create_or_reject("thread-1", user_id=None, metadata=metadata)
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
answered = await manager.mark_approval_answered("thread-1", expected_request_ids={"call-1": "approval-1"})
|
||||
assert answered is not None
|
||||
assert paused.status == RunStatus.success
|
||||
|
||||
body = RunCreateRequest(command={"resume": {"responses": [{"option_id": "reject", "tool_call_id": "call-1", "request_id": "approval-1"}]}})
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new=AsyncMock(return_value={"call-1": "approval-1"})),
|
||||
patch("app.gateway.hitl_approval.current_approval_request_ids", new=AsyncMock(return_value={"call-1": "approval-1"})),
|
||||
patch("app.gateway.services.run_agent", new_callable=AsyncMock) as worker,
|
||||
):
|
||||
retried = await start_run(body, "thread-1", request)
|
||||
assert retried.task is not None
|
||||
await retried.task
|
||||
|
||||
worker.assert_awaited_once()
|
||||
assert (await store.get(paused.run_id))["status"] == "success"
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_resume_persistence_failure_allows_same_key_retry_to_resume(_stub_app_config):
|
||||
import sqlite3
|
||||
|
||||
from app.gateway.services import start_run
|
||||
from deerflow.runtime.runs.manager import PersistenceRetryPolicy, RunStartupError
|
||||
|
||||
store = MemoryRunStore()
|
||||
manager = RunManager(
|
||||
store=store,
|
||||
worker_id="worker-a",
|
||||
persistence_retry_policy=PersistenceRetryPolicy(max_attempts=2, initial_delay=0, max_delay=0),
|
||||
)
|
||||
paused = await manager.create_or_reject(
|
||||
"thread-1",
|
||||
user_id=None,
|
||||
metadata={"hitl_approval": {"kind": "hitl_approval", "pending_tool_call_ids": ["call-1"]}},
|
||||
kwargs={"input": None, "config": None},
|
||||
)
|
||||
await manager.set_status(paused.run_id, RunStatus.waiting_for_input)
|
||||
original_answer = store.answer_approval
|
||||
paused_writes = 0
|
||||
|
||||
async def fail_paused_write(run_id, *, metadata, answered_updated_at, expected_request_id=None):
|
||||
nonlocal paused_writes
|
||||
if run_id == paused.run_id:
|
||||
paused_writes += 1
|
||||
raise sqlite3.OperationalError("database is locked")
|
||||
return await original_answer(
|
||||
run_id,
|
||||
metadata=metadata,
|
||||
answered_updated_at=answered_updated_at,
|
||||
expected_request_id=expected_request_id,
|
||||
)
|
||||
|
||||
store.answer_approval = fail_paused_write
|
||||
body = RunCreateRequest(command={"resume": {"answer": "approve"}})
|
||||
request = _make_start_run_request(manager)
|
||||
request.app.state.run_store = store
|
||||
key = "http-run:resume-retry"
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new_callable=AsyncMock),
|
||||
patch("app.gateway.services.run_agent", new_callable=AsyncMock) as worker,
|
||||
pytest.raises(RunStartupError, match="Failed to mark approval run"),
|
||||
):
|
||||
await start_run(
|
||||
body,
|
||||
"thread-1",
|
||||
request,
|
||||
idempotency_key=key,
|
||||
)
|
||||
|
||||
assert paused_writes == 2
|
||||
worker.assert_not_called()
|
||||
assert paused.status == RunStatus.waiting_for_input
|
||||
assert paused.metadata["hitl_approval"]["pending_tool_call_ids"] == ["call-1"]
|
||||
paused_row = await store.get(paused.run_id)
|
||||
assert paused_row["status"] == "waiting_for_input"
|
||||
assert "hitl_approval" in paused_row["metadata"]
|
||||
admitted = next(record for record in await manager.list_by_thread("thread-1") if record.run_id != paused.run_id)
|
||||
assert admitted.status == RunStatus.error
|
||||
assert admitted.task is None
|
||||
assert (await store.get(admitted.run_id))["status"] == "error"
|
||||
|
||||
failed_run_id = admitted.run_id
|
||||
store.answer_approval = original_answer
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new_callable=AsyncMock) as validate,
|
||||
patch("app.gateway.services.run_agent", new_callable=AsyncMock) as worker,
|
||||
):
|
||||
retried = await start_run(body, "thread-1", request, idempotency_key=key)
|
||||
assert retried.task is not None
|
||||
await retried.task
|
||||
|
||||
assert retried.run_id != failed_run_id
|
||||
assert retried.idempotency_reused is False
|
||||
validate.assert_awaited_once()
|
||||
worker.assert_awaited_once()
|
||||
assert await store.get(failed_run_id) is None
|
||||
assert paused.status == RunStatus.success
|
||||
assert (await store.get(paused.run_id))["status"] == "success"
|
||||
|
||||
|
||||
async def _pre_6499_resume_retry(session_factory, body, idempotency_key):
|
||||
"""Freeze the null-input resume reuse policy from 02ce9ab2, without mapping the new column.
|
||||
|
||||
@ -1105,6 +1645,7 @@ async def test_mixed_version_resume_admission_requires_upgraded_routing(tmp_path
|
||||
key = "http-run:mixed-resume"
|
||||
with (
|
||||
patch("app.gateway.services.resolve_agent_factory", return_value=object()),
|
||||
patch("app.gateway.hitl_approval.validate_resume_request", new=AsyncMock()),
|
||||
patch("app.gateway.services.run_agent", new_callable=AsyncMock) as run_agent,
|
||||
):
|
||||
if writer_version == "current":
|
||||
|
||||
@ -32,6 +32,7 @@ from deerflow.persistence.thread_meta import (
|
||||
)
|
||||
from deerflow.persistence.thread_meta.memory import THREADS_NS, MemoryThreadMetaStore
|
||||
from deerflow.runtime import ConflictError, ThreadOperationKind
|
||||
from deerflow.runtime.checkpoint_mode import CheckpointModeMismatchError, CheckpointModeReconfigurationError
|
||||
from deerflow.runtime.checkpoint_state import CheckpointStateAccessor
|
||||
from deerflow.runtime.context_keys import CHECKPOINT_AGENT_NAME_METADATA_KEY
|
||||
from deerflow.runtime.goal import build_goal_outcome, build_goal_state, write_thread_goal
|
||||
@ -146,6 +147,68 @@ def test_compact_rejects_run_owned_by_another_worker(monkeypatch) -> None:
|
||||
assert response.json()["detail"] == "Thread has a run in flight. Compact after the run finishes."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "suffix", "payload"),
|
||||
[
|
||||
("put", "/goal", {"objective": "ship safely"}),
|
||||
("delete", "/goal", None),
|
||||
("post", "/state", {"values": {"title": "must not write"}}),
|
||||
],
|
||||
)
|
||||
def test_pending_approval_guard_uses_effective_checkpoint_access(monkeypatch, method, suffix, payload) -> None:
|
||||
app, _store, _checkpointer = _build_thread_app()
|
||||
thread_id = f"approval-accessor-{method}-{suffix.strip('/')}"
|
||||
accessor = SimpleNamespace()
|
||||
read_config = {"configurable": {"thread_id": thread_id}}
|
||||
guard = AsyncMock(side_effect=HTTPException(status_code=409, detail="approval_pending"))
|
||||
builder = AsyncMock(return_value=(accessor, read_config))
|
||||
monkeypatch.setattr(threads, "build_thread_checkpoint_state_accessor", builder)
|
||||
|
||||
with (
|
||||
patch("app.gateway.hitl_approval.reject_new_run_when_approval_pending", guard),
|
||||
TestClient(app) as client,
|
||||
):
|
||||
assert client.post("/api/threads", json={"thread_id": thread_id}).status_code == 200
|
||||
response = client.request(method.upper(), f"/api/threads/{thread_id}{suffix}", json=payload)
|
||||
|
||||
assert response.status_code == 409, response.text
|
||||
if suffix == "/state":
|
||||
assert guard.await_args.kwargs == {}
|
||||
builder.assert_not_awaited()
|
||||
else:
|
||||
assert guard.await_args.kwargs["accessor"] is accessor
|
||||
assert guard.await_args.kwargs["read_config"] is read_config
|
||||
|
||||
|
||||
def test_compact_pending_approval_guard_uses_raw_checkpoint_signal(monkeypatch) -> None:
|
||||
app, _store, _checkpointer = _build_thread_app()
|
||||
thread_id = "approval-accessor-compact"
|
||||
accessor = SimpleNamespace()
|
||||
read_config = {"configurable": {"thread_id": thread_id}}
|
||||
guard = AsyncMock(side_effect=HTTPException(status_code=409, detail="approval_pending"))
|
||||
monkeypatch.setattr(threads, "build_checkpoint_state_mutation_accessor", lambda *_args, **_kwargs: (accessor, read_config))
|
||||
|
||||
with (
|
||||
patch("app.gateway.hitl_approval.reject_new_run_when_approval_pending", guard),
|
||||
TestClient(app) as client,
|
||||
):
|
||||
assert client.post("/api/threads", json={"thread_id": thread_id}).status_code == 200
|
||||
response = client.post(f"/api/threads/{thread_id}/compact", json={"force": True})
|
||||
|
||||
assert response.status_code == 409, response.text
|
||||
assert guard.await_args.kwargs == {}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_hitl_helpers_degrade_when_app_state_is_absent() -> None:
|
||||
from app.gateway.hitl_approval import find_pending_interrupts, has_waiting_for_input_run, mark_approval_answered
|
||||
|
||||
request = SimpleNamespace()
|
||||
assert await has_waiting_for_input_run(request, "thread-1") is False
|
||||
assert await find_pending_interrupts(request, "thread-1") == {}
|
||||
await mark_approval_answered(request, "thread-1")
|
||||
|
||||
|
||||
def test_update_state_rejects_run_owned_by_another_worker(monkeypatch) -> None:
|
||||
"""All out-of-run writes share the same durable thread-operation admission."""
|
||||
from deerflow.runtime import RunManager, RunStatus
|
||||
@ -1284,6 +1347,44 @@ def test_goal_status_and_clear_round_trip() -> None:
|
||||
assert "goal" not in state_response.json()["values"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["put", "delete"])
|
||||
@pytest.mark.parametrize("failure_source", ["accessor", "approval_guard"])
|
||||
@pytest.mark.parametrize(
|
||||
("error_type", "expected_status"),
|
||||
[
|
||||
(CheckpointModeMismatchError, 409),
|
||||
(CheckpointModeReconfigurationError, 503),
|
||||
],
|
||||
)
|
||||
def test_goal_mutations_map_checkpoint_mode_errors(monkeypatch, method, failure_source, error_type, expected_status) -> None:
|
||||
app, _store, _checkpointer = _build_thread_app()
|
||||
thread_id = f"goal-mode-{method}-{failure_source}-{expected_status}"
|
||||
read_config = {"configurable": {"thread_id": thread_id}}
|
||||
error = error_type("checkpoint mode unavailable")
|
||||
builder = AsyncMock(
|
||||
side_effect=error if failure_source == "accessor" else None,
|
||||
return_value=(SimpleNamespace(), read_config),
|
||||
)
|
||||
guard = AsyncMock(side_effect=error if failure_source == "approval_guard" else None)
|
||||
monkeypatch.setattr(threads, "build_thread_checkpoint_state_accessor", builder)
|
||||
|
||||
with (
|
||||
patch("app.gateway.hitl_approval.reject_new_run_when_approval_pending", guard),
|
||||
TestClient(app) as client,
|
||||
):
|
||||
response = client.request(
|
||||
method.upper(),
|
||||
f"/api/threads/{thread_id}/goal",
|
||||
json={"objective": "Ship safely"} if method == "put" else None,
|
||||
)
|
||||
|
||||
assert response.status_code == expected_status, response.text
|
||||
expected_detail = f"Thread {thread_id}: {error}" if expected_status == 409 else str(error)
|
||||
assert response.json()["detail"] == expected_detail
|
||||
if failure_source == "accessor":
|
||||
guard.assert_not_awaited()
|
||||
|
||||
|
||||
def test_goal_mutations_reject_run_owned_by_another_worker() -> None:
|
||||
"""PUT and DELETE goal writes share the durable thread-operation boundary."""
|
||||
import httpx
|
||||
@ -1547,7 +1648,8 @@ def test_get_thread_status_uses_raw_pending_writes_for_materialized_checkpoint()
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["status"] == "idle"
|
||||
assert requested_configs == [snapshot.config]
|
||||
assert requested_configs[0] == snapshot.config
|
||||
assert requested_configs[-1]["configurable"].get("checkpoint_id") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stored_status", ["running", "error"])
|
||||
@ -1846,6 +1948,141 @@ def test_get_thread_history_returns_iso_for_legacy_checkpoint_metadata() -> None
|
||||
assert _ISO_TIMESTAMP_RE.match(entry["created_at"]), entry
|
||||
|
||||
|
||||
def test_get_thread_state_projects_pending_interrupt_tasks(monkeypatch) -> None:
|
||||
from langgraph.types import Interrupt
|
||||
|
||||
app, _store, _checkpointer = _build_thread_app()
|
||||
thread_id = "state-head-interrupt"
|
||||
config = {"configurable": {"thread_id": thread_id}}
|
||||
snapshot = SimpleNamespace(
|
||||
values={},
|
||||
config={"configurable": {**config["configurable"], "checkpoint_id": "checkpoint-head"}},
|
||||
parent_config=None,
|
||||
metadata={"created_at": "2026-10-26T01:02:03+00:00"},
|
||||
next=("approval",),
|
||||
tasks=(SimpleNamespace(id="task-head", name="approval"),),
|
||||
created_at="2026-10-26T01:02:03+00:00",
|
||||
)
|
||||
accessor = SimpleNamespace(aget=AsyncMock(return_value=snapshot))
|
||||
interrupt = Interrupt(value={"kind": "hitl_approval"}, id="interrupt-head")
|
||||
fallback_interrupt = Interrupt(value={"kind": "hitl_approval", "request_id": "approval-store-only"}, id="approval-store-only")
|
||||
monkeypatch.setattr(threads, "build_thread_checkpoint_state_accessor", AsyncMock(return_value=(accessor, config)))
|
||||
find_pending = AsyncMock(return_value={"task-head": [interrupt], "paused-run": [fallback_interrupt]})
|
||||
monkeypatch.setattr(threads, "find_pending_interrupts", find_pending)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get(f"/api/threads/{thread_id}/state")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["tasks"] == [
|
||||
{
|
||||
"id": "task-head",
|
||||
"name": "approval",
|
||||
"interrupts": [{"value": {"kind": "hitl_approval"}, "id": "interrupt-head"}],
|
||||
},
|
||||
{
|
||||
"id": "paused-run",
|
||||
"name": "",
|
||||
"interrupts": [{"value": {"kind": "hitl_approval", "request_id": "approval-store-only"}, "id": "approval-store-only"}],
|
||||
},
|
||||
]
|
||||
assert find_pending.await_args.kwargs["snapshot"] is snapshot
|
||||
|
||||
|
||||
def test_get_thread_state_serializes_run_store_fallback_interrupt(monkeypatch) -> None:
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
|
||||
app, _store, _checkpointer = _build_thread_app()
|
||||
thread_id = "state-run-store-interrupt"
|
||||
payload = {"kind": "hitl_approval", "request_id": "approval-store-only"}
|
||||
run_store = MemoryRunStore()
|
||||
asyncio.run(
|
||||
run_store.put(
|
||||
"paused-run",
|
||||
thread_id=thread_id,
|
||||
status="waiting_for_input",
|
||||
metadata={"hitl_approval": payload},
|
||||
)
|
||||
)
|
||||
app.state.run_store = run_store
|
||||
config = {"configurable": {"thread_id": thread_id}}
|
||||
snapshot = SimpleNamespace(
|
||||
values={},
|
||||
config={"configurable": {**config["configurable"], "checkpoint_id": "checkpoint-head"}},
|
||||
parent_config=None,
|
||||
metadata={},
|
||||
next=(),
|
||||
tasks=(),
|
||||
created_at=None,
|
||||
)
|
||||
accessor = SimpleNamespace(aget=AsyncMock(return_value=snapshot))
|
||||
monkeypatch.setattr(threads, "build_thread_checkpoint_state_accessor", AsyncMock(return_value=(accessor, config)))
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.get(f"/api/threads/{thread_id}/state")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["tasks"] == [
|
||||
{
|
||||
"id": "paused-run",
|
||||
"name": "",
|
||||
"interrupts": [{"value": payload, "id": "approval-store-only"}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_get_thread_history_projects_head_interrupt_tasks_and_created_at(monkeypatch) -> None:
|
||||
from langgraph.types import Interrupt
|
||||
|
||||
app, _store, _checkpointer = _build_thread_app()
|
||||
thread_id = "history-head-interrupt"
|
||||
config = {"configurable": {"thread_id": thread_id}}
|
||||
|
||||
def snapshot(checkpoint_id, created_at, task_id):
|
||||
return SimpleNamespace(
|
||||
values={},
|
||||
config={"configurable": {**config["configurable"], "checkpoint_id": checkpoint_id}},
|
||||
parent_config=None,
|
||||
metadata={"created_at": created_at},
|
||||
next=(),
|
||||
tasks=(SimpleNamespace(id=task_id, name="approval"),),
|
||||
created_at=created_at,
|
||||
)
|
||||
|
||||
snapshots = [
|
||||
snapshot("checkpoint-head", "2026-10-26T01:02:03+00:00", "task-head"),
|
||||
snapshot("checkpoint-old", "2026-10-25T01:02:03+00:00", "task-old"),
|
||||
]
|
||||
accessor = SimpleNamespace(ahistory=AsyncMock(return_value=snapshots))
|
||||
interrupt = Interrupt(value={"kind": "hitl_approval"}, id="interrupt-head")
|
||||
fallback_interrupt = Interrupt(value={"kind": "hitl_approval", "request_id": "approval-store-only"}, id="approval-store-only")
|
||||
monkeypatch.setattr(threads, "build_thread_checkpoint_state_accessor", AsyncMock(return_value=(accessor, config)))
|
||||
find_pending = AsyncMock(return_value={"task-head": [interrupt], "paused-run": [fallback_interrupt]})
|
||||
monkeypatch.setattr(threads, "find_pending_interrupts", find_pending)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post(f"/api/threads/{thread_id}/history", json={"limit": 2})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
head, older = response.json()
|
||||
assert head["created_at"] == "2026-10-26T01:02:03+00:00"
|
||||
assert older["created_at"] == "2026-10-25T01:02:03+00:00"
|
||||
assert head["tasks"] == [
|
||||
{
|
||||
"id": "task-head",
|
||||
"name": "approval",
|
||||
"interrupts": [{"value": {"kind": "hitl_approval"}, "id": "interrupt-head"}],
|
||||
},
|
||||
{
|
||||
"id": "paused-run",
|
||||
"name": "",
|
||||
"interrupts": [{"value": {"kind": "hitl_approval", "request_id": "approval-store-only"}, "id": "approval-store-only"}],
|
||||
},
|
||||
]
|
||||
assert older["tasks"] == [{"id": "task-old", "name": "approval"}]
|
||||
assert find_pending.await_args.kwargs["snapshot"] is snapshots[0]
|
||||
|
||||
|
||||
def test_get_thread_history_associates_tool_messages_from_checkpoint_turn() -> None:
|
||||
app, _store, checkpointer = _build_thread_app()
|
||||
app.state.run_event_store = SimpleNamespace(find_latest_ai_message_run_ids=AsyncMock(return_value={}))
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
# ============================================================================
|
||||
# Bump this number when the config schema changes.
|
||||
# Run `make config-upgrade` to merge new fields into your local config.yaml.
|
||||
config_version: 64
|
||||
config_version: 65
|
||||
|
||||
# Optional operator-owned literal prompt extensions (see CONFIGURATION.md):
|
||||
# lead_prompt_overlay:
|
||||
@ -2126,6 +2126,24 @@ verification:
|
||||
judge_enabled: false
|
||||
judge_model_name: null
|
||||
|
||||
# ============================================================================
|
||||
# HITL tool approval
|
||||
# ============================================================================
|
||||
# Pause high-risk tool calls for human approval before execution.
|
||||
# enabled=false (default) leaves all behavior unchanged.
|
||||
# The high-risk list is host-owned security policy edited by the operator.
|
||||
# non_interactive: deny | allow — behavior for IM channels, GitHub runs,
|
||||
# scheduled tasks, and other non-interactive entry points. Subagents and
|
||||
# runtimes without resume support always fail closed regardless of this setting.
|
||||
# Approval is not authorization: read_before_write, guardrails, and sandbox
|
||||
# authorization still apply after approval ("approve to try executing").
|
||||
hitl_approval:
|
||||
enabled: false
|
||||
tools: [] # e.g. [write_file, str_replace, bash]
|
||||
non_interactive: deny
|
||||
# Maximum matching tool calls in one approval batch; 0 disables the count limit.
|
||||
max_actions: 32
|
||||
|
||||
# ============================================================================
|
||||
# ACP Agents Configuration
|
||||
# ============================================================================
|
||||
|
||||
@ -114,7 +114,7 @@
|
||||
"content_schema": {
|
||||
"type": "object",
|
||||
"required": ["status"],
|
||||
"properties": {"status": {"enum": ["success", "error", "interrupted", "timeout"]}},
|
||||
"properties": {"status": {"enum": ["success", "error", "interrupted", "timeout", "waiting_for_input"]}},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"metadata_schema": {
|
||||
@ -493,7 +493,7 @@
|
||||
{
|
||||
"id": "terminal-run-status",
|
||||
"status": "split source of truth",
|
||||
"notes": "run.end is only a root graph completion marker and always says success. RunRow.status is authoritative for success, error, interrupted, and timeout; worker loss may leave no terminal run event."
|
||||
"notes": "run.end is only a root graph completion marker and always says success. RunRow.status is authoritative for success, error, interrupted, timeout, and waiting_for_input (added for HITL approval pauses); worker loss may leave no terminal run event."
|
||||
},
|
||||
{
|
||||
"id": "run-end-backend-serialization",
|
||||
|
||||
@ -135,7 +135,7 @@ they resolve from the `secrets` map):
|
||||
|
||||
```yaml
|
||||
config: |
|
||||
config_version: 64
|
||||
config_version: 65
|
||||
models:
|
||||
- name: gpt-4
|
||||
use: langchain_openai:ChatOpenAI
|
||||
|
||||
@ -367,7 +367,7 @@ ingress:
|
||||
# -- DeerFlow config.yaml content. Secrets MUST stay as $VAR references — never
|
||||
# inline literal secret values here. The default enables provisioner sandbox.
|
||||
config: |
|
||||
config_version: 64
|
||||
config_version: 65
|
||||
log_level: info
|
||||
recursion_limit: 100
|
||||
max_recursion_limit: 1000
|
||||
|
||||
@ -204,6 +204,10 @@ export default function AgentChatPage() {
|
||||
isHistoryLoading,
|
||||
hasMoreHistory,
|
||||
loadMoreHistory,
|
||||
approvalRequests,
|
||||
hasPendingApproval,
|
||||
submitHitlApproval,
|
||||
waitingForInputRunIds,
|
||||
} = useThreadStream({
|
||||
threadId: isNewThread ? undefined : threadId,
|
||||
displayThreadId: threadId,
|
||||
@ -236,6 +240,9 @@ export default function AgentChatPage() {
|
||||
});
|
||||
}
|
||||
},
|
||||
restInterrupts: threadMetadata.data?.interrupts,
|
||||
restInterruptsLoading: threadMetadata.isLoading,
|
||||
restInterruptsError: threadMetadata.isError,
|
||||
});
|
||||
|
||||
const hasThreadMessages = thread.messages.length > 0;
|
||||
@ -421,6 +428,13 @@ export default function AgentChatPage() {
|
||||
},
|
||||
[agent_name, currentKnowledgeScopeSnapshot, sendMessage, threadId],
|
||||
);
|
||||
const handleSubmitApproval = useCallback(
|
||||
(
|
||||
request: (typeof approvalRequests)[number],
|
||||
responses: Parameters<typeof submitHitlApproval>[2],
|
||||
) => submitHitlApproval(threadId, request, responses),
|
||||
[submitHitlApproval, threadId],
|
||||
);
|
||||
|
||||
const handleStop = useCallback(async () => {
|
||||
await thread.stop();
|
||||
@ -473,7 +487,8 @@ export default function AgentChatPage() {
|
||||
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY !== "true" &&
|
||||
!isUploading &&
|
||||
!thread.isLoading &&
|
||||
!hasOpenHumanInputCard;
|
||||
!hasOpenHumanInputCard &&
|
||||
!hasPendingApproval;
|
||||
|
||||
return (
|
||||
<ThreadContext.Provider value={{ thread, isMock }}>
|
||||
@ -585,12 +600,25 @@ export default function AgentChatPage() {
|
||||
loadMoreHistory={loadMoreHistory}
|
||||
isHistoryLoading={isHistoryLoading}
|
||||
tokenUsageInlineMode={tokenUsageInlineMode}
|
||||
approvalRequests={approvalRequests}
|
||||
approvalMetadataLoading={
|
||||
hasPendingApproval && approvalRequests.length === 0
|
||||
}
|
||||
approvalMetadataError={
|
||||
hasPendingApproval &&
|
||||
approvalRequests.length === 0 &&
|
||||
threadMetadata.isError
|
||||
}
|
||||
onReloadApprovalMetadata={() => threadMetadata.refetch()}
|
||||
waitingForInputRunIds={waitingForInputRunIds}
|
||||
onSubmitApproval={handleSubmitApproval}
|
||||
canRegenerate={
|
||||
!isNewThread &&
|
||||
!isMock &&
|
||||
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY !== "true" &&
|
||||
!isUploading &&
|
||||
!thread.isLoading
|
||||
!thread.isLoading &&
|
||||
!hasPendingApproval
|
||||
}
|
||||
onRegenerateMessage={handleRegenerate}
|
||||
canEdit={editBase && !hasGoal}
|
||||
@ -638,6 +666,12 @@ export default function AgentChatPage() {
|
||||
outcome={goalOutcome}
|
||||
isRunning={thread.isLoading}
|
||||
hasOpenHumanInputCard={hasOpenHumanInputCard}
|
||||
hasPendingApproval={hasPendingApproval}
|
||||
approvalMetadataError={
|
||||
hasPendingApproval &&
|
||||
approvalRequests.length === 0 &&
|
||||
threadMetadata.isError
|
||||
}
|
||||
/>
|
||||
{hasTodos && (
|
||||
<TodoList
|
||||
@ -686,7 +720,11 @@ export default function AgentChatPage() {
|
||||
selectorVisible && knowledgeScope ? (
|
||||
<KnowledgeScopeSelector
|
||||
agentName={agent_name}
|
||||
disabled={thread.isLoading || isUploading}
|
||||
disabled={
|
||||
thread.isLoading ||
|
||||
isUploading ||
|
||||
hasPendingApproval
|
||||
}
|
||||
selection={knowledgeScope}
|
||||
unavailableReason={
|
||||
agentKnowledgeEnabled
|
||||
@ -718,7 +756,8 @@ export default function AgentChatPage() {
|
||||
isUploading ||
|
||||
modelControl.busy ||
|
||||
(selectorVisible && agent === null) ||
|
||||
(!isNewThread && isHistoryLoading)
|
||||
(!isNewThread && isHistoryLoading) ||
|
||||
hasPendingApproval
|
||||
}
|
||||
onContextChange={(context, options) => {
|
||||
if (options?.automatic)
|
||||
|
||||
@ -216,6 +216,10 @@ export default function ChatPage() {
|
||||
isHistoryLoading,
|
||||
hasMoreHistory,
|
||||
loadMoreHistory,
|
||||
approvalRequests,
|
||||
hasPendingApproval,
|
||||
submitHitlApproval,
|
||||
waitingForInputRunIds,
|
||||
} = useThreadStream({
|
||||
threadId: isNewThread ? undefined : threadId,
|
||||
displayThreadId: threadId,
|
||||
@ -246,6 +250,9 @@ export default function ChatPage() {
|
||||
});
|
||||
}
|
||||
},
|
||||
restInterrupts: threadMetadata.data?.interrupts,
|
||||
restInterruptsLoading: threadMetadata.isLoading,
|
||||
restInterruptsError: threadMetadata.isError,
|
||||
});
|
||||
|
||||
const hasThreadMessages = thread.messages.length > 0;
|
||||
@ -499,6 +506,13 @@ export default function ChatPage() {
|
||||
},
|
||||
[currentKnowledgeScopeSnapshot, sendMessage, threadId],
|
||||
);
|
||||
const handleSubmitApproval = useCallback(
|
||||
(
|
||||
request: (typeof approvalRequests)[number],
|
||||
responses: Parameters<typeof submitHitlApproval>[2],
|
||||
) => submitHitlApproval(threadId, request, responses),
|
||||
[submitHitlApproval, threadId],
|
||||
);
|
||||
const handleStop = useCallback(async () => {
|
||||
await thread.stop();
|
||||
}, [thread]);
|
||||
@ -581,7 +595,8 @@ export default function ChatPage() {
|
||||
!isUploading &&
|
||||
!thread.isLoading &&
|
||||
!branchThread.isPending &&
|
||||
!hasOpenHumanInputCard;
|
||||
!hasOpenHumanInputCard &&
|
||||
!hasPendingApproval;
|
||||
|
||||
return (
|
||||
<ThreadContext.Provider value={{ thread, isMock }}>
|
||||
@ -675,12 +690,25 @@ export default function ChatPage() {
|
||||
loadMoreHistory={loadMoreHistory}
|
||||
isHistoryLoading={isHistoryLoading}
|
||||
tokenUsageInlineMode={tokenUsageInlineMode}
|
||||
approvalRequests={approvalRequests}
|
||||
approvalMetadataLoading={
|
||||
hasPendingApproval && approvalRequests.length === 0
|
||||
}
|
||||
approvalMetadataError={
|
||||
hasPendingApproval &&
|
||||
approvalRequests.length === 0 &&
|
||||
threadMetadata.isError
|
||||
}
|
||||
onReloadApprovalMetadata={() => threadMetadata.refetch()}
|
||||
waitingForInputRunIds={waitingForInputRunIds}
|
||||
onSubmitApproval={handleSubmitApproval}
|
||||
canRegenerate={
|
||||
!isNewThread &&
|
||||
!isMock &&
|
||||
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY !== "true" &&
|
||||
!isUploading &&
|
||||
!thread.isLoading
|
||||
!thread.isLoading &&
|
||||
!hasPendingApproval
|
||||
}
|
||||
onRegenerateMessage={handleRegenerate}
|
||||
canEdit={editBase && !hasGoal}
|
||||
@ -697,7 +725,8 @@ export default function ChatPage() {
|
||||
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY !== "true" &&
|
||||
!isUploading &&
|
||||
!thread.isLoading &&
|
||||
!branchThread.isPending
|
||||
!branchThread.isPending &&
|
||||
!hasPendingApproval
|
||||
}
|
||||
onBranchTurn={handleBranchTurn}
|
||||
/>
|
||||
@ -736,6 +765,12 @@ export default function ChatPage() {
|
||||
outcome={goalOutcome}
|
||||
isRunning={thread.isLoading}
|
||||
hasOpenHumanInputCard={hasOpenHumanInputCard}
|
||||
hasPendingApproval={hasPendingApproval}
|
||||
approvalMetadataError={
|
||||
hasPendingApproval &&
|
||||
approvalRequests.length === 0 &&
|
||||
threadMetadata.isError
|
||||
}
|
||||
/>
|
||||
{hasTodos && (
|
||||
<TodoList
|
||||
@ -785,7 +820,11 @@ export default function ChatPage() {
|
||||
selectorVisible && knowledgeScope ? (
|
||||
<KnowledgeScopeSelector
|
||||
agentName="lead_agent"
|
||||
disabled={thread.isLoading || isUploading}
|
||||
disabled={
|
||||
thread.isLoading ||
|
||||
isUploading ||
|
||||
hasPendingApproval
|
||||
}
|
||||
selection={knowledgeScope}
|
||||
onChange={setKnowledgeScope}
|
||||
/>
|
||||
@ -810,7 +849,8 @@ export default function ChatPage() {
|
||||
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" ||
|
||||
isUploading ||
|
||||
modelControl.busy ||
|
||||
(!isNewThread && isHistoryLoading)
|
||||
(!isNewThread && isHistoryLoading) ||
|
||||
hasPendingApproval
|
||||
}
|
||||
onContextChange={(context, options) => {
|
||||
if (options?.automatic)
|
||||
|
||||
@ -105,7 +105,10 @@ export function isGoalOutcomeCurrent(
|
||||
|
||||
/** A row of the stand-down copy in `inputBox.goalBar.next`. */
|
||||
export type GoalNextKey = keyof Translations["inputBox"]["goalBar"]["next"];
|
||||
export type GoalStopKey = Exclude<GoalNextKey, "paused">;
|
||||
export type GoalStopKey = Exclude<
|
||||
GoalNextKey,
|
||||
"paused" | "needsApproval" | "needsApprovalReload"
|
||||
>;
|
||||
/** Who acts next: amber "Stopped", primary "Waiting for you", muted otherwise. */
|
||||
export type GoalBarChip = "stopped" | "waitingForYou" | "waiting" | "unchecked";
|
||||
|
||||
@ -114,6 +117,11 @@ export type GoalBarView =
|
||||
| { kind: "set"; goal: GoalState }
|
||||
| { kind: "inProgress"; goal: GoalState }
|
||||
| { kind: "continuing"; goal: GoalState; count: number; max: number }
|
||||
| {
|
||||
kind: "awaitingApproval";
|
||||
goal: GoalState;
|
||||
metadataError?: boolean;
|
||||
}
|
||||
| {
|
||||
kind: "stopped";
|
||||
goal: GoalState;
|
||||
@ -194,12 +202,23 @@ export function describeGoalBar({
|
||||
outcome,
|
||||
isRunning,
|
||||
hasOpenHumanInputCard = false,
|
||||
hasPendingApproval = false,
|
||||
approvalMetadataError = false,
|
||||
}: {
|
||||
goal: GoalState | null;
|
||||
outcome: ThreadGoalOutcome | null;
|
||||
isRunning: boolean;
|
||||
hasOpenHumanInputCard?: boolean;
|
||||
hasPendingApproval?: boolean;
|
||||
approvalMetadataError?: boolean;
|
||||
}): GoalBarView {
|
||||
if (goal && hasPendingApproval) {
|
||||
return {
|
||||
kind: "awaitingApproval",
|
||||
goal,
|
||||
...(approvalMetadataError ? { metadataError: true } : {}),
|
||||
};
|
||||
}
|
||||
if (goal) {
|
||||
const evaluation = goal.last_evaluation;
|
||||
const code = evaluation?.stand_down_reason?.trim() ?? "";
|
||||
@ -314,6 +333,13 @@ export function describeGoalBarCopy(
|
||||
return copy;
|
||||
case "inProgress":
|
||||
return { ...copy, chip: bar.inProgress };
|
||||
case "awaitingApproval":
|
||||
return withDetail(
|
||||
bar.waitingApproval,
|
||||
view.metadataError
|
||||
? bar.next.needsApprovalReload
|
||||
: bar.next.needsApproval,
|
||||
);
|
||||
case "continuing": {
|
||||
const chip = fill(t.inputBox.goalContinuing, {
|
||||
count: view.count,
|
||||
@ -359,7 +385,12 @@ export function goalBarAnnounceKey(view: GoalBarView): string {
|
||||
if (view.kind === "met") {
|
||||
return `met:${goalOutcomeKey(view.outcome)}`;
|
||||
}
|
||||
const state = view.kind === "stopped" ? `stopped:${view.stop}` : view.kind;
|
||||
const state =
|
||||
view.kind === "stopped"
|
||||
? `stopped:${view.stop}`
|
||||
: view.kind === "awaitingApproval" && view.metadataError
|
||||
? "awaitingApproval:metadataError"
|
||||
: view.kind;
|
||||
return [state, view.goal.objective, view.goal.created_at].join("|");
|
||||
}
|
||||
|
||||
|
||||
@ -64,12 +64,16 @@ export function GoalStatus({
|
||||
outcome = null,
|
||||
isRunning = false,
|
||||
hasOpenHumanInputCard = false,
|
||||
hasPendingApproval = false,
|
||||
approvalMetadataError = false,
|
||||
}: {
|
||||
className?: string;
|
||||
goal: GoalState | null;
|
||||
outcome?: ThreadGoalOutcome | null;
|
||||
isRunning?: boolean;
|
||||
hasOpenHumanInputCard?: boolean;
|
||||
hasPendingApproval?: boolean;
|
||||
approvalMetadataError?: boolean;
|
||||
}) {
|
||||
const { locale, t } = useI18n();
|
||||
const bar = t.inputBox.goalBar;
|
||||
@ -78,6 +82,8 @@ export function GoalStatus({
|
||||
outcome,
|
||||
isRunning,
|
||||
hasOpenHumanInputCard,
|
||||
hasPendingApproval,
|
||||
approvalMetadataError,
|
||||
});
|
||||
const copy = describeGoalBarCopy(view, t);
|
||||
const announceKey = goalBarAnnounceKey(view);
|
||||
@ -103,16 +109,26 @@ export function GoalStatus({
|
||||
return null;
|
||||
}
|
||||
const met = view.kind === "met";
|
||||
const hasDetails = view.kind === "stopped" || view.kind === "paused" || met;
|
||||
const awaitingApproval = view.kind === "awaitingApproval";
|
||||
const hasDetails =
|
||||
view.kind === "stopped" ||
|
||||
view.kind === "paused" ||
|
||||
view.kind === "awaitingApproval" ||
|
||||
met;
|
||||
const note = "note" in view ? view.note : null;
|
||||
const rawCode = "rawCode" in view ? view.rawCode : null;
|
||||
const reliedOnAssumption = met && view.reliedOnAssumption;
|
||||
const chipStyle =
|
||||
view.kind === "stopped"
|
||||
? { Icon: STOP_ICONS[view.stop], tone: CHIP_TONES[view.chip] }
|
||||
: view.kind === "paused"
|
||||
? { Icon: PauseIcon, tone: CHIP_TONES.paused }
|
||||
: null;
|
||||
: view.kind === "awaitingApproval"
|
||||
? {
|
||||
Icon: MessageCircleQuestionIcon,
|
||||
tone: CHIP_TONES.waitingForYou,
|
||||
}
|
||||
: view.kind === "paused"
|
||||
? { Icon: PauseIcon, tone: CHIP_TONES.paused }
|
||||
: null;
|
||||
|
||||
return (
|
||||
<section
|
||||
@ -134,6 +150,8 @@ export function GoalStatus({
|
||||
>
|
||||
{met ? (
|
||||
<CircleCheckIcon className="size-4" />
|
||||
) : awaitingApproval ? (
|
||||
<MessageCircleQuestionIcon className="size-4" />
|
||||
) : (
|
||||
<TargetIcon className="size-4" />
|
||||
)}
|
||||
|
||||
@ -5,6 +5,8 @@ import {
|
||||
CheckIcon,
|
||||
Loader2Icon,
|
||||
MessageCircleQuestionMarkIcon,
|
||||
ShieldCheckIcon,
|
||||
XIcon,
|
||||
} from "lucide-react";
|
||||
import { useId, useMemo, useState, type KeyboardEvent } from "react";
|
||||
|
||||
@ -24,9 +26,14 @@ import {
|
||||
buildHumanInputFormSubmissionValue,
|
||||
buildHumanInputFormSummary,
|
||||
buildInitialHumanInputFormValues,
|
||||
createHitlApprovalResponses,
|
||||
createHumanInputOptionResponse,
|
||||
createHumanInputTextResponse,
|
||||
parseApprovalArgsSummary,
|
||||
readHumanInputFormValue,
|
||||
type HitlApprovalDecision,
|
||||
type HitlApprovalRequest,
|
||||
type HitlApprovalResponse,
|
||||
type HumanInputField,
|
||||
type HumanInputFormValue,
|
||||
type HumanInputOption,
|
||||
@ -38,6 +45,7 @@ import { cn } from "@/lib/utils";
|
||||
|
||||
import { MarkdownContent } from "./markdown-content";
|
||||
import { createMarkdownLinkComponent } from "./markdown-link";
|
||||
import { ToolCallDetails } from "./tool-call-details";
|
||||
|
||||
export type HumanInputSubmitResult = boolean | void;
|
||||
|
||||
@ -598,3 +606,270 @@ export function HumanInputCard({
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export function ToolApprovalCard({
|
||||
request,
|
||||
disabled = false,
|
||||
onSubmit,
|
||||
}: {
|
||||
request: HitlApprovalRequest;
|
||||
disabled?: boolean;
|
||||
onSubmit?: (
|
||||
request: HitlApprovalRequest,
|
||||
responses: HitlApprovalResponse[],
|
||||
) => HumanInputSubmitResult | Promise<HumanInputSubmitResult>;
|
||||
}) {
|
||||
const { t } = useI18n();
|
||||
const copy = t.humanInput.approval;
|
||||
const titleId = useId();
|
||||
const reasonId = useId();
|
||||
const [decisionState, setDecisionState] = useState<{
|
||||
requestId: string;
|
||||
values: Record<string, HitlApprovalDecision | undefined>;
|
||||
}>({ requestId: request.request_id, values: {} });
|
||||
const [rejectionReason, setRejectionReason] = useState("");
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
const [answeredRequestId, setAnsweredRequestId] = useState<string | null>(
|
||||
null,
|
||||
);
|
||||
const [error, setError] = useState("");
|
||||
const decisions =
|
||||
decisionState.requestId === request.request_id ? decisionState.values : {};
|
||||
const answered = answeredRequestId === request.request_id;
|
||||
const readOnly = !onSubmit;
|
||||
const isDisabled = disabled || submitting || answered || readOnly;
|
||||
const allowApprove = request.allowed_decisions.includes("approve");
|
||||
const allowReject = request.allowed_decisions.includes("reject");
|
||||
const responses = createHitlApprovalResponses(
|
||||
request,
|
||||
decisions,
|
||||
rejectionReason,
|
||||
);
|
||||
|
||||
const submitBatch = async (batch: HitlApprovalResponse[] | null) => {
|
||||
if (isDisabled || !onSubmit || batch?.length !== request.actions.length) {
|
||||
return;
|
||||
}
|
||||
setSubmitting(true);
|
||||
setError("");
|
||||
try {
|
||||
const result = await onSubmit(request, batch);
|
||||
if (result === false) {
|
||||
setError(copy.submitFailed);
|
||||
} else {
|
||||
setAnsweredRequestId(request.request_id);
|
||||
}
|
||||
} catch (cause) {
|
||||
setError(cause instanceof Error ? cause.message : String(cause));
|
||||
} finally {
|
||||
setSubmitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const setDecision = (toolCallId: string, decision: HitlApprovalDecision) => {
|
||||
setDecisionState((current) => ({
|
||||
requestId: request.request_id,
|
||||
values: {
|
||||
...(current.requestId === request.request_id ? current.values : {}),
|
||||
[toolCallId]: decision,
|
||||
},
|
||||
}));
|
||||
};
|
||||
|
||||
const submitAll = (decision: HitlApprovalDecision) =>
|
||||
submitBatch(
|
||||
createHitlApprovalResponses(
|
||||
request,
|
||||
Object.fromEntries(
|
||||
request.actions.map((action) => [action.tool_call_id, decision]),
|
||||
),
|
||||
rejectionReason,
|
||||
),
|
||||
);
|
||||
|
||||
return (
|
||||
<section
|
||||
aria-labelledby={titleId}
|
||||
className="border-border bg-card/70 text-card-foreground rounded-lg border p-4 shadow-xs"
|
||||
data-testid="tool-approval-card"
|
||||
>
|
||||
<div className="flex items-start gap-3">
|
||||
<div className="bg-primary/10 text-primary mt-0.5 flex size-8 shrink-0 items-center justify-center rounded-md">
|
||||
<ShieldCheckIcon className="size-4" />
|
||||
</div>
|
||||
<div className="min-w-0 flex-1 space-y-3">
|
||||
<div className="flex flex-wrap items-start justify-between gap-2">
|
||||
<div className="space-y-1">
|
||||
<h2 id={titleId} className="text-sm leading-5 font-medium">
|
||||
{copy.title}
|
||||
</h2>
|
||||
<p className="text-muted-foreground text-sm leading-6">
|
||||
{copy.description}
|
||||
</p>
|
||||
</div>
|
||||
{answered || submitting || readOnly ? (
|
||||
<Badge variant={answered ? "outline" : "secondary"}>
|
||||
{submitting ? (
|
||||
<Loader2Icon className="size-3 animate-spin" />
|
||||
) : null}
|
||||
{answered ? <CheckCircle2Icon className="size-3" /> : null}
|
||||
{answered
|
||||
? t.humanInput.answered
|
||||
: submitting
|
||||
? t.humanInput.pending
|
||||
: t.humanInput.readOnly}
|
||||
</Badge>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="space-y-3">
|
||||
{request.actions.map((action) => {
|
||||
const args = parseApprovalArgsSummary(action.args_summary);
|
||||
const oldString = args.old_str;
|
||||
const newString = args.new_str;
|
||||
const hasReplacementDiff =
|
||||
action.tool_name === "str_replace" &&
|
||||
typeof oldString === "string" &&
|
||||
typeof newString === "string";
|
||||
return (
|
||||
<div
|
||||
className="border-border/70 rounded-md border p-3"
|
||||
data-testid={`approval-action-${action.tool_call_id}`}
|
||||
key={action.tool_call_id}
|
||||
>
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<span className="font-mono text-sm font-medium">
|
||||
{action.tool_name}
|
||||
</span>
|
||||
<div className="flex gap-1" aria-label={action.tool_name}>
|
||||
{allowApprove ? (
|
||||
<Button
|
||||
aria-pressed={
|
||||
decisions[action.tool_call_id] === "approve"
|
||||
}
|
||||
disabled={isDisabled}
|
||||
onClick={() =>
|
||||
setDecision(action.tool_call_id, "approve")
|
||||
}
|
||||
size="sm"
|
||||
type="button"
|
||||
variant={
|
||||
decisions[action.tool_call_id] === "approve"
|
||||
? "secondary"
|
||||
: "outline"
|
||||
}
|
||||
>
|
||||
<CheckIcon className="size-3.5" />
|
||||
{copy.approve}
|
||||
</Button>
|
||||
) : null}
|
||||
{allowReject ? (
|
||||
<Button
|
||||
aria-pressed={
|
||||
decisions[action.tool_call_id] === "reject"
|
||||
}
|
||||
disabled={isDisabled}
|
||||
onClick={() =>
|
||||
setDecision(action.tool_call_id, "reject")
|
||||
}
|
||||
size="sm"
|
||||
type="button"
|
||||
variant={
|
||||
decisions[action.tool_call_id] === "reject"
|
||||
? "secondary"
|
||||
: "outline"
|
||||
}
|
||||
>
|
||||
<XIcon className="size-3.5" />
|
||||
{copy.reject}
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
<ToolCallDetails
|
||||
args={args}
|
||||
callId={action.tool_call_id}
|
||||
name={action.tool_name}
|
||||
pendingResultLabel={copy.waiting}
|
||||
>
|
||||
{hasReplacementDiff ? (
|
||||
<div className="grid gap-2" data-testid="approval-diff">
|
||||
<pre className="bg-destructive/10 max-h-64 overflow-auto rounded p-2 font-mono text-xs whitespace-pre-wrap">
|
||||
{oldString
|
||||
.split("\n")
|
||||
.map((line) => `- ${line}`)
|
||||
.join("\n")}
|
||||
</pre>
|
||||
<pre className="bg-primary/10 max-h-64 overflow-auto rounded p-2 font-mono text-xs whitespace-pre-wrap">
|
||||
{newString
|
||||
.split("\n")
|
||||
.map((line) => `+ ${line}`)
|
||||
.join("\n")}
|
||||
</pre>
|
||||
</div>
|
||||
) : null}
|
||||
</ToolCallDetails>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{allowReject ? (
|
||||
<div className="space-y-1.5">
|
||||
<label className="text-sm font-medium" htmlFor={reasonId}>
|
||||
{copy.reasonLabel}
|
||||
</label>
|
||||
<Textarea
|
||||
className="min-h-20 resize-y text-sm"
|
||||
disabled={isDisabled}
|
||||
id={reasonId}
|
||||
maxLength={2000}
|
||||
placeholder={copy.reasonPlaceholder}
|
||||
value={rejectionReason}
|
||||
onChange={(event) => setRejectionReason(event.target.value)}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{error ? (
|
||||
<p className="text-destructive text-sm" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
<div className="flex flex-wrap items-center justify-end gap-2">
|
||||
{allowReject ? (
|
||||
<Button
|
||||
disabled={isDisabled}
|
||||
onClick={() => void submitAll("reject")}
|
||||
type="button"
|
||||
variant="outline"
|
||||
>
|
||||
{copy.rejectAll}
|
||||
</Button>
|
||||
) : null}
|
||||
{allowApprove ? (
|
||||
<Button
|
||||
disabled={isDisabled}
|
||||
onClick={() => void submitAll("approve")}
|
||||
type="button"
|
||||
variant="secondary"
|
||||
>
|
||||
{copy.approveAll}
|
||||
</Button>
|
||||
) : null}
|
||||
<Button
|
||||
disabled={isDisabled || responses === null}
|
||||
onClick={() => void submitBatch(responses)}
|
||||
type="button"
|
||||
>
|
||||
{submitting ? (
|
||||
<Loader2Icon className="size-4 animate-spin" />
|
||||
) : null}
|
||||
{copy.submitDecisions}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@ import {
|
||||
NotebookPenIcon,
|
||||
SearchIcon,
|
||||
SquareTerminalIcon,
|
||||
ShieldCheckIcon,
|
||||
WrenchIcon,
|
||||
} from "lucide-react";
|
||||
import { memo, useEffect, useMemo, useState } from "react";
|
||||
@ -73,6 +74,7 @@ interface MessageGroupProps {
|
||||
showTokenDebugSummaries?: boolean;
|
||||
threadId?: string;
|
||||
toolArtifacts?: ArtifactEntry[];
|
||||
awaitingApproval?: boolean;
|
||||
}
|
||||
|
||||
function MessageGroupComponent({
|
||||
@ -85,6 +87,7 @@ function MessageGroupComponent({
|
||||
showTokenDebugSummaries = false,
|
||||
threadId,
|
||||
toolArtifacts,
|
||||
awaitingApproval = false,
|
||||
}: MessageGroupProps) {
|
||||
const { t } = useI18n();
|
||||
const markdownComponents = useMemo(
|
||||
@ -516,6 +519,16 @@ function MessageGroupComponent({
|
||||
|
||||
return (
|
||||
<>
|
||||
{awaitingApproval ? (
|
||||
<p
|
||||
className="text-muted-foreground flex items-center gap-1.5 text-sm"
|
||||
data-testid="awaiting-approval"
|
||||
role="status"
|
||||
>
|
||||
<ShieldCheckIcon className="size-4" />
|
||||
{t.humanInput.approval.waiting}
|
||||
</p>
|
||||
) : null}
|
||||
{processingPanel}
|
||||
{clarificationTextSteps.map((step) => (
|
||||
<div key={step.id} className="w-full">
|
||||
@ -547,6 +560,7 @@ function areMessageGroupPropsEqual(
|
||||
previous.className === next.className &&
|
||||
previous.durationSeconds === next.durationSeconds &&
|
||||
Boolean(previous.isLoading) === Boolean(next.isLoading) &&
|
||||
Boolean(previous.awaitingApproval) === Boolean(next.awaitingApproval) &&
|
||||
Boolean(previous.deferBrowserPreviews) ===
|
||||
Boolean(next.deferBrowserPreviews) &&
|
||||
Boolean(previous.showTokenDebugSummaries) ===
|
||||
|
||||
@ -7,6 +7,7 @@ import {
|
||||
MessageCircleIcon,
|
||||
MessageSquarePlusIcon,
|
||||
RefreshCcwIcon,
|
||||
ShieldCheckIcon,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
useCallback,
|
||||
@ -50,6 +51,8 @@ import {
|
||||
deriveHumanInputThreadState,
|
||||
extractHumanInputRequest,
|
||||
shouldClearPendingHumanInputOnThreadError,
|
||||
type HitlApprovalRequest,
|
||||
type HitlApprovalResponse,
|
||||
type HumanInputRequest,
|
||||
type HumanInputResponse,
|
||||
} from "@/core/messages/human-input";
|
||||
@ -113,6 +116,7 @@ import { Tooltip } from "../tooltip";
|
||||
import { ConversationOutline } from "./conversation-outline";
|
||||
import {
|
||||
HumanInputCard,
|
||||
ToolApprovalCard,
|
||||
type HumanInputSubmitResult,
|
||||
} from "./human-input-card";
|
||||
import { MarkdownContent } from "./markdown-content";
|
||||
@ -334,6 +338,12 @@ export function MessageList({
|
||||
onRegenerateMessage,
|
||||
onEditAndRegenerateMessage,
|
||||
onSubmitHumanInput,
|
||||
approvalRequests = [],
|
||||
approvalMetadataLoading = false,
|
||||
approvalMetadataError = false,
|
||||
onReloadApprovalMetadata,
|
||||
waitingForInputRunIds = new Set<string>(),
|
||||
onSubmitApproval,
|
||||
onBranchTurn,
|
||||
canRegenerate = false,
|
||||
activeRunId,
|
||||
@ -370,6 +380,15 @@ export function MessageList({
|
||||
request: HumanInputRequest,
|
||||
response: HumanInputResponse,
|
||||
) => HumanInputSubmitResult | Promise<HumanInputSubmitResult>;
|
||||
approvalRequests?: HitlApprovalRequest[];
|
||||
approvalMetadataLoading?: boolean;
|
||||
approvalMetadataError?: boolean;
|
||||
onReloadApprovalMetadata?: () => unknown;
|
||||
waitingForInputRunIds?: ReadonlySet<string>;
|
||||
onSubmitApproval?: (
|
||||
request: HitlApprovalRequest,
|
||||
responses: HitlApprovalResponse[],
|
||||
) => HumanInputSubmitResult | Promise<HumanInputSubmitResult>;
|
||||
onBranchTurn?: (
|
||||
messageId: string,
|
||||
messageIds: string[],
|
||||
@ -1450,11 +1469,29 @@ export function MessageList({
|
||||
),
|
||||
group.id !== undefined &&
|
||||
branchableAssistantGroupIds.has(group.id),
|
||||
group.id === latestAssistantGroupId,
|
||||
!group.messages.some(
|
||||
(message) =>
|
||||
message.type === "ai" &&
|
||||
"run_id" in message &&
|
||||
typeof message.run_id === "string" &&
|
||||
waitingForInputRunIds.has(message.run_id),
|
||||
) && group.id === latestAssistantGroupId,
|
||||
sidecarSurface
|
||||
? undefined
|
||||
: skillUsageByGroupIndex.get(groupIndex),
|
||||
)}
|
||||
{group.type === "assistant" &&
|
||||
approvalRequests.length > 0 &&
|
||||
groupIndex === lastGroupIndex ? (
|
||||
<p
|
||||
className="text-muted-foreground flex items-center gap-1.5 text-sm"
|
||||
data-testid="awaiting-approval"
|
||||
role="status"
|
||||
>
|
||||
<ShieldCheckIcon className="size-4" />
|
||||
{t.humanInput.approval.waiting}
|
||||
</p>
|
||||
) : null}
|
||||
</div>,
|
||||
inlineDurationRunIds,
|
||||
);
|
||||
@ -1707,6 +1744,10 @@ export function MessageList({
|
||||
)}
|
||||
showTokenDebugSummaries={showTokenDebugSummaries}
|
||||
toolArtifacts={thread.values?.tool_artifacts}
|
||||
awaitingApproval={
|
||||
approvalRequests.length > 0 &&
|
||||
groupIndex === lastGroupIndex
|
||||
}
|
||||
/>
|
||||
</MessageArtifactPathsContext.Provider>
|
||||
{renderTokenUsage({
|
||||
@ -1719,6 +1760,47 @@ export function MessageList({
|
||||
);
|
||||
}}
|
||||
/>
|
||||
{approvalRequests.map((request) => (
|
||||
<ToolApprovalCard
|
||||
disabled={thread.isLoading}
|
||||
key={request.request_id}
|
||||
request={request}
|
||||
onSubmit={onSubmitApproval}
|
||||
/>
|
||||
))}
|
||||
{approvalMetadataError && approvalRequests.length === 0 ? (
|
||||
<div
|
||||
className="border-destructive/30 bg-destructive/5 space-y-3 rounded-lg border p-4"
|
||||
data-testid="approval-metadata-error"
|
||||
>
|
||||
<p className="text-destructive text-sm" role="alert">
|
||||
{t.humanInput.approval.loadFailed}
|
||||
</p>
|
||||
{onReloadApprovalMetadata ? (
|
||||
<Button
|
||||
size="sm"
|
||||
type="button"
|
||||
variant="outline"
|
||||
onClick={() => void onReloadApprovalMetadata()}
|
||||
>
|
||||
<RefreshCcwIcon className="size-4" />
|
||||
{t.humanInput.approval.reload}
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
{approvalMetadataLoading &&
|
||||
!approvalMetadataError &&
|
||||
approvalRequests.length === 0 ? (
|
||||
<p
|
||||
className="text-muted-foreground flex items-center gap-1.5 text-sm"
|
||||
data-testid="approval-metadata-loading"
|
||||
role="status"
|
||||
>
|
||||
<ShieldCheckIcon className="size-4" />
|
||||
{t.humanInput.approval.loading}
|
||||
</p>
|
||||
) : null}
|
||||
{renderTaskEventLines(placedTaskEvents?.tail)}
|
||||
{thread.isLoading && !hasActiveAssistantText && (
|
||||
<div className="w-full">
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
"use client";
|
||||
|
||||
import type { Message } from "@langchain/langgraph-sdk";
|
||||
import { useEffect, useId, useMemo, useState } from "react";
|
||||
import { useEffect, useId, useMemo, useState, type ReactNode } from "react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { writeTextToClipboard } from "@/core/clipboard";
|
||||
@ -14,11 +14,15 @@ export function ToolCallDetails({
|
||||
callId,
|
||||
args,
|
||||
resultMessage,
|
||||
pendingResultLabel,
|
||||
children,
|
||||
}: {
|
||||
name: string;
|
||||
callId?: string;
|
||||
args: Record<string, unknown>;
|
||||
resultMessage?: Extract<Message, { type: "tool" }>;
|
||||
pendingResultLabel?: string;
|
||||
children?: ReactNode;
|
||||
}) {
|
||||
const { t } = useI18n();
|
||||
const [open, setOpen] = useState(false);
|
||||
@ -41,6 +45,7 @@ export function ToolCallDetails({
|
||||
<Payload label={t.toolCalls.toolName} value={name} />
|
||||
<Payload label={t.toolCalls.callId} value={callId ?? "—"} />
|
||||
<Payload label={t.toolCalls.input} value={args} />
|
||||
{children}
|
||||
{resultMessage ? (
|
||||
<Payload
|
||||
label={
|
||||
@ -51,7 +56,7 @@ export function ToolCallDetails({
|
||||
value={resultMessage.content}
|
||||
/>
|
||||
) : (
|
||||
<p>{t.toolCalls.noResult}</p>
|
||||
<p>{pendingResultLabel ?? t.toolCalls.noResult}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@ -74,6 +74,7 @@ import {
|
||||
import { createSidecarThread } from "@/core/sidecar/api";
|
||||
import {
|
||||
useDeleteThread,
|
||||
useThreadMetadata,
|
||||
useThreadStream,
|
||||
type ThreadStreamOptions,
|
||||
} from "@/core/threads/hooks";
|
||||
@ -160,6 +161,11 @@ export function SidecarPanel({ className }: { className?: string }) {
|
||||
);
|
||||
}, [models, sidecar.context.model_name]);
|
||||
|
||||
const threadMetadata = useThreadMetadata(sidecar.sidecarThreadId, {
|
||||
enabled: !sidecar.isMock,
|
||||
isMock: sidecar.isMock,
|
||||
});
|
||||
|
||||
const {
|
||||
thread,
|
||||
sendMessage,
|
||||
@ -167,6 +173,10 @@ export function SidecarPanel({ className }: { className?: string }) {
|
||||
isHistoryLoading,
|
||||
hasMoreHistory,
|
||||
loadMoreHistory,
|
||||
approvalRequests,
|
||||
hasPendingApproval,
|
||||
submitHitlApproval,
|
||||
waitingForInputRunIds,
|
||||
} = useThreadStream({
|
||||
threadId: sidecar.sidecarThreadId ?? undefined,
|
||||
displayThreadId: sidecar.sidecarThreadId ?? undefined,
|
||||
@ -175,6 +185,9 @@ export function SidecarPanel({ className }: { className?: string }) {
|
||||
onStart: (createdThreadId) => {
|
||||
sidecar.setSidecarThreadId(createdThreadId);
|
||||
},
|
||||
restInterrupts: threadMetadata.data?.interrupts,
|
||||
restInterruptsLoading: threadMetadata.isLoading,
|
||||
restInterruptsError: threadMetadata.isError,
|
||||
});
|
||||
|
||||
const referenceCountLabel = useMemo(() => {
|
||||
@ -203,6 +216,7 @@ export function SidecarPanel({ className }: { className?: string }) {
|
||||
isUploading ||
|
||||
(hasSidecarThread && isHistoryLoading) ||
|
||||
(sidecar.isMock ?? false) ||
|
||||
hasPendingApproval ||
|
||||
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true";
|
||||
|
||||
useEffect(() => {
|
||||
@ -393,6 +407,17 @@ export function SidecarPanel({ className }: { className?: string }) {
|
||||
[sidecar, submitToSidecarThread],
|
||||
);
|
||||
|
||||
const handleSubmitApproval = useCallback(
|
||||
(
|
||||
request: (typeof approvalRequests)[number],
|
||||
responses: Parameters<typeof submitHitlApproval>[2],
|
||||
) =>
|
||||
sidecar.sidecarThreadId
|
||||
? submitHitlApproval(sidecar.sidecarThreadId, request, responses)
|
||||
: false,
|
||||
[sidecar.sidecarThreadId, submitHitlApproval],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!queuedSubmit || !sidecar.sidecarThreadId || thread.isLoading) {
|
||||
return;
|
||||
@ -574,6 +599,18 @@ export function SidecarPanel({ className }: { className?: string }) {
|
||||
sidecarSurface
|
||||
initialScroll="instant"
|
||||
resizeScroll="instant"
|
||||
approvalRequests={approvalRequests}
|
||||
approvalMetadataLoading={
|
||||
hasPendingApproval && approvalRequests.length === 0
|
||||
}
|
||||
approvalMetadataError={
|
||||
hasPendingApproval &&
|
||||
approvalRequests.length === 0 &&
|
||||
threadMetadata.isError
|
||||
}
|
||||
onReloadApprovalMetadata={() => threadMetadata.refetch()}
|
||||
waitingForInputRunIds={waitingForInputRunIds}
|
||||
onSubmitApproval={handleSubmitApproval}
|
||||
onSubmitHumanInput={
|
||||
sidecar.isMock || env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true"
|
||||
? undefined
|
||||
|
||||
@ -68,6 +68,7 @@ const TERMINAL_RUN_STATUSES = new Set([
|
||||
"error",
|
||||
"timeout",
|
||||
"interrupted",
|
||||
"waiting_for_input",
|
||||
]);
|
||||
|
||||
// This is a rejoin budget: the original stream is not counted, so exhausting
|
||||
|
||||
@ -623,6 +623,7 @@ export const enUS: Translations = {
|
||||
waitingForYou: "Waiting for you",
|
||||
waiting: "Waiting",
|
||||
paused: "Paused",
|
||||
waitingApproval: "Waiting for approval",
|
||||
autoContinuedOnce: "Auto-continued once",
|
||||
autoContinuedMany: "Auto-continued {count} times",
|
||||
details: "Details",
|
||||
@ -654,6 +655,9 @@ export const enUS: Translations = {
|
||||
"Goal check: the run did not finish the work. Check the reply for errors, then send a message to retry.",
|
||||
needsInputCard: "Answer the question above to continue.",
|
||||
needsInputReply: "Reply with the missing details to continue.",
|
||||
needsApproval: "Review the tool approval above to continue.",
|
||||
needsApprovalReload:
|
||||
"Approval details failed to load. Use the reload-error panel above to try again.",
|
||||
external:
|
||||
"Goal check: waiting on something external. Send a message when it's ready.",
|
||||
unchecked: "{reason}. It's checked again after your next message.",
|
||||
@ -1922,6 +1926,31 @@ export const enUS: Translations = {
|
||||
requiredA11yLabel: "required",
|
||||
selectPlaceholder: "Select...",
|
||||
answeredValue: (value: string) => `Answered: ${value}`,
|
||||
approval: {
|
||||
title: "Tool approval required",
|
||||
description:
|
||||
"Review every action before continuing. Long values may be truncated for display, but omitted content still executes in full. Approving will attempt to execute the tool; other execution checks still apply.",
|
||||
waiting: "Waiting for approval",
|
||||
loading: "Loading approval details...",
|
||||
loadFailed:
|
||||
"Approval details could not be loaded. This conversation is still waiting for approval; reload the details before continuing.",
|
||||
reload: "Reload approval details",
|
||||
approve: "Approve",
|
||||
reject: "Reject",
|
||||
approveAll: "Approve all",
|
||||
rejectAll: "Reject all and finish",
|
||||
submitDecisions: "Submit decisions",
|
||||
reasonLabel: "Rejection reason (optional)",
|
||||
reasonPlaceholder: "Tell the agent why it should not run these tools...",
|
||||
submitFailed:
|
||||
"The approval could not be submitted. Review your decisions and try again.",
|
||||
submitTimeout:
|
||||
"The approval submission timed out and was stopped. Review your decisions and try again.",
|
||||
featureDisabled:
|
||||
"Tool approval is now disabled. You can still reject all and finish safely.",
|
||||
noPending:
|
||||
"This approval is no longer pending. Reload the conversation before retrying.",
|
||||
},
|
||||
},
|
||||
|
||||
// Subtasks
|
||||
|
||||
@ -491,6 +491,7 @@ export interface Translations {
|
||||
waitingForYou: string;
|
||||
waiting: string;
|
||||
paused: string;
|
||||
waitingApproval: string;
|
||||
autoContinuedOnce: string;
|
||||
autoContinuedMany: string;
|
||||
details: string;
|
||||
@ -512,6 +513,8 @@ export interface Translations {
|
||||
runFailed: string;
|
||||
needsInputCard: string;
|
||||
needsInputReply: string;
|
||||
needsApproval: string;
|
||||
needsApprovalReload: string;
|
||||
external: string;
|
||||
unchecked: string;
|
||||
unknown: string;
|
||||
@ -1591,6 +1594,25 @@ export interface Translations {
|
||||
requiredA11yLabel: string;
|
||||
selectPlaceholder: string;
|
||||
answeredValue: (value: string) => string;
|
||||
approval: {
|
||||
title: string;
|
||||
description: string;
|
||||
waiting: string;
|
||||
loading: string;
|
||||
loadFailed: string;
|
||||
reload: string;
|
||||
approve: string;
|
||||
reject: string;
|
||||
approveAll: string;
|
||||
rejectAll: string;
|
||||
submitDecisions: string;
|
||||
reasonLabel: string;
|
||||
reasonPlaceholder: string;
|
||||
submitFailed: string;
|
||||
submitTimeout: string;
|
||||
featureDisabled: string;
|
||||
noPending: string;
|
||||
};
|
||||
};
|
||||
|
||||
// Uploads
|
||||
|
||||
@ -585,6 +585,7 @@ export const zhCN: Translations = {
|
||||
waitingForYou: "等你回复",
|
||||
waiting: "等待中",
|
||||
paused: "已暂停",
|
||||
waitingApproval: "等待审批",
|
||||
autoContinuedOnce: "自动续跑了 1 次",
|
||||
autoContinuedMany: "自动续跑了 {count} 次",
|
||||
details: "详情",
|
||||
@ -614,6 +615,9 @@ export const zhCN: Translations = {
|
||||
"目标检查:这次运行没有完成任务。先看看回复里的错误,再发送消息重试。",
|
||||
needsInputCard: "回答上面的问题即可继续。",
|
||||
needsInputReply: "回复补充缺少的信息即可继续。",
|
||||
needsApproval: "查看并处理上面的工具审批即可继续。",
|
||||
needsApprovalReload:
|
||||
"审批详情加载失败,请使用上方的重新加载错误面板重试。",
|
||||
external: "目标检查:正在等待外部条件,就绪后发送消息即可继续。",
|
||||
unchecked: "{reason}。下次发消息后会重新检查。",
|
||||
unknown: "发送消息即可继续。",
|
||||
@ -1821,6 +1825,27 @@ export const zhCN: Translations = {
|
||||
requiredA11yLabel: "必填",
|
||||
selectPlaceholder: "请选择...",
|
||||
answeredValue: (value: string) => `已回答:${value}`,
|
||||
approval: {
|
||||
title: "工具调用需要审批",
|
||||
description:
|
||||
"继续前请检查每个操作。较长的值可能在显示时被截断,但省略的内容仍会完整参与执行。批准后将尝试执行工具,其他执行检查仍然有效。",
|
||||
waiting: "等待审批",
|
||||
loading: "正在加载审批详情...",
|
||||
loadFailed:
|
||||
"无法加载审批详情。此对话仍在等待审批,请重新加载详情后再继续。",
|
||||
reload: "重新加载审批详情",
|
||||
approve: "批准",
|
||||
reject: "拒绝",
|
||||
approveAll: "全部批准",
|
||||
rejectAll: "全部拒绝并结束",
|
||||
submitDecisions: "提交所选决定",
|
||||
reasonLabel: "拒绝理由(可选)",
|
||||
reasonPlaceholder: "告诉智能体不应执行这些工具的原因……",
|
||||
submitFailed: "审批提交失败。请检查所选决定后重试。",
|
||||
submitTimeout: "审批提交超时,已停止本次提交。请检查所选决定后重试。",
|
||||
featureDisabled: "工具审批现已关闭,仍可安全地全部拒绝并结束。",
|
||||
noPending: "该审批已不再挂起,请重新加载对话后重试。",
|
||||
},
|
||||
},
|
||||
|
||||
uploads: {
|
||||
|
||||
@ -66,6 +66,37 @@ export type HumanInputResponse =
|
||||
value: string;
|
||||
};
|
||||
|
||||
export type HitlApprovalDecision = "approve" | "reject";
|
||||
|
||||
export type HitlApprovalAction = {
|
||||
tool_call_id: string;
|
||||
tool_name: string;
|
||||
args_summary: string | Record<string, unknown>;
|
||||
request_id: string;
|
||||
};
|
||||
|
||||
export type HitlApprovalRequest = {
|
||||
version: 1;
|
||||
kind: "hitl_approval";
|
||||
request_id: string;
|
||||
actions: HitlApprovalAction[];
|
||||
allowed_decisions: HitlApprovalDecision[];
|
||||
pending_tool_call_ids: string[];
|
||||
};
|
||||
|
||||
export type HitlApprovalResponse = {
|
||||
version: 1;
|
||||
kind: "human_input_response";
|
||||
source: "hitl_approval";
|
||||
request_id: string;
|
||||
tool_call_id: string;
|
||||
response_kind: "option";
|
||||
option_id: HitlApprovalDecision;
|
||||
value: string;
|
||||
};
|
||||
|
||||
export type HitlApprovalResume = { responses: HitlApprovalResponse[] };
|
||||
|
||||
export type HumanInputThreadState = {
|
||||
answeredResponses: Map<string, HumanInputResponse>;
|
||||
latestOpenRequestId: string | null;
|
||||
@ -95,6 +126,240 @@ function isNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function isApprovalDecision(value: unknown): value is HitlApprovalDecision {
|
||||
return value === "approve" || value === "reject";
|
||||
}
|
||||
|
||||
/** Parse only the approval interrupt payload; static breakpoints stay invisible. */
|
||||
export function parseHitlApprovalRequest(
|
||||
value: unknown,
|
||||
): HitlApprovalRequest | null {
|
||||
if (
|
||||
!isRecord(value) ||
|
||||
value.kind !== "hitl_approval" ||
|
||||
value.version !== 1 ||
|
||||
!isNonEmptyString(value.request_id) ||
|
||||
!Array.isArray(value.actions) ||
|
||||
value.actions.length === 0 ||
|
||||
!Array.isArray(value.allowed_decisions) ||
|
||||
!Array.isArray(value.pending_tool_call_ids)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const allowedDecisions = value.allowed_decisions.filter(isApprovalDecision);
|
||||
if (
|
||||
allowedDecisions.length === 0 ||
|
||||
allowedDecisions.length !== value.allowed_decisions.length ||
|
||||
new Set(allowedDecisions).size !== allowedDecisions.length
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const actions: HitlApprovalAction[] = [];
|
||||
const toolCallIds = new Set<string>();
|
||||
const requestIds = new Set<string>();
|
||||
for (const action of value.actions) {
|
||||
if (
|
||||
!isRecord(action) ||
|
||||
!isNonEmptyString(action.tool_call_id) ||
|
||||
!isNonEmptyString(action.tool_name) ||
|
||||
(typeof action.args_summary !== "string" &&
|
||||
!isRecord(action.args_summary)) ||
|
||||
!isNonEmptyString(action.request_id) ||
|
||||
toolCallIds.has(action.tool_call_id) ||
|
||||
requestIds.has(action.request_id)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
toolCallIds.add(action.tool_call_id);
|
||||
requestIds.add(action.request_id);
|
||||
actions.push({
|
||||
tool_call_id: action.tool_call_id,
|
||||
tool_name: action.tool_name,
|
||||
args_summary: action.args_summary,
|
||||
request_id: action.request_id,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
value.pending_tool_call_ids.length !== actions.length ||
|
||||
value.pending_tool_call_ids.some(
|
||||
(id) => typeof id !== "string" || !toolCallIds.has(id),
|
||||
) ||
|
||||
new Set(value.pending_tool_call_ids).size !== actions.length
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
version: 1,
|
||||
kind: "hitl_approval",
|
||||
request_id: value.request_id,
|
||||
actions,
|
||||
allowed_decisions: allowedDecisions,
|
||||
pending_tool_call_ids: [...value.pending_tool_call_ids] as string[],
|
||||
};
|
||||
}
|
||||
|
||||
/** Accept SDK live Interrupt objects and REST Record<task, Interrupt[]> shapes. */
|
||||
export function extractHitlApprovalRequests(source: unknown) {
|
||||
const candidates: unknown[] = Array.isArray(source) ? source : [source];
|
||||
const requests: HitlApprovalRequest[] = [];
|
||||
const seen = new Set<string>();
|
||||
const append = (candidate: unknown): void => {
|
||||
if (Array.isArray(candidate)) {
|
||||
candidate.forEach(append);
|
||||
return;
|
||||
}
|
||||
if (!isRecord(candidate)) return;
|
||||
const value = "value" in candidate ? candidate.value : candidate;
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(append);
|
||||
return;
|
||||
}
|
||||
const request = parseHitlApprovalRequest(value);
|
||||
if (request && !seen.has(request.request_id)) {
|
||||
seen.add(request.request_id);
|
||||
requests.push(request);
|
||||
}
|
||||
};
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
isRecord(candidate) &&
|
||||
candidate.kind !== "hitl_approval" &&
|
||||
!("value" in candidate)
|
||||
) {
|
||||
for (const interrupts of Object.values(candidate)) {
|
||||
if (Array.isArray(interrupts)) interrupts.forEach(append);
|
||||
}
|
||||
} else {
|
||||
append(candidate);
|
||||
}
|
||||
}
|
||||
return requests;
|
||||
}
|
||||
|
||||
/** Build one atomic resume batch, or null while any action is undecided. */
|
||||
export function createHitlApprovalResponses(
|
||||
request: HitlApprovalRequest,
|
||||
decisions: Readonly<Record<string, HitlApprovalDecision | undefined>>,
|
||||
rejectionReason = "",
|
||||
): HitlApprovalResponse[] | null {
|
||||
const responses: HitlApprovalResponse[] = [];
|
||||
for (const action of request.actions) {
|
||||
const decision = decisions[action.tool_call_id];
|
||||
if (!decision || !request.allowed_decisions.includes(decision)) return null;
|
||||
responses.push({
|
||||
version: 1,
|
||||
kind: "human_input_response",
|
||||
source: "hitl_approval",
|
||||
request_id: action.request_id,
|
||||
tool_call_id: action.tool_call_id,
|
||||
response_kind: "option",
|
||||
option_id: decision,
|
||||
value: decision === "reject" ? rejectionReason : "",
|
||||
});
|
||||
}
|
||||
return responses;
|
||||
}
|
||||
|
||||
export function parseApprovalArgsSummary(summary: unknown) {
|
||||
if (isRecord(summary)) return summary;
|
||||
if (typeof summary !== "string") return { summary: String(summary) };
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(summary);
|
||||
if (isRecord(parsed)) return parsed;
|
||||
} catch {
|
||||
// The backend serializes its flat string map with Python repr.
|
||||
}
|
||||
return parsePythonStringDict(summary) ?? { summary };
|
||||
}
|
||||
|
||||
const PYTHON_STRING_ESCAPES: Record<string, string> = {
|
||||
"\\": "\\",
|
||||
"'": "'",
|
||||
'"': '"',
|
||||
a: "\u0007",
|
||||
b: "\b",
|
||||
f: "\f",
|
||||
n: "\n",
|
||||
r: "\r",
|
||||
t: "\t",
|
||||
v: "\v",
|
||||
"\n": "",
|
||||
};
|
||||
function readPythonQuotedString(text: string, start: number) {
|
||||
const quote = text[start];
|
||||
if (quote !== "'" && quote !== '"') return null;
|
||||
let value = "";
|
||||
let index = start + 1;
|
||||
while (index < text.length) {
|
||||
const char = text[index++]!;
|
||||
if (char === quote) return { value, end: index };
|
||||
if (char !== "\\") {
|
||||
value += char;
|
||||
continue;
|
||||
}
|
||||
if (index >= text.length) return null;
|
||||
const escape = text[index++]!;
|
||||
if (escape in PYTHON_STRING_ESCAPES) {
|
||||
value += PYTHON_STRING_ESCAPES[escape];
|
||||
continue;
|
||||
}
|
||||
const hexLength =
|
||||
escape === "x" ? 2 : escape === "u" ? 4 : escape === "U" ? 8 : 0;
|
||||
if (hexLength > 0) {
|
||||
const hex = text.slice(index, index + hexLength);
|
||||
if (hex.length !== hexLength || !/^[0-9a-fA-F]+$/.test(hex)) return null;
|
||||
const codePoint = Number.parseInt(hex, 16);
|
||||
if (codePoint > 0x10ffff) return null;
|
||||
value += String.fromCodePoint(codePoint);
|
||||
index += hexLength;
|
||||
continue;
|
||||
}
|
||||
if (/^[0-7]$/.test(escape)) {
|
||||
let octal = escape;
|
||||
while (octal.length < 3 && /^[0-7]$/.test(text[index] ?? "")) {
|
||||
octal += text[index++]!;
|
||||
}
|
||||
value += String.fromCharCode(Number.parseInt(octal, 8));
|
||||
continue;
|
||||
}
|
||||
value += escape;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parsePythonStringDict(text: string) {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) return null;
|
||||
const result: Record<string, unknown> = Object.create(null);
|
||||
let index = 1;
|
||||
const skipSpace = () => {
|
||||
while (/\s/.test(trimmed[index] ?? "")) index += 1;
|
||||
};
|
||||
while (index < trimmed.length - 1) {
|
||||
skipSpace();
|
||||
if (trimmed[index] === "}") return result;
|
||||
const key = readPythonQuotedString(trimmed, index);
|
||||
if (!key) return null;
|
||||
index = key.end;
|
||||
skipSpace();
|
||||
if (trimmed[index++] !== ":") return null;
|
||||
skipSpace();
|
||||
const value = readPythonQuotedString(trimmed, index);
|
||||
if (!value) return null;
|
||||
index = value.end;
|
||||
result[key.value] = value.value;
|
||||
skipSpace();
|
||||
if (trimmed[index] === "}") return result;
|
||||
if (trimmed[index++] !== ",") return null;
|
||||
}
|
||||
return index === trimmed.length - 1 ? result : null;
|
||||
}
|
||||
|
||||
function isHumanInputMode(value: unknown): value is HumanInputMode {
|
||||
return (
|
||||
value === "free_text" ||
|
||||
|
||||
@ -30,6 +30,11 @@ import { getBackendBaseURL } from "../config";
|
||||
import { useRunInputsEnabled } from "../features";
|
||||
import { useI18n } from "../i18n/hooks";
|
||||
import type { Locale } from "../i18n/locale";
|
||||
import {
|
||||
extractHitlApprovalRequests,
|
||||
type HitlApprovalRequest,
|
||||
type HitlApprovalResponse,
|
||||
} from "../messages/human-input";
|
||||
import { getMessageRunId, parseRunStartTime } from "../messages/run-duration";
|
||||
import { isErrorToolMessage } from "../messages/tool-error";
|
||||
import { indexToolResults } from "../messages/tool-result";
|
||||
@ -89,6 +94,7 @@ import {
|
||||
import type {
|
||||
AgentThread,
|
||||
AgentThreadState,
|
||||
AppRun,
|
||||
RunMessage,
|
||||
ThreadTokenUsageResponse,
|
||||
} from "./types";
|
||||
@ -112,6 +118,10 @@ export type ThreadStreamOptions = {
|
||||
onSend?: (threadId: string) => void;
|
||||
onStart?: (threadId: string, runId: string) => void;
|
||||
onFinish?: (state: AgentThreadState) => void;
|
||||
/** REST detail fallback for reloads and other tabs; live uses thread.interrupt. */
|
||||
restInterrupts?: unknown;
|
||||
restInterruptsLoading?: boolean;
|
||||
restInterruptsError?: boolean;
|
||||
};
|
||||
|
||||
type SendMessageOptions = {
|
||||
@ -138,6 +148,80 @@ type SendMessageOptions = {
|
||||
onSent?: () => void;
|
||||
};
|
||||
|
||||
export const THREAD_RUN_RECURSION_LIMIT = 1000;
|
||||
export const HITL_APPROVAL_ACCEPTANCE_TIMEOUT_MS = 10_000;
|
||||
|
||||
type PausedRunContext = {
|
||||
extraContext?: Record<string, unknown>;
|
||||
conversationReferences?: string[];
|
||||
};
|
||||
|
||||
type PendingApprovalSubmission = {
|
||||
threadId: string;
|
||||
requestId: string;
|
||||
resolve: (accepted: boolean) => void;
|
||||
timeout: number | undefined;
|
||||
};
|
||||
|
||||
export function deriveHitlApprovalState({
|
||||
liveInterrupt,
|
||||
restInterrupts,
|
||||
historyLoading,
|
||||
restInterruptsError = false,
|
||||
next,
|
||||
answeredRequestIds,
|
||||
}: {
|
||||
liveInterrupt: unknown;
|
||||
restInterrupts?: unknown;
|
||||
historyLoading: boolean;
|
||||
restInterruptsError?: boolean;
|
||||
next: readonly string[];
|
||||
answeredRequestIds: ReadonlySet<string>;
|
||||
}) {
|
||||
const requests = extractHitlApprovalRequests([
|
||||
...extractHitlApprovalRequests(liveInterrupt),
|
||||
...extractHitlApprovalRequests(restInterrupts),
|
||||
]);
|
||||
const pendingRequests = requests.filter(
|
||||
(request) => !answeredRequestIds.has(request.request_id),
|
||||
);
|
||||
return {
|
||||
requests: pendingRequests,
|
||||
hasPendingApproval:
|
||||
pendingRequests.length > 0 ||
|
||||
((historyLoading || restInterruptsError) &&
|
||||
next.includes("HitlApprovalMiddleware.after_model")),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildHitlApprovalSubmitOptions({
|
||||
threadId,
|
||||
settings,
|
||||
responses,
|
||||
extraContext,
|
||||
conversationReferences,
|
||||
}: {
|
||||
threadId: string;
|
||||
settings: LocalSettings["context"];
|
||||
responses: HitlApprovalResponse[];
|
||||
} & PausedRunContext) {
|
||||
return {
|
||||
threadId,
|
||||
// The SDK otherwise injects the pre-interrupt history head in this window.
|
||||
checkpoint: null,
|
||||
command: { resume: { responses } },
|
||||
streamMode: ["values" as const],
|
||||
streamResumable: true,
|
||||
config: { recursion_limit: THREAD_RUN_RECURSION_LIMIT },
|
||||
context: buildRunContext({
|
||||
settings,
|
||||
threadId,
|
||||
extraContext,
|
||||
conversationReferences,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
type ThreadDeleteClient = {
|
||||
threads: {
|
||||
delete: (threadId: string) => Promise<unknown>;
|
||||
@ -1986,6 +2070,9 @@ export function useThreadStream({
|
||||
onSend,
|
||||
onStart,
|
||||
onFinish,
|
||||
restInterrupts,
|
||||
restInterruptsLoading = false,
|
||||
restInterruptsError = false,
|
||||
}: ThreadStreamOptions) {
|
||||
const { t, locale } = useI18n();
|
||||
const currentViewThreadId = displayThreadId ?? threadId ?? null;
|
||||
@ -2020,6 +2107,29 @@ export function useThreadStream({
|
||||
onStart,
|
||||
onFinish,
|
||||
});
|
||||
const [answeredApprovalState, setAnsweredApprovalState] = useState<{
|
||||
threadId: string | null;
|
||||
requestIds: Set<string>;
|
||||
}>({ threadId: null, requestIds: new Set() });
|
||||
const pendingApprovalSubmissionRef = useRef<PendingApprovalSubmission | null>(
|
||||
null,
|
||||
);
|
||||
const timedOutApprovalSubmissionRef = useRef<{
|
||||
threadId: string;
|
||||
requestId: string;
|
||||
} | null>(null);
|
||||
const liveInterruptRef = useRef<unknown>(undefined);
|
||||
const restInterruptsRef = useRef<unknown>(restInterrupts);
|
||||
restInterruptsRef.current = restInterrupts;
|
||||
|
||||
const settlePendingApproval = useCallback((accepted: boolean) => {
|
||||
const pending = pendingApprovalSubmissionRef.current;
|
||||
if (!pending) return;
|
||||
pendingApprovalSubmissionRef.current = null;
|
||||
clearTimeout(pending.timeout);
|
||||
pending.resolve(accepted);
|
||||
}, []);
|
||||
const pausedRunContextsRef = useRef(new Map<string, PausedRunContext>());
|
||||
|
||||
const {
|
||||
messages: history,
|
||||
@ -2039,6 +2149,15 @@ export function useThreadStream({
|
||||
?.run_id,
|
||||
[runsQuery.data],
|
||||
);
|
||||
const waitingForInputRunIds = useMemo(
|
||||
() =>
|
||||
new Set(
|
||||
(runsQuery.data ?? [])
|
||||
.filter((run) => run.status === "waiting_for_input")
|
||||
.map((run) => run.run_id),
|
||||
),
|
||||
[runsQuery.data],
|
||||
);
|
||||
const activeRunRejoinRef = useRef<ActiveRunRejoinState>(
|
||||
createActiveRunRejoinState(),
|
||||
);
|
||||
@ -2058,6 +2177,7 @@ export function useThreadStream({
|
||||
|
||||
useEffect(() => {
|
||||
const normalizedThreadId = threadId ?? null;
|
||||
timedOutApprovalSubmissionRef.current = null;
|
||||
if (!normalizedThreadId) {
|
||||
// Reset when the UI moves back to a brand new unsaved thread.
|
||||
startedRef.current = false;
|
||||
@ -2067,6 +2187,25 @@ export function useThreadStream({
|
||||
}
|
||||
threadIdRef.current = normalizedThreadId;
|
||||
}, [threadId]);
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
timedOutApprovalSubmissionRef.current = null;
|
||||
settlePendingApproval(false);
|
||||
};
|
||||
}, [settlePendingApproval]);
|
||||
|
||||
useEffect(() => {
|
||||
const timedOutApproval = timedOutApprovalSubmissionRef.current;
|
||||
if (!timedOutApproval || restInterruptsLoading || restInterruptsError) {
|
||||
return;
|
||||
}
|
||||
const stillPending = extractHitlApprovalRequests(restInterrupts).some(
|
||||
(request) => request.request_id === timedOutApproval.requestId,
|
||||
);
|
||||
if (!stillPending) {
|
||||
timedOutApprovalSubmissionRef.current = null;
|
||||
}
|
||||
}, [restInterrupts, restInterruptsError, restInterruptsLoading]);
|
||||
|
||||
const handleStreamStart = useCallback((_threadId: string, _runId: string) => {
|
||||
threadIdRef.current = _threadId;
|
||||
@ -2191,6 +2330,27 @@ export function useThreadStream({
|
||||
// Keep explicit: SDK types claim @default true, but runtime uses throttle ?? false.
|
||||
throttle: true,
|
||||
onCreated(meta) {
|
||||
const pendingApproval = pendingApprovalSubmissionRef.current;
|
||||
const timedOutApproval = timedOutApprovalSubmissionRef.current;
|
||||
const acceptedRequestId =
|
||||
pendingApproval?.threadId === meta.thread_id
|
||||
? pendingApproval.requestId
|
||||
: timedOutApproval?.threadId === meta.thread_id
|
||||
? timedOutApproval.requestId
|
||||
: null;
|
||||
if (pendingApproval?.threadId === meta.thread_id) {
|
||||
settlePendingApproval(true);
|
||||
}
|
||||
if (acceptedRequestId) {
|
||||
timedOutApprovalSubmissionRef.current = null;
|
||||
setAnsweredApprovalState((current) => {
|
||||
const requestIds = new Set(
|
||||
current.threadId === meta.thread_id ? current.requestIds : [],
|
||||
);
|
||||
requestIds.add(acceptedRequestId);
|
||||
return { threadId: meta.thread_id, requestIds };
|
||||
});
|
||||
}
|
||||
submittedRunIdRef.current = meta.run_id;
|
||||
handleStreamStart(meta.thread_id, meta.run_id);
|
||||
const now = new Date().toISOString();
|
||||
@ -2335,6 +2495,32 @@ export function useThreadStream({
|
||||
}
|
||||
},
|
||||
onError(error, run) {
|
||||
timedOutApprovalSubmissionRef.current = null;
|
||||
if (pendingApprovalSubmissionRef.current) {
|
||||
settlePendingApproval(false);
|
||||
}
|
||||
const erroredThreadId = run?.thread_id ?? threadIdRef.current;
|
||||
// Acceptance does not prove the replay completed. Reopen only requests
|
||||
// still advertised as pending; completed historical answers stay hidden.
|
||||
const stillPendingRequestIds = new Set(
|
||||
extractHitlApprovalRequests([
|
||||
...extractHitlApprovalRequests(liveInterruptRef.current),
|
||||
...extractHitlApprovalRequests(restInterruptsRef.current),
|
||||
]).map((request) => request.request_id),
|
||||
);
|
||||
setAnsweredApprovalState((current) => {
|
||||
if (current.threadId !== erroredThreadId) {
|
||||
return current;
|
||||
}
|
||||
const requestIds = new Set(
|
||||
[...current.requestIds].filter(
|
||||
(requestId) => !stillPendingRequestIds.has(requestId),
|
||||
),
|
||||
);
|
||||
return requestIds.size === current.requestIds.size
|
||||
? current
|
||||
: { threadId: current.threadId, requestIds };
|
||||
});
|
||||
scheduleActiveRunRejoinRetry(run);
|
||||
setOptimisticMessages([]);
|
||||
setOptimisticThreadId(null);
|
||||
@ -2342,7 +2528,16 @@ export function useThreadStream({
|
||||
pendingPreparedReplayRef.current = null;
|
||||
setPendingSupersededRunIds(new Set());
|
||||
setPendingSupersededMessageIds(new Set());
|
||||
toast.error(getStreamErrorMessage(error));
|
||||
const streamErrorText = `${JSON.stringify(error) ?? ""} ${getStreamErrorMessage(
|
||||
error,
|
||||
)}`;
|
||||
toast.error(
|
||||
streamErrorText.includes("hitl_approval_disabled")
|
||||
? t.humanInput.approval.featureDisabled
|
||||
: streamErrorText.includes("no_pending_approval")
|
||||
? t.humanInput.approval.noPending
|
||||
: getStreamErrorMessage(error),
|
||||
);
|
||||
pendingUsageBaselineMessageIdsRef.current = new Set(
|
||||
messagesRef.current
|
||||
.map(messageIdentity)
|
||||
@ -2361,6 +2556,7 @@ export function useThreadStream({
|
||||
}
|
||||
},
|
||||
onFinish(state, run) {
|
||||
settlePendingApproval(false);
|
||||
if (run) {
|
||||
completedRunIdsRef.current.add(run.run_id);
|
||||
}
|
||||
@ -2375,6 +2571,8 @@ export function useThreadStream({
|
||||
invalidateStoppedThreadCaches(queryClient, threadIdRef.current, isMock);
|
||||
},
|
||||
});
|
||||
liveInterruptRef.current = thread.interrupt;
|
||||
|
||||
const { isLoading: isThreadLoading, joinStream } = thread;
|
||||
const runInputAccount = useContext(AuthQueryScopeContext);
|
||||
const inputCapability = useRunInputsEnabled(!!runInputAccount && !isMock);
|
||||
@ -2589,6 +2787,7 @@ export function useThreadStream({
|
||||
);
|
||||
|
||||
const stopThread = useCallback(async () => {
|
||||
settlePendingApproval(false);
|
||||
const stoppedThreadId =
|
||||
threadIdRef.current ?? displayThreadId ?? threadId ?? null;
|
||||
const pendingReplay = pendingPreparedReplayRef.current;
|
||||
@ -2612,6 +2811,7 @@ export function useThreadStream({
|
||||
knownActiveRunId,
|
||||
isMock,
|
||||
queryClient,
|
||||
settlePendingApproval,
|
||||
thread,
|
||||
threadId,
|
||||
]);
|
||||
@ -2693,6 +2893,7 @@ export function useThreadStream({
|
||||
useEffect(() => {
|
||||
startedRef.current = false;
|
||||
sendInFlightRef.current = false;
|
||||
settlePendingApproval(false);
|
||||
messagesRef.current = [];
|
||||
transientHistoryBridgeRef.current = [];
|
||||
transientHistoryOrderRef.current = [];
|
||||
@ -2709,7 +2910,8 @@ export function useThreadStream({
|
||||
setPendingSupersededMessageIds(new Set());
|
||||
prevHumanMsgCountRef.current =
|
||||
latestMessageCountsRef.current.humanMessageCount;
|
||||
}, [threadId]);
|
||||
return () => settlePendingApproval(false);
|
||||
}, [settlePendingApproval, threadId]);
|
||||
|
||||
// Confirming a new thread only assigns its SDK id; the displayed
|
||||
// conversation and its submitted human anchor have not changed.
|
||||
@ -2948,6 +3150,12 @@ export function useThreadStream({
|
||||
// Attachments are uploaded, so the send is genuinely dispatched and
|
||||
// callers can drop one-time state. A failed upload rejected above
|
||||
// without reaching here, leaving that state for a retry.
|
||||
pausedRunContextsRef.current.set(threadId, {
|
||||
extraContext,
|
||||
conversationReferences: options?.conversationReferences
|
||||
? [...options.conversationReferences]
|
||||
: undefined,
|
||||
});
|
||||
options?.onSent?.();
|
||||
await thread.submit(
|
||||
{
|
||||
@ -2966,7 +3174,7 @@ export function useThreadStream({
|
||||
// subagent's values/messages into the thread view (#4399).
|
||||
streamResumable: true,
|
||||
config: {
|
||||
recursion_limit: 1000,
|
||||
recursion_limit: THREAD_RUN_RECURSION_LIMIT,
|
||||
},
|
||||
context: buildRunContext({
|
||||
settings: context,
|
||||
@ -3004,6 +3212,138 @@ export function useThreadStream({
|
||||
],
|
||||
);
|
||||
|
||||
const submitHitlApproval = useCallback(
|
||||
async (
|
||||
threadId: string,
|
||||
request: HitlApprovalRequest,
|
||||
responses: HitlApprovalResponse[],
|
||||
) => {
|
||||
if (thread.isLoading || sendInFlightRef.current) return false;
|
||||
const expected = new Set(
|
||||
request.actions.map((action) => action.tool_call_id),
|
||||
);
|
||||
const responseIds = new Set<string>();
|
||||
const valid =
|
||||
responses.length === request.actions.length &&
|
||||
responses.every((response) => {
|
||||
const action = request.actions.find(
|
||||
(candidate) => candidate.tool_call_id === response.tool_call_id,
|
||||
);
|
||||
if (
|
||||
!action ||
|
||||
responseIds.has(response.tool_call_id) ||
|
||||
response.request_id !== action.request_id ||
|
||||
response.source !== "hitl_approval" ||
|
||||
response.response_kind !== "option" ||
|
||||
!request.allowed_decisions.includes(response.option_id) ||
|
||||
response.value.length > 2000 ||
|
||||
(response.option_id === "approve" && response.value !== "")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
responseIds.add(response.tool_call_id);
|
||||
return true;
|
||||
});
|
||||
if (!valid || responseIds.size !== expected.size) return false;
|
||||
|
||||
const timedOutApproval = timedOutApprovalSubmissionRef.current;
|
||||
if (
|
||||
timedOutApproval?.threadId === threadId &&
|
||||
timedOutApproval.requestId !== request.request_id
|
||||
) {
|
||||
// Without a request ID echo, a different resume cannot safely share
|
||||
// the thread until the timed-out request resolves or disappears.
|
||||
void queryClient.invalidateQueries({
|
||||
queryKey: ["thread", "metadata", threadId],
|
||||
});
|
||||
return false;
|
||||
}
|
||||
|
||||
sendInFlightRef.current = true;
|
||||
// ES2022 target: Promise.withResolvers is not in the configured libs.
|
||||
let resolveAccepted!: (accepted: boolean) => void;
|
||||
const accepted = new Promise<boolean>((resolve) => {
|
||||
resolveAccepted = resolve;
|
||||
});
|
||||
const submission: PendingApprovalSubmission = {
|
||||
threadId,
|
||||
requestId: request.request_id,
|
||||
resolve: resolveAccepted,
|
||||
timeout: undefined,
|
||||
};
|
||||
pendingApprovalSubmissionRef.current = submission;
|
||||
try {
|
||||
submission.timeout = window.setTimeout(() => {
|
||||
if (pendingApprovalSubmissionRef.current === submission) {
|
||||
timedOutApprovalSubmissionRef.current = {
|
||||
threadId,
|
||||
requestId: request.request_id,
|
||||
};
|
||||
toast.error(t.humanInput.approval.submitTimeout);
|
||||
void stopThread().catch((error: unknown) => {
|
||||
toast.error(getStreamErrorMessage(error));
|
||||
});
|
||||
}
|
||||
}, HITL_APPROVAL_ACCEPTANCE_TIMEOUT_MS);
|
||||
const pausedContext = pausedRunContextsRef.current.get(threadId);
|
||||
// assistantId is fixed on this useStream instance and is therefore the
|
||||
// same lead/custom agent used by the paused run.
|
||||
const queued = thread.submit(
|
||||
null,
|
||||
buildHitlApprovalSubmitOptions({
|
||||
threadId,
|
||||
settings: context,
|
||||
responses,
|
||||
extraContext: pausedContext?.extraContext,
|
||||
conversationReferences: pausedContext?.conversationReferences,
|
||||
}),
|
||||
);
|
||||
// submit() resolves when the SDK has queued consumption, not when the
|
||||
// Gateway has accepted the resume. Acceptance can arrive first, while
|
||||
// onError, onFinish, abort, and the deadline can unblock a submit that
|
||||
// never settles. Only onCreated proves acceptance.
|
||||
const outcome = await Promise.race([
|
||||
queued.then(() => "queued" as const),
|
||||
accepted,
|
||||
]);
|
||||
const wasAccepted = outcome === "queued" ? await accepted : outcome;
|
||||
if (wasAccepted) {
|
||||
void queryClient.invalidateQueries({
|
||||
queryKey: ["thread", "metadata", threadId],
|
||||
});
|
||||
void queryClient.invalidateQueries({
|
||||
queryKey: ["thread", threadId],
|
||||
});
|
||||
}
|
||||
return wasAccepted;
|
||||
} catch (error) {
|
||||
if (
|
||||
timedOutApprovalSubmissionRef.current?.threadId === threadId &&
|
||||
timedOutApprovalSubmissionRef.current.requestId === request.request_id
|
||||
) {
|
||||
timedOutApprovalSubmissionRef.current = null;
|
||||
}
|
||||
if (pendingApprovalSubmissionRef.current === submission) {
|
||||
settlePendingApproval(false);
|
||||
}
|
||||
const serialized = `${JSON.stringify(error) ?? ""} ${getStreamErrorMessage(
|
||||
error,
|
||||
)}`;
|
||||
toast.error(
|
||||
serialized.includes("hitl_approval_disabled")
|
||||
? t.humanInput.approval.featureDisabled
|
||||
: serialized.includes("no_pending_approval")
|
||||
? t.humanInput.approval.noPending
|
||||
: getStreamErrorMessage(error),
|
||||
);
|
||||
return false;
|
||||
} finally {
|
||||
sendInFlightRef.current = false;
|
||||
}
|
||||
},
|
||||
[context, queryClient, settlePendingApproval, stopThread, t, thread],
|
||||
);
|
||||
|
||||
const submitPreparedReplay = useCallback(
|
||||
async <TPrepared extends RegeneratePrepareResponse>({
|
||||
threadId,
|
||||
@ -3100,6 +3440,7 @@ export function useThreadStream({
|
||||
setOptimisticMessages(nextOptimisticMessages);
|
||||
}
|
||||
|
||||
pausedRunContextsRef.current.set(threadId, {});
|
||||
await thread.submit(prepared.input, {
|
||||
threadId,
|
||||
checkpoint: prepared.checkpoint,
|
||||
@ -3107,7 +3448,7 @@ export function useThreadStream({
|
||||
// No streamSubgraphs — same contract as the main submit path (#4399).
|
||||
streamResumable: true,
|
||||
config: {
|
||||
recursion_limit: 1000,
|
||||
recursion_limit: THREAD_RUN_RECURSION_LIMIT,
|
||||
},
|
||||
// Replaying a turn never carries conversation references: the grant
|
||||
// is per send, so a regenerate or edit runs without them unless the
|
||||
@ -3430,6 +3771,35 @@ export function useThreadStream({
|
||||
pendingUsageBaselineMessageIdsRef.current,
|
||||
)
|
||||
: [];
|
||||
const answeredApprovalRequestIds = useMemo(
|
||||
() =>
|
||||
answeredApprovalState.threadId === currentViewThreadId
|
||||
? answeredApprovalState.requestIds
|
||||
: new Set<string>(),
|
||||
[answeredApprovalState, currentViewThreadId],
|
||||
);
|
||||
const approvalState = useMemo(
|
||||
() =>
|
||||
deriveHitlApprovalState({
|
||||
liveInterrupt: thread.interrupt,
|
||||
restInterrupts,
|
||||
restInterruptsError,
|
||||
historyLoading:
|
||||
isHistoryLoading || restInterruptsLoading || thread.isThreadLoading,
|
||||
next: thread.history?.[0]?.next ?? [],
|
||||
answeredRequestIds: answeredApprovalRequestIds,
|
||||
}),
|
||||
[
|
||||
answeredApprovalRequestIds,
|
||||
isHistoryLoading,
|
||||
restInterrupts,
|
||||
restInterruptsError,
|
||||
restInterruptsLoading,
|
||||
thread.history,
|
||||
thread.interrupt,
|
||||
thread.isThreadLoading,
|
||||
],
|
||||
);
|
||||
|
||||
// Merge history, live stream, and optimistic messages for display
|
||||
// History messages may overlap with thread.messages; thread.messages take precedence
|
||||
@ -3450,6 +3820,10 @@ export function useThreadStream({
|
||||
pendingUsageMessages,
|
||||
sendMessage,
|
||||
regenerateMessage,
|
||||
approvalRequests: approvalState.requests,
|
||||
hasPendingApproval: approvalState.hasPendingApproval,
|
||||
submitHitlApproval,
|
||||
waitingForInputRunIds,
|
||||
editAndRegenerateMessage,
|
||||
isUploading,
|
||||
isHistoryLoading,
|
||||
@ -3859,14 +4233,14 @@ export function useThreadRuns(
|
||||
{ enabled = true }: { enabled?: boolean } = {},
|
||||
) {
|
||||
const apiClient = getAPIClient();
|
||||
return useQuery<Run[]>({
|
||||
return useQuery<AppRun[]>({
|
||||
queryKey: ["thread", threadId],
|
||||
queryFn: async () => {
|
||||
if (!threadId) {
|
||||
return [];
|
||||
}
|
||||
const response = await apiClient.runs.list(threadId);
|
||||
return response;
|
||||
return response as AppRun[];
|
||||
},
|
||||
enabled: enabled && Boolean(threadId),
|
||||
refetchOnWindowFocus: false,
|
||||
@ -4036,11 +4410,11 @@ export function useMoveThreadToProject(options?: {
|
||||
|
||||
export function useRunDetail(threadId: string, runId: string) {
|
||||
const apiClient = getAPIClient();
|
||||
return useQuery<Run>({
|
||||
return useQuery<AppRun>({
|
||||
queryKey: ["thread", threadId, "run", runId],
|
||||
queryFn: async () => {
|
||||
const response = await apiClient.runs.get(threadId, runId);
|
||||
return response;
|
||||
return response as AppRun;
|
||||
},
|
||||
refetchOnWindowFocus: false,
|
||||
});
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
import type { Message, Thread } from "@langchain/langgraph-sdk";
|
||||
import type { Message, Run, Thread } from "@langchain/langgraph-sdk";
|
||||
|
||||
import type { ReasoningEffortValue } from "@/core/models/reasoning";
|
||||
|
||||
import type { Todo } from "../todos";
|
||||
|
||||
/** Gateway status outside the SDK's closed RunStatus union. */
|
||||
export type AppRunStatus = Run["status"] | "waiting_for_input";
|
||||
export type AppRun = Omit<Run, "status"> & { status: AppRunStatus };
|
||||
|
||||
export interface GoalState {
|
||||
objective: string;
|
||||
status: "active";
|
||||
|
||||
@ -3,6 +3,7 @@ import { describe, expect, it } from "@rstest/core";
|
||||
|
||||
import {
|
||||
describeGoalBar,
|
||||
describeGoalBarCopy,
|
||||
GOAL_HOST_REASON_TEXTS,
|
||||
goalBarAnnounceKey,
|
||||
goalOutcomeKey,
|
||||
@ -11,6 +12,7 @@ import {
|
||||
isThreadGoalOutcome,
|
||||
sameGoalInstance,
|
||||
} from "@/components/workspace/goal-status-helpers";
|
||||
import { enUS } from "@/core/i18n/locales/en-US";
|
||||
import type { GoalState, ThreadGoalOutcome } from "@/core/threads/types";
|
||||
|
||||
function makeGoal(overrides: Partial<GoalState> = {}): GoalState {
|
||||
@ -79,6 +81,45 @@ describe("describeGoalBar", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("shows waiting for approval instead of reply or in-progress states", () => {
|
||||
const goal = makeGoal();
|
||||
for (const isRunning of [false, true]) {
|
||||
expect(
|
||||
describeGoalBar({
|
||||
goal,
|
||||
outcome: null,
|
||||
isRunning,
|
||||
hasPendingApproval: true,
|
||||
}),
|
||||
).toEqual({ kind: "awaitingApproval", goal });
|
||||
}
|
||||
});
|
||||
|
||||
it("prioritizes approval over a stood-down human-input state", () => {
|
||||
const goal = stoodDown("blocked:needs_user_input");
|
||||
expect(
|
||||
describeGoalBar({
|
||||
goal,
|
||||
outcome: null,
|
||||
isRunning: false,
|
||||
hasOpenHumanInputCard: true,
|
||||
hasPendingApproval: true,
|
||||
}),
|
||||
).toEqual({ kind: "awaitingApproval", goal });
|
||||
});
|
||||
|
||||
it("uses approval-specific waiting copy and next step", () => {
|
||||
const goal = makeGoal();
|
||||
expect(
|
||||
describeGoalBarCopy({ kind: "awaitingApproval", goal }, enUS),
|
||||
).toMatchObject({
|
||||
chip: "Waiting for approval",
|
||||
detail: "Review the tool approval above to continue.",
|
||||
announcement:
|
||||
"Waiting for approval: Review the tool approval above to continue.",
|
||||
});
|
||||
});
|
||||
|
||||
it("counts automatic follow-ups while one is running", () => {
|
||||
const goal = makeGoal({
|
||||
continuation_count: 1,
|
||||
|
||||
@ -518,6 +518,24 @@ describe("GoalStatus a11y", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("GoalStatus approval metadata errors", () => {
|
||||
it("directs users to the reload-error panel instead of an approval card", () => {
|
||||
render(
|
||||
bar({
|
||||
goal: makeGoal(),
|
||||
hasPendingApproval: true,
|
||||
approvalMetadataError: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(visibleRowText()).toContain(
|
||||
"Approval details failed to load. Use the reload-error panel above to try again.",
|
||||
);
|
||||
expect(visibleRowText()).not.toContain("Review the tool approval above");
|
||||
expect(liveRegion().textContent).toContain("reload-error panel");
|
||||
});
|
||||
});
|
||||
|
||||
describe("GoalStatus visuals", () => {
|
||||
function chipClass() {
|
||||
return screen.getByTestId("goal-status-chip").className;
|
||||
|
||||
@ -1,11 +1,25 @@
|
||||
import { afterEach, describe, expect, it, rs } from "@rstest/core";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import {
|
||||
act,
|
||||
cleanup,
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
waitFor,
|
||||
} from "@testing-library/react";
|
||||
|
||||
import { HumanInputCard } from "@/components/workspace/messages/human-input-card";
|
||||
import {
|
||||
HumanInputCard,
|
||||
ToolApprovalCard,
|
||||
} from "@/components/workspace/messages/human-input-card";
|
||||
import { MessageArtifactPathsContext } from "@/components/workspace/messages/markdown-link";
|
||||
import { I18nContext } from "@/core/i18n/context";
|
||||
import { enUS } from "@/core/i18n/locales/en-US";
|
||||
import type { HumanInputRequest } from "@/core/messages/human-input";
|
||||
import type {
|
||||
HitlApprovalRequest,
|
||||
HitlApprovalResponse,
|
||||
HumanInputRequest,
|
||||
} from "@/core/messages/human-input";
|
||||
|
||||
const formRequest: HumanInputRequest = {
|
||||
version: 2,
|
||||
@ -20,6 +34,56 @@ const formRequest: HumanInputRequest = {
|
||||
],
|
||||
};
|
||||
|
||||
const approvalRequest: HitlApprovalRequest = {
|
||||
version: 1,
|
||||
kind: "hitl_approval",
|
||||
request_id: "hitl_approval:ai-1:call-1",
|
||||
actions: [
|
||||
{
|
||||
tool_call_id: "call-1",
|
||||
tool_name: "write_file",
|
||||
args_summary: JSON.stringify({ path: "/tmp/a.txt", content: "A" }),
|
||||
request_id: "hitl_approval:ai-1:call-1",
|
||||
},
|
||||
{
|
||||
tool_call_id: "call-2",
|
||||
tool_name: "bash",
|
||||
args_summary: JSON.stringify({ command: "pwd" }),
|
||||
request_id: "hitl_approval:ai-1:call-2",
|
||||
},
|
||||
],
|
||||
allowed_decisions: ["approve", "reject"],
|
||||
pending_tool_call_ids: ["call-1", "call-2"],
|
||||
};
|
||||
|
||||
function renderApprovalCard({
|
||||
request = approvalRequest,
|
||||
onSubmit = async () => undefined,
|
||||
disabled = false,
|
||||
}: {
|
||||
request?: HitlApprovalRequest;
|
||||
onSubmit?: (
|
||||
request: HitlApprovalRequest,
|
||||
responses: HitlApprovalResponse[],
|
||||
) => Promise<boolean | void>;
|
||||
disabled?: boolean;
|
||||
} = {}) {
|
||||
return {
|
||||
onSubmit,
|
||||
...render(
|
||||
<I18nContext.Provider
|
||||
value={{ locale: "en-US", setLocale: () => undefined, t: enUS }}
|
||||
>
|
||||
<ToolApprovalCard
|
||||
disabled={disabled}
|
||||
request={request}
|
||||
onSubmit={onSubmit}
|
||||
/>
|
||||
</I18nContext.Provider>,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function renderCard() {
|
||||
return render(
|
||||
<I18nContext.Provider
|
||||
@ -130,3 +194,131 @@ describe("HumanInputCard form validation (DOM)", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ToolApprovalCard", () => {
|
||||
it("collects every action before one atomic mixed-decision submit", async () => {
|
||||
const onSubmit = rs.fn(
|
||||
async (
|
||||
_request: HitlApprovalRequest,
|
||||
_responses: HitlApprovalResponse[],
|
||||
) => undefined,
|
||||
);
|
||||
renderApprovalCard({ onSubmit });
|
||||
const submit = screen.getByRole("button", { name: "Submit decisions" });
|
||||
|
||||
expect(submit.hasAttribute("disabled")).toBe(true);
|
||||
const approveButtons = screen.getAllByRole("button", { name: "Approve" });
|
||||
fireEvent.click(approveButtons[0]!);
|
||||
expect(submit.hasAttribute("disabled")).toBe(true);
|
||||
expect(onSubmit).not.toHaveBeenCalled();
|
||||
|
||||
fireEvent.click(screen.getAllByRole("button", { name: "Reject" })[1]!);
|
||||
fireEvent.change(screen.getByLabelText("Rejection reason (optional)"), {
|
||||
target: { value: "Do not run this command" },
|
||||
});
|
||||
expect(submit.hasAttribute("disabled")).toBe(false);
|
||||
fireEvent.click(submit);
|
||||
|
||||
await waitFor(() => expect(onSubmit).toHaveBeenCalledTimes(1));
|
||||
const responses = onSubmit.mock.calls[0]![1];
|
||||
expect(responses).toHaveLength(2);
|
||||
expect(
|
||||
responses.map((response: { option_id: string }) => response.option_id),
|
||||
).toEqual(["approve", "reject"]);
|
||||
expect(responses[1]?.value).toBe("Do not run this command");
|
||||
await screen.findByText("Answered");
|
||||
expect(submit.hasAttribute("disabled")).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps mixed decisions and rejection reason editable after failure", async () => {
|
||||
let resolveAccepted!: (accepted: boolean) => void;
|
||||
const accepted = new Promise<boolean>((resolve) => {
|
||||
resolveAccepted = resolve;
|
||||
});
|
||||
const onSubmit = rs.fn(() => accepted);
|
||||
renderApprovalCard({ onSubmit });
|
||||
const approveButtons = screen.getAllByRole("button", { name: "Approve" });
|
||||
fireEvent.click(approveButtons[0]!);
|
||||
fireEvent.click(screen.getAllByRole("button", { name: "Reject" })[1]!);
|
||||
const reason = screen.getByLabelText<HTMLTextAreaElement>(
|
||||
"Rejection reason (optional)",
|
||||
);
|
||||
fireEvent.change(reason, {
|
||||
target: { value: "Do not run this command" },
|
||||
});
|
||||
const submit = screen.getByRole("button", { name: "Submit decisions" });
|
||||
fireEvent.click(submit);
|
||||
|
||||
// The submit promise deliberately remains unresolved; assert the in-flight
|
||||
// frame synchronously instead of wrapping that pending operation in findBy*.
|
||||
expect(screen.getByText("Sending...")).not.toBeNull();
|
||||
expect(submit.hasAttribute("disabled")).toBe(true);
|
||||
expect(onSubmit).toHaveBeenCalledTimes(1);
|
||||
act(() => resolveAccepted(false));
|
||||
|
||||
await waitFor(() => expect(submit.hasAttribute("disabled")).toBe(false));
|
||||
expect(screen.getByTestId("tool-approval-card")).not.toBeNull();
|
||||
expect(approveButtons[0]!.getAttribute("aria-pressed")).toBe("true");
|
||||
expect(
|
||||
screen
|
||||
.getAllByRole("button", { name: "Reject" })[1]!
|
||||
.getAttribute("aria-pressed"),
|
||||
).toBe("true");
|
||||
expect(reason.value).toBe("Do not run this command");
|
||||
expect(screen.queryByText("Answered")).toBeNull();
|
||||
expect(screen.queryByText("Sending...")).toBeNull();
|
||||
});
|
||||
|
||||
it("submits approve-all once and disables an in-flight interrupt frame", async () => {
|
||||
const onSubmit = rs.fn(
|
||||
async (
|
||||
_request: HitlApprovalRequest,
|
||||
_responses: HitlApprovalResponse[],
|
||||
) => undefined,
|
||||
);
|
||||
renderApprovalCard({ onSubmit, disabled: true });
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Approve all" }));
|
||||
expect(onSubmit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses allowed_decisions and keeps all-reject available by itself", () => {
|
||||
renderApprovalCard({
|
||||
request: { ...approvalRequest, allowed_decisions: ["reject"] },
|
||||
});
|
||||
|
||||
expect(screen.queryByRole("button", { name: "Approve all" })).toBeNull();
|
||||
expect(screen.queryByRole("button", { name: "Approve" })).toBeNull();
|
||||
expect(
|
||||
screen.getByRole("button", { name: "Reject all and finish" }),
|
||||
).not.toBeNull();
|
||||
});
|
||||
|
||||
it("shows the complete str_replace old and new strings as a diff", () => {
|
||||
renderApprovalCard({
|
||||
request: {
|
||||
...approvalRequest,
|
||||
actions: [
|
||||
{
|
||||
tool_call_id: "replace-1",
|
||||
tool_name: "str_replace",
|
||||
args_summary: JSON.stringify({
|
||||
path: "/tmp/a.txt",
|
||||
old_str: "old line\nsecond old line",
|
||||
new_str: "new line\nsecond new line",
|
||||
}),
|
||||
request_id: "hitl_approval:ai-1:replace-1",
|
||||
},
|
||||
],
|
||||
pending_tool_call_ids: ["replace-1"],
|
||||
},
|
||||
});
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: /Tool details/ }));
|
||||
const diff = screen.getByTestId("approval-diff");
|
||||
expect(diff.textContent).toContain("- old line");
|
||||
expect(diff.textContent).toContain("- second old line");
|
||||
expect(diff.textContent).toContain("+ new line");
|
||||
expect(diff.textContent).toContain("+ second new line");
|
||||
});
|
||||
});
|
||||
|
||||
@ -0,0 +1,208 @@
|
||||
import type { Message } from "@langchain/langgraph-sdk";
|
||||
import { afterEach, describe, expect, it, rs } from "@rstest/core";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
import { MessageList } from "@/components/workspace/messages/message-list";
|
||||
import { I18nContext } from "@/core/i18n/context";
|
||||
import { enUS } from "@/core/i18n/locales/en-US";
|
||||
import type {
|
||||
HitlApprovalRequest,
|
||||
HitlApprovalResponse,
|
||||
} from "@/core/messages/human-input";
|
||||
import type { MessageGroup } from "@/core/messages/utils";
|
||||
|
||||
rs.mock("@/components/ai-elements/conversation", () => ({
|
||||
Conversation: ({ children }: { children: ReactNode }) => (
|
||||
<div>{children}</div>
|
||||
),
|
||||
ConversationContent: ({ children }: { children: ReactNode }) => (
|
||||
<div>{children}</div>
|
||||
),
|
||||
}));
|
||||
rs.mock("@/components/workspace/messages/virtual-message-list", () => ({
|
||||
VirtualMessageList: ({
|
||||
groups,
|
||||
renderGroup,
|
||||
}: {
|
||||
groups: MessageGroup[];
|
||||
renderGroup: (group: MessageGroup, index: number) => ReactNode;
|
||||
}) => (
|
||||
<div>
|
||||
{groups.map((group, index) => (
|
||||
<div key={group.id}>{renderGroup(group, index)}</div>
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
}));
|
||||
rs.mock("@/components/workspace/messages/message-list-item", () => ({
|
||||
MessageListItem: () => null,
|
||||
}));
|
||||
rs.mock("@/components/workspace/messages/subtask-card", () => ({
|
||||
SubtaskCard: () => null,
|
||||
}));
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
rs.restoreAllMocks();
|
||||
});
|
||||
|
||||
const approvalRequest: HitlApprovalRequest = {
|
||||
version: 1,
|
||||
kind: "hitl_approval",
|
||||
request_id: "hitl_approval:ai-1:call-1",
|
||||
actions: [
|
||||
{
|
||||
tool_call_id: "call-1",
|
||||
tool_name: "write_file",
|
||||
args_summary: JSON.stringify({ path: "/tmp/a.txt", content: "A" }),
|
||||
request_id: "hitl_approval:ai-1:call-1",
|
||||
},
|
||||
],
|
||||
allowed_decisions: ["approve", "reject"],
|
||||
pending_tool_call_ids: ["call-1"],
|
||||
};
|
||||
|
||||
const messages: Message[] = [
|
||||
{ id: "ai-1", type: "ai", content: "I need approval first." },
|
||||
];
|
||||
const getMessagesMetadata = () => undefined;
|
||||
|
||||
type ViewOptions = {
|
||||
approvalRequests?: HitlApprovalRequest[];
|
||||
error?: unknown;
|
||||
approvalMetadataLoading?: boolean;
|
||||
approvalMetadataError?: boolean;
|
||||
onReloadApprovalMetadata?: () => unknown;
|
||||
onSubmitApproval?: (
|
||||
request: HitlApprovalRequest,
|
||||
responses: HitlApprovalResponse[],
|
||||
) => Promise<boolean | void>;
|
||||
};
|
||||
|
||||
function view({
|
||||
approvalRequests = [approvalRequest],
|
||||
error,
|
||||
approvalMetadataLoading = false,
|
||||
approvalMetadataError = false,
|
||||
onReloadApprovalMetadata,
|
||||
onSubmitApproval = async () => undefined,
|
||||
}: ViewOptions = {}) {
|
||||
return (
|
||||
<I18nContext.Provider
|
||||
value={{ locale: "en-US", setLocale: () => undefined, t: enUS }}
|
||||
>
|
||||
<MessageList
|
||||
approvalRequests={approvalRequests}
|
||||
approvalMetadataLoading={approvalMetadataLoading}
|
||||
approvalMetadataError={approvalMetadataError}
|
||||
onReloadApprovalMetadata={onReloadApprovalMetadata}
|
||||
onSubmitApproval={onSubmitApproval}
|
||||
threadId="thread-1"
|
||||
thread={
|
||||
{
|
||||
messages,
|
||||
error,
|
||||
isLoading: false,
|
||||
isThreadLoading: false,
|
||||
values: {},
|
||||
getMessagesMetadata,
|
||||
} as unknown as React.ComponentProps<typeof MessageList>["thread"]
|
||||
}
|
||||
/>
|
||||
</I18nContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
describe("MessageList HITL approvals", () => {
|
||||
it("keeps a false submission visible and preserves the card across stream errors", async () => {
|
||||
const onSubmitApproval = rs.fn(async () => false);
|
||||
const { rerender } = render(view({ onSubmitApproval }));
|
||||
const card = screen.getByTestId("tool-approval-card");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Approve" }));
|
||||
const reason = screen.getByLabelText<HTMLTextAreaElement>(
|
||||
"Rejection reason (optional)",
|
||||
);
|
||||
fireEvent.change(reason, { target: { value: "Review the path first" } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Submit decisions" }));
|
||||
|
||||
const alert = await screen.findByRole("alert");
|
||||
expect(alert.textContent).toBe(enUS.humanInput.approval.submitFailed);
|
||||
expect(onSubmitApproval).toHaveBeenCalledTimes(1);
|
||||
|
||||
rerender(view({ error: new Error("stream failed"), onSubmitApproval }));
|
||||
expect(screen.getByTestId("tool-approval-card")).toBe(card);
|
||||
expect(screen.getByRole("alert")).toBe(alert);
|
||||
expect(
|
||||
screen
|
||||
.getByRole("button", { name: "Approve" })
|
||||
.getAttribute("aria-pressed"),
|
||||
).toBe("true");
|
||||
expect(reason.value).toBe("Review the path first");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Reject" }));
|
||||
fireEvent.change(reason, { target: { value: "Use a different path" } });
|
||||
expect(
|
||||
screen
|
||||
.getByRole("button", { name: "Reject" })
|
||||
.getAttribute("aria-pressed"),
|
||||
).toBe("true");
|
||||
expect(reason.value).toBe("Use a different path");
|
||||
expect(screen.getByRole("alert")).toBe(alert);
|
||||
});
|
||||
|
||||
it("shows truthful loading copy until the approval payload mounts its card", () => {
|
||||
const { rerender } = render(
|
||||
view({ approvalRequests: [], approvalMetadataLoading: true }),
|
||||
);
|
||||
expect(screen.queryByTestId("tool-approval-card")).toBeNull();
|
||||
expect(screen.queryByText("Waiting for approval")).toBeNull();
|
||||
expect(screen.getByTestId("approval-metadata-loading").textContent).toBe(
|
||||
"Loading approval details...",
|
||||
);
|
||||
|
||||
rerender(view({ approvalMetadataLoading: true }));
|
||||
expect(screen.getByTestId("tool-approval-card")).not.toBeNull();
|
||||
expect(screen.getByText("Waiting for approval")).not.toBeNull();
|
||||
expect(screen.queryByTestId("approval-metadata-loading")).toBeNull();
|
||||
});
|
||||
|
||||
it("shows a truthful reload path instead of a blank approval region after metadata failure", () => {
|
||||
const onReloadApprovalMetadata = rs.fn();
|
||||
render(
|
||||
view({
|
||||
approvalRequests: [],
|
||||
approvalMetadataLoading: true,
|
||||
approvalMetadataError: true,
|
||||
onReloadApprovalMetadata,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(screen.queryByTestId("tool-approval-card")).toBeNull();
|
||||
expect(screen.queryByTestId("approval-metadata-loading")).toBeNull();
|
||||
expect(screen.getByRole("alert").textContent).toBe(
|
||||
enUS.humanInput.approval.loadFailed,
|
||||
);
|
||||
fireEvent.click(
|
||||
screen.getByRole("button", {
|
||||
name: enUS.humanInput.approval.reload,
|
||||
}),
|
||||
);
|
||||
expect(onReloadApprovalMetadata).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("removes stale failure state when an answered approval card unmounts", async () => {
|
||||
const onSubmitApproval = rs.fn(async () => false);
|
||||
const { rerender } = render(view({ onSubmitApproval }));
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Approve" }));
|
||||
fireEvent.click(screen.getByRole("button", { name: "Submit decisions" }));
|
||||
expect(await screen.findByRole("alert")).not.toBeNull();
|
||||
|
||||
rerender(view({ approvalRequests: [], onSubmitApproval }));
|
||||
expect(screen.queryByTestId("tool-approval-card")).toBeNull();
|
||||
expect(screen.queryByRole("alert")).toBeNull();
|
||||
expect(screen.queryByText("Waiting for approval")).toBeNull();
|
||||
});
|
||||
});
|
||||
@ -28,6 +28,12 @@ const mocks = rs.hoisted(() => ({
|
||||
createSidecarThread: rs.fn(),
|
||||
sendMessage: rs.fn(),
|
||||
toastError: rs.fn(),
|
||||
streamOptions: undefined as unknown,
|
||||
threadMetadata: {
|
||||
data: undefined as unknown,
|
||||
isLoading: false,
|
||||
isFetching: false,
|
||||
},
|
||||
}));
|
||||
|
||||
rs.mock("next/navigation", () => ({
|
||||
@ -58,14 +64,18 @@ rs.mock("@/core/sidecar/api", () => ({
|
||||
|
||||
rs.mock("@/core/threads/hooks", () => ({
|
||||
useDeleteThread: () => ({ mutateAsync: rs.fn(), isPending: false }),
|
||||
useThreadStream: () => ({
|
||||
thread: { isLoading: false, messages: [], values: {} },
|
||||
sendMessage: mocks.sendMessage,
|
||||
isUploading: false,
|
||||
isHistoryLoading: false,
|
||||
hasMoreHistory: false,
|
||||
loadMoreHistory: rs.fn(),
|
||||
}),
|
||||
useThreadMetadata: () => mocks.threadMetadata,
|
||||
useThreadStream: (options: unknown) => {
|
||||
mocks.streamOptions = options;
|
||||
return {
|
||||
thread: { isLoading: false, messages: [], values: {} },
|
||||
sendMessage: mocks.sendMessage,
|
||||
isUploading: false,
|
||||
isHistoryLoading: false,
|
||||
hasMoreHistory: false,
|
||||
loadMoreHistory: rs.fn(),
|
||||
};
|
||||
},
|
||||
}));
|
||||
|
||||
const USER = {
|
||||
@ -141,6 +151,10 @@ beforeEach(() => {
|
||||
mocks.createSidecarThread.mockReset();
|
||||
mocks.sendMessage.mockReset();
|
||||
mocks.toastError.mockReset();
|
||||
mocks.streamOptions = undefined;
|
||||
mocks.threadMetadata.data = undefined;
|
||||
mocks.threadMetadata.isLoading = false;
|
||||
mocks.threadMetadata.isFetching = false;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@ -148,6 +162,36 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("SidecarPanel keeps the draft when a send fails", () => {
|
||||
it("forwards restored interrupts and initial metadata loading to the stream", async () => {
|
||||
const interrupts = { task: [{ value: { kind: "hitl_approval" } }] };
|
||||
mocks.threadMetadata.data = { interrupts };
|
||||
mocks.threadMetadata.isLoading = true;
|
||||
mocks.threadMetadata.isFetching = true;
|
||||
|
||||
render(renderPanel("sidecar-1"));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.streamOptions).toMatchObject({
|
||||
threadId: "sidecar-1",
|
||||
restInterrupts: interrupts,
|
||||
restInterruptsLoading: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not treat a background metadata refetch as initial loading", async () => {
|
||||
mocks.threadMetadata.isFetching = true;
|
||||
|
||||
render(renderPanel("sidecar-1"));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mocks.streamOptions).toMatchObject({
|
||||
threadId: "sidecar-1",
|
||||
restInterruptsLoading: false,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the draft when creating the side chat fails", async () => {
|
||||
mocks.createSidecarThread.mockRejectedValue(new Error("create failed"));
|
||||
render(renderPanel(null));
|
||||
|
||||
1069
frontend/tests/unit/core/threads/hitl-approval.dom.test.tsx
Normal file
1069
frontend/tests/unit/core/threads/hitl-approval.dom.test.tsx
Normal file
File diff suppressed because it is too large
Load Diff
206
frontend/tests/unit/core/threads/hitl-approval.test.ts
Normal file
206
frontend/tests/unit/core/threads/hitl-approval.test.ts
Normal file
@ -0,0 +1,206 @@
|
||||
import { describe, expect, it } from "@rstest/core";
|
||||
|
||||
import {
|
||||
createHitlApprovalResponses,
|
||||
extractHitlApprovalRequests,
|
||||
parseApprovalArgsSummary,
|
||||
parseHitlApprovalRequest,
|
||||
type HitlApprovalRequest,
|
||||
} from "@/core/messages/human-input";
|
||||
import { DEFAULT_LOCAL_SETTINGS } from "@/core/settings/local";
|
||||
import {
|
||||
buildHitlApprovalSubmitOptions,
|
||||
deriveHitlApprovalState,
|
||||
THREAD_RUN_RECURSION_LIMIT,
|
||||
} from "@/core/threads/hooks";
|
||||
|
||||
const request: HitlApprovalRequest = {
|
||||
version: 1,
|
||||
kind: "hitl_approval",
|
||||
request_id: "hitl_approval:ai-1:call-1",
|
||||
actions: [
|
||||
{
|
||||
tool_call_id: "call-1",
|
||||
tool_name: "write_file",
|
||||
args_summary: JSON.stringify({ path: "/tmp/a.txt" }),
|
||||
request_id: "hitl_approval:ai-1:call-1",
|
||||
},
|
||||
{
|
||||
tool_call_id: "call-2",
|
||||
tool_name: "bash",
|
||||
args_summary: JSON.stringify({ command: "pwd" }),
|
||||
request_id: "hitl_approval:ai-1:call-2",
|
||||
},
|
||||
],
|
||||
allowed_decisions: ["approve", "reject"],
|
||||
pending_tool_call_ids: ["call-1", "call-2"],
|
||||
};
|
||||
|
||||
describe("parseHitlApprovalRequest", () => {
|
||||
it.each([
|
||||
null,
|
||||
{ ...request, kind: "human_input_request" },
|
||||
{ ...request, version: 2 },
|
||||
{ ...request, request_id: " " },
|
||||
{ ...request, actions: [], pending_tool_call_ids: [] },
|
||||
{ ...request, allowed_decisions: ["approve", "approve"] },
|
||||
{ ...request, allowed_decisions: ["maybe"] },
|
||||
{
|
||||
...request,
|
||||
actions: [request.actions[0], request.actions[0]],
|
||||
},
|
||||
{
|
||||
...request,
|
||||
actions: [
|
||||
request.actions[0],
|
||||
{ ...request.actions[1], tool_call_id: "call-1" },
|
||||
],
|
||||
},
|
||||
{
|
||||
...request,
|
||||
actions: [
|
||||
request.actions[0],
|
||||
{ ...request.actions[1], args_summary: null },
|
||||
],
|
||||
},
|
||||
{ ...request, pending_tool_call_ids: ["call-1", "missing"] },
|
||||
{ ...request, pending_tool_call_ids: ["call-1", "call-1"] },
|
||||
])("rejects malformed request %#", (value) => {
|
||||
expect(parseHitlApprovalRequest(value)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("HITL approval interrupt state", () => {
|
||||
it("filters static breakpoints and locks the reload window by default", () => {
|
||||
expect(extractHitlApprovalRequests({ when: "breakpoint" })).toEqual([]);
|
||||
const loading = deriveHitlApprovalState({
|
||||
liveInterrupt: { when: "breakpoint" },
|
||||
historyLoading: true,
|
||||
next: ["HitlApprovalMiddleware.after_model"],
|
||||
answeredRequestIds: new Set(),
|
||||
});
|
||||
expect(loading.requests).toEqual([]);
|
||||
expect(loading.hasPendingApproval).toBe(true);
|
||||
|
||||
const loaded = deriveHitlApprovalState({
|
||||
liveInterrupt: { when: "breakpoint" },
|
||||
historyLoading: false,
|
||||
next: ["tools"],
|
||||
answeredRequestIds: new Set(),
|
||||
});
|
||||
expect(loaded.requests).toEqual([]);
|
||||
expect(loaded.hasPendingApproval).toBe(false);
|
||||
});
|
||||
|
||||
it("stays locked when approval metadata fails for a paused snapshot", () => {
|
||||
const failed = deriveHitlApprovalState({
|
||||
liveInterrupt: undefined,
|
||||
restInterrupts: undefined,
|
||||
restInterruptsError: true,
|
||||
historyLoading: false,
|
||||
next: ["HitlApprovalMiddleware.after_model"],
|
||||
answeredRequestIds: new Set(),
|
||||
});
|
||||
expect(failed.requests).toEqual([]);
|
||||
expect(failed.hasPendingApproval).toBe(true);
|
||||
|
||||
const notPaused = deriveHitlApprovalState({
|
||||
liveInterrupt: undefined,
|
||||
restInterruptsError: true,
|
||||
historyLoading: false,
|
||||
next: ["tools"],
|
||||
answeredRequestIds: new Set(),
|
||||
});
|
||||
expect(notPaused.hasPendingApproval).toBe(false);
|
||||
});
|
||||
|
||||
it("does not show approval waiting for clarification-only history loading", () => {
|
||||
const loading = deriveHitlApprovalState({
|
||||
liveInterrupt: undefined,
|
||||
historyLoading: true,
|
||||
next: ["ask_clarification"],
|
||||
answeredRequestIds: new Set(),
|
||||
});
|
||||
expect(loading.requests).toEqual([]);
|
||||
expect(loading.hasPendingApproval).toBe(false);
|
||||
});
|
||||
|
||||
it("reads live and REST interrupt shapes and honors local answered state", () => {
|
||||
const live = deriveHitlApprovalState({
|
||||
liveInterrupt: { id: "task-local-id", value: request },
|
||||
historyLoading: false,
|
||||
next: ["tools"],
|
||||
answeredRequestIds: new Set(),
|
||||
});
|
||||
expect(live.requests).toEqual([request]);
|
||||
expect(live.hasPendingApproval).toBe(true);
|
||||
|
||||
const rest = deriveHitlApprovalState({
|
||||
liveInterrupt: undefined,
|
||||
restInterrupts: { "task-id": [{ id: "interrupt-id", value: request }] },
|
||||
historyLoading: false,
|
||||
next: ["tools"],
|
||||
answeredRequestIds: new Set([request.request_id]),
|
||||
});
|
||||
expect(rest.requests).toEqual([]);
|
||||
expect(rest.hasPendingApproval).toBe(false);
|
||||
});
|
||||
|
||||
it("parses the backend's Python-repr string map without truncating replacements", () => {
|
||||
expect(
|
||||
parseApprovalArgsSummary(
|
||||
String.raw`{'path': '/tmp/a.txt', 'old_str': 'old\nsecond', 'new_str': "it's new"}`,
|
||||
),
|
||||
).toEqual({
|
||||
path: "/tmp/a.txt",
|
||||
old_str: "old\nsecond",
|
||||
new_str: "it's new",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses to construct a partial response batch", () => {
|
||||
expect(
|
||||
createHitlApprovalResponses(request, { "call-1": "approve" }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
createHitlApprovalResponses(request, {
|
||||
"call-1": "approve",
|
||||
"call-2": "reject",
|
||||
}),
|
||||
).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HITL approval resume options", () => {
|
||||
it("uses the current head, resumable values stream, normal recursion limit, and paused send context", () => {
|
||||
const responses = createHitlApprovalResponses(request, {
|
||||
"call-1": "approve",
|
||||
"call-2": "approve",
|
||||
})!;
|
||||
const options = buildHitlApprovalSubmitOptions({
|
||||
threadId: "thread-1",
|
||||
settings: DEFAULT_LOCAL_SETTINGS.context,
|
||||
responses,
|
||||
extraContext: {
|
||||
model_name: "model-for-paused-turn",
|
||||
model_selection_requested: "model-for-paused-turn",
|
||||
},
|
||||
conversationReferences: ["referenced-thread"],
|
||||
});
|
||||
|
||||
expect(options).toMatchObject({
|
||||
threadId: "thread-1",
|
||||
checkpoint: null,
|
||||
command: { resume: { responses } },
|
||||
streamMode: ["values"],
|
||||
streamResumable: true,
|
||||
config: { recursion_limit: THREAD_RUN_RECURSION_LIMIT },
|
||||
context: {
|
||||
thread_id: "thread-1",
|
||||
model_name: "model-for-paused-turn",
|
||||
conversation_references: ["referenced-thread"],
|
||||
},
|
||||
});
|
||||
expect("streamSubgraphs" in options).toBe(false);
|
||||
});
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user