From 5991ab936ed2bc3f22175270a71aa6f135879df5 Mon Sep 17 00:00:00 2001 From: kongxin Date: Thu, 19 Nov 2026 05:26:58 -0800 Subject: [PATCH] fix(runtime): keep an edit replay that pauses for tool approval (#8390) The edit replay finalizer rolled back every run that did not end in success, including one waiting for tool approval. The edit was lost, the resume had nothing to run, and the unanswered interrupt kept blocking new runs on the thread. Treat a native pause as the current attempt: skip the rollback and let the paused attempt replace its source turn in history. --- backend/docs/RUN_INTERACTION_POLICY.md | 3 +- backend/docs/THREAD_LIFECYCLE.md | 4 + .../harness/deerflow/runtime/runs/manager.py | 2 +- .../harness/deerflow/runtime/runs/worker.py | 4 +- backend/tests/test_history_batch_queries.py | 7 +- backend/tests/test_hitl_approval_api.py | 126 +++++++++++++++++- 6 files changed, 141 insertions(+), 5 deletions(-) diff --git a/backend/docs/RUN_INTERACTION_POLICY.md b/backend/docs/RUN_INTERACTION_POLICY.md index 8e49d4cf73..4d48c5e39e 100644 --- a/backend/docs/RUN_INTERACTION_POLICY.md +++ b/backend/docs/RUN_INTERACTION_POLICY.md @@ -80,7 +80,8 @@ rejected, so continue the work in a new thread. In-place recovery is tracked in [#8071](https://github.com/bytedance/deer-flow/issues/8071). `waiting_for_input` is not terminal success or failure. Goals, delivery checks, -and batch continuation must preserve a native pause. New runs cannot discard it. +batch continuation and edit-replay rollback must preserve a native pause. New runs +cannot discard it. Root checkpoint approvals do not expire and block new runs until resolved; live child approvals retain the time limit described above. See [HITL Tool Approval](CONFIGURATION.md#hitl-tool-approval) for grant scoping, diff --git a/backend/docs/THREAD_LIFECYCLE.md b/backend/docs/THREAD_LIFECYCLE.md index 902dffa640..0f8b63e094 100644 --- a/backend/docs/THREAD_LIFECYCLE.md +++ b/backend/docs/THREAD_LIFECYCLE.md @@ -26,6 +26,10 @@ tasks; absence of evidence stays permissive there rather than failing closed. Edit replay resolves its base through the same lineage-first path as regenerate; it must pass `head_checkpoint` or it silently degrades to the chronological scan that cannot tell sibling branches apart. +An edit replay that pauses for tool approval stays the current attempt: the edit +remains in thread state and replaces the source turn in history. Only a failed, +timed-out or cancelled attempt restores the pre-edit checkpoint; once the pause is +answered, the resume run is an ordinary run and its failure keeps the edit. ### Chat archive diff --git a/backend/packages/harness/deerflow/runtime/runs/manager.py b/backend/packages/harness/deerflow/runtime/runs/manager.py index 3002411e28..98225fb097 100644 --- a/backend/packages/harness/deerflow/runtime/runs/manager.py +++ b/backend/packages/harness/deerflow/runtime/runs/manager.py @@ -1379,7 +1379,7 @@ class RunManager: hidden_sources: set[str] = set() for source, (_, status) in latest_attempt_by_source.items(): - if status in {RunStatus.pending.value, RunStatus.running.value, RunStatus.success.value}: + if status in {RunStatus.pending.value, RunStatus.running.value, RunStatus.waiting_for_input.value, RunStatus.success.value}: hidden_sources.add(source) return EditReplayVisibility( hidden_source_run_ids=hidden_sources, diff --git a/backend/packages/harness/deerflow/runtime/runs/worker.py b/backend/packages/harness/deerflow/runtime/runs/worker.py index 6307be023b..f783a41fc1 100644 --- a/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -2076,7 +2076,9 @@ async def run_agent( # A rejected resume has not written state; compensating it through # the state-only graph would itself discard the pending Sends. - if not record.ownership_lost and _is_edit_replay_run(record) and record.status != RunStatus.success and not checkpoint_resume_rejected: + # A native pause has not failed: restoring the pre-run checkpoint + # would drop the edit and strand the interrupt behind the new head. + if not record.ownership_lost and _is_edit_replay_run(record) and record.status not in (RunStatus.success, RunStatus.waiting_for_input) and not checkpoint_resume_rejected: if not record.finalizing: await run_manager.set_finalizing(run_id, True) try: diff --git a/backend/tests/test_history_batch_queries.py b/backend/tests/test_history_batch_queries.py index 75db6e2958..fb9ae2c640 100644 --- a/backend/tests/test_history_batch_queries.py +++ b/backend/tests/test_history_batch_queries.py @@ -431,10 +431,15 @@ async def test_run_manager_computes_edit_replay_visibility_by_latest_attempt(): metadata={"replay_kind": "edit", "regenerate_from_run_id": "source-retry-success"}, ) newer_success_after_failure.status = RunStatus.success + paused = await manager.create( + "t1", + metadata={"replay_kind": "edit", "regenerate_from_run_id": "source-paused"}, + ) + paused.status = RunStatus.waiting_for_input visibility = await manager.list_edit_replay_visibility("t1", user_id=None) assert visibility == EditReplayVisibility( - hidden_source_run_ids={"source-running", "source-success", "source-retry-success"}, + hidden_source_run_ids={"source-running", "source-success", "source-retry-success", "source-paused"}, hidden_attempt_run_ids={failed.run_id, newer_failed.run_id, older_failed_then_success.run_id}, ) diff --git a/backend/tests/test_hitl_approval_api.py b/backend/tests/test_hitl_approval_api.py index 077a5763a5..71e372ee8d 100644 --- a/backend/tests/test_hitl_approval_api.py +++ b/backend/tests/test_hitl_approval_api.py @@ -12,7 +12,7 @@ from unittest.mock import patch import pytest from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel -from langchain_core.messages import AIMessage, ToolMessage +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from langchain_core.outputs import ChatGeneration, ChatResult from starlette.testclient import TestClient from test_runtime_lifecycle_e2e import ( # noqa: F401 @@ -853,3 +853,127 @@ async def test_pending_interrupts_fall_back_to_sync_saver_history(): assert recovered_task_id in pending assert recovered_interrupt.value == payload assert await has_any_pending_interrupt(SimpleNamespace(checkpointer=saver), config) is True + + +_ORIGINAL_PROMPT = "original prompt" +_EDITED_PROMPT = "edited prompt" + + +class _EditReplayModel(_StateAwareModel): + """Answer the original prompt in text; the edited prompt needs an approved tool.""" + + def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult: + if not any(isinstance(message, HumanMessage) and _EDITED_PROMPT in str(message.content) for message in messages): + message = AIMessage(content="original answer") + elif any(isinstance(message, ToolMessage) for message in messages): + message = AIMessage(content="edited answer") + else: + message = AIMessage(content="", tool_calls=[{"name": "write_file", "args": {"path": "/mnt/user-data/workspace/edited.txt", "content": "ok"}, "id": "call_edit_1"}]) + return ChatResult(generations=[ChatGeneration(message=message)]) + + +def _fake_edit_replay_model(*args, **kwargs): + return _EditReplayModel(responses=[]) + + +def _pause_edit_replay(client, csrf, tid) -> str: + """Finish a turn, edit it, and return the source run once the replay's worker has finalized its pause.""" + source = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": _ORIGINAL_PROMPT}]})) + assert source.status_code == 200, source.text + source_run_id = source.json()["run_id"] + assert _wait_run_terminal(client, tid, source_run_id)["status"] == "success" + + messages = client.get(f"/api/threads/{tid}").json()["values"]["messages"] + human_id = next(message["id"] for message in messages if message.get("type") == "human" and _ORIGINAL_PROMPT in str(message.get("content"))) + prepared = client.post( + f"/api/threads/{tid}/runs/edit-regenerate/prepare", + json={"human_message_id": human_id, "replacement_text": _EDITED_PROMPT}, + headers={"X-CSRF-Token": csrf}, + ) + assert prepared.status_code == 200, prepared.text + body = prepared.json() + replay = _run(client, csrf, tid, _run_body(input=body["input"], checkpoint=body["checkpoint"], metadata=body["metadata"])) + assert replay.status_code == 200, replay.text + replay_run_id = replay.json()["run_id"] + assert _wait_run_status(client, tid, replay_run_id, "waiting_for_input")["status"] == "waiting_for_input" + # Join returns after the worker's finalizer, which is where a failed edit rolls back. + assert client.get(f"/api/threads/{tid}/runs/{replay_run_id}/join").status_code == 200 + return source_run_id + + +def _human_texts(client, tid) -> list[str]: + return [str(message.get("content")) for message in client.get(f"/api/threads/{tid}").json()["values"]["messages"] if message.get("type") == "human"] + + +def _assert_history_shows_only_the_edit(client, tid, source_run_id) -> None: + page = client.get(f"/api/threads/{tid}/messages/page") + assert page.status_code == 200, page.text + rows = page.json()["data"] + assert not [row for row in rows if row.get("run_id") == source_run_id] + assert any(_EDITED_PROMPT in str(row.get("content")) for row in rows) + + +def test_edit_replay_paused_for_approval_keeps_the_edit(isolated_app): + _enable_approval() + with patch(_MODEL_PATCH, new=_fake_edit_replay_model), TestClient(isolated_app) as client: + csrf = _register_user(client) + tid = _create_thread(client, csrf) + source_run_id = _pause_edit_replay(client, csrf, tid) + + humans = _human_texts(client, tid) + assert any(_EDITED_PROMPT in text for text in humans), humans + assert not any(_ORIGINAL_PROMPT in text for text in humans), humans + assert _approval_request_id(client, tid, "call_edit_1") + _assert_history_shows_only_the_edit(client, tid, source_run_id) + + +@pytest.mark.parametrize("option_id", ["approve", "reject"]) +def test_answering_a_paused_edit_replay_finishes_the_edit_and_frees_the_thread(isolated_app, option_id): + _enable_approval() + with patch(_MODEL_PATCH, new=_fake_edit_replay_model), TestClient(isolated_app) as client: + csrf = _register_user(client) + tid = _create_thread(client, csrf) + source_run_id = _pause_edit_replay(client, csrf, tid) + + resumed = _resume(client, csrf, tid, option_id, tool_call_id="call_edit_1") + assert resumed.status_code == 200, resumed.text + assert _wait_run_terminal(client, tid, resumed.json()["run_id"])["status"] == "success" + + state = client.get(f"/api/threads/{tid}").json() + tool_results = [message["content"] for message in state["values"]["messages"] if message.get("type") == "tool"] + assert len(tool_results) == 1 + assert (tool_results[0] == "OK") is (option_id == "approve") + assert state["values"]["messages"][-1]["content"] == "edited answer" + assert not state.get("interrupts") + _assert_history_shows_only_the_edit(client, tid, source_run_id) + + next_turn = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "a later question"}]})) + assert next_turn.status_code == 200, next_turn.text + assert _wait_run_terminal(client, tid, next_turn.json()["run_id"])["status"] == "success" + + +def test_failed_resume_of_an_approved_edit_replay_keeps_the_edit(isolated_app): + class FailingAfterApprovalModel(_EditReplayModel): + def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult: + if any(isinstance(message, ToolMessage) for message in messages) and not any(isinstance(message, HumanMessage) and "a later question" in str(message.content) for message in messages): + raise RuntimeError("provider failed after approval") + return super()._generate(messages, stop, run_manager, **kwargs) + + _enable_approval() + with patch(_MODEL_PATCH, new=lambda *args, **kwargs: FailingAfterApprovalModel(responses=[])), TestClient(isolated_app) as client: + csrf = _register_user(client) + tid = _create_thread(client, csrf) + source_run_id = _pause_edit_replay(client, csrf, tid) + + resumed = _resume(client, csrf, tid, "approve", tool_call_id="call_edit_1") + assert resumed.status_code == 200, resumed.text + assert _wait_run_terminal(client, tid, resumed.json()["run_id"], timeout=60.0)["status"] == "error" + + humans = _human_texts(client, tid) + assert any(_EDITED_PROMPT in text for text in humans), humans + assert not any(_ORIGINAL_PROMPT in text for text in humans), humans + _assert_history_shows_only_the_edit(client, tid, source_run_id) + + next_turn = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "a later question"}]})) + assert next_turn.status_code == 200, next_turn.text + assert _wait_run_terminal(client, tid, next_turn.json()["run_id"], timeout=60.0)["status"] == "success"