diff --git a/backend/docs/RUN_INTERACTION_POLICY.md b/backend/docs/RUN_INTERACTION_POLICY.md index 8e49d4cf73..4d48c5e39e 100644 --- a/backend/docs/RUN_INTERACTION_POLICY.md +++ b/backend/docs/RUN_INTERACTION_POLICY.md @@ -80,7 +80,8 @@ rejected, so continue the work in a new thread. In-place recovery is tracked in [#8071](https://github.com/bytedance/deer-flow/issues/8071). `waiting_for_input` is not terminal success or failure. Goals, delivery checks, -and batch continuation must preserve a native pause. New runs cannot discard it. +batch continuation and edit-replay rollback must preserve a native pause. New runs +cannot discard it. Root checkpoint approvals do not expire and block new runs until resolved; live child approvals retain the time limit described above. See [HITL Tool Approval](CONFIGURATION.md#hitl-tool-approval) for grant scoping, diff --git a/backend/docs/THREAD_LIFECYCLE.md b/backend/docs/THREAD_LIFECYCLE.md index 902dffa640..0f8b63e094 100644 --- a/backend/docs/THREAD_LIFECYCLE.md +++ b/backend/docs/THREAD_LIFECYCLE.md @@ -26,6 +26,10 @@ tasks; absence of evidence stays permissive there rather than failing closed. Edit replay resolves its base through the same lineage-first path as regenerate; it must pass `head_checkpoint` or it silently degrades to the chronological scan that cannot tell sibling branches apart. +An edit replay that pauses for tool approval stays the current attempt: the edit +remains in thread state and replaces the source turn in history. Only a failed, +timed-out or cancelled attempt restores the pre-edit checkpoint; once the pause is +answered, the resume run is an ordinary run and its failure keeps the edit. ### Chat archive diff --git a/backend/packages/harness/deerflow/runtime/runs/manager.py b/backend/packages/harness/deerflow/runtime/runs/manager.py index 3002411e28..98225fb097 100644 --- a/backend/packages/harness/deerflow/runtime/runs/manager.py +++ b/backend/packages/harness/deerflow/runtime/runs/manager.py @@ -1379,7 +1379,7 @@ class RunManager: hidden_sources: set[str] = set() for source, (_, status) in latest_attempt_by_source.items(): - if status in {RunStatus.pending.value, RunStatus.running.value, RunStatus.success.value}: + if status in {RunStatus.pending.value, RunStatus.running.value, RunStatus.waiting_for_input.value, RunStatus.success.value}: hidden_sources.add(source) return EditReplayVisibility( hidden_source_run_ids=hidden_sources, diff --git a/backend/packages/harness/deerflow/runtime/runs/worker.py b/backend/packages/harness/deerflow/runtime/runs/worker.py index 6307be023b..f783a41fc1 100644 --- a/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -2076,7 +2076,9 @@ async def run_agent( # A rejected resume has not written state; compensating it through # the state-only graph would itself discard the pending Sends. - if not record.ownership_lost and _is_edit_replay_run(record) and record.status != RunStatus.success and not checkpoint_resume_rejected: + # A native pause has not failed: restoring the pre-run checkpoint + # would drop the edit and strand the interrupt behind the new head. + if not record.ownership_lost and _is_edit_replay_run(record) and record.status not in (RunStatus.success, RunStatus.waiting_for_input) and not checkpoint_resume_rejected: if not record.finalizing: await run_manager.set_finalizing(run_id, True) try: diff --git a/backend/tests/test_history_batch_queries.py b/backend/tests/test_history_batch_queries.py index 75db6e2958..fb9ae2c640 100644 --- a/backend/tests/test_history_batch_queries.py +++ b/backend/tests/test_history_batch_queries.py @@ -431,10 +431,15 @@ async def test_run_manager_computes_edit_replay_visibility_by_latest_attempt(): metadata={"replay_kind": "edit", "regenerate_from_run_id": "source-retry-success"}, ) newer_success_after_failure.status = RunStatus.success + paused = await manager.create( + "t1", + metadata={"replay_kind": "edit", "regenerate_from_run_id": "source-paused"}, + ) + paused.status = RunStatus.waiting_for_input visibility = await manager.list_edit_replay_visibility("t1", user_id=None) assert visibility == EditReplayVisibility( - hidden_source_run_ids={"source-running", "source-success", "source-retry-success"}, + hidden_source_run_ids={"source-running", "source-success", "source-retry-success", "source-paused"}, hidden_attempt_run_ids={failed.run_id, newer_failed.run_id, older_failed_then_success.run_id}, ) diff --git a/backend/tests/test_hitl_approval_api.py b/backend/tests/test_hitl_approval_api.py index 077a5763a5..71e372ee8d 100644 --- a/backend/tests/test_hitl_approval_api.py +++ b/backend/tests/test_hitl_approval_api.py @@ -12,7 +12,7 @@ from unittest.mock import patch import pytest from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel -from langchain_core.messages import AIMessage, ToolMessage +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from langchain_core.outputs import ChatGeneration, ChatResult from starlette.testclient import TestClient from test_runtime_lifecycle_e2e import ( # noqa: F401 @@ -853,3 +853,127 @@ async def test_pending_interrupts_fall_back_to_sync_saver_history(): assert recovered_task_id in pending assert recovered_interrupt.value == payload assert await has_any_pending_interrupt(SimpleNamespace(checkpointer=saver), config) is True + + +_ORIGINAL_PROMPT = "original prompt" +_EDITED_PROMPT = "edited prompt" + + +class _EditReplayModel(_StateAwareModel): + """Answer the original prompt in text; the edited prompt needs an approved tool.""" + + def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult: + if not any(isinstance(message, HumanMessage) and _EDITED_PROMPT in str(message.content) for message in messages): + message = AIMessage(content="original answer") + elif any(isinstance(message, ToolMessage) for message in messages): + message = AIMessage(content="edited answer") + else: + message = AIMessage(content="", tool_calls=[{"name": "write_file", "args": {"path": "/mnt/user-data/workspace/edited.txt", "content": "ok"}, "id": "call_edit_1"}]) + return ChatResult(generations=[ChatGeneration(message=message)]) + + +def _fake_edit_replay_model(*args, **kwargs): + return _EditReplayModel(responses=[]) + + +def _pause_edit_replay(client, csrf, tid) -> str: + """Finish a turn, edit it, and return the source run once the replay's worker has finalized its pause.""" + source = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": _ORIGINAL_PROMPT}]})) + assert source.status_code == 200, source.text + source_run_id = source.json()["run_id"] + assert _wait_run_terminal(client, tid, source_run_id)["status"] == "success" + + messages = client.get(f"/api/threads/{tid}").json()["values"]["messages"] + human_id = next(message["id"] for message in messages if message.get("type") == "human" and _ORIGINAL_PROMPT in str(message.get("content"))) + prepared = client.post( + f"/api/threads/{tid}/runs/edit-regenerate/prepare", + json={"human_message_id": human_id, "replacement_text": _EDITED_PROMPT}, + headers={"X-CSRF-Token": csrf}, + ) + assert prepared.status_code == 200, prepared.text + body = prepared.json() + replay = _run(client, csrf, tid, _run_body(input=body["input"], checkpoint=body["checkpoint"], metadata=body["metadata"])) + assert replay.status_code == 200, replay.text + replay_run_id = replay.json()["run_id"] + assert _wait_run_status(client, tid, replay_run_id, "waiting_for_input")["status"] == "waiting_for_input" + # Join returns after the worker's finalizer, which is where a failed edit rolls back. + assert client.get(f"/api/threads/{tid}/runs/{replay_run_id}/join").status_code == 200 + return source_run_id + + +def _human_texts(client, tid) -> list[str]: + return [str(message.get("content")) for message in client.get(f"/api/threads/{tid}").json()["values"]["messages"] if message.get("type") == "human"] + + +def _assert_history_shows_only_the_edit(client, tid, source_run_id) -> None: + page = client.get(f"/api/threads/{tid}/messages/page") + assert page.status_code == 200, page.text + rows = page.json()["data"] + assert not [row for row in rows if row.get("run_id") == source_run_id] + assert any(_EDITED_PROMPT in str(row.get("content")) for row in rows) + + +def test_edit_replay_paused_for_approval_keeps_the_edit(isolated_app): + _enable_approval() + with patch(_MODEL_PATCH, new=_fake_edit_replay_model), TestClient(isolated_app) as client: + csrf = _register_user(client) + tid = _create_thread(client, csrf) + source_run_id = _pause_edit_replay(client, csrf, tid) + + humans = _human_texts(client, tid) + assert any(_EDITED_PROMPT in text for text in humans), humans + assert not any(_ORIGINAL_PROMPT in text for text in humans), humans + assert _approval_request_id(client, tid, "call_edit_1") + _assert_history_shows_only_the_edit(client, tid, source_run_id) + + +@pytest.mark.parametrize("option_id", ["approve", "reject"]) +def test_answering_a_paused_edit_replay_finishes_the_edit_and_frees_the_thread(isolated_app, option_id): + _enable_approval() + with patch(_MODEL_PATCH, new=_fake_edit_replay_model), TestClient(isolated_app) as client: + csrf = _register_user(client) + tid = _create_thread(client, csrf) + source_run_id = _pause_edit_replay(client, csrf, tid) + + resumed = _resume(client, csrf, tid, option_id, tool_call_id="call_edit_1") + assert resumed.status_code == 200, resumed.text + assert _wait_run_terminal(client, tid, resumed.json()["run_id"])["status"] == "success" + + state = client.get(f"/api/threads/{tid}").json() + tool_results = [message["content"] for message in state["values"]["messages"] if message.get("type") == "tool"] + assert len(tool_results) == 1 + assert (tool_results[0] == "OK") is (option_id == "approve") + assert state["values"]["messages"][-1]["content"] == "edited answer" + assert not state.get("interrupts") + _assert_history_shows_only_the_edit(client, tid, source_run_id) + + next_turn = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "a later question"}]})) + assert next_turn.status_code == 200, next_turn.text + assert _wait_run_terminal(client, tid, next_turn.json()["run_id"])["status"] == "success" + + +def test_failed_resume_of_an_approved_edit_replay_keeps_the_edit(isolated_app): + class FailingAfterApprovalModel(_EditReplayModel): + def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult: + if any(isinstance(message, ToolMessage) for message in messages) and not any(isinstance(message, HumanMessage) and "a later question" in str(message.content) for message in messages): + raise RuntimeError("provider failed after approval") + return super()._generate(messages, stop, run_manager, **kwargs) + + _enable_approval() + with patch(_MODEL_PATCH, new=lambda *args, **kwargs: FailingAfterApprovalModel(responses=[])), TestClient(isolated_app) as client: + csrf = _register_user(client) + tid = _create_thread(client, csrf) + source_run_id = _pause_edit_replay(client, csrf, tid) + + resumed = _resume(client, csrf, tid, "approve", tool_call_id="call_edit_1") + assert resumed.status_code == 200, resumed.text + assert _wait_run_terminal(client, tid, resumed.json()["run_id"], timeout=60.0)["status"] == "error" + + humans = _human_texts(client, tid) + assert any(_EDITED_PROMPT in text for text in humans), humans + assert not any(_ORIGINAL_PROMPT in text for text in humans), humans + _assert_history_shows_only_the_edit(client, tid, source_run_id) + + next_turn = _run(client, csrf, tid, _run_body(input={"messages": [{"role": "user", "content": "a later question"}]})) + assert next_turn.status_code == 200, next_turn.text + assert _wait_run_terminal(client, tid, next_turn.json()["run_id"], timeout=60.0)["status"] == "success"