From c48447a4aee5c69d6e0d796f5790e9d2fe9759de Mon Sep 17 00:00:00 2001 From: Totoro Date: Tue, 27 Oct 2026 10:34:34 +0800 Subject: [PATCH] fix(runtime): present outputs before a loop wrap-up and explain step-limit stops (#7588) * fix(runtime): present outputs before a loop wrap-up and explain step-limit stops Loop-detection warnings now ask the agent to present the files it already created in /mnt/user-data/outputs before its final answer, when present_files is available. A run that exhausts recursion_limit now ends with stop_reason recursion_capped and a plain message that lists outputs it created but did not present. An active goal stands down without an evaluator call. The /wait error envelope keeps the GraphRecursionError name, and IM channels reply with the step-limit message instead of a generic failure. * docs(changelog): note run-end delivery and step-limit stops (#7588) * test(runtime): stop pinning LangGraph's step count in the step-limit test The produced paths and the message already follow the observed model calls, so the exact count only pinned LangGraph's per-step accounting. Also note on the step-limit goal branch that its caller passes no evaluator factory, so it has to stay ahead of the evaluator section. --------- Co-authored-by: Totoro-qaq <279883115+Totoro-qaq@users.noreply.github.com> Co-authored-by: Willem Jiang --- CHANGELOG.md | 11 ++ CHANGELOG_zh.md | 8 + README.md | 9 +- backend/app/channels/manager.py | 35 ++++ backend/app/gateway/routers/thread_runs.py | 4 +- backend/app/gateway/services.py | 6 +- backend/app/scheduler/notification_text.py | 2 + backend/docs/API.md | 6 +- backend/docs/CONFIGURATION.md | 5 + .../deerflow/agents/middlewares/AGENTS.md | 2 +- .../middlewares/loop_detection_middleware.py | 11 +- .../harness/deerflow/runtime/AGENTS.md | 2 +- .../harness/deerflow/runtime/runs/worker.py | 82 ++++++++- backend/tests/AGENTS.md | 2 + backend/tests/test_channel_stream_errors.py | 46 +++++ backend/tests/test_channels.py | 34 ++++ .../tests/test_loop_detection_middleware.py | 42 +++++ backend/tests/test_run_recursion_capped.py | 164 ++++++++++++++++++ .../test_scheduled_goal_notes_contract.py | 1 + backend/tests/test_thread_goal_contract.py | 2 +- backend/tests/test_thread_run_wait_errors.py | 27 +++ contracts/scheduled_goal_notes_contract.json | 1 + contracts/thread_goal_contract.json | 4 +- .../workspace/goal-status-helpers.ts | 2 + .../src/components/workspace/goal-status.tsx | 1 + frontend/src/core/i18n/locales/en-US.ts | 3 + frontend/src/core/i18n/locales/types.ts | 2 + frontend/src/core/i18n/locales/zh-CN.ts | 3 + .../src/core/scheduled-tasks/goal-outcome.ts | 2 + frontend/tests/e2e/goal-status.spec.ts | 21 +++ frontend/tests/e2e/utils/raw-identifiers.ts | 2 +- 31 files changed, 527 insertions(+), 15 deletions(-) create mode 100644 backend/tests/test_run_recursion_capped.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b37212814d..3187695341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1402,6 +1402,16 @@ As of 2026-10-21, the milestone includes **829 merged pull requests**. ### Fixed +- **runtime:** Loop-detection warnings now ask the agent to present the files + it already created in `/mnt/user-data/outputs` before its final answer, so a + warned run no longer fails its delivery check with finished files left + undelivered. A run that reaches `recursion_limit` now ends with + `stop_reason: recursion_capped` and a plain message that lists the outputs it + created but did not present, instead of LangGraph's raw error. An active goal + stands down without an evaluator call and stays active, `/wait` names the + error `GraphRecursionError`, and IM channels reply with the message instead + of a generic failure. ([#7588]) + - **channels:** Connecting Slack, Discord, Feishu, DingTalk, WeCom, Buzz or QQ now opens a dialog that keeps the `/connect` command on screen with a copy button, its expiry time and the binding status, instead of a toast that @@ -13167,4 +13177,5 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#7560]: https://github.com/bytedance/deer-flow/pull/7560 [#7576]: https://github.com/bytedance/deer-flow/pull/7576 [#7580]: https://github.com/bytedance/deer-flow/pull/7580 +[#7588]: https://github.com/bytedance/deer-flow/pull/7588 [#7591]: https://github.com/bytedance/deer-flow/pull/7591 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 354e256c0a..6cb3b16868 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -1116,6 +1116,13 @@ ### 修复 +- **运行时：** 循环检测警告现在会让 agent 在作答前，先交付已经写入 + `/mnt/user-data/outputs` 的文件，被警告的运行不会再因为做好的文件没交付而判为失败。 + 运行达到 `recursion_limit` 时，现在以 `stop_reason: recursion_capped` 结束，并给出 + 一条说明，列出已生成但未交付的文件，而不是 LangGraph 的原始报错。进行中的目标会直接 + 停止续跑（不调用评估模型）并保持活跃；`/wait` 把这个错误命名为 + `GraphRecursionError`；IM 渠道也会回复这条说明，而不是笼统的错误提示。([#7588]) + - **渠道：** 连接 Slack、Discord、飞书、钉钉、企业微信、Buzz 或 QQ 时，现在会打开 一个对话框，持续显示 `/connect` 命令，并提供复制按钮、过期时间和绑定状态，不再只是 一闪而过的提示。机器人对 `/connect` 和 `/goal` 的回复会跟随用户在网页端选择的语言 @@ -10510,4 +10517,5 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#7580]: https://github.com/bytedance/deer-flow/pull/7580 [#7576]: https://github.com/bytedance/deer-flow/pull/7576 [#7580]: https://github.com/bytedance/deer-flow/pull/7580 +[#7588]: https://github.com/bytedance/deer-flow/pull/7588 [#7591]: https://github.com/bytedance/deer-flow/pull/7591 diff --git a/README.md b/README.md index 8bd7b5371c..8172e5e190 100644 --- a/README.md +++ b/README.md @@ -542,7 +542,10 @@ apply to the next run without restarting the Gateway. This top-level setting applies to Gateway API and IM channel runs. A `recursion_limit` in an IM channel's `session.config` takes precedence, and GitHub channel runs keep their own floor. Embedded `DeerFlowClient` runs retain their own defaults and -per-call override paths. +per-call override paths. A run that reaches its limit stops with a plain message +that names up to five files it created in `/mnt/user-data/outputs` without +presenting them; send a message to continue. IM channels reply with the same +message. A custom frontend can add an optional `context.client` to a Gateway run request: a client name, capabilities such as `artifacts` or `charts`, and display @@ -2474,7 +2477,7 @@ After each Gateway-backed run, DeerFlow evaluates the visible conversation, incl The evaluator keeps recent work and the latest user request within a fixed evidence budget, using spare room to retain more of long user input. If a normally completed evaluator response has an invalid verdict format, DeerFlow evaluates the same evidence once more unless you cancel or change the thread. Both calls are counted; a second failure stops automatic continuation without repeating the task or its tools. -For tool-using turns, successful operation results drive the no-progress count: changing only reply wording or a sandbox tool's UI explanation does not reset it, while newly observed work can keep it going. Text-only turns keep their reply-based check. An unmet goal still stops automatic continuation after a token or loop-detection hard stop and stays active for a later user turn. +For tool-using turns, successful operation results drive the no-progress count: changing only reply wording or a sandbox tool's UI explanation does not reset it, while newly observed work can keep it going. Text-only turns keep their reply-based check. An unmet goal still stops automatic continuation after a token or loop-detection hard stop, or when the run reaches its step limit, and stays active for a later user turn. Regenerating an earlier response keeps the goal currently set for the chat and its used continuation budget. It does not restore a goal that was cleared or achieved. The earlier answer's completion record is removed when its historical checkpoint is replayed. @@ -2941,7 +2944,7 @@ The read-before-write gate ties each read mark to that `read_file` call's result On Linux and macOS, local sandbox `write_file` replacements and `str_replace` stage complete text before replacing the destination, so write, flush or publication errors retain the previous file. Existing write access is checked; ownership, permission bits, ACLs and extended attributes are transferred before writing staging content. Failure to transfer metadata aborts the save. Successful revisions get a fresh modification time, including empty saves; source-descriptor cleanup errors are warnings and do not change the save result. Files with set-id bits, macOS file flags or content-bound Linux privilege attributes are refused unchanged. Replacement requires parent-directory write access and publishes a new inode; other hard links retain the previous version. Symlinks follow the existing path-validation rules. Append writes, other host platforms (including Windows), shell writes and remote providers keep their existing behavior. This is not a concurrent-writer or power-loss recovery guarantee. -Loop warnings let agents save collected results before their final answer. If the save is blocked by the read-before-write gate, the agent may read the target file and retry the save once. A looping or stopped tool must be replaced by an available alternative; if no permitted save path exists or the retry fails, the final answer includes the collected results and reports that the file was not saved. Recovery calls still count toward the loop limits, and a hard stop prevents further tool execution. +Loop warnings let agents save collected results before their final answer. If the save is blocked by the read-before-write gate, the agent may read the target file and retry the save once. A looping or stopped tool must be replaced by an available alternative; if no permitted save path exists or the retry fails, the final answer includes the collected results and reports that the file was not saved. When `present_files` is available, the warning also asks the agent to present the files it already created in `/mnt/user-data/outputs` before answering, so finished outputs are delivered. Recovery calls still count toward the loop limits, and a hard stop prevents further tool execution. Ranged `read_file` calls count lines the same way on every sandbox provider: a line ends only at a newline. On E2B, BoxLite, Tenki, and OpenSandbox, a file with bare carriage returns (such as a saved progress log), form feeds, or Unicode line separators therefore returns the same lines as on the local sandbox, and the `start_line` a truncated read suggests points at the next unread line. diff --git a/backend/app/channels/manager.py b/backend/app/channels/manager.py index 373b8301a6..829e0d293c 100644 --- a/backend/app/channels/manager.py +++ b/backend/app/channels/manager.py @@ -567,6 +567,28 @@ def _is_admitted_run_conflict(exc: BaseException | None) -> bool: return isinstance(exc, ConflictError) and any(detail in str(exc) for detail in _ADMITTED_RUN_CONFLICT_DETAILS) +# The Gateway names a run that stopped at its step limit after LangGraph's +# GraphRecursionError, in its stream error frame and its /wait error. The +# message says the run stopped and lists the outputs it never presented. +_STEP_LIMIT_ERROR_NAME = "GraphRecursionError" + + +def _step_limit_text(error: Any) -> str | None: + """The message of a step-limit stream error frame or ``/wait`` error, else ``None``. + + The SDK raises a ``/wait`` error envelope as ``Exception(": ")``. + """ + if isinstance(error, BaseException): + name, _, message = str(error).partition(": ") + elif isinstance(error, dict): + name, message = error.get("name"), error.get("message") + else: + return None + if name != _STEP_LIMIT_ERROR_NAME or not isinstance(message, str) or not message.strip(): + return None + return message.strip() + + def _is_run_status_result(result: Any) -> bool: """A ``/wait`` response carrying run status instead of thread state. @@ -3414,6 +3436,13 @@ class ChannelManager: await self._release_inbound_dedupe_key(msg) await self._send_error(msg, THREAD_BUSY_MESSAGE) return + if step_limit_text := _step_limit_text(exc): + logger.warning("[Manager] run stopped at its step limit: thread_id=%s", thread_id) + await self._send_error(msg, step_limit_text) + # The run ended, as after a terminal error frame: release the key + # only after the reply, so a redelivery cannot overtake it. + await self._release_inbound_dedupe_key(msg) + return if recovery := await self._stale_pin_recovery(msg, exc, expected_pin=expected_pin, thread_id=thread_id): await self._release_inbound_dedupe_key(msg) await self._send_error(msg, recovery.text) @@ -3508,6 +3537,7 @@ class ChannelManager: last_published_len = 0 last_publish_at = 0.0 stream_error: BaseException | None = None + step_limit_text: str | None = None replay_gap = False pin_recovery_text: str | None = None pinned_model_value = run_context.get("model_name") @@ -3551,6 +3581,7 @@ class ChannelManager: # The v1 SDK yields backend errors as frames, not exceptions. # Keep draining the stream and use the existing failure path. stream_error = _RunFailedFrame("Agent run failed.") + step_limit_text = _step_limit_text(error_data) elif event == "gap": # A reused run whose events are no longer replayable. replay_gap = True @@ -3616,6 +3647,10 @@ class ChannelManager: # matches the upload bucket and we avoid re-running _safe_user_id_for_run # (and its possible filesystem touch) on the streaming-error path. response_text, attachments = _prepare_artifact_delivery(thread_id, response_text, artifacts, user_id=storage_user_id) + if step_limit_text: + # The last text is mid-task narration: say the run stopped, in + # place of the generic failure, so the user knows to continue. + response_text = f"{response_text}\n\n{step_limit_text}" if response_text else step_limit_text if not response_text: if attachments: diff --git a/backend/app/gateway/routers/thread_runs.py b/backend/app/gateway/routers/thread_runs.py index ade649cd33..e1e041d718 100644 --- a/backend/app/gateway/routers/thread_runs.py +++ b/backend/app/gateway/routers/thread_runs.py @@ -130,7 +130,7 @@ def _scope_http_run_idempotency_key(request: Request, thread_id: str, key: str | async def _refresh_store_backed_run(run_mgr: Any, record: Any) -> Any: - """Overlay durable status/error onto a hydrated store-only record.""" + """Overlay durable status/error/stop reason onto a hydrated store-only record.""" if not getattr(record, "store_only", False): return record store = getattr(run_mgr, "_store", None) @@ -154,6 +154,8 @@ async def _refresh_store_backed_run(run_mgr: Any, record: Any) -> Any: pass if "error" in row: record.error = row.get("error") + if "stop_reason" in row: + record.stop_reason = row.get("stop_reason") return record diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index 435b500bc3..083555dccd 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -3076,7 +3076,11 @@ def serialize_wait_run_status(record: RunRecord) -> dict[str, Any]: """Return durable status and the error envelope recognized by the SDK.""" result: dict[str, Any] = {"status": record.status.value, "error": record.error} if record.status == RunStatus.error: - result["__error__"] = {"error": "RunError", "message": record.error or "Run failed."} + # A run stopped at its step limit keeps the error name of its stream + # error frame, so callers such as IM channels can show its message + # instead of a generic failure. + name = "GraphRecursionError" if getattr(record, "stop_reason", None) == "recursion_capped" else "RunError" + result["__error__"] = {"error": name, "message": record.error or "Run failed."} return result diff --git a/backend/app/scheduler/notification_text.py b/backend/app/scheduler/notification_text.py index 4ad4231e3c..1ff673f1f7 100644 --- a/backend/app/scheduler/notification_text.py +++ b/backend/app/scheduler/notification_text.py @@ -86,6 +86,7 @@ _GOAL_REASON_TEXT: dict[str, dict[str, str]] = { "no_progress_detected": "no progress was made between turns", "token_capped": "the token budget was reached", "loop_capped": "repeated tool calls reached the safety limit", + "recursion_capped": "the run reached its step limit", "no_durable_end_of_turn": "no final reply was saved", "thread_changed_after_evaluation": "the conversation changed during the goal check", "thread_changed_before_continuation": "the conversation changed during the goal check", @@ -104,6 +105,7 @@ _GOAL_REASON_TEXT: dict[str, dict[str, str]] = { "no_progress_detected": "多轮之间没有进展", "token_capped": "已达到 token 预算", "loop_capped": "重复工具调用触发了安全停止", + "recursion_capped": "运行达到了步数上限", "no_durable_end_of_turn": "没有保存最终回复", "thread_changed_after_evaluation": "目标检查期间对话发生了变化", "thread_changed_before_continuation": "目标检查期间对话发生了变化", diff --git a/backend/docs/API.md b/backend/docs/API.md index c131397f88..10c6d8d5f6 100644 --- a/backend/docs/API.md +++ b/backend/docs/API.md @@ -385,7 +385,11 @@ the next request without restarting the Gateway. IM channel runs get the same default unless a channel `session.config` sets `recursion_limit` (GitHub channel runs have their own floor; see [CONFIGURATION.md](CONFIGURATION.md#recursion-limits)); embedded -`DeerFlowClient` runs retain their own defaults and override paths. +`DeerFlowClient` runs retain their own defaults and override paths. A run that +reaches the limit ends with status `error` and `stop_reason: "recursion_capped"`; +its stream `error` event and the `/wait` error envelope use the name +`GraphRecursionError`, and the message lists up to five outputs the run created +but did not present. **Configurable Options:** - `model_name` (string): Override the default model diff --git a/backend/docs/CONFIGURATION.md b/backend/docs/CONFIGURATION.md index 2a4bdd5344..d7edaba106 100644 --- a/backend/docs/CONFIGURATION.md +++ b/backend/docs/CONFIGURATION.md @@ -145,6 +145,11 @@ configured default. `max_recursion_limit` (default `1000`) caps both sources to limit runaway LLM cost. Both settings are read per run, so changes apply to the next request without a Gateway restart. +A run that reaches its limit ends with status `error` and +`stop_reason: recursion_capped`. Its error message lists up to five outputs it +created but did not present and asks the user to send a message to continue. +An active goal stands down without an evaluator call and stays active. + These settings also apply to IM channel runs. A `recursion_limit` in an IM channel's `session.config` (for all channels, one channel or one user) takes precedence. The GitHub channel instead uses `250`, or a higher diff --git a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md index 05e63052f1..e6a845af02 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md @@ -154,7 +154,7 @@ Before changing a later authorization phase, read the [authorization RFC](../../ Capture/retain delegations by `(run_id, tool_call_id)`. Later-run replies cannot close older orphans; missing run IDs match conservatively. 31. **LoopDetectionMiddleware** - `loop_detection.enabled`: detects repeated tool-call loops. Hard stops clear structured/raw/content-block calls, force a final text answer, and stamp `loop_capped` via `consume_stop_reason` (#3875 Phase 2, matching `TokenBudgetMiddleware`). Persist first warned-state transitions per call hash/tool-frequency burst and hard stops as `middleware:loop_detection`, with `is_subagent`/optional `agent_id`; omit arguments, message content, results and argument-derived hashes. Ordinary task subagents use dedicated recorder keys via a parent-loop proxy; never pass `RunJournal` into their isolated loop. Durable batch subagents have no parent journal and persist no transitions. New user runs reset budgets/history; same-run goal continuations share it. - Warnings allow a final save and one read-before-write retry through non-looping tools; otherwise report unsaved results. Hard stops never permit saves. + Warnings allow a final save, one read-before-write retry and presenting outputs via non-looping tools; otherwise report unsaved results. Hard stops never permit saves. Isolate sibling warnings and keep lifecycle hooks topology-stable. Before changing this guard, read [Loop detection lifecycle](../../../../../docs/LOOP_DETECTION.md) for fallback identity, cleanup/LRU/reset, severity ordering, and test invariants. diff --git a/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py index 0886c6eb18..3cdf40ea17 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py @@ -232,8 +232,10 @@ def _hash_tool_calls(tool_calls: list[dict]) -> str: # otherwise stop *all* tool calls, including the write that delivers its # result, and finish as "completed" with nothing on disk. So the wrap-up # allows a bounded save/read/retry before the final answer, without reusing -# a looping writer or bypassing the read-before-write gate. The hard-stop -# path below is unchanged (it strips tool calls). +# a looping writer or bypassing the read-before-write gate. It also names +# present_files: an agent told only to stop calling tools skipped presenting +# outputs it had already created, so the run failed its delivery check. The +# hard-stop path below is unchanged (it strips tool calls). _WRAP_UP_INSTRUCTIONS = ( "If your task asks you to save results to a file, first write what you have collected so far to that file " "with a file-writing tool (for example write_file, unless it is the looping tool). " @@ -241,7 +243,10 @@ _WRAP_UP_INSTRUCTIONS = ( "If a save is blocked by a read-before-write check, read the target file and retry the save once. " "Do not use a tool that this warning tells you to stop, or repeat a failed save loop. " "If no permitted save path is available or the retry fails, include the collected results in your final answer " - "and state that the file was not saved. Then stop calling tools and produce your final answer. " + "and state that the file was not saved. " + "If present_files is available, present the files you already created in /mnt/user-data/outputs " + "with one present_files call before answering, unless it is the looping tool. " + "Then stop calling tools and produce your final answer. " "If you cannot complete the task, summarize what you accomplished so far." ) diff --git a/backend/packages/harness/deerflow/runtime/AGENTS.md b/backend/packages/harness/deerflow/runtime/AGENTS.md index bec7c0d1e5..2b475ef7db 100644 --- a/backend/packages/harness/deerflow/runtime/AGENTS.md +++ b/backend/packages/harness/deerflow/runtime/AGENTS.md @@ -126,7 +126,7 @@ answers for paginated history. See `docs/skill-usage-ui.md`. **Workspace change detection:** Truncated snapshots cannot prove absence. Mark skipped delivery verification in the receipt. Details: `backend/docs/runtime-guidance-details.md`. -Unmet goals stand down on run-level token/loop hard stops and retain the original stop reason; hidden continuations must not re-enter that run's shared safety state. Operation-based no-progress counting: `backend/docs/runtime-guidance-details.md`. +Unmet goals stand down on run-level token/loop hard stops and retain the original stop reason; a step-limit stop (`recursion_capped`) stands down without an evaluator call. Hidden continuations must not re-enter that run's shared safety state. Operation-based no-progress counting: `backend/docs/runtime-guidance-details.md`. **Deferred terminal commit:** Keep terminal commit authority on the owning Worker's confirmed ACK; row reads never grant it. Renew leases through receipt, diff --git a/backend/packages/harness/deerflow/runtime/runs/worker.py b/backend/packages/harness/deerflow/runtime/runs/worker.py index adecf29549..56ae8aab78 100644 --- a/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -36,6 +36,7 @@ from typing import Any, Final, Literal, cast from deerflow_extension_api.agent_runs import AGENT_RUNS_CONTEXT_KEY from langchain_core.messages import AIMessage, AIMessageChunk from langgraph.checkpoint.base import empty_checkpoint +from langgraph.errors import GraphRecursionError from langgraph.types import Command, Overwrite from deerflow.agents.goal_state import GoalEvaluation, GoalState @@ -349,6 +350,26 @@ async def _persist_delivery_receipt( _DELIVERY_INCOMPLETE_ERROR = "Artifact delivery incomplete: no produced output artifact was presented" _DELIVERY_RECEIPT_FAILED_ERROR = "Artifact delivery verification failed: terminal delivery receipt could not be persisted" +_RECURSION_CAPPED_STOP_REASON = "recursion_capped" +_RECURSION_CAPPED_LISTED_OUTPUTS = 5 + + +def _recursion_capped_message(recursion_limit: object, delivery_content: dict[str, Any] | None) -> str: + """Say a run hit its step budget, listing outputs it created but never presented. + + LangGraph's own text points at a config key and a troubleshooting URL; users + see it as an unexplained failure and assume the files they waited for are lost. + """ + limit = f" (recursion_limit={recursion_limit})" if isinstance(recursion_limit, int) and not isinstance(recursion_limit, bool) else "" + message = f"The run stopped at its step limit{limit} before finishing." + content = delivery_content or {} + presented = set(content.get("matched_paths") or []) + unpresented = [path for path in content.get("produced_paths") or [] if path not in presented] + if unpresented: + listed = ", ".join(unpresented[:_RECURSION_CAPPED_LISTED_OUTPUTS]) + more = len(unpresented) - _RECURSION_CAPPED_LISTED_OUTPUTS + message += f" Created but not presented: {listed}{f' and {more} more' if more > 0 else ''}." + return f"{message} Send a message to continue." def _empty_delivery_content() -> dict[str, Any]: @@ -1755,13 +1776,54 @@ async def run_agent( except Exception as exc: checkpoint_resume_rejected = isinstance(exc, CheckpointResumeRejected) error_msg = f"{exc}" - logger.exception("Run %s failed: %s", run_id, error_msg) + stop_status_kwargs: dict[str, Any] = {} + if isinstance(exc, GraphRecursionError) and evidence_execution_started: + # The run used up its step budget: an expected stop, not a crash. + # Report it plainly with the outputs it never presented, and stand + # the goal down without an evaluator call. + logger.warning("Run %s stopped at its step limit: %s", run_id, error_msg) + produced_output_paths = await _produced_output_paths( + pre_run_workspace_snapshot, + thread_id=thread_id, + user_id=workspace_changes_user_id, + extra_excluded_dir_names=workspace_excluded_dir_names, + ) + delivery_content = _delivery_content_with_outputs( + journal.get_delivery_content() if journal is not None else _empty_delivery_content(), + produced_output_paths, + pre_run_snapshot=pre_run_workspace_snapshot, + ) + error_msg = _recursion_capped_message(config.get("recursion_limit"), delivery_content) + if not record.abort_event.is_set(): + await _prepare_goal_continuation_input( + bridge=bridge, + accessor=accessor, + checkpointer=checkpointer, + thread_id=thread_id, + run_id=run_id, + model_name=record.model_name, + app_config=ctx.app_config, + abort_event=record.abort_event, + user_id=resolve_runtime_user_id(runtime), + deerflow_trace_id=deerflow_trace_id, + task_store=task_store, + extensions=extensions, + run_stop_reason=_RECURSION_CAPPED_STOP_REASON, + interaction_policy=resolve_run_interaction_policy(config), + expected_goal=scheduled_goal, + verdict_callback=capture_goal_verdict, + ) + record.goal_verdict = final_goal_verdict + stop_status_kwargs = {"stop_reason": _RECURSION_CAPPED_STOP_REASON, "goal_verdict": record.goal_verdict} + else: + logger.exception("Run %s failed: %s", run_id, error_msg) await _ensure_stream_producer_lease() await _ensure_finalizing_before_edit_failure(run_manager, record) cancel_action = await run_manager.set_status_if_not_cancelled( run_id, RunStatus.error, error=error_msg, + **stop_status_kwargs, **terminal_status_kwargs, ) if cancel_action is not None: @@ -2241,6 +2303,9 @@ async def run_agent( # --------------------------------------------------------------------------- _GOAL_RUN_HARD_STOP_REASONS = frozenset({"token_capped", "loop_capped"}) +# Stops that end the run with an error before the turn has a final reply: the +# goal stands down without an evaluator call, since there is nothing to judge. +_GOAL_RUN_ERROR_STOP_REASONS = frozenset({_RECURSION_CAPPED_STOP_REASON}) @dataclass(frozen=True) @@ -2656,6 +2721,21 @@ async def _prepare_goal_continuation_input( evidence_signature = latest_visible_assistant_signature(messages) operation_progress = observe_goal_operations(goal, messages) + if run_stop_reason in _GOAL_RUN_ERROR_STOP_REASONS: + # Checked first: otherwise the stop would read as a missing receipt + # or cost an evaluator call, depending on the last message's text. + # Keep it ahead of the evaluator section: the step-limit caller in + # run_agent passes no evaluator_model_factory or usage_callback. + evaluation = GoalEvaluation( + satisfied=False, + blocker="run_failed", + reason="The run reached its step limit before finishing.", + evidence_summary="", + ) + no_progress_count = compute_no_progress_count(goal, evaluation, evidence_signature=evidence_signature, operation_progress=operation_progress) + await _persist(goal, evaluation, no_progress_count, stand_down_reason=run_stop_reason) + return None + if _ends_on_human_input_request(messages): # The agent asked the user something. Continuing would tell it to keep # going while the question is still open on screen. diff --git a/backend/tests/AGENTS.md b/backend/tests/AGENTS.md index 596fbafb20..2eefa38638 100644 --- a/backend/tests/AGENTS.md +++ b/backend/tests/AGENTS.md @@ -51,6 +51,8 @@ cleanup, preserve healthy controls in refusal cases, and retain Journal/Seal compatibility coverage. Protocol: [Run outcome](../docs/RUN_EVENT_STREAM.md#authoritative-run-outcome). `test_goal_loop_stop_probe.py` runs the standalone Goal-loop reproduction through the native graph, loop middleware, worker and checkpoints. Only model responses are scripted; do not replace the Goal decision or simulate the stop-reason write. +`test_run_recursion_capped.py` drives a real graph through the worker until LangGraph raises `GraphRecursionError`. Only model responses are scripted; do not raise the error or write the stop reason by hand. + InfoQuest fetch log assertions install the real `UrlRedactionFilter` on the capture handler with fixture-scoped cleanup. Expect scheme/host context with redacted paths, so isolated runs and full-suite logging configuration agree; diff --git a/backend/tests/test_channel_stream_errors.py b/backend/tests/test_channel_stream_errors.py index 498cf037b2..5413e9d0c9 100644 --- a/backend/tests/test_channel_stream_errors.py +++ b/backend/tests/test_channel_stream_errors.py @@ -105,3 +105,49 @@ async def test_channel_consumes_error_sse_and_releases_dedupe_after_final_publis else: assert "Error" in logged assert "unknown" in logged + + +_STEP_LIMIT_TEXT = "The run stopped at its step limit (recursion_limit=6) before finishing. Created but not presented: /mnt/user-data/outputs/report.md. Send a message to continue." + + +@pytest.mark.asyncio +@pytest.mark.parametrize("has_partial", [False, True], ids=["empty", "partial-text"]) +async def test_channel_reports_a_step_limit_stop_instead_of_a_generic_failure(monkeypatch, tmp_path, has_partial): + bridge = MemoryStreamBridge() + record = RunRecord( + run_id="stream-run", + thread_id="thread-1", + assistant_id=None, + status=RunStatus.error, + on_disconnect=DisconnectMode.continue_, + error=_STEP_LIMIT_TEXT, + stop_reason="recursion_capped", + ) + + async def start_run(*args, **kwargs): + messages = [{"type": "ai", "content": "Writing the report.", "tool_calls": [{"id": "w-1", "name": "write_file", "args": {}}]}] if has_partial else [] + await bridge.publish(record.run_id, "values", {"messages": messages}) + await bridge.publish(record.run_id, "error", {"name": "GraphRecursionError", "message": _STEP_LIMIT_TEXT}) + await bridge.publish_end(record.run_id) + return record + + monkeypatch.setattr(thread_runs, "start_run", start_run) + app = make_authed_test_app() + app.include_router(thread_runs.router) + app.state.stream_bridge = bridge + app.state.run_manager = SimpleNamespace(get=AsyncMock(return_value=record)) + bus = MessageBus() + manager = ChannelManager(bus=bus, store=JsonChannelStore(path=tmp_path / "channels.json")) + inbound = InboundMessage(channel_name="feishu", chat_id="chat-1", user_id="user-1", text="Write the report", metadata={"message_id": "message-1"}) + assert await manager._is_duplicate_inbound(inbound) is False + outbound = [] + bus.subscribe_outbound(outbound.append) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test/api") as http: + await manager._handle_streaming_chat(LangGraphClient(http), inbound, "thread-1", "lead_agent", {}, {}, {"role": "user", "content": "Write the report"}) + + final = outbound[-1] + assert final.is_final + # The narration stays; the stop replaces the generic failure. + assert final.text == (f"Writing the report.\n\n{_STEP_LIMIT_TEXT}" if has_partial else _STEP_LIMIT_TEXT) + # The run ended, so a redelivery may retry it, as after any error frame. + assert await manager._is_duplicate_inbound(inbound) is False diff --git a/backend/tests/test_channels.py b/backend/tests/test_channels.py index 14dc1261f0..ecca2e7104 100644 --- a/backend/tests/test_channels.py +++ b/backend/tests/test_channels.py @@ -1567,6 +1567,40 @@ class TestChannelManager: _run(go()) + def test_wait_reports_a_step_limit_stop_and_releases_dedupe_key(self, tmp_path): + """``/wait`` returns a step-limit stop only as an error: reply with its message, not a generic failure.""" + from app.channels.manager import ChannelManager + + step_limit_text = "The run stopped at its step limit (recursion_limit=100) before finishing. Created but not presented: /mnt/user-data/outputs/report.md. Send a message to continue." + + async def go(): + bus = MessageBus() + manager = ChannelManager(bus=bus, store=JsonChannelStore(path=tmp_path / "store.json")) + outbound_received: list[OutboundMessage] = [] + + async def capture_outbound(msg: OutboundMessage) -> None: + outbound_received.append(msg) + + bus.subscribe_outbound(capture_outbound) + manager._client = _make_mock_langgraph_client() + # The SDK's exception for the Gateway's /wait error envelope. + manager._client.runs.wait = AsyncMock(side_effect=Exception(f"GraphRecursionError: {step_limit_text}")) + await manager.start() + + def _inbound() -> InboundMessage: + return InboundMessage(channel_name="wechat", chat_id="wx_user_1", user_id="wx_user_1", text="write the report", metadata={"message_id": "m-1"}) + + await bus.publish_inbound(_inbound()) + await _wait_for(lambda: len(outbound_received) == 1) + assert [m.text for m in outbound_received] == [step_limit_text] + + # The run ended, so a provider redelivery may retry it. + await bus.publish_inbound(_inbound()) + await _wait_for(lambda: manager._client.runs.wait.call_count == 2) + await manager.stop() + + _run(go()) + def test_fire_and_forget_thread_busy_releases_dedupe_key(self, tmp_path): """Same invariant for a fire-and-forget channel that does not buffer follow-ups.""" import httpx diff --git a/backend/tests/test_loop_detection_middleware.py b/backend/tests/test_loop_detection_middleware.py index f5145365b3..d1e174eb4e 100644 --- a/backend/tests/test_loop_detection_middleware.py +++ b/backend/tests/test_loop_detection_middleware.py @@ -2391,6 +2391,20 @@ class TestWarningLetsAgentSaveResultsFirst: assert "include the collected results in your final answer" in warning assert "file was not saved" in warning + @pytest.mark.parametrize("layer", ["hash", "frequency"]) + def test_warning_asks_to_present_created_outputs_before_answering(self, layer): + # Told only to stop calling tools, agents skipped present_files for + # outputs they had already created, so the run failed its delivery check. + mw = LoopDetectionMiddleware(warn_threshold=3 if layer == "hash" else 10, hard_limit=20, tool_freq_warn=3 if layer == "frequency" else 10, tool_freq_hard_limit=20) + runtime = _make_runtime() + for i in range(3): + mw._apply(_make_state(tool_calls=[_bash_call("ls" if layer == "hash" else f"cmd_{i}")]), runtime) + warning = mw._pending_warnings[_pending_key()][0] + assert "If present_files is available" in warning + assert "already created in /mnt/user-data/outputs" in warning + present = warning.index("present_files call before answering, unless it is the looping tool") + assert warning.index("retry the save once") < present < warning.index("produce your final answer") + @pytest.mark.asyncio async def test_warned_agent_can_recover_an_existing_file_save_through_the_real_gate(self, tmp_path): from deerflow.agents.middlewares.read_before_write_middleware import ReadBeforeWriteMiddleware @@ -2437,6 +2451,34 @@ class TestWarningLetsAgentSaveResultsFirst: assert result["messages"][-1].content == "Saved partial results." assert loop.consume_stop_reason("save-run") is None + @pytest.mark.asyncio + async def test_warned_agent_presents_created_outputs_before_answering(self): + presented = [] + + @as_tool + def bash(command: str) -> str: + """Build an output file without running a shell.""" + return "built" + + @as_tool + def present_files(filepaths: list[str]) -> str: + """Record the files presented to the user.""" + presented.extend(filepaths) + return "presented" + + loop = LoopDetectionMiddleware(warn_threshold=10, hard_limit=20, tool_freq_warn=3, tool_freq_hard_limit=10) + calls = [dict(_bash_call(f"python build_{i}.py"), id=f"build_{i}") for i in range(3)] + calls.append({"name": "present_files", "id": "present", "args": {"filepaths": ["/mnt/user-data/outputs/report.docx"]}}) + model = _CapturingFakeMessagesListChatModel(responses=[*[AIMessage(content="", tool_calls=[call]) for call in calls], AIMessage(content="The report is ready.")]) + graph = create_agent(model=model, tools=[bash, present_files], middleware=[loop]) + result = await graph.ainvoke({"messages": [("user", "Build the report.")]}, context={"thread_id": "present-thread", "run_id": "present-run"}, config={"recursion_limit": 30}) + + warnings = [message for message in model.seen_messages[3] if isinstance(message, HumanMessage) and message.name == "loop_warning"] + assert len(warnings) == 1 and "present_files call before answering" in warnings[0].content + assert presented == ["/mnt/user-data/outputs/report.docx"] + assert result["messages"][-1].content == "The report is ready." + assert loop.consume_stop_reason("present-run") is None + @pytest.mark.parametrize( "name,expected", [ diff --git a/backend/tests/test_run_recursion_capped.py b/backend/tests/test_run_recursion_capped.py new file mode 100644 index 0000000000..91fd4a4c5d --- /dev/null +++ b/backend/tests/test_run_recursion_capped.py @@ -0,0 +1,164 @@ +"""A run that exhausts recursion_limit reports a plain stop, its unpresented outputs, and stands its goal down.""" + +import asyncio +import json +import socket +from pathlib import Path +from types import SimpleNamespace +from typing import Any +from unittest.mock import patch + +import pytest +from langchain.agents import create_agent +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from pydantic import ConfigDict + +from deerflow.agents.thread_state import ThreadState +from deerflow.config.app_config import AppConfig +from deerflow.config.paths import Paths +from deerflow.config.sandbox_config import SandboxConfig +from deerflow.runtime.events.store.memory import MemoryRunEventStore +from deerflow.runtime.goal import build_goal_state, read_thread_goal, write_thread_goal +from deerflow.runtime.runs import worker +from deerflow.runtime.runs.manager import RunManager +from deerflow.runtime.runs.schemas import RunStatus +from deerflow.runtime.runs.store.memory import MemoryRunStore +from deerflow.runtime.user_context import get_effective_user_id, reset_current_user, set_current_user + + +class _ToolLoopModel(BaseChatModel): + """Calls the writer on every turn, so the graph only stops at its recursion limit.""" + + calls: list[int] + model_config = ConfigDict(arbitrary_types_allowed=True) + + @property + def _llm_type(self) -> str: + return "scripted-recursion-cap" + + def bind_tools(self, tools, **kwargs): + return self + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): + self.calls.append(len(self.calls) + 1) + n = len(self.calls) + message = AIMessage(content=f"Working on part {n}.", tool_calls=[{"id": f"w-{n}", "name": "write_part", "args": {"n": n}}]) + return ChatResult(generations=[ChatGeneration(message=message)]) + + +async def _run_until_recursion_cap(tmp_path: Path, *, with_goal: bool) -> dict[str, Any]: + evaluator_calls: list[Any] = [] + published: list[tuple[str, Any]] = [] + + async def publish(run_id, event, data): + published.append((event, data)) + + bridge = SimpleNamespace(publish=publish, publish_end=lambda run_id: asyncio.sleep(0), cleanup=lambda run_id, delay=0: asyncio.sleep(0)) + saver = InMemorySaver() + events = MemoryRunEventStore() + manager = RunManager(store=MemoryRunStore()) + paths = Paths(base_dir=tmp_path) + token = set_current_user(SimpleNamespace(id="recursion-cap-user", role="admin")) + try: + with ( + patch("deerflow.config.paths._paths", paths), + patch.dict("os.environ", {"DEER_FLOW_HOME": str(tmp_path), "LANGSMITH_TRACING": "false", "LANGCHAIN_TRACING_V2": "false"}), + patch.object(worker, "create_goal_evaluator_model", lambda **kwargs: evaluator_calls.append(kwargs)), + ): + record = await manager.create("recursion-cap-thread", user_id="recursion-cap-user") + outputs = paths.sandbox_outputs_dir(record.thread_id, user_id=get_effective_user_id()) + outputs.mkdir(parents=True, exist_ok=True) + + @tool + def write_part(n: int) -> str: + """Write one part of the report.""" + (outputs / f"part-{n}.md").write_text(f"part {n}\n", encoding="utf-8") + return f"wrote part-{n}.md" + + model = _ToolLoopModel(calls=[]) + graph = create_agent(model=model, tools=[write_part], state_schema=ThreadState, context_schema=dict, checkpointer=saver) + app_config = AppConfig(sandbox=SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider")) + app_config.title.enabled = app_config.memory.enabled = app_config.memory.injection_enabled = False + if with_goal: + await write_thread_goal(saver, record.thread_id, build_goal_state("Write the report.", max_continuations=2), create_if_missing=True) + await worker.run_agent( + bridge, + manager, + record, + ctx=worker.RunContext(checkpointer=saver, event_store=events, app_config=app_config), + agent_factory=lambda config: graph, + graph_input={"messages": [HumanMessage(content="Write the report in parts.")]}, + config={"configurable": {"thread_id": record.thread_id}, "recursion_limit": 6}, + stream_modes=["values"], + ) + goal = await read_thread_goal(saver, record.thread_id) + recorded = await events.list_events(record.thread_id, record.run_id) + finally: + reset_current_user(token) + return { + "record": record, + "goal": goal, + "error_frames": [data for event, data in published if event == "error"], + "deliveries": [event["content"] for event in recorded if event["event_type"] == "run.delivery"], + "model_calls": len(model.calls), + "evaluator_calls": len(evaluator_calls), + } + + +@pytest.fixture(autouse=True) +def _no_network(monkeypatch): + def deny(*args, **kwargs): + raise AssertionError("this regression must not use the network") + + monkeypatch.setattr(socket.socket, "connect", deny) + + +@pytest.mark.asyncio +async def test_recursion_cap_reports_a_plain_stop_with_unpresented_outputs(tmp_path): + result = await _run_until_recursion_cap(tmp_path, with_goal=False) + record = result["record"] + + assert record.status == RunStatus.error + assert record.stop_reason == "recursion_capped" + produced = [f"/mnt/user-data/outputs/part-{n}.md" for n in range(1, result["model_calls"] + 1)] + assert result["model_calls"] >= 1 + assert record.error == (f"The run stopped at its step limit (recursion_limit=6) before finishing. Created but not presented: {', '.join(produced)}. Send a message to continue.") + assert "config key" not in record.error and "troubleshooting" not in record.error + assert result["error_frames"] == [{"message": record.error, "name": "GraphRecursionError"}] + # One scan feeds both the message and the delivery receipt. + assert len(result["deliveries"]) == 1 + assert result["deliveries"][0]["produced_paths"] == produced + assert result["deliveries"][0]["satisfied"] is False + + +@pytest.mark.asyncio +async def test_recursion_cap_stands_the_goal_down_without_an_evaluator_call(tmp_path): + result = await _run_until_recursion_cap(tmp_path, with_goal=True) + record, goal = result["record"], result["goal"] + + assert result["evaluator_calls"] == 0 + assert goal is not None and goal["status"] == "active" + evaluation = goal["last_evaluation"] + assert evaluation["stand_down_reason"] == "recursion_capped" + assert evaluation["blocker"] == "run_failed" + assert evaluation["reason"] == "The run reached its step limit before finishing." + assert goal["continuation_count"] == 0 + assert record.goal_verdict is not None + assert record.goal_verdict["stand_down_reason"] == "recursion_capped" + assert json.dumps(record.goal_verdict) # stored as plain data on the run record + + +def test_recursion_cap_message_without_outputs_or_limit(): + assert worker._recursion_capped_message(None, None) == "The run stopped at its step limit before finishing. Send a message to continue." + presented = {"produced_paths": ["/mnt/user-data/outputs/a.md"], "matched_paths": ["/mnt/user-data/outputs/a.md"]} + assert worker._recursion_capped_message(100, presented) == "The run stopped at its step limit (recursion_limit=100) before finishing. Send a message to continue." + + +def test_recursion_cap_message_lists_at_most_five_outputs(): + produced = [f"/mnt/user-data/outputs/{n}.md" for n in range(7)] + message = worker._recursion_capped_message(1000, {"produced_paths": produced, "matched_paths": []}) + assert message.endswith(f"Created but not presented: {', '.join(produced[:5])} and 2 more. Send a message to continue.") diff --git a/backend/tests/test_scheduled_goal_notes_contract.py b/backend/tests/test_scheduled_goal_notes_contract.py index 33a55d0a4b..e9e235e700 100644 --- a/backend/tests/test_scheduled_goal_notes_contract.py +++ b/backend/tests/test_scheduled_goal_notes_contract.py @@ -40,6 +40,7 @@ def test_cap_stand_down_reasons_are_in_contract(): assert worker._stand_down_reason(at_cap, evaluation, 0) in _CODES assert worker._stand_down_reason({"continuation_count": 0}, evaluation, DEFAULT_MAX_NO_PROGRESS_CONTINUATIONS) in _CODES assert worker._GOAL_RUN_HARD_STOP_REASONS <= _CODES + assert worker._GOAL_RUN_ERROR_STOP_REASONS <= _CODES def test_literal_stand_down_reasons_and_no_verdict_are_in_contract(): diff --git a/backend/tests/test_thread_goal_contract.py b/backend/tests/test_thread_goal_contract.py index 8a24ba5342..14b10dad21 100644 --- a/backend/tests/test_thread_goal_contract.py +++ b/backend/tests/test_thread_goal_contract.py @@ -42,7 +42,7 @@ def test_stand_down_reason_codes_match_the_worker(): } literals = set(re.findall(r'stand_down_reason\s*=\s*"([a-z_:]+)"', Path(worker.__file__).read_text(encoding="utf-8"))) assert literals, "pattern no longer matches worker.py; update this test" - assert set(_CONTRACT["stand_down_reason_codes"]) == blocked | caps | literals | worker._GOAL_RUN_HARD_STOP_REASONS + assert set(_CONTRACT["stand_down_reason_codes"]) == blocked | caps | literals | worker._GOAL_RUN_HARD_STOP_REASONS | worker._GOAL_RUN_ERROR_STOP_REASONS def test_check_failure_codes_match_the_scheduled_tasks_page(): diff --git a/backend/tests/test_thread_run_wait_errors.py b/backend/tests/test_thread_run_wait_errors.py index ecc314fe4a..ee3611997c 100644 --- a/backend/tests/test_thread_run_wait_errors.py +++ b/backend/tests/test_thread_run_wait_errors.py @@ -87,3 +87,30 @@ async def test_successful_wait_preserves_checkpoint_and_reused_status_contract(w else: assert response["messages"][0]["content"] == "Current answer." state.accessor.aget.assert_awaited_once() + + +_STEP_LIMIT_ERROR = "The run stopped at its step limit (recursion_limit=6) before finishing. Send a message to continue." + + +@pytest.mark.asyncio +async def test_step_limit_wait_error_keeps_the_stream_frame_name(wait_client): + state = wait_client + state.record.status = RunStatus.error + state.record.error = _STEP_LIMIT_ERROR + state.record.stop_reason = "recursion_capped" + + response = await state.client.runs.wait("thread-1", "lead_agent", input={"messages": []}, raise_error=False) + assert response["__error__"] == {"error": "GraphRecursionError", "message": _STEP_LIMIT_ERROR} + # IM channels read the name from the SDK's exception text. + with pytest.raises(Exception, match=r"^GraphRecursionError: The run stopped at its step limit"): + await state.client.runs.wait("thread-1", "lead_agent", input={"messages": []}) + + +@pytest.mark.asyncio +async def test_store_backed_wait_reads_the_durable_stop_reason(): + record = RunRecord(run_id="run-1", thread_id="thread-1", assistant_id=None, status=RunStatus.running, on_disconnect=DisconnectMode.continue_, store_only=True) + store = SimpleNamespace(get=AsyncMock(return_value={"status": "error", "error": _STEP_LIMIT_ERROR, "stop_reason": "recursion_capped"})) + + refreshed = await thread_runs._refresh_store_backed_run(SimpleNamespace(_store=store), record) + + assert thread_runs.serialize_wait_run_status(refreshed)["__error__"] == {"error": "GraphRecursionError", "message": _STEP_LIMIT_ERROR} diff --git a/contracts/scheduled_goal_notes_contract.json b/contracts/scheduled_goal_notes_contract.json index 6a85c3c02e..df47076541 100644 --- a/contracts/scheduled_goal_notes_contract.json +++ b/contracts/scheduled_goal_notes_contract.json @@ -13,6 +13,7 @@ "no_progress_detected", "token_capped", "loop_capped", + "recursion_capped", "evaluator_failed", "no_durable_end_of_turn", "thread_changed_after_evaluation", diff --git a/contracts/thread_goal_contract.json b/contracts/thread_goal_contract.json index 0bdcbe8e5f..22d3e7c406 100644 --- a/contracts/thread_goal_contract.json +++ b/contracts/thread_goal_contract.json @@ -10,6 +10,7 @@ "no_progress_detected", "token_capped", "loop_capped", + "recursion_capped", "evaluator_failed", "no_durable_end_of_turn", "thread_changed_after_evaluation", @@ -40,6 +41,7 @@ "host_reason_texts": [ "No visible assistant evidence is available yet.", "The turn ended on a question to the user that has not been answered.", - "No durable assistant end-of-turn receipt was available." + "No durable assistant end-of-turn receipt was available.", + "The run reached its step limit before finishing." ] } diff --git a/frontend/src/components/workspace/goal-status-helpers.ts b/frontend/src/components/workspace/goal-status-helpers.ts index 62aeb811c1..88f59e13cb 100644 --- a/frontend/src/components/workspace/goal-status-helpers.ts +++ b/frontend/src/components/workspace/goal-status-helpers.ts @@ -34,6 +34,7 @@ export const GOAL_HOST_REASON_TEXTS: readonly string[] = [ "No visible assistant evidence is available yet.", "The turn ended on a question to the user that has not been answered.", "No durable assistant end-of-turn receipt was available.", + "The run reached its step limit before finishing.", ]; export function isThreadGoalOutcome( @@ -154,6 +155,7 @@ const STOP_BY_REASON: Partial> = { noProgress: { stop: "noProgress", chip: "stopped" }, tokenCapped: { stop: "tokenCapped", chip: "stopped" }, loopCapped: { stop: "loopCapped", chip: "stopped" }, + recursionCapped: { stop: "recursionCapped", chip: "stopped" }, missingEvidence: { stop: "missingEvidence", chip: "stopped" }, runFailed: { stop: "runFailed", chip: "stopped" }, externalWait: { stop: "external", chip: "waiting" }, diff --git a/frontend/src/components/workspace/goal-status.tsx b/frontend/src/components/workspace/goal-status.tsx index b2d2828dec..b33132a03f 100644 --- a/frontend/src/components/workspace/goal-status.tsx +++ b/frontend/src/components/workspace/goal-status.tsx @@ -36,6 +36,7 @@ const STOP_ICONS: Record = { noProgress: CircleStopIcon, tokenCapped: GaugeIcon, loopCapped: CircleStopIcon, + recursionCapped: CircleStopIcon, missingEvidence: CircleStopIcon, runFailed: TriangleAlertIcon, needsInputCard: MessageCircleQuestionIcon, diff --git a/frontend/src/core/i18n/locales/en-US.ts b/frontend/src/core/i18n/locales/en-US.ts index e901204419..6e0ca5fa9f 100644 --- a/frontend/src/core/i18n/locales/en-US.ts +++ b/frontend/src/core/i18n/locales/en-US.ts @@ -581,6 +581,8 @@ export const enUS: Translations = { "Token budget reached. Send a message to continue; a new run starts with a fresh budget.", loopCapped: "Repeated tool calls triggered a safety stop. Adjust the approach and send a new message to retry; your goal stays active.", + recursionCapped: + "The run reached its step limit before finishing. Send a new message to continue; your goal stays active.", missingEvidence: "Goal check: evidence missing. Ask it to show the result or explain what's missing; if the goal no longer applies, run /goal clear.", runFailed: @@ -1261,6 +1263,7 @@ export const enUS: Translations = { noProgress: "No progress between turns", tokenCapped: "Token budget reached", loopCapped: "Repeated tool calls reached the safety limit", + recursionCapped: "The run reached its step limit", evaluatorFailed: "Couldn't check the goal; the run itself may be fine", noDurableEndOfTurn: "Couldn't check the goal: no final reply was saved", threadChanged: diff --git a/frontend/src/core/i18n/locales/types.ts b/frontend/src/core/i18n/locales/types.ts index 8784c8a6a6..9b2616b1de 100644 --- a/frontend/src/core/i18n/locales/types.ts +++ b/frontend/src/core/i18n/locales/types.ts @@ -449,6 +449,7 @@ export interface Translations { noProgress: string; tokenCapped: string; loopCapped: string; + recursionCapped: string; missingEvidence: string; runFailed: string; needsInputCard: string; @@ -1026,6 +1027,7 @@ export interface Translations { noProgress: string; tokenCapped: string; loopCapped: string; + recursionCapped: string; evaluatorFailed: string; noDurableEndOfTurn: string; threadChanged: string; diff --git a/frontend/src/core/i18n/locales/zh-CN.ts b/frontend/src/core/i18n/locales/zh-CN.ts index 1b88636704..d2f1b55d61 100644 --- a/frontend/src/core/i18n/locales/zh-CN.ts +++ b/frontend/src/core/i18n/locales/zh-CN.ts @@ -542,6 +542,8 @@ export const zhCN: Translations = { "已达到 token 预算。发送消息即可继续，新的运行会重新计算预算。", loopCapped: "重复工具调用触发了安全停止。调整要求后发送新消息即可重试，目标会保留。", + recursionCapped: + "运行达到了步数上限，未能完成。发送新消息即可继续，目标会保留。", missingEvidence: "目标检查：缺少依据。可以让它展示结果或说明卡在哪里；如果目标已不适用，用 /goal clear 清除目标。", runFailed: @@ -1187,6 +1189,7 @@ export const zhCN: Translations = { noProgress: "连续多轮没有进展", tokenCapped: "已达到 token 预算", loopCapped: "重复工具调用触发了安全停止", + recursionCapped: "运行达到了步数上限", evaluatorFailed: "未能检查目标，这次运行本身可能没问题", noDurableEndOfTurn: "未能检查目标：没有保存最终回复", threadChanged: "未能检查目标：检查期间对话发生了变化", diff --git a/frontend/src/core/scheduled-tasks/goal-outcome.ts b/frontend/src/core/scheduled-tasks/goal-outcome.ts index 8b52aa1053..783f6eabeb 100644 --- a/frontend/src/core/scheduled-tasks/goal-outcome.ts +++ b/frontend/src/core/scheduled-tasks/goal-outcome.ts @@ -10,6 +10,7 @@ export type GoalReasonKey = | "noProgress" | "tokenCapped" | "loopCapped" + | "recursionCapped" | "evaluatorFailed" | "noDurableEndOfTurn" | "threadChanged" @@ -29,6 +30,7 @@ const REASON_KEYS: Record = { no_progress_detected: "noProgress", token_capped: "tokenCapped", loop_capped: "loopCapped", + recursion_capped: "recursionCapped", evaluator_failed: "evaluatorFailed", no_durable_end_of_turn: "noDurableEndOfTurn", thread_changed_after_evaluation: "threadChanged", diff --git a/frontend/tests/e2e/goal-status.spec.ts b/frontend/tests/e2e/goal-status.spec.ts index 0a2819386f..e7a5bc95a4 100644 --- a/frontend/tests/e2e/goal-status.spec.ts +++ b/frontend/tests/e2e/goal-status.spec.ts @@ -187,6 +187,27 @@ const STOOD_DOWN = [ "Repeated tool calls triggered a safety stop. Adjust the approach and send a new message to retry; your goal stays active.", note: "The report is unfinished.", }, + { + name: "a step-limit stop", + objective: "Write the report in six parts", + goal: { + continuation_count: 0, + // The host writes this reason itself; it is not the checker's note. + last_evaluation: { + satisfied: false, + blocker: "run_failed", + reason: "The run reached its step limit before finishing.", + stand_down_reason: "recursion_capped", + }, + }, + reply: [ + { type: "ai", id: "goal-ai", content: "Writing part 3 of the report." }, + ], + chip: "Stopped", + detail: + "The run reached its step limit before finishing. Send a new message to continue; your goal stays active.", + note: null, + }, { name: "a goal that used every automatic follow-up", objective: "Make every test in the suite pass", diff --git a/frontend/tests/e2e/utils/raw-identifiers.ts b/frontend/tests/e2e/utils/raw-identifiers.ts index f61e4a1453..787a97d6cf 100644 --- a/frontend/tests/e2e/utils/raw-identifiers.ts +++ b/frontend/tests/e2e/utils/raw-identifiers.ts @@ -13,7 +13,7 @@ export const RAW_IDENTIFIER_PATTERNS: readonly RegExp[] = [ /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/, /(^|\s)[\d*/,-]+(\s[\d*/,-]+){4}(\s|$)/, /\b(fresh_thread_per_run|reuse_thread|lead_agent|goal_objective|max_runs|end_at|unmet|stop_scheduled_task|schedule_task)\b/, - /\b(blocked:[a-z_]+|max_continuations_reached|no_progress_detected|token_capped|loop_capped|evaluator_failed|no_durable_end_of_turn|thread_changed_(after_evaluation|before_continuation)|goal_outcome|stand_down_reason|last_evaluation)\b/, + /\b(blocked:[a-z_]+|max_continuations_reached|no_progress_detected|token_capped|loop_capped|recursion_capped|evaluator_failed|no_durable_end_of_turn|thread_changed_(after_evaluation|before_continuation)|goal_outcome|stand_down_reason|last_evaluation)\b/, /\b(goal_not_met_yet|needs_user_input|missing_evidence|external_wait|run_failed|goal_created_at|reply_message_id|relied_on_assumption|continuation_count|max_continuations)\b/, ];