mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
fix(runtime): stop Goal continuations after loop hard stops (#7167)
* fix(runtime): stop Goal continuations after loop hard stops * chore: remove redundant SVG sources from Goal fix * test(goal): remove PR-only screenshot generation --------- Co-authored-by: Totoro-qaq <279883115+Totoro-qaq@users.noreply.github.com>
This commit is contained in:
parent
d85e0e8608
commit
059758f321
@ -2186,6 +2186,8 @@ Supported commands:
|
||||
|
||||
After each Gateway-backed run, DeerFlow evaluates the visible conversation, including the assistant's tool calls and shortened tool results, against the active goal with a non-thinking evaluator model. A successful tool result alone does not satisfy a goal, and when the assistant had to guess missing or ambiguous information the evaluator reports `needs_user_input`. The evaluator must return a typed blocker (`missing_evidence`, `needs_user_input`, `run_failed`, `external_wait`, or `goal_not_met_yet`) plus visible evidence. DeerFlow only injects a hidden continuation when the latest assistant turn is durably checkpointed, the blocker is `goal_not_met_yet`, the thread did not change during evaluation, and the no-progress breaker has not fired. The safety cap defaults to 8 hidden continuations, and repeated identical non-progress evaluations stop after 2 attempts. `/goal clear` and any user-authored new input win over queued continuations. When the evaluator considers the goal satisfied, DeerFlow clears it after the run successfully finalizes and publishes the updated thread state. The same checkpoint records the completion as `goal_outcome`, which stays until the next goal change. If required artifact delivery or its receipt persistence fails, the run reports an error and the goal remains active for a later retry; this does not start another hidden continuation.
|
||||
|
||||
An unmet goal also stops automatic continuation after a token or loop-detection hard stop; it stays active for a later user turn.
|
||||
|
||||
Regenerating an earlier response keeps the goal currently set for the chat and its used continuation budget. It does not restore a goal that was cleared or achieved. The earlier answer's completion record is removed when its historical checkpoint is replayed.
|
||||
|
||||
The Web UI shows the active goal above the composer, says why auto-continue stopped, and shows a "Goal met" notice after a goal is met. Editing the last message is off while a goal is set. The same command is available from the TUI and supported IM channels. In the Web UI and supported IM channels, setting `/goal <completion condition>` also starts a run with the condition as the task; status and clear commands only manage goal state. Setting or clearing a goal is rejected while that thread has a run in flight, including a run owned by another Gateway worker, so the goal checkpoint cannot branch away from an active run's checkpoint lineage.
|
||||
|
||||
@ -85,6 +85,7 @@ _GOAL_REASON_TEXT: dict[str, dict[str, str]] = {
|
||||
"max_continuations_reached": "the continuation limit was reached",
|
||||
"no_progress_detected": "no progress was made between turns",
|
||||
"token_capped": "the token budget was reached",
|
||||
"loop_capped": "repeated tool calls reached the safety limit",
|
||||
"no_durable_end_of_turn": "no final reply was saved",
|
||||
"thread_changed_after_evaluation": "the conversation changed during the goal check",
|
||||
"thread_changed_before_continuation": "the conversation changed during the goal check",
|
||||
@ -102,6 +103,7 @@ _GOAL_REASON_TEXT: dict[str, dict[str, str]] = {
|
||||
"max_continuations_reached": "已达到继续处理的次数上限",
|
||||
"no_progress_detected": "多轮之间没有进展",
|
||||
"token_capped": "已达到 token 预算",
|
||||
"loop_capped": "重复工具调用触发了安全停止",
|
||||
"no_durable_end_of_turn": "没有保存最终回复",
|
||||
"thread_changed_after_evaluation": "目标检查期间对话发生了变化",
|
||||
"thread_changed_before_continuation": "目标检查期间对话发生了变化",
|
||||
|
||||
@ -108,6 +108,8 @@ answers for paginated history. See `docs/skill-usage-ui.md`.
|
||||
|
||||
**Run delivery receipts:** Finalize artifact evidence and terminal status before clearing an ordinary satisfied goal, under a checkpoint-write reservation. Delivery failure retains the goal without continuing. Only that clear writes `goal_outcome`; other goal writes remove it. Historical replay keeps the current goal and budget, and drops the old outcome. Scheduled exception below; details: `backend/docs/runtime-guidance-details.md`.
|
||||
|
||||
Unmet goals stand down on run-level token/loop hard stops and retain the original stop reason; hidden continuations must not re-enter that run's shared safety state.
|
||||
|
||||
**Deferred terminal commit:** With an event store, the worker stages its terminal
|
||||
status locally and commits it only after finalization's receipt and duration
|
||||
writes. `RunRecord.terminal_commit_pending` keeps `_renew_leases()` renewing that
|
||||
|
||||
@ -2160,6 +2160,8 @@ async def run_agent(
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_GOAL_RUN_HARD_STOP_REASONS = frozenset({"token_capped", "loop_capped"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _GoalCompletionCandidate:
|
||||
@ -2660,11 +2662,10 @@ async def _prepare_goal_continuation_input(
|
||||
return _GoalCompletionCandidate(copy.deepcopy(current_goal), conversation_signature_before, evaluation)
|
||||
|
||||
stand_down_reason = _stand_down_reason(goal, evaluation, no_progress_count)
|
||||
if stand_down_reason is None and run_stop_reason == "token_capped":
|
||||
# The run already used up its token budget, and continuations share that
|
||||
# budget, so another hidden turn would spend one more model call only to
|
||||
# have its tool calls stripped.
|
||||
stand_down_reason = "token_capped"
|
||||
if stand_down_reason is None and run_stop_reason in _GOAL_RUN_HARD_STOP_REASONS:
|
||||
# Both guards keep their safety state across hidden turns in this run.
|
||||
# Respect the hard stop instead of re-entering the same run budget.
|
||||
stand_down_reason = run_stop_reason
|
||||
if stand_down_reason is not None or not should_continue_goal(goal, evaluation, no_progress_count=no_progress_count):
|
||||
await _persist(goal, evaluation, no_progress_count, stand_down_reason=stand_down_reason)
|
||||
return None
|
||||
|
||||
103
backend/scripts/benchmark/goal_loop_stop/README.md
Normal file
103
backend/scripts/benchmark/goal_loop_stop/README.md
Normal file
@ -0,0 +1,103 @@
|
||||
# Goal continuation after a loop hard stop
|
||||
|
||||
This offline reproduction exercises the native loop guard, LangChain graph and
|
||||
ToolNode, Gateway worker, in-memory run/event stores and checkpoints. The model
|
||||
always requests the same harmless marker inspection; the Goal evaluator always
|
||||
reports the synthetic task unfinished. Neither model calls a provider.
|
||||
|
||||
On the affected main commit `1fd266e0c03e1d998918990727de43769136e7f6`, an unmet Goal
|
||||
starts another hidden turn after the loop guard has already stopped the run.
|
||||
Adding `loop_capped` to the existing Goal stand-down condition prevents that
|
||||
re-entry and keeps the Goal active for a later user turn.
|
||||
|
||||
## Recorded result
|
||||
|
||||
| Observed count or state | Before | After |
|
||||
| --- | ---: | ---: |
|
||||
| Lead model calls | 6 | 5 |
|
||||
| Actual tool executions | 4 | 4 |
|
||||
| Goal evaluator calls | 2 | 1 |
|
||||
| Hidden Goal continuations | 1 | 0 |
|
||||
| Goal state | active | active |
|
||||
| Last Goal stand-down reason | max_continuations_reached | loop_capped |
|
||||
| Run stop reason | loop_capped | loop_capped |
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
The fixture deliberately sets the identical-call hard limit to 5 and the maximum
|
||||
Goal continuations to 1. Both chat-only runs retain the existing run status
|
||||
`success`; that status does not establish Goal achievement. The fifth repeated
|
||||
call is stripped by the real middleware, so only four tools execute. Before the
|
||||
fix, the sixth call is stripped again in a hidden turn.
|
||||
|
||||
These are deterministic boundary observations, not real-model task success,
|
||||
latency, token pricing or an estimate of production incident frequency.
|
||||
|
||||
## Run
|
||||
|
||||
From `backend/`, with the repository's locked environment installed:
|
||||
|
||||
```bash
|
||||
uv run --locked python -m scripts.benchmark.goal_loop_stop --output /tmp/goal-loop-stop.json
|
||||
uv run --locked pytest tests/test_goal_loop_stop_probe.py tests/test_goal_worker.py -q
|
||||
```
|
||||
|
||||
The reproduction creates temporary storage and no sandbox/container, shell tool,
|
||||
Gateway server or background service. It replaces only the evaluator provider
|
||||
factory with a scripted evaluator; Goal decisions and the loop guard are native.
|
||||
|
||||
To compare the affected source and this fix, start from a checkout containing
|
||||
this reproduction, at the repository root:
|
||||
|
||||
```bash
|
||||
git worktree add --detach ../goal-loop-before 1fd266e0c03e1d998918990727de43769136e7f6
|
||||
cp -R backend/scripts/benchmark/goal_loop_stop ../goal-loop-before/backend/scripts/benchmark/
|
||||
(cd ../goal-loop-before/backend && uv sync --locked && uv run --locked python -m scripts.benchmark.goal_loop_stop --output /tmp/goal-loop-before.json)
|
||||
(cd backend && uv run --locked python -m scripts.benchmark.goal_loop_stop --output /tmp/goal-loop-after.json)
|
||||
```
|
||||
|
||||
Only the reproduction module is copied into the affected checkout; its runtime
|
||||
source is left unchanged. The locked dependency manifests are the same in both
|
||||
recorded runs: Python 3.12.13, LangGraph 1.2.9, langgraph-checkpoint 4.2.0 and
|
||||
LangChain 1.3.14 on macOS arm64.
|
||||
|
||||
## User-facing stop reason
|
||||
|
||||
The browser regression checks both ordinary and custom-agent chat pages, including
|
||||
a reload, against mocked Gateway responses. The production frontend renders the
|
||||
localized stop reason and retains the active Goal; this is separate from the
|
||||
native worker reproduction above.
|
||||
|
||||

|
||||
|
||||
## Investigation context and scope
|
||||
|
||||
This follow-up came from our completion/delivery investigation. In the earlier
|
||||
12-unit real-task comparison (three task types, two repetitions per arm), the
|
||||
ordinary runtime qualified in 4/6 units and the broader experimental intervention
|
||||
in 1/6. All failures were retained. That experiment did not support adopting the
|
||||
broader intervention, so this fix isolates a separately reproduced native Goal
|
||||
control-flow problem.
|
||||
|
||||
Our earlier [Goal investigation (#6555)](https://github.com/bytedance/deer-flow/issues/6555)
|
||||
also found evaluator replays returning both `run_failed` and `goal_not_met_yet`
|
||||
after a loop stop. This fix makes an unmet Goal respect the explicit hard stop
|
||||
without depending on which of those blocker labels the evaluator chooses.
|
||||
|
||||
The historical A/B figures are background, not validation of this patch. This
|
||||
change introduces no step-budget middleware, warning prompt, tool-schema rewrite,
|
||||
automatic file presentation, provider integration or dependency upgrade.
|
||||
Satisfied Goal candidates still go through the existing delivery and persistence
|
||||
checks; other blockers and unclassified stop reasons keep their existing behavior.
|
||||
|
||||
## Data
|
||||
|
||||
- [Before](evidence/before.json), [after](evidence/after.json), and [CSV counts](evidence/comparison.csv)
|
||||
- [Source/version provenance](evidence/provenance.json)
|
||||
- [Historical investigation summary](evidence/investigation-context.json)
|
||||
- Figures: [call counts](figures/call-counts.png), [flow](figures/flow.png)
|
||||
|
||||
All committed results are synthetic counts and non-secret version/source metadata.
|
||||
They contain no provider requests, credentials, user messages or production logs.
|
||||
147
backend/scripts/benchmark/goal_loop_stop/__init__.py
Normal file
147
backend/scripts/benchmark/goal_loop_stop/__init__.py
Normal file
@ -0,0 +1,147 @@
|
||||
"""Offline reproduction of Goal re-entry after the native loop guard stops."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
from langchain.agents import create_agent
|
||||
from langchain_core.language_models.chat_models import BaseChatModel
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
from langchain_core.outputs import ChatGeneration, ChatResult
|
||||
from langchain_core.tools import tool
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
|
||||
from deerflow.agents.thread_state import ThreadState
|
||||
from deerflow.config.app_config import AppConfig
|
||||
from deerflow.config.paths import Paths
|
||||
from deerflow.config.sandbox_config import SandboxConfig
|
||||
from deerflow.runtime.events.store.memory import MemoryRunEventStore
|
||||
from deerflow.runtime.goal import build_goal_state, read_thread_goal, write_thread_goal
|
||||
from deerflow.runtime.runs import worker
|
||||
from deerflow.runtime.runs.manager import RunManager
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
|
||||
@dataclass
|
||||
class Counts:
|
||||
model_calls: int = 0
|
||||
tool_executions: int = 0
|
||||
evaluator_calls: int = 0
|
||||
|
||||
|
||||
class RepeatingModel(BaseChatModel):
|
||||
"""Always request the same harmless tool, including after a hard stop."""
|
||||
|
||||
counts: Counts
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "offline-loop-stop-reproduction"
|
||||
|
||||
def bind_tools(self, tools: Any, **kwargs: Any) -> RepeatingModel:
|
||||
return self
|
||||
|
||||
def _generate(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any) -> ChatResult:
|
||||
self.counts.model_calls += 1
|
||||
message = AIMessage(
|
||||
content="",
|
||||
tool_calls=[{"id": f"inspect-{self.counts.model_calls}", "name": "inspect_marker", "args": {"marker": "fixed"}}],
|
||||
)
|
||||
return ChatResult(generations=[ChatGeneration(message=message)])
|
||||
|
||||
|
||||
class UnmetEvaluator:
|
||||
def __init__(self, counts: Counts) -> None:
|
||||
self.counts = counts
|
||||
|
||||
async def ainvoke(self, messages: Any, config: Any = None) -> AIMessage:
|
||||
self.counts.evaluator_calls += 1
|
||||
return AIMessage(
|
||||
content=json.dumps({"satisfied": False, "blocker": "goal_not_met_yet", "reason": "The synthetic task remains unfinished.", "evidence_summary": "Only repeated marker inspections ran.", "relied_on_assumption": False})
|
||||
)
|
||||
|
||||
|
||||
async def run_case(*, max_continuations: int = 1) -> dict[str, Any]:
|
||||
"""Run a real graph and worker with scripted models and temporary storage.
|
||||
|
||||
The sole provider replacement is the Goal evaluator factory. Goal decisions,
|
||||
loop detection, tool execution, checkpoints and finalization stay native.
|
||||
"""
|
||||
counts = Counts()
|
||||
|
||||
@tool
|
||||
def inspect_marker(marker: str) -> str:
|
||||
"""Record a synthetic inspection without shell or network access."""
|
||||
counts.tool_executions += 1
|
||||
return f"Observed marker: {marker}"
|
||||
|
||||
saver = InMemorySaver()
|
||||
events = MemoryRunEventStore()
|
||||
manager = RunManager(store=MemoryRunStore())
|
||||
graph = create_agent(
|
||||
model=RepeatingModel(counts=counts),
|
||||
tools=[inspect_marker],
|
||||
middleware=[LoopDetectionMiddleware(warn_threshold=3, hard_limit=5, window_size=20)],
|
||||
state_schema=ThreadState,
|
||||
context_schema=dict,
|
||||
checkpointer=saver,
|
||||
)
|
||||
cfg = AppConfig(sandbox=SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider"))
|
||||
cfg.title.enabled = cfg.memory.enabled = cfg.memory.injection_enabled = False
|
||||
bridge = SimpleNamespace(publish=AsyncMock(), publish_end=AsyncMock(), cleanup=AsyncMock())
|
||||
token = set_current_user(SimpleNamespace(id="goal-loop-probe", role="admin"))
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="goal-loop-stop-") as folder:
|
||||
paths = Paths(base_dir=Path(folder))
|
||||
with (
|
||||
patch("deerflow.config.paths._paths", paths),
|
||||
patch.dict(os.environ, {"DEER_FLOW_HOME": folder, "LANGSMITH_TRACING": "false", "LANGCHAIN_TRACING_V2": "false"}),
|
||||
patch.object(worker, "create_goal_evaluator_model", lambda **kwargs: UnmetEvaluator(counts)),
|
||||
):
|
||||
record = await manager.create(f"loop-stop-{uuid4().hex}", user_id="goal-loop-probe")
|
||||
await write_thread_goal(saver, record.thread_id, build_goal_state("Finish the synthetic task.", max_continuations=max_continuations), create_if_missing=True)
|
||||
await worker.run_agent(
|
||||
bridge,
|
||||
manager,
|
||||
record,
|
||||
ctx=worker.RunContext(checkpointer=saver, event_store=events, app_config=cfg),
|
||||
agent_factory=lambda config: graph,
|
||||
graph_input={"messages": [HumanMessage(content="Inspect the marker until the synthetic task is complete.")]},
|
||||
config={"configurable": {"thread_id": record.thread_id}, "recursion_limit": 100},
|
||||
stream_modes=["values"],
|
||||
)
|
||||
goal = await read_thread_goal(saver, record.thread_id)
|
||||
if goal is None:
|
||||
raise AssertionError("The unfinished goal was cleared")
|
||||
return {
|
||||
"scenario": "native_loop_hard_stop_with_unmet_goal",
|
||||
"fixture": "scripted models; real loop middleware, ToolNode, worker and in-memory checkpoints",
|
||||
"guard_hard_limit": 5,
|
||||
"max_goal_continuations": max_continuations,
|
||||
"lead_model_calls": counts.model_calls,
|
||||
"tool_executions": counts.tool_executions,
|
||||
"goal_evaluator_calls": counts.evaluator_calls,
|
||||
"goal_continuations": goal["continuation_count"],
|
||||
"goal_status": goal["status"],
|
||||
"stand_down_reason": goal.get("last_evaluation", {}).get("stand_down_reason"),
|
||||
"run_status": record.status.value,
|
||||
"run_stop_reason": record.stop_reason,
|
||||
"run_error": record.error,
|
||||
"provider_requests": 0,
|
||||
"versions": {name: version(name) for name in ["langgraph", "langgraph-checkpoint", "langchain"]},
|
||||
}
|
||||
finally:
|
||||
reset_current_user(token)
|
||||
25
backend/scripts/benchmark/goal_loop_stop/__main__.py
Normal file
25
backend/scripts/benchmark/goal_loop_stop/__main__.py
Normal file
@ -0,0 +1,25 @@
|
||||
"""Run from backend: python -m scripts.benchmark.goal_loop_stop --output result.json."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from . import run_case
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Reproduce Goal continuation after a native loop hard stop, without provider calls.")
|
||||
parser.add_argument("--output", type=Path, help="Write the synthetic result JSON here; otherwise print it.")
|
||||
args = parser.parse_args()
|
||||
result = asyncio.run(run_case())
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(text, encoding="utf-8")
|
||||
else:
|
||||
print(text, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
21
backend/scripts/benchmark/goal_loop_stop/evidence/after.json
Normal file
21
backend/scripts/benchmark/goal_loop_stop/evidence/after.json
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"scenario": "native_loop_hard_stop_with_unmet_goal",
|
||||
"fixture": "scripted models; real loop middleware, ToolNode, worker and in-memory checkpoints",
|
||||
"guard_hard_limit": 5,
|
||||
"max_goal_continuations": 1,
|
||||
"lead_model_calls": 5,
|
||||
"tool_executions": 4,
|
||||
"goal_evaluator_calls": 1,
|
||||
"goal_continuations": 0,
|
||||
"goal_status": "active",
|
||||
"stand_down_reason": "loop_capped",
|
||||
"run_status": "success",
|
||||
"run_stop_reason": "loop_capped",
|
||||
"run_error": null,
|
||||
"provider_requests": 0,
|
||||
"versions": {
|
||||
"langgraph": "1.2.9",
|
||||
"langgraph-checkpoint": "4.2.0",
|
||||
"langchain": "1.3.14"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
{
|
||||
"scenario": "native_loop_hard_stop_with_unmet_goal",
|
||||
"fixture": "scripted models; real loop middleware, ToolNode, worker and in-memory checkpoints",
|
||||
"guard_hard_limit": 5,
|
||||
"max_goal_continuations": 1,
|
||||
"lead_model_calls": 6,
|
||||
"tool_executions": 4,
|
||||
"goal_evaluator_calls": 2,
|
||||
"goal_continuations": 1,
|
||||
"goal_status": "active",
|
||||
"stand_down_reason": "max_continuations_reached",
|
||||
"run_status": "success",
|
||||
"run_stop_reason": "loop_capped",
|
||||
"run_error": null,
|
||||
"provider_requests": 0,
|
||||
"versions": {
|
||||
"langgraph": "1.2.9",
|
||||
"langgraph-checkpoint": "4.2.0",
|
||||
"langchain": "1.3.14"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
metric,before,after
|
||||
lead_model_calls,6,5
|
||||
tool_executions,4,4
|
||||
goal_evaluator_calls,2,1
|
||||
goal_continuations,1,0
|
||||
|
@ -0,0 +1,19 @@
|
||||
{
|
||||
"description": "Earlier real-task investigation; historical context, not validation of this narrow Goal fix.",
|
||||
"paid_units": 12,
|
||||
"task_types": 3,
|
||||
"repeats_per_task_per_arm": 2,
|
||||
"model": "deepseek-flash / DeepSeek-V4.1-Flash-1017",
|
||||
"recursion_limit": 1000,
|
||||
"output_ceiling": 65536,
|
||||
"A_qualified": 4,
|
||||
"A_total": 6,
|
||||
"B_qualified": 1,
|
||||
"B_total": 6,
|
||||
"B_step_capped_units": 3,
|
||||
"B_presentation_repair_units": 0,
|
||||
"usage_priced_usd": 4.65303372,
|
||||
"conclusion": "The broader experimental intervention was not supported for adoption. This patch addresses only a separately reproduced native loop-stop/Goal interaction.",
|
||||
"local_source_summary_sha256": "aa24f93b4207af888556ac27a573d3cbc8355e7afd62b190d05f50e5c4d660e3",
|
||||
"earlier_goal_investigation": "https://github.com/bytedance/deer-flow/issues/6555"
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
{
|
||||
"affected_base_commit": "1fd266e0c03e1d998918990727de43769136e7f6",
|
||||
"before_worker_sha256": "379041b126140d91223230e5bd200e713ac746d5d18927db668a3656b5257b64",
|
||||
"after_worker_sha256": "c6dec31e2c30281cd736d70086065d702756c87659f115328d3c993e424275ed",
|
||||
"reproduction_module_sha256": "3b392029b31ea4eced3cf734dc692355ee66b70cbe01eb084818c5fc5206ee8d",
|
||||
"python": "3.12.13",
|
||||
"platform": "macOS arm64",
|
||||
"provider_requests": 0,
|
||||
"method": "One deterministic repetition fixture executed once before and once after the worker change; not a real-model task-success benchmark."
|
||||
}
|
||||
BIN
backend/scripts/benchmark/goal_loop_stop/figures/call-counts.png
Normal file
BIN
backend/scripts/benchmark/goal_loop_stop/figures/call-counts.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 98 KiB |
BIN
backend/scripts/benchmark/goal_loop_stop/figures/flow.png
Normal file
BIN
backend/scripts/benchmark/goal_loop_stop/figures/flow.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 168 KiB |
BIN
backend/scripts/benchmark/goal_loop_stop/figures/goal-status.png
Normal file
BIN
backend/scripts/benchmark/goal_loop_stop/figures/goal-status.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 58 KiB |
@ -10,6 +10,8 @@ ownership and capture cleanup. Retain the real XLSX conversion coverage.
|
||||
|
||||
Backend tests must preserve the runtime invariants they exercise without changing production execution topology.
|
||||
|
||||
`test_goal_loop_stop_probe.py` runs the standalone Goal-loop reproduction through the native graph, loop middleware, worker and checkpoints. Only model responses are scripted; do not replace the Goal decision or simulate the stop-reason write.
|
||||
|
||||
InfoQuest fetch log assertions install the real `UrlRedactionFilter` on the
|
||||
capture handler with fixture-scoped cleanup. Expect scheme/host context with
|
||||
redacted paths, so isolated runs and full-suite logging configuration agree;
|
||||
|
||||
25
backend/tests/test_goal_loop_stop_probe.py
Normal file
25
backend/tests/test_goal_loop_stop_probe.py
Normal file
@ -0,0 +1,25 @@
|
||||
"""The standalone reproduction also exercises the real runtime in CI."""
|
||||
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts.benchmark.goal_loop_stop import run_case
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_goal_stands_down_after_the_native_loop_guard_hard_stop(monkeypatch):
|
||||
def deny_network(*args, **kwargs):
|
||||
raise AssertionError("This controlled regression must not use the network")
|
||||
|
||||
monkeypatch.setattr(socket.socket, "connect", deny_network)
|
||||
result = await run_case()
|
||||
assert result["run_error"] is None
|
||||
assert result["run_stop_reason"] == "loop_capped"
|
||||
assert result["lead_model_calls"] == result["guard_hard_limit"] == 5
|
||||
assert result["tool_executions"] == 4
|
||||
assert result["goal_evaluator_calls"] == 1
|
||||
assert result["goal_status"] == "active"
|
||||
assert result["goal_continuations"] == 0
|
||||
assert result["stand_down_reason"] == "loop_capped"
|
||||
assert result["provider_requests"] == 0
|
||||
@ -147,7 +147,8 @@ async def _write_messages(checkpointer: InMemorySaver, *, thread_id: str, messag
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_goal_worker_returns_hidden_continuation_when_goal_is_unmet(monkeypatch):
|
||||
@pytest.mark.parametrize("run_stop_reason", [None, "subagent_limit_capped", "unknown_reason"])
|
||||
async def test_goal_worker_returns_hidden_continuation_when_goal_is_unmet(monkeypatch, run_stop_reason):
|
||||
checkpointer = InMemorySaver()
|
||||
thread_id = "goal-thread"
|
||||
await _seed_goal_thread(checkpointer, thread_id=thread_id, goal_text="Finish all tests")
|
||||
@ -177,6 +178,7 @@ async def test_goal_worker_returns_hidden_continuation_when_goal_is_unmet(monkey
|
||||
run_id="run-1",
|
||||
model_name="test-model",
|
||||
app_config=None,
|
||||
run_stop_reason=run_stop_reason,
|
||||
task_store=task_store,
|
||||
extensions=extensions,
|
||||
)
|
||||
@ -275,7 +277,8 @@ async def test_goal_worker_evaluates_materialized_messages_in_delta_mode(monkeyp
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_goal_worker_stands_down_for_non_continuable_blocker(monkeypatch):
|
||||
@pytest.mark.parametrize("run_stop_reason", [None, "loop_capped"])
|
||||
async def test_goal_worker_stands_down_for_non_continuable_blocker(monkeypatch, run_stop_reason):
|
||||
checkpointer = InMemorySaver()
|
||||
thread_id = "blocked-goal-thread"
|
||||
await _seed_goal_thread(checkpointer, thread_id=thread_id, goal_text="Finish all tests")
|
||||
@ -299,6 +302,7 @@ async def test_goal_worker_stands_down_for_non_continuable_blocker(monkeypatch):
|
||||
run_id="run-3",
|
||||
model_name="test-model",
|
||||
app_config=None,
|
||||
run_stop_reason=run_stop_reason,
|
||||
)
|
||||
|
||||
assert continuation is None
|
||||
@ -310,9 +314,10 @@ async def test_goal_worker_stands_down_for_non_continuable_blocker(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_goal_worker_stands_down_after_the_run_hit_its_token_budget(monkeypatch):
|
||||
@pytest.mark.parametrize("run_stop_reason", ["token_capped", "loop_capped"])
|
||||
async def test_goal_worker_stands_down_after_run_hard_stop(monkeypatch, run_stop_reason):
|
||||
checkpointer = InMemorySaver()
|
||||
thread_id = "token-capped-goal-thread"
|
||||
thread_id = "hard-stop-goal-thread"
|
||||
await _seed_goal_thread(checkpointer, thread_id=thread_id, goal_text="Finish all tests")
|
||||
bridge = _CollectingBridge()
|
||||
|
||||
@ -334,7 +339,7 @@ async def test_goal_worker_stands_down_after_the_run_hit_its_token_budget(monkey
|
||||
run_id="run-capped",
|
||||
model_name="test-model",
|
||||
app_config=None,
|
||||
run_stop_reason="token_capped",
|
||||
run_stop_reason=run_stop_reason,
|
||||
)
|
||||
|
||||
assert continuation is None
|
||||
@ -342,7 +347,7 @@ async def test_goal_worker_stands_down_after_the_run_hit_its_token_budget(monkey
|
||||
assert latest_goal is not None
|
||||
assert latest_goal["continuation_count"] == 0
|
||||
assert latest_goal["last_evaluation"]["blocker"] == "goal_not_met_yet"
|
||||
assert latest_goal["last_evaluation"]["stand_down_reason"] == "token_capped"
|
||||
assert latest_goal["last_evaluation"]["stand_down_reason"] == run_stop_reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@ -457,7 +462,8 @@ async def test_goal_worker_resumes_after_the_user_answers_clarification(monkeypa
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_goal_worker_defers_satisfied_goal_completion_even_after_token_cap(monkeypatch):
|
||||
@pytest.mark.parametrize("run_stop_reason", ["token_capped", "loop_capped"])
|
||||
async def test_goal_worker_defers_satisfied_goal_completion_even_after_hard_stop(monkeypatch, run_stop_reason):
|
||||
checkpointer = InMemorySaver()
|
||||
thread_id = "token-capped-done-goal-thread"
|
||||
await _seed_goal_thread(checkpointer, thread_id=thread_id, goal_text="Finish all tests")
|
||||
@ -481,10 +487,10 @@ async def test_goal_worker_defers_satisfied_goal_completion_even_after_token_cap
|
||||
run_id="run-capped-done",
|
||||
model_name="test-model",
|
||||
app_config=None,
|
||||
run_stop_reason="token_capped",
|
||||
run_stop_reason=run_stop_reason,
|
||||
)
|
||||
|
||||
# A token cap does not veto completion, but delivery must still succeed.
|
||||
# A hard stop does not veto completion, but delivery must still succeed.
|
||||
assert isinstance(continuation, worker._GoalCompletionCandidate)
|
||||
assert await read_thread_goal(checkpointer, thread_id) == continuation.goal
|
||||
|
||||
|
||||
@ -39,6 +39,7 @@ def test_cap_stand_down_reasons_are_in_contract():
|
||||
at_cap = {"continuation_count": DEFAULT_MAX_GOAL_CONTINUATIONS}
|
||||
assert worker._stand_down_reason(at_cap, evaluation, 0) in _CODES
|
||||
assert worker._stand_down_reason({"continuation_count": 0}, evaluation, DEFAULT_MAX_NO_PROGRESS_CONTINUATIONS) in _CODES
|
||||
assert worker._GOAL_RUN_HARD_STOP_REASONS <= _CODES
|
||||
|
||||
|
||||
def test_literal_stand_down_reasons_and_no_verdict_are_in_contract():
|
||||
|
||||
@ -42,7 +42,7 @@ def test_stand_down_reason_codes_match_the_worker():
|
||||
}
|
||||
literals = set(re.findall(r'stand_down_reason\s*=\s*"([a-z_:]+)"', Path(worker.__file__).read_text(encoding="utf-8")))
|
||||
assert literals, "pattern no longer matches worker.py; update this test"
|
||||
assert set(_CONTRACT["stand_down_reason_codes"]) == blocked | caps | literals
|
||||
assert set(_CONTRACT["stand_down_reason_codes"]) == blocked | caps | literals | worker._GOAL_RUN_HARD_STOP_REASONS
|
||||
|
||||
|
||||
def test_check_failure_codes_match_the_scheduled_tasks_page():
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
"max_continuations_reached",
|
||||
"no_progress_detected",
|
||||
"token_capped",
|
||||
"loop_capped",
|
||||
"evaluator_failed",
|
||||
"no_durable_end_of_turn",
|
||||
"thread_changed_after_evaluation",
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
"max_continuations_reached",
|
||||
"no_progress_detected",
|
||||
"token_capped",
|
||||
"loop_capped",
|
||||
"evaluator_failed",
|
||||
"no_durable_end_of_turn",
|
||||
"thread_changed_after_evaluation",
|
||||
|
||||
@ -153,6 +153,7 @@ type StopRow = { stop: GoalStopKey; chip: GoalBarChip };
|
||||
const STOP_BY_REASON: Partial<Record<GoalReasonKey, StopRow>> = {
|
||||
noProgress: { stop: "noProgress", chip: "stopped" },
|
||||
tokenCapped: { stop: "tokenCapped", chip: "stopped" },
|
||||
loopCapped: { stop: "loopCapped", chip: "stopped" },
|
||||
missingEvidence: { stop: "missingEvidence", chip: "stopped" },
|
||||
runFailed: { stop: "runFailed", chip: "stopped" },
|
||||
externalWait: { stop: "external", chip: "waiting" },
|
||||
|
||||
@ -35,6 +35,7 @@ const STOP_ICONS: Record<GoalStopKey, LucideIcon> = {
|
||||
autoOff: CircleStopIcon,
|
||||
noProgress: CircleStopIcon,
|
||||
tokenCapped: GaugeIcon,
|
||||
loopCapped: CircleStopIcon,
|
||||
missingEvidence: CircleStopIcon,
|
||||
runFailed: TriangleAlertIcon,
|
||||
needsInputCard: MessageCircleQuestionIcon,
|
||||
|
||||
@ -545,6 +545,8 @@ export const enUS: Translations = {
|
||||
"No progress between turns. Reply with what's missing, or rephrase the goal with /goal.",
|
||||
tokenCapped:
|
||||
"Token budget reached. Send a message to continue; a new run starts with a fresh budget.",
|
||||
loopCapped:
|
||||
"Repeated tool calls triggered a safety stop. Adjust the approach and send a new message to retry; your goal stays active.",
|
||||
missingEvidence:
|
||||
"Goal check: evidence missing. Ask it to show the result or explain what's missing; if the goal no longer applies, run /goal clear.",
|
||||
runFailed:
|
||||
@ -1212,6 +1214,7 @@ export const enUS: Translations = {
|
||||
maxContinuations: "Continuation limit reached",
|
||||
noProgress: "No progress between turns",
|
||||
tokenCapped: "Token budget reached",
|
||||
loopCapped: "Repeated tool calls reached the safety limit",
|
||||
evaluatorFailed: "Couldn't check the goal; the run itself may be fine",
|
||||
noDurableEndOfTurn: "Couldn't check the goal: no final reply was saved",
|
||||
threadChanged:
|
||||
|
||||
@ -432,6 +432,7 @@ export interface Translations {
|
||||
autoOff: string;
|
||||
noProgress: string;
|
||||
tokenCapped: string;
|
||||
loopCapped: string;
|
||||
missingEvidence: string;
|
||||
runFailed: string;
|
||||
needsInputCard: string;
|
||||
@ -996,6 +997,7 @@ export interface Translations {
|
||||
maxContinuations: string;
|
||||
noProgress: string;
|
||||
tokenCapped: string;
|
||||
loopCapped: string;
|
||||
evaluatorFailed: string;
|
||||
noDurableEndOfTurn: string;
|
||||
threadChanged: string;
|
||||
|
||||
@ -508,6 +508,8 @@ export const zhCN: Translations = {
|
||||
"连续几轮没有进展。回复说明还缺什么,或用 /goal 换个说法重新设置目标。",
|
||||
tokenCapped:
|
||||
"已达到 token 预算。发送消息即可继续,新的运行会重新计算预算。",
|
||||
loopCapped:
|
||||
"重复工具调用触发了安全停止。调整要求后发送新消息即可重试,目标会保留。",
|
||||
missingEvidence:
|
||||
"目标检查:缺少依据。可以让它展示结果或说明卡在哪里;如果目标已不适用,用 /goal clear 清除目标。",
|
||||
runFailed:
|
||||
@ -1140,6 +1142,7 @@ export const zhCN: Translations = {
|
||||
maxContinuations: "已达到继续处理的次数上限",
|
||||
noProgress: "连续多轮没有进展",
|
||||
tokenCapped: "已达到 token 预算",
|
||||
loopCapped: "重复工具调用触发了安全停止",
|
||||
evaluatorFailed: "未能检查目标,这次运行本身可能没问题",
|
||||
noDurableEndOfTurn: "未能检查目标:没有保存最终回复",
|
||||
threadChanged: "未能检查目标:检查期间对话发生了变化",
|
||||
|
||||
@ -9,6 +9,7 @@ export type GoalReasonKey =
|
||||
| "maxContinuations"
|
||||
| "noProgress"
|
||||
| "tokenCapped"
|
||||
| "loopCapped"
|
||||
| "evaluatorFailed"
|
||||
| "noDurableEndOfTurn"
|
||||
| "threadChanged"
|
||||
@ -27,6 +28,7 @@ const REASON_KEYS: Record<string, GoalReasonKey> = {
|
||||
max_continuations_reached: "maxContinuations",
|
||||
no_progress_detected: "noProgress",
|
||||
token_capped: "tokenCapped",
|
||||
loop_capped: "loopCapped",
|
||||
evaluator_failed: "evaluatorFailed",
|
||||
no_durable_end_of_turn: "noDurableEndOfTurn",
|
||||
thread_changed_after_evaluation: "threadChanged",
|
||||
|
||||
@ -163,6 +163,30 @@ const OPEN_QUESTION = [
|
||||
];
|
||||
|
||||
const STOOD_DOWN = [
|
||||
{
|
||||
name: "a loop safety stop",
|
||||
objective: "Finish the report",
|
||||
goal: {
|
||||
continuation_count: 0,
|
||||
last_evaluation: {
|
||||
satisfied: false,
|
||||
blocker: "goal_not_met_yet",
|
||||
reason: "The report is unfinished.",
|
||||
stand_down_reason: "loop_capped",
|
||||
},
|
||||
},
|
||||
reply: [
|
||||
{
|
||||
type: "ai",
|
||||
id: "goal-ai",
|
||||
content: "Repeated inspections stopped before the report was finished.",
|
||||
},
|
||||
],
|
||||
chip: "Stopped",
|
||||
detail:
|
||||
"Repeated tool calls triggered a safety stop. Adjust the approach and send a new message to retry; your goal stays active.",
|
||||
note: "The report is unfinished.",
|
||||
},
|
||||
{
|
||||
name: "a goal that used every automatic follow-up",
|
||||
objective: "Make every test in the suite pass",
|
||||
|
||||
@ -13,7 +13,7 @@ export const RAW_IDENTIFIER_PATTERNS: readonly RegExp[] = [
|
||||
/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/,
|
||||
/(^|\s)[\d*/,-]+(\s[\d*/,-]+){4}(\s|$)/,
|
||||
/\b(fresh_thread_per_run|reuse_thread|lead_agent|goal_objective|max_runs|end_at|unmet|stop_scheduled_task|schedule_task)\b/,
|
||||
/\b(blocked:[a-z_]+|max_continuations_reached|no_progress_detected|token_capped|evaluator_failed|no_durable_end_of_turn|thread_changed_(after_evaluation|before_continuation)|goal_outcome|stand_down_reason|last_evaluation)\b/,
|
||||
/\b(blocked:[a-z_]+|max_continuations_reached|no_progress_detected|token_capped|loop_capped|evaluator_failed|no_durable_end_of_turn|thread_changed_(after_evaluation|before_continuation)|goal_outcome|stand_down_reason|last_evaluation)\b/,
|
||||
/\b(goal_not_met_yet|needs_user_input|missing_evidence|external_wait|run_failed|goal_created_at|reply_message_id|relied_on_assumption|continuation_count|max_continuations)\b/,
|
||||
];
|
||||
|
||||
|
||||
@ -138,6 +138,7 @@ describe("describeGoalBar", () => {
|
||||
],
|
||||
["no_progress_detected", {}, "noProgress", "stopped"],
|
||||
["token_capped", {}, "tokenCapped", "stopped"],
|
||||
["loop_capped", {}, "loopCapped", "stopped"],
|
||||
["blocked:missing_evidence", {}, "missingEvidence", "stopped"],
|
||||
["blocked:run_failed", {}, "runFailed", "stopped"],
|
||||
["blocked:external_wait", {}, "external", "waiting"],
|
||||
|
||||
@ -162,6 +162,12 @@ const CASES: Case[] = [
|
||||
en: `Goal${OBJECTIVE}StoppedDetails | Token budget reached. Send a message to continue; a new run starts with a fresh budget.`,
|
||||
zh: `目标${OBJECTIVE}已停止详情 | 已达到 token 预算。发送消息即可继续,新的运行会重新计算预算。`,
|
||||
},
|
||||
{
|
||||
name: "stopped at the loop safety limit",
|
||||
props: { goal: stoodDown("loop_capped") },
|
||||
en: `Goal${OBJECTIVE}StoppedDetails | Repeated tool calls triggered a safety stop. Adjust the approach and send a new message to retry; your goal stays active.`,
|
||||
zh: `目标${OBJECTIVE}已停止详情 | 重复工具调用触发了安全停止。调整要求后发送新消息即可重试,目标会保留。`,
|
||||
},
|
||||
{
|
||||
name: "stopped without evidence (count 0)",
|
||||
props: { goal: stoodDown("blocked:missing_evidence") },
|
||||
|
||||
@ -80,6 +80,7 @@ test.each([
|
||||
["max_continuations_reached", "maxContinuations"],
|
||||
["no_progress_detected", "noProgress"],
|
||||
["token_capped", "tokenCapped"],
|
||||
["loop_capped", "loopCapped"],
|
||||
["no_verdict", "noVerdict"],
|
||||
])("unmet reason %s maps to %s", (code, reasonKey) => {
|
||||
expect(describeGoalOutcome(run({ status: "unmet", error: code }))).toEqual({
|
||||
|
||||
@ -30,6 +30,7 @@ describe("raw identifier guard", () => {
|
||||
"Stopped: blocked:needs_user_input",
|
||||
"max_continuations_reached",
|
||||
"Reason token_capped",
|
||||
"Reason loop_capped",
|
||||
"thread_changed_after_evaluation",
|
||||
"goal_outcome missing",
|
||||
"last_evaluation.stand_down_reason",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user