deer-flow/backend/tests/test_loop_detection_result_progress.py
Eilen Shin 25a744a99a
fix(harness): reflect progress in loop detection and plan todos (#8248)
* feat(frontend): improve file attachment recovery and artifact presentation

* feat(extensions): add Office artifact previews with search and navigation

* fix: preserve public viewers and HTTP Office previews

* fix: address file preview and upload review findings

* fix: use typed upload errors and linear bookmark allocation

* fix(office-preview): isolate parsing and bound preview resources

* fix(ci): install Office preview fixture dependencies for E2E

* fix(ci): provision matching Playwright browsers before E2E tests

* fix(ci): isolate Python libraries and include Office documentation

* test(frontend): wait for download status to finish rendering

* fix(harness): account for tool results in loop detection

* fix(harness): make tool frequency detection observational

* fix(harness): support blocked and waiting plan todos

* style(harness): wrap overlong system prompt lines

* Improve loop detection and todo message formatting

Refactor middleware logic for handling repeated results and update messages for todo list.

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Address plan progress and loop detection review

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
2026-11-19 08:09:43 +08:00

387 lines
17 KiB
Python

"""Result changes distinguish useful repeated calls from stalled tool loops."""
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import pytest
from langchain.agents import create_agent
from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_core.tools import tool
from pydantic import PrivateAttr
from deerflow.agents.middlewares.loop_detection_middleware import LoopDetectionMiddleware
from deerflow.agents.middlewares.read_before_write_middleware import READ_MARK_KEY, ReadBeforeWriteMiddleware
from deerflow.agents.middlewares.subagent_limit_middleware import SubagentLimitMiddleware
class _ScriptedModel(FakeMessagesListChatModel):
_requests: list[list[Any]] = PrivateAttr(default_factory=list)
def bind_tools(self, tools, **kwargs):
return self
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
self._requests.append(list(messages))
return super()._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
def _call(name, args, call_id="call_0"):
# Providers may reuse the same ID in every response.
return {"name": name, "args": args, "id": call_id}
async def _invoke(graph, async_mode, context):
state = {"messages": [("user", "Continue until the requested work is complete.")]}
if async_mode:
return await graph.ainvoke(state, context=context, config={"recursion_limit": 100})
return graph.invoke(state, context=context, config={"recursion_limit": 100})
def _warnings(model):
return [message for messages in model._requests for message in messages if isinstance(message, HumanMessage) and message.name == "loop_warning"]
@pytest.mark.asyncio
@pytest.mark.parametrize("async_mode", [False, True])
@pytest.mark.parametrize("ranged_read", [False, True])
async def test_read_modify_reread_keeps_making_progress(tmp_path, async_mode, ranged_read):
report = tmp_path / "report.md"
report.write_text("Report\nv1\n", encoding="utf-8", newline="\n")
edits = []
@tool
def read_file(path: str, start_line: int = 1, end_line: int | None = None) -> str:
"""Read the requested lines of the temporary report."""
return "".join(Path(path).read_text(encoding="utf-8").splitlines(keepends=True)[start_line - 1 : end_line])
@tool
def str_replace(path: str, old_str: str, new_str: str) -> str:
"""Apply one edit to the temporary report."""
target = Path(path)
content = target.read_text(encoding="utf-8")
assert old_str in content
target.write_text(content.replace(old_str, new_str), encoding="utf-8", newline="\n")
edits.append(new_str)
return "Successfully replaced the requested text."
responses = []
for version in range(1, 7):
read_args = {"path": str(report), **({"start_line": 1, "end_line": 1} if ranged_read else {})}
responses.append(AIMessage(content="", tool_calls=[_call("read_file", read_args)]))
responses.append(AIMessage(content="", tool_calls=[_call("str_replace", {"path": str(report), "old_str": f"v{version}", "new_str": f"v{version + 1}"})]))
responses.append(AIMessage(content="Done."))
model = _ScriptedModel(responses=responses)
loop = LoopDetectionMiddleware()
gate = ReadBeforeWriteMiddleware(content_reader=lambda _runtime, path: Path(path).read_text(encoding="utf-8"))
graph = create_agent(model=model, tools=[read_file, str_replace], middleware=[gate, loop])
context = {"thread_id": "editing", "run_id": "editing"}
result = await _invoke(graph, async_mode, context)
assert edits == [f"v{version}" for version in range(2, 8)]
assert report.read_text(encoding="utf-8") == "Report\nv7\n"
tool_results = [message for message in result["messages"] if isinstance(message, ToolMessage)]
reads = [message for message in tool_results if message.name == "read_file"]
assert len({message.additional_kwargs[READ_MARK_KEY]["hash"] for message in reads}) == 6
if ranged_read:
# The visible first line stays identical; the full-file version changes.
assert {message.content for message in reads} == {"Report\n"}
assert all(message.status == "success" for message in tool_results)
assert result["messages"][-1].content == "Done."
assert not _warnings(model)
assert context.get("stop_reason") is None
@pytest.mark.asyncio
@pytest.mark.parametrize("async_mode", [False, True])
@pytest.mark.parametrize("edit_result", ["same_file", "other_file", "failed"])
async def test_reread_near_limit_after_file_edit(tmp_path, async_mode, edit_result):
report = tmp_path / "report.md"
other = tmp_path / "other.md"
report.write_text("v1", encoding="utf-8")
reads = []
@tool
def read_file(path: str) -> str:
"""Read the report."""
content = Path(path).read_text(encoding="utf-8")
reads.append(content)
return content
@tool
def str_replace(path: str) -> ToolMessage:
"""Apply a test edit."""
if edit_result == "failed":
return ToolMessage(content="Edit failed", status="error", tool_call_id="call_0")
Path(path).write_text("v2", encoding="utf-8")
return ToolMessage(content="Edited", tool_call_id="call_0")
read = _call("read_file", {"path": str(report)})
edit = _call("str_replace", {"path": str(other if edit_result == "other_file" else report)})
responses = [AIMessage(content="", tool_calls=[call]) for call in [read, read, read, read, edit, read]]
model = _ScriptedModel(responses=[*responses, AIMessage(content="Done.")])
graph = create_agent(model=model, tools=[read_file, str_replace], middleware=[LoopDetectionMiddleware()])
context = {"thread_id": "edit-at-limit", "run_id": "edit-at-limit"}
result = await _invoke(graph, async_mode, context)
if edit_result == "same_file":
assert reads == ["v1"] * 4 + ["v2"]
assert result["messages"][-1].content == "Done."
assert context.get("stop_reason") is None
else:
assert reads == ["v1"] * 4
assert context["stop_reason"] == "loop_capped"
@pytest.mark.asyncio
@pytest.mark.parametrize("async_mode", [False, True])
async def test_trimmed_delegations_preserve_progress_in_admitted_batch(async_mode):
from deerflow.agents.lead_agent.agent import build_middlewares
from deerflow.config.app_config import AppConfig
from deerflow.config.sandbox_config import SandboxConfig
observed = []
delegated = []
@tool
def inspect_job() -> str:
"""Return the next observed job state."""
observed.append(len(observed) + 1)
return str(observed[-1])
@tool
def task(description: str) -> str:
"""Record the admitted delegation."""
delegated.append(description)
return "Done."
config = {"configurable": {"subagent_enabled": True, "max_concurrent_subagents": 1}}
app_config = AppConfig(sandbox=SandboxConfig(use="deerflow.sandbox.local:LocalSandboxProvider"))
chain = build_middlewares(config, model_name=None, app_config=app_config, memory_enabled=False, available_skills=set())
middlewares = [mw for mw in chain if isinstance(mw, (LoopDetectionMiddleware, SubagentLimitMiddleware))]
calls = [
_call("inspect_job", {}),
_call("task", {"description": "admitted"}, "task-1"),
_call("task", {"description": "trimmed"}, "task-2"),
]
model = _ScriptedModel(responses=[AIMessage(content="", tool_calls=calls) for _ in range(6)] + [AIMessage(content="Done.")])
graph = create_agent(model=model, tools=[inspect_job, task], middleware=middlewares)
context = {"thread_id": "admitted-batch", "run_id": "admitted-batch"}
result = await _invoke(graph, async_mode, context)
assert observed == list(range(1, 7))
assert delegated == ["admitted"] * 6
assert result["messages"][-1].content == "Done."
assert context.get("stop_reason") is None
@pytest.mark.asyncio
@pytest.mark.parametrize("async_mode", [False, True])
@pytest.mark.parametrize("batched", [False, True])
async def test_changing_results_work_for_arbitrary_tools(async_mode, batched):
observations = []
@tool
def inspect_job(job: str) -> list[dict]:
"""Return the next observed job state."""
observations.append(len(observations) + 1)
return [{"type": "text", "text": f"Completed {observations[-1]} steps"}]
@tool
def lookup_owner(job: str) -> str:
"""Return the stable owner of the job."""
return "owner"
responses = []
for index in range(6):
calls = [_call("inspect_job", {"job": "demo"})]
if batched:
calls.append(_call("lookup_owner", {"job": "demo"}, "call_1"))
if index % 2:
calls.reverse()
responses.append(AIMessage(content="", tool_calls=calls))
responses.append(AIMessage(content="Done."))
model = _ScriptedModel(responses=responses)
loop = LoopDetectionMiddleware()
graph = create_agent(model=model, tools=[inspect_job, lookup_owner], middleware=[loop])
result = await _invoke(graph, async_mode, {"thread_id": "jobs", "run_id": "jobs"})
assert observations == list(range(1, 7))
assert result["messages"][-1].content == "Done."
assert not _warnings(model)
assert loop.consume_stop_reason("jobs") is None
@pytest.mark.asyncio
@pytest.mark.parametrize("async_mode", [False, True])
@pytest.mark.parametrize("failure", [None, "native", "metadata"])
async def test_unchanged_results_and_changing_errors_still_stop(async_mode, failure):
executions = []
@tool
def inspect_job(job: str) -> ToolMessage:
"""Inspect the same stalled job through the tool boundary."""
executions.append(len(executions) + 1)
metadata = {"receipt": {"id": len(executions)}}
if failure == "metadata":
metadata["deerflow_tool_meta"] = {"status": "error", "error_type": "transient"}
return ToolMessage(
content=f"Attempt {executions[-1]} failed" if failure else "No change",
status="error" if failure == "native" else "success",
additional_kwargs=metadata,
tool_call_id="call_0",
)
model = _ScriptedModel(responses=[AIMessage(content="", tool_calls=[_call("inspect_job", {"job": "demo"})]) for _ in range(6)])
loop = LoopDetectionMiddleware()
graph = create_agent(model=model, tools=[inspect_job], middleware=[loop])
context = {"thread_id": "stalled", "run_id": "stalled"}
result = await _invoke(graph, async_mode, context)
assert len(executions) == 4
assert len(model._requests) == 5
assert len(_warnings(model)) == 1
assert not result["messages"][-1].tool_calls
assert context["stop_reason"] == "loop_capped"
@pytest.mark.asyncio
async def test_repeated_result_cycle_still_stops():
executions = []
@tool
def inspect_job(job: str) -> str:
"""Observe a job oscillating between two states."""
executions.append(len(executions) % 2)
return str(executions[-1])
model = _ScriptedModel(responses=[AIMessage(content="", tool_calls=[_call("inspect_job", {"job": "demo"})]) for _ in range(12)])
graph = create_agent(model=model, tools=[inspect_job], middleware=[LoopDetectionMiddleware()])
context = {"thread_id": "cycling", "run_id": "cycling"}
await _invoke(graph, True, context)
# Four observed zeroes plus the next predicted zero reach the hard limit.
assert executions == [0, 1, 0, 1, 0, 1, 0]
assert context["stop_reason"] == "loop_capped"
@pytest.mark.asyncio
async def test_changing_results_are_not_stopped_by_tool_frequency_metrics():
executions = []
@tool
def inspect_job(job: str) -> str:
"""Observe a job that always returns a new state."""
executions.append(len(executions))
return str(executions[-1])
model = _ScriptedModel(responses=[AIMessage(content="", tool_calls=[_call("inspect_job", {"job": "demo"})]) for _ in range(8)] + [AIMessage(content="Done.")])
loop = LoopDetectionMiddleware(tool_freq_warn=4, tool_freq_hard_limit=6)
graph = create_agent(model=model, tools=[inspect_job], middleware=[loop])
context = {"thread_id": "budget", "run_id": "budget"}
result = await _invoke(graph, True, context)
assert len(executions) == 8
assert result["messages"][-1].content == "Done."
assert not _warnings(model)
assert context.get("stop_reason") is None
@pytest.mark.asyncio
async def test_noop_writes_still_hit_the_loop_limit(tmp_path):
report = tmp_path / "report.md"
report.write_text("unchanged", encoding="utf-8", newline="\n")
reads, writes = [], []
@tool
def read_file(path: str) -> str:
"""Read the temporary report."""
reads.append(path)
return Path(path).read_text(encoding="utf-8")
@tool
def write_file(path: str, content: str) -> str:
"""Write the same report contents."""
writes.append(path)
Path(path).write_text(content, encoding="utf-8", newline="\n")
return "Saved."
calls = [AIMessage(content="", tool_calls=[_call(name, args)]) for _ in range(6) for name, args in [("read_file", {"path": str(report)}), ("write_file", {"path": str(report), "content": "unchanged"})]]
model = _ScriptedModel(responses=calls)
loop = LoopDetectionMiddleware()
gate = ReadBeforeWriteMiddleware(content_reader=lambda _runtime, path: Path(path).read_text(encoding="utf-8"))
graph = create_agent(model=model, tools=[read_file, write_file], middleware=[gate, loop])
context = {"thread_id": "noop", "run_id": "noop"}
await _invoke(graph, True, context)
# The last successful write warrants one more read. Its result settles to
# the old version, and the unchanged write loop still reaches the cap.
assert len(reads) == 5
assert len(writes) == 4
assert report.read_text(encoding="utf-8") == "unchanged"
assert context["stop_reason"] == "loop_capped"
def _runtime():
return SimpleNamespace(context={"thread_id": "history", "run_id": "history"})
@pytest.mark.parametrize("metadata", [None, [], {"deerflow_tool_meta": ["error"]}, {READ_MARK_KEY: {"path": [], "hash": {}}}, {READ_MARK_KEY: {"path": "/report", "hash": "not-a-digest"}}])
def test_malformed_persisted_metadata_cannot_disable_loop_protection(metadata):
loop = LoopDetectionMiddleware()
messages = []
for index in range(5):
messages.append(AIMessage(id=f"turn-{index}", content="", tool_calls=[_call("inspect_job", {})]))
update = loop.after_model({"messages": messages}, _runtime())
if index < 4:
result = ToolMessage(content="unchanged", tool_call_id="call_0").model_copy(update={"additional_kwargs": metadata})
messages.append(result)
assert not update["messages"][0].tool_calls
assert loop.consume_stop_reason("history") == "loop_capped"
@pytest.mark.parametrize("malformation", ["missing", "duplicate_id", "unserializable"])
def test_incomplete_batches_do_not_borrow_results_from_reused_call_ids(malformation):
loop = LoopDetectionMiddleware()
messages = []
for index in range(5):
calls = [_call("inspect_job", {}), _call("lookup_owner", {}, "call_0" if malformation == "duplicate_id" else "call_1")]
messages.append(AIMessage(id=f"turn-{index}", content="", tool_calls=calls))
update = loop.after_model({"messages": messages}, _runtime())
if index < 4:
messages.append(ToolMessage(content=f"state {index}", tool_call_id="call_0"))
if malformation != "missing":
result = ToolMessage(content="owner", tool_call_id=calls[1]["id"])
if malformation == "unserializable":
cyclic = []
cyclic.append(cyclic)
result = result.model_copy(update={"content": cyclic})
messages.append(result)
assert not update["messages"][0].tool_calls
assert loop.consume_stop_reason("history") == "loop_capped"
def test_compacted_history_cannot_settle_a_different_call_occurrence():
loop = LoopDetectionMiddleware(hard_limit=3)
call = _call("inspect_job", {})
latest = AIMessage(id="current", content="", tool_calls=[call])
loop.after_model({"messages": [latest]}, _runtime())
for index in range(2):
# A stale checkpoint retains the same call ID, but a different AIMessage.
stale = AIMessage(id="old", content="", tool_calls=[call])
result = ToolMessage(content=f"old state {index}", tool_call_id="call_0")
latest = AIMessage(id=f"next-{index}", content="", tool_calls=[call])
update = loop.after_model({"messages": [stale, result, latest]}, _runtime())
assert not update["messages"][0].tool_calls
assert loop.consume_stop_reason("history") == "loop_capped"