deer-flow/backend/tests/test_loop_detection_middleware.py
Eilen Shin 25a744a99a
fix(harness): reflect progress in loop detection and plan todos (#8248)
* feat(frontend): improve file attachment recovery and artifact presentation

* feat(extensions): add Office artifact previews with search and navigation

* fix: preserve public viewers and HTTP Office previews

* fix: address file preview and upload review findings

* fix: use typed upload errors and linear bookmark allocation

* fix(office-preview): isolate parsing and bound preview resources

* fix(ci): install Office preview fixture dependencies for E2E

* fix(ci): provision matching Playwright browsers before E2E tests

* fix(ci): isolate Python libraries and include Office documentation

* test(frontend): wait for download status to finish rendering

* fix(harness): account for tool results in loop detection

* fix(harness): make tool frequency detection observational

* fix(harness): support blocked and waiting plan todos

* style(harness): wrap overlong system prompt lines

* Improve loop detection and todo message formatting

Refactor middleware logic for handling repeated results and update messages for todo list.

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Address plan progress and loop detection review

---------

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
2026-11-19 08:09:43 +08:00

2036 lines
89 KiB
Python

"""Tests for LoopDetectionMiddleware."""
import copy
from collections import OrderedDict
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import pytest
from langchain.agents import create_agent
from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
from langchain_core.runnables import Runnable
from langchain_core.tools import tool as as_tool
from pydantic import PrivateAttr
from deerflow.agents.middlewares import loop_detection_middleware as loop_detection_module
from deerflow.agents.middlewares.loop_detection_middleware import (
_HARD_STOP_MSG,
_MAX_PENDING_WARNINGS_PER_RUN,
LoopDetectionMiddleware,
_hash_tool_calls,
)
def _make_runtime(thread_id="test-thread", run_id="test-run"):
"""Build a minimal Runtime mock with context."""
runtime = MagicMock()
runtime.context = {"thread_id": thread_id, "run_id": run_id}
return runtime
def _pending_key(thread_id="test-thread", run_id="test-run"):
return (thread_id, run_id)
def _scope_key(thread_id="test-thread", run_id="test-run"):
return (thread_id, run_id)
def _make_request(messages, runtime):
"""Build a minimal ModelRequest stand-in for wrap_model_call tests."""
request = MagicMock()
request.messages = list(messages)
request.runtime = runtime
request.override = lambda **updates: _override_request(request, updates)
return request
def _override_request(request, updates):
"""Mimic ModelRequest.override(): return a copy with fields replaced."""
new = MagicMock()
new.messages = updates.get("messages", request.messages)
new.runtime = updates.get("runtime", request.runtime)
new.override = lambda **u: _override_request(new, u)
return new
def _capture_handler():
"""Build a sync handler that records the request it was called with."""
captured: list = []
def handler(req):
captured.append(req)
return MagicMock()
return captured, handler
class _CapturingFakeMessagesListChatModel(FakeMessagesListChatModel):
"""Fake chat model that records each model request's messages."""
_seen_messages: list[list[Any]] = PrivateAttr(default_factory=list)
@property
def seen_messages(self) -> list[list[Any]]:
return self._seen_messages
def bind_tools(
self,
tools: Any,
*,
tool_choice: Any = None,
**kwargs: Any,
) -> Runnable:
return self
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
self._seen_messages.append(list(messages))
return super()._generate(
messages,
stop=stop,
run_manager=run_manager,
**kwargs,
)
def _make_state(tool_calls=None, content=""):
"""Build a minimal AgentState dict with an AIMessage.
Deep-copies *content* when it is mutable (e.g. list) so that
successive calls never share the same object reference.
"""
safe_content = copy.deepcopy(content) if isinstance(content, list) else content
msg = AIMessage(content=safe_content, tool_calls=tool_calls or [])
return {"messages": [msg]}
def _bash_call(cmd="ls"):
return {"name": "bash", "id": f"call_{cmd}", "args": {"command": cmd}}
class TestHashToolCalls:
def test_same_calls_same_hash(self):
a = _hash_tool_calls([_bash_call("ls")])
b = _hash_tool_calls([_bash_call("ls")])
assert a == b
def test_different_calls_different_hash(self):
a = _hash_tool_calls([_bash_call("ls")])
b = _hash_tool_calls([_bash_call("pwd")])
assert a != b
def test_order_independent(self):
a = _hash_tool_calls([_bash_call("ls"), {"name": "read_file", "args": {"path": "/tmp"}}])
b = _hash_tool_calls([{"name": "read_file", "args": {"path": "/tmp"}}, _bash_call("ls")])
assert a == b
def test_empty_calls(self):
h = _hash_tool_calls([])
assert isinstance(h, str)
assert len(h) > 0
def test_stringified_dict_args_match_dict_args(self):
dict_call = {
"name": "read_file",
"args": {"path": "/tmp/demo.py", "start_line": "1", "end_line": "150"},
}
string_call = {
"name": "read_file",
"args": '{"path":"/tmp/demo.py","start_line":"1","end_line":"150"}',
}
assert _hash_tool_calls([dict_call]) == _hash_tool_calls([string_call])
def test_reversed_read_file_range_matches_forward_range(self):
forward_call = {
"name": "read_file",
"args": {"path": "/tmp/demo.py", "start_line": 10, "end_line": 300},
}
reversed_call = {
"name": "read_file",
"args": {"path": "/tmp/demo.py", "start_line": 300, "end_line": 10},
}
assert _hash_tool_calls([forward_call]) == _hash_tool_calls([reversed_call])
def test_non_finite_read_file_line_bound_does_not_crash(self):
"""A model-emitted ``1e999`` parses to ``float('inf')`` via ``json.loads``.
``int(float('inf'))`` raises ``OverflowError``, which the line-bound
coercion must treat as an unusable value — falling back to the
open-ended read key — instead of crashing ``after_model`` and the run.
"""
inf_call = {
"name": "read_file",
"args": {"path": "/tmp/demo.py", "start_line": float("inf")},
}
open_ended_call = {"name": "read_file", "args": {"path": "/tmp/demo.py"}}
assert _hash_tool_calls([inf_call]) == _hash_tool_calls([open_ended_call])
def test_non_finite_end_line_does_not_crash(self):
inf_call = {
"name": "read_file",
"args": {"path": "/tmp/demo.py", "end_line": float("inf")},
}
assert isinstance(_hash_tool_calls([inf_call]), str)
def test_stringified_non_dict_args_do_not_crash(self):
non_dict_json_call = {"name": "bash", "args": '"echo hello"'}
plain_string_call = {"name": "bash", "args": "echo hello"}
json_hash = _hash_tool_calls([non_dict_json_call])
plain_hash = _hash_tool_calls([plain_string_call])
assert isinstance(json_hash, str)
assert isinstance(plain_hash, str)
assert json_hash
assert plain_hash
def test_grep_pattern_affects_hash(self):
grep_foo = {"name": "grep", "args": {"path": "/tmp", "pattern": "foo"}}
grep_bar = {"name": "grep", "args": {"path": "/tmp", "pattern": "bar"}}
assert _hash_tool_calls([grep_foo]) != _hash_tool_calls([grep_bar])
def test_glob_pattern_affects_hash(self):
glob_py = {"name": "glob", "args": {"path": "/tmp", "pattern": "*.py"}}
glob_ts = {"name": "glob", "args": {"path": "/tmp", "pattern": "*.ts"}}
assert _hash_tool_calls([glob_py]) != _hash_tool_calls([glob_ts])
def test_write_file_content_affects_hash(self):
v1 = {"name": "write_file", "args": {"path": "/tmp/a.py", "content": "v1"}}
v2 = {"name": "write_file", "args": {"path": "/tmp/a.py", "content": "v2"}}
assert _hash_tool_calls([v1]) != _hash_tool_calls([v2])
def test_str_replace_content_affects_hash(self):
a = {
"name": "str_replace",
"args": {"path": "/tmp/a.py", "old_str": "foo", "new_str": "bar"},
}
b = {
"name": "str_replace",
"args": {"path": "/tmp/a.py", "old_str": "foo", "new_str": "baz"},
}
assert _hash_tool_calls([a]) != _hash_tool_calls([b])
class TestReadFileRangeKey:
"""``read_file`` keys must separate paging progress from re-reading.
Line ranges used to be quantized into 200-line buckets, which erased the
offset inside a bucket: every read shorter than 200 lines collapsed onto its
neighbours, so paging one file in 40-line chunks looked like five identical
calls and tripped the hard stop — on reads ``read_file``'s own truncation
notice tells the model to make.
"""
@staticmethod
def _read_call(path="/w/app.py", **range_args):
return {"name": "read_file", "id": "call_read", "args": {"path": path, **range_args}}
def test_adjacent_pages_are_distinct_calls(self):
pages = [self._read_call(start_line=start, end_line=start + 39) for start in (1, 41, 81, 121, 161)]
hashes = {_hash_tool_calls([page]) for page in pages}
assert len(hashes) == len(pages)
def test_paging_through_a_file_does_not_hard_stop(self):
"""Regression: five sequential 40-line reads used to force a final answer."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=5)
runtime = _make_runtime()
for start in range(1, 401, 40):
decision = mw._apply(_make_state(tool_calls=[self._read_call(start_line=start, end_line=start + 39)]), runtime)
assert decision is None, f"read of lines {start}-{start + 39} was treated as a loop"
assert mw.consume_stop_reason("test-run") is None
def test_repeating_one_range_still_hard_stops(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=5)
runtime = _make_runtime()
call = [self._read_call(start_line=1, end_line=40)]
for _ in range(4):
assert mw._apply(_make_state(tool_calls=call), runtime) is None
hard_stop = mw._apply(_make_state(tool_calls=call), runtime)
assert hard_stop is not None
assert hard_stop["messages"][0].tool_calls == []
assert mw.consume_stop_reason("test-run") == "loop_capped"
def test_rereading_the_same_page_counts_even_between_new_pages(self):
"""Interleaving fresh pages must not hide a repeated read: the window counts by key."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=3)
runtime = _make_runtime()
repeated = [self._read_call(start_line=1, end_line=40)]
assert mw._apply(_make_state(tool_calls=repeated), runtime) is None
assert mw._apply(_make_state(tool_calls=[self._read_call(start_line=41, end_line=80)]), runtime) is None
assert mw._apply(_make_state(tool_calls=repeated), runtime) is None
assert mw._apply(_make_state(tool_calls=[self._read_call(start_line=81, end_line=120)]), runtime) is None
assert mw._apply(_make_state(tool_calls=repeated), runtime) is not None
def test_omitted_end_line_matches_a_bare_read_of_the_same_file(self):
"""Both read to the last line, so they are one read written two ways."""
open_ended = self._read_call(start_line=1)
bare = self._read_call()
assert _hash_tool_calls([open_ended]) == _hash_tool_calls([bare])
def test_omitted_end_line_is_not_a_single_line_read(self):
open_ended = self._read_call(start_line=10)
single_line = self._read_call(start_line=10, end_line=10)
assert _hash_tool_calls([open_ended]) != _hash_tool_calls([single_line])
def test_line_bounds_are_clamped_to_the_first_line(self):
assert _hash_tool_calls([self._read_call(start_line=0, end_line=40)]) == _hash_tool_calls([self._read_call(start_line=1, end_line=40)])
def test_unparsable_end_line_reads_as_open_ended(self):
"""The tool rejects such a call anyway; the key must stay stable, not crash."""
assert _hash_tool_calls([self._read_call(start_line=5, end_line="oops")]) == _hash_tool_calls([self._read_call(start_line=5)])
def test_different_paths_stay_distinct(self):
assert _hash_tool_calls([self._read_call(path="/w/a.py", start_line=1, end_line=40)]) != _hash_tool_calls([self._read_call(path="/w/b.py", start_line=1, end_line=40)])
class TestGenericToolKey:
"""Tools without a dedicated key rule must keep every argument that changes the call.
Keying only the first salient field (``path``/``url``/``query``/...) collapsed
distinct calls onto one key: paging a URL or a search, or rewriting one
skill file with new content, hard-stopped on its fifth *distinct* call.
Only a sandbox tool's UI narration (``description``) may vary without
making a call new.
"""
@staticmethod
def _call(tool_name, args):
return {"name": tool_name, "id": f"call_{tool_name}", "args": args}
@pytest.mark.parametrize(
("name", "first", "second"),
[
("fetch", {"url": "https://example.com/doc", "start_index": 0}, {"url": "https://example.com/doc", "start_index": 5000}),
("search_issues", {"query": "is:open label:bug", "page": 1}, {"query": "is:open label:bug", "page": 2}),
("list_uploaded_files", {"query": "report"}, {"query": "report", "cursor": "next-page"}),
("skill_manage", {"action": "write_file", "name": "etl", "path": "scripts/run.py", "content": "v1"}, {"action": "write_file", "name": "etl", "path": "scripts/run.py", "content": "v2"}),
("grep", {"path": "/w", "pattern": "foo", "case_sensitive": False}, {"path": "/w", "pattern": "foo", "case_sensitive": True}),
],
)
def test_non_salient_args_affect_hash(self, name, first, second):
assert _hash_tool_calls([self._call(name, first)]) != _hash_tool_calls([self._call(name, second)])
@pytest.mark.parametrize(
("name", "args"),
[
("bash", {"command": "make test"}),
("ls", {"path": "/w"}),
("glob", {"path": "/w", "pattern": "*.py"}),
("grep", {"path": "/w", "pattern": "foo"}),
],
)
def test_sandbox_ui_narration_does_not_affect_hash(self, name, args):
"""Regression guard for #1905: rewording the narration must not dodge detection."""
first = self._call(name, {**args, "description": "run it"})
second = self._call(name, {**args, "description": "run it once more"})
assert _hash_tool_calls([first]) == _hash_tool_calls([second])
@pytest.mark.parametrize(
("name", "args"),
[
("update_agent", {}),
("setup_agent", {"soul": "You are helpful."}),
("create_issue", {"title": "Crash on start"}),
],
)
def test_description_payload_affects_hash(self, name, args):
"""``description`` is the operation's payload outside the sandbox narration tools."""
first = self._call(name, {**args, "description": "Reviews pull requests."})
second = self._call(name, {**args, "description": "Reviews pull requests and triages issues."})
assert _hash_tool_calls([first]) != _hash_tool_calls([second])
def test_description_only_agent_updates_do_not_hard_stop(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=5)
runtime = _make_runtime()
for version in range(6):
call = self._call("update_agent", {"description": f"Assistant v{version}"})
assert mw._apply(_make_state(tool_calls=[call]), runtime) is None, f"update to v{version} was treated as a loop"
assert mw.consume_stop_reason("test-run") is None
def test_paging_a_url_does_not_hard_stop(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=5)
runtime = _make_runtime()
for start in range(0, 50_000, 5_000):
call = self._call("fetch", {"url": "https://example.com/doc", "max_length": 5_000, "start_index": start})
assert mw._apply(_make_state(tool_calls=[call]), runtime) is None, f"page at start_index={start} was treated as a loop"
assert mw.consume_stop_reason("test-run") is None
def test_repeating_one_page_still_hard_stops(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=5)
runtime = _make_runtime()
call = [self._call("fetch", {"url": "https://example.com/doc", "start_index": 5_000})]
for _ in range(4):
assert mw._apply(_make_state(tool_calls=call), runtime) is None
hard_stop = mw._apply(_make_state(tool_calls=call), runtime)
assert hard_stop is not None
assert hard_stop["messages"][0].tool_calls == []
assert mw.consume_stop_reason("test-run") == "loop_capped"
class TestLoopDetection:
def test_no_tool_calls_returns_none(self):
mw = LoopDetectionMiddleware()
runtime = _make_runtime()
state = {"messages": [AIMessage(content="hello")]}
result = mw._apply(state, runtime)
assert result is None
def test_below_threshold_returns_none(self):
mw = LoopDetectionMiddleware(warn_threshold=3)
runtime = _make_runtime()
call = [_bash_call("ls")]
# First two identical calls — no warning
for _ in range(2):
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is None
def test_non_finite_line_bound_does_not_break_detection(self):
"""``after_model`` must survive a read_file call whose JSON args carry
``start_line: 1e999`` (parsed to ``float('inf')``): the middleware keeps
tracking the call instead of raising ``OverflowError`` out of the hook.
"""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=5)
runtime = _make_runtime()
call = [{"name": "read_file", "id": "call_inf", "args": {"path": "/tmp/demo.py", "start_line": float("inf")}}]
for _ in range(2):
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is None
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is None
assert mw._pending_warnings[_pending_key()]
assert "LOOP DETECTED" in mw._pending_warnings[_pending_key()][0]
def test_warn_at_threshold_queues_but_does_not_mutate_state(self):
"""At warn threshold, ``after_model`` enqueues but returns None.
Detection observes the just-emitted AIMessage(tool_calls=...). The
tools node hasn't run yet, so injecting any non-tool message here
would split the assistant's tool_calls from their ToolMessage
responses and break OpenAI/Moonshot pairing. The warning is
delivered later from ``wrap_model_call``.
"""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=5)
runtime = _make_runtime()
call = [_bash_call("ls")]
for _ in range(2):
mw._apply(_make_state(tool_calls=call), runtime)
# Third identical call triggers warning detection.
result = mw._apply(_make_state(tool_calls=call), runtime)
# Detection must not mutate state — the AIMessage with tool_calls is
# left untouched so the tools node runs normally.
assert result is None
# ...but a warning is queued for the next model call.
assert mw._pending_warnings[_pending_key()]
assert "LOOP DETECTED" in mw._pending_warnings[_pending_key()][0]
def test_warn_injected_at_next_model_call(self):
"""``wrap_model_call`` appends a HumanMessage(loop_warning) to the
outgoing messages — *after* every existing message — so that the
AIMessage(tool_calls=...) -> ToolMessage(...) pairing stays intact.
"""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime = _make_runtime()
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
# Build the messages the agent runtime would assemble for the next
# turn: prior AIMessage(tool_calls), its ToolMessage responses, ...
ai_msg = AIMessage(content="", tool_calls=call)
tool_msg = ToolMessage(content="ok", tool_call_id=call[0]["id"], name="bash")
request = _make_request([ai_msg, tool_msg], runtime)
captured, handler = _capture_handler()
mw.wrap_model_call(request, handler)
sent = captured[0].messages
# AIMessage and ToolMessage stay in order, untouched.
assert sent[0] is ai_msg
assert sent[1] is tool_msg
# HumanMessage(warning) appears AFTER the ToolMessage — pairing intact.
assert isinstance(sent[2], HumanMessage)
assert sent[2].name == "loop_warning"
assert "LOOP DETECTED" in sent[2].content
def test_warn_queue_drained_after_injection(self):
"""A queued warning must be emitted exactly once per detection event."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime = _make_runtime()
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
request = _make_request([AIMessage(content="hi")], runtime)
captured, handler = _capture_handler()
# First call: warning is appended.
mw.wrap_model_call(request, handler)
first = captured[0].messages
assert any(isinstance(m, HumanMessage) for m in first)
# Subsequent call without new detection: no warning re-emitted.
request2 = _make_request([AIMessage(content="hi")], runtime)
mw.wrap_model_call(request2, handler)
second = captured[1].messages
assert not any(isinstance(m, HumanMessage) for m in second)
def test_warn_queue_scoped_by_run_id(self):
"""A warning queued for one run must not be injected into another run."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime_a = _make_runtime(run_id="run-A")
runtime_b = _make_runtime(run_id="run-B")
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime_a)
request_b = _make_request([AIMessage(content="hi")], runtime_b)
captured, handler = _capture_handler()
mw.wrap_model_call(request_b, handler)
assert not any(isinstance(m, HumanMessage) for m in captured[0].messages)
assert mw._pending_warnings.get(_pending_key(run_id="run-A"))
request_a = _make_request([AIMessage(content="hi")], runtime_a)
mw.wrap_model_call(request_a, handler)
assert any(isinstance(message, HumanMessage) and message.name == "loop_warning" for message in captured[1].messages)
def test_missing_run_id_uses_per_runtime_pending_scope(self):
"""When runtime.context has no ``run_id`` key at all, warning handling
falls back to a key scoped to the LangGraph invocation instead of a
shared literal like the old ``"default"``."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime = SimpleNamespace(
context={"thread_id": "test-thread"},
control=object(),
execution_info=None,
)
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
fallback_run_id = mw._get_run_id(runtime)
assert mw._pending_warnings.get(_pending_key(run_id=fallback_run_id))
request = _make_request([AIMessage(content="hi")], runtime)
captured, handler = _capture_handler()
mw.wrap_model_call(request, handler)
loop_warnings = [message for message in captured[0].messages if isinstance(message, HumanMessage) and message.name == "loop_warning"]
assert len(loop_warnings) == 1
assert "LOOP DETECTED" in loop_warnings[0].content
assert not mw._pending_warnings.get(_pending_key(run_id=fallback_run_id))
def test_missing_run_id_shares_scope_across_runtime_wrappers(self):
"""LangGraph replaces ``Runtime`` per node but preserves ``control``."""
mw = LoopDetectionMiddleware()
invocation_control = object()
first_runtime = SimpleNamespace(
context={"thread_id": "test-thread"},
control=invocation_control,
execution_info=None,
)
later_runtime = SimpleNamespace(
context={"thread_id": "test-thread"},
control=invocation_control,
execution_info=None,
)
assert mw._get_run_id(first_runtime) == mw._get_run_id(later_runtime)
def test_missing_run_id_fallback_survives_reused_object_address(self, monkeypatch):
"""A later invocation must not inherit a freed anchor's fallback key."""
mw = LoopDetectionMiddleware()
monkeypatch.setattr(loop_detection_module, "id", lambda _value: 42, raising=False)
first_runtime = SimpleNamespace(
context={"thread_id": "test-thread"},
control=object(),
execution_info=None,
)
first_run_id = mw._get_run_id(first_runtime)
mw.after_agent({"messages": []}, first_runtime)
assert not mw._fallback_run_ids
later_runtime = SimpleNamespace(
context={"thread_id": "test-thread"},
control=object(),
execution_info=None,
)
assert mw._get_run_id(later_runtime) != first_run_id
def test_missing_run_id_fallback_map_is_bounded_on_abnormal_exits(self):
mw = LoopDetectionMiddleware(max_tracked_threads=2)
for _ in range(10):
runtime = SimpleNamespace(
context={"thread_id": "test-thread"},
control=object(),
execution_info=None,
)
mw._get_run_id(runtime)
assert len(mw._fallback_run_ids) == mw._max_fallback_run_ids == 4
def test_before_agent_preserves_pending_warning_for_sibling_run(self):
"""An overlapping run must not erase a warning owned by another run."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime_a = _make_runtime(run_id="run-A")
runtime_b = _make_runtime(run_id="run-B")
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime_a)
assert mw._pending_warnings.get(_pending_key(run_id="run-A"))
mw.before_agent({"messages": []}, runtime_b)
assert mw._pending_warnings.get(_pending_key(run_id="run-A"))
request = _make_request([AIMessage(content="hi")], runtime_a)
captured, handler = _capture_handler()
mw.wrap_model_call(request, handler)
loop_warnings = [message for message in captured[0].messages if isinstance(message, HumanMessage) and message.name == "loop_warning"]
assert len(loop_warnings) == 1
assert not mw._pending_warnings.get(_pending_key(run_id="run-A"))
def test_after_agent_clears_current_run_pending_warnings(self):
"""Run cleanup should drop warnings that never reached wrap_model_call."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime = _make_runtime()
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
assert mw._pending_warnings.get(_pending_key())
mw.after_agent({"messages": []}, runtime)
assert not mw._pending_warnings.get(_pending_key())
def test_multiple_pending_warnings_are_merged_into_one_message(self):
"""Edge-case drains should produce one loop_warning prompt message."""
mw = LoopDetectionMiddleware()
runtime = _make_runtime()
mw._pending_warnings[_pending_key()] = ["first warning", "second warning", "first warning"]
request = _make_request([AIMessage(content="hi")], runtime)
captured, handler = _capture_handler()
mw.wrap_model_call(request, handler)
loop_warnings = [message for message in captured[0].messages if isinstance(message, HumanMessage) and message.name == "loop_warning"]
assert len(loop_warnings) == 1
assert loop_warnings[0].content == "first warning\n\nsecond warning"
def test_warn_only_queued_once_per_hash(self):
"""Same hash repeated past the threshold should warn only once."""
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime = _make_runtime()
call = [_bash_call("ls")]
# First two — no warning
for _ in range(2):
mw._apply(_make_state(tool_calls=call), runtime)
# Third — warning queued
mw._apply(_make_state(tool_calls=call), runtime)
assert len(mw._pending_warnings[_pending_key()]) == 1
# Fourth — already warned for this hash, no additional enqueue.
mw._apply(_make_state(tool_calls=call), runtime)
assert len(mw._pending_warnings[_pending_key()]) == 1
def test_hard_stop_at_limit(self):
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = _make_runtime()
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
# Fourth call triggers hard stop
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is not None
msgs = result["messages"]
assert len(msgs) == 1
# Hard stop strips tool_calls
assert isinstance(msgs[0], AIMessage)
assert msgs[0].tool_calls == []
assert _HARD_STOP_MSG in msgs[0].content
def test_hard_stop_stamps_loop_capped_stop_reason(self):
"""#3875 Phase 2 (ggnnggez review): the loop hard-stop stamps
``loop_capped`` on ``consume_stop_reason`` so the executor can surface
``completed + loop_capped`` instead of a clean completion. Mirrors
``TokenBudgetMiddleware.consume_stop_reason``."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = _make_runtime() # run_id="test-run"
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
# Fourth call triggers the hard stop -> stamps loop_capped.
hard_stop_result = mw._apply(_make_state(tool_calls=call), runtime)
assert hard_stop_result is not None
assert mw.consume_stop_reason("test-run") == "loop_capped"
# Popped on read — a second read is None (no double-report on reuse).
assert mw.consume_stop_reason("test-run") is None
def test_warn_only_does_not_stamp_stop_reason(self):
"""Crossing the warn threshold (not the hard limit) keeps the run going
and must NOT stamp ``loop_capped`` — the run is not capped."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=10)
runtime = _make_runtime()
call = [_bash_call("ls")]
# Two identical calls cross warn (2) but not hard (10).
mw._apply(_make_state(tool_calls=call), runtime)
mw._apply(_make_state(tool_calls=call), runtime)
assert mw.consume_stop_reason("test-run") is None
def test_hard_stop_stamps_loop_capped_with_explicit_none_run_id(self):
"""Regression: a subagent whose ``run_id`` is genuinely ``None`` must
still round-trip its ``loop_capped`` stop reason.
``SubagentExecutor`` sets ``context["run_id"] = self.run_id``
unconditionally (no truthiness guard), so an embedded/TUI-dispatched
subagent — whose ``run_id`` is never assigned per ``AGENTS.md``'s
description of the embedded ``DeerFlowClient`` — runs with a context
that legitimately carries ``run_id=None`` (the key is *present*, not
absent). The executor later reads the reason back with the raw
attribute: ``consume_stop_reason(self.run_id)``, i.e.
``consume_stop_reason(None)``. Before the fix, ``_get_run_id`` used a
truthiness check (``if run_id:``) that collapsed this present-but-None
state to the same literal ``"default"`` key used for a totally absent
run_id, so the write (``self._stop_reason["default"] = "loop_capped"``)
and this read (keyed by the raw ``None``) disagreed and the signal was
silently lost. Mirrors ``TokenBudgetMiddleware``'s key-presence-based
``_get_run_id``, which does not have this bug."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = SimpleNamespace(context={"thread_id": "t", "run_id": None})
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
hard_stop = mw._apply(_make_state(tool_calls=call), runtime)
assert hard_stop is not None
# Exactly what SubagentExecutor._consume_guard_stop_reason does:
# consume_stop_reason(self.run_id), where self.run_id is the raw,
# un-normalized (possibly-None) attribute value.
assert mw.consume_stop_reason(None) == "loop_capped"
# Popped on read — a second read is None (no double-report on reuse).
assert mw.consume_stop_reason(None) is None
def test_different_calls_dont_trigger(self):
mw = LoopDetectionMiddleware(warn_threshold=2)
runtime = _make_runtime()
# Each call is different
for i in range(10):
result = mw._apply(_make_state(tool_calls=[_bash_call(f"cmd_{i}")]), runtime)
assert result is None
def test_window_sliding(self):
mw = LoopDetectionMiddleware(warn_threshold=3, window_size=5)
runtime = _make_runtime()
call = [_bash_call("ls")]
# Fill with 2 identical calls
mw._apply(_make_state(tool_calls=call), runtime)
mw._apply(_make_state(tool_calls=call), runtime)
# Push them out of the window with different calls
for i in range(5):
mw._apply(_make_state(tool_calls=[_bash_call(f"other_{i}")]), runtime)
# Now the original call should be fresh again — no warning
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is None
def test_reset_clears_state(self):
mw = LoopDetectionMiddleware(warn_threshold=2)
runtime = _make_runtime()
call = [_bash_call("ls")]
mw._apply(_make_state(tool_calls=call), runtime)
mw._apply(_make_state(tool_calls=call), runtime)
# Would trigger warning, but reset first
mw.reset()
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is None
assert not mw._pending_warnings.get(_pending_key())
def test_non_ai_message_ignored(self):
mw = LoopDetectionMiddleware()
runtime = _make_runtime()
state = {"messages": [SystemMessage(content="hello")]}
result = mw._apply(state, runtime)
assert result is None
def test_empty_messages_ignored(self):
mw = LoopDetectionMiddleware()
runtime = _make_runtime()
result = mw._apply({"messages": []}, runtime)
assert result is None
def test_thread_id_from_runtime_context(self):
"""Thread ID should come from runtime.context, not state."""
mw = LoopDetectionMiddleware(warn_threshold=2)
runtime_a = _make_runtime("thread-A")
runtime_b = _make_runtime("thread-B")
call = [_bash_call("ls")]
# One call on thread A
mw._apply(_make_state(tool_calls=call), runtime_a)
# One call on thread B
mw._apply(_make_state(tool_calls=call), runtime_b)
# Second call on thread A — queues warning under thread-A only.
mw._apply(_make_state(tool_calls=call), runtime_a)
assert mw._pending_warnings.get(_pending_key("thread-A"))
assert "LOOP DETECTED" in mw._pending_warnings[_pending_key("thread-A")][0]
assert not mw._pending_warnings.get(_pending_key("thread-B"))
# Second call on thread B — independent queue.
mw._apply(_make_state(tool_calls=call), runtime_b)
assert mw._pending_warnings.get(_pending_key("thread-B"))
assert "LOOP DETECTED" in mw._pending_warnings[_pending_key("thread-B")][0]
def test_lru_eviction(self):
"""Old threads should be evicted when max_tracked_threads is exceeded."""
mw = LoopDetectionMiddleware(warn_threshold=2, max_tracked_threads=3)
call = [_bash_call("ls")]
# Fill up 3 threads
for i in range(3):
runtime = _make_runtime(f"thread-{i}")
mw._apply(_make_state(tool_calls=call), runtime)
# Add a 4th thread — should evict thread-0
runtime_new = _make_runtime("thread-new")
mw._apply(_make_state(tool_calls=call), runtime_new)
assert _scope_key("thread-0") not in mw._history
assert _scope_key("thread-0") not in mw._tool_name_history
assert _scope_key("thread-new") in mw._history
assert len(mw._history) == 3
def test_warned_hashes_are_pruned_to_sliding_window(self):
"""A long-lived thread should not keep every historical warned hash."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=100, window_size=4)
runtime = _make_runtime()
for i in range(12):
call = [_bash_call(f"cmd_{i}")]
mw._apply(_make_state(tool_calls=call), runtime)
mw._apply(_make_state(tool_calls=call), runtime)
assert len(mw._history[_scope_key()]) <= 4
assert set(mw._warned[_scope_key()]).issubset({entry.key for entry in mw._history[_scope_key()]})
assert len(mw._warned[_scope_key()]) <= 4
def test_pending_warning_keys_are_capped(self):
"""Abnormal same-thread runs cannot grow pending-warning keys forever."""
mw = LoopDetectionMiddleware(warn_threshold=2, max_tracked_threads=2)
for i in range(10):
runtime = _make_runtime(thread_id="same-thread", run_id=f"run-{i}")
mw._queue_pending_warning(runtime, f"warning-{i}")
assert len(mw._pending_warnings) == mw._max_pending_warning_keys
assert len(mw._pending_warning_touch_order) == mw._max_pending_warning_keys
assert _pending_key("same-thread", "run-9") in mw._pending_warnings
def test_pending_warning_list_is_capped_and_deduped(self):
"""One run cannot accumulate an unbounded warning list."""
mw = LoopDetectionMiddleware()
runtime = _make_runtime()
for i in range(_MAX_PENDING_WARNINGS_PER_RUN + 4):
mw._queue_pending_warning(runtime, f"warning-{i}")
mw._queue_pending_warning(runtime, f"warning-{_MAX_PENDING_WARNINGS_PER_RUN + 3}")
warnings = mw._pending_warnings[_pending_key()]
assert len(warnings) == _MAX_PENDING_WARNINGS_PER_RUN
assert warnings == [f"warning-{i}" for i in range(4, _MAX_PENDING_WARNINGS_PER_RUN + 4)]
def test_pending_warning_touch_order_cleared_with_pending_key(self):
mw = LoopDetectionMiddleware()
runtime = _make_runtime()
mw._queue_pending_warning(runtime, "warning")
mw.after_agent({"messages": []}, runtime)
assert mw._pending_warnings == {}
assert mw._pending_warning_touch_order == OrderedDict()
def test_thread_safe_mutations(self):
"""Verify lock is used for mutations (basic structural test)."""
mw = LoopDetectionMiddleware()
# The middleware should have a lock attribute
assert hasattr(mw, "_lock")
assert isinstance(mw._lock, type(mw._lock))
def test_fallback_thread_id_when_missing(self):
"""When runtime context has no thread_id, should use 'default'."""
mw = LoopDetectionMiddleware(warn_threshold=2)
runtime = SimpleNamespace(context={}, control=object(), execution_info=None)
call = [_bash_call("ls")]
mw._apply(_make_state(tool_calls=call), runtime)
assert ("default", mw._get_run_id(runtime)) in mw._history
class TestRunScopedTracking:
def test_identical_call_history_isolated_between_runs_on_same_thread(self):
"""Separate user runs on a cached graph must not share repeat counts."""
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=3,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
first_run = _make_runtime("shared-thread", "run-1")
second_run = _make_runtime("shared-thread", "run-2")
state = _make_state(tool_calls=[_bash_call("pwd")])
assert mw._apply(state, first_run) is None
mw.after_agent({"messages": []}, first_run)
assert mw._apply(state, second_run) is None
assert not mw._pending_warnings.get(_pending_key("shared-thread", "run-2"))
assert mw._history[_scope_key("shared-thread", "run-1")] == mw._history[_scope_key("shared-thread", "run-2")]
assert len(mw._history[_scope_key("shared-thread", "run-1")]) == 1
def test_same_run_accumulates_across_agent_hook_cycles_for_goal_continuation(self):
"""One Gateway run may re-enter the graph for hidden goal continuations."""
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=10,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
runtime = _make_runtime("goal-thread", "goal-run")
state = _make_state(tool_calls=[_bash_call("pwd")])
assert mw._apply(state, runtime) is None
mw.after_agent({"messages": []}, runtime)
assert mw._apply(state, runtime) is None
assert mw._pending_warnings.get(_pending_key("goal-thread", "goal-run"))
assert len(mw._history[_scope_key("goal-thread", "goal-run")]) == 2
def test_tool_frequency_history_isolated_between_runs_on_same_thread(self):
mw = LoopDetectionMiddleware(
warn_threshold=100,
hard_limit=200,
tool_freq_warn=2,
tool_freq_hard_limit=3,
)
first_run = _make_runtime("shared-thread", "run-1")
second_run = _make_runtime("shared-thread", "run-2")
assert mw._apply(_make_state(tool_calls=[_bash_call("first")]), first_run) is None
mw.after_agent({"messages": []}, first_run)
assert mw._apply(_make_state(tool_calls=[_bash_call("second")]), second_run) is None
assert not mw._pending_warnings.get(_pending_key("shared-thread", "run-2"))
assert mw._tool_name_counter[_scope_key("shared-thread", "run-1")]["bash"] == 1
assert mw._tool_name_counter[_scope_key("shared-thread", "run-2")]["bash"] == 1
def test_warned_hash_is_run_scoped(self):
"""A warning in one run must not suppress the same warning in a later run."""
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=100,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
state = _make_state(tool_calls=[_bash_call("pwd")])
for run_id in ("run-1", "run-2"):
runtime = _make_runtime("shared-thread", run_id)
assert mw._apply(state, runtime) is None
assert mw._apply(state, runtime) is None
assert mw._pending_warnings.get(_pending_key("shared-thread", run_id))
mw.after_agent({"messages": []}, runtime)
assert len(mw._warned[_scope_key("shared-thread", "run-1")]) == 1
assert len(mw._warned[_scope_key("shared-thread", "run-2")]) == 1
def test_reset_thread_clears_every_run_scope_and_preserves_other_threads(self):
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=100,
tool_freq_warn=3,
tool_freq_hard_limit=100,
)
target_runs = [
_make_runtime("thread-A", "run-A1"),
_make_runtime("thread-A", "run-A2"),
]
other_run = _make_runtime("thread-B", "run-B1")
state = _make_state(tool_calls=[_bash_call("pwd")])
for runtime in [*target_runs, other_run]:
mw._apply(state, runtime)
mw._apply(state, runtime)
mw.reset(thread_id="thread-A")
expected_keys = {_scope_key("thread-B", "run-B1")}
for mapping in (
mw._history,
mw._warned,
mw._tool_name_history,
mw._tool_name_counter,
mw._tool_freq_reported,
mw._pending_warnings,
mw._pending_warning_touch_order,
):
assert set(mapping) == expected_keys
def test_reset_thread_clears_its_unconsumed_stop_reasons_only(self):
mw = LoopDetectionMiddleware(
warn_threshold=1,
hard_limit=2,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
runtimes = [
_make_runtime("thread-A", "run-A1"),
_make_runtime("thread-A", "run-A2"),
_make_runtime("thread-B", "run-B1"),
]
state = _make_state(tool_calls=[_bash_call("pwd")])
for runtime in runtimes:
assert mw._apply(state, runtime) is None
result = mw._apply(state, runtime)
assert result is not None
mw.reset(thread_id="thread-A")
assert mw.consume_stop_reason("run-A1") is None
assert mw.consume_stop_reason("run-A2") is None
assert mw.consume_stop_reason("run-B1") == "loop_capped"
def test_lru_evicts_one_run_scope_without_dropping_sibling_run_warning(self):
mw = LoopDetectionMiddleware(
warn_threshold=100,
hard_limit=200,
max_tracked_threads=2,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
run_1 = _make_runtime("shared-thread", "run-1")
run_2 = _make_runtime("shared-thread", "run-2")
run_3 = _make_runtime("shared-thread", "run-3")
mw._apply(_make_state(tool_calls=[_bash_call("one")]), run_1)
mw._queue_pending_warning(run_1, "run-1 warning")
mw._apply(_make_state(tool_calls=[_bash_call("two")]), run_2)
mw._queue_pending_warning(run_2, "run-2 warning")
mw._apply(_make_state(tool_calls=[_bash_call("one-again")]), run_1)
mw._apply(_make_state(tool_calls=[_bash_call("three")]), run_3)
assert list(mw._history) == [
_scope_key("shared-thread", "run-1"),
_scope_key("shared-thread", "run-3"),
]
assert _scope_key("shared-thread", "run-2") not in mw._tool_name_history
assert _scope_key("shared-thread", "run-2") not in mw._tool_name_counter
assert _pending_key("shared-thread", "run-1") in mw._pending_warnings
assert _pending_key("shared-thread", "run-2") not in mw._pending_warnings
class TestLoopDetectionRunEvents:
@staticmethod
def _runtime_with_journal(journal):
runtime = _make_runtime()
runtime.context["__run_journal"] = journal
return runtime
def test_identical_call_warning_records_once_without_arguments(self):
journal = MagicMock()
runtime = self._runtime_with_journal(journal)
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=10,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
call = [_bash_call("SUPER_SECRET_COMMAND")]
# The second identical call reaches the warning threshold.
for _ in range(2):
assert mw._apply(_make_state(tool_calls=call), runtime) is None
# A subsequent occurrence must not duplicate the warning event.
assert mw._apply(_make_state(tool_calls=call), runtime) is None
journal.record_middleware.assert_called_once()
recorded = journal.record_middleware.call_args
assert recorded.kwargs["tag"] == "loop_detection"
assert recorded.kwargs["name"] == "LoopDetectionMiddleware"
assert recorded.kwargs["hook"] == "after_model"
assert recorded.kwargs["action"] == "warn"
assert recorded.kwargs["changes"] == {
"is_subagent": False,
"agent_id": None,
"detection_layer": "identical_call_set",
"tool_names": ["bash"],
"count": 2,
"threshold": 2,
}
# Tool arguments and argument-derived values must not be persisted.
assert "SUPER_SECRET_COMMAND" not in repr(recorded)
assert "args" not in recorded.kwargs["changes"]
def test_narrow_subagent_recorder_key_records_without_shared_journal(self):
recorder = MagicMock()
runtime = _make_runtime()
runtime.context["__run_loop_detection_recorder"] = recorder
runtime.context["agent_id"] = "general-purpose"
assert "__run_journal" not in runtime.context
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=10,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
call = [_bash_call("ls")]
assert mw._apply(_make_state(tool_calls=call), runtime) is None
assert mw._apply(_make_state(tool_calls=call), runtime) is None
recorder.record_middleware.assert_called_once()
assert recorder.record_middleware.call_args.kwargs["action"] == "warn"
assert recorder.record_middleware.call_args.kwargs["changes"]["is_subagent"] is True
assert recorder.record_middleware.call_args.kwargs["changes"]["agent_id"] == "general-purpose"
def test_lead_attribution_ignores_caller_supplied_subagent_fields(self):
journal = MagicMock()
runtime = self._runtime_with_journal(journal)
runtime.context["is_subagent"] = True
runtime.context["agent_id"] = "forged-agent"
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=10,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
call = [_bash_call("ls")]
assert mw._apply(_make_state(tool_calls=call), runtime) is None
assert mw._apply(_make_state(tool_calls=call), runtime) is None
changes = journal.record_middleware.call_args.kwargs["changes"]
assert changes["is_subagent"] is False
assert changes["agent_id"] is None
def test_identical_call_hard_stop_records_event(self):
journal = MagicMock()
runtime = self._runtime_with_journal(journal)
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=3,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
call = [_bash_call("ls")]
for _ in range(2):
assert mw._apply(_make_state(tool_calls=call), runtime) is None
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is not None
assert result["messages"][0].tool_calls == []
# One warning transition followed by one hard-stop transition.
assert journal.record_middleware.call_count == 2
recorded = journal.record_middleware.call_args_list[-1]
assert recorded.kwargs["tag"] == "loop_detection"
assert recorded.kwargs["action"] == "hard_stop"
assert recorded.kwargs["changes"] == {
"is_subagent": False,
"agent_id": None,
"detection_layer": "identical_call_set",
"tool_names": ["bash"],
"count": 3,
"threshold": 3,
}
def test_journal_failure_warns_without_breaking_detection(self, caplog):
journal = MagicMock()
journal.record_middleware.side_effect = RuntimeError("db down")
runtime = self._runtime_with_journal(journal)
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=10,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
call = [_bash_call("ls")]
assert mw._apply(_make_state(tool_calls=call), runtime) is None
with caplog.at_level("WARNING"):
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is None
assert mw._pending_warnings[_pending_key()]
assert "Failed to record middleware:loop_detection event" in caplog.text
@pytest.mark.asyncio
async def test_async_after_model_records_warning_event(self):
journal = MagicMock()
runtime = self._runtime_with_journal(journal)
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=10,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
call = [_bash_call("ls")]
assert await mw.aafter_model(_make_state(tool_calls=call), runtime) is None
assert await mw.aafter_model(_make_state(tool_calls=call), runtime) is None
journal.record_middleware.assert_called_once()
assert journal.record_middleware.call_args.kwargs["action"] == "warn"
class TestLoopDetectionAgentGraphIntegration:
def test_reused_agent_graph_without_run_id_gets_one_budget_per_invocation(self):
"""Library embedders may omit run_id while reusing one compiled graph."""
@as_tool
def bash(command: str) -> str:
"""Run a fake shell command."""
return f"ran: {command}"
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=3,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
model = _CapturingFakeMessagesListChatModel(
responses=[
AIMessage(content="", tool_calls=[{"name": "bash", "id": "run-1-a", "args": {"command": "pwd"}}]),
AIMessage(content="", tool_calls=[{"name": "bash", "id": "run-1-b", "args": {"command": "pwd"}}]),
AIMessage(content="first final answer"),
AIMessage(content="", tool_calls=[{"name": "bash", "id": "run-2-a", "args": {"command": "pwd"}}]),
AIMessage(content="", tool_calls=[{"name": "bash", "id": "run-2-b", "args": {"command": "pwd"}}]),
AIMessage(content="second final answer"),
],
)
graph = create_agent(model=model, tools=[bash], middleware=[mw])
shared_context = {"thread_id": "library-thread"}
first = graph.invoke(
{"messages": [("user", "where am I?")]},
context=shared_context,
config={"recursion_limit": 15},
)
second = graph.invoke(
{"messages": [("user", "where am I now?")]},
context=shared_context,
config={"recursion_limit": 15},
)
assert first["messages"][-1].content == "first final answer"
assert second["messages"][-1].content == "second final answer"
loop_warnings_by_call = [[message for message in messages if isinstance(message, HumanMessage) and message.name == "loop_warning"] for messages in model.seen_messages]
assert [len(warnings) for warnings in loop_warnings_by_call] == [0, 0, 1, 0, 0, 1]
assert not mw._fallback_run_ids
def test_reused_agent_graph_isolates_loop_history_between_runs(self):
"""A cached graph must give each new run a fresh loop-detection budget."""
@as_tool
def bash(command: str) -> str:
"""Run a fake shell command."""
return f"ran: {command}"
mw = LoopDetectionMiddleware(
warn_threshold=2,
hard_limit=3,
tool_freq_warn=100,
tool_freq_hard_limit=200,
)
model = _CapturingFakeMessagesListChatModel(
responses=[
AIMessage(content="", tool_calls=[{"name": "bash", "id": "run-1-call", "args": {"command": "pwd"}}]),
AIMessage(content="first final answer"),
AIMessage(content="", tool_calls=[{"name": "bash", "id": "run-2-call", "args": {"command": "pwd"}}]),
AIMessage(content="second final answer"),
],
)
graph = create_agent(model=model, tools=[bash], middleware=[mw])
first = graph.invoke(
{"messages": [("user", "where am I?")]},
context={"thread_id": "cached-thread", "run_id": "run-1"},
config={"recursion_limit": 10},
)
second = graph.invoke(
{"messages": [("user", "where am I now?")]},
context={"thread_id": "cached-thread", "run_id": "run-2"},
config={"recursion_limit": 10},
)
assert first["messages"][-1].content == "first final answer"
assert second["messages"][-1].content == "second final answer"
assert len(model.seen_messages) == 4
assert not any(isinstance(message, HumanMessage) and message.name == "loop_warning" for request_messages in model.seen_messages for message in request_messages)
assert set(mw._history) == {
_scope_key("cached-thread", "run-1"),
_scope_key("cached-thread", "run-2"),
}
def test_loop_warning_survives_a_retried_model_call_in_real_agent_graph(self):
"""LLMErrorHandlingMiddleware retries a failed call by running the inner wraps again; the retry must still carry the warning."""
from deerflow.agents.middlewares.llm_error_handling_middleware import LLMErrorHandlingMiddleware
from deerflow.config.app_config import AppConfig, LlmCallConfig
from deerflow.config.sandbox_config import SandboxConfig
class ProviderUnavailable(Exception):
def __init__(self) -> None:
super().__init__("503 Service Unavailable")
self.status_code = 503
self.response = SimpleNamespace(status_code=503, headers={})
class FailsOnceOnWarning(_CapturingFakeMessagesListChatModel):
_failed: bool = PrivateAttr(default=False)
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
if not self._failed and any(isinstance(message, HumanMessage) and message.name == "loop_warning" for message in messages):
self._failed = True
self._seen_messages.append(list(messages))
raise ProviderUnavailable()
return super()._generate(messages, stop=stop, run_manager=run_manager, **kwargs)
@as_tool
def bash(command: str) -> str:
"""Run a fake shell command."""
return f"ran: {command}"
repeated_calls = [[{"name": "bash", "id": f"call_ls_{i}", "args": {"command": "ls"}}] for i in range(3)]
model = FailsOnceOnWarning(
responses=[
AIMessage(content="", tool_calls=repeated_calls[0]),
AIMessage(content="", tool_calls=repeated_calls[1]),
AIMessage(content="", tool_calls=repeated_calls[2]),
AIMessage(content="final answer"),
],
)
app_config = AppConfig(
sandbox=SandboxConfig(use="test"),
llm_call=LlmCallConfig(retry_max_attempts=3, retry_base_delay_ms=0, retry_cap_delay_ms=0),
)
graph = create_agent(
model=model,
tools=[bash],
middleware=[LLMErrorHandlingMiddleware(app_config=app_config), LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)],
)
result = graph.invoke(
{"messages": [("user", "inspect the directory")]},
context={"thread_id": "retry-thread", "run_id": "retry-run"},
config={"recursion_limit": 20},
)
# Three tool-calling requests, then the failed attempt and its retry.
assert len(model.seen_messages) == 5
has_warning = [any(isinstance(message, HumanMessage) and message.name == "loop_warning" for message in messages) for messages in model.seen_messages]
assert has_warning == [False, False, False, True, True]
assert result["messages"][-1].content == "final answer"
def test_loop_warning_is_transient_in_real_agent_graph(self):
"""after_model queues the warning; wrap_model_call injects it request-only."""
@as_tool
def bash(command: str) -> str:
"""Run a fake shell command."""
return f"ran: {command}"
repeated_calls = [[{"name": "bash", "id": f"call_ls_{i}", "args": {"command": "ls"}}] for i in range(3)]
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
model = _CapturingFakeMessagesListChatModel(
responses=[
AIMessage(content="", tool_calls=repeated_calls[0]),
AIMessage(content="", tool_calls=repeated_calls[1]),
AIMessage(content="", tool_calls=repeated_calls[2]),
AIMessage(content="final answer"),
],
)
graph = create_agent(model=model, tools=[bash], middleware=[mw])
result = graph.invoke(
{"messages": [("user", "inspect the directory")]},
context={"thread_id": "integration-thread", "run_id": "integration-run"},
config={"recursion_limit": 20},
)
assert len(model.seen_messages) == 4
loop_warnings_by_call = [[message for message in messages if isinstance(message, HumanMessage) and message.name == "loop_warning"] for messages in model.seen_messages]
assert loop_warnings_by_call[0] == []
assert loop_warnings_by_call[1] == []
assert loop_warnings_by_call[2] == []
assert len(loop_warnings_by_call[3]) == 1
assert "LOOP DETECTED" in loop_warnings_by_call[3][0].content
fourth_request = model.seen_messages[3]
assert isinstance(fourth_request[-2], ToolMessage)
assert fourth_request[-2].tool_call_id == "call_ls_2"
assert fourth_request[-1] is loop_warnings_by_call[3][0]
persisted_loop_warnings = [message for message in result["messages"] if isinstance(message, HumanMessage) and message.name == "loop_warning"]
assert persisted_loop_warnings == []
assert result["messages"][-1].content == "final answer"
assert mw._pending_warnings == {}
assert mw._pending_warning_touch_order == OrderedDict()
@pytest.mark.asyncio
async def test_loop_warning_is_transient_in_async_agent_graph(self):
"""awrap_model_call injects loop_warning request-only in async graph runs."""
@as_tool
async def bash(command: str) -> str:
"""Run a fake shell command."""
return f"ran: {command}"
repeated_calls = [[{"name": "bash", "id": f"call_async_ls_{i}", "args": {"command": "ls"}}] for i in range(3)]
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
model = _CapturingFakeMessagesListChatModel(
responses=[
AIMessage(content="", tool_calls=repeated_calls[0]),
AIMessage(content="", tool_calls=repeated_calls[1]),
AIMessage(content="", tool_calls=repeated_calls[2]),
AIMessage(content="async final answer"),
],
)
graph = create_agent(model=model, tools=[bash], middleware=[mw])
result = await graph.ainvoke(
{"messages": [("user", "inspect the directory asynchronously")]},
context={"thread_id": "async-integration-thread", "run_id": "async-integration-run"},
config={"recursion_limit": 20},
)
assert len(model.seen_messages) == 4
loop_warnings_by_call = [[message for message in messages if isinstance(message, HumanMessage) and message.name == "loop_warning"] for messages in model.seen_messages]
assert loop_warnings_by_call[0] == []
assert loop_warnings_by_call[1] == []
assert loop_warnings_by_call[2] == []
assert len(loop_warnings_by_call[3]) == 1
assert "LOOP DETECTED" in loop_warnings_by_call[3][0].content
fourth_request = model.seen_messages[3]
assert isinstance(fourth_request[-2], ToolMessage)
assert fourth_request[-2].tool_call_id == "call_async_ls_2"
assert fourth_request[-1] is loop_warnings_by_call[3][0]
persisted_loop_warnings = [message for message in result["messages"] if isinstance(message, HumanMessage) and message.name == "loop_warning"]
assert persisted_loop_warnings == []
assert result["messages"][-1].content == "async final answer"
assert mw._pending_warnings == {}
assert mw._pending_warning_touch_order == OrderedDict()
class TestAppendText:
"""Unit tests for LoopDetectionMiddleware._append_text."""
def test_none_content_returns_text(self):
result = LoopDetectionMiddleware._append_text(None, "hello")
assert result == "hello"
def test_str_content_concatenates(self):
result = LoopDetectionMiddleware._append_text("existing", "appended")
assert result == "existing\n\nappended"
def test_empty_str_content_concatenates(self):
result = LoopDetectionMiddleware._append_text("", "appended")
assert result == "\n\nappended"
def test_list_content_appends_text_block(self):
"""List content (e.g. Anthropic thinking mode) should get a new text block."""
content = [
{"type": "thinking", "text": "Let me think..."},
{"type": "text", "text": "Here is my answer"},
]
result = LoopDetectionMiddleware._append_text(content, "stop msg")
assert isinstance(result, list)
assert len(result) == 3
assert result[0] == content[0]
assert result[1] == content[1]
assert result[2] == {"type": "text", "text": "\n\nstop msg"}
def test_empty_list_content_appends_text_block(self):
result = LoopDetectionMiddleware._append_text([], "stop msg")
assert isinstance(result, list)
assert len(result) == 1
assert result[0] == {"type": "text", "text": "\n\nstop msg"}
def test_unexpected_type_coerced_to_str(self):
"""Unexpected content types should be coerced to str as a fallback."""
result = LoopDetectionMiddleware._append_text(42, "stop msg")
assert isinstance(result, str)
assert result == "42\n\nstop msg"
def test_list_content_not_mutated_in_place(self):
"""_append_text must not modify the original list."""
original = [{"type": "text", "text": "hello"}]
result = LoopDetectionMiddleware._append_text(original, "appended")
assert len(original) == 1 # original unchanged
assert len(result) == 2 # new list has the appended block
class TestHardStopWithListContent:
"""Regression tests: hard stop must not crash when AIMessage.content is a list."""
def test_hard_stop_with_list_content(self):
"""Hard stop on list content should not raise TypeError (regression)."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = _make_runtime()
call = [_bash_call("ls")]
# Build state with list content (e.g. Anthropic thinking mode)
list_content = [
{"type": "thinking", "text": "Let me think..."},
{"type": "text", "text": "I'll run ls"},
]
for _ in range(3):
mw._apply(_make_state(tool_calls=call, content=list_content), runtime)
# Fourth call triggers hard stop — must not raise TypeError
result = mw._apply(_make_state(tool_calls=call, content=list_content), runtime)
assert result is not None
msg = result["messages"][0]
assert isinstance(msg, AIMessage)
assert msg.tool_calls == []
# Content should remain a list with the stop message appended
assert isinstance(msg.content, list)
assert len(msg.content) == 3
assert msg.content[2]["type"] == "text"
assert _HARD_STOP_MSG in msg.content[2]["text"]
def test_hard_stop_drops_provider_tool_use_blocks(self):
"""A stripped call's Anthropic tool_use block must not outlive it in content."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = _make_runtime()
call = [_bash_call("ls")]
list_content = [
{"type": "text", "text": "I'll run ls"},
{"type": "tool_use", "id": "call_ls", "name": "bash", "input": {"command": "ls"}},
]
for _ in range(3):
mw._apply(_make_state(tool_calls=call, content=list_content), runtime)
result = mw._apply(_make_state(tool_calls=call, content=list_content), runtime)
msg = result["messages"][0]
assert [block["type"] for block in msg.content] == ["text", "text"]
assert _HARD_STOP_MSG in msg.content[-1]["text"]
def test_hard_stop_with_none_content(self):
"""Hard stop on None content should produce a plain string."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = _make_runtime()
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call), runtime)
# Fourth call with default empty-string content
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is not None
msg = result["messages"][0]
assert isinstance(msg.content, str)
assert _HARD_STOP_MSG in msg.content
def test_hard_stop_with_str_content(self):
"""Hard stop on str content should concatenate the stop message."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = _make_runtime()
call = [_bash_call("ls")]
for _ in range(3):
mw._apply(_make_state(tool_calls=call, content="thinking..."), runtime)
result = mw._apply(_make_state(tool_calls=call, content="thinking..."), runtime)
assert result is not None
msg = result["messages"][0]
assert isinstance(msg.content, str)
assert msg.content.startswith("thinking...")
assert _HARD_STOP_MSG in msg.content
def test_hard_stop_clears_raw_tool_call_metadata(self):
"""Forced-stop messages must not retain provider-level raw tool-call payloads."""
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=4)
runtime = _make_runtime()
call = [_bash_call("ls")]
def _make_provider_state():
return {
"messages": [
AIMessage(
content="thinking...",
tool_calls=call,
additional_kwargs={
"tool_calls": [
{
"id": "call_ls",
"type": "function",
"function": {"name": "bash", "arguments": '{"command":"ls"}'},
"thought_signature": "sig-1",
}
],
"function_call": {"name": "bash", "arguments": '{"command":"ls"}'},
},
response_metadata={"finish_reason": "tool_calls"},
)
]
}
for _ in range(3):
mw._apply(_make_provider_state(), runtime)
result = mw._apply(_make_provider_state(), runtime)
assert result is not None
msg = result["messages"][0]
assert msg.tool_calls == []
assert "tool_calls" not in msg.additional_kwargs
assert "function_call" not in msg.additional_kwargs
assert msg.response_metadata["finish_reason"] == "stop"
class TestToolFrequencyDetection:
"""Tool-name frequency is an observability metric, never a loop decision."""
def _read_call(self, path):
return {"name": "read_file", "id": f"call_read_{path}", "args": {"path": path}}
def test_thresholds_are_observed_without_warning_or_stop(self, caplog):
mw = LoopDetectionMiddleware(tool_freq_warn=3, tool_freq_hard_limit=5)
runtime = _make_runtime()
with caplog.at_level("INFO"):
for i in range(6):
result = mw._apply(
_make_state(tool_calls=[self._read_call(f"/file_{i}.py")]),
runtime,
)
assert result is None
assert not mw._pending_warnings
assert not mw._stop_reason
assert mw._tool_name_counter[_scope_key()]["read_file"] == 6
records = [record for record in caplog.records if record.message == "Tool frequency metric reached"]
assert [(record.metric, record.threshold, record.action) for record in records] == [
("warn", 3, "observe_only"),
("hard_limit", 5, "observe_only"),
]
def test_window_decay_rearms_each_metric(self, caplog):
mw = LoopDetectionMiddleware(
tool_freq_warn=2,
tool_freq_hard_limit=3,
window_size=4,
)
runtime = _make_runtime()
with caplog.at_level("INFO"):
sequence = [
("read_file", 0),
("read_file", 1),
("read_file", 2),
("bash", 0),
("bash", 1),
("bash", 2),
("bash", 3),
("read_file", 3),
("read_file", 4),
("read_file", 5),
]
for name, value in sequence:
call = self._read_call(f"/{value}.py") if name == "read_file" else _bash_call(str(value))
assert mw._apply(_make_state(tool_calls=[call]), runtime) is None
records = [record for record in caplog.records if record.message == "Tool frequency metric reached"]
assert [(record.tool_name, record.metric) for record in records] == [
("read_file", "warn"),
("read_file", "hard_limit"),
("bash", "warn"),
("bash", "hard_limit"),
("read_file", "warn"),
("read_file", "hard_limit"),
]
def test_different_tools_and_runs_are_isolated(self):
mw = LoopDetectionMiddleware(tool_freq_warn=3, tool_freq_hard_limit=5)
runtime_a = _make_runtime("thread-A", "run-A")
runtime_b = _make_runtime("thread-B", "run-B")
for i in range(2):
assert mw._apply(_make_state(tool_calls=[self._read_call(f"/a_{i}.py")]), runtime_a) is None
assert mw._apply(_make_state(tool_calls=[_bash_call(f"b_{i}")]), runtime_b) is None
assert mw._tool_name_counter[_scope_key("thread-A", "run-A")]["read_file"] == 2
assert mw._tool_name_counter[_scope_key("thread-B", "run-B")]["bash"] == 2
assert _scope_key("thread-B", "run-B") not in mw._tool_name_counter or "read_file" not in mw._tool_name_counter[_scope_key("thread-B", "run-B")]
def test_reset_and_lru_eviction_clear_metric_state(self):
mw = LoopDetectionMiddleware(tool_freq_warn=2, tool_freq_hard_limit=3, max_tracked_threads=2)
evicted = _make_runtime("evicted", "run")
for i in range(2):
mw._apply(_make_state(tool_calls=[self._read_call(f"/old_{i}.py")]), evicted)
mw._apply(_make_state(tool_calls=[_bash_call("a")]), _make_runtime("a", "run"))
mw._apply(_make_state(tool_calls=[_bash_call("b")]), _make_runtime("b", "run"))
assert _scope_key("evicted", "run") not in mw._tool_freq_reported
mw.reset(thread_id="a")
assert _scope_key("a", "run") not in mw._tool_freq_reported
mw.reset()
assert not mw._tool_freq_reported
def test_overrides_only_change_observation_thresholds(self, caplog):
mw = LoopDetectionMiddleware(
tool_freq_warn=2,
tool_freq_hard_limit=3,
tool_freq_overrides={"bash": (4, 6)},
)
runtime = _make_runtime()
with caplog.at_level("INFO"):
for i in range(6):
assert mw._apply(_make_state(tool_calls=[_bash_call(f"cmd-{i}")]), runtime) is None
records = [r for r in caplog.records if r.message == "Tool frequency metric reached"]
assert [(r.metric, r.threshold) for r in records] == [("warn", 4), ("hard_limit", 6)]
class TestToolCallBatchDecisions:
"""Frequency accounting must not mask identical-call detection."""
@staticmethod
def _call(name, value):
if name == "read_file":
args = {"path": f"/{value}.py"}
elif name == "bash":
args = {"command": f"echo {value}"}
else:
args = {"value": value}
return {"name": name, "id": f"call_{name}_{value}", "args": args}
def test_batch_counts_all_tool_names_without_creating_decision(self, caplog):
mw = LoopDetectionMiddleware(tool_freq_warn=2, tool_freq_hard_limit=3)
runtime = _make_runtime()
calls = [self._call("read_file", i) for i in range(3)] + [self._call("bash", i) for i in range(2)]
with caplog.at_level("INFO"):
decision = mw._track_and_check(_make_state(tool_calls=calls), runtime)
assert decision is None
assert list(mw._tool_name_history[_scope_key()]) == ["read_file"] * 3 + ["bash"] * 2
assert dict(mw._tool_name_counter[_scope_key()]) == {"read_file": 3, "bash": 2}
assert not mw._pending_warnings
assert {record.tool_name for record in caplog.records if record.message == "Tool frequency metric reached"} == {"read_file", "bash"}
def test_identical_call_detection_still_hard_stops(self):
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=3, tool_freq_warn=1, tool_freq_hard_limit=2)
runtime = _make_runtime()
calls = [self._call("read_file", 0)]
mw._apply(_make_state(tool_calls=calls), runtime)
mw._apply(_make_state(tool_calls=calls), runtime)
result = mw._apply(_make_state(tool_calls=calls), runtime)
assert result is not None
assert mw.consume_stop_reason("test-run") == "loop_capped"
def test_hash_warning_preserves_frequency_metrics(self, caplog):
mw = LoopDetectionMiddleware(warn_threshold=2, hard_limit=10, tool_freq_warn=2, tool_freq_hard_limit=5)
runtime = _make_runtime()
calls = [self._call("read_file", 0)]
assert mw._track_and_check(_make_state(tool_calls=calls), runtime) is None
with caplog.at_level("INFO"):
warning = mw._track_and_check(_make_state(tool_calls=calls), runtime)
assert warning is not None and warning.detection_layer == "identical_call_set"
assert mw._tool_name_counter[_scope_key()]["read_file"] == 2
assert ("read_file", "warn") in mw._tool_freq_reported[_scope_key()]
metric_records = [record for record in caplog.records if record.message == "Tool frequency metric reached"]
assert [(record.tool_name, record.metric, record.threshold) for record in metric_records] == [
("read_file", "warn", 2),
]
class TestFromConfig:
"""Tests for LoopDetectionMiddleware.from_config — the sole validated construction path."""
@staticmethod
def _config(**kwargs):
from deerflow.config.loop_detection_config import LoopDetectionConfig
return LoopDetectionConfig(**kwargs)
def test_scalar_fields_mapped(self):
config = self._config(
warn_threshold=4,
hard_limit=8,
window_size=15,
max_tracked_threads=50,
tool_freq_warn=20,
tool_freq_hard_limit=40,
)
mw = LoopDetectionMiddleware.from_config(config)
assert mw.warn_threshold == 4
assert mw.hard_limit == 8
assert mw.window_size == 15
assert mw.max_tracked_threads == 50
assert mw.tool_freq_warn == 20
assert mw.tool_freq_hard_limit == 40
def test_overrides_converted_to_tuples(self):
config = self._config(tool_freq_overrides={"bash": {"warn": 50, "hard_limit": 100}})
mw = LoopDetectionMiddleware.from_config(config)
assert mw._tool_freq_overrides == {"bash": (50, 100)}
def test_empty_overrides(self):
mw = LoopDetectionMiddleware.from_config(self._config())
assert mw._tool_freq_overrides == {}
def test_constructed_middleware_queues_loop_warning(self):
mw = LoopDetectionMiddleware.from_config(self._config(warn_threshold=2, hard_limit=4))
runtime = _make_runtime()
call = [_bash_call("ls")]
mw._apply(_make_state(tool_calls=call), runtime)
result = mw._apply(_make_state(tool_calls=call), runtime)
assert result is None
queued = mw._pending_warnings.get(_pending_key(), [])
assert queued
assert "LOOP DETECTED" in queued[0]
def test_freq_window_sized_to_hard_limit_under_defaults(self):
"""Regression for #4072: the Layer-2 frequency window must be >= the
largest threshold, or the warn/hard branches are dead code. With the
shipped defaults (window 20 < warn 30 < hard 50) the freq deque must be
sized to 50, not 20."""
mw = LoopDetectionMiddleware.from_config(self._config())
assert mw._tool_freq_window >= mw.tool_freq_hard_limit
assert mw._tool_freq_window >= mw.tool_freq_warn
def test_freq_window_covers_largest_override_hard_limit(self):
mw = LoopDetectionMiddleware.from_config(self._config(tool_freq_overrides={"bash": {"warn": 60, "hard_limit": 120}}))
assert mw._tool_freq_window >= 120
def test_tight_burst_records_metric_under_default_config(self, caplog):
"""The default frequency threshold is observable and never stops work."""
mw = LoopDetectionMiddleware.from_config(self._config())
runtime = _make_runtime()
hard = mw.tool_freq_hard_limit # 50 by default
# Distinct args every call -> unique hashes -> Layer 1 (hash) never trips.
with caplog.at_level("INFO"):
for i in range(hard):
result = mw._apply(_make_state(tool_calls=[_bash_call(f"cmd_{i}")]), runtime)
assert result is None, f"frequency metric unexpectedly changed control flow at call {i}"
records = [record for record in caplog.records if record.message == "Tool frequency metric reached"]
assert [(record.metric, record.threshold, record.action) for record in records] == [
("warn", mw.tool_freq_warn, "observe_only"),
("hard_limit", hard, "observe_only"),
]
assert mw.consume_stop_reason("test-run") is None
class TestWarningLetsAgentSaveResultsFirst:
"""F67: a warn-level loop message must not make a subagent drop the
file it was asked to write. The warning asks for one last save through a
file-writing tool before the final answer."""
@staticmethod
def _write_call(path="/mnt/user-data/workspace/out.md"):
return {"name": "write_file", "id": "call_write", "args": {"path": path, "content": "partial results"}}
@staticmethod
def _read_call(path):
return {"name": "read_file", "id": f"call_read_{path}", "args": {"path": path}}
def test_hash_warning_asks_to_save_before_final_answer(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime = _make_runtime()
for _ in range(3):
mw._apply(_make_state(tool_calls=[_bash_call("ls")]), runtime)
warning = mw._pending_warnings[_pending_key()][0]
assert "LOOP DETECTED" in warning
assert "Stop calling tools and produce your final answer now" not in warning
assert "write_file" in warning
assert warning.index("write what you have collected") < warning.index("produce your final answer")
def test_save_call_after_hash_warning_is_not_stripped(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10)
runtime = _make_runtime()
for _ in range(3):
mw._apply(_make_state(tool_calls=[_bash_call("ls")]), runtime)
assert mw._pending_warnings[_pending_key()]
# The follow-up write is a different tool and call set, so it reaches the tools node.
assert mw._apply(_make_state(tool_calls=[self._write_call()]), runtime) is None
def test_warning_allows_a_bounded_read_before_write_recovery(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=20)
runtime = _make_runtime()
for i in range(3):
mw._apply(_make_state(tool_calls=[_bash_call("ls")]), runtime)
warning = mw._pending_warnings[_pending_key()][0]
assert "read-before-write" in warning
assert "read the target file" in warning
assert "retry the save once" in warning
assert warning.index("retry the save once") < warning.index("produce your final answer")
def test_looping_writer_requires_an_alternative_or_an_unsaved_result(self):
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=20)
runtime = _make_runtime()
for i in range(3):
mw._apply(_make_state(tool_calls=[self._write_call("/out.md")]), runtime)
warning = mw._pending_warnings[_pending_key()][0]
assert "unless it is the looping tool" in warning
assert "different available file-writing tool" in warning
assert "Do not use a tool that this warning tells you to stop" in warning
assert "include the collected results in your final answer" in warning
assert "file was not saved" in warning
def test_warning_asks_to_present_created_outputs_before_answering(self):
# Told only to stop calling tools, agents skipped present_files for
# outputs they had already created, so the run failed its delivery check.
mw = LoopDetectionMiddleware(warn_threshold=3, hard_limit=20)
runtime = _make_runtime()
for i in range(3):
mw._apply(_make_state(tool_calls=[_bash_call("ls")]), runtime)
warning = mw._pending_warnings[_pending_key()][0]
assert "If present_files is available" in warning
assert "already created in /mnt/user-data/outputs" in warning
present = warning.index("present_files call before answering, unless it is the looping tool")
assert warning.index("retry the save once") < present < warning.index("produce your final answer")
@pytest.mark.asyncio
async def test_warned_agent_can_recover_an_existing_file_save_through_the_real_gate(self, tmp_path):
from deerflow.agents.middlewares.read_before_write_middleware import ReadBeforeWriteMiddleware
output = tmp_path / "out.md"
output.write_text("previous attempt", encoding="utf-8")
executed = []
@as_tool
def bash(command: str) -> str:
"""Return a deterministic research result without running a shell."""
return "partial results"
@as_tool
def read_file(path: str) -> str:
"""Read the output from the previous attempt."""
content = Path(path).read_text(encoding="utf-8")
executed.append(("read", content))
return content
@as_tool
def write_file(path: str, content: str) -> str:
"""Save the collected results."""
Path(path).write_text(content, encoding="utf-8")
executed.append(("write", content))
return "saved"
loop = LoopDetectionMiddleware(warn_threshold=3, hard_limit=10, tool_freq_warn=10, tool_freq_hard_limit=20)
gate = ReadBeforeWriteMiddleware(content_reader=lambda _runtime, path: Path(path).read_text(encoding="utf-8"))
calls = [dict(_bash_call("ls"), id=f"research_{i}") for i in range(3)]
calls.extend([dict(self._write_call(str(output)), id="blocked_save"), self._read_call(str(output)), dict(self._write_call(str(output)), id="final_save")])
model = _CapturingFakeMessagesListChatModel(responses=[*[AIMessage(content="", tool_calls=[call]) for call in calls], AIMessage(content="Saved partial results.")])
graph = create_agent(model=model, tools=[bash, read_file, write_file], middleware=[gate, loop])
result = await graph.ainvoke({"messages": [("user", f"Save your research to {output}.")]}, context={"thread_id": "save-thread", "run_id": "save-run"}, config={"recursion_limit": 30})
warnings = [message for message in model.seen_messages[3] if isinstance(message, HumanMessage) and message.name == "loop_warning"]
assert len(warnings) == 1 and "retry the save once" in warnings[0].content
blocked = next(message for message in result["messages"] if isinstance(message, ToolMessage) and message.tool_call_id == "blocked_save")
assert blocked.status == "error" and "deerflow_write_block" in blocked.additional_kwargs
read = next(message for message in result["messages"] if isinstance(message, ToolMessage) and message.name == "read_file")
assert read.additional_kwargs["deerflow_read_mark"]["path"] == str(output)
assert executed == [("read", "previous attempt"), ("write", "partial results")]
assert output.read_text(encoding="utf-8") == "partial results"
assert result["messages"][-1].content == "Saved partial results."
assert loop.consume_stop_reason("save-run") is None
@pytest.mark.asyncio
async def test_warned_agent_presents_created_outputs_before_answering(self):
presented = []
@as_tool
def bash(command: str) -> str:
"""Build an output file without running a shell."""
return "built"
@as_tool
def present_files(filepaths: list[str]) -> str:
"""Record the files presented to the user."""
presented.extend(filepaths)
return "presented"
loop = LoopDetectionMiddleware(warn_threshold=3, hard_limit=20, tool_freq_warn=10, tool_freq_hard_limit=20)
calls = [dict(_bash_call("python build.py"), id=f"build_{i}") for i in range(3)]
calls.append({"name": "present_files", "id": "present", "args": {"filepaths": ["/mnt/user-data/outputs/report.docx"]}})
model = _CapturingFakeMessagesListChatModel(responses=[*[AIMessage(content="", tool_calls=[call]) for call in calls], AIMessage(content="The report is ready.")])
graph = create_agent(model=model, tools=[bash, present_files], middleware=[loop])
result = await graph.ainvoke({"messages": [("user", "Build the report.")]}, context={"thread_id": "present-thread", "run_id": "present-run"}, config={"recursion_limit": 30})
warnings = [message for message in model.seen_messages[3] if isinstance(message, HumanMessage) and message.name == "loop_warning"]
assert len(warnings) == 1 and "present_files call before answering" in warnings[0].content
assert presented == ["/mnt/user-data/outputs/report.docx"]
assert result["messages"][-1].content == "The report is ready."
assert loop.consume_stop_reason("present-run") is None
@pytest.mark.parametrize(
"name,expected",
[
("_HARD_STOP_MSG", "[FORCED STOP] Repeated tool calls exceeded the safety limit. Producing final answer with results collected so far."),
],
)
def test_hard_stop_message_unchanged(self, name, expected):
message = getattr(loop_detection_module, name)
assert message == expected
assert loop_detection_module._WRAP_UP_INSTRUCTIONS not in message