fix(agents): tolerate deeply nested JSON in tool-call normalization (#7742)

Loop detection, clarification and dangling-tool-call repair parse model-provided
tool arguments and options. A JSON string, Python structure or native option
label nested past the interpreter recursion limit raised RecursionError and
aborted the run; these paths now treat it like any other unusable payload
(opaque args, an empty replay object, or a skipped option), and the dict-option
flatten and _stable_tool_key use guarded serialization.

Rebased onto current main; the stale-base docs commit is superseded by main and
the AGENTS note is trimmed to stay inside the guidance chain budget.

README.md

@@ -3012,6 +3012,7 @@ Each task gets its own execution environment with a full filesystem view — ski
 The read-before-write gate ties each read mark to that `read_file` call's result, including custom tools returning multi-message `Command` updates. An unrelated result cannot authorize a write after a failed read or hide a successful read.
 
 Deeply nested JSON returned by tools, including `read_file`, keeps its original content when result analysis reaches the JSON parser's depth limit. Explicit tool failures remain errors with an unknown category when depth prevents analysis.
+Tool-call arguments and clarification options nested past the JSON parser's recursion limit are treated as unusable opaque values instead of aborting the run.
 
 On Linux and macOS, local sandbox `write_file` replacements and `str_replace` stage complete text before replacing the destination, so write, flush or publication errors retain the previous file. Existing write access is checked; ownership, permission bits, ACLs and extended attributes are transferred before writing staging content. Failure to transfer metadata aborts the save. Successful revisions get a fresh modification time, including empty saves; source-descriptor cleanup errors are warnings and do not change the save result. Files with set-id bits, macOS file flags or content-bound Linux privilege attributes are refused unchanged. Replacement requires parent-directory write access and publishes a new inode; other hard links retain the previous version. Symlinks follow the existing path-validation rules. Append writes, other host platforms (including Windows), shell writes and remote providers keep their existing behavior. This is not a concurrent-writer or power-loss recovery guarantee.
 

backend/packages/harness/deerflow/agents/middlewares/AGENTS.md

@@ -8,6 +8,8 @@ depth regressions in `tests/test_tool_result_meta_json_depth.py`, including real
 LocalSandbox reads through both synchronous and asynchronous agent graphs.
 Deep graph cases must establish a real parser depth failure or report a skip.
 
+Tool-call argument depth handling is best-effort; see `tests/test_tool_call_json_depth.py`.
+
 Async gate waiters own separate cross-loop futures. Cancel only their signal;
 notify outside the state lock. `_await_off_thread` drains started I/O across
 repeated cancellation.

backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py

@@ -143,7 +143,7 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
         if isinstance(options, str):
             try:
                 options = json.loads(options)
-            except (json.JSONDecodeError, TypeError):
+            except (json.JSONDecodeError, TypeError, RecursionError):
                 options = [options]
 
         if options is None:
@@ -159,7 +159,11 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
         normalized: list[str] = []
         seen: set[str] = set()
         for option in options:
-            text = _XML_TAG_RE.sub("", str(option)).strip()
+            try:
+                text = _XML_TAG_RE.sub("", str(option)).strip()
+            except RecursionError:
+                # A native option nested past the limit cannot become a label.
+                continue
             if not text or text in seen:
                 continue
             seen.add(text)
@@ -170,18 +174,17 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
     def _flatten_dict_option_values(value: dict[str, Any]) -> list[str | int | float]:
         """Flatten scalar leaves from XML-to-dict option payloads in source order."""
         flattened: list[str | int | float] = []
-
-        def collect(nested: Any) -> None:
-            if isinstance(nested, dict):
-                for item in nested.values():
-                    collect(item)
-            elif isinstance(nested, list):
-                for item in nested:
-                    collect(item)
-            elif isinstance(nested, str | int | float):
-                flattened.append(nested)
-
-        collect(value)
+        # Explicit stack instead of recursion: XML-to-dict payloads are
+        # model-controlled and can nest past the interpreter's limit.
+        stack: list[Any] = [value]
+        while stack:
+            node = stack.pop()
+            if isinstance(node, dict):
+                stack.extend(reversed(list(node.values())))
+            elif isinstance(node, list):
+                stack.extend(reversed(node))
+            elif isinstance(node, str | int | float):
+                flattened.append(node)
         return flattened
 
     @staticmethod
@@ -208,7 +211,7 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]):
         if isinstance(fields, str):
             try:
                 fields = json.loads(fields)
-            except (json.JSONDecodeError, TypeError):
+            except (json.JSONDecodeError, TypeError, RecursionError):
                 return []
         if not isinstance(fields, list):
             return []

backend/packages/harness/deerflow/agents/middlewares/dangling_tool_call_middleware.py

@@ -143,7 +143,7 @@ def _parse_json_object(value: object) -> dict | None:
         return None
     try:
         parsed = json.loads(value)
-    except ValueError:
+    except (ValueError, RecursionError):
         return None
     return parsed if isinstance(parsed, dict) else None
 
@@ -153,7 +153,7 @@ def _normalize_tool_arguments(arguments: object) -> str:
     if isinstance(arguments, dict):
         try:
             return json.dumps(arguments, ensure_ascii=False, allow_nan=False)
-        except (TypeError, ValueError):
+        except (TypeError, ValueError, RecursionError):
             return "{}"
     return arguments if _parse_json_object(arguments) is not None else "{}"
 

backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py

@@ -109,6 +109,17 @@ _UI_NARRATION_TOOLS = frozenset({"bash", "ls", "glob", "grep"})
 
 type _RunScopeKey = tuple[str, str | None]
 
+# Deeply nested args cannot be serialized for a key; they collapse to one opaque key.
+_UNSERIALIZABLE_ARGS = "<unserializable>"
+
+
+def _stable_args_key(value: object) -> str | None:
+    """Serialize opaque args for a stable key, or None past the recursion limit."""
+    try:
+        return json.dumps(value, sort_keys=True, default=str)
+    except (TypeError, ValueError, RecursionError):
+        return None
+
 
 def _normalize_tool_call_args(raw_args: object) -> tuple[dict, str | None]:
     """Normalize tool call args to a dict plus an optional fallback key.
@@ -123,17 +134,17 @@ def _normalize_tool_call_args(raw_args: object) -> tuple[dict, str | None]:
     if isinstance(raw_args, str):
         try:
             parsed = json.loads(raw_args)
-        except (TypeError, ValueError, json.JSONDecodeError):
+        except (TypeError, ValueError, json.JSONDecodeError, RecursionError):
             return {}, raw_args
 
         if isinstance(parsed, dict):
             return parsed, None
-        return {}, json.dumps(parsed, sort_keys=True, default=str)
+        return {}, _stable_args_key(parsed)
 
     if raw_args is None:
         return {}, None
 
-    return {}, json.dumps(raw_args, sort_keys=True, default=str)
+    return {}, _stable_args_key(raw_args)
 
 
 def _coerce_line_number(value: object) -> int | None:
@@ -181,6 +192,11 @@ def _stable_tool_key(name: str, args: dict, fallback_key: str | None) -> str:
     """Derive a stable key from the args that change what a call does."""
     if name == "read_file" and fallback_key is None:
         path = args.get("path") or ""
+        if not isinstance(path, str):
+            # Formatting a nested structure into the key calls repr(), which
+            # recurses past the limit for the same payloads _stable_args_key
+            # already treats as opaque.
+            path = _stable_args_key(path) or _UNSERIALIZABLE_ARGS
         start_line, end_line = _normalized_read_range(args)
         return f"{path}:{start_line}-{end_line if end_line is not None else _OPEN_ENDED_READ}"
 
@@ -190,7 +206,7 @@ def _stable_tool_key(name: str, args: dict, fallback_key: str | None) -> str:
     if name in {"write_file", "str_replace"}:
         if fallback_key is not None:
             return fallback_key
-        return json.dumps(args, sort_keys=True, default=str)
+        return _stable_args_key(args) or _UNSERIALIZABLE_ARGS
 
     if fallback_key is not None:
         return fallback_key
@@ -203,7 +219,7 @@ def _stable_tool_key(name: str, args: dict, fallback_key: str | None) -> str:
     # call look new (#1905).
     if name in _UI_NARRATION_TOOLS:
         args = {field: value for field, value in args.items() if field != "description"}
-    return json.dumps(args, sort_keys=True, default=str)
+    return _stable_args_key(args) or _UNSERIALIZABLE_ARGS
 
 
 def _hash_tool_calls(tool_calls: list[dict]) -> str:

backend/tests/test_tool_call_json_depth.py

@@ -0,0 +1,80 @@
+"""Tool-call argument normalization must tolerate deeply nested JSON.
+
+A model (or a replayed history) can hand the middlewares a JSON string or a
+Python structure nested beyond the interpreter's recursion limit. These
+payloads are unusable for loop detection or replay repair, but they must not
+abort the graph with ``RecursionError``; the parsers fall back to the same
+"opaque" path they use for any unparsable input.
+"""
+
+from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware
+from deerflow.agents.middlewares.dangling_tool_call_middleware import (
+    _normalize_tool_arguments,
+    _parse_json_object,
+)
+from deerflow.agents.middlewares.loop_detection_middleware import (
+    _hash_tool_calls,
+    _normalize_tool_call_args,
+)
+
+# Above the JSON C parser's recursion limit on CPython 3.12 for the platforms this
+# repository builds on (Linux/macOS 64-bit allow roughly 8,000 levels).
+DEEP_DEPTH = 20_000
+DEEP_ARRAY = "[" * DEEP_DEPTH + "]" * DEEP_DEPTH
+
+
+def _deep_dict(depth: int = DEEP_DEPTH) -> dict:
+    root: dict = {}
+    node = root
+    for _ in range(depth):
+        child: dict = {}
+        node["x"] = child
+        node = child
+    return root
+
+
+def _deep_list(depth: int = DEEP_DEPTH) -> list:
+    root: list = []
+    node = root
+    for _ in range(depth):
+        child: list = []
+        node.append(child)
+        node = child
+    return root
+
+
+def test_loop_detection_keeps_a_deep_json_string_as_opaque_args():
+    assert _normalize_tool_call_args(DEEP_ARRAY) == ({}, DEEP_ARRAY)
+
+
+def test_loop_detection_keeps_deep_structures_as_opaque_args():
+    assert _normalize_tool_call_args(_deep_list()) == ({}, None)
+
+
+def test_loop_detection_hash_keeps_deep_dict_args_serializable():
+    assert _hash_tool_calls([{"name": "bash", "args": _deep_dict()}]) is not None
+
+
+def test_clarification_options_keep_a_deep_json_string_as_one_option():
+    assert ClarificationMiddleware._normalize_options(None, DEEP_ARRAY) == [DEEP_ARRAY]
+
+
+def test_clarification_native_deep_options_are_skipped():
+    assert ClarificationMiddleware._normalize_options(None, [_deep_list()]) == []
+
+
+def test_clarification_dict_options_flatten_without_recursion():
+    assert ClarificationMiddleware._flatten_dict_option_values(_deep_dict()) == []
+
+
+def test_loop_detection_read_file_key_tolerates_a_deep_path():
+    assert _hash_tool_calls([{"name": "read_file", "args": {"path": _deep_list()}}]) is not None
+
+
+def test_dangling_parser_rejects_a_deep_json_string():
+    assert _parse_json_object(DEEP_ARRAY) is None
+
+
+def test_dangling_arguments_repair_falls_back_to_empty_object():
+    assert _normalize_tool_arguments(DEEP_ARRAY) == "{}"
+    assert _normalize_tool_arguments(_deep_dict()) == "{}"