diff --git a/README.md b/README.md index b70c4d133f..df2f55f408 100644 --- a/README.md +++ b/README.md @@ -3012,6 +3012,7 @@ Each task gets its own execution environment with a full filesystem view — ski The read-before-write gate ties each read mark to that `read_file` call's result, including custom tools returning multi-message `Command` updates. An unrelated result cannot authorize a write after a failed read or hide a successful read. Deeply nested JSON returned by tools, including `read_file`, keeps its original content when result analysis reaches the JSON parser's depth limit. Explicit tool failures remain errors with an unknown category when depth prevents analysis. +Tool-call arguments and clarification options nested past the JSON parser's recursion limit are treated as unusable opaque values instead of aborting the run. On Linux and macOS, local sandbox `write_file` replacements and `str_replace` stage complete text before replacing the destination, so write, flush or publication errors retain the previous file. Existing write access is checked; ownership, permission bits, ACLs and extended attributes are transferred before writing staging content. Failure to transfer metadata aborts the save. Successful revisions get a fresh modification time, including empty saves; source-descriptor cleanup errors are warnings and do not change the save result. Files with set-id bits, macOS file flags or content-bound Linux privilege attributes are refused unchanged. Replacement requires parent-directory write access and publishes a new inode; other hard links retain the previous version. Symlinks follow the existing path-validation rules. Append writes, other host platforms (including Windows), shell writes and remote providers keep their existing behavior. This is not a concurrent-writer or power-loss recovery guarantee. diff --git a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md index 5d47434369..ed8147d528 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md +++ b/backend/packages/harness/deerflow/agents/middlewares/AGENTS.md @@ -8,6 +8,8 @@ depth regressions in `tests/test_tool_result_meta_json_depth.py`, including real LocalSandbox reads through both synchronous and asynchronous agent graphs. Deep graph cases must establish a real parser depth failure or report a skip. +Tool-call argument depth handling is best-effort; see `tests/test_tool_call_json_depth.py`. + Async gate waiters own separate cross-loop futures. Cancel only their signal; notify outside the state lock. `_await_off_thread` drains started I/O across repeated cancellation. diff --git a/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py index d05b66fb4c..975a47b357 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/clarification_middleware.py @@ -143,7 +143,7 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]): if isinstance(options, str): try: options = json.loads(options) - except (json.JSONDecodeError, TypeError): + except (json.JSONDecodeError, TypeError, RecursionError): options = [options] if options is None: @@ -159,7 +159,11 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]): normalized: list[str] = [] seen: set[str] = set() for option in options: - text = _XML_TAG_RE.sub("", str(option)).strip() + try: + text = _XML_TAG_RE.sub("", str(option)).strip() + except RecursionError: + # A native option nested past the limit cannot become a label. + continue if not text or text in seen: continue seen.add(text) @@ -170,18 +174,17 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]): def _flatten_dict_option_values(value: dict[str, Any]) -> list[str | int | float]: """Flatten scalar leaves from XML-to-dict option payloads in source order.""" flattened: list[str | int | float] = [] - - def collect(nested: Any) -> None: - if isinstance(nested, dict): - for item in nested.values(): - collect(item) - elif isinstance(nested, list): - for item in nested: - collect(item) - elif isinstance(nested, str | int | float): - flattened.append(nested) - - collect(value) + # Explicit stack instead of recursion: XML-to-dict payloads are + # model-controlled and can nest past the interpreter's limit. + stack: list[Any] = [value] + while stack: + node = stack.pop() + if isinstance(node, dict): + stack.extend(reversed(list(node.values()))) + elif isinstance(node, list): + stack.extend(reversed(node)) + elif isinstance(node, str | int | float): + flattened.append(node) return flattened @staticmethod @@ -208,7 +211,7 @@ class ClarificationMiddleware(AgentMiddleware[ClarificationMiddlewareState]): if isinstance(fields, str): try: fields = json.loads(fields) - except (json.JSONDecodeError, TypeError): + except (json.JSONDecodeError, TypeError, RecursionError): return [] if not isinstance(fields, list): return [] diff --git a/backend/packages/harness/deerflow/agents/middlewares/dangling_tool_call_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/dangling_tool_call_middleware.py index 5a5603dd3a..4f50ebebcc 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/dangling_tool_call_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/dangling_tool_call_middleware.py @@ -143,7 +143,7 @@ def _parse_json_object(value: object) -> dict | None: return None try: parsed = json.loads(value) - except ValueError: + except (ValueError, RecursionError): return None return parsed if isinstance(parsed, dict) else None @@ -153,7 +153,7 @@ def _normalize_tool_arguments(arguments: object) -> str: if isinstance(arguments, dict): try: return json.dumps(arguments, ensure_ascii=False, allow_nan=False) - except (TypeError, ValueError): + except (TypeError, ValueError, RecursionError): return "{}" return arguments if _parse_json_object(arguments) is not None else "{}" diff --git a/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py b/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py index 3cdf40ea17..3ca05c57d3 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py +++ b/backend/packages/harness/deerflow/agents/middlewares/loop_detection_middleware.py @@ -109,6 +109,17 @@ _UI_NARRATION_TOOLS = frozenset({"bash", "ls", "glob", "grep"}) type _RunScopeKey = tuple[str, str | None] +# Deeply nested args cannot be serialized for a key; they collapse to one opaque key. +_UNSERIALIZABLE_ARGS = "" + + +def _stable_args_key(value: object) -> str | None: + """Serialize opaque args for a stable key, or None past the recursion limit.""" + try: + return json.dumps(value, sort_keys=True, default=str) + except (TypeError, ValueError, RecursionError): + return None + def _normalize_tool_call_args(raw_args: object) -> tuple[dict, str | None]: """Normalize tool call args to a dict plus an optional fallback key. @@ -123,17 +134,17 @@ def _normalize_tool_call_args(raw_args: object) -> tuple[dict, str | None]: if isinstance(raw_args, str): try: parsed = json.loads(raw_args) - except (TypeError, ValueError, json.JSONDecodeError): + except (TypeError, ValueError, json.JSONDecodeError, RecursionError): return {}, raw_args if isinstance(parsed, dict): return parsed, None - return {}, json.dumps(parsed, sort_keys=True, default=str) + return {}, _stable_args_key(parsed) if raw_args is None: return {}, None - return {}, json.dumps(raw_args, sort_keys=True, default=str) + return {}, _stable_args_key(raw_args) def _coerce_line_number(value: object) -> int | None: @@ -181,6 +192,11 @@ def _stable_tool_key(name: str, args: dict, fallback_key: str | None) -> str: """Derive a stable key from the args that change what a call does.""" if name == "read_file" and fallback_key is None: path = args.get("path") or "" + if not isinstance(path, str): + # Formatting a nested structure into the key calls repr(), which + # recurses past the limit for the same payloads _stable_args_key + # already treats as opaque. + path = _stable_args_key(path) or _UNSERIALIZABLE_ARGS start_line, end_line = _normalized_read_range(args) return f"{path}:{start_line}-{end_line if end_line is not None else _OPEN_ENDED_READ}" @@ -190,7 +206,7 @@ def _stable_tool_key(name: str, args: dict, fallback_key: str | None) -> str: if name in {"write_file", "str_replace"}: if fallback_key is not None: return fallback_key - return json.dumps(args, sort_keys=True, default=str) + return _stable_args_key(args) or _UNSERIALIZABLE_ARGS if fallback_key is not None: return fallback_key @@ -203,7 +219,7 @@ def _stable_tool_key(name: str, args: dict, fallback_key: str | None) -> str: # call look new (#1905). if name in _UI_NARRATION_TOOLS: args = {field: value for field, value in args.items() if field != "description"} - return json.dumps(args, sort_keys=True, default=str) + return _stable_args_key(args) or _UNSERIALIZABLE_ARGS def _hash_tool_calls(tool_calls: list[dict]) -> str: diff --git a/backend/tests/test_tool_call_json_depth.py b/backend/tests/test_tool_call_json_depth.py new file mode 100644 index 0000000000..d4b6d1defa --- /dev/null +++ b/backend/tests/test_tool_call_json_depth.py @@ -0,0 +1,80 @@ +"""Tool-call argument normalization must tolerate deeply nested JSON. + +A model (or a replayed history) can hand the middlewares a JSON string or a +Python structure nested beyond the interpreter's recursion limit. These +payloads are unusable for loop detection or replay repair, but they must not +abort the graph with ``RecursionError``; the parsers fall back to the same +"opaque" path they use for any unparsable input. +""" + +from deerflow.agents.middlewares.clarification_middleware import ClarificationMiddleware +from deerflow.agents.middlewares.dangling_tool_call_middleware import ( + _normalize_tool_arguments, + _parse_json_object, +) +from deerflow.agents.middlewares.loop_detection_middleware import ( + _hash_tool_calls, + _normalize_tool_call_args, +) + +# Above the JSON C parser's recursion limit on CPython 3.12 for the platforms this +# repository builds on (Linux/macOS 64-bit allow roughly 8,000 levels). +DEEP_DEPTH = 20_000 +DEEP_ARRAY = "[" * DEEP_DEPTH + "]" * DEEP_DEPTH + + +def _deep_dict(depth: int = DEEP_DEPTH) -> dict: + root: dict = {} + node = root + for _ in range(depth): + child: dict = {} + node["x"] = child + node = child + return root + + +def _deep_list(depth: int = DEEP_DEPTH) -> list: + root: list = [] + node = root + for _ in range(depth): + child: list = [] + node.append(child) + node = child + return root + + +def test_loop_detection_keeps_a_deep_json_string_as_opaque_args(): + assert _normalize_tool_call_args(DEEP_ARRAY) == ({}, DEEP_ARRAY) + + +def test_loop_detection_keeps_deep_structures_as_opaque_args(): + assert _normalize_tool_call_args(_deep_list()) == ({}, None) + + +def test_loop_detection_hash_keeps_deep_dict_args_serializable(): + assert _hash_tool_calls([{"name": "bash", "args": _deep_dict()}]) is not None + + +def test_clarification_options_keep_a_deep_json_string_as_one_option(): + assert ClarificationMiddleware._normalize_options(None, DEEP_ARRAY) == [DEEP_ARRAY] + + +def test_clarification_native_deep_options_are_skipped(): + assert ClarificationMiddleware._normalize_options(None, [_deep_list()]) == [] + + +def test_clarification_dict_options_flatten_without_recursion(): + assert ClarificationMiddleware._flatten_dict_option_values(_deep_dict()) == [] + + +def test_loop_detection_read_file_key_tolerates_a_deep_path(): + assert _hash_tool_calls([{"name": "read_file", "args": {"path": _deep_list()}}]) is not None + + +def test_dangling_parser_rejects_a_deep_json_string(): + assert _parse_json_object(DEEP_ARRAY) is None + + +def test_dangling_arguments_repair_falls_back_to_empty_object(): + assert _normalize_tool_arguments(DEEP_ARRAY) == "{}" + assert _normalize_tool_arguments(_deep_dict()) == "{}"