mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-19 19:16:17 +00:00
49f2197ba1
* fix(sandbox): report the line a read_file truncation lands on
The read_file tool head-truncates at a character offset and tells the
model to continue with start_line/end_line, but the marker reported only
character counts ("showing first N of M chars"), so the model had no way
to know which line the cut fell on — the cut almost always lands mid-line
and read_file output carries no line numbers (#5475).
The marker now also reports the 1-indexed line holding the first hidden
character and the file's total line count, and names the exact resume
point: "... [truncated: showing first N of M chars (cut lands in line L
of T). Use start_line=L — optionally with end_line — to continue without
a gap] ...". Resuming at the reported line is gap-free whether the cut
lands mid-line or exactly after a newline.
The marker length budget accounts for the new fields, so the
len(result) <= max_chars contract still holds.
* fix(sandbox): report absolute lines in ranged-read truncation markers
Review follow-up on #5478: read_file_tool runs the same truncation on
ranged reads (start_line/end_line), where the slice's line 1 is the
requested start_line, not the file's first line. The marker's reported
lines were slice-relative while the model reasons in absolute file lines,
so the resume hint could re-issue the identical start_line forever
(repro: resume at 831 -> "cut lands in line 831 of 5170" -> start_line=831).
_truncate_read_file_output gains a line_offset parameter (the 0-based
absolute line of the slice's first line) and reports absolute lines for
both the cut position and the range end; read_file_tool threads
effective_start - 1 through. Full reads pass the default offset 0 and are
byte-identical.
Regression tests pin the absolute coordinates and that the resume point
strictly advances past the slice start.
361 lines
15 KiB
Python
361 lines
15 KiB
Python
"""Unit tests for tool output truncation functions.
|
|
|
|
These functions truncate long tool outputs to prevent context window overflow.
|
|
- _truncate_bash_output: middle-truncation (head + tail), for bash tool
|
|
- _truncate_read_file_output: head-truncation, for read_file tool
|
|
- _truncate_ls_output: head-truncation, for ls tool
|
|
"""
|
|
|
|
import re
|
|
|
|
from deerflow.sandbox.tools import _truncate_bash_output, _truncate_ls_output, _truncate_read_file_output
|
|
|
|
|
|
def _head_and_marker(result: str) -> tuple[str, str]:
|
|
"""Split a truncated read_file result into shown head and trailing marker."""
|
|
idx = result.rfind("\n... [truncated:")
|
|
assert idx != -1, "truncation marker missing"
|
|
return result[:idx], result[idx:]
|
|
|
|
|
|
def _line_containing(output: str, char_index: int) -> int:
|
|
"""Return the 0-based line index holding ``output[char_index]``.
|
|
|
|
Computed from line spans independently of the truncation marker, so the
|
|
tests verify the marker's reported line rather than restating its formula.
|
|
"""
|
|
assert 0 <= char_index < len(output)
|
|
starts = [0]
|
|
for line in output.splitlines(keepends=True):
|
|
starts.append(starts[-1] + len(line))
|
|
candidate = 0
|
|
for i, start in enumerate(starts):
|
|
if start <= char_index:
|
|
candidate = i
|
|
return candidate
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _truncate_bash_output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTruncateBashOutput:
|
|
def test_short_output_returned_unchanged(self):
|
|
output = "hello world"
|
|
assert _truncate_bash_output(output, 20000) == output
|
|
|
|
def test_trailing_exit_marker_survives_truncation(self):
|
|
"""PR review: a failing command's pass-shaped text must not lose the
|
|
authoritative exit marker to truncation — evidence consumers parse it."""
|
|
output = "1 passed\n" + "M" * 30000 + "\nExit Code: 1"
|
|
result = _truncate_bash_output(output, 20000)
|
|
assert result.endswith("\nExit Code: 1")
|
|
assert len(result) <= 20000
|
|
|
|
def test_trailing_exit_marker_survives_a_tiny_budget(self):
|
|
output = "x" * 5000 + "\nExit Code: 124"
|
|
result = _truncate_bash_output(output, 100)
|
|
assert result.endswith("\nExit Code: 124")
|
|
assert len(result) <= 100
|
|
|
|
def test_command_exited_with_code_form_is_preserved(self):
|
|
output = "M" * 5000 + "\nCommand exited with code 3"
|
|
result = _truncate_bash_output(output, 1000)
|
|
assert result.endswith("Command exited with code 3")
|
|
|
|
def test_signed_signal_exit_marker_is_preserved(self):
|
|
"""Signal-killed processes report signed codes (Exit Code: -9)."""
|
|
output = "5 passed\n" + "M" * 5000 + "\nExit Code: -9"
|
|
result = _truncate_bash_output(output, 100)
|
|
assert result.endswith("\nExit Code: -9")
|
|
assert len(result) <= 100
|
|
|
|
def test_marker_preserved_when_limit_is_below_marker_length(self):
|
|
"""PR review: a configured limit smaller than the exit marker must
|
|
not silently discard failure status — the floor keeps it."""
|
|
result = _truncate_bash_output("1 passed\nExit Code: 1", 10)
|
|
assert result.endswith("\nExit Code: 1")
|
|
assert len(result) <= 32 # the marker-preserving floor
|
|
|
|
def test_small_limit_without_marker_uses_the_floor(self):
|
|
result = _truncate_bash_output("x" * 500, 10)
|
|
assert len(result) <= 32
|
|
|
|
def test_output_without_marker_truncates_as_before(self):
|
|
output = "A" * 30000
|
|
result = _truncate_bash_output(output, 20000)
|
|
assert len(result) <= 20000
|
|
assert not result.endswith("Exit Code: 1")
|
|
|
|
def test_output_equal_to_limit_returned_unchanged(self):
|
|
output = "A" * 20000
|
|
assert _truncate_bash_output(output, 20000) == output
|
|
|
|
def test_long_output_is_truncated(self):
|
|
output = "A" * 30000
|
|
result = _truncate_bash_output(output, 20000)
|
|
assert len(result) < len(output)
|
|
|
|
def test_result_never_exceeds_max_chars(self):
|
|
output = "A" * 30000
|
|
max_chars = 20000
|
|
result = _truncate_bash_output(output, max_chars)
|
|
assert len(result) <= max_chars
|
|
|
|
def test_head_is_preserved(self):
|
|
head = "HEAD_CONTENT"
|
|
output = head + "M" * 30000
|
|
result = _truncate_bash_output(output, 20000)
|
|
assert result.startswith(head)
|
|
|
|
def test_tail_is_preserved(self):
|
|
tail = "TAIL_CONTENT"
|
|
output = "M" * 30000 + tail
|
|
result = _truncate_bash_output(output, 20000)
|
|
assert result.endswith(tail)
|
|
|
|
def test_middle_truncation_marker_present(self):
|
|
output = "A" * 30000
|
|
result = _truncate_bash_output(output, 20000)
|
|
assert "[middle truncated:" in result
|
|
assert "chars skipped" in result
|
|
|
|
def test_skipped_chars_count_is_correct(self):
|
|
output = "A" * 25000
|
|
result = _truncate_bash_output(output, 20000)
|
|
# Extract the reported skipped count and verify it equals len(output) - kept.
|
|
# (kept = max_chars - marker_max_len, where marker_max_len is computed from
|
|
# the worst-case marker string — so the exact value is implementation-defined,
|
|
# but it must equal len(output) minus the chars actually preserved.)
|
|
import re
|
|
|
|
m = re.search(r"(\d+) chars skipped", result)
|
|
assert m is not None
|
|
reported_skipped = int(m.group(1))
|
|
# Verify the number is self-consistent: head + skipped + tail == total
|
|
assert reported_skipped > 0
|
|
# The marker reports exactly the chars between head and tail
|
|
head_and_tail = len(output) - reported_skipped
|
|
assert result.startswith(output[: head_and_tail // 2])
|
|
|
|
def test_max_chars_zero_disables_truncation(self):
|
|
output = "A" * 100000
|
|
assert _truncate_bash_output(output, 0) == output
|
|
|
|
def test_50_50_split(self):
|
|
# head and tail should each be roughly max_chars // 2
|
|
output = "H" * 20000 + "M" * 10000 + "T" * 20000
|
|
result = _truncate_bash_output(output, 20000)
|
|
assert result[:100] == "H" * 100
|
|
assert result[-100:] == "T" * 100
|
|
|
|
def test_small_max_chars_does_not_crash(self):
|
|
output = "A" * 1000
|
|
result = _truncate_bash_output(output, 10)
|
|
# Limits below the marker-preserving floor (32) are raised so a
|
|
# failing command's exit marker always fits; see
|
|
# _BASH_OUTPUT_MIN_LIMIT_CHARS.
|
|
assert len(result) <= 32
|
|
|
|
def test_result_never_exceeds_max_chars_various_sizes(self):
|
|
output = "X" * 50000
|
|
for max_chars in [100, 1000, 5000, 20000, 49999]:
|
|
result = _truncate_bash_output(output, max_chars)
|
|
assert len(result) <= max_chars, f"failed for max_chars={max_chars}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _truncate_read_file_output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTruncateReadFileOutput:
|
|
def test_short_output_returned_unchanged(self):
|
|
output = "def foo():\n pass\n"
|
|
assert _truncate_read_file_output(output, 50000) == output
|
|
|
|
def test_output_equal_to_limit_returned_unchanged(self):
|
|
output = "X" * 50000
|
|
assert _truncate_read_file_output(output, 50000) == output
|
|
|
|
def test_long_output_is_truncated(self):
|
|
output = "X" * 60000
|
|
result = _truncate_read_file_output(output, 50000)
|
|
assert len(result) < len(output)
|
|
|
|
def test_result_never_exceeds_max_chars(self):
|
|
output = "X" * 60000
|
|
max_chars = 50000
|
|
result = _truncate_read_file_output(output, max_chars)
|
|
assert len(result) <= max_chars
|
|
|
|
def test_head_is_preserved(self):
|
|
head = "import os\nimport sys\n"
|
|
output = head + "X" * 60000
|
|
result = _truncate_read_file_output(output, 50000)
|
|
assert result.startswith(head)
|
|
|
|
def test_truncation_marker_present(self):
|
|
output = "X" * 60000
|
|
result = _truncate_read_file_output(output, 50000)
|
|
assert "[truncated:" in result
|
|
assert "showing first" in result
|
|
|
|
def test_total_chars_reported_correctly(self):
|
|
output = "X" * 60000
|
|
result = _truncate_read_file_output(output, 50000)
|
|
assert "of 60000 chars" in result
|
|
|
|
def test_start_line_hint_present(self):
|
|
output = "X" * 60000
|
|
result = _truncate_read_file_output(output, 50000)
|
|
assert "start_line" in result
|
|
assert "end_line" in result
|
|
|
|
def test_marker_reports_the_line_the_cut_lands_in(self):
|
|
# Shape from #5475: a multi-line file cut mid-line, where the old
|
|
# marker gave only a character count so the model could not compute
|
|
# which line to resume from.
|
|
output = "".join(f"def fn_{i}():\n return {i}\n" for i in range(3000))
|
|
result = _truncate_read_file_output(output, 50000)
|
|
head, marker = _head_and_marker(result)
|
|
cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1))
|
|
total_lines = output.count("\n")
|
|
assert f"cut lands in line {cut_line} of {total_lines}" in marker
|
|
assert f"start_line={cut_line}" in marker
|
|
# The no-gap contract: the reported line is the one holding the first
|
|
# hidden character (computed independently from line spans).
|
|
assert _line_containing(output, len(head)) + 1 == cut_line
|
|
|
|
def test_first_hidden_character_always_belongs_to_reported_line(self):
|
|
# Whatever size the cut lands at — mid-line or exactly after a
|
|
# newline — the reported line is the one holding the first hidden
|
|
# character, so resuming at start_line=<reported> leaves no gap.
|
|
lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)]
|
|
output = "".join(lines)
|
|
for max_chars in [1000, 5000, 20000, 50000]:
|
|
result = _truncate_read_file_output(output, max_chars)
|
|
head, marker = _head_and_marker(result)
|
|
cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1))
|
|
assert 1 <= cut_line <= len(lines)
|
|
assert _line_containing(output, len(head)) + 1 == cut_line
|
|
assert f"start_line={cut_line}" in marker
|
|
assert len(result) <= max_chars
|
|
|
|
def test_marker_reports_total_lines(self):
|
|
lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)]
|
|
output = "".join(lines)
|
|
result = _truncate_read_file_output(output, 50000)
|
|
_, marker = _head_and_marker(result)
|
|
assert f"of {output.count(chr(10))}" in marker
|
|
|
|
def test_ranged_read_reports_absolute_lines(self):
|
|
# Ranged reads hand a slice to the truncator; reported lines must be
|
|
# absolute file lines or the resume hint loops back onto the slice's
|
|
# own start (the repro from the #5478 review: start_line=831 kept
|
|
# suggesting start_line=831 forever).
|
|
slice_lines = [f"row-{i}-content\n" for i in range(1, 5171)]
|
|
slice_output = "".join(slice_lines)
|
|
offset = 830 # slice starts at absolute line 831
|
|
result = _truncate_read_file_output(slice_output, 50000, line_offset=offset)
|
|
head, marker = _head_and_marker(result)
|
|
absolute_cut = offset + head.count("\n") + 1
|
|
absolute_last = offset + len(slice_lines)
|
|
assert f"cut lands in line {absolute_cut} of {absolute_last}" in marker
|
|
assert f"start_line={absolute_cut}" in marker
|
|
assert absolute_cut > offset + 1 # resume strictly advances
|
|
|
|
def test_ranged_read_zero_offset_keeps_full_read_semantics(self):
|
|
output = "".join(f"line-{i}-with-padding\n" for i in range(3000))
|
|
assert _truncate_read_file_output(output, 50000) == _truncate_read_file_output(output, 50000, line_offset=0)
|
|
|
|
def test_max_chars_zero_disables_truncation(self):
|
|
output = "X" * 100000
|
|
assert _truncate_read_file_output(output, 0) == output
|
|
|
|
def test_tail_is_not_preserved(self):
|
|
# head-truncation: tail should be cut off
|
|
output = "H" * 50000 + "TAIL_SHOULD_NOT_APPEAR"
|
|
result = _truncate_read_file_output(output, 50000)
|
|
assert "TAIL_SHOULD_NOT_APPEAR" not in result
|
|
|
|
def test_small_max_chars_does_not_crash(self):
|
|
output = "X" * 1000
|
|
result = _truncate_read_file_output(output, 10)
|
|
assert len(result) <= 10
|
|
|
|
def test_result_never_exceeds_max_chars_various_sizes(self):
|
|
output = "X" * 50000
|
|
for max_chars in [100, 1000, 5000, 20000, 49999]:
|
|
result = _truncate_read_file_output(output, max_chars)
|
|
assert len(result) <= max_chars, f"failed for max_chars={max_chars}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _truncate_ls_output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTruncateLsOutput:
|
|
def test_short_output_returned_unchanged(self):
|
|
output = "dir1\ndir2\nfile1.txt"
|
|
assert _truncate_ls_output(output, 20000) == output
|
|
|
|
def test_output_equal_to_limit_returned_unchanged(self):
|
|
output = "X" * 20000
|
|
assert _truncate_ls_output(output, 20000) == output
|
|
|
|
def test_long_output_is_truncated(self):
|
|
output = "\n".join(f"file_{i}.txt" for i in range(5000))
|
|
result = _truncate_ls_output(output, 20000)
|
|
assert len(result) < len(output)
|
|
|
|
def test_result_never_exceeds_max_chars(self):
|
|
output = "\n".join(f"subdir/file_{i}.txt" for i in range(5000))
|
|
max_chars = 20000
|
|
result = _truncate_ls_output(output, max_chars)
|
|
assert len(result) <= max_chars
|
|
|
|
def test_head_is_preserved(self):
|
|
head = "first_dir\nsecond_dir\n"
|
|
output = head + "\n".join(f"file_{i}" for i in range(5000))
|
|
result = _truncate_ls_output(output, 20000)
|
|
assert result.startswith(head)
|
|
|
|
def test_truncation_marker_present(self):
|
|
output = "\n".join(f"file_{i}.txt" for i in range(5000))
|
|
result = _truncate_ls_output(output, 20000)
|
|
assert "[truncated:" in result
|
|
assert "showing first" in result
|
|
|
|
def test_total_chars_reported_correctly(self):
|
|
output = "X" * 30000
|
|
result = _truncate_ls_output(output, 20000)
|
|
assert "of 30000 chars" in result
|
|
|
|
def test_hint_suggests_specific_path(self):
|
|
output = "X" * 30000
|
|
result = _truncate_ls_output(output, 20000)
|
|
assert "Use a more specific path" in result
|
|
|
|
def test_max_chars_zero_disables_truncation(self):
|
|
output = "\n".join(f"file_{i}.txt" for i in range(10000))
|
|
assert _truncate_ls_output(output, 0) == output
|
|
|
|
def test_tail_is_not_preserved(self):
|
|
output = "H" * 20000 + "TAIL_SHOULD_NOT_APPEAR"
|
|
result = _truncate_ls_output(output, 20000)
|
|
assert "TAIL_SHOULD_NOT_APPEAR" not in result
|
|
|
|
def test_small_max_chars_does_not_crash(self):
|
|
output = "\n".join(f"file_{i}.txt" for i in range(100))
|
|
result = _truncate_ls_output(output, 10)
|
|
assert len(result) <= 10
|
|
|
|
def test_result_never_exceeds_max_chars_various_sizes(self):
|
|
output = "\n".join(f"file_{i}.txt" for i in range(5000))
|
|
for max_chars in [100, 1000, 5000, 20000, len(output) - 1]:
|
|
result = _truncate_ls_output(output, max_chars)
|
|
assert len(result) <= max_chars, f"failed for max_chars={max_chars}"
|