From 49f2197ba1086412367dc85b6a16cf9306527b1e Mon Sep 17 00:00:00 2001 From: xiaodu55 <1704788159@qq.com> Date: Wed, 16 Sep 2026 22:24:01 +0800 Subject: [PATCH] fix(sandbox): report the line a read_file truncation lands on (#5478) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sandbox): report the line a read_file truncation lands on The read_file tool head-truncates at a character offset and tells the model to continue with start_line/end_line, but the marker reported only character counts ("showing first N of M chars"), so the model had no way to know which line the cut fell on — the cut almost always lands mid-line and read_file output carries no line numbers (#5475). The marker now also reports the 1-indexed line holding the first hidden character and the file's total line count, and names the exact resume point: "... [truncated: showing first N of M chars (cut lands in line L of T). Use start_line=L — optionally with end_line — to continue without a gap] ...". Resuming at the reported line is gap-free whether the cut lands mid-line or exactly after a newline. The marker length budget accounts for the new fields, so the len(result) <= max_chars contract still holds. * fix(sandbox): report absolute lines in ranged-read truncation markers Review follow-up on #5478: read_file_tool runs the same truncation on ranged reads (start_line/end_line), where the slice's line 1 is the requested start_line, not the file's first line. The marker's reported lines were slice-relative while the model reasons in absolute file lines, so the resume hint could re-issue the identical start_line forever (repro: resume at 831 -> "cut lands in line 831 of 5170" -> start_line=831). _truncate_read_file_output gains a line_offset parameter (the 0-based absolute line of the slice's first line) and reports absolute lines for both the cut position and the range end; read_file_tool threads effective_start - 1 through. Full reads pass the default offset 0 and are byte-identical. Regression tests pin the absolute coordinates and that the resume point strictly advances past the slice start. --- .../harness/deerflow/sandbox/tools.py | 30 +++++-- backend/tests/test_tool_output_truncation.py | 84 +++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/backend/packages/harness/deerflow/sandbox/tools.py b/backend/packages/harness/deerflow/sandbox/tools.py index 4748af32b..463656d4b 100644 --- a/backend/packages/harness/deerflow/sandbox/tools.py +++ b/backend/packages/harness/deerflow/sandbox/tools.py @@ -1840,7 +1840,7 @@ def _truncate_bash_output(output: str, max_chars: int) -> str: return f"{output[:head_len]}{marker}{output[-tail_len:] if tail_len > 0 else ''}" + preserved -def _truncate_read_file_output(output: str, max_chars: int) -> str: +def _truncate_read_file_output(output: str, max_chars: int, line_offset: int = 0) -> str: """Head-truncate read_file output, preserving the beginning of the file. Source code and documents are read top-to-bottom; the head contains the @@ -1849,19 +1849,34 @@ def _truncate_read_file_output(output: str, max_chars: int) -> str: The returned string (including the truncation marker) is guaranteed to be no longer than max_chars characters. Pass max_chars=0 to disable truncation and return the full output unchanged. + + The marker reports the 1-indexed line the character cut lands in, so the + model can resume with ``start_line`` without re-reading or skipping + content (#5475). ``line_offset`` is the 0-based absolute line number of + ``output``'s first line — ranged reads pass the requested ``start_line - + 1`` so the reported lines are absolute file lines, keeping the resume + point from looping back onto the slice's own beginning. """ if max_chars == 0: return output if len(output) <= max_chars: return output total = len(output) - # Compute the exact worst-case marker length: both numeric fields are at - # their maximum (total chars), so this is a tight upper bound. - marker_max_len = len(f"\n... [truncated: showing first {total} of {total} chars. Use start_line/end_line to read a specific range] ...") + total_lines = output.count("\n") + (0 if output.endswith("\n") else 1) + # Compute the exact worst-case marker length: every numeric field is at + # its maximum (total + offset), so this is a tight upper bound. + bound = total + max(line_offset, 0) + marker_max_len = len(f"\n... [truncated: showing first {total} of {total} chars (cut lands in line {bound} of {bound}). Use start_line={bound} — optionally with end_line — to continue without a gap] ...") kept = max(0, max_chars - marker_max_len) if kept == 0: return output[:max_chars] - marker = f"\n... [truncated: showing first {kept} of {total} chars. Use start_line/end_line to read a specific range] ..." + # 1-indexed line holding the first hidden character: a cut mid-line lands + # in the partially shown line, a cut exactly after a newline lands in the + # next line — either way resuming at this line leaves no gap. Expressed in + # absolute file lines via line_offset. + cut_line = line_offset + output[:kept].count("\n") + 1 + last_line = line_offset + total_lines + marker = f"\n... [truncated: showing first {kept} of {total} chars (cut lands in line {cut_line} of {last_line}). Use start_line={cut_line} — optionally with end_line — to continue without a gap] ..." return f"{output[:kept]}{marker}" @@ -2445,7 +2460,10 @@ def read_file_tool( max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000 except Exception: max_chars = 50000 - return _truncate_read_file_output(content, max_chars) + # Ranged reads hand a slice to the truncator; line_offset converts the + # slice-relative cut line back to the absolute file lines the tool's + # start_line/end_line arguments speak in (#5475). + return _truncate_read_file_output(content, max_chars, line_offset=effective_start - 1) except SandboxError as e: return f"Error: {e}" except FileNotFoundError: diff --git a/backend/tests/test_tool_output_truncation.py b/backend/tests/test_tool_output_truncation.py index 4b2bf4652..1d4094ac4 100644 --- a/backend/tests/test_tool_output_truncation.py +++ b/backend/tests/test_tool_output_truncation.py @@ -6,8 +6,35 @@ These functions truncate long tool outputs to prevent context window overflow. - _truncate_ls_output: head-truncation, for ls tool """ +import re + from deerflow.sandbox.tools import _truncate_bash_output, _truncate_ls_output, _truncate_read_file_output + +def _head_and_marker(result: str) -> tuple[str, str]: + """Split a truncated read_file result into shown head and trailing marker.""" + idx = result.rfind("\n... [truncated:") + assert idx != -1, "truncation marker missing" + return result[:idx], result[idx:] + + +def _line_containing(output: str, char_index: int) -> int: + """Return the 0-based line index holding ``output[char_index]``. + + Computed from line spans independently of the truncation marker, so the + tests verify the marker's reported line rather than restating its formula. + """ + assert 0 <= char_index < len(output) + starts = [0] + for line in output.splitlines(keepends=True): + starts.append(starts[-1] + len(line)) + candidate = 0 + for i, start in enumerate(starts): + if start <= char_index: + candidate = i + return candidate + + # --------------------------------------------------------------------------- # _truncate_bash_output # --------------------------------------------------------------------------- @@ -186,6 +213,63 @@ class TestTruncateReadFileOutput: assert "start_line" in result assert "end_line" in result + def test_marker_reports_the_line_the_cut_lands_in(self): + # Shape from #5475: a multi-line file cut mid-line, where the old + # marker gave only a character count so the model could not compute + # which line to resume from. + output = "".join(f"def fn_{i}():\n return {i}\n" for i in range(3000)) + result = _truncate_read_file_output(output, 50000) + head, marker = _head_and_marker(result) + cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1)) + total_lines = output.count("\n") + assert f"cut lands in line {cut_line} of {total_lines}" in marker + assert f"start_line={cut_line}" in marker + # The no-gap contract: the reported line is the one holding the first + # hidden character (computed independently from line spans). + assert _line_containing(output, len(head)) + 1 == cut_line + + def test_first_hidden_character_always_belongs_to_reported_line(self): + # Whatever size the cut lands at — mid-line or exactly after a + # newline — the reported line is the one holding the first hidden + # character, so resuming at start_line= leaves no gap. + lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)] + output = "".join(lines) + for max_chars in [1000, 5000, 20000, 50000]: + result = _truncate_read_file_output(output, max_chars) + head, marker = _head_and_marker(result) + cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1)) + assert 1 <= cut_line <= len(lines) + assert _line_containing(output, len(head)) + 1 == cut_line + assert f"start_line={cut_line}" in marker + assert len(result) <= max_chars + + def test_marker_reports_total_lines(self): + lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)] + output = "".join(lines) + result = _truncate_read_file_output(output, 50000) + _, marker = _head_and_marker(result) + assert f"of {output.count(chr(10))}" in marker + + def test_ranged_read_reports_absolute_lines(self): + # Ranged reads hand a slice to the truncator; reported lines must be + # absolute file lines or the resume hint loops back onto the slice's + # own start (the repro from the #5478 review: start_line=831 kept + # suggesting start_line=831 forever). + slice_lines = [f"row-{i}-content\n" for i in range(1, 5171)] + slice_output = "".join(slice_lines) + offset = 830 # slice starts at absolute line 831 + result = _truncate_read_file_output(slice_output, 50000, line_offset=offset) + head, marker = _head_and_marker(result) + absolute_cut = offset + head.count("\n") + 1 + absolute_last = offset + len(slice_lines) + assert f"cut lands in line {absolute_cut} of {absolute_last}" in marker + assert f"start_line={absolute_cut}" in marker + assert absolute_cut > offset + 1 # resume strictly advances + + def test_ranged_read_zero_offset_keeps_full_read_semantics(self): + output = "".join(f"line-{i}-with-padding\n" for i in range(3000)) + assert _truncate_read_file_output(output, 50000) == _truncate_read_file_output(output, 50000, line_offset=0) + def test_max_chars_zero_disables_truncation(self): output = "X" * 100000 assert _truncate_read_file_output(output, 0) == output