mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-19 11:06:18 +00:00
fix(sandbox): report the line a read_file truncation lands on (#5478)
* fix(sandbox): report the line a read_file truncation lands on
The read_file tool head-truncates at a character offset and tells the
model to continue with start_line/end_line, but the marker reported only
character counts ("showing first N of M chars"), so the model had no way
to know which line the cut fell on — the cut almost always lands mid-line
and read_file output carries no line numbers (#5475).
The marker now also reports the 1-indexed line holding the first hidden
character and the file's total line count, and names the exact resume
point: "... [truncated: showing first N of M chars (cut lands in line L
of T). Use start_line=L — optionally with end_line — to continue without
a gap] ...". Resuming at the reported line is gap-free whether the cut
lands mid-line or exactly after a newline.
The marker length budget accounts for the new fields, so the
len(result) <= max_chars contract still holds.
* fix(sandbox): report absolute lines in ranged-read truncation markers
Review follow-up on #5478: read_file_tool runs the same truncation on
ranged reads (start_line/end_line), where the slice's line 1 is the
requested start_line, not the file's first line. The marker's reported
lines were slice-relative while the model reasons in absolute file lines,
so the resume hint could re-issue the identical start_line forever
(repro: resume at 831 -> "cut lands in line 831 of 5170" -> start_line=831).
_truncate_read_file_output gains a line_offset parameter (the 0-based
absolute line of the slice's first line) and reports absolute lines for
both the cut position and the range end; read_file_tool threads
effective_start - 1 through. Full reads pass the default offset 0 and are
byte-identical.
Regression tests pin the absolute coordinates and that the resume point
strictly advances past the slice start.
This commit is contained in:
parent
4387dce7be
commit
49f2197ba1
@ -1840,7 +1840,7 @@ def _truncate_bash_output(output: str, max_chars: int) -> str:
|
||||
return f"{output[:head_len]}{marker}{output[-tail_len:] if tail_len > 0 else ''}" + preserved
|
||||
|
||||
|
||||
def _truncate_read_file_output(output: str, max_chars: int) -> str:
|
||||
def _truncate_read_file_output(output: str, max_chars: int, line_offset: int = 0) -> str:
|
||||
"""Head-truncate read_file output, preserving the beginning of the file.
|
||||
|
||||
Source code and documents are read top-to-bottom; the head contains the
|
||||
@ -1849,19 +1849,34 @@ def _truncate_read_file_output(output: str, max_chars: int) -> str:
|
||||
The returned string (including the truncation marker) is guaranteed to be
|
||||
no longer than max_chars characters. Pass max_chars=0 to disable truncation
|
||||
and return the full output unchanged.
|
||||
|
||||
The marker reports the 1-indexed line the character cut lands in, so the
|
||||
model can resume with ``start_line`` without re-reading or skipping
|
||||
content (#5475). ``line_offset`` is the 0-based absolute line number of
|
||||
``output``'s first line — ranged reads pass the requested ``start_line -
|
||||
1`` so the reported lines are absolute file lines, keeping the resume
|
||||
point from looping back onto the slice's own beginning.
|
||||
"""
|
||||
if max_chars == 0:
|
||||
return output
|
||||
if len(output) <= max_chars:
|
||||
return output
|
||||
total = len(output)
|
||||
# Compute the exact worst-case marker length: both numeric fields are at
|
||||
# their maximum (total chars), so this is a tight upper bound.
|
||||
marker_max_len = len(f"\n... [truncated: showing first {total} of {total} chars. Use start_line/end_line to read a specific range] ...")
|
||||
total_lines = output.count("\n") + (0 if output.endswith("\n") else 1)
|
||||
# Compute the exact worst-case marker length: every numeric field is at
|
||||
# its maximum (total + offset), so this is a tight upper bound.
|
||||
bound = total + max(line_offset, 0)
|
||||
marker_max_len = len(f"\n... [truncated: showing first {total} of {total} chars (cut lands in line {bound} of {bound}). Use start_line={bound} — optionally with end_line — to continue without a gap] ...")
|
||||
kept = max(0, max_chars - marker_max_len)
|
||||
if kept == 0:
|
||||
return output[:max_chars]
|
||||
marker = f"\n... [truncated: showing first {kept} of {total} chars. Use start_line/end_line to read a specific range] ..."
|
||||
# 1-indexed line holding the first hidden character: a cut mid-line lands
|
||||
# in the partially shown line, a cut exactly after a newline lands in the
|
||||
# next line — either way resuming at this line leaves no gap. Expressed in
|
||||
# absolute file lines via line_offset.
|
||||
cut_line = line_offset + output[:kept].count("\n") + 1
|
||||
last_line = line_offset + total_lines
|
||||
marker = f"\n... [truncated: showing first {kept} of {total} chars (cut lands in line {cut_line} of {last_line}). Use start_line={cut_line} — optionally with end_line — to continue without a gap] ..."
|
||||
return f"{output[:kept]}{marker}"
|
||||
|
||||
|
||||
@ -2445,7 +2460,10 @@ def read_file_tool(
|
||||
max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
|
||||
except Exception:
|
||||
max_chars = 50000
|
||||
return _truncate_read_file_output(content, max_chars)
|
||||
# Ranged reads hand a slice to the truncator; line_offset converts the
|
||||
# slice-relative cut line back to the absolute file lines the tool's
|
||||
# start_line/end_line arguments speak in (#5475).
|
||||
return _truncate_read_file_output(content, max_chars, line_offset=effective_start - 1)
|
||||
except SandboxError as e:
|
||||
return f"Error: {e}"
|
||||
except FileNotFoundError:
|
||||
|
||||
@ -6,8 +6,35 @@ These functions truncate long tool outputs to prevent context window overflow.
|
||||
- _truncate_ls_output: head-truncation, for ls tool
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from deerflow.sandbox.tools import _truncate_bash_output, _truncate_ls_output, _truncate_read_file_output
|
||||
|
||||
|
||||
def _head_and_marker(result: str) -> tuple[str, str]:
|
||||
"""Split a truncated read_file result into shown head and trailing marker."""
|
||||
idx = result.rfind("\n... [truncated:")
|
||||
assert idx != -1, "truncation marker missing"
|
||||
return result[:idx], result[idx:]
|
||||
|
||||
|
||||
def _line_containing(output: str, char_index: int) -> int:
|
||||
"""Return the 0-based line index holding ``output[char_index]``.
|
||||
|
||||
Computed from line spans independently of the truncation marker, so the
|
||||
tests verify the marker's reported line rather than restating its formula.
|
||||
"""
|
||||
assert 0 <= char_index < len(output)
|
||||
starts = [0]
|
||||
for line in output.splitlines(keepends=True):
|
||||
starts.append(starts[-1] + len(line))
|
||||
candidate = 0
|
||||
for i, start in enumerate(starts):
|
||||
if start <= char_index:
|
||||
candidate = i
|
||||
return candidate
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _truncate_bash_output
|
||||
# ---------------------------------------------------------------------------
|
||||
@ -186,6 +213,63 @@ class TestTruncateReadFileOutput:
|
||||
assert "start_line" in result
|
||||
assert "end_line" in result
|
||||
|
||||
def test_marker_reports_the_line_the_cut_lands_in(self):
|
||||
# Shape from #5475: a multi-line file cut mid-line, where the old
|
||||
# marker gave only a character count so the model could not compute
|
||||
# which line to resume from.
|
||||
output = "".join(f"def fn_{i}():\n return {i}\n" for i in range(3000))
|
||||
result = _truncate_read_file_output(output, 50000)
|
||||
head, marker = _head_and_marker(result)
|
||||
cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1))
|
||||
total_lines = output.count("\n")
|
||||
assert f"cut lands in line {cut_line} of {total_lines}" in marker
|
||||
assert f"start_line={cut_line}" in marker
|
||||
# The no-gap contract: the reported line is the one holding the first
|
||||
# hidden character (computed independently from line spans).
|
||||
assert _line_containing(output, len(head)) + 1 == cut_line
|
||||
|
||||
def test_first_hidden_character_always_belongs_to_reported_line(self):
|
||||
# Whatever size the cut lands at — mid-line or exactly after a
|
||||
# newline — the reported line is the one holding the first hidden
|
||||
# character, so resuming at start_line=<reported> leaves no gap.
|
||||
lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)]
|
||||
output = "".join(lines)
|
||||
for max_chars in [1000, 5000, 20000, 50000]:
|
||||
result = _truncate_read_file_output(output, max_chars)
|
||||
head, marker = _head_and_marker(result)
|
||||
cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1))
|
||||
assert 1 <= cut_line <= len(lines)
|
||||
assert _line_containing(output, len(head)) + 1 == cut_line
|
||||
assert f"start_line={cut_line}" in marker
|
||||
assert len(result) <= max_chars
|
||||
|
||||
def test_marker_reports_total_lines(self):
|
||||
lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)]
|
||||
output = "".join(lines)
|
||||
result = _truncate_read_file_output(output, 50000)
|
||||
_, marker = _head_and_marker(result)
|
||||
assert f"of {output.count(chr(10))}" in marker
|
||||
|
||||
def test_ranged_read_reports_absolute_lines(self):
|
||||
# Ranged reads hand a slice to the truncator; reported lines must be
|
||||
# absolute file lines or the resume hint loops back onto the slice's
|
||||
# own start (the repro from the #5478 review: start_line=831 kept
|
||||
# suggesting start_line=831 forever).
|
||||
slice_lines = [f"row-{i}-content\n" for i in range(1, 5171)]
|
||||
slice_output = "".join(slice_lines)
|
||||
offset = 830 # slice starts at absolute line 831
|
||||
result = _truncate_read_file_output(slice_output, 50000, line_offset=offset)
|
||||
head, marker = _head_and_marker(result)
|
||||
absolute_cut = offset + head.count("\n") + 1
|
||||
absolute_last = offset + len(slice_lines)
|
||||
assert f"cut lands in line {absolute_cut} of {absolute_last}" in marker
|
||||
assert f"start_line={absolute_cut}" in marker
|
||||
assert absolute_cut > offset + 1 # resume strictly advances
|
||||
|
||||
def test_ranged_read_zero_offset_keeps_full_read_semantics(self):
|
||||
output = "".join(f"line-{i}-with-padding\n" for i in range(3000))
|
||||
assert _truncate_read_file_output(output, 50000) == _truncate_read_file_output(output, 50000, line_offset=0)
|
||||
|
||||
def test_max_chars_zero_disables_truncation(self):
|
||||
output = "X" * 100000
|
||||
assert _truncate_read_file_output(output, 0) == output
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user