mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-09-23 21:16:17 +00:00
fix(sandbox): report the line a read_file truncation lands on (#5478)
* fix(sandbox): report the line a read_file truncation lands on
The read_file tool head-truncates at a character offset and tells the
model to continue with start_line/end_line, but the marker reported only
character counts ("showing first N of M chars"), so the model had no way
to know which line the cut fell on — the cut almost always lands mid-line
and read_file output carries no line numbers (#5475).
The marker now also reports the 1-indexed line holding the first hidden
character and the file's total line count, and names the exact resume
point: "... [truncated: showing first N of M chars (cut lands in line L
of T). Use start_line=L — optionally with end_line — to continue without
a gap] ...". Resuming at the reported line is gap-free whether the cut
lands mid-line or exactly after a newline.
The marker length budget accounts for the new fields, so the
len(result) <= max_chars contract still holds.
* fix(sandbox): report absolute lines in ranged-read truncation markers
Review follow-up on #5478: read_file_tool runs the same truncation on
ranged reads (start_line/end_line), where the slice's line 1 is the
requested start_line, not the file's first line. The marker's reported
lines were slice-relative while the model reasons in absolute file lines,
so the resume hint could re-issue the identical start_line forever
(repro: resume at 831 -> "cut lands in line 831 of 5170" -> start_line=831).
_truncate_read_file_output gains a line_offset parameter (the 0-based
absolute line of the slice's first line) and reports absolute lines for
both the cut position and the range end; read_file_tool threads
effective_start - 1 through. Full reads pass the default offset 0 and are
byte-identical.
Regression tests pin the absolute coordinates and that the resume point
strictly advances past the slice start.
This commit is contained in:
parent
4387dce7be
commit
49f2197ba1
@ -1840,7 +1840,7 @@ def _truncate_bash_output(output: str, max_chars: int) -> str:
|
|||||||
return f"{output[:head_len]}{marker}{output[-tail_len:] if tail_len > 0 else ''}" + preserved
|
return f"{output[:head_len]}{marker}{output[-tail_len:] if tail_len > 0 else ''}" + preserved
|
||||||
|
|
||||||
|
|
||||||
def _truncate_read_file_output(output: str, max_chars: int) -> str:
|
def _truncate_read_file_output(output: str, max_chars: int, line_offset: int = 0) -> str:
|
||||||
"""Head-truncate read_file output, preserving the beginning of the file.
|
"""Head-truncate read_file output, preserving the beginning of the file.
|
||||||
|
|
||||||
Source code and documents are read top-to-bottom; the head contains the
|
Source code and documents are read top-to-bottom; the head contains the
|
||||||
@ -1849,19 +1849,34 @@ def _truncate_read_file_output(output: str, max_chars: int) -> str:
|
|||||||
The returned string (including the truncation marker) is guaranteed to be
|
The returned string (including the truncation marker) is guaranteed to be
|
||||||
no longer than max_chars characters. Pass max_chars=0 to disable truncation
|
no longer than max_chars characters. Pass max_chars=0 to disable truncation
|
||||||
and return the full output unchanged.
|
and return the full output unchanged.
|
||||||
|
|
||||||
|
The marker reports the 1-indexed line the character cut lands in, so the
|
||||||
|
model can resume with ``start_line`` without re-reading or skipping
|
||||||
|
content (#5475). ``line_offset`` is the 0-based absolute line number of
|
||||||
|
``output``'s first line — ranged reads pass the requested ``start_line -
|
||||||
|
1`` so the reported lines are absolute file lines, keeping the resume
|
||||||
|
point from looping back onto the slice's own beginning.
|
||||||
"""
|
"""
|
||||||
if max_chars == 0:
|
if max_chars == 0:
|
||||||
return output
|
return output
|
||||||
if len(output) <= max_chars:
|
if len(output) <= max_chars:
|
||||||
return output
|
return output
|
||||||
total = len(output)
|
total = len(output)
|
||||||
# Compute the exact worst-case marker length: both numeric fields are at
|
total_lines = output.count("\n") + (0 if output.endswith("\n") else 1)
|
||||||
# their maximum (total chars), so this is a tight upper bound.
|
# Compute the exact worst-case marker length: every numeric field is at
|
||||||
marker_max_len = len(f"\n... [truncated: showing first {total} of {total} chars. Use start_line/end_line to read a specific range] ...")
|
# its maximum (total + offset), so this is a tight upper bound.
|
||||||
|
bound = total + max(line_offset, 0)
|
||||||
|
marker_max_len = len(f"\n... [truncated: showing first {total} of {total} chars (cut lands in line {bound} of {bound}). Use start_line={bound} — optionally with end_line — to continue without a gap] ...")
|
||||||
kept = max(0, max_chars - marker_max_len)
|
kept = max(0, max_chars - marker_max_len)
|
||||||
if kept == 0:
|
if kept == 0:
|
||||||
return output[:max_chars]
|
return output[:max_chars]
|
||||||
marker = f"\n... [truncated: showing first {kept} of {total} chars. Use start_line/end_line to read a specific range] ..."
|
# 1-indexed line holding the first hidden character: a cut mid-line lands
|
||||||
|
# in the partially shown line, a cut exactly after a newline lands in the
|
||||||
|
# next line — either way resuming at this line leaves no gap. Expressed in
|
||||||
|
# absolute file lines via line_offset.
|
||||||
|
cut_line = line_offset + output[:kept].count("\n") + 1
|
||||||
|
last_line = line_offset + total_lines
|
||||||
|
marker = f"\n... [truncated: showing first {kept} of {total} chars (cut lands in line {cut_line} of {last_line}). Use start_line={cut_line} — optionally with end_line — to continue without a gap] ..."
|
||||||
return f"{output[:kept]}{marker}"
|
return f"{output[:kept]}{marker}"
|
||||||
|
|
||||||
|
|
||||||
@ -2445,7 +2460,10 @@ def read_file_tool(
|
|||||||
max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
|
max_chars = sandbox_cfg.read_file_output_max_chars if sandbox_cfg else 50000
|
||||||
except Exception:
|
except Exception:
|
||||||
max_chars = 50000
|
max_chars = 50000
|
||||||
return _truncate_read_file_output(content, max_chars)
|
# Ranged reads hand a slice to the truncator; line_offset converts the
|
||||||
|
# slice-relative cut line back to the absolute file lines the tool's
|
||||||
|
# start_line/end_line arguments speak in (#5475).
|
||||||
|
return _truncate_read_file_output(content, max_chars, line_offset=effective_start - 1)
|
||||||
except SandboxError as e:
|
except SandboxError as e:
|
||||||
return f"Error: {e}"
|
return f"Error: {e}"
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
|
|||||||
@ -6,8 +6,35 @@ These functions truncate long tool outputs to prevent context window overflow.
|
|||||||
- _truncate_ls_output: head-truncation, for ls tool
|
- _truncate_ls_output: head-truncation, for ls tool
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
from deerflow.sandbox.tools import _truncate_bash_output, _truncate_ls_output, _truncate_read_file_output
|
from deerflow.sandbox.tools import _truncate_bash_output, _truncate_ls_output, _truncate_read_file_output
|
||||||
|
|
||||||
|
|
||||||
|
def _head_and_marker(result: str) -> tuple[str, str]:
|
||||||
|
"""Split a truncated read_file result into shown head and trailing marker."""
|
||||||
|
idx = result.rfind("\n... [truncated:")
|
||||||
|
assert idx != -1, "truncation marker missing"
|
||||||
|
return result[:idx], result[idx:]
|
||||||
|
|
||||||
|
|
||||||
|
def _line_containing(output: str, char_index: int) -> int:
|
||||||
|
"""Return the 0-based line index holding ``output[char_index]``.
|
||||||
|
|
||||||
|
Computed from line spans independently of the truncation marker, so the
|
||||||
|
tests verify the marker's reported line rather than restating its formula.
|
||||||
|
"""
|
||||||
|
assert 0 <= char_index < len(output)
|
||||||
|
starts = [0]
|
||||||
|
for line in output.splitlines(keepends=True):
|
||||||
|
starts.append(starts[-1] + len(line))
|
||||||
|
candidate = 0
|
||||||
|
for i, start in enumerate(starts):
|
||||||
|
if start <= char_index:
|
||||||
|
candidate = i
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# _truncate_bash_output
|
# _truncate_bash_output
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@ -186,6 +213,63 @@ class TestTruncateReadFileOutput:
|
|||||||
assert "start_line" in result
|
assert "start_line" in result
|
||||||
assert "end_line" in result
|
assert "end_line" in result
|
||||||
|
|
||||||
|
def test_marker_reports_the_line_the_cut_lands_in(self):
|
||||||
|
# Shape from #5475: a multi-line file cut mid-line, where the old
|
||||||
|
# marker gave only a character count so the model could not compute
|
||||||
|
# which line to resume from.
|
||||||
|
output = "".join(f"def fn_{i}():\n return {i}\n" for i in range(3000))
|
||||||
|
result = _truncate_read_file_output(output, 50000)
|
||||||
|
head, marker = _head_and_marker(result)
|
||||||
|
cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1))
|
||||||
|
total_lines = output.count("\n")
|
||||||
|
assert f"cut lands in line {cut_line} of {total_lines}" in marker
|
||||||
|
assert f"start_line={cut_line}" in marker
|
||||||
|
# The no-gap contract: the reported line is the one holding the first
|
||||||
|
# hidden character (computed independently from line spans).
|
||||||
|
assert _line_containing(output, len(head)) + 1 == cut_line
|
||||||
|
|
||||||
|
def test_first_hidden_character_always_belongs_to_reported_line(self):
|
||||||
|
# Whatever size the cut lands at — mid-line or exactly after a
|
||||||
|
# newline — the reported line is the one holding the first hidden
|
||||||
|
# character, so resuming at start_line=<reported> leaves no gap.
|
||||||
|
lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)]
|
||||||
|
output = "".join(lines)
|
||||||
|
for max_chars in [1000, 5000, 20000, 50000]:
|
||||||
|
result = _truncate_read_file_output(output, max_chars)
|
||||||
|
head, marker = _head_and_marker(result)
|
||||||
|
cut_line = int(re.search(r"cut lands in line (\d+) of", marker).group(1))
|
||||||
|
assert 1 <= cut_line <= len(lines)
|
||||||
|
assert _line_containing(output, len(head)) + 1 == cut_line
|
||||||
|
assert f"start_line={cut_line}" in marker
|
||||||
|
assert len(result) <= max_chars
|
||||||
|
|
||||||
|
def test_marker_reports_total_lines(self):
|
||||||
|
lines = [f"line-{i}-with-padding\n" for i in range(1, 3001)]
|
||||||
|
output = "".join(lines)
|
||||||
|
result = _truncate_read_file_output(output, 50000)
|
||||||
|
_, marker = _head_and_marker(result)
|
||||||
|
assert f"of {output.count(chr(10))}" in marker
|
||||||
|
|
||||||
|
def test_ranged_read_reports_absolute_lines(self):
|
||||||
|
# Ranged reads hand a slice to the truncator; reported lines must be
|
||||||
|
# absolute file lines or the resume hint loops back onto the slice's
|
||||||
|
# own start (the repro from the #5478 review: start_line=831 kept
|
||||||
|
# suggesting start_line=831 forever).
|
||||||
|
slice_lines = [f"row-{i}-content\n" for i in range(1, 5171)]
|
||||||
|
slice_output = "".join(slice_lines)
|
||||||
|
offset = 830 # slice starts at absolute line 831
|
||||||
|
result = _truncate_read_file_output(slice_output, 50000, line_offset=offset)
|
||||||
|
head, marker = _head_and_marker(result)
|
||||||
|
absolute_cut = offset + head.count("\n") + 1
|
||||||
|
absolute_last = offset + len(slice_lines)
|
||||||
|
assert f"cut lands in line {absolute_cut} of {absolute_last}" in marker
|
||||||
|
assert f"start_line={absolute_cut}" in marker
|
||||||
|
assert absolute_cut > offset + 1 # resume strictly advances
|
||||||
|
|
||||||
|
def test_ranged_read_zero_offset_keeps_full_read_semantics(self):
|
||||||
|
output = "".join(f"line-{i}-with-padding\n" for i in range(3000))
|
||||||
|
assert _truncate_read_file_output(output, 50000) == _truncate_read_file_output(output, 50000, line_offset=0)
|
||||||
|
|
||||||
def test_max_chars_zero_disables_truncation(self):
|
def test_max_chars_zero_disables_truncation(self):
|
||||||
output = "X" * 100000
|
output = "X" * 100000
|
||||||
assert _truncate_read_file_output(output, 0) == output
|
assert _truncate_read_file_output(output, 0) == output
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user