diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d270058d5..b310885461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4346,6 +4346,15 @@ This release closes that milestone with **439 merged pull requests**. always available, so skill tool policy could not remove it. Local targets are now confined to the configured skills root and the caller's own user directory; `skill://` and `inline://` targets are unchanged. ([#6580]) +- **sandbox:** `glob` and `grep` on the BoxLite, OpenSandbox and Tenki providers frame + their records with `str.splitlines()`, which also ends a line at a bare carriage + return, form feed, vertical tab, file/group/record separator, next-line, and + U+2028/U+2029 — all legal inside Linux filenames and inside matched text. A file + named `notes\x0bdraft.txt` was therefore reported as two unrelated paths (one of + them nonexistent), and a matched line such as `const s = "a\u2028b";` came back + truncated at that character. These providers now split on `" +"` only, as the shared parser already documents and as + LocalSandbox, the AIO backend and E2B already do. ([#6595]) ### Documentation @@ -9473,3 +9482,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#6582]: https://github.com/bytedance/deer-flow/pull/6582 [#6588]: https://github.com/bytedance/deer-flow/pull/6588 [#6590]: https://github.com/bytedance/deer-flow/pull/6590 +[#6595]: https://github.com/bytedance/deer-flow/pull/6595 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index e20f236390..fd2f330681 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -3360,6 +3360,14 @@ `SKILL.md` 与 `references/` 内容。该工具始终可用，技能工具策略也无法移除它。 现在本地目标仅限于配置的技能根目录和调用者自己的用户目录；`skill://` 与 `inline://` 目标不受影响。([#6580]) +- **沙箱：** BoxLite、OpenSandbox、Tenki 三个提供者的 `glob` 与 `grep` 用 + `str.splitlines()` 切分记录，而该函数还会在裸回车、换页符、垂直制表符、 + 文件/组分/记录分隔符、下一行符以及 U+2028/U+2029 处断行——这些字符在 Linux + 文件名与被匹配文本中都是合法内容。因此名为 `notes\x0bdraft.txt` 的文件会被 + 报告成两条互不相关的路径（其中一条并不存在），而形如 `const s = "a\u2028b";` + 的匹配行会在该字符处被截断。现在这三个提供者只按 `" +"` 切分，与共享解析器 + 既有的约定以及 LocalSandbox、AIO 后端、E2B 的行为一致。([#6595]) ### 文档 @@ -7630,3 +7638,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写，核心包含子 [#6582]: https://github.com/bytedance/deer-flow/pull/6582 [#6588]: https://github.com/bytedance/deer-flow/pull/6588 [#6590]: https://github.com/bytedance/deer-flow/pull/6590 +[#6595]: https://github.com/bytedance/deer-flow/pull/6595 diff --git a/README.md b/README.md index 08376e08f7..1da22d3187 100644 --- a/README.md +++ b/README.md @@ -2155,7 +2155,7 @@ Concurrent reads and writes to the same file share a gate across synchronous and The built-in `grep` tool searches either one text file or all matching text files below a directory, so an agent can search an uploaded document directly without first broadening the request to the entire uploads directory. E2B's `glob` filter preserves spaces, quotes, and dollar signs in filename patterns, while wildcard matching and root-relative directory scoping remain unchanged. E2B `grep` also preserves colons in file and directory names when reporting matching paths and line numbers. -E2B `glob` and `grep` preserve non-LF separator characters, such as vertical tabs and Unicode line separators, inside returned paths; `grep` also preserves them inside matched text. +`glob` and `grep` preserve non-LF separator characters, such as vertical tabs and Unicode line separators, inside returned paths on E2B, BoxLite, Tenki, and OpenSandbox; `grep` also preserves them inside matched text. Remote `ls` excludes ignored descendants before applying its 500-entry listing limit, so dependency and build trees do not crowd out visible files. Explicitly listing an ignored directory still lists its contents; normal depth and output limits remain in effect. diff --git a/backend/packages/harness/deerflow/community/boxlite/box.py b/backend/packages/harness/deerflow/community/boxlite/box.py index 274d4e8ac2..f912e6cfaf 100644 --- a/backend/packages/harness/deerflow/community/boxlite/box.py +++ b/backend/packages/harness/deerflow/community/boxlite/box.py @@ -319,7 +319,8 @@ class BoxliteBox(Sandbox): matches: list[str] = [] root = resolved.rstrip("/") or "/" root_prefix = root if root == "/" else f"{root}/" - for entry in output.text.splitlines(): + # Records are LF-delimited; other splitlines() boundaries can be path characters. + for entry in output.text.split("\n"): # Do NOT strip: trailing whitespace can be part of the filename. if not entry or (entry != root and not entry.startswith(root_prefix)): continue @@ -373,7 +374,8 @@ class BoxliteBox(Sandbox): root_prefix = root if root == "/" else f"{root}/" matches: list[GrepMatch] = [] truncated = output.truncated - for raw in output.text.splitlines(): + # Keep non-LF separators inside filenames and matched text intact. + for raw in output.text.split("\n"): try: file_path, line_no_str, line_text = raw.split(":", 2) except ValueError: diff --git a/backend/packages/harness/deerflow/community/opensandbox/sandbox.py b/backend/packages/harness/deerflow/community/opensandbox/sandbox.py index 0279d24118..7866342449 100644 --- a/backend/packages/harness/deerflow/community/opensandbox/sandbox.py +++ b/backend/packages/harness/deerflow/community/opensandbox/sandbox.py @@ -356,7 +356,8 @@ class OpenSandboxSandbox(Sandbox): matches: list[str] = [] root = resolved.rstrip("/") or "/" root_prefix = root if root == "/" else f"{root}/" - for entry in output.text.splitlines(): + # Records are LF-delimited; other splitlines() boundaries can be path characters. + for entry in output.text.split("\n"): # Do NOT strip: trailing whitespace can be part of the filename. if not entry or (entry != root and not entry.startswith(root_prefix)) or should_ignore_path_under_root(entry, root): continue @@ -411,7 +412,8 @@ class OpenSandboxSandbox(Sandbox): root_prefix = root if root == "/" else f"{root}/" matches: list[GrepMatch] = [] seen_positions: set[tuple[str, int]] = set() - for raw in output.text.splitlines(): + # Keep non-LF separators inside filenames and matched text intact. + for raw in output.text.split("\n"): try: file_path, line_number_text, line = raw.split(":", 2) line_number = int(line_number_text) diff --git a/backend/packages/harness/deerflow/community/tenki/sandbox.py b/backend/packages/harness/deerflow/community/tenki/sandbox.py index bcd8c2d575..35ec3d0306 100644 --- a/backend/packages/harness/deerflow/community/tenki/sandbox.py +++ b/backend/packages/harness/deerflow/community/tenki/sandbox.py @@ -417,7 +417,8 @@ class TenkiSandbox(Sandbox): matches: list[str] = [] root = resolved.rstrip("/") or "/" root_prefix = root if root == "/" else f"{root}/" - for entry in output.text.splitlines(): + # Records are LF-delimited; other splitlines() boundaries can be path characters. + for entry in output.text.split("\n"): # Do NOT strip: trailing whitespace can be part of the filename. if not entry or (entry != root and not entry.startswith(root_prefix)): continue @@ -473,7 +474,8 @@ class TenkiSandbox(Sandbox): root_prefix = root if root == "/" else f"{root}/" matches: list[GrepMatch] = [] truncated = output.truncated - for raw in output.text.splitlines(): + # Keep non-LF separators inside filenames and matched text intact. + for raw in output.text.split("\n"): try: file_path, line_no_str, line_text = raw.split(":", 2) except ValueError: diff --git a/backend/tests/test_remote_search_record_framing.py b/backend/tests/test_remote_search_record_framing.py new file mode 100644 index 0000000000..8c2a457c21 --- /dev/null +++ b/backend/tests/test_remote_search_record_framing.py @@ -0,0 +1,287 @@ +"""Remote ``grep``/``glob`` record framing on BoxLite, OpenSandbox and Tenki. + +``parse_remote_search_output`` hands callers LF-joined records and documents the +rule: split on ``"\\n"`` only, because ``str.splitlines()`` also breaks on a bare +``"\\r"``, ``\\f``, ``\\v``, ``\\x1c``-``\\x1e``, ``\\x85``, ``\\u2028`` and +``\\u2029`` -- every one of which is legal inside a Linux filename and inside +matched text. E2B was moved onto that rule in #6573; these three providers kept +the ``splitlines()`` framing, so a file such as ``notes\\x0bdraft.txt`` was +reported as two unrelated paths (one of them nonexistent) and a matched line +containing U+2028 came back truncated at that character. + +The canned-output tests pin the framing directly and run on any host. The +real-shell tests build the fixture through a POSIX shell (a Windows host cannot +create ``\\x0b`` in a name at all) and are skipped where the shell tools are +missing. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from deerflow.community.boxlite.box import BoxliteBox +from deerflow.community.opensandbox.sandbox import OpenSandboxSandbox +from deerflow.community.tenki.sandbox import TenkiSandbox + +_ROOT = "/mnt/user-data/workspace" + +# Characters that end a line for ``str.splitlines()`` but are legal in a Linux +# filename and inside grep output. +NON_LF_SEPARATORS = [ + pytest.param("\r", id="carriage-return"), + pytest.param("\v", id="vertical-tab"), + pytest.param("\f", id="form-feed"), + pytest.param("\x1c", id="file-separator"), + pytest.param("\x1d", id="group-separator"), + pytest.param("\x1e", id="record-separator"), + pytest.param("\x85", id="next-line"), + pytest.param("\u2028", id="line-separator"), + pytest.param("\u2029", id="paragraph-separator"), +] + +_RS_POSIX = pytest.mark.skipif( + os.name == "nt" or any(shutil.which(tool) is None for tool in ("sh", "head", "grep", "find")), + reason="POSIX sh, head, grep and find required", +) + + +def _find_stdout(*entries: str) -> str: + """A ``find`` result in the framing ``parse_remote_search_output`` expects.""" + return "".join(f"{entry}\n" for entry in entries) + "\n__DF_SEARCH_STATUS__:0\n" + + +def _grep_stdout(*records: str) -> str: + return "".join(f"{record}\n" for record in records) + "\n__DF_SEARCH_STATUS__:0\n" + + +class _BoxliteTransport: + """BoxLite's ``SimpleBox`` seam returning one canned result.""" + + def __init__(self, stdout: str) -> None: + self._stdout = stdout + + async def exec(self, *argv, env=None, timeout=None): + return SimpleNamespace(stdout=self._stdout, stderr="", exit_code=0) + + +def _boxlite(stdout: str) -> BoxliteBox: + """BoxLite with its shell seam replaced, as the provider's own tests do. + + Patching ``_sh`` (rather than injecting a fake ``SimpleBox``) keeps the + production command string, the status marker and the record framing on the + real path; only the transport is canned. + """ + + def sh(script: str, env=None, timeout=None): + return SimpleNamespace(stdout=stdout, stderr="", exit_code=0) + + box = BoxliteBox("box-id", box=_BoxliteTransport(stdout), run=lambda coro, timeout=None: coro) + box._sh = sh + return box + + +class _OpenSandboxRemote: + """OpenSandbox's remote handle; the command transport is patched below.""" + + def __init__(self) -> None: + self.id = "remote" + + +class _OpenSandboxEvent: + def __init__(self, text: str) -> None: + self.text = text + + +class _OpenSandboxLogs: + def __init__(self, stdout: str) -> None: + self.stdout = [_OpenSandboxEvent(stdout)] + self.stderr: list[_OpenSandboxEvent] = [] + + +class _OpenSandboxExecution: + """The ``logs.stdout[*].text`` shape ``execution_stdout`` reads.""" + + def __init__(self, stdout: str) -> None: + self.exit_code = 0 + self.logs = _OpenSandboxLogs(stdout) + self.result: list[_OpenSandboxEvent] = [] + self.error = None + + +def _opensandbox(stdout: str) -> OpenSandboxSandbox: + box = OpenSandboxSandbox("sandbox-id", _OpenSandboxRemote(), run_command_opts_cls=lambda **kwargs: None) + box._run = lambda command, env=None, timeout=None: _OpenSandboxExecution(stdout) + return box + + +class _TenkiSandbox: + """Tenki's live-sandbox handle; ``_sh`` is patched below.""" + + def __init__(self) -> None: + self.id = "tenki-remote" + + +# Tenki rewrites the DeerFlow virtual prefix into its writable home dir, so its +# search output (and the paths it reports back through ``_virtual_path``) carry +# the home-dir form rather than ``/mnt/user-data``. +_TENKI_HOME = "/home/tenki" + + +def _tenki(stdout: str) -> TenkiSandbox: + box = TenkiSandbox("sb", _TenkiSandbox(), home_dir=_TENKI_HOME) + + def sh(script, env=None, timeout=None): + return SimpleNamespace(stdout=stdout, stderr="", stdout_text=stdout, stderr_text="", exit_code=0) + + box._sh = sh + return box + + +_PROVIDER_FACTORIES = [ + pytest.param(_boxlite, id="boxlite"), + pytest.param(_opensandbox, id="opensandbox"), + pytest.param(_tenki, id="tenki"), +] + + +def _search_root(factory) -> str: + """The sandbox-side root each provider actually searches. + + Tenki rewrites the DeerFlow virtual prefix into its writable home dir, so a + search for ``/mnt/user-data/workspace`` reaches ``/home/tenki/workspace``. + """ + return f"{_TENKI_HOME}/workspace" if factory is _tenki else _ROOT + + +def _reported_path(name: str) -> str: + """The path every provider reports back for ``name``. + + Tenki reverses its home-dir rewrite through ``_virtual_path``, and the other + two report the DeerFlow path verbatim, so all three answer under the prefix. + """ + return f"{_ROOT}/{name}" + + +@pytest.mark.parametrize("factory", _PROVIDER_FACTORIES) +@pytest.mark.parametrize("separator", NON_LF_SEPARATORS) +def test_glob_keeps_non_lf_separators_in_paths(factory, separator: str) -> None: + """A path holding a non-LF separator must come back as one intact entry.""" + root = _search_root(factory) + box = factory(_find_stdout(f"{root}/notes{separator}draft.txt", f"{root}/plain.txt")) + + found, truncated = box.glob(_ROOT, "**/*.txt") + + assert found == [_reported_path(f"notes{separator}draft.txt"), _reported_path("plain.txt")] + assert truncated is False + + +@pytest.mark.parametrize("factory", _PROVIDER_FACTORIES) +@pytest.mark.parametrize("separator", NON_LF_SEPARATORS) +def test_grep_keeps_non_lf_separators_in_paths(factory, separator: str) -> None: + """A matched file whose path holds a non-LF separator is reported intact.""" + root = _search_root(factory) + box = factory(_grep_stdout(f"{root}/notes{separator}draft.txt:1:needle here")) + + matches, truncated = box.grep(_ROOT, "needle") + + expected = _reported_path(f"notes{separator}draft.txt") + assert [(match.path, match.line_number, match.line) for match in matches] == [(expected, 1, "needle here")] + assert truncated is False + + +@pytest.mark.parametrize("factory", _PROVIDER_FACTORIES) +@pytest.mark.parametrize("separator", NON_LF_SEPARATORS) +def test_grep_keeps_non_lf_separators_in_matched_text(factory, separator: str) -> None: + """A matched line must not be cut at a separator that is legal in content.""" + line = f"needle: before{separator}after" + box = factory(_grep_stdout(f"{_search_root(factory)}/plain.txt:2:{line}")) + + matches, _truncated = box.grep(_ROOT, "needle") + + assert len(matches) == 1 + assert matches[0].line == line + + +@pytest.mark.parametrize("factory", _PROVIDER_FACTORIES) +def test_glob_still_reads_plain_lf_records(factory) -> None: + """No regression: ordinary LF-delimited records keep parsing as before.""" + root = _search_root(factory) + box = factory(_find_stdout(f"{root}/a.txt", f"{root}/b.txt")) + + found, truncated = box.glob(_ROOT, "**/*.txt") + + assert found == [_reported_path("a.txt"), _reported_path("b.txt")] + assert truncated is False + + +@pytest.mark.parametrize("factory", _PROVIDER_FACTORIES) +def test_grep_empty_result_is_reported_as_no_matches(factory) -> None: + """No regression: an empty result set stays ``([], False)``.""" + assert factory(_grep_stdout()).grep(_ROOT, "needle") == ([], False) + + +def _rs_shell(env: dict[str, str]): + def sh(script: str, env_=None, timeout=None): + proc = subprocess.run(["sh", "-c", script], capture_output=True, text=True, env=env, check=False) + return SimpleNamespace(stdout=proc.stdout, stderr=proc.stderr, stdout_text=proc.stdout, stderr_text=proc.stderr, exit_code=proc.returncode) + + return sh + + +def _rs_box() -> BoxliteBox: + box = BoxliteBox("box-id", box=object(), run=lambda coro, timeout=None: coro) + box._sh = _rs_shell(os.environ.copy()) + return box + + +@_RS_POSIX +@pytest.mark.parametrize("separator", ["\v", "\u2028"]) +def test_real_shell_glob_keeps_separator_named_file(tmp_path: Path, separator: str) -> None: + """End-to-end through a real ``find``: such a name stays one entry. + + The fixture is created by the shell because Windows cannot create a name + containing ``\\x0b`` at all. + + ``printf`` converts octal escapes in its *format* string, so the separator is + emitted as one octal escape per UTF-8 byte (``\\342\\200\\250`` for U+2028). + ``%03o`` on a code point above 0xFF would emit ``\\20050``, which printf reads + as ``\\200`` plus the literal digits ``50``. + """ + if ord(separator) <= 0xFF: + octal = f"\\{ord(separator):03o}" + else: + octal = "".join(f"\\{byte:03o}" for byte in separator.encode("utf-8")) + script = rf"mkdir -p {tmp_path} && printf 'x\n' > {tmp_path}/notes$(printf '{octal}')draft.txt" + subprocess.run(["sh", "-c", script], capture_output=True, check=True, env=os.environ.copy()) + listing = subprocess.run(["sh", "-c", f"ls {tmp_path}"], capture_output=True, text=True, env=os.environ.copy()).stdout.strip() + target = f"{tmp_path}/{listing}" + + found, truncated = _rs_box().glob(str(tmp_path), "**/*.txt") + + assert found == [target] + assert truncated is False + + +@_RS_POSIX +def test_real_shell_grep_keeps_separator_in_matched_text(tmp_path: Path) -> None: + """End-to-end through a real ``grep``: matched text is not cut at U+2028. + + The escape belongs in ``printf``'s *format* string: ``%s`` does not interpret + backslash escapes, and the payload's own double quotes would otherwise close + the shell word and be eaten. The octals are assembled from the code point's + UTF-8 bytes so Python never decodes them into literal characters. + """ + escapes = "".join(f"\\{byte:03o}" for byte in "\u2028".encode("utf-8")) + script = f"mkdir -p {tmp_path} && printf 'const s = \"a{escapes}b\";\\n' > {tmp_path}/mod.js" + subprocess.run(["sh", "-c", script], capture_output=True, check=True, env=os.environ.copy()) + + matches, _truncated = _rs_box().grep(str(tmp_path), "const") + + assert len(matches) == 1 + assert matches[0].line == 'const s = "a\u2028b";'