diff --git a/README.md b/README.md index 7afa038b3c..88dedaef90 100644 --- a/README.md +++ b/README.md @@ -2154,6 +2154,8 @@ Concurrent reads and writes to the same file share a gate across synchronous and The built-in `grep` tool searches either one text file or all matching text files below a directory, so an agent can search an uploaded document directly without first broadening the request to the entire uploads directory. E2B's `glob` filter preserves spaces, quotes, and dollar signs in filename patterns, while wildcard matching and root-relative directory scoping remain unchanged. +E2B `grep` also preserves colons in file and directory names when reporting matching paths and line numbers. +E2B `glob` and `grep` preserve non-LF separator characters, such as vertical tabs and Unicode line separators, inside returned paths; `grep` also preserves them inside matched text. Remote `ls` excludes ignored descendants before applying its 500-entry listing limit, so dependency and build trees do not crowd out visible files. Explicitly listing an ignored directory still lists its contents; normal depth and output limits remain in effect. diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md b/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md index a1cf2be2e6..6d4172bd69 100644 --- a/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md @@ -44,3 +44,15 @@ passing it to the shell. The shell must not split or expand filename patterns; grep owns basename globbing and `path_matches` owns root-relative directory scoping. `tests/test_sandbox_search_contract.py` pins this across providers using real temporary files and offline shell transports. + +Use `grep --null` to separate the filename from `line:text`, then parse the +line number only after that NUL delimiter. Colons are valid path characters, +so never split the whole record on them. Real-shell regressions in +`tests/test_e2b_sandbox_provider.py` cover colon-containing files and directories. + +Split `grep` and `glob` output records only on `"\n"`, matching +`parse_remote_search_output`. Do not use `splitlines()`: its other boundaries +(carriage returns, vertical tabs, form feeds, and control/Unicode separators) +can occur inside filenames or matched text. The real-shell tests cover these +characters in filenames, parent directories, and grep content; their transport +decodes captured bytes without universal-newline conversion. diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox.py b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox.py index 28ac559944..18caef4d94 100644 --- a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox.py +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox.py @@ -440,7 +440,8 @@ class E2BSandbox(Sandbox): matches: list[str] = [] root = resolved.rstrip("/") or "/" root_prefix = root if root == "/" else f"{root}/" - for entry in output.text.splitlines(): + # Records are LF-delimited; other splitlines() boundaries can be path characters. + for entry in output.text.split("\n"): # Do NOT strip: trailing whitespace can be part of the filename. if not entry: continue @@ -478,7 +479,8 @@ class E2BSandbox(Sandbox): # Build a portable ``grep`` invocation: # -r recursive, -n line numbers, -H always print filename, -I skip # binary files, -E extended regex (or -F for literal/fixed strings). - flags = ["-r", "-n", "-H", "-I"] + # --null separates filenames from line numbers even when a path contains colons. + flags = ["-r", "-n", "-H", "-I", "--null"] if not case_sensitive: flags.append("-i") if literal: @@ -519,9 +521,11 @@ class E2BSandbox(Sandbox): matches: list[GrepMatch] = [] truncated = output.truncated - for raw in output.text.splitlines(): + # Keep non-LF separators inside filenames and matched text intact. + for raw in output.text.split("\n"): try: - file_path, line_no_str, line_text = raw.split(":", 2) + file_path, match_text = raw.split("\0", 1) + line_no_str, line_text = match_text.split(":", 1) except ValueError: continue try: diff --git a/backend/tests/test_e2b_sandbox_provider.py b/backend/tests/test_e2b_sandbox_provider.py index 9d3e07f7a1..35b5f867a5 100644 --- a/backend/tests/test_e2b_sandbox_provider.py +++ b/backend/tests/test_e2b_sandbox_provider.py @@ -4043,7 +4043,7 @@ def _search_stdout(raw: str, *, status: int = 0) -> str: def test_grep_scoped_glob_excludes_unrelated_directory_matches(): """Regression: grep(glob="src/*.js") must not leak matches from sibling directories that merely share the file extension.""" - raw_stdout = "/home/user/workspace/other_dir/unrelated.js:1:console.log('needle in other_dir');\n/home/user/workspace/src/app.js:1:console.log('needle in src');\n" + raw_stdout = "/home/user/workspace/other_dir/unrelated.js\x001:console.log('needle in other_dir');\n/home/user/workspace/src/app.js\x001:console.log('needle in src');\n" client = FakeClient(commands=FakeCommandsAPI([SimpleNamespace(stdout=_search_stdout(raw_stdout), stderr="", exit_code=0)])) sb = _make_sandbox(client) @@ -4059,7 +4059,7 @@ def test_grep_plain_glob_matches_files_in_any_directory(): """No regression: a plain non-scoped glob (no ``/`` in the pattern) must keep matching files at any depth, same as before the directory-scoping fix.""" - raw_stdout = "/home/user/workspace/other_dir/deep/mod.py:1:needle in a deeply nested file\n/home/user/workspace/src/app.py:1:needle in a python file too\n" + raw_stdout = "/home/user/workspace/other_dir/deep/mod.py\x001:needle in a deeply nested file\n/home/user/workspace/src/app.py\x001:needle in a python file too\n" client = FakeClient(commands=FakeCommandsAPI([SimpleNamespace(stdout=_search_stdout(raw_stdout), stderr="", exit_code=0)])) sb = _make_sandbox(client) @@ -4089,7 +4089,7 @@ def test_grep_scoped_glob_still_passes_coarse_include_flag(): def test_grep_without_glob_is_unaffected(): """No regression: omitting ``glob`` entirely must return every match with no path-based post-filtering.""" - raw_stdout = "/home/user/workspace/anywhere/file.txt:3:needle here\n" + raw_stdout = "/home/user/workspace/anywhere/file.txt\x003:needle here\n" client = FakeClient(commands=FakeCommandsAPI([SimpleNamespace(stdout=_search_stdout(raw_stdout), stderr="", exit_code=0)])) sb = _make_sandbox(client) @@ -4101,7 +4101,7 @@ def test_grep_without_glob_is_unaffected(): def test_grep_single_file_path_with_matching_glob(): """A basename glob must also apply when the search root is one file.""" - raw_stdout = "/home/user/uploads/report.md:2:needle here\n" + raw_stdout = "/home/user/uploads/report.md\x002:needle here\n" client = FakeClient(commands=FakeCommandsAPI([SimpleNamespace(stdout=_search_stdout(raw_stdout), stderr="", exit_code=0)])) sb = _make_sandbox(client) @@ -5894,6 +5894,17 @@ _RS_POSIX = pytest.mark.skipif( os.name == "nt" or any(shutil.which(tool) is None for tool in ("sh", "head", "grep", "find")), reason="POSIX sh, head, grep and find required", ) +_RS_NON_LF_SEPARATORS = [ + pytest.param("\r", id="carriage-return"), + pytest.param("\v", id="vertical-tab"), + pytest.param("\f", id="form-feed"), + pytest.param("\x1c", id="file-separator"), + pytest.param("\x1d", id="group-separator"), + pytest.param("\x1e", id="record-separator"), + pytest.param("\x85", id="next-line"), + pytest.param("\u2028", id="line-separator"), + pytest.param("\u2029", id="paragraph-separator"), +] def _rs_env(tmp_path, failing: str | None = None) -> dict[str, str]: @@ -5918,8 +5929,9 @@ class _RsShellCommands: def run(self, cmd: str, envs: dict[str, str] | None = None, **kwargs) -> SimpleNamespace: self.calls.append(cmd) # ``sh -c`` (not ``-lc``) keeps a login profile from overriding the fake PATH. - proc = subprocess.run(["sh", "-c", cmd], capture_output=True, text=True, env=self._env, check=False) - return SimpleNamespace(stdout=proc.stdout, stderr=proc.stderr, exit_code=proc.returncode) + # Decode bytes ourselves so text-mode universal newlines cannot alter CR in paths/content. + proc = subprocess.run(["sh", "-c", cmd], capture_output=True, env=self._env, check=False) + return SimpleNamespace(stdout=proc.stdout.decode("utf-8"), stderr=proc.stderr.decode("utf-8"), exit_code=proc.returncode) def _rs_sandbox(tmp_path, failing: str | None = None): @@ -5930,6 +5942,66 @@ def _rs_search(sb, op: str, root: str): return sb.grep(root, "needle") if op == "grep" else sb.glob(root, "**/*.py") +@_RS_POSIX +@pytest.mark.parametrize("relative_path", ["plain.txt", "report:2026.txt", "report:2:2026.txt", "2026-10-09T13:45:00.txt", "reports:2026/result.txt"]) +@pytest.mark.parametrize("search_scope", ["directory", "single_file", "filtered_directory"]) +def test_remote_grep_preserves_colons_in_paths(tmp_path, relative_path, search_scope): + target = tmp_path / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("header\nneedle: hello:world\n", encoding="utf-8") + root = target if search_scope == "single_file" else tmp_path + glob = "*.txt" if search_scope == "filtered_directory" else None + + matches, truncated = _rs_sandbox(tmp_path).grep(str(root), "needle", glob=glob) + + assert [(match.path, match.line_number, match.line) for match in matches] == [(str(target), 2, "needle: hello:world")] + assert truncated is False + + +@_RS_POSIX +@pytest.mark.parametrize("separator", _RS_NON_LF_SEPARATORS) +@pytest.mark.parametrize("path_template", ["report{}2026.txt", "reports{}2026/result.txt"], ids=["filename", "parent-directory"]) +@pytest.mark.parametrize("search_scope", ["directory", "single_file", "filtered_directory"]) +def test_remote_grep_preserves_non_lf_separators_in_paths(tmp_path, separator, path_template, search_scope): + target = tmp_path / path_template.format(separator) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("header\nneedle: hello:world\n", encoding="utf-8") + root = target if search_scope == "single_file" else tmp_path + glob = "*.txt" if search_scope == "filtered_directory" else None + + matches, truncated = _rs_sandbox(tmp_path).grep(str(root), "needle", glob=glob) + + assert [(match.path, match.line_number, match.line) for match in matches] == [(str(target), 2, "needle: hello:world")] + assert truncated is False + + +@_RS_POSIX +@pytest.mark.parametrize("separator", _RS_NON_LF_SEPARATORS) +def test_remote_grep_preserves_non_lf_separators_in_content(tmp_path, separator): + target = tmp_path / "plain.txt" + line = f"needle: before{separator}after" + target.write_bytes(f"header\n{line}\n".encode()) + + matches, truncated = _rs_sandbox(tmp_path).grep(str(tmp_path), "needle") + + assert [(match.path, match.line_number, match.line) for match in matches] == [(str(target), 2, line)] + assert truncated is False + + +@_RS_POSIX +@pytest.mark.parametrize("separator", _RS_NON_LF_SEPARATORS) +@pytest.mark.parametrize("path_template", ["report{}2026.txt", "reports{}2026/result.txt"], ids=["filename", "parent-directory"]) +def test_remote_glob_preserves_non_lf_separators_in_paths(tmp_path, separator, path_template): + target = tmp_path / path_template.format(separator) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("content\n", encoding="utf-8") + + matches, truncated = _rs_sandbox(tmp_path).glob(str(tmp_path), "*.txt") + + assert matches == [str(target)] + assert truncated is False + + @_RS_POSIX @pytest.mark.parametrize("op", ["grep", "glob"]) def test_remote_search_missing_root_raises_file_not_found(tmp_path, op):