diff --git a/backend/packages/harness/deerflow/sandbox/remote_list_dir.py b/backend/packages/harness/deerflow/sandbox/remote_list_dir.py index e45b70f4a..367270410 100644 --- a/backend/packages/harness/deerflow/sandbox/remote_list_dir.py +++ b/backend/packages/harness/deerflow/sandbox/remote_list_dir.py @@ -16,6 +16,8 @@ from __future__ import annotations import shlex +from deerflow.sandbox.search import should_ignore_path + _STATUS_PREFIX = "__DF_FIND_STATUS__:" _MISSING_ROOT = "missing" _LIST_LIMIT = 500 @@ -59,9 +61,18 @@ def parse_remote_list_dir_output( ) -> list[str]: """Parse listing stdout, preferring the find-status marker over pipeline status. + Entries under ignored directories (``IGNORE_PATTERNS``) are dropped, matching + the local ``list_dir`` and the remote ``glob``/``grep`` implementations, which + already skip those paths. Patterns apply to the path relative to the listing + root, so an ignored name only hides that directory's *descendants*: explicitly + listing ``build`` — or a path below an ignored ancestor — still returns its + contents, as the local walk does. Filtering happens after the empty-output + check, so a directory whose entries are all ignored returns an empty list + rather than a missing-path error. + Raises: OSError: Command/client failure or an incomplete traversal. - FileNotFoundError: The root is missing or no listable entries exist. + FileNotFoundError: The root is missing or ``find`` produced no output. """ # find delimits records with "\n" only. splitlines() would also split on # \v, \f, \x1c-\x1e and \x85, which are legal in Linux filenames. Do not @@ -98,4 +109,20 @@ def parse_remote_list_dir_output( if not entries: raise FileNotFoundError(resolved) - return entries + root = resolved.rstrip("/") or "/" + prefix = "/" if root == "/" else f"{root}/" + kept: list[str] = [] + for entry in entries: + if entry.rstrip("/") == root: + # The requested root is what the caller asked for; keep it even when + # its own name matches an ignore pattern. + kept.append(entry) + elif entry.startswith(prefix): + if not should_ignore_path(entry[len(prefix) :]): + kept.append(entry) + else: + # ``find -H`` prints the resolved target when the root is a symlink, + # so an entry may not carry the requested prefix. Keep it: a path + # that cannot be placed relative to the root must not disappear. + kept.append(entry) + return kept diff --git a/backend/tests/test_remote_list_dir.py b/backend/tests/test_remote_list_dir.py index 83d9c6eb9..40aafea85 100644 --- a/backend/tests/test_remote_list_dir.py +++ b/backend/tests/test_remote_list_dir.py @@ -223,3 +223,94 @@ def test_list_dir_existing_root_with_no_output_is_incomplete_failure(tmp_path) - assert proc.returncode == 1 with pytest.raises(OSError, match="results would be incomplete"): parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode) + + +def test_parse_drops_entries_under_ignored_directories() -> None: + """A remote listing must skip the same directories remote glob/grep already skip.""" + stdout = "/root\n/root/.git\n/root/.git/config\n/root/node_modules/pkg/index.js\n/root/src\n/root/src/app.py\n\n__DF_FIND_STATUS__:0\n" + assert parse_remote_list_dir_output(stdout, "/root", pipeline_exit_code=0) == [ + "/root", + "/root/src", + "/root/src/app.py", + ] + + +def test_parse_keeps_names_that_only_resemble_ignore_patterns() -> None: + stdout = "/root/node_modules_backup/keep.txt\n/root/builds/keep.txt\n/root/src/environment.py\n\n__DF_FIND_STATUS__:0\n" + assert parse_remote_list_dir_output(stdout, "/root", pipeline_exit_code=0) == [ + "/root/node_modules_backup/keep.txt", + "/root/builds/keep.txt", + "/root/src/environment.py", + ] + + +def test_parse_all_entries_ignored_returns_empty_list_not_missing_path() -> None: + """An existing directory whose entries are all ignored is empty, not missing.""" + stdout = "/data/node_modules\n/data/node_modules/pkg/index.js\n\n__DF_FIND_STATUS__:0\n" + assert parse_remote_list_dir_output(stdout, "/data", pipeline_exit_code=0) == [] + + +def test_parse_keeps_contents_of_an_explicitly_requested_ignored_root() -> None: + """Explicitly listing a directory whose own name is ignored must not come back empty.""" + stdout = "/data/node_modules\n/data/node_modules/pkg\n/data/node_modules/pkg/index.js\n\n__DF_FIND_STATUS__:0\n" + assert parse_remote_list_dir_output(stdout, "/data/node_modules", pipeline_exit_code=0) == [ + "/data/node_modules", + "/data/node_modules/pkg", + "/data/node_modules/pkg/index.js", + ] + + +def test_parse_ignores_only_descendants_of_the_listing_root() -> None: + """An ignored name *outside* the root hides nothing; an ignored name inside it still does.""" + stdout = "/tmp/build/workspace\n/tmp/build/workspace/report.txt\n/tmp/build/workspace/nested/node_modules/dep.js\n\n__DF_FIND_STATUS__:0\n" + assert parse_remote_list_dir_output(stdout, "/tmp/build/workspace", pipeline_exit_code=0) == [ + "/tmp/build/workspace", + "/tmp/build/workspace/report.txt", + ] + stdout_env = "/srv/env/project\n/srv/env/project/src/main.py\n\n__DF_FIND_STATUS__:0\n" + assert parse_remote_list_dir_output(stdout_env, "/srv/env/project", pipeline_exit_code=0) == [ + "/srv/env/project", + "/srv/env/project/src/main.py", + ] + + +@_POSIX_SH +@pytest.mark.skipif(shutil.which("find") is None, reason="system find required") +def test_list_dir_command_drops_ignored_directories(tmp_path) -> None: + root = tmp_path / "workspace" + (root / "src").mkdir(parents=True) + (root / "src" / "app.py").write_text("print('x')", encoding="utf-8") + (root / "node_modules" / "pkg").mkdir(parents=True) + (root / "node_modules" / "pkg" / "index.js").write_text("module.exports = {}", encoding="utf-8") + (root / ".git").mkdir() + (root / ".git" / "config").write_text("[core]", encoding="utf-8") + proc = _run_list_dir_script(remote_list_dir_command(str(root), 2)) + entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode) + assert str(root / "src" / "app.py") in entries + assert not any("node_modules" in entry for entry in entries) + assert not any(entry.endswith("/.git") or "/.git/" in entry for entry in entries) + + +@_POSIX_SH +@pytest.mark.skipif(shutil.which("find") is None, reason="system find required") +def test_list_dir_command_lists_inside_an_ignored_ancestor(tmp_path) -> None: + """Real find: an ignored-looking ancestor of the root hides nothing.""" + root = tmp_path / "build" / "workspace" + root.mkdir(parents=True) + (root / "report.txt").write_text("x", encoding="utf-8") + proc = _run_list_dir_script(remote_list_dir_command(str(root), 2)) + entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode) + assert str(root / "report.txt") in entries + + +@_POSIX_SH +@pytest.mark.skipif(shutil.which("find") is None, reason="system find required") +def test_list_dir_command_lists_an_explicitly_requested_ignored_directory(tmp_path) -> None: + """Real find: `ls logs` returns the directory and its files, not emptiness.""" + root = tmp_path / "logs" + root.mkdir() + (root / "notes.txt").write_text("x", encoding="utf-8") + proc = _run_list_dir_script(remote_list_dir_command(str(root), 2)) + entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode) + assert str(root) in entries + assert str(root / "notes.txt") in entries