fix(sandbox): drop ignored directories from remote list_dir (#5612)

* fix(sandbox): drop ignored directories from remote list_dir

Remote providers listed node_modules/.git and similar entries through the
shared remote_list_dir parser, while the local list_dir and the remote
glob/grep implementations all skip them via should_ignore_path. Apply the
same rule in the shared parser so remote listings match both, filtering
after the empty-output check so an all-ignored directory returns an empty
list instead of a missing-path error.

* fix(sandbox): apply ignore patterns relative to the listing root

should_ignore_path checked every component of the absolute entry path, so an
ancestor of the listing root whose name matches a pattern (`build`, `env`,
`logs`, …) hid the root's contents: `ls /srv/build/workspace` returned []
even though the directory had files. The local walk only filters descendants
of the requested root and still returns that file.

Match patterns against the path relative to `resolved` instead, keep the
requested root itself, and keep entries that cannot be placed relative to the
root (a symlinked root is printed resolved by `find -H`) rather than dropping
them. Regression tests cover an explicitly requested ignored root, ignored
ancestors outside the root, and both cases through the real find pipeline.

backend/packages/harness/deerflow/sandbox/remote_list_dir.py

@@ -16,6 +16,8 @@ from __future__ import annotations
 
 import shlex
 
+from deerflow.sandbox.search import should_ignore_path
+
 _STATUS_PREFIX = "__DF_FIND_STATUS__:"
 _MISSING_ROOT = "missing"
 _LIST_LIMIT = 500
@@ -59,9 +61,18 @@ def parse_remote_list_dir_output(
 ) -> list[str]:
     """Parse listing stdout, preferring the find-status marker over pipeline status.
 
+    Entries under ignored directories (``IGNORE_PATTERNS``) are dropped, matching
+    the local ``list_dir`` and the remote ``glob``/``grep`` implementations, which
+    already skip those paths. Patterns apply to the path relative to the listing
+    root, so an ignored name only hides that directory's *descendants*: explicitly
+    listing ``build`` — or a path below an ignored ancestor — still returns its
+    contents, as the local walk does. Filtering happens after the empty-output
+    check, so a directory whose entries are all ignored returns an empty list
+    rather than a missing-path error.
+
     Raises:
         OSError: Command/client failure or an incomplete traversal.
-        FileNotFoundError: The root is missing or no listable entries exist.
+        FileNotFoundError: The root is missing or ``find`` produced no output.
     """
     # find delimits records with "\n" only. splitlines() would also split on
     # \v, \f, \x1c-\x1e and \x85, which are legal in Linux filenames. Do not
@@ -98,4 +109,20 @@ def parse_remote_list_dir_output(
 
     if not entries:
         raise FileNotFoundError(resolved)
-    return entries
+    root = resolved.rstrip("/") or "/"
+    prefix = "/" if root == "/" else f"{root}/"
+    kept: list[str] = []
+    for entry in entries:
+        if entry.rstrip("/") == root:
+            # The requested root is what the caller asked for; keep it even when
+            # its own name matches an ignore pattern.
+            kept.append(entry)
+        elif entry.startswith(prefix):
+            if not should_ignore_path(entry[len(prefix) :]):
+                kept.append(entry)
+        else:
+            # ``find -H`` prints the resolved target when the root is a symlink,
+            # so an entry may not carry the requested prefix. Keep it: a path
+            # that cannot be placed relative to the root must not disappear.
+            kept.append(entry)
+    return kept

backend/tests/test_remote_list_dir.py

@@ -223,3 +223,94 @@ def test_list_dir_existing_root_with_no_output_is_incomplete_failure(tmp_path) -
     assert proc.returncode == 1
     with pytest.raises(OSError, match="results would be incomplete"):
         parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
+
+
+def test_parse_drops_entries_under_ignored_directories() -> None:
+    """A remote listing must skip the same directories remote glob/grep already skip."""
+    stdout = "/root\n/root/.git\n/root/.git/config\n/root/node_modules/pkg/index.js\n/root/src\n/root/src/app.py\n\n__DF_FIND_STATUS__:0\n"
+    assert parse_remote_list_dir_output(stdout, "/root", pipeline_exit_code=0) == [
+        "/root",
+        "/root/src",
+        "/root/src/app.py",
+    ]
+
+
+def test_parse_keeps_names_that_only_resemble_ignore_patterns() -> None:
+    stdout = "/root/node_modules_backup/keep.txt\n/root/builds/keep.txt\n/root/src/environment.py\n\n__DF_FIND_STATUS__:0\n"
+    assert parse_remote_list_dir_output(stdout, "/root", pipeline_exit_code=0) == [
+        "/root/node_modules_backup/keep.txt",
+        "/root/builds/keep.txt",
+        "/root/src/environment.py",
+    ]
+
+
+def test_parse_all_entries_ignored_returns_empty_list_not_missing_path() -> None:
+    """An existing directory whose entries are all ignored is empty, not missing."""
+    stdout = "/data/node_modules\n/data/node_modules/pkg/index.js\n\n__DF_FIND_STATUS__:0\n"
+    assert parse_remote_list_dir_output(stdout, "/data", pipeline_exit_code=0) == []
+
+
+def test_parse_keeps_contents_of_an_explicitly_requested_ignored_root() -> None:
+    """Explicitly listing a directory whose own name is ignored must not come back empty."""
+    stdout = "/data/node_modules\n/data/node_modules/pkg\n/data/node_modules/pkg/index.js\n\n__DF_FIND_STATUS__:0\n"
+    assert parse_remote_list_dir_output(stdout, "/data/node_modules", pipeline_exit_code=0) == [
+        "/data/node_modules",
+        "/data/node_modules/pkg",
+        "/data/node_modules/pkg/index.js",
+    ]
+
+
+def test_parse_ignores_only_descendants_of_the_listing_root() -> None:
+    """An ignored name *outside* the root hides nothing; an ignored name inside it still does."""
+    stdout = "/tmp/build/workspace\n/tmp/build/workspace/report.txt\n/tmp/build/workspace/nested/node_modules/dep.js\n\n__DF_FIND_STATUS__:0\n"
+    assert parse_remote_list_dir_output(stdout, "/tmp/build/workspace", pipeline_exit_code=0) == [
+        "/tmp/build/workspace",
+        "/tmp/build/workspace/report.txt",
+    ]
+    stdout_env = "/srv/env/project\n/srv/env/project/src/main.py\n\n__DF_FIND_STATUS__:0\n"
+    assert parse_remote_list_dir_output(stdout_env, "/srv/env/project", pipeline_exit_code=0) == [
+        "/srv/env/project",
+        "/srv/env/project/src/main.py",
+    ]
+
+
+@_POSIX_SH
+@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
+def test_list_dir_command_drops_ignored_directories(tmp_path) -> None:
+    root = tmp_path / "workspace"
+    (root / "src").mkdir(parents=True)
+    (root / "src" / "app.py").write_text("print('x')", encoding="utf-8")
+    (root / "node_modules" / "pkg").mkdir(parents=True)
+    (root / "node_modules" / "pkg" / "index.js").write_text("module.exports = {}", encoding="utf-8")
+    (root / ".git").mkdir()
+    (root / ".git" / "config").write_text("[core]", encoding="utf-8")
+    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
+    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
+    assert str(root / "src" / "app.py") in entries
+    assert not any("node_modules" in entry for entry in entries)
+    assert not any(entry.endswith("/.git") or "/.git/" in entry for entry in entries)
+
+
+@_POSIX_SH
+@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
+def test_list_dir_command_lists_inside_an_ignored_ancestor(tmp_path) -> None:
+    """Real find: an ignored-looking ancestor of the root hides nothing."""
+    root = tmp_path / "build" / "workspace"
+    root.mkdir(parents=True)
+    (root / "report.txt").write_text("x", encoding="utf-8")
+    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
+    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
+    assert str(root / "report.txt") in entries
+
+
+@_POSIX_SH
+@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
+def test_list_dir_command_lists_an_explicitly_requested_ignored_directory(tmp_path) -> None:
+    """Real find: `ls logs` returns the directory and its files, not emptiness."""
+    root = tmp_path / "logs"
+    root.mkdir()
+    (root / "notes.txt").write_text("x", encoding="utf-8")
+    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
+    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
+    assert str(root) in entries
+    assert str(root / "notes.txt") in entries