deer-flow/backend/tests/test_remote_list_dir.py

"""Remote list_dir command/parser contract, including a real POSIX sh pipeline."""

from __future__ import annotations

import os
import shlex
import shutil
import stat
import subprocess

import pytest

from deerflow.sandbox.remote_list_dir import parse_remote_list_dir_output, remote_list_dir_command

_POSIX_SH = pytest.mark.skipif(
    os.name == "nt" or shutil.which("sh") is None or shutil.which("head") is None,
    reason="POSIX sh pipeline required",
)


def test_parse_marker_127_is_command_failure_not_missing_path() -> None:
    stdout = "\n__DF_FIND_STATUS__:127\n"
    with pytest.raises(OSError, match="exited with code 127"):
        parse_remote_list_dir_output(stdout, "/dir", pipeline_exit_code=0)


def test_parse_missing_marker_is_missing_path() -> None:
    stdout = "__DF_FIND_STATUS__:missing\n"
    with pytest.raises(FileNotFoundError):
        parse_remote_list_dir_output(stdout, "/dir", pipeline_exit_code=0)


def test_parse_marker_1_empty_is_incomplete_failure() -> None:
    with pytest.raises(OSError, match="results would be incomplete"):
        parse_remote_list_dir_output("\n__DF_FIND_STATUS__:1\n", "/dir", pipeline_exit_code=1)


def test_parse_marker_1_with_entries_is_incomplete_failure() -> None:
    stdout = "/dir/visible.txt\n\n__DF_FIND_STATUS__:1\n"
    with pytest.raises(OSError, match="results would be incomplete"):
        parse_remote_list_dir_output(stdout, "/dir", pipeline_exit_code=1)


def test_parse_marker_0_returns_listing_and_keeps_trailing_space() -> None:
    stdout = "/dir/notes.txt \n/dir/sub\n\n__DF_FIND_STATUS__:0\n"
    assert parse_remote_list_dir_output(stdout, "/dir", pipeline_exit_code=0) == [
        "/dir/notes.txt ",
        "/dir/sub",
    ]


def test_parse_sigpipe_truncated_listing_is_success() -> None:
    lines = "\n".join(f"/dir/f{i}" for i in range(500))
    stdout = f"{lines}\n\n__DF_FIND_STATUS__:141\n"
    entries = parse_remote_list_dir_output(stdout, "/dir", pipeline_exit_code=0)
    assert len(entries) == 500
    assert entries[0] == "/dir/f0"
    assert entries[-1] == "/dir/f499"


def test_parse_falls_back_to_pipeline_exit_without_marker() -> None:
    with pytest.raises(OSError, match="exited with code 127"):
        parse_remote_list_dir_output("", "/dir", pipeline_exit_code=127)
    with pytest.raises(OSError, match="marker missing"):
        parse_remote_list_dir_output("", "/dir", pipeline_exit_code=0)
    with pytest.raises(OSError, match="marker missing"):
        parse_remote_list_dir_output("/dir\n", "/dir", pipeline_exit_code=0)


@_POSIX_SH
def test_parse_without_marker_real_subprocess_status_is_not_always_ok() -> None:
    """``rm -f`` exits 0/1. A process status without a marker must not become FileNotFoundError."""
    for script in ("exit 0", "exit 1"):
        proc = subprocess.run(["sh", "-c", script], capture_output=True, text=True, check=False)
        with pytest.raises(OSError, match="marker missing"):
            parse_remote_list_dir_output(
                proc.stdout,
                "/dir",
                pipeline_exit_code=proc.returncode,
            )


def test_command_records_find_status_after_head() -> None:
    command = remote_list_dir_command("/mnt/acp-workspace", 2)
    assert command.startswith("set +e; ")
    assert "find -H " in command
    assert "\\( -type f -o -type d \\)" in command
    assert "head -n 500" in command
    assert "__DF_FIND_STATUS__:" in command
    assert command.index("find -H ") < command.index("head -n")
    assert command.index("head -n") < command.rindex("__DF_FIND_STATUS__:")
    assert 'exit "${st:-126}"' in command
    # Mirror pin (review on PR #5546): the exit must stay subshell-wrapped — a
    # bare top-level exit wedges the implicit persistent session. A silent
    # revert of the subshell wrap must fail this test.
    assert command.endswith('exit "${st:-126}" )')


def _run_list_dir_script(command: str, *, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
    # Providers invoke ``sh -lc``; tests use ``sh -c`` so an injected fake
    # ``find`` on PATH is not overwritten by a login profile.
    return subprocess.run(
        ["sh", "-c", command],
        capture_output=True,
        text=True,
        env=env,
        check=False,
    )


def _env_with_bin(bin_dir: str) -> dict[str, str]:
    env = os.environ.copy()
    env["PATH"] = bin_dir + os.pathsep + env.get("PATH", "")
    return env


@_POSIX_SH
def test_naive_find_head_pipeline_hides_find_127(tmp_path) -> None:
    """Reproduction: without capturing find's status, head's 0 wins."""
    fake_bin = tmp_path / "bin"
    fake_bin.mkdir()
    fake_find = fake_bin / "find"
    fake_find.write_text("#!/bin/sh\nexit 127\n", encoding="utf-8")
    fake_find.chmod(fake_find.stat().st_mode | stat.S_IEXEC)

    naive = "find -H /dir -maxdepth 2 \\( -type f -o -type d \\) 2>/dev/null | head -n 500"
    proc = _run_list_dir_script(naive, env=_env_with_bin(str(fake_bin)))
    assert proc.returncode == 0
    assert proc.stdout == ""


def _write_fake_find(tmp_path, script: str):
    fake_bin = tmp_path / "bin"
    fake_bin.mkdir()
    fake_find = fake_bin / "find"
    fake_find.write_text(script, encoding="utf-8")
    fake_find.chmod(fake_find.stat().st_mode | stat.S_IEXEC)
    return fake_bin


@_POSIX_SH
def test_list_dir_command_surfaces_find_127_not_head_0(tmp_path) -> None:
    fake_bin = _write_fake_find(tmp_path, "#!/bin/sh\nexit 127\n")
    proc = _run_list_dir_script(
        remote_list_dir_command(str(tmp_path), 2),
        env=_env_with_bin(str(fake_bin)),
    )
    assert proc.returncode == 127
    with pytest.raises(OSError, match="exited with code 127"):
        parse_remote_list_dir_output(proc.stdout, "/dir", pipeline_exit_code=proc.returncode)


@_POSIX_SH
def test_list_dir_command_surfaces_partial_find_failure(tmp_path) -> None:
    fake_bin = _write_fake_find(tmp_path, '#!/bin/sh\nprintf "/dir/visible.txt\\n"\nexit 1\n')
    proc = _run_list_dir_script(
        remote_list_dir_command(str(tmp_path), 2),
        env=_env_with_bin(str(fake_bin)),
    )
    assert proc.returncode == 1
    with pytest.raises(OSError, match="results would be incomplete"):
        parse_remote_list_dir_output(proc.stdout, "/dir", pipeline_exit_code=proc.returncode)


@_POSIX_SH
def test_list_dir_command_records_find_127_under_set_e(tmp_path) -> None:
    fake_bin = _write_fake_find(tmp_path, "#!/bin/sh\nexit 127\n")
    proc = _run_list_dir_script(
        "set -e; " + remote_list_dir_command(str(tmp_path), 2),
        env=_env_with_bin(str(fake_bin)),
    )
    assert proc.returncode == 127
    with pytest.raises(OSError, match="exited with code 127"):
        parse_remote_list_dir_output(proc.stdout, "/dir", pipeline_exit_code=proc.returncode)


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_list_dir_command_missing_path_is_file_not_found(tmp_path) -> None:
    missing = tmp_path / "no-such-dir"
    proc = _run_list_dir_script(remote_list_dir_command(str(missing), 2))
    assert proc.stdout.strip() == "__DF_FIND_STATUS__:missing"
    with pytest.raises(FileNotFoundError):
        parse_remote_list_dir_output(proc.stdout, str(missing), pipeline_exit_code=proc.returncode)


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_list_dir_command_empty_dir_lists_the_start_point(tmp_path) -> None:
    empty = tmp_path / "empty"
    empty.mkdir()
    proc = _run_list_dir_script(remote_list_dir_command(str(empty), 2))
    entries = parse_remote_list_dir_output(proc.stdout, str(empty), pipeline_exit_code=proc.returncode)
    assert str(empty) in entries


@_POSIX_SH
def test_list_dir_command_head_truncation_is_not_an_error(tmp_path) -> None:
    fake_bin = tmp_path / "bin"
    fake_bin.mkdir()
    fake_find = fake_bin / "find"
    fake_find.write_text(
        '#!/bin/sh\ni=1\nwhile [ "$i" -le 5000 ]; do\n  echo "/dir/f$i"\n  i=$((i+1))\ndone\nexit 0\n',
        encoding="utf-8",
    )
    fake_find.chmod(fake_find.stat().st_mode | stat.S_IEXEC)

    proc = _run_list_dir_script(
        remote_list_dir_command(str(tmp_path), 2),
        env=_env_with_bin(str(fake_bin)),
    )
    entries = parse_remote_list_dir_output(proc.stdout, "/dir", pipeline_exit_code=proc.returncode)
    assert len(entries) == 500
    # The explicitly emitted root shares the same 500-entry output budget.
    assert entries[0] == str(tmp_path)
    assert entries[1] == "/dir/f1"
    assert entries[-1] == "/dir/f499"


@_POSIX_SH
def test_list_dir_existing_root_with_no_output_is_incomplete_failure(tmp_path) -> None:
    root = tmp_path / "unreadable 'directory"
    root.mkdir()
    fake_bin = _write_fake_find(tmp_path, "#!/bin/sh\nexit 1\n")
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2), env=_env_with_bin(str(fake_bin)))
    assert proc.returncode == 1
    with pytest.raises(OSError, match="results would be incomplete"):
        parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)


def test_parse_drops_entries_under_ignored_directories() -> None:
    """A remote listing must skip the same directories remote glob/grep already skip."""
    stdout = "/root\n/root/.git\n/root/.git/config\n/root/node_modules/pkg/index.js\n/root/src\n/root/src/app.py\n\n__DF_FIND_STATUS__:0\n"
    assert parse_remote_list_dir_output(stdout, "/root", pipeline_exit_code=0) == [
        "/root",
        "/root/src",
        "/root/src/app.py",
    ]


def test_parse_keeps_names_that_only_resemble_ignore_patterns() -> None:
    stdout = "/root/node_modules_backup/keep.txt\n/root/builds/keep.txt\n/root/src/environment.py\n\n__DF_FIND_STATUS__:0\n"
    assert parse_remote_list_dir_output(stdout, "/root", pipeline_exit_code=0) == [
        "/root/node_modules_backup/keep.txt",
        "/root/builds/keep.txt",
        "/root/src/environment.py",
    ]


def test_parse_all_entries_ignored_returns_empty_list_not_missing_path() -> None:
    """An existing directory whose entries are all ignored is empty, not missing."""
    stdout = "/data/node_modules\n/data/node_modules/pkg/index.js\n\n__DF_FIND_STATUS__:0\n"
    assert parse_remote_list_dir_output(stdout, "/data", pipeline_exit_code=0) == []


def test_parse_keeps_contents_of_an_explicitly_requested_ignored_root() -> None:
    """Explicitly listing a directory whose own name is ignored must not come back empty."""
    stdout = "/data/node_modules\n/data/node_modules/pkg\n/data/node_modules/pkg/index.js\n\n__DF_FIND_STATUS__:0\n"
    assert parse_remote_list_dir_output(stdout, "/data/node_modules", pipeline_exit_code=0) == [
        "/data/node_modules",
        "/data/node_modules/pkg",
        "/data/node_modules/pkg/index.js",
    ]


def test_parse_ignores_only_descendants_of_the_listing_root() -> None:
    """An ignored name *outside* the root hides nothing; an ignored name inside it still does."""
    stdout = "/tmp/build/workspace\n/tmp/build/workspace/report.txt\n/tmp/build/workspace/nested/node_modules/dep.js\n\n__DF_FIND_STATUS__:0\n"
    assert parse_remote_list_dir_output(stdout, "/tmp/build/workspace", pipeline_exit_code=0) == [
        "/tmp/build/workspace",
        "/tmp/build/workspace/report.txt",
    ]
    stdout_env = "/srv/env/project\n/srv/env/project/src/main.py\n\n__DF_FIND_STATUS__:0\n"
    assert parse_remote_list_dir_output(stdout_env, "/srv/env/project", pipeline_exit_code=0) == [
        "/srv/env/project",
        "/srv/env/project/src/main.py",
    ]


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_list_dir_command_drops_ignored_directories(tmp_path) -> None:
    root = tmp_path / "workspace"
    (root / "src").mkdir(parents=True)
    (root / "src" / "app.py").write_text("print('x')", encoding="utf-8")
    (root / "node_modules" / "pkg").mkdir(parents=True)
    (root / "node_modules" / "pkg" / "index.js").write_text("module.exports = {}", encoding="utf-8")
    (root / ".git").mkdir()
    (root / ".git" / "config").write_text("[core]", encoding="utf-8")
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
    assert str(root / "src" / "app.py") in entries
    assert not any("node_modules" in entry for entry in entries)
    assert not any(entry.endswith("/.git") or "/.git/" in entry for entry in entries)


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_list_dir_command_lists_inside_an_ignored_ancestor(tmp_path) -> None:
    """Real find: an ignored-looking ancestor of the root hides nothing."""
    root = tmp_path / "build" / "workspace"
    root.mkdir(parents=True)
    (root / "report.txt").write_text("x", encoding="utf-8")
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
    assert str(root / "report.txt") in entries


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_list_dir_command_lists_an_explicitly_requested_ignored_directory(tmp_path) -> None:
    """Real find: `ls logs` returns the directory and its files, not emptiness."""
    root = tmp_path / "logs"
    root.mkdir()
    (root / "notes.txt").write_text("x", encoding="utf-8")
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
    assert str(root) in entries
    assert str(root / "notes.txt") in entries


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None or shutil.which("sort") is None, reason="system find and sort required")
@pytest.mark.parametrize("root_name", ["workspace", "build", "project [one]'s"])
@pytest.mark.parametrize("ignored_kind", ["directory", "files"])
def test_ignored_entries_do_not_consume_listing_budget(tmp_path, root_name, ignored_kind) -> None:
    """A deterministic real-find order puts ignored entries before the useful file."""
    root = tmp_path / root_name
    root.mkdir()
    if ignored_kind == "directory":
        ignored = root / "node_modules"
        ignored.mkdir()
        for index in range(600):
            (ignored / f"dependency_{index:04}.js").touch()
    else:
        for index in range(600):
            (root / f"ignored_{index:04}.log").touch()
    visible = root / "zz_report.txt"
    visible.write_text("report", encoding="utf-8")
    # Only enumeration order is normalized. The real find still evaluates the
    # production arguments and traversal/pruning expression against real files.
    find = shlex.quote(shutil.which("find"))
    sort = shlex.quote(shutil.which("sort"))
    fake_bin = _write_fake_find(tmp_path, f'#!/bin/sh\n{find} "$@" | LC_ALL=C {sort}\n')
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2), env=_env_with_bin(str(fake_bin)))
    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
    assert entries == [str(root), str(visible)]


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_visible_listing_still_obeys_depth_and_output_limit(tmp_path) -> None:
    root = tmp_path / "workspace"
    root.mkdir()
    for index in range(8):
        (root / f"visible_{index}.txt").touch()
    (root / "nested" / "too-deep").mkdir(parents=True)
    (root / "nested" / "too-deep" / "report.txt").touch()
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 1, limit=4))
    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
    assert len(entries) == 4
    assert entries[0] == str(root)
    assert all("too-deep" not in entry for entry in entries)


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_pruning_preserves_an_ignored_symlinked_root(tmp_path) -> None:
    target = tmp_path / "actual"
    target.mkdir()
    (target / "report.txt").touch()
    (target / "node_modules").mkdir()
    (target / "node_modules" / "dependency.js").touch()
    root = tmp_path / "build"
    root.symlink_to(target, target_is_directory=True)
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
    assert entries == [str(root), str(root / "report.txt")]


@_POSIX_SH
@pytest.mark.skipif(shutil.which("find") is None, reason="system find required")
def test_pruning_uses_the_parsers_case_policy(tmp_path, monkeypatch) -> None:
    root = tmp_path / "workspace"
    (root / "BUILD").mkdir(parents=True)
    (root / "BUILD" / "ignored.txt").touch()
    (root / "report.txt").touch()
    # Exercise the policy a Windows Gateway applies to a POSIX remote sandbox.
    monkeypatch.setattr(os.path, "normcase", lambda value: value.lower())
    proc = _run_list_dir_script(remote_list_dir_command(str(root), 2))
    assert "BUILD" not in proc.stdout
    entries = parse_remote_list_dir_output(proc.stdout, str(root), pipeline_exit_code=proc.returncode)
    assert entries == [str(root), str(root / "report.txt")]