"""Tests for deerflow.uploads.manager — shared upload management logic.""" import errno import logging import os import shutil import stat import time from concurrent.futures import ThreadPoolExecutor from datetime import timedelta from threading import Barrier, local from unittest.mock import patch import pytest from deerflow.uploads.manager import ( PathTraversalError, UnsafeUploadPathError, _staging_name_shares_inode_with_sibling, apply_upload_sandbox_permits, claim_unique_filename, cleanup_stale_upload_staging_files, copy_upload_file_no_replace, copy_upload_file_no_symlink, delete_file_safe, list_files_in_dir, normalize_filename, validate_path_traversal, validate_upload_destination, write_upload_file_no_symlink, ) @pytest.mark.skipif(not (hasattr(os, "O_NOFOLLOW") and hasattr(os, "fchmod")), reason="POSIX-only: O_NOFOLLOW + fchmod") def test_apply_upload_sandbox_permits_propagates_permission_errors(tmp_path): upload = tmp_path / "attachment.bin" upload.write_bytes(b"attachment") upload.chmod(0o600) with patch.object(os, "fchmod", side_effect=PermissionError("permission denied")): with pytest.raises(PermissionError, match="permission denied"): apply_upload_sandbox_permits(upload, stat.S_IRGRP | stat.S_IROTH) assert stat.S_IMODE(upload.stat().st_mode) == 0o600 def test_apply_upload_sandbox_permits_fallback_propagates_permission_errors(tmp_path, monkeypatch): upload = tmp_path / "attachment.bin" upload.write_bytes(b"attachment") upload.chmod(0o600) monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) with patch.object(os, "chmod", side_effect=PermissionError("permission denied")): with pytest.raises(PermissionError, match="permission denied"): apply_upload_sandbox_permits(upload, stat.S_IRGRP | stat.S_IROTH) # --------------------------------------------------------------------------- # normalize_filename # --------------------------------------------------------------------------- class TestNormalizeFilename: def test_safe_filename(self): assert normalize_filename("report.pdf") == "report.pdf" def test_strips_path_components(self): assert normalize_filename("../../etc/passwd") == "passwd" def test_rejects_empty(self): with pytest.raises(ValueError, match="empty"): normalize_filename("") def test_rejects_dot_dot(self): with pytest.raises(ValueError, match="unsafe"): normalize_filename("..") def test_strips_separators(self): assert normalize_filename("path/to/file.txt") == "file.txt" def test_dot_only(self): with pytest.raises(ValueError, match="unsafe"): normalize_filename(".") def test_rejects_embedded_nul(self): with pytest.raises(ValueError, match="NUL"): normalize_filename("report\x00.pdf") @pytest.mark.parametrize("filename", [".upload-notes.part", ".upload-.part", "folder/.upload-notes.part"]) def test_rejects_reserved_staging_names(self, filename): with pytest.raises(ValueError, match="reserved upload staging"): normalize_filename(filename) @pytest.mark.parametrize("filename", [".upload-notes.txt", "notes.part", ".env", ".UPLOAD-notes.txt", "notes.PART"]) def test_keeps_non_staging_names(self, filename): assert normalize_filename(filename) == filename def test_reserved_name_rejected_before_writing_upload(self, tmp_path): with pytest.raises(ValueError, match="reserved upload staging"): write_upload_file_no_symlink(tmp_path, ".upload-notes.part", b"user document") assert list(tmp_path.iterdir()) == [] @pytest.mark.parametrize( "filename", ["CON", "con.txt", "PRN", "AUX", "NUL", "COM1", "COM9", "LPT1", "LPT9", "COM¹", "com².txt", "COM³.log", "LPT¹", "lpt².md", "LPT³.pdf", "file.txt.", "file.txt ", "a.", "folder/CON"], ) def test_rejects_windows_incompatible_names(self, filename): with pytest.raises(ValueError, match="not portable to Windows"): normalize_filename(filename) @pytest.mark.parametrize("filename", ["CONIN$", "conin$", "CONOUT$", "conout$", "CONIN$.txt", "folder/CONOUT$.log"]) def test_rejects_windows_console_device_names(self, filename): with pytest.raises(ValueError, match="reserved Windows device name"): normalize_filename(filename) @pytest.mark.parametrize("filename", ["NUL .txt", "con .log", "COM1 .txt", "lpt² .md", "CONIN$ .txt", "conout$ .log"]) def test_rejects_windows_device_names_with_spaces_before_extension(self, filename): with pytest.raises(ValueError, match="reserved Windows device name"): normalize_filename(filename) @pytest.mark.parametrize("filename", ["report .txt", "NULnotes .txt", "COM¹notes .txt", "NUL\u00a0.txt"]) def test_preserves_spaces_in_ordinary_filename_stems(self, filename): assert normalize_filename(filename) == filename @pytest.mark.parametrize("filename", ["CONIN", "CONOUT", "CONIN$notes.txt", "CONOUT$notes.log"]) def test_allows_names_resembling_console_devices(self, filename): assert normalize_filename(filename) == filename @pytest.mark.parametrize("filename", ["report.pdf", "contour.txt", "console.log", "COM0", "COM10", "COM⁴.txt", "LPT⁵", "COM¹notes.txt", "LPT²0.md", ".gitignore"]) def test_allows_portable_names(self, filename): assert normalize_filename(filename) == filename def test_reserved_parent_is_stripped_with_the_directory(self): assert normalize_filename("CON/notes.md") == "notes.md" # --------------------------------------------------------------------------- # claim_unique_filename # --------------------------------------------------------------------------- class TestDeduplicateFilename: def test_no_collision(self): seen: set[str] = set() assert claim_unique_filename("data.txt", seen) == "data.txt" assert "data.txt" in seen def test_single_collision(self): seen = {"data.txt"} assert claim_unique_filename("data.txt", seen) == "data_1.txt" assert "data_1.txt" in seen def test_case_insensitive_collision_including_suffix(self): seen = {"Report.txt", "report_1.TXT"} assert claim_unique_filename("report.txt", seen) == "report_2.txt" assert "report_2.txt" in seen def test_triple_collision(self): seen = {"data.txt", "data_1.txt", "data_2.txt"} assert claim_unique_filename("data.txt", seen) == "data_3.txt" assert "data_3.txt" in seen def test_mutates_seen(self): seen: set[str] = set() claim_unique_filename("a.txt", seen) claim_unique_filename("a.txt", seen) assert seen == {"a.txt", "a_1.txt"} def test_max_length_name_stays_within_filename_limit(self): # A 255-byte name passes normalize_filename; the deduplicated name # must not exceed that limit, or the write path rejects it. name = "a" * 251 + ".txt" seen = {name} deduped = claim_unique_filename(name, seen) assert deduped != name assert deduped.endswith("_1.txt") assert len(deduped.encode("utf-8")) <= 255 # The truncated result must round-trip through normalize_filename. assert normalize_filename(deduped) == deduped def test_max_length_collisions_stay_unique_across_truncation(self): name = "a" * 251 + ".txt" seen = {name} first = claim_unique_filename(name, seen) second = claim_unique_filename(name, seen) assert first != second assert len(second.encode("utf-8")) <= 255 def test_multibyte_stem_is_truncated_on_a_codepoint_boundary(self): # 85 CJK chars × 3 bytes = 255 bytes. name = "深" * 85 seen = {name} deduped = claim_unique_filename(name, seen) assert len(deduped.encode("utf-8")) <= 255 assert deduped.endswith("_1") # No replacement characters / decode artifacts. deduped.encode("utf-8").decode("utf-8") def test_short_names_keep_existing_dedupe_shape(self): seen = {"data.txt"} assert claim_unique_filename("data.txt", seen) == "data_1.txt" # --------------------------------------------------------------------------- # validate_path_traversal # --------------------------------------------------------------------------- class TestValidatePathTraversal: def test_inside_base_ok(self, tmp_path): child = tmp_path / "file.txt" child.touch() validate_path_traversal(child, tmp_path) # no exception def test_outside_base_raises(self, tmp_path): outside = tmp_path / ".." / "evil.txt" with pytest.raises(PathTraversalError, match="traversal"): validate_path_traversal(outside, tmp_path) def test_symlink_escape(self, tmp_path): target = tmp_path.parent / "secret.txt" target.touch() link = tmp_path / "escape" try: link.symlink_to(target) except OSError as exc: if getattr(exc, "winerror", None) == 1314: pytest.skip("symlink creation requires Developer Mode or elevated privileges on Windows") raise with pytest.raises(PathTraversalError, match="traversal"): validate_path_traversal(link, tmp_path) # --------------------------------------------------------------------------- # write_upload_file_no_symlink # --------------------------------------------------------------------------- class TestWriteUploadFileNoSymlink: @pytest.mark.parametrize("without_nofollow", [False, True]) def test_exclusive_write_treats_existing_hardlink_as_collision(self, tmp_path, monkeypatch, without_nofollow): if without_nofollow: monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) source = tmp_path / "source.txt" source.write_bytes(b"existing upload") os.link(source, tmp_path / "notes.txt") with pytest.raises(FileExistsError): write_upload_file_no_symlink(tmp_path, "notes.txt", b"new attachment", exclusive=True) # Default collision retries keep both aliases intact. POSIX still # rejects a pre-existing multi-link destination; Windows cannot use its # stale link count to reject files published through a staging link. if os.name == "nt": renamed = write_upload_file_no_symlink(tmp_path, "notes.txt", b"new attachment") assert renamed.name == "notes_1.txt" assert renamed.read_bytes() == b"new attachment" assert source.read_bytes() == b"existing upload" assert (tmp_path / "notes.txt").read_bytes() == b"existing upload" else: with pytest.raises(UnsafeUploadPathError, match="multiple links"): write_upload_file_no_symlink(tmp_path, "notes.txt", b"replacement") assert source.read_bytes() == b"existing upload" assert (tmp_path / "notes.txt").read_bytes() == b"existing upload" @pytest.mark.parametrize("without_nofollow", [False, True]) def test_exclusive_write_preserves_existing_file(self, tmp_path, monkeypatch, without_nofollow): if without_nofollow: monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) dest = tmp_path / "notes.txt" dest.write_bytes(b"previous upload") with pytest.raises(FileExistsError): write_upload_file_no_symlink(tmp_path, "notes.txt", b"new attachment", exclusive=True) assert dest.read_bytes() == b"previous upload" @pytest.mark.parametrize("without_nofollow", [False, True]) def test_exclusive_write_creates_new_file(self, tmp_path, monkeypatch, without_nofollow): if without_nofollow: monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) dest = write_upload_file_no_symlink(tmp_path, "notes.txt", b"new attachment", exclusive=True) assert dest == tmp_path / "notes.txt" assert dest.read_bytes() == b"new attachment" def test_writes_new_file(self, tmp_path): dest = write_upload_file_no_symlink(tmp_path, "notes.txt", b"hello") assert dest == tmp_path / "notes.txt" assert dest.read_bytes() == b"hello" def test_collision_appends_suffix_and_preserves_existing_file(self, tmp_path): """A same-name upload must never truncate the file already on disk (issue #3750).""" existing = tmp_path / "notes.txt" existing.write_bytes(b"old contents") assert os.stat(existing).st_nlink == 1 result = write_upload_file_no_symlink(tmp_path, "notes.txt", b"new contents") assert result == tmp_path / "notes_1.txt" assert existing.read_bytes() == b"old contents" assert result.read_bytes() == b"new contents" def test_repeated_same_name_uploads_keep_every_version(self, tmp_path): write_upload_file_no_symlink(tmp_path, "notes.txt", b"v1") write_upload_file_no_symlink(tmp_path, "notes.txt", b"v2") result = write_upload_file_no_symlink(tmp_path, "notes.txt", b"v3") assert result == tmp_path / "notes_2.txt" assert {p.read_bytes() for p in tmp_path.glob("notes*.txt")} == {b"v1", b"v2", b"v3"} @pytest.mark.parametrize("without_nofollow", [False, True], ids=["native", "without-nofollow"]) def test_concurrent_same_name_uploads_keep_every_payload(self, tmp_path, monkeypatch, without_nofollow): if without_nofollow: monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) filename = "notes.txt" payloads = [f"upload-{index}:".encode() + bytes([index]) * 4096 for index in range(12)] first_claim = Barrier(len(payloads), timeout=15) real_open = os.open def synchronized_open(path, flags, *args, **kwargs): if os.path.basename(path) == filename and flags & os.O_CREAT: # All writers complete their pre-open checks before any can # create the shared name. Keep the actual filesystem open. first_claim.wait() return real_open(path, flags, *args, **kwargs) monkeypatch.setattr(os, "open", synchronized_open) with ThreadPoolExecutor(max_workers=len(payloads)) as executor: futures = [executor.submit(write_upload_file_no_symlink, tmp_path, filename, payload) for payload in payloads] try: paths = [future.result(timeout=20) for future in futures] finally: first_claim.abort() assert len(set(paths)) == len(payloads) assert {path.name for path in paths} == {filename, *(f"notes_{index}.txt" for index in range(1, len(payloads)))} assert set(tmp_path.iterdir()) == set(paths) for path, payload in zip(paths, payloads, strict=True): assert path.read_bytes() == payload def test_collision_fallback_without_no_follow_support_preserves_existing_file(self, tmp_path, monkeypatch): # When O_NOFOLLOW is absent (Windows), the function falls back to a # dual-lstat + fstat approach — which must also refuse to truncate. monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) existing = tmp_path / "notes.txt" existing.write_bytes(b"old contents") result = write_upload_file_no_symlink(tmp_path, "notes.txt", b"new contents") assert result == tmp_path / "notes_1.txt" assert existing.read_bytes() == b"old contents" assert result.read_bytes() == b"new contents" def test_fallback_without_no_follow_support_succeeds(self, tmp_path, monkeypatch): monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) # When O_NOFOLLOW is absent (Windows), the function falls back to # a dual-lstat + fstat approach and succeeds. result = write_upload_file_no_symlink(tmp_path, "notes.txt", b"hello") assert result == tmp_path / "notes.txt" assert (tmp_path / "notes.txt").read_bytes() == b"hello" @pytest.mark.parametrize("exclusive", [False, True]) def test_write_keeps_post_open_link_guard_without_no_follow(self, tmp_path, monkeypatch, exclusive): """Both O_EXCL creation modes require a new file with one link.""" monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) real_fstat = os.fstat def twin_linked_fstat(fd): st = real_fstat(fd) return os.stat_result((st.st_mode, st.st_ino, st.st_dev, 2, st.st_uid, st.st_gid, st.st_size, st.st_atime, st.st_mtime, st.st_ctime)) monkeypatch.setattr(os, "fstat", twin_linked_fstat) with pytest.raises(UnsafeUploadPathError, match="not an exclusive regular file"): write_upload_file_no_symlink(tmp_path, "notes.txt", b"new attachment", exclusive=exclusive) def test_open_uses_nonblocking_flag_when_available(self, tmp_path): if not hasattr(os, "O_NONBLOCK"): pytest.skip("O_NONBLOCK not available on this platform") with patch("deerflow.uploads.manager.os.open", side_effect=OSError(errno.ENXIO, "no reader")) as open_mock: with pytest.raises(UnsafeUploadPathError, match="Unsafe upload destination"): write_upload_file_no_symlink(tmp_path, "pipe.txt", b"hello") flags = open_mock.call_args.args[1] assert flags & os.O_NONBLOCK @pytest.mark.parametrize("open_errno", [errno.ENXIO, errno.EAGAIN]) def test_nonblocking_special_file_open_errors_are_unsafe(self, tmp_path, open_errno): if not hasattr(os, "O_NONBLOCK"): pytest.skip("O_NONBLOCK not available on this platform") with patch("deerflow.uploads.manager.os.open", side_effect=OSError(open_errno, "would block")): with pytest.raises(UnsafeUploadPathError, match="Unsafe upload destination"): write_upload_file_no_symlink(tmp_path, "pipe.txt", b"hello") assert not (tmp_path / "pipe.txt").exists() # --------------------------------------------------------------------------- # copy_upload_file_no_symlink # --------------------------------------------------------------------------- class TestCopyUploadFileNoSymlink: def test_copies_content_mode_and_timestamps(self, tmp_path): uploads = tmp_path / "uploads" uploads.mkdir() src = tmp_path / "notes.txt" src.write_bytes(b"hello") os.chmod(src, 0o640) os.utime(src, ns=(1_700_000_000_000_000_000, 1_700_000_000_000_000_000)) dest = copy_upload_file_no_symlink(uploads, "notes.txt", src) assert dest == uploads / "notes.txt" assert dest.read_bytes() == b"hello" if os.chmod in os.supports_fd: assert stat.S_IMODE(os.stat(dest).st_mode) == 0o640 if os.utime in os.supports_fd: assert os.stat(dest).st_mtime_ns == 1_700_000_000_000_000_000 def test_collision_copies_to_unique_name_and_preserves_existing(self, tmp_path): """Re-uploading an existing name must not destroy the earlier copy (issue #3750).""" uploads = tmp_path / "uploads" uploads.mkdir() (uploads / "notes.txt").write_bytes(b"old contents") src = tmp_path / "notes.txt" src.write_bytes(b"new contents") dest = copy_upload_file_no_symlink(uploads, "notes.txt", src) assert dest == uploads / "notes_1.txt" assert (uploads / "notes.txt").read_bytes() == b"old contents" assert dest.read_bytes() == b"new contents" @pytest.mark.parametrize("capture", [False, True], ids=["direct", "snapshot"]) def test_known_collision_stages_payload_once(self, tmp_path, capture): uploads = tmp_path / "uploads" uploads.mkdir() existing = {"notes.txt": b"first upload", "NOTES_1.TXT": b"second upload"} for name, data in existing.items(): (uploads / name).write_bytes(data) src = tmp_path / "source.txt" src.write_bytes(b"new contents" * 4096) snapshot = tmp_path / "snapshot" if capture else None with patch("deerflow.uploads.manager.shutil.copyfileobj", wraps=shutil.copyfileobj) as copy: dest = copy_upload_file_no_symlink(uploads, "notes.txt", src, snapshot_path=snapshot) assert copy.call_count == 1 + int(capture) assert dest.name == "notes_2.txt" assert dest.read_bytes() == src.read_bytes() for name, data in existing.items(): assert (uploads / name).read_bytes() == data assert {path.name for path in uploads.iterdir()} == {*existing, dest.name} if snapshot is not None: assert snapshot.read_bytes() == src.read_bytes() @pytest.mark.parametrize("existing", [False, True], ids=["fresh", "occupied"]) @pytest.mark.parametrize("nofollow", [False, True], ids=["link-fallback", "link-nofollow"]) def test_concurrent_same_name_copies_keep_every_payload(self, tmp_path, monkeypatch, existing, nofollow): uploads = tmp_path / "uploads" uploads.mkdir() if existing: (uploads / "notes.txt").write_bytes(b"original upload") payloads = [f"copy-{index}:".encode() + bytes([index]) * 4096 for index in range(8)] sources = [tmp_path / f"source-{index}.txt" for index in range(len(payloads))] for source, payload in zip(sources, payloads, strict=True): source.write_bytes(payload) first_publication = Barrier(len(payloads), timeout=15) worker_state = local() real_link = os.link if nofollow and real_link not in os.supports_follow_symlinks: pytest.skip("host does not support no-follow hard links") def synchronized_link(staging, dest, **kwargs): if not getattr(worker_state, "published", False): worker_state.published = True first_publication.wait() return real_link(staging, dest, **kwargs) monkeypatch.setattr(os, "link", synchronized_link) monkeypatch.setattr(os, "supports_follow_symlinks", {synchronized_link} if nofollow else set()) with ThreadPoolExecutor(max_workers=len(payloads)) as executor: futures = [executor.submit(copy_upload_file_no_symlink, uploads, "notes.txt", source) for source in sources] try: paths = [future.result(timeout=20) for future in futures] finally: first_publication.abort() assert len(set(paths)) == len(payloads) start = int(existing) expected = {"notes.txt" if index == 0 else f"notes_{index}.txt" for index in range(start, start + len(payloads))} assert {path.name for path in paths} == expected assert {path.name for path in uploads.iterdir()} == expected | ({"notes.txt"} if existing else set()) for path, payload in zip(paths, payloads, strict=True): assert path.read_bytes() == payload if existing: assert (uploads / "notes.txt").read_bytes() == b"original upload" @pytest.mark.skipif(os.name == "nt", reason="POSIX destination hardlink guard") def test_existing_hardlinked_destination_is_not_silently_renamed(self, tmp_path): uploads = tmp_path / "uploads" uploads.mkdir() original = tmp_path / "original.txt" original.write_bytes(b"original upload") os.link(original, uploads / "notes.txt") src = tmp_path / "source.txt" src.write_bytes(b"new contents") with pytest.raises(UnsafeUploadPathError, match="multiple links"): copy_upload_file_no_symlink(uploads, "notes.txt", src) assert (uploads / "notes.txt").read_bytes() == b"original upload" assert [path.name for path in uploads.iterdir()] == ["notes.txt"] def test_no_replace_copy_preserves_existing_file_and_cleans_stage(self, tmp_path): uploads = tmp_path / "uploads" uploads.mkdir() dest = uploads / "notes.txt" dest.write_bytes(b"old contents") src = tmp_path / "notes.txt" src.write_bytes(b"new contents") with pytest.raises(FileExistsError): copy_upload_file_no_replace(uploads, "notes.txt", src) assert dest.read_bytes() == b"old contents" assert sorted(path.name for path in uploads.iterdir()) == ["notes.txt"] def test_no_replace_copy_failure_removes_stage_without_publishing(self, tmp_path): uploads = tmp_path / "uploads" uploads.mkdir() src = tmp_path / "notes.txt" src.write_bytes(b"new contents") def fail_copy(source, staged): staged.write(b"partial") raise OSError("copy failed") with patch("deerflow.uploads.manager.shutil.copyfileobj", side_effect=fail_copy): with pytest.raises(OSError, match="copy failed"): copy_upload_file_no_replace(uploads, "notes.txt", src) assert list(uploads.iterdir()) == [] def test_no_replace_copy_publishes_complete_file_and_preserves_metadata(self, tmp_path): uploads = tmp_path / "threads" / "t1" / "user-data" / "uploads" uploads.mkdir(parents=True) src = tmp_path / "notes.txt" src.write_bytes(b"complete contents") src.chmod(0o640) old_time = time.time_ns() - 3 * 24 * 60 * 60 * 1_000_000_000 os.utime(src, ns=(old_time, old_time)) source_stat = src.stat() real_link = os.link def publish(staged, dest, **kwargs): assert not dest.exists() assert list_files_in_dir(uploads)["files"] == [] assert staged.read_bytes() == b"complete contents" assert cleanup_stale_upload_staging_files(tmp_path) == 0 real_link(staged, dest, **kwargs) with patch("deerflow.uploads.manager.os.link", side_effect=publish): dest = copy_upload_file_no_replace(uploads, "notes.txt", src) assert dest.read_bytes() == b"complete contents" assert sorted(path.name for path in uploads.iterdir()) == ["notes.txt"] if os.chmod in os.supports_fd: assert stat.S_IMODE(dest.stat().st_mode) == stat.S_IMODE(source_stat.st_mode) if os.utime in os.supports_fd: assert dest.stat().st_mtime_ns == source_stat.st_mtime_ns def test_rejects_symlink_destination(self, tmp_path): uploads = tmp_path / "uploads" uploads.mkdir() outside = tmp_path / "outside.txt" outside.write_bytes(b"original") link = uploads / "notes.txt" try: link.symlink_to(outside) except OSError as exc: if getattr(exc, "winerror", None) == 1314: pytest.skip("symlink creation requires Developer Mode or elevated privileges on Windows") raise src = tmp_path / "notes.txt" src.write_bytes(b"attacker-chosen target") with pytest.raises(UnsafeUploadPathError): copy_upload_file_no_symlink(uploads, "notes.txt", src) assert outside.read_bytes() == b"original" assert link.is_symlink() def test_rejects_copying_a_file_onto_itself(self, tmp_path): uploads = tmp_path / "uploads" uploads.mkdir() src = uploads / "notes.txt" src.write_bytes(b"IMPORTANT") with pytest.raises(shutil.SameFileError): copy_upload_file_no_symlink(uploads, "notes.txt", src) assert src.read_bytes() == b"IMPORTANT" def test_rejects_a_hardlink_to_the_destination(self, tmp_path): """Identity, not path text: another name for the same inode is the same file.""" uploads = tmp_path / "uploads" uploads.mkdir() dest = uploads / "notes.txt" dest.write_bytes(b"IMPORTANT") src = uploads / "same-inode.txt" try: os.link(dest, src) except (OSError, NotImplementedError) as exc: pytest.skip(f"hardlinks unavailable on this platform: {exc}") with pytest.raises(shutil.SameFileError): copy_upload_file_no_symlink(uploads, "notes.txt", src) assert dest.read_bytes() == b"IMPORTANT" def test_missing_source_leaves_existing_destination_untouched(self, tmp_path): uploads = tmp_path / "uploads" uploads.mkdir() (uploads / "notes.txt").write_bytes(b"keep me") with pytest.raises(FileNotFoundError): copy_upload_file_no_symlink(uploads, "notes.txt", tmp_path / "missing.txt") assert (uploads / "notes.txt").read_bytes() == b"keep me" # --------------------------------------------------------------------------- # list_files_in_dir # --------------------------------------------------------------------------- class _VanishingDirEntry: """DirEntry stand-in whose stat() raises, simulating a file removed mid-scan.""" def __init__(self, real, error): self._real = real self._error = error @property def name(self): return self._real.name @property def path(self): return self._real.path def is_file(self, follow_symlinks=False): return True def stat(self, follow_symlinks=False): raise self._error class _StaticScandir: """Context manager mimicking os.scandir over a fixed entry list.""" def __init__(self, entries): self._entries = entries def __enter__(self): return iter(self._entries) def __exit__(self, *exc_info): return False class TestListFilesInDir: def test_empty_dir(self, tmp_path): result = list_files_in_dir(tmp_path) assert result == {"files": [], "count": 0} def test_nonexistent_dir(self, tmp_path): result = list_files_in_dir(tmp_path / "nope") assert result == {"files": [], "count": 0} @pytest.mark.parametrize("replacement", ["removed", "file"]) def test_directory_replaced_after_is_dir_check(self, tmp_path, monkeypatch, caplog, replacement): directory = tmp_path / "uploads" directory.mkdir() real_scandir = os.scandir def replace_before_scandir(path): assert path == directory directory.rmdir() if replacement == "file": directory.write_text("replacement", encoding="utf-8") return real_scandir(path) monkeypatch.setattr(os, "scandir", replace_before_scandir) with caplog.at_level(logging.DEBUG, logger="deerflow.uploads.manager"): assert list_files_in_dir(directory) == {"files": [], "count": 0} assert ( "deerflow.uploads.manager", logging.DEBUG, f"Uploads directory vanished mid-scan, keeping partial snapshot of 0 entries: {directory}", ) in caplog.record_tuples @pytest.mark.parametrize("error_code", [errno.EACCES, errno.EIO]) def test_scandir_operational_error_still_propagates(self, tmp_path, monkeypatch, error_code): def fail_scandir(path): raise OSError(error_code, os.strerror(error_code), str(path)) monkeypatch.setattr(os, "scandir", fail_scandir) with pytest.raises(OSError) as excinfo: list_files_in_dir(tmp_path) assert excinfo.value.errno == error_code def test_multiple_files_sorted(self, tmp_path): (tmp_path / "b.txt").write_text("b") (tmp_path / "a.txt").write_text("a") result = list_files_in_dir(tmp_path) assert result["count"] == 2 assert result["files"][0]["filename"] == "a.txt" assert result["files"][1]["filename"] == "b.txt" for f in result["files"]: assert set(f.keys()) == {"filename", "size", "path", "extension", "modified"} def test_ignores_subdirectories(self, tmp_path): (tmp_path / "file.txt").write_text("data") (tmp_path / "subdir").mkdir() result = list_files_in_dir(tmp_path) assert result["count"] == 1 assert result["files"][0]["filename"] == "file.txt" def test_filters_only_upload_staging_files(self, tmp_path): (tmp_path / ".env").write_text("intentional dotfile") (tmp_path / ".upload-active.part").write_text("partial") (tmp_path / ".upload-note.txt").write_text("intentional upload") (tmp_path / "draft.part").write_text("intentional upload") (tmp_path / "visible.txt").write_text("visible") result = list_files_in_dir(tmp_path) assert result["count"] == 4 assert [f["filename"] for f in result["files"]] == [".env", ".upload-note.txt", "draft.part", "visible.txt"] @pytest.mark.parametrize("error_code", [errno.ENOENT, errno.ENOTDIR, errno.ELOOP]) def test_skips_entries_vanishing_mid_scan(self, tmp_path, monkeypatch, caplog, error_code): (tmp_path / "kept.txt").write_text("kept") (tmp_path / "gone.txt").write_text("gone") with os.scandir(tmp_path) as it: real_entries = {e.name: e for e in it} vanished = _VanishingDirEntry( real_entries["gone.txt"], OSError(error_code, os.strerror(error_code), str(tmp_path / "gone.txt")), ) monkeypatch.setattr(os, "scandir", lambda path: _StaticScandir([real_entries["kept.txt"], vanished])) with caplog.at_level(logging.DEBUG, logger="deerflow.uploads.manager"): result = list_files_in_dir(tmp_path) assert result["count"] == 1 assert result["files"][0]["filename"] == "kept.txt" assert any("vanished mid-scan" in record.message and str(tmp_path / "gone.txt") in record.message for record in caplog.records) def test_stat_permission_error_still_propagates(self, tmp_path, monkeypatch): (tmp_path / "locked.txt").write_text("locked") with os.scandir(tmp_path) as it: real_entry = next(iter(it)) locked = _VanishingDirEntry(real_entry, PermissionError(errno.EACCES, "Permission denied", str(tmp_path / "locked.txt"))) monkeypatch.setattr(os, "scandir", lambda path: _StaticScandir([locked])) with pytest.raises(PermissionError): list_files_in_dir(tmp_path) # --------------------------------------------------------------------------- # cleanup_stale_upload_staging_files # --------------------------------------------------------------------------- def _set_age(path, age: timedelta) -> None: """Backdate *path*'s mtime so the cleanup sees it as *age* old.""" timestamp = time.time() - age.total_seconds() os.utime(path, (timestamp, timestamp)) class TestCleanupStaleUploadStagingFiles: def test_removes_only_stale_staging_files_from_all_upload_layouts(self, tmp_path): legacy_uploads = tmp_path / "threads" / "thread-legacy" / "user-data" / "uploads" user_uploads = tmp_path / "users" / "owner-1" / "threads" / "thread-owned" / "user-data" / "uploads" unrelated_uploads = tmp_path / "misc" / "thread-other" / "user-data" / "uploads" for uploads_dir in (legacy_uploads, user_uploads, unrelated_uploads): uploads_dir.mkdir(parents=True) (legacy_uploads / ".upload-old.part").write_text("legacy partial") (user_uploads / ".upload-new.part").write_text("user partial") (unrelated_uploads / ".upload-ignore.part").write_text("outside layout") (legacy_uploads / ".env").write_text("intentional dotfile") (legacy_uploads / ".upload-note.txt").write_text("intentional upload") (legacy_uploads / "draft.part").write_text("intentional upload") for path in ( legacy_uploads / ".upload-old.part", user_uploads / ".upload-new.part", unrelated_uploads / ".upload-ignore.part", legacy_uploads / ".env", legacy_uploads / ".upload-note.txt", legacy_uploads / "draft.part", ): _set_age(path, timedelta(days=2)) removed = cleanup_stale_upload_staging_files(tmp_path) assert removed == 2 assert not (legacy_uploads / ".upload-old.part").exists() assert not (user_uploads / ".upload-new.part").exists() assert (unrelated_uploads / ".upload-ignore.part").exists() assert (legacy_uploads / ".env").exists() assert (legacy_uploads / ".upload-note.txt").exists() assert (legacy_uploads / "draft.part").exists() def test_keeps_staging_files_younger_than_the_default_guard(self, tmp_path): """On a volume shared by several Gateway replicas, a young ``.part`` may be an upload another replica is still writing; only old ones are orphans.""" uploads_dir = tmp_path / "users" / "owner-1" / "threads" / "thread-1" / "user-data" / "uploads" uploads_dir.mkdir(parents=True) fresh = uploads_dir / ".upload-fresh.part" recent = uploads_dir / ".upload-recent.part" old = uploads_dir / ".upload-old.part" for path in (fresh, recent, old): path.write_text("partial") _set_age(recent, timedelta(hours=23)) _set_age(old, timedelta(hours=25)) removed = cleanup_stale_upload_staging_files(tmp_path) assert removed == 1 assert fresh.exists() assert recent.exists() assert not old.exists() def test_min_age_is_configurable(self, tmp_path): uploads_dir = tmp_path / "threads" / "thread-1" / "user-data" / "uploads" uploads_dir.mkdir(parents=True) young = uploads_dir / ".upload-young.part" old = uploads_dir / ".upload-old.part" for path in (young, old): path.write_text("partial") _set_age(young, timedelta(minutes=1)) _set_age(old, timedelta(minutes=10)) removed = cleanup_stale_upload_staging_files(tmp_path, min_age=timedelta(minutes=5)) assert removed == 1 assert young.exists() assert not old.exists() def test_sibling_inode_match_ignores_degenerate_zero_inodes(self): """On volumes that cannot answer inode identity (FAT/exFAT, some SMB shares) ``st_ino`` reads as 0; a zero-inode sibling must never mark a staging file as a published alias, or the sweep would delete in-flight uploads at any age.""" def fake_stat(ino, dev=1): return os.stat_result((stat.S_IFREG | 0o644, ino, dev, 1, 0, 0, 4, 0.0, 0.0, 0.0)) siblings = { "published.txt": fake_stat(42), ".upload-zero.part": fake_stat(0), } # A degenerate zero inode matches nothing, not even a sibling reporting # the same degenerate zero. assert not _staging_name_shares_inode_with_sibling(siblings, ".upload-zero.part", fake_stat(0)) assert not _staging_name_shares_inode_with_sibling(siblings, ".upload-new.part", fake_stat(0)) # A real shared-inode alias is still detected. assert _staging_name_shares_inode_with_sibling(siblings, ".upload-alias.part", fake_stat(42)) # A different device never matches, even with the same inode number. assert not _staging_name_shares_inode_with_sibling(siblings, ".upload-alias.part", fake_stat(42, dev=2)) def test_sweep_scans_each_uploads_dir_once(self, tmp_path, monkeypatch): """The sweep builds its inode map once per uploads directory instead of rescanning the directory for every staging file, so N orphan parts cost one directory scan, not N.""" uploads_dir = tmp_path / "threads" / "thread-1" / "user-data" / "uploads" uploads_dir.mkdir(parents=True) for i in range(5): part = uploads_dir / f".upload-{i}.part" part.write_text("partial") _set_age(part, timedelta(days=2)) real_scandir = os.scandir sweep_scans: list[str] = [] def counting_scandir(path, /): if os.path.abspath(os.fspath(path)) == os.path.abspath(os.fspath(uploads_dir)): sweep_scans.append(os.fspath(path)) return real_scandir(path) monkeypatch.setattr(os, "scandir", counting_scandir) removed = cleanup_stale_upload_staging_files(tmp_path) assert removed == 5 assert len(sweep_scans) == 1 @pytest.mark.parametrize("error_code", [errno.EACCES, errno.EIO, errno.ENOENT]) def test_failed_directory_scan_preserves_files_and_continues_cleanup(self, tmp_path, monkeypatch, caplog, error_code): failed_dir = tmp_path / "threads" / "failed" / "user-data" / "uploads" healthy_dir = tmp_path / "threads" / "healthy" / "user-data" / "uploads" for directory in (failed_dir, healthy_dir): directory.mkdir(parents=True) part = directory / ".upload-old.part" part.write_text("partial", encoding="utf-8") _set_age(part, timedelta(days=2)) real_scandir = os.scandir def fail_one_scan(path): if os.path.abspath(os.fspath(path)) == os.path.abspath(os.fspath(failed_dir)): raise OSError(error_code, "scan failed") return real_scandir(path) monkeypatch.setattr(os, "scandir", fail_one_scan) with caplog.at_level(logging.WARNING, logger="deerflow.uploads.manager"): removed = cleanup_stale_upload_staging_files(tmp_path) assert removed == 1 assert (failed_dir / ".upload-old.part").exists() assert not (healthy_dir / ".upload-old.part").exists() warnings = [record for record in caplog.records if record.name == "deerflow.uploads.manager" and record.levelno >= logging.WARNING] if error_code == errno.ENOENT: assert warnings == [] else: assert len(warnings) == 1 assert "scan uploads directory for stale staging files" in warnings[0].getMessage() assert str(failed_dir) in warnings[0].getMessage() assert warnings[0].exc_info[1].errno == error_code def test_keeps_staging_file_when_its_age_cannot_be_read(self, tmp_path, caplog): uploads_dir = tmp_path / "threads" / "thread-1" / "user-data" / "uploads" uploads_dir.mkdir(parents=True) old = uploads_dir / ".upload-old.part" old.write_text("partial") _set_age(old, timedelta(days=2)) with ( patch("deerflow.uploads.manager._staging_entry_stat", side_effect=PermissionError(errno.EACCES, "denied")), caplog.at_level("WARNING", logger="deerflow.uploads.manager"), ): removed = cleanup_stale_upload_staging_files(tmp_path) assert removed == 0 assert old.exists() assert any("keeping it" in record.getMessage() for record in caplog.records) def test_removes_published_alias_at_any_age_but_keeps_lone_young_part(self, tmp_path): """A crash between the commit's ``os.link`` and the staged-name removal leaves the final file and its ``.part`` alias sharing one inode. The alias is reclaimed on the next startup regardless of age, or the destination would fail the multi-link check on its next replacement; a lone young part next to it is still treated as in flight. The multi-link check reads ``st_nlink``, which Windows never decrements when a hard link is removed, so the pre-sweep rejection is only asserted where that count is authoritative; the Windows path is exercised by the replacement at the end of the test.""" uploads_dir = tmp_path / "users" / "owner-1" / "threads" / "thread-1" / "user-data" / "uploads" uploads_dir.mkdir(parents=True) published = uploads_dir / "notes.txt" published.write_bytes(b"published bytes") alias = uploads_dir / ".upload-abc123.part" try: os.link(published, alias) except OSError as exc: # pragma: no cover - filesystems without hard links pytest.skip(f"hard links unsupported here: {exc}") in_flight = uploads_dir / ".upload-inflight.part" in_flight.write_bytes(b"partial") if os.name != "nt": with pytest.raises(UnsafeUploadPathError, match="multiple links"): validate_upload_destination(uploads_dir, "notes.txt") removed = cleanup_stale_upload_staging_files(tmp_path) assert removed == 1 assert not alias.exists() assert in_flight.exists() assert published.read_bytes() == b"published bytes" if os.name != "nt": assert os.lstat(published).st_nlink == 1 assert validate_upload_destination(uploads_dir, "notes.txt") == published replacement = tmp_path / "replacement.txt" replacement.write_bytes(b"replacement bytes") copied = copy_upload_file_no_symlink(uploads_dir, "notes.txt", replacement) assert copied == uploads_dir / "notes_1.txt" assert copied.read_bytes() == b"replacement bytes" assert published.read_bytes() == b"published bytes" def test_skips_staging_file_that_vanished_before_its_age_was_read(self, tmp_path, caplog): uploads_dir = tmp_path / "threads" / "thread-1" / "user-data" / "uploads" uploads_dir.mkdir(parents=True) (uploads_dir / ".upload-gone.part").write_text("partial") with ( patch("deerflow.uploads.manager._staging_entry_stat", side_effect=FileNotFoundError), caplog.at_level("WARNING", logger="deerflow.uploads.manager"), ): removed = cleanup_stale_upload_staging_files(tmp_path) assert removed == 0 assert caplog.records == [] # --------------------------------------------------------------------------- # delete_file_safe # --------------------------------------------------------------------------- class TestDeleteFileSafe: def test_delete_existing_file(self, tmp_path): f = tmp_path / "test.txt" f.write_text("data") result = delete_file_safe(tmp_path, "test.txt") assert result["success"] is True assert not f.exists() def test_delete_nonexistent_raises(self, tmp_path): with pytest.raises(FileNotFoundError): delete_file_safe(tmp_path, "nope.txt") def test_delete_traversal_raises(self, tmp_path): with pytest.raises(PathTraversalError, match="traversal"): delete_file_safe(tmp_path, "../outside.txt") def test_delete_keeps_the_converted_markdown(self, tmp_path): """Companion ownership cannot be proven from the name, so nothing is guessed at.""" (tmp_path / "a.docx").write_bytes(b"DOCX") (tmp_path / "a.md").write_text("converted from the docx", encoding="utf-8") (tmp_path / "a.pdf").write_bytes(b"PDF") (tmp_path / "a_1.md").write_text("converted from the pdf", encoding="utf-8") result = delete_file_safe(tmp_path, "a.pdf") assert result["success"] is True assert not (tmp_path / "a.pdf").exists() # a.md belongs to a.docx; deleting a.pdf used to remove it. assert (tmp_path / "a.md").read_text(encoding="utf-8") == "converted from the docx" assert (tmp_path / "a_1.md").read_text(encoding="utf-8") == "converted from the pdf" def test_delete_symlink_to_sibling_upload_keeps_target(self, tmp_path): """A symlink planted in the uploads dir must not delete the upload it aliases.""" victim = tmp_path / "victim.pdf" victim.write_bytes(b"pdf-bytes") companion = tmp_path / "victim.md" companion.write_text("converted", encoding="utf-8") alias = tmp_path / "alias.pdf" try: alias.symlink_to(victim.name) except OSError as exc: if getattr(exc, "winerror", None) == 1314: pytest.skip("symlink creation requires Developer Mode or elevated privileges on Windows") raise with pytest.raises(FileNotFoundError): delete_file_safe(tmp_path, "alias.pdf") assert victim.read_bytes() == b"pdf-bytes" assert companion.exists() assert alias.is_symlink()