diff --git a/backend/packages/harness/deerflow/skills/installer.py b/backend/packages/harness/deerflow/skills/installer.py index 8460d43aab..2288fa553f 100644 --- a/backend/packages/harness/deerflow/skills/installer.py +++ b/backend/packages/harness/deerflow/skills/installer.py @@ -11,6 +11,7 @@ import os import posixpath import shutil import stat +import unicodedata import zipfile from pathlib import Path, PurePosixPath, PureWindowsPath @@ -29,6 +30,50 @@ from deerflow.utils.host_paths import windows_incompatible_segment logger = logging.getLogger(__name__) +# Mirror the path limits export.py applies to installed skills, so an archive +# that cannot be listed or extracted fails as a validation error instead of an +# OS error during installation. Export limits depth relative to the skill root, +# so the archive limit adds the enclosing skill directory component. +_MAX_MEMBER_PATH_BYTES = 1024 +_MAX_MEMBER_DEPTH = 33 +_MAX_MEMBER_SEGMENT_BYTES = 255 + + +def _host_path_length(path: Path, *, windows: bool | None = None) -> int: + """Return a path's length in the units the host filesystem limits.""" + if windows is None: + windows = os.name == "nt" + if windows: + # Win32 path limits count UTF-16 code units, not UTF-8 bytes. + return len(str(path).encode("utf-16-le")) // 2 + return len(os.fsencode(path)) + + +def _host_path_limit(root: Path, *, windows: bool | None = None) -> int: + """Return the destination filesystem's maximum path length in host units.""" + if windows is None: + windows = os.name == "nt" + if windows: + try: + import winreg + + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SYSTEM\CurrentControlSet\Control\FileSystem", + ) as key: + if winreg.QueryValueEx(key, "LongPathsEnabled")[0]: + return 32767 + except (ImportError, OSError): + pass + return 260 + if hasattr(os, "pathconf"): + try: + return os.pathconf(root, "PC_PATH_MAX") + except (OSError, ValueError): + pass + return 4096 + + _PROMPT_INPUT_DIRS = {"references", "templates"} _PROMPT_INPUT_SUFFIXES = frozenset({".json", ".markdown", ".md", ".rst", ".txt", ".yaml", ".yml"}) @@ -137,6 +182,7 @@ def safe_extract_skill_archive( ValueError: If unsafe members, executable binaries, entry count, or size limit exceeded. """ dest_root = dest_path.resolve() + host_path_limit = _host_path_limit(dest_root) total_written = 0 infos = zip_ref.infolist() @@ -149,6 +195,11 @@ def safe_extract_skill_archive( # it lives in the extraction path every install goes through. raise ValueError(f"Skill archive contains too many entries ({len(infos)} > {max_entries}).") + # Validate every member before writing anything, so an archive whose paths + # cannot form a file tree fails without leaving a partial install. + planned: list[tuple[zipfile.ZipInfo, tuple[str, ...]]] = [] + seen_files: set[tuple[str, ...]] = set() + seen_dirs: set[tuple[str, ...]] = set() for info in infos: if is_unsafe_zip_member(info): raise ValueError(f"Archive contains unsafe member path: {info.filename!r}") @@ -158,14 +209,45 @@ def safe_extract_skill_archive( continue normalized_name = posixpath.normpath(info.filename.replace("\\", "/")) - for segment in PurePosixPath(normalized_name).parts: + parts = PurePosixPath(normalized_name).parts + if not parts or normalized_name in ("", "."): + raise ValueError(f"Archive member path is not extractable: {info.filename!r}") + if len(parts) > _MAX_MEMBER_DEPTH: + raise ValueError(f"Archive member path is too deep: {info.filename!r}") + if len(normalized_name.encode("utf-8")) > _MAX_MEMBER_PATH_BYTES: + raise ValueError(f"Archive member path is too long: {info.filename!r}") + for segment in parts: + if len(segment.encode("utf-8")) > _MAX_MEMBER_SEGMENT_BYTES: + raise ValueError(f"Archive member path segment is too long: {info.filename!r}") reason = windows_incompatible_segment(segment) if reason is not None: # Extraction creates one host path component per segment; a # Windows device name cannot become one, and the same names are # already refused for support-file writes and skill names. raise ValueError(f"Archive member path is not portable to Windows: {info.filename!r} ({reason})") - member_path = dest_root.joinpath(*PurePosixPath(normalized_name).parts) + + if _host_path_length(dest_root.joinpath(*parts)) >= host_path_limit: + raise ValueError(f"Archive member path is too long for the destination filesystem: {info.filename!r}") + + # Compare NFC+casefolded keys so case-insensitive and normalization- + # sensitive filesystems cannot turn a distinct path into a collision. + key = tuple(unicodedata.normalize("NFC", part).casefold() for part in parts) + if any(key[:index] in seen_files for index in range(1, len(key))): + raise ValueError(f"Archive member path conflicts with a file member: {info.filename!r}") + if info.is_dir(): + if key in seen_files: + raise ValueError(f"Archive contains conflicting member paths: {info.filename!r}") + seen_dirs.add(key) + else: + if key in seen_dirs or key in seen_files: + raise ValueError(f"Archive contains conflicting member paths: {info.filename!r}") + seen_files.add(key) + for index in range(1, len(key)): + seen_dirs.add(key[:index]) + planned.append((info, parts)) + + for info, parts in planned: + member_path = dest_root.joinpath(*parts) if not member_path.resolve().is_relative_to(dest_root): raise ValueError(f"Zip entry escapes destination: {info.filename!r}") member_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/backend/packages/harness/deerflow/skills/skillscan/orchestrator.py b/backend/packages/harness/deerflow/skills/skillscan/orchestrator.py index f19531114a..29b9af0dad 100644 --- a/backend/packages/harness/deerflow/skills/skillscan/orchestrator.py +++ b/backend/packages/harness/deerflow/skills/skillscan/orchestrator.py @@ -1381,6 +1381,8 @@ def _relative_file(path: Path, root: Path) -> str: def _is_hidden_sensitive_path(rel_path: str) -> bool: parts = PurePosixPath(rel_path).parts + if not parts: + return False if ".aws" in parts and parts[-1] == "credentials": return True if ".git" in parts and parts[-1] == "config": diff --git a/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py b/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py index 75b86657eb..e578492509 100644 --- a/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py +++ b/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py @@ -21,6 +21,14 @@ from deerflow.utils.atomic_write import cleanup_failed_atomic_write logger = logging.getLogger(__name__) + +def _publication_path_limit(root: Path) -> int: + """Host path limit for the staged publication directory (indirection for tests).""" + from deerflow.skills.installer import _host_path_limit + + return _host_path_limit(root) + + # Bound for the best-effort temp-dir cleanup so a stalled filesystem (e.g. NFS) # cannot hold back the install outcome propagating out of the finally block. _INSTALL_TMP_CLEANUP_TIMEOUT_SECONDS = 5.0 @@ -218,13 +226,30 @@ class LocalSkillStorage(SkillStorage): def _commit_skill_install(self, skill_dir: Path, skill_name: str, custom_dir: Path, target: Path) -> None: """Stage and move the validated skill into place (blocking; runs off the event loop).""" - from deerflow.skills.installer import _move_staged_skill_into_reserved_target + from deerflow.skills.installer import _host_path_length, _move_staged_skill_into_reserved_target with self._skill_projection_mutation(names=(skill_name,)): with tempfile.TemporaryDirectory(prefix=f".installing-{skill_name}-", dir=custom_dir) as staging_root: staging_target = Path(staging_root) / skill_name - shutil.copytree(skill_dir, staging_target) - _move_staged_skill_into_reserved_target(staging_target, target) + publication_limit = _publication_path_limit(custom_dir) + if _host_path_length(staging_target) >= publication_limit: + raise ValueError(f"Skill '{skill_name}' cannot be published: the temporary publication path is too long for this filesystem") + # The staging root's temp-dir name lengthens every member path, so + # a nested member can outgrow the limit while the skill root fits. + # Check members before copying: copytree wraps a child's + # ENAMETOOLONG in shutil.Error, which carries no errno/winerror for + # the translation below to recognize. + for member in skill_dir.rglob("*"): + relative = member.relative_to(skill_dir) + if _host_path_length(staging_target / relative) >= publication_limit: + raise ValueError(f"Skill '{skill_name}' cannot be published: the temporary publication path for '{relative.as_posix()}' is too long for this filesystem") + try: + shutil.copytree(skill_dir, staging_target) + _move_staged_skill_into_reserved_target(staging_target, target) + except OSError as exc: + if exc.errno == errno.ENAMETOOLONG or getattr(exc, "winerror", None) == 206: + raise ValueError(f"Skill '{skill_name}' cannot be published: the path is too long for this filesystem") from exc + raise make_skill_written_path_sandbox_readable(custom_dir, target) def delete_custom_skill(self, name: str, *, history_meta: dict | None = None) -> None: diff --git a/backend/tests/test_skills_installer.py b/backend/tests/test_skills_installer.py index 8b62e4183f..46f0f3df41 100644 --- a/backend/tests/test_skills_installer.py +++ b/backend/tests/test_skills_installer.py @@ -463,6 +463,42 @@ class TestInstallSkillFromArchive: ) return zip_path + def test_rejects_publication_path_too_long(self, tmp_path, monkeypatch): + zip_path = self._make_skill_zip(tmp_path) + skills_root = tmp_path / "skills" + skills_root.mkdir() + monkeypatch.setattr( + "deerflow.skills.storage.local_skill_storage._publication_path_limit", + lambda root: 1, + ) + with pytest.raises(ValueError, match="publication path is too long"): + get_or_new_skill_storage(skills_path=skills_root).install_skill_from_archive(zip_path) + + def test_rejects_nested_member_whose_staging_path_is_too_long(self, tmp_path, monkeypatch): + zip_path = tmp_path / "nested-skill.skill" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr( + "nested-skill/SKILL.md", + "---\nname: nested-skill\ndescription: A test skill\n---\n\n# nested-skill\n", + ) + zf.writestr("nested-skill/references/very/deep/guide.md", "# Guide\n") + skills_root = tmp_path / "skills" + skills_root.mkdir() + # The staged skill root fits the limit; only the nested member exceeds it, + # as when the staging temp-dir name lengthens a path that fits at its + # final location. + monkeypatch.setattr( + "deerflow.skills.installer._host_path_length", + lambda path, **kwargs: 100 if path.name == "guide.md" else 10, + ) + monkeypatch.setattr( + "deerflow.skills.storage.local_skill_storage._publication_path_limit", + lambda root: 50, + ) + with pytest.raises(ValueError, match="publication path for 'references/very/deep/guide.md' is too long"): + get_or_new_skill_storage(skills_path=skills_root).install_skill_from_archive(zip_path) + assert not (skills_root / "custom" / "nested-skill").exists() + def test_success(self, tmp_path): zip_path = self._make_skill_zip(tmp_path) skills_root = tmp_path / "skills" @@ -948,3 +984,180 @@ class TestInstallSkillFromArchive: result = get_or_new_skill_storage(skills_path=skills_root).install_skill_from_archive(zip_path) assert result["success"] is True assert result["skill_name"] == "my-skill" + + +# --------------------------------------------------------------------------- +# Unextractable member trees +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "members", + [ + {"a": "file", "a/b": "nested"}, + {"a/b": "nested", "a": "file"}, + ], +) +def test_safe_extract_rejects_file_and_directory_collisions(tmp_path, members): + zip_path = tmp_path / "collision.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + for name, content in members.items(): + zf.writestr(name, content) + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="conflict"): + safe_extract_skill_archive(zf, dest) + assert not any(dest.rglob("*")) + + +@pytest.mark.parametrize("name", ["", "."]) +def test_safe_extract_rejects_members_that_normalize_to_the_root(tmp_path, name): + zip_path = tmp_path / "root.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr(name, "x") + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="not extractable"): + safe_extract_skill_archive(zf, dest) + + +def test_safe_extract_allows_explicit_directory_before_nested_file(tmp_path): + zip_path = tmp_path / "ok.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("a/", "") + zf.writestr("a/b.txt", "x") + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + safe_extract_skill_archive(zf, dest) + assert (dest / "a" / "b.txt").read_text() == "x" + + +def test_scan_archive_preflight_tolerates_empty_member_name(tmp_path): + from deerflow.skills.skillscan.orchestrator import scan_archive_preflight + + zip_path = tmp_path / "empty.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("", "x") + result = scan_archive_preflight(zip_path) + assert result is not None + + +@pytest.mark.parametrize( + "members", + [ + {"Report": "x", "report/slides/lesson.pptx": "y"}, + {"report/slides/lesson.pptx": "y", "Report": "x"}, + {"Café": "x", "Café/slides/a.pptx": "y"}, + {"Café/slides/a.pptx": "y", "Café": "x"}, + ], +) +def test_safe_extract_rejects_case_and_unicode_collisions(tmp_path, members): + zip_path = tmp_path / "fold.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + for name, content in members.items(): + zf.writestr(name, content) + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="conflict"): + safe_extract_skill_archive(zf, dest) + assert not any(dest.rglob("*")) + + +def test_safe_extract_rejects_overlong_path_segment(tmp_path): + zip_path = tmp_path / "long-segment.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("x" * 300 + "/a.txt", "x") + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="segment is too long"): + safe_extract_skill_archive(zf, dest) + + +def test_safe_extract_rejects_deep_paths(tmp_path): + zip_path = tmp_path / "deep.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("/".join(["d"] * 33) + "/a.txt", "x") + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="too deep"): + safe_extract_skill_archive(zf, dest) + + +def test_safe_extract_accepts_the_export_depth_boundary(tmp_path): + """A skill exported at export.py's 32-component limit must reinstall. + + The archive member adds the enclosing skill directory on top of those 32 + components, so the installer limit is 33. + """ + skill_name = "demo" + relative = "/".join(["d"] * 31) + "/a.txt" + zip_path = tmp_path / "boundary.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr(f"{skill_name}/{relative}", "x") + assert len(relative.split("/")) == 32 + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + safe_extract_skill_archive(zf, dest) + assert (dest / skill_name / relative).read_text() == "x" + + +def test_safe_extract_rejects_overlong_paths(tmp_path): + zip_path = tmp_path / "long-path.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("/".join(["d" * 40] * 30) + "/a.txt", "x") + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="too long"): + safe_extract_skill_archive(zf, dest) + + +@pytest.mark.parametrize( + "members", + [ + {"Notes": "first", "notes": "second"}, + {"notes": "second", "Notes": "first"}, + ], +) +def test_safe_extract_rejects_duplicate_folded_file_paths(tmp_path, members): + zip_path = tmp_path / "dup.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + for name, content in members.items(): + zf.writestr(name, content) + dest = tmp_path / "out" + dest.mkdir() + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="conflict"): + safe_extract_skill_archive(zf, dest) + assert not any(dest.rglob("*")) + + +def test_safe_extract_rejects_paths_over_the_host_limit(tmp_path, monkeypatch): + import os + + zip_path = tmp_path / "host-limit.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("skill/a.txt", "x") + dest = tmp_path / "out" + dest.mkdir() + monkeypatch.setattr( + "deerflow.skills.installer._host_path_limit", + lambda root: len(os.fsencode(str(root))) + 5, + ) + with zipfile.ZipFile(zip_path) as zf: + with pytest.raises(ValueError, match="destination filesystem"): + safe_extract_skill_archive(zf, dest) + + +def test_host_path_length_counts_windows_utf16_units(): + from deerflow.skills.installer import _host_path_length + + assert _host_path_length(Path("𝄞"), windows=True) == 2 + assert _host_path_length(Path("𝄞"), windows=False) == 4