mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
fix(skills): reject unextractable archive member trees (#8302)
* fix(skills): reject unextractable archive member trees An archive with a file that is also an ancestor directory, or a member that normalizes to the extraction root, crashed extraction with OSError and the scan preflight raised IndexError on an empty member name, so the install route returned 500 instead of a validation 400. Validate the whole member tree before writing anything, comparing NFC+casefolded keys so case-insensitive and normalization-sensitive filesystems cannot create the same collision, and enforce the export path/depth/segment limits so overlong paths fail as validation errors instead of OS errors. * fix(skills): align archive depth limit with export export.py limits paths to 32 components relative to the skill root; an archive member adds the enclosing skill directory, so a valid 32-component export was rejected on reinstall. Allow 33 archive components and cover the boundary with a round-trip extraction test. * fix(skills): reject folded duplicates and enforce the host path limit Two case-variant file entries collapse into one file on a case-insensitive filesystem, and a long member path can still exceed the destination filesystem limit once the staging root is prefixed, raising ENAMETOOLONG after a partial write. Reject duplicate folded file keys and validate the full destination path before extraction. * fix(skills): measure host paths in platform units and guard publication Win32 limits are counted in UTF-16 code units and long paths are opt-in, while the previous check compared UTF-8 bytes against 260. Measure with the platform units and read LongPathsEnabled. Also reject a publication staging path that exceeds the host limit and translate ENAMETOOLONG during staging into the installer validation error. * fix(skills): reject nested members whose staged path is too long The staging temp-dir name lengthens every member path, so a nested member can exceed the filesystem path limit while the skill root and the final path fit. shutil.copytree wraps a child's ENAMETOOLONG in shutil.Error, which carries no errno or winerror, so the existing translation let it escape as a 500. Check each member's full staging path before copying.
This commit is contained in:
parent
2cc30714c9
commit
b6515b1415
@ -11,6 +11,7 @@ import os
|
||||
import posixpath
|
||||
import shutil
|
||||
import stat
|
||||
import unicodedata
|
||||
import zipfile
|
||||
from pathlib import Path, PurePosixPath, PureWindowsPath
|
||||
|
||||
@ -29,6 +30,50 @@ from deerflow.utils.host_paths import windows_incompatible_segment
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Mirror the path limits export.py applies to installed skills, so an archive
|
||||
# that cannot be listed or extracted fails as a validation error instead of an
|
||||
# OS error during installation. Export limits depth relative to the skill root,
|
||||
# so the archive limit adds the enclosing skill directory component.
|
||||
_MAX_MEMBER_PATH_BYTES = 1024
|
||||
_MAX_MEMBER_DEPTH = 33
|
||||
_MAX_MEMBER_SEGMENT_BYTES = 255
|
||||
|
||||
|
||||
def _host_path_length(path: Path, *, windows: bool | None = None) -> int:
|
||||
"""Return a path's length in the units the host filesystem limits."""
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
if windows:
|
||||
# Win32 path limits count UTF-16 code units, not UTF-8 bytes.
|
||||
return len(str(path).encode("utf-16-le")) // 2
|
||||
return len(os.fsencode(path))
|
||||
|
||||
|
||||
def _host_path_limit(root: Path, *, windows: bool | None = None) -> int:
|
||||
"""Return the destination filesystem's maximum path length in host units."""
|
||||
if windows is None:
|
||||
windows = os.name == "nt"
|
||||
if windows:
|
||||
try:
|
||||
import winreg
|
||||
|
||||
with winreg.OpenKey(
|
||||
winreg.HKEY_LOCAL_MACHINE,
|
||||
r"SYSTEM\CurrentControlSet\Control\FileSystem",
|
||||
) as key:
|
||||
if winreg.QueryValueEx(key, "LongPathsEnabled")[0]:
|
||||
return 32767
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
return 260
|
||||
if hasattr(os, "pathconf"):
|
||||
try:
|
||||
return os.pathconf(root, "PC_PATH_MAX")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return 4096
|
||||
|
||||
|
||||
_PROMPT_INPUT_DIRS = {"references", "templates"}
|
||||
_PROMPT_INPUT_SUFFIXES = frozenset({".json", ".markdown", ".md", ".rst", ".txt", ".yaml", ".yml"})
|
||||
|
||||
@ -137,6 +182,7 @@ def safe_extract_skill_archive(
|
||||
ValueError: If unsafe members, executable binaries, entry count, or size limit exceeded.
|
||||
"""
|
||||
dest_root = dest_path.resolve()
|
||||
host_path_limit = _host_path_limit(dest_root)
|
||||
total_written = 0
|
||||
|
||||
infos = zip_ref.infolist()
|
||||
@ -149,6 +195,11 @@ def safe_extract_skill_archive(
|
||||
# it lives in the extraction path every install goes through.
|
||||
raise ValueError(f"Skill archive contains too many entries ({len(infos)} > {max_entries}).")
|
||||
|
||||
# Validate every member before writing anything, so an archive whose paths
|
||||
# cannot form a file tree fails without leaving a partial install.
|
||||
planned: list[tuple[zipfile.ZipInfo, tuple[str, ...]]] = []
|
||||
seen_files: set[tuple[str, ...]] = set()
|
||||
seen_dirs: set[tuple[str, ...]] = set()
|
||||
for info in infos:
|
||||
if is_unsafe_zip_member(info):
|
||||
raise ValueError(f"Archive contains unsafe member path: {info.filename!r}")
|
||||
@ -158,14 +209,45 @@ def safe_extract_skill_archive(
|
||||
continue
|
||||
|
||||
normalized_name = posixpath.normpath(info.filename.replace("\\", "/"))
|
||||
for segment in PurePosixPath(normalized_name).parts:
|
||||
parts = PurePosixPath(normalized_name).parts
|
||||
if not parts or normalized_name in ("", "."):
|
||||
raise ValueError(f"Archive member path is not extractable: {info.filename!r}")
|
||||
if len(parts) > _MAX_MEMBER_DEPTH:
|
||||
raise ValueError(f"Archive member path is too deep: {info.filename!r}")
|
||||
if len(normalized_name.encode("utf-8")) > _MAX_MEMBER_PATH_BYTES:
|
||||
raise ValueError(f"Archive member path is too long: {info.filename!r}")
|
||||
for segment in parts:
|
||||
if len(segment.encode("utf-8")) > _MAX_MEMBER_SEGMENT_BYTES:
|
||||
raise ValueError(f"Archive member path segment is too long: {info.filename!r}")
|
||||
reason = windows_incompatible_segment(segment)
|
||||
if reason is not None:
|
||||
# Extraction creates one host path component per segment; a
|
||||
# Windows device name cannot become one, and the same names are
|
||||
# already refused for support-file writes and skill names.
|
||||
raise ValueError(f"Archive member path is not portable to Windows: {info.filename!r} ({reason})")
|
||||
member_path = dest_root.joinpath(*PurePosixPath(normalized_name).parts)
|
||||
|
||||
if _host_path_length(dest_root.joinpath(*parts)) >= host_path_limit:
|
||||
raise ValueError(f"Archive member path is too long for the destination filesystem: {info.filename!r}")
|
||||
|
||||
# Compare NFC+casefolded keys so case-insensitive and normalization-
|
||||
# sensitive filesystems cannot turn a distinct path into a collision.
|
||||
key = tuple(unicodedata.normalize("NFC", part).casefold() for part in parts)
|
||||
if any(key[:index] in seen_files for index in range(1, len(key))):
|
||||
raise ValueError(f"Archive member path conflicts with a file member: {info.filename!r}")
|
||||
if info.is_dir():
|
||||
if key in seen_files:
|
||||
raise ValueError(f"Archive contains conflicting member paths: {info.filename!r}")
|
||||
seen_dirs.add(key)
|
||||
else:
|
||||
if key in seen_dirs or key in seen_files:
|
||||
raise ValueError(f"Archive contains conflicting member paths: {info.filename!r}")
|
||||
seen_files.add(key)
|
||||
for index in range(1, len(key)):
|
||||
seen_dirs.add(key[:index])
|
||||
planned.append((info, parts))
|
||||
|
||||
for info, parts in planned:
|
||||
member_path = dest_root.joinpath(*parts)
|
||||
if not member_path.resolve().is_relative_to(dest_root):
|
||||
raise ValueError(f"Zip entry escapes destination: {info.filename!r}")
|
||||
member_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@ -1381,6 +1381,8 @@ def _relative_file(path: Path, root: Path) -> str:
|
||||
|
||||
def _is_hidden_sensitive_path(rel_path: str) -> bool:
|
||||
parts = PurePosixPath(rel_path).parts
|
||||
if not parts:
|
||||
return False
|
||||
if ".aws" in parts and parts[-1] == "credentials":
|
||||
return True
|
||||
if ".git" in parts and parts[-1] == "config":
|
||||
|
||||
@ -21,6 +21,14 @@ from deerflow.utils.atomic_write import cleanup_failed_atomic_write
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _publication_path_limit(root: Path) -> int:
|
||||
"""Host path limit for the staged publication directory (indirection for tests)."""
|
||||
from deerflow.skills.installer import _host_path_limit
|
||||
|
||||
return _host_path_limit(root)
|
||||
|
||||
|
||||
# Bound for the best-effort temp-dir cleanup so a stalled filesystem (e.g. NFS)
|
||||
# cannot hold back the install outcome propagating out of the finally block.
|
||||
_INSTALL_TMP_CLEANUP_TIMEOUT_SECONDS = 5.0
|
||||
@ -218,13 +226,30 @@ class LocalSkillStorage(SkillStorage):
|
||||
|
||||
def _commit_skill_install(self, skill_dir: Path, skill_name: str, custom_dir: Path, target: Path) -> None:
|
||||
"""Stage and move the validated skill into place (blocking; runs off the event loop)."""
|
||||
from deerflow.skills.installer import _move_staged_skill_into_reserved_target
|
||||
from deerflow.skills.installer import _host_path_length, _move_staged_skill_into_reserved_target
|
||||
|
||||
with self._skill_projection_mutation(names=(skill_name,)):
|
||||
with tempfile.TemporaryDirectory(prefix=f".installing-{skill_name}-", dir=custom_dir) as staging_root:
|
||||
staging_target = Path(staging_root) / skill_name
|
||||
shutil.copytree(skill_dir, staging_target)
|
||||
_move_staged_skill_into_reserved_target(staging_target, target)
|
||||
publication_limit = _publication_path_limit(custom_dir)
|
||||
if _host_path_length(staging_target) >= publication_limit:
|
||||
raise ValueError(f"Skill '{skill_name}' cannot be published: the temporary publication path is too long for this filesystem")
|
||||
# The staging root's temp-dir name lengthens every member path, so
|
||||
# a nested member can outgrow the limit while the skill root fits.
|
||||
# Check members before copying: copytree wraps a child's
|
||||
# ENAMETOOLONG in shutil.Error, which carries no errno/winerror for
|
||||
# the translation below to recognize.
|
||||
for member in skill_dir.rglob("*"):
|
||||
relative = member.relative_to(skill_dir)
|
||||
if _host_path_length(staging_target / relative) >= publication_limit:
|
||||
raise ValueError(f"Skill '{skill_name}' cannot be published: the temporary publication path for '{relative.as_posix()}' is too long for this filesystem")
|
||||
try:
|
||||
shutil.copytree(skill_dir, staging_target)
|
||||
_move_staged_skill_into_reserved_target(staging_target, target)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ENAMETOOLONG or getattr(exc, "winerror", None) == 206:
|
||||
raise ValueError(f"Skill '{skill_name}' cannot be published: the path is too long for this filesystem") from exc
|
||||
raise
|
||||
make_skill_written_path_sandbox_readable(custom_dir, target)
|
||||
|
||||
def delete_custom_skill(self, name: str, *, history_meta: dict | None = None) -> None:
|
||||
|
||||
@ -463,6 +463,42 @@ class TestInstallSkillFromArchive:
|
||||
)
|
||||
return zip_path
|
||||
|
||||
def test_rejects_publication_path_too_long(self, tmp_path, monkeypatch):
|
||||
zip_path = self._make_skill_zip(tmp_path)
|
||||
skills_root = tmp_path / "skills"
|
||||
skills_root.mkdir()
|
||||
monkeypatch.setattr(
|
||||
"deerflow.skills.storage.local_skill_storage._publication_path_limit",
|
||||
lambda root: 1,
|
||||
)
|
||||
with pytest.raises(ValueError, match="publication path is too long"):
|
||||
get_or_new_skill_storage(skills_path=skills_root).install_skill_from_archive(zip_path)
|
||||
|
||||
def test_rejects_nested_member_whose_staging_path_is_too_long(self, tmp_path, monkeypatch):
|
||||
zip_path = tmp_path / "nested-skill.skill"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr(
|
||||
"nested-skill/SKILL.md",
|
||||
"---\nname: nested-skill\ndescription: A test skill\n---\n\n# nested-skill\n",
|
||||
)
|
||||
zf.writestr("nested-skill/references/very/deep/guide.md", "# Guide\n")
|
||||
skills_root = tmp_path / "skills"
|
||||
skills_root.mkdir()
|
||||
# The staged skill root fits the limit; only the nested member exceeds it,
|
||||
# as when the staging temp-dir name lengthens a path that fits at its
|
||||
# final location.
|
||||
monkeypatch.setattr(
|
||||
"deerflow.skills.installer._host_path_length",
|
||||
lambda path, **kwargs: 100 if path.name == "guide.md" else 10,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"deerflow.skills.storage.local_skill_storage._publication_path_limit",
|
||||
lambda root: 50,
|
||||
)
|
||||
with pytest.raises(ValueError, match="publication path for 'references/very/deep/guide.md' is too long"):
|
||||
get_or_new_skill_storage(skills_path=skills_root).install_skill_from_archive(zip_path)
|
||||
assert not (skills_root / "custom" / "nested-skill").exists()
|
||||
|
||||
def test_success(self, tmp_path):
|
||||
zip_path = self._make_skill_zip(tmp_path)
|
||||
skills_root = tmp_path / "skills"
|
||||
@ -948,3 +984,180 @@ class TestInstallSkillFromArchive:
|
||||
result = get_or_new_skill_storage(skills_path=skills_root).install_skill_from_archive(zip_path)
|
||||
assert result["success"] is True
|
||||
assert result["skill_name"] == "my-skill"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unextractable member trees
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"members",
|
||||
[
|
||||
{"a": "file", "a/b": "nested"},
|
||||
{"a/b": "nested", "a": "file"},
|
||||
],
|
||||
)
|
||||
def test_safe_extract_rejects_file_and_directory_collisions(tmp_path, members):
|
||||
zip_path = tmp_path / "collision.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
for name, content in members.items():
|
||||
zf.writestr(name, content)
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
assert not any(dest.rglob("*"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["", "."])
|
||||
def test_safe_extract_rejects_members_that_normalize_to_the_root(tmp_path, name):
|
||||
zip_path = tmp_path / "root.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr(name, "x")
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="not extractable"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
|
||||
|
||||
def test_safe_extract_allows_explicit_directory_before_nested_file(tmp_path):
|
||||
zip_path = tmp_path / "ok.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr("a/", "")
|
||||
zf.writestr("a/b.txt", "x")
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
assert (dest / "a" / "b.txt").read_text() == "x"
|
||||
|
||||
|
||||
def test_scan_archive_preflight_tolerates_empty_member_name(tmp_path):
|
||||
from deerflow.skills.skillscan.orchestrator import scan_archive_preflight
|
||||
|
||||
zip_path = tmp_path / "empty.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr("", "x")
|
||||
result = scan_archive_preflight(zip_path)
|
||||
assert result is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"members",
|
||||
[
|
||||
{"Report": "x", "report/slides/lesson.pptx": "y"},
|
||||
{"report/slides/lesson.pptx": "y", "Report": "x"},
|
||||
{"Café": "x", "Café/slides/a.pptx": "y"},
|
||||
{"Café/slides/a.pptx": "y", "Café": "x"},
|
||||
],
|
||||
)
|
||||
def test_safe_extract_rejects_case_and_unicode_collisions(tmp_path, members):
|
||||
zip_path = tmp_path / "fold.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
for name, content in members.items():
|
||||
zf.writestr(name, content)
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
assert not any(dest.rglob("*"))
|
||||
|
||||
|
||||
def test_safe_extract_rejects_overlong_path_segment(tmp_path):
|
||||
zip_path = tmp_path / "long-segment.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr("x" * 300 + "/a.txt", "x")
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="segment is too long"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
|
||||
|
||||
def test_safe_extract_rejects_deep_paths(tmp_path):
|
||||
zip_path = tmp_path / "deep.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr("/".join(["d"] * 33) + "/a.txt", "x")
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="too deep"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
|
||||
|
||||
def test_safe_extract_accepts_the_export_depth_boundary(tmp_path):
|
||||
"""A skill exported at export.py's 32-component limit must reinstall.
|
||||
|
||||
The archive member adds the enclosing skill directory on top of those 32
|
||||
components, so the installer limit is 33.
|
||||
"""
|
||||
skill_name = "demo"
|
||||
relative = "/".join(["d"] * 31) + "/a.txt"
|
||||
zip_path = tmp_path / "boundary.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr(f"{skill_name}/{relative}", "x")
|
||||
assert len(relative.split("/")) == 32
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
assert (dest / skill_name / relative).read_text() == "x"
|
||||
|
||||
|
||||
def test_safe_extract_rejects_overlong_paths(tmp_path):
|
||||
zip_path = tmp_path / "long-path.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr("/".join(["d" * 40] * 30) + "/a.txt", "x")
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="too long"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"members",
|
||||
[
|
||||
{"Notes": "first", "notes": "second"},
|
||||
{"notes": "second", "Notes": "first"},
|
||||
],
|
||||
)
|
||||
def test_safe_extract_rejects_duplicate_folded_file_paths(tmp_path, members):
|
||||
zip_path = tmp_path / "dup.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
for name, content in members.items():
|
||||
zf.writestr(name, content)
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="conflict"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
assert not any(dest.rglob("*"))
|
||||
|
||||
|
||||
def test_safe_extract_rejects_paths_over_the_host_limit(tmp_path, monkeypatch):
|
||||
import os
|
||||
|
||||
zip_path = tmp_path / "host-limit.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
zf.writestr("skill/a.txt", "x")
|
||||
dest = tmp_path / "out"
|
||||
dest.mkdir()
|
||||
monkeypatch.setattr(
|
||||
"deerflow.skills.installer._host_path_limit",
|
||||
lambda root: len(os.fsencode(str(root))) + 5,
|
||||
)
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
with pytest.raises(ValueError, match="destination filesystem"):
|
||||
safe_extract_skill_archive(zf, dest)
|
||||
|
||||
|
||||
def test_host_path_length_counts_windows_utf16_units():
|
||||
from deerflow.skills.installer import _host_path_length
|
||||
|
||||
assert _host_path_length(Path("𝄞"), windows=True) == 2
|
||||
assert _host_path_length(Path("𝄞"), windows=False) == 4
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user