deer-flow/backend/app/gateway/artifact_archive.py
Arif Rahman 9134cf26c6
fix(gateway): reject artifact archives where a file collides with an implicit directory (#7977) (#8232)
* fix(gateway): reject artifact archives where a file collides with an implicit directory

When a run outputs both a file and a deeper file path whose parent directory
matches the first file's normalized name (e.g. 'Report' and
'report/slides/lesson.pptx', or Unicode NFC/NFD variations), the ZIP archive
builder allowed both entries because it only checked for exact full path
equality. On case-insensitive filesystems, extracting such archives fails because
the file entry blocks directory creation or vice versa.

This change collects all implicit directory prefixes from normalized casefolded
entries and rejects the archive with HTTP 409 (ArtifactArchiveError) if any file
entry name collides with an implicit directory, regardless of receipt order.

Fixes #7977

* fix(gateway): address review feedback on test portability and loop readability
2026-11-15 20:28:29 +08:00

279 lines
10 KiB
Python

"""Fail-closed ZIP construction for files presented by one run."""
from __future__ import annotations
import os
import stat
import tempfile
import time
import unicodedata
import zipfile
from collections.abc import Iterable
from dataclasses import dataclass
from hashlib import sha256
from pathlib import Path
from typing import BinaryIO
from deerflow.constants import BROWSER_FRAMES_DIRNAME, TOOL_RESULTS_DIRNAME
_VIRTUAL_PREFIX = "mnt/user-data/outputs/"
_EDIT_TEMP_PREFIX = ".artifact-edit-"
_ALLOWED_FORMAT_CHARS = frozenset({"\u200c", "\u200d"})
_WINDOWS_INVALID_CHARS = frozenset('<>:"|?*')
_WINDOWS_DEVICE_NAMES = frozenset({"con", "prn", "aux", "nul"} | {f"com{number}" for number in range(1, 10)} | {f"lpt{number}" for number in range(1, 10)})
MAX_FILES = 50
MAX_FILE_BYTES = 50 * 1024 * 1024
MAX_TOTAL_BYTES = 100 * 1024 * 1024
MAX_ENTRY_BYTES = 1024
BUILD_TIMEOUT_SECONDS = 60.0
_CHUNK_BYTES = 1024 * 1024
class ArtifactArchiveError(ValueError):
def __init__(self, detail: str, status_code: int = 409) -> None:
super().__init__(detail)
self.detail = detail
self.status_code = status_code
@dataclass(frozen=True)
class ArtifactArchiveResult:
file: BinaryIO
size: int
member_count: int
input_bytes: int
@dataclass(frozen=True)
class _ArchiveMember:
path: Path
entry: str
initial: os.stat_result
components: tuple[tuple[Path, int, int], ...]
def _reject() -> ArtifactArchiveError:
return ArtifactArchiveError("The files listed by this response are not available for archive download")
def _too_large(detail: str) -> ArtifactArchiveError:
return ArtifactArchiveError(detail, 413)
def _check_deadline(deadline: float) -> None:
if time.monotonic() > deadline:
raise ArtifactArchiveError("Artifact archive creation timed out", 503)
def _is_link_like(path: Path, metadata: os.stat_result) -> bool:
return stat.S_ISLNK(metadata.st_mode) or path.is_junction()
def _member(
root: Path,
virtual_path: str,
reserved: frozenset[str],
deadline: float,
root_components: tuple[tuple[Path, int, int], ...],
) -> _ArchiveMember:
_check_deadline(deadline)
if not virtual_path or virtual_path.startswith("//") or "\\" in virtual_path or "\x00" in virtual_path:
raise _reject()
stripped = virtual_path.removeprefix("/")
if not stripped.startswith(_VIRTUAL_PREFIX):
raise _reject()
parts = stripped.removeprefix(_VIRTUAL_PREFIX).split("/")
if any(part in {"", ".", ".."} for part in parts):
raise _reject()
if any(any(char in _WINDOWS_INVALID_CHARS for char in part) or part.endswith((" ", ".")) or part.split(".", 1)[0].rstrip().casefold() in _WINDOWS_DEVICE_NAMES for part in parts):
raise _reject()
if any(any(unicodedata.category(char).startswith("C") and char not in _ALLOWED_FORMAT_CHARS for char in part) for part in parts):
raise _reject()
if any(part.casefold() in reserved or part.casefold().startswith(_EDIT_TEMP_PREFIX) for part in parts):
raise _reject()
if any(part.casefold().endswith(".skill") for part in parts[:-1]):
raise _reject()
entry = "/".join(parts)
if len(entry.encode()) > MAX_ENTRY_BYTES:
raise _too_large("An artifact path is too long to include in an archive")
candidate = root.joinpath(*parts)
current = root
components = list(root_components)
try:
for part in parts:
_check_deadline(deadline)
current /= part
metadata = os.lstat(current)
if _is_link_like(current, metadata):
raise _reject()
components.append((current, metadata.st_dev, metadata.st_ino))
initial = os.lstat(candidate)
if not stat.S_ISREG(initial.st_mode) or initial.st_nlink != 1:
raise _reject()
resolved = candidate.resolve(strict=True)
resolved.relative_to(root)
except ArtifactArchiveError:
raise
except (OSError, ValueError) as exc:
raise _reject() from exc
_check_deadline(deadline)
return _ArchiveMember(resolved, entry, initial, tuple(components))
def _hash_descriptor(descriptor: int, size: int, deadline: float) -> bytes:
digest = sha256()
remaining = size
try:
os.lseek(descriptor, 0, os.SEEK_SET)
while remaining:
_check_deadline(deadline)
chunk = os.read(descriptor, min(_CHUNK_BYTES, remaining))
if not chunk:
raise _reject()
digest.update(chunk)
remaining -= len(chunk)
if os.read(descriptor, 1):
raise _reject()
except OSError as exc:
raise _reject() from exc
return digest.digest()
def _copy_member(
archive: zipfile.ZipFile,
member: _ArchiveMember,
deadline: float,
remaining_total_bytes: int,
) -> int:
_check_deadline(deadline)
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0)
try:
descriptor = os.open(member.path, flags)
except OSError as exc:
raise _reject() from exc
try:
_check_deadline(deadline)
before = os.fstat(descriptor)
identity = (before.st_dev, before.st_ino)
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or identity != (member.initial.st_dev, member.initial.st_ino):
raise _reject()
if before.st_size > MAX_FILE_BYTES:
raise _too_large(f"Each archived artifact must be at most {MAX_FILE_BYTES} bytes")
if before.st_size > remaining_total_bytes:
raise _too_large(f"Archived artifacts must total at most {MAX_TOTAL_BYTES} bytes")
info = zipfile.ZipInfo(member.entry)
info.create_system = 0
info.compress_type = zipfile.ZIP_STORED
remaining = before.st_size
copied_digest = sha256()
with archive.open(info, "w", force_zip64=False) as destination:
while remaining:
_check_deadline(deadline)
chunk = os.read(descriptor, min(_CHUNK_BYTES, remaining))
if not chunk:
raise _reject()
destination.write(chunk)
copied_digest.update(chunk)
remaining -= len(chunk)
if os.read(descriptor, 1):
raise _reject()
_check_deadline(deadline)
after = os.fstat(descriptor)
if after.st_nlink != 1 or (after.st_dev, after.st_ino) != identity or (after.st_size, after.st_mtime_ns) != (before.st_size, before.st_mtime_ns):
raise _reject()
if _hash_descriptor(descriptor, before.st_size, deadline) != copied_digest.digest():
raise _reject()
after_verification = os.fstat(descriptor)
if after_verification.st_nlink != 1 or (after_verification.st_dev, after_verification.st_ino) != identity or (after_verification.st_size, after_verification.st_mtime_ns) != (before.st_size, before.st_mtime_ns):
raise _reject()
try:
for component, device, inode in member.components:
current = os.lstat(component)
if _is_link_like(component, current) or (current.st_dev, current.st_ino) != (device, inode):
raise _reject()
except OSError as exc:
raise _reject() from exc
return before.st_size
finally:
os.close(descriptor)
def build_artifact_archive(
outputs_dir: Path,
virtual_paths: Iterable[str],
*,
user_data_dir: Path,
extra_reserved_dir_names: Iterable[str] = (),
) -> ArtifactArchiveResult:
deadline = time.monotonic() + BUILD_TIMEOUT_SECONDS
try:
if outputs_dir.parent != user_data_dir:
raise _reject()
user_data_metadata = os.lstat(user_data_dir)
outputs_metadata = os.lstat(outputs_dir)
if _is_link_like(user_data_dir, user_data_metadata) or not stat.S_ISDIR(user_data_metadata.st_mode) or _is_link_like(outputs_dir, outputs_metadata) or not stat.S_ISDIR(outputs_metadata.st_mode):
raise _reject()
user_data_root = user_data_dir.resolve(strict=True)
root = outputs_dir.resolve(strict=True)
if root.parent != user_data_root:
raise _reject()
except ArtifactArchiveError:
raise
except OSError as exc:
raise _reject() from exc
_check_deadline(deadline)
root_components = (
(user_data_dir, user_data_metadata.st_dev, user_data_metadata.st_ino),
(outputs_dir, outputs_metadata.st_dev, outputs_metadata.st_ino),
)
paths = list(dict.fromkeys(virtual_paths))
if not paths:
raise _reject()
if len(paths) > MAX_FILES:
raise _too_large(f"An artifact archive can contain at most {MAX_FILES} files")
reserved = frozenset(name.casefold() for name in {BROWSER_FRAMES_DIRNAME, TOOL_RESULTS_DIRNAME, *extra_reserved_dir_names})
members = [_member(root, path, reserved, deadline, root_components) for path in paths]
collision_keys = [unicodedata.normalize("NFC", member.entry).casefold() for member in members]
collision_set = set(collision_keys)
if len(collision_keys) != len(collision_set):
raise _reject()
implicit_dirs = set()
for key in collision_keys:
parts = key.split("/")
for i in range(1, len(parts)):
implicit_dirs.add("/".join(parts[:i]))
if collision_set & implicit_dirs:
raise _reject()
sizes = [member.initial.st_size for member in members]
if any(size > MAX_FILE_BYTES for size in sizes):
raise _too_large(f"Each archived artifact must be at most {MAX_FILE_BYTES} bytes")
if sum(sizes) > MAX_TOTAL_BYTES:
raise _too_large(f"Archived artifacts must total at most {MAX_TOTAL_BYTES} bytes")
_check_deadline(deadline)
output = tempfile.TemporaryFile("w+b")
try:
if hasattr(os, "fchmod"):
os.fchmod(output.fileno(), 0o600)
input_bytes = 0
with zipfile.ZipFile(output, "w", zipfile.ZIP_STORED, allowZip64=False) as archive:
for member in members:
input_bytes += _copy_member(archive, member, deadline, MAX_TOTAL_BYTES - input_bytes)
_check_deadline(deadline)
size = output.tell()
output.seek(0)
return ArtifactArchiveResult(output, size, len(members), input_bytes)
except Exception:
output.close()
raise