fix(uploads): keep dedupe counter before compound archive suffix (#8042)

Re-uploading a file whose name collides with a compound archive suffix
(e.g. .tar.gz, .tar.bz2) previously inserted the dedupe counter inside
the suffix: a second report.tar.gz became report.tar_1.gz, which broke
file-type detection. Treat the compound suffix as a single extension so
the counter lands before it (report_1.tar.gz), matching single-suffix
behavior (report_1.tar, report_1.txt).

- Add _COMPOUND_SUFFIXES and split stem/suffix as one unit, sorted
  longest-first with a closed, non-overlapping allowlist (.tar.br added).
- Expand tests to all 13 suffixes plus counter-escalation and plain .tar.
- Add a CHANGELOG [Unreleased] Fixed entry.

Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
hdu_zhou 2026-11-10 09:52:01 +08:00 committed by GitHub
parent 54753636b1
commit bfa2794c76
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 101 additions and 1 deletions

View File

@ -58,6 +58,12 @@ This section accumulates work toward the **2.3.0** milestone
they carry per-call environment variables, including Lark CLI commands.
Commands within one subagent remain ordered and client cleanup drains them.
- **uploads:** Re-uploading a file whose name collides with a compound archive
suffix no longer inserts the dedupe counter inside the suffix. A second
`report.tar.gz` is now named `report_1.tar.gz` (matching `report_1.tar` and
`report_1.txt`), instead of `report.tar_1.gz`, so the extension stays intact
for file-type detection. ([#8042](https://github.com/bytedance/deer-flow/pull/8042))
- **gateway:** Deleting a thread no longer reports success while its
conversation stays readable. When the checkpoint store failed during
`DELETE /api/threads/{id}`, the route logged it at DEBUG, removed the

View File

@ -42,6 +42,44 @@ UPLOAD_STAGING_MIN_AGE = timedelta(hours=24)
_MAX_FILENAME_BYTES = 255
# Multi-segment archive suffixes that must be treated as a single extension
# when deduplicating, so ``report.tar.gz`` becomes ``report_1.tar.gz`` instead
# of ``report.tar_1.gz``. Sorted longest-first so the longest match wins; the
# list is deliberately closed (not every compression format is listed), and no
# entry is a proper suffix of another, so a future addition (e.g. ``.tar.lzo``
# beside ``.tar.lz``) stays safe regardless of position.
_COMPOUND_SUFFIXES = (
".tar.lzma",
".tar.bz2",
".tar.zst",
".tar.lz4",
".tar.br",
".tar.gz",
".tar.lz",
".tar.xz",
".tar.z",
".tbz2",
".tzst",
".tgz",
".txz",
)
def _split_stem_suffix(name: str) -> tuple[str, str]:
"""Split *name* into ``(stem, suffix)``.
Unlike :meth:`pathlib.Path.stem`/:meth:`pathlib.Path.suffix`, which only
strip the final extension, this keeps known compound archive suffixes
(``.tar.gz``, ``.tgz``, ...) intact so a deduplication counter lands
before the whole archive suffix.
"""
lowered = name.casefold()
for compound in _COMPOUND_SUFFIXES:
if lowered.endswith(compound):
return name[: -len(compound)], name[-len(compound) :]
path = Path(name)
return path.stem, path.suffix
def get_uploads_dir(thread_id: str, *, user_id: str | None = None) -> Path:
"""Return the uploads directory path for a thread (no side effects)."""
@ -122,7 +160,7 @@ def claim_unique_filename(name: str, seen: set[str]) -> str:
if name.casefold() not in claimed:
seen.add(name)
return name
stem, suffix = Path(name).stem, Path(name).suffix
stem, suffix = _split_stem_suffix(name)
counter = 1
while True:
tag = f"_{counter}"

View File

@ -200,6 +200,62 @@ class TestDeduplicateFilename:
seen = {"data.txt"}
assert claim_unique_filename("data.txt", seen) == "data_1.txt"
@pytest.mark.parametrize(
("name", "expected"),
[
("report.tar.gz", "report_1.tar.gz"),
("report.tar.bz2", "report_1.tar.bz2"),
("report.tar.xz", "report_1.tar.xz"),
("report.tar.zst", "report_1.tar.zst"),
("report.tar.lz", "report_1.tar.lz"),
("report.tar.lz4", "report_1.tar.lz4"),
("report.tar.lzma", "report_1.tar.lzma"),
("report.tar.z", "report_1.tar.z"),
("report.tar.br", "report_1.tar.br"),
("report.tgz", "report_1.tgz"),
("report.tbz2", "report_1.tbz2"),
("report.txz", "report_1.txz"),
("report.tzst", "report_1.tzst"),
],
)
def test_compound_archive_suffix_keeps_counter_before_suffix(
self,
name: str,
expected: str,
):
"""The ``_N`` counter lands before the whole archive suffix, not
inside it (``report.tar_1.gz`` is wrong)."""
seen = {name}
assert claim_unique_filename(name, seen) == expected
assert expected in seen
def test_plain_tar_suffix_keeps_original_dedupe_shape(self):
"""A plain ``.tar`` (single suffix, not in the compound list) still
dedupes as ``report_1.tar``."""
seen = {"report.tar"}
assert claim_unique_filename("report.tar", seen) == "report_1.tar"
assert "report_1.tar" in seen
def test_compound_suffix_counter_escalation(self):
"""A second collision re-enters the compound split, producing
``report_2.tar.gz`` rather than mangling the suffix."""
seen = {"report.tar.gz", "report_1.tar.gz"}
assert claim_unique_filename("report.tar.gz", seen) == "report_2.tar.gz"
assert "report_2.tar.gz" in seen
def test_compound_suffix_collision_is_case_insensitive(self):
seen = {"Report.Tar.Gz"}
assert claim_unique_filename("report.tar.gz", seen) == "report_1.tar.gz"
def test_compound_suffix_max_length_stays_within_filename_limit(self):
# 250-byte stem + 7-byte ".tar.gz" = 257 bytes; dedupe must truncate
# the stem so the counter + suffix still fit in 255 bytes.
name = "a" * 250 + ".tar.gz"
seen = {name}
deduped = claim_unique_filename(name, seen)
assert deduped.endswith(".tar.gz")
assert len(deduped.encode("utf-8")) <= 255
# ---------------------------------------------------------------------------
# validate_path_traversal