mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-20 03:10:34 +00:00
fix(storage): reject trailing newlines in blob kinds (#8287)
* fix(storage): anchor the blob-kind pattern at end of string, not before a newline In Python, "$" also matches just before a trailing newline, so the blob kind pattern accepted "tool\n" and 64 chars + "\n" (65 bytes) even though the contract grammar is 1-64 chars and the kind is used as a filesystem path segment by the local_fs backend. Anchor with "\Z" instead. * fix(storage): report the enforced blob-kind pattern --------- Co-authored-by: laoyyu <laoyyu@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
parent
f95d4f6353
commit
1cd1a8ac63
@ -21,7 +21,7 @@ On a single gateway both are correct. Behind a load balancer, the instance handl
|
||||
|
||||
Reads **verify the digest**. A content-addressed store that silently returns wrong bytes is indistinguishable from a corrupt checkpoint, so it fails loudly instead.
|
||||
|
||||
`kind` is validated by `validate_blob_kind` against `^[a-z0-9]([a-z0-9-]*[a-z0-9])?$` (1-64 chars, no leading or trailing hyphen) at the contract level because it is also a path segment in the `local_fs` backend — traversal and separator surprises are ruled out once, not per backend, and the single validator is what a rejected producer sees.
|
||||
`validate_blob_kind` uses the Python regex `^(?=.{1,64}\Z)[a-z0-9]([a-z0-9-]*[a-z0-9])?\Z` and reports the same pattern in validation errors. A kind is 1–64 characters long and may contain lowercase ASCII letters, digits, and hyphens. It must start and end with a letter or digit. The `\Z` anchor requires the match to end at the last character, so trailing newlines are rejected. This grammar keeps `kind` safe as a path segment in the `local_fs` backend.
|
||||
|
||||
## Backend
|
||||
|
||||
|
||||
@ -29,11 +29,12 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
# trailing hyphen, 1-64 chars. That rules out traversal (".."), separators and
|
||||
# case-sensitivity surprises on NTFS/APFS at the contract level instead of
|
||||
# leaving every backend to re-check it.
|
||||
_KIND_PATTERN = re.compile(r"^(?=.{1,64}$)[a-z0-9]([a-z0-9-]*[a-z0-9])?$")
|
||||
# The grammar as a caller sees it. One text, one place: this message is the only
|
||||
# thing a producer gets when a kind is rejected, and a second copy of the
|
||||
# pattern (the local_fs backend had one) drifts from the enforced one.
|
||||
_KIND_GRAMMAR = "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ (1-64 chars, no leading or trailing hyphen)"
|
||||
# ``\Z`` anchors the match at the very end of the string: a ``$`` anchor
|
||||
# would also match just before a trailing newline, letting kinds like
|
||||
# ``"tool\n"`` or 64 chars + ``"\n"`` (65 bytes) through the 1-64 rule.
|
||||
_KIND_PATTERN = re.compile(r"^(?=.{1,64}\Z)[a-z0-9]([a-z0-9-]*[a-z0-9])?\Z")
|
||||
# Use the validation pattern in the error so the advertised grammar stays in sync.
|
||||
_KIND_GRAMMAR = f"{_KIND_PATTERN.pattern} (1-64 chars, no leading or trailing hyphen)"
|
||||
|
||||
|
||||
def is_valid_blob_kind(kind: str) -> bool:
|
||||
|
||||
@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import errno
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
@ -69,7 +70,7 @@ def test_blob_ref_validates_digest_shape():
|
||||
|
||||
|
||||
def test_blob_ref_kind_is_path_safe():
|
||||
for bad in ("../etc", "a/b", "Aupper", "", "-lead", "trailing-", "has space", "x" * 65):
|
||||
for bad in ("../etc", "a/b", "Aupper", "", "-lead", "trailing-", "has space", "x" * 65, "tool\n", "a" * 64 + "\n"):
|
||||
with pytest.raises(ValidationError):
|
||||
BlobRef(sha256=hashlib.sha256(b"x").hexdigest(), size=1, kind=bad)
|
||||
for good in ("viewed-image", "tool-output", "a", "0", "a" * 64):
|
||||
@ -133,14 +134,19 @@ def test_dedup_records_only_the_first_writer_as_advisory_provenance(store: Local
|
||||
def test_kind_rejection_quotes_the_enforced_grammar():
|
||||
"""One validator, one message: the pattern a producer is shown must be the
|
||||
pattern that is enforced (it was not, before this was centralised)."""
|
||||
assert validate_blob_kind("tool-output") == "tool-output"
|
||||
for bad in ("trailing-", "a--", "-lead", "Aupper", "a/b", "", "x" * 65):
|
||||
valid_kind = "tool-output"
|
||||
assert validate_blob_kind(valid_kind) == valid_kind
|
||||
for bad in ("trailing-", "a--", "-lead", "Aupper", "a/b", "", "x" * 65, "tool\n", "a" * 64 + "\n"):
|
||||
with pytest.raises(ValueError) as caught:
|
||||
validate_blob_kind(bad)
|
||||
message = str(caught.value)
|
||||
assert "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" in message, message
|
||||
assert "1-64 chars" in message, message
|
||||
assert "no leading or trailing hyphen" in message, message
|
||||
prefix = "kind must match "
|
||||
assert message.startswith(prefix), message
|
||||
pattern, _, description = message.removeprefix(prefix).partition(" ")
|
||||
assert re.match(pattern, valid_kind)
|
||||
assert not re.match(pattern, bad), message
|
||||
assert "1-64 chars" in description, message
|
||||
assert "no leading or trailing hyphen" in description, message
|
||||
assert validate_blob_kind is contract_module.validate_blob_kind
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user