From 1cd1a8ac6374d20ea4912c5cd54bdbc5a8dd303d Mon Sep 17 00:00:00 2001 From: laoyyu Date: Tue, 17 Nov 2026 15:59:09 +0800 Subject: [PATCH] fix(storage): reject trailing newlines in blob kinds (#8287) * fix(storage): anchor the blob-kind pattern at end of string, not before a newline In Python, "$" also matches just before a trailing newline, so the blob kind pattern accepted "tool\n" and 64 chars + "\n" (65 bytes) even though the contract grammar is 1-64 chars and the kind is used as a filesystem path segment by the local_fs backend. Anchor with "\Z" instead. * fix(storage): report the enforced blob-kind pattern --------- Co-authored-by: laoyyu Co-authored-by: Willem Jiang --- backend/docs/blob-storage.md | 2 +- .../harness/deerflow/storage/contract.py | 11 ++++++----- backend/tests/test_blob_store.py | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/backend/docs/blob-storage.md b/backend/docs/blob-storage.md index c825c210cf..fa1f61a695 100644 --- a/backend/docs/blob-storage.md +++ b/backend/docs/blob-storage.md @@ -21,7 +21,7 @@ On a single gateway both are correct. Behind a load balancer, the instance handl Reads **verify the digest**. A content-addressed store that silently returns wrong bytes is indistinguishable from a corrupt checkpoint, so it fails loudly instead. -`kind` is validated by `validate_blob_kind` against `^[a-z0-9]([a-z0-9-]*[a-z0-9])?$` (1-64 chars, no leading or trailing hyphen) at the contract level because it is also a path segment in the `local_fs` backend — traversal and separator surprises are ruled out once, not per backend, and the single validator is what a rejected producer sees. +`validate_blob_kind` uses the Python regex `^(?=.{1,64}\Z)[a-z0-9]([a-z0-9-]*[a-z0-9])?\Z` and reports the same pattern in validation errors. A kind is 1–64 characters long and may contain lowercase ASCII letters, digits, and hyphens. It must start and end with a letter or digit. The `\Z` anchor requires the match to end at the last character, so trailing newlines are rejected. This grammar keeps `kind` safe as a path segment in the `local_fs` backend. ## Backend diff --git a/backend/packages/harness/deerflow/storage/contract.py b/backend/packages/harness/deerflow/storage/contract.py index 9c6b889d40..2be15224cc 100644 --- a/backend/packages/harness/deerflow/storage/contract.py +++ b/backend/packages/harness/deerflow/storage/contract.py @@ -29,11 +29,12 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator # trailing hyphen, 1-64 chars. That rules out traversal (".."), separators and # case-sensitivity surprises on NTFS/APFS at the contract level instead of # leaving every backend to re-check it. -_KIND_PATTERN = re.compile(r"^(?=.{1,64}$)[a-z0-9]([a-z0-9-]*[a-z0-9])?$") -# The grammar as a caller sees it. One text, one place: this message is the only -# thing a producer gets when a kind is rejected, and a second copy of the -# pattern (the local_fs backend had one) drifts from the enforced one. -_KIND_GRAMMAR = "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ (1-64 chars, no leading or trailing hyphen)" +# ``\Z`` anchors the match at the very end of the string: a ``$`` anchor +# would also match just before a trailing newline, letting kinds like +# ``"tool\n"`` or 64 chars + ``"\n"`` (65 bytes) through the 1-64 rule. +_KIND_PATTERN = re.compile(r"^(?=.{1,64}\Z)[a-z0-9]([a-z0-9-]*[a-z0-9])?\Z") +# Use the validation pattern in the error so the advertised grammar stays in sync. +_KIND_GRAMMAR = f"{_KIND_PATTERN.pattern} (1-64 chars, no leading or trailing hyphen)" def is_valid_blob_kind(kind: str) -> bool: diff --git a/backend/tests/test_blob_store.py b/backend/tests/test_blob_store.py index 7e325223d2..da4e2f7506 100644 --- a/backend/tests/test_blob_store.py +++ b/backend/tests/test_blob_store.py @@ -10,6 +10,7 @@ from __future__ import annotations import errno import hashlib import json +import re import sys import threading from pathlib import Path @@ -69,7 +70,7 @@ def test_blob_ref_validates_digest_shape(): def test_blob_ref_kind_is_path_safe(): - for bad in ("../etc", "a/b", "Aupper", "", "-lead", "trailing-", "has space", "x" * 65): + for bad in ("../etc", "a/b", "Aupper", "", "-lead", "trailing-", "has space", "x" * 65, "tool\n", "a" * 64 + "\n"): with pytest.raises(ValidationError): BlobRef(sha256=hashlib.sha256(b"x").hexdigest(), size=1, kind=bad) for good in ("viewed-image", "tool-output", "a", "0", "a" * 64): @@ -133,14 +134,19 @@ def test_dedup_records_only_the_first_writer_as_advisory_provenance(store: Local def test_kind_rejection_quotes_the_enforced_grammar(): """One validator, one message: the pattern a producer is shown must be the pattern that is enforced (it was not, before this was centralised).""" - assert validate_blob_kind("tool-output") == "tool-output" - for bad in ("trailing-", "a--", "-lead", "Aupper", "a/b", "", "x" * 65): + valid_kind = "tool-output" + assert validate_blob_kind(valid_kind) == valid_kind + for bad in ("trailing-", "a--", "-lead", "Aupper", "a/b", "", "x" * 65, "tool\n", "a" * 64 + "\n"): with pytest.raises(ValueError) as caught: validate_blob_kind(bad) message = str(caught.value) - assert "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" in message, message - assert "1-64 chars" in message, message - assert "no leading or trailing hyphen" in message, message + prefix = "kind must match " + assert message.startswith(prefix), message + pattern, _, description = message.removeprefix(prefix).partition(" ") + assert re.match(pattern, valid_kind) + assert not re.match(pattern, bad), message + assert "1-64 chars" in description, message + assert "no leading or trailing hyphen" in description, message assert validate_blob_kind is contract_module.validate_blob_kind