mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
82ac1612e3
* feat(storage): add a content-addressed blob store contract (#4189 item 2) Addresses the review on the previous head: * the retention/dedup claim in the doc now states the deletion rule (liveness from the durable references) instead of promising refcount-free deletion, and the sidecar records the first writer as advisory provenance only; * one validate_blob_kind() owns the kind grammar, so the message a producer sees is the pattern that is enforced; * adds the missing storage/AGENTS.md (registry entry included) that both docstrings already pointed at; * drops the dead _import_backend_module() by routing the folder scan through it; * exports BlobNotConfiguredError from the package; * renames _KIND_MAX_BYTES to _MAX_BLOB_BYTES (it caps one put, not a kind); * delete failures raise the neutral BlobStoreError instead of BlobReadError. Rebased onto current main so the guidance-budget test matches upstream. Signed-off-by: zeng-bohan <zengbh1@gmail.com> * fix(storage): restore the config module and backend packages dropped by the squash The force-push to e72b435 silently lost four items the surviving code imports, breaking `import deerflow.storage` and the test suite at collection: - deerflow/config/blob_storage_config.py (BlobStorageConfig + get/set_blob_storage_config + load_blob_storage_config_from_dict, mirroring memory_config's host-shared schema and hot-reload pattern) - app_config.py wiring: the blob_storage AppConfig field, the import, and the _apply_singleton_configs -> load_blob_storage_config_from_dict call (memory stayed wired; blob did not) - storage/backends/__init__.py (drop-in contract docstring) - storage/backends/local_fs/__init__.py (LocalFsBlobStore re-export + STORE_CLASS for _scan_backends) Verified: pytest tests/test_blob_store.py tests/test_app_config_reload.py tests/test_app_config_name_indexes.py -> 67 passed; ruff format/check clean on all four files. * fix(storage): name the failing digest check; document the 64 MiB per-put cap - get_bytes: split the corruption message into size mismatch (truncation / partial write) vs digest mismatch (same-size bit rot, reporting what the content hashes to vs what the ref names) — the combined message reported 'got 1024 bytes, expected 1024' on same-size corruption and sent the debugger in the wrong direction. - docs/blob-storage.md + storage/AGENTS.md: state the _MAX_BLOB_BYTES cap (64 MiB) and that it is backend-level defense-in-depth rather than part of the BlobStore contract, so the externalized-tool-results follow-up sees the limit before the first production put fails. * test(storage): pin both get_bytes corruption branches separately The pre-split test corrupted the blob with different-size bytes, so only the size-mismatch branch ever ran and neither new message was asserted. Now: different-size corruption asserts 'size mismatch', and a same-size reversal (bit rot) asserts 'digest mismatch' — the branch the split was actually about. * docs(config): document blob_storage in config.example.yaml Add the blob_storage section (enabled/backend/backend_config) mirroring the memory section's shape, matching the Configuration section of backend/docs/blob-storage.md, and bump config_version 46 -> 47 per the example-file convention for schema additions. Default state is unchanged: enabled: false, purely opt-in. * fix(helm): align chart config version with example * fix(storage): refresh blob store on config changes --------- Signed-off-by: zeng-bohan <zengbh1@gmail.com> Co-authored-by: zengbohan1 <310902929+zengbohan1@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com> Co-authored-by: zeng-bohan <310902929+zeng-bohan@users.noreply.github.com>
201 lines
8.4 KiB
Python
201 lines
8.4 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
CHECKER_PATH = REPO_ROOT / "scripts" / "check_agent_guidance.py"
|
|
EXPECTED_GUIDANCE_PATHS = {
|
|
"AGENTS.md",
|
|
"backend/AGENTS.md",
|
|
"backend/tests/AGENTS.md",
|
|
"frontend/AGENTS.md",
|
|
"backend/app/gateway/AGENTS.md",
|
|
"backend/app/channels/AGENTS.md",
|
|
"backend/packages/harness/deerflow/AGENTS.md",
|
|
"backend/packages/harness/deerflow/agents/AGENTS.md",
|
|
"backend/packages/harness/deerflow/agents/middlewares/AGENTS.md",
|
|
"backend/packages/harness/deerflow/agents/memory/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/ragflow/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/tavily/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md",
|
|
"backend/packages/harness/deerflow/config/AGENTS.md",
|
|
"backend/packages/harness/deerflow/extensions/AGENTS.md",
|
|
"backend/packages/harness/deerflow/runtime/AGENTS.md",
|
|
"backend/packages/harness/deerflow/sandbox/AGENTS.md",
|
|
"backend/packages/harness/deerflow/mcp/AGENTS.md",
|
|
"backend/packages/harness/deerflow/models/AGENTS.md",
|
|
"backend/packages/harness/deerflow/persistence/AGENTS.md",
|
|
"backend/packages/harness/deerflow/persistence/migrations/AGENTS.md",
|
|
"backend/packages/harness/deerflow/persistence/user/AGENTS.md",
|
|
"backend/packages/harness/deerflow/reflection/AGENTS.md",
|
|
"backend/packages/harness/deerflow/skills/AGENTS.md",
|
|
"backend/packages/harness/deerflow/storage/AGENTS.md",
|
|
"backend/packages/harness/deerflow/subagents/AGENTS.md",
|
|
"backend/packages/harness/deerflow/tools/AGENTS.md",
|
|
"backend/packages/harness/deerflow/tracing/AGENTS.md",
|
|
"backend/packages/harness/deerflow/tui/AGENTS.md",
|
|
"backend/packages/harness/deerflow/utils/AGENTS.md",
|
|
"frontend/src/AGENTS.md",
|
|
"scripts/AGENTS.md",
|
|
}
|
|
|
|
|
|
def _load_checker():
|
|
assert CHECKER_PATH.exists(), f"{CHECKER_PATH} must exist"
|
|
spec = importlib.util.spec_from_file_location("deerflow_agent_guidance_check", CHECKER_PATH)
|
|
assert spec is not None
|
|
assert spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
checker = _load_checker()
|
|
|
|
|
|
def _files(**overrides: str) -> dict[PurePosixPath, str]:
|
|
files = {PurePosixPath("AGENTS.md"): "# Repository instructions\n"}
|
|
files.update({PurePosixPath(path): text for path, text in overrides.items()})
|
|
return files
|
|
|
|
|
|
def _codes(findings, severity: str | None = None) -> list[str]:
|
|
return [finding.code for finding in findings if severity is None or finding.severity == severity]
|
|
|
|
|
|
def test_normalized_utf8_size_uses_lf_and_counts_non_ascii_bytes() -> None:
|
|
assert checker.normalized_utf8_size("a\r\n中\r") == len("a\n中\n".encode())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("path", "soft", "hard"),
|
|
[
|
|
("AGENTS.md", 16 * 1024, 20 * 1024),
|
|
("backend/AGENTS.md", 28 * 1024, 32 * 1024),
|
|
("backend/app/gateway/AGENTS.md", 40 * 1024, 48 * 1024),
|
|
],
|
|
)
|
|
def test_budget_depends_on_directory_level(path: str, soft: int, hard: int) -> None:
|
|
assert checker.agent_budget(PurePosixPath(path)) == (soft, hard)
|
|
|
|
|
|
def test_single_file_over_hard_limit_is_an_error(tmp_path: Path) -> None:
|
|
findings = checker.analyze(tmp_path, _files(**{"AGENTS.md": "x" * (20 * 1024 + 1)}))
|
|
|
|
assert "AG001" in _codes(findings, "error")
|
|
|
|
|
|
def test_effective_ancestor_chain_can_fail_when_each_file_is_valid(tmp_path: Path) -> None:
|
|
files = _files(
|
|
**{
|
|
"AGENTS.md": "r" * (15 * 1024),
|
|
"backend/AGENTS.md": "m" * (31 * 1024),
|
|
"backend/app/AGENTS.md": "l" * (47 * 1024),
|
|
"backend/app/gateway/AGENTS.md": "g" * (47 * 1024),
|
|
}
|
|
)
|
|
|
|
findings = checker.analyze(tmp_path, files)
|
|
|
|
assert "AG001" not in _codes(findings, "error")
|
|
assert "AG002" in _codes(findings, "error")
|
|
|
|
|
|
def test_legacy_hard_violation_may_shrink_but_may_not_grow(tmp_path: Path) -> None:
|
|
base = _files(**{"AGENTS.md": "x" * (20 * 1024)})
|
|
smaller = _files(**{"AGENTS.md": "x" * (20 * 1024 - 1)})
|
|
grown = _files(**{"AGENTS.md": "x" * (20 * 1024 + 1)})
|
|
|
|
smaller_findings = checker.analyze(tmp_path, smaller, base_files=base)
|
|
grown_findings = checker.analyze(tmp_path, grown, base_files=base)
|
|
|
|
assert "AG001" not in _codes(smaller_findings, "error")
|
|
assert "AG001" in _codes(grown_findings, "error")
|
|
|
|
|
|
def test_discovery_uses_exact_agents_basename() -> None:
|
|
paths = [
|
|
PurePosixPath("AGENTS.md"),
|
|
PurePosixPath("backend/AGENTS.md"),
|
|
PurePosixPath("backend/CLAUDE.md"),
|
|
PurePosixPath("backend/docs/GITHUB_AGENTS.md"),
|
|
]
|
|
|
|
assert checker.guidance_paths(paths) == {
|
|
PurePosixPath("AGENTS.md"),
|
|
PurePosixPath("backend/AGENTS.md"),
|
|
}
|
|
|
|
|
|
def test_repository_has_the_approved_scoped_guidance_shape() -> None:
|
|
actual = {path.as_posix() for path in checker.guidance_paths(checker._worktree_paths(REPO_ROOT))}
|
|
|
|
assert actual == EXPECTED_GUIDANCE_PATHS
|
|
|
|
|
|
def test_repository_guidance_stays_below_hard_budgets_and_avoids_doc_indexes() -> None:
|
|
for relative_text in EXPECTED_GUIDANCE_PATHS:
|
|
relative = PurePosixPath(relative_text)
|
|
path = REPO_ROOT / relative_text
|
|
assert path.is_file(), relative
|
|
_, hard = checker.agent_budget(relative)
|
|
text = path.read_text(encoding="utf-8")
|
|
assert checker.normalized_utf8_size(text) <= hard, relative
|
|
assert "Subsystem Index" not in text
|
|
|
|
|
|
def test_local_guidance_files_contain_the_split_original_sections() -> None:
|
|
expected_headings = {
|
|
"backend/app/gateway/AGENTS.md": "### Gateway API (`app/gateway/`)",
|
|
"backend/app/channels/AGENTS.md": "### IM Channels System (`app/channels/`)",
|
|
"backend/packages/harness/deerflow/agents/AGENTS.md": "### Agent System",
|
|
"backend/packages/harness/deerflow/agents/middlewares/AGENTS.md": "### Middleware Chain",
|
|
"backend/packages/harness/deerflow/agents/memory/AGENTS.md": "### Memory System",
|
|
"backend/packages/harness/deerflow/config/AGENTS.md": "### Configuration System",
|
|
"backend/packages/harness/deerflow/extensions/AGENTS.md": "### Python Extension System",
|
|
"backend/packages/harness/deerflow/runtime/AGENTS.md": "### Checkpoint Channel Modes",
|
|
"backend/packages/harness/deerflow/sandbox/AGENTS.md": "### Sandbox System",
|
|
"backend/packages/harness/deerflow/mcp/AGENTS.md": "### MCP System",
|
|
"backend/packages/harness/deerflow/models/AGENTS.md": "### Model Factory",
|
|
"backend/packages/harness/deerflow/persistence/migrations/AGENTS.md": "### Schema Migrations",
|
|
"backend/packages/harness/deerflow/reflection/AGENTS.md": "### Reflection System",
|
|
"backend/packages/harness/deerflow/skills/AGENTS.md": "### Skills System",
|
|
"backend/packages/harness/deerflow/subagents/AGENTS.md": "### Subagent System",
|
|
"backend/packages/harness/deerflow/tools/AGENTS.md": "### Tool System",
|
|
"backend/packages/harness/deerflow/tracing/AGENTS.md": "### Tracing System",
|
|
"backend/packages/harness/deerflow/tui/AGENTS.md": "### Terminal Workbench / TUI",
|
|
"frontend/src/AGENTS.md": "### Data Flow",
|
|
}
|
|
|
|
for relative_text, heading in expected_headings.items():
|
|
text = (REPO_ROOT / relative_text).read_text(encoding="utf-8")
|
|
assert heading in text, relative_text
|
|
assert "Before changing files in this directory" not in text, relative_text
|
|
|
|
|
|
def test_mcp_task_lease_token_migration_is_documented() -> None:
|
|
guidance = (REPO_ROOT / "backend" / "packages" / "harness" / "deerflow" / "persistence" / "migrations" / "AGENTS.md").read_text(encoding="utf-8")
|
|
|
|
for required in (
|
|
"0026_mcp_task_lease_tokens.py",
|
|
"0016_subagent_batches",
|
|
"lease_token",
|
|
"notification_lease_token",
|
|
):
|
|
assert required in guidance
|
|
|
|
|
|
def test_repository_exposes_one_local_and_one_ci_entrypoint() -> None:
|
|
makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
|
|
workflow = (REPO_ROOT / ".github" / "workflows" / "lint-check.yml").read_text(encoding="utf-8")
|
|
|
|
assert "check-agent-guidance:" in makefile
|
|
assert "scripts/check_agent_guidance.py" in makefile
|
|
assert workflow.count("agent-guidance:") == 1
|
|
assert "fetch-depth: 0" in workflow
|
|
assert "--base-ref" in workflow
|
|
assert "--before" in workflow
|