mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-11-16 00:10:42 +00:00
* fix(uploads): skip files that vanish mid-scan in list_files_in_dir The listing iterates scandir entries and stats each survivor. A concurrent upload delete or the staging sweep can remove a file in between, and the resulting FileNotFoundError escapes as a 500 on the listing endpoint. is_file() does not cover this: on POSIX it answers from the scandir d_type without touching the filesystem again. Catch OSError around the stat and skip the entry on ENOENT/ENOTDIR/ ELOOP, the same tolerance the chmod path just above already applies to replacement races. Operational errors such as EACCES still propagate. Regression tests pin both directions: a vanished entry is skipped while the rest of the listing survives, and a permission error still raises. Fixes #6097 * fix(uploads): log vanished entries and fix comment per review Address willem-bd's review on #6533: the skip now emits a debug line so "my file disappeared from the list" reports are diagnosable without changing the response contract, and the race comment names the actual concurrent removers (the DELETE endpoint, possibly from another replica, and sandbox processes) instead of the staging sweep, which cannot remove a listed non-staging file. The directory-level race between the is_dir() guard and os.scandir() stays out of scope; it is covered by the still-open #6096 and its PR. Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> * fix(uploads): tolerate directory races during listing * docs(uploads): scope listing guidance to avoid inherited size limits * fix(uploads): log directory races during listing --------- Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
209 lines
8.8 KiB
Python
209 lines
8.8 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
CHECKER_PATH = REPO_ROOT / "scripts" / "check_agent_guidance.py"
|
|
EXPECTED_GUIDANCE_PATHS = {
|
|
"AGENTS.md",
|
|
"backend/AGENTS.md",
|
|
"backend/tests/AGENTS.md",
|
|
"frontend/AGENTS.md",
|
|
"backend/app/gateway/AGENTS.md",
|
|
"backend/app/gateway/routers/AGENTS.md",
|
|
"backend/app/channels/AGENTS.md",
|
|
"backend/packages/harness/deerflow/AGENTS.md",
|
|
"backend/packages/harness/deerflow/agents/AGENTS.md",
|
|
"backend/packages/harness/deerflow/agents/middlewares/AGENTS.md",
|
|
"backend/packages/harness/deerflow/agents/memory/AGENTS.md",
|
|
"backend/packages/harness/deerflow/agents/task_continuity/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/jina_ai/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/ragflow/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/serper/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/tavily/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/e2b_sandbox/AGENTS.md",
|
|
"backend/packages/harness/deerflow/community/aio_sandbox/AGENTS.md",
|
|
"backend/packages/harness/deerflow/config/AGENTS.md",
|
|
"backend/packages/harness/deerflow/extensions/AGENTS.md",
|
|
"backend/packages/harness/deerflow/runtime/AGENTS.md",
|
|
"backend/packages/harness/deerflow/sandbox/AGENTS.md",
|
|
"backend/packages/harness/deerflow/mcp/AGENTS.md",
|
|
"backend/packages/harness/deerflow/models/AGENTS.md",
|
|
"backend/packages/harness/deerflow/persistence/AGENTS.md",
|
|
"backend/packages/harness/deerflow/persistence/migrations/AGENTS.md",
|
|
"backend/packages/harness/deerflow/persistence/user/AGENTS.md",
|
|
"backend/packages/harness/deerflow/reflection/AGENTS.md",
|
|
"backend/packages/harness/deerflow/skills/AGENTS.md",
|
|
"backend/packages/harness/deerflow/storage/AGENTS.md",
|
|
"backend/packages/harness/deerflow/subagents/AGENTS.md",
|
|
"backend/packages/harness/deerflow/tools/AGENTS.md",
|
|
"backend/packages/harness/deerflow/tracing/AGENTS.md",
|
|
"backend/packages/harness/deerflow/tui/AGENTS.md",
|
|
"backend/packages/harness/deerflow/typesafe/AGENTS.md",
|
|
"backend/packages/harness/deerflow/uploads/AGENTS.md",
|
|
"backend/packages/harness/deerflow/utils/AGENTS.md",
|
|
"frontend/src/AGENTS.md",
|
|
"scripts/AGENTS.md",
|
|
"examples/deerflow-extension-agent-teams/AGENTS.md",
|
|
}
|
|
|
|
|
|
def _load_checker():
|
|
assert CHECKER_PATH.exists(), f"{CHECKER_PATH} must exist"
|
|
spec = importlib.util.spec_from_file_location("deerflow_agent_guidance_check", CHECKER_PATH)
|
|
assert spec is not None
|
|
assert spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
checker = _load_checker()
|
|
|
|
|
|
def _files(**overrides: str) -> dict[PurePosixPath, str]:
|
|
files = {PurePosixPath("AGENTS.md"): "# Repository instructions\n"}
|
|
files.update({PurePosixPath(path): text for path, text in overrides.items()})
|
|
return files
|
|
|
|
|
|
def _codes(findings, severity: str | None = None) -> list[str]:
|
|
return [finding.code for finding in findings if severity is None or finding.severity == severity]
|
|
|
|
|
|
def test_normalized_utf8_size_uses_lf_and_counts_non_ascii_bytes() -> None:
|
|
assert checker.normalized_utf8_size("a\r\n中\r") == len("a\n中\n".encode())
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("path", "soft", "hard"),
|
|
[
|
|
("AGENTS.md", 16 * 1024, 20 * 1024),
|
|
("backend/AGENTS.md", 28 * 1024, 32 * 1024),
|
|
("backend/app/gateway/AGENTS.md", 40 * 1024, 48 * 1024),
|
|
],
|
|
)
|
|
def test_budget_depends_on_directory_level(path: str, soft: int, hard: int) -> None:
|
|
assert checker.agent_budget(PurePosixPath(path)) == (soft, hard)
|
|
|
|
|
|
def test_single_file_over_hard_limit_is_an_error(tmp_path: Path) -> None:
|
|
findings = checker.analyze(tmp_path, _files(**{"AGENTS.md": "x" * (20 * 1024 + 1)}))
|
|
|
|
assert "AG001" in _codes(findings, "error")
|
|
|
|
|
|
def test_effective_ancestor_chain_can_fail_when_each_file_is_valid(tmp_path: Path) -> None:
|
|
files = _files(
|
|
**{
|
|
"AGENTS.md": "r" * (15 * 1024),
|
|
"backend/AGENTS.md": "m" * (31 * 1024),
|
|
"backend/app/AGENTS.md": "l" * (47 * 1024),
|
|
"backend/app/gateway/AGENTS.md": "g" * (47 * 1024),
|
|
}
|
|
)
|
|
|
|
findings = checker.analyze(tmp_path, files)
|
|
|
|
assert "AG001" not in _codes(findings, "error")
|
|
assert "AG002" in _codes(findings, "error")
|
|
|
|
|
|
def test_legacy_hard_violation_may_shrink_but_may_not_grow(tmp_path: Path) -> None:
|
|
base = _files(**{"AGENTS.md": "x" * (20 * 1024)})
|
|
smaller = _files(**{"AGENTS.md": "x" * (20 * 1024 - 1)})
|
|
grown = _files(**{"AGENTS.md": "x" * (20 * 1024 + 1)})
|
|
|
|
smaller_findings = checker.analyze(tmp_path, smaller, base_files=base)
|
|
grown_findings = checker.analyze(tmp_path, grown, base_files=base)
|
|
|
|
assert "AG001" not in _codes(smaller_findings, "error")
|
|
assert "AG001" in _codes(grown_findings, "error")
|
|
|
|
|
|
def test_discovery_uses_exact_agents_basename() -> None:
|
|
paths = [
|
|
PurePosixPath("AGENTS.md"),
|
|
PurePosixPath("backend/AGENTS.md"),
|
|
PurePosixPath("backend/CLAUDE.md"),
|
|
PurePosixPath("backend/docs/GITHUB_AGENTS.md"),
|
|
]
|
|
|
|
assert checker.guidance_paths(paths) == {
|
|
PurePosixPath("AGENTS.md"),
|
|
PurePosixPath("backend/AGENTS.md"),
|
|
}
|
|
|
|
|
|
def test_repository_has_the_approved_scoped_guidance_shape() -> None:
|
|
actual = {path.as_posix() for path in checker.guidance_paths(checker._worktree_paths(REPO_ROOT))}
|
|
|
|
assert actual == EXPECTED_GUIDANCE_PATHS
|
|
|
|
|
|
def test_repository_guidance_stays_below_hard_budgets_and_avoids_doc_indexes() -> None:
|
|
for relative_text in EXPECTED_GUIDANCE_PATHS:
|
|
relative = PurePosixPath(relative_text)
|
|
path = REPO_ROOT / relative_text
|
|
assert path.is_file(), relative
|
|
_, hard = checker.agent_budget(relative)
|
|
text = path.read_text(encoding="utf-8")
|
|
assert checker.normalized_utf8_size(text) <= hard, relative
|
|
assert "Subsystem Index" not in text
|
|
|
|
|
|
def test_local_guidance_files_contain_the_split_original_sections() -> None:
|
|
expected_headings = {
|
|
"backend/app/gateway/AGENTS.md": "### Gateway API (`app/gateway/`)",
|
|
"backend/app/channels/AGENTS.md": "### IM Channels System (`app/channels/`)",
|
|
"backend/packages/harness/deerflow/agents/AGENTS.md": "### Agent System",
|
|
"backend/packages/harness/deerflow/agents/middlewares/AGENTS.md": "### Middleware Chain",
|
|
"backend/packages/harness/deerflow/agents/memory/AGENTS.md": "### Memory System",
|
|
"backend/packages/harness/deerflow/config/AGENTS.md": "### Configuration System",
|
|
"backend/packages/harness/deerflow/extensions/AGENTS.md": "### Python Extension System",
|
|
"backend/packages/harness/deerflow/runtime/AGENTS.md": "### Checkpoint Channel Modes",
|
|
"backend/packages/harness/deerflow/sandbox/AGENTS.md": "### Sandbox System",
|
|
"backend/packages/harness/deerflow/mcp/AGENTS.md": "### MCP System",
|
|
"backend/packages/harness/deerflow/models/AGENTS.md": "### Model Factory",
|
|
"backend/packages/harness/deerflow/persistence/migrations/AGENTS.md": "### Schema Migrations",
|
|
"backend/packages/harness/deerflow/reflection/AGENTS.md": "### Reflection System",
|
|
"backend/packages/harness/deerflow/skills/AGENTS.md": "### Skills System",
|
|
"backend/packages/harness/deerflow/subagents/AGENTS.md": "### Subagent System",
|
|
"backend/packages/harness/deerflow/tools/AGENTS.md": "### Tool System",
|
|
"backend/packages/harness/deerflow/tracing/AGENTS.md": "### Tracing System",
|
|
"backend/packages/harness/deerflow/tui/AGENTS.md": "### Terminal Workbench / TUI",
|
|
"frontend/src/AGENTS.md": "### Data Flow",
|
|
}
|
|
|
|
for relative_text, heading in expected_headings.items():
|
|
text = (REPO_ROOT / relative_text).read_text(encoding="utf-8")
|
|
assert heading in text, relative_text
|
|
assert "Before changing files in this directory" not in text, relative_text
|
|
|
|
|
|
def test_mcp_task_lease_token_migration_is_documented() -> None:
|
|
guidance = (REPO_ROOT / "backend" / "packages" / "harness" / "deerflow" / "persistence" / "migrations" / "AGENTS.md").read_text(encoding="utf-8")
|
|
|
|
for required in (
|
|
"0026_mcp_task_lease_tokens.py",
|
|
"0016_subagent_batches",
|
|
"lease_token",
|
|
"notification_lease_token",
|
|
):
|
|
assert required in guidance
|
|
|
|
|
|
def test_repository_exposes_one_local_and_one_ci_entrypoint() -> None:
|
|
makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
|
|
workflow = (REPO_ROOT / ".github" / "workflows" / "lint-check.yml").read_text(encoding="utf-8")
|
|
|
|
assert "check-agent-guidance:" in makefile
|
|
assert "scripts/check_agent_guidance.py" in makefile
|
|
assert workflow.count("agent-guidance:") == 1
|
|
assert "fetch-depth: 0" in workflow
|
|
assert "--base-ref" in workflow
|
|
assert "--before" in workflow
|