diff --git a/AGENTS.md b/AGENTS.md index 5d8f012258..6ef1880c45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,7 +156,7 @@ Run `make help` for the full list. ```bash # Backend (see backend/AGENTS.md for the full set) cd backend && make dev # Gateway API with reload (port 8001) -cd backend && make test # Default backend suite; excludes live and blocking-I/O tests +cd backend && make test # Four parallel shards; excludes live and blocking-I/O tests cd backend && make test-blocking-io # Strict blocking-I/O suite cd backend && make lint # ruff check cd backend && make format # ruff format diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0205c61548..8bff80c896 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -333,10 +333,13 @@ before review. ## Testing ```bash -# Default backend tests (excludes live and blocking-I/O tests) +# Default backend tests (four parallel shards; excludes live and blocking-I/O tests) cd backend make test +# Run the same shards sequentially +make test TEST_JOBS=1 + # Strict blocking-I/O tests make test-blocking-io diff --git a/README.md b/README.md index 7aac7a9bf7..d23398fc93 100644 --- a/README.md +++ b/README.md @@ -2617,7 +2617,9 @@ immediately after starting any deployment that is not loopback-only. We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, workflow, and guidelines. -Backend `make test` excludes live external-API and blocking-I/O coverage. +Backend `make test` runs four duration-balanced shards concurrently and reports +failure if any shard fails. Use `make test TEST_JOBS=1` to run the shards sequentially. +It excludes live external-API and blocking-I/O coverage. Run `cd backend && make test-blocking-io` for strict blocking-I/O checks. Maintainers can run the real `DeerFlowClient` suite with `cd backend && make test-live`. This command requires a valid root `config.yaml` and API credentials. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index cbcc524823..2c1070aead 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -170,7 +170,7 @@ make stop # Stop all services make install # Install backend dependencies make dev # Gateway API, reload (port 8001) make gateway # Gateway API only (port 8001) -make test # offline tests (no live/blocking-io) +make test # four offline workers; TEST_JOBS=1 for serial make test-live # live tests (real APIs) make test-blocking-io # strict Blockbuster gate on tests/blocking_io/ make test-shard SPLITS=4 GROUP=2 # one duration-aware shard @@ -252,14 +252,13 @@ only `AUTH_SOURCE_INTERNAL` run input may retain them. - If a module causes circular import issues in tests, add a `sys.modules` mock in `tests/conftest.py` (see existing example for `deerflow.subagents.executor`) ```bash -# Run default offline tests +# Four duration-balanced offline shards make test # Run strict blocking-I/O tests make test-blocking-io -# Explicit live integration tests (requires config.yaml and credentials; -# calls real APIs and may create local side effects) +# Live API tests: require config.yaml, credentials, and explicit opt-in make test-live # Run a specific test file diff --git a/backend/Makefile b/backend/Makefile index 1f715f46b2..14d7bdd6e8 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -19,8 +19,15 @@ dev: gateway: PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run --locked uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 +# Run all four duration-balanced shards locally; wait for every shard on failure. +TEST_JOBS ?= 4 +TEST_SHARD_TARGETS := test-shard-1 test-shard-2 test-shard-3 test-shard-4 +.PHONY: test test-shard $(TEST_SHARD_TARGETS) test: - PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest -m "not live" --ignore=tests/blocking_io tests/ -v + $(MAKE) --no-print-directory -k -j$(TEST_JOBS) $(TEST_SHARD_TARGETS) + +$(TEST_SHARD_TARGETS): + $(MAKE) --no-print-directory test-shard SPLITS=4 GROUP=$(patsubst test-shard-%,%,$@) test-live: DEER_FLOW_RUN_LIVE_TESTS=1 PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest -m live tests/ -v -s @@ -35,7 +42,7 @@ test-blocking-io: # so concurrent CI jobs cannot race writes on it. Refresh it with # `make test-shard-durations` after a meaningful change to the test set. # `make test-shard SPLITS=4 GROUP=2` runs shard 2 of 4; a bare `make test-shard` -# runs the whole (non-live) suite as a single split, matching `make test`. +# runs the whole offline suite as a single split; `make test` runs all four shards. SPLITS ?= 1 GROUP ?= 1 DURATIONS_FILE ?= .test_durations diff --git a/backend/README.md b/backend/README.md index fc0d68b664..435a645488 100644 --- a/backend/README.md +++ b/backend/README.md @@ -517,9 +517,12 @@ the only execution path, which keeps operational mistakes off the table. See ### Testing ```bash -# Default offline backend suite (live external-API and blocking-I/O tests are excluded) +# Default offline backend suite (four parallel shards; excludes live and blocking-I/O tests) make test +# Run the same shards sequentially +make test TEST_JOBS=1 + # Strict blocking-I/O suite make test-blocking-io diff --git a/backend/tests/test_make_test_parallel.py b/backend/tests/test_make_test_parallel.py new file mode 100644 index 0000000000..25534394fe --- /dev/null +++ b/backend/tests/test_make_test_parallel.py @@ -0,0 +1,110 @@ +"""Exercise the backend test scheduler with offline worker doubles.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +BACKEND_ROOT = Path(__file__).resolve().parents[1] +MAKE = shutil.which("make") +pytestmark = pytest.mark.skipif(MAKE is None or os.name == "nt", reason="requires POSIX make") + + +@pytest.fixture +def test_project(tmp_path: Path) -> Path: + project = tmp_path / "backend" + project.mkdir() + shutil.copyfile(BACKEND_ROOT / "Makefile", project / "Makefile") + (project / ".test_durations").write_text("{}\n", encoding="utf-8") + bin_dir = project / "bin" + bin_dir.mkdir() + worker = project / "worker.py" + worker.write_text( + """import json +import os +import sys +import time +from pathlib import Path + +args = sys.argv[1:] +group = args[args.index('--group') + 1] if '--group' in args else '0' +Path(f'{group}.started').write_text(json.dumps(args), encoding='utf-8') +if group == '0': + sys.exit(0) +if os.environ.get('EXPECT_PARALLEL') == '1': + deadline = time.monotonic() + 10 + while not all(Path(f'{other}.started').exists() for other in range(1, 5)): + if time.monotonic() > deadline: + sys.exit('workers did not start concurrently') + time.sleep(0.01) +Path(f'{group}.finished').write_text('done', encoding='utf-8') +sys.exit(7 if group == os.environ.get('FAIL_GROUP') else 0) +""", + encoding="utf-8", + ) + uv = bin_dir / "uv" + uv.write_text('#!/bin/sh\nexec "$TEST_WORKER_PYTHON" "$TEST_WORKER_SCRIPT" "$@"\n', encoding="utf-8") + uv.chmod(0o755) + return project + + +def run_make(project: Path, *args: str, fail_group: str = "", parallel: bool = True) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "PATH": str(project / "bin") + os.pathsep + os.environ.get("PATH", ""), + "TEST_WORKER_PYTHON": sys.executable, + "TEST_WORKER_SCRIPT": str(project / "worker.py"), + "EXPECT_PARALLEL": "1" if parallel else "0", + "FAIL_GROUP": fail_group, + } + env.pop("MAKEFLAGS", None) + env.pop("MFLAGS", None) + env.pop("MAKELEVEL", None) + return subprocess.run([MAKE, *args], cwd=project, env=env, capture_output=True, text=True, encoding="utf-8", check=False, timeout=30) + + +@pytest.mark.parametrize("fail_group", ["", "1"]) +def test_make_test_runs_all_four_shards_and_reports_any_failure(test_project: Path, fail_group: str) -> None: + result = run_make(test_project, "test", fail_group=fail_group) + + assert (result.returncode == 0) == (not fail_group), result.stdout + result.stderr + assert {path.stem for path in test_project.glob("*.finished")} == {"1", "2", "3", "4"} + for group in range(1, 5): + args = json.loads((test_project / f"{group}.started").read_text(encoding="utf-8")) + assert args[:3] == ["run", "pytest", "-m"] + assert args[args.index("-m") + 1] == "not live" + assert "--ignore=tests/blocking_io" in args + assert args[args.index("--splits") + 1] == "4" + assert args[args.index("--group") + 1] == str(group) + assert args[args.index("--splitting-algorithm") + 1] == "least_duration" + assert "--store-durations" not in args + + +def test_ci_can_still_run_one_shard(test_project: Path) -> None: + result = run_make(test_project, "test-shard", "SPLITS=4", "GROUP=2", parallel=False) + + assert result.returncode == 0, result.stdout + result.stderr + assert {path.stem for path in test_project.glob("*.finished")} == {"2"} + + +def test_make_test_can_limit_concurrency(test_project: Path) -> None: + result = run_make(test_project, "test", "TEST_JOBS=1", parallel=False) + + assert result.returncode == 0, result.stdout + result.stderr + assert {path.stem for path in test_project.glob("*.finished")} == {"1", "2", "3", "4"} + + +def test_make_test_reports_missing_durations(test_project: Path) -> None: + (test_project / ".test_durations").unlink() + + result = run_make(test_project, "test") + + assert result.returncode != 0 + assert "make test-shard-durations" in result.stdout + result.stderr + assert not list(test_project.glob("*.started")) diff --git a/backend/tests/test_skill_review_core.py b/backend/tests/test_skill_review_core.py index f27f5fa0d6..1869c3750f 100644 --- a/backend/tests/test_skill_review_core.py +++ b/backend/tests/test_skill_review_core.py @@ -529,13 +529,21 @@ def test_skillscan_scans_eval_fixture_files_other_than_skill_markdown(tmp_path, assert f"{fixture_dir}/SKILL.md" not in scanned_paths +def test_case_sensitivity_probe_preserves_existing_files(tmp_path): + existing = tmp_path / "CaseProbe" + _write(existing, "keep this file") + + _is_case_insensitive_directory(tmp_path) + + assert existing.read_text(encoding="utf-8") == "keep this file" + assert list(tmp_path.iterdir()) == [existing] + + def _is_case_insensitive_directory(path: Path) -> bool: - probe = path / "CaseProbe" - probe.touch() - try: - return (path / "caseprobe").exists() - finally: - probe.unlink() + with tempfile.TemporaryDirectory(dir=path) as probe_dir: + directory = Path(probe_dir) + (directory / "CaseProbe").touch() + return (directory / "caseprobe").exists() @pytest.mark.parametrize(