test(backend): run make test in four parallel shards (#6324)

AGENTS.md

@@ -156,7 +156,7 @@ Run `make help` for the full list.
 ```bash
 # Backend (see backend/AGENTS.md for the full set)
 cd backend && make dev        # Gateway API with reload (port 8001)
-cd backend && make test       # Default backend suite; excludes live and blocking-I/O tests
+cd backend && make test       # Four parallel shards; excludes live and blocking-I/O tests
 cd backend && make test-blocking-io  # Strict blocking-I/O suite
 cd backend && make lint       # ruff check
 cd backend && make format     # ruff format

CONTRIBUTING.md

@@ -333,10 +333,13 @@ before review.
 ## Testing
 
 ```bash
-# Default backend tests (excludes live and blocking-I/O tests)
+# Default backend tests (four parallel shards; excludes live and blocking-I/O tests)
 cd backend
 make test
 
+# Run the same shards sequentially
+make test TEST_JOBS=1
+
 # Strict blocking-I/O tests
 make test-blocking-io
 

README.md

@@ -2617,7 +2617,9 @@ immediately after starting any deployment that is not loopback-only.
 
 We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, workflow, and guidelines.
 
-Backend `make test` excludes live external-API and blocking-I/O coverage.
+Backend `make test` runs four duration-balanced shards concurrently and reports
+failure if any shard fails. Use `make test TEST_JOBS=1` to run the shards sequentially.
+It excludes live external-API and blocking-I/O coverage.
 Run `cd backend && make test-blocking-io` for strict blocking-I/O checks.
 Maintainers can run the real `DeerFlowClient` suite with `cd backend && make test-live`.
 This command requires a valid root `config.yaml` and API credentials.

backend/AGENTS.md

@@ -170,7 +170,7 @@ make stop       # Stop all services
 make install            # Install backend dependencies
 make dev                # Gateway API, reload (port 8001)
 make gateway            # Gateway API only (port 8001)
-make test               # offline tests (no live/blocking-io)
+make test               # four offline workers; TEST_JOBS=1 for serial
 make test-live          # live tests (real APIs)
 make test-blocking-io   # strict Blockbuster gate on tests/blocking_io/
 make test-shard SPLITS=4 GROUP=2  # one duration-aware shard
@@ -252,14 +252,13 @@ only `AUTH_SOURCE_INTERNAL` run input may retain them.
 - If a module causes circular import issues in tests, add a `sys.modules` mock in `tests/conftest.py` (see existing example for `deerflow.subagents.executor`)
 
 ```bash
-# Run default offline tests
+# Four duration-balanced offline shards
 make test
 
 # Run strict blocking-I/O tests
 make test-blocking-io
 
-# Explicit live integration tests (requires config.yaml and credentials;
-# calls real APIs and may create local side effects)
+# Live API tests: require config.yaml, credentials, and explicit opt-in
 make test-live
 
 # Run a specific test file

backend/Makefile

@@ -19,8 +19,15 @@ dev:
 gateway:
 	PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run --locked uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001
 
+# Run all four duration-balanced shards locally; wait for every shard on failure.
+TEST_JOBS ?= 4
+TEST_SHARD_TARGETS := test-shard-1 test-shard-2 test-shard-3 test-shard-4
+.PHONY: test test-shard $(TEST_SHARD_TARGETS)
 test:
-	PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest -m "not live" --ignore=tests/blocking_io tests/ -v
+	$(MAKE) --no-print-directory -k -j$(TEST_JOBS) $(TEST_SHARD_TARGETS)
+
+$(TEST_SHARD_TARGETS):
+	$(MAKE) --no-print-directory test-shard SPLITS=4 GROUP=$(patsubst test-shard-%,%,$@)
 
 test-live:
 	DEER_FLOW_RUN_LIVE_TESTS=1 PYTHONPATH=. PYTHONIOENCODING=utf-8 PYTHONUTF8=1 uv run pytest -m live tests/ -v -s
@@ -35,7 +42,7 @@ test-blocking-io:
 # so concurrent CI jobs cannot race writes on it. Refresh it with
 # `make test-shard-durations` after a meaningful change to the test set.
 # `make test-shard SPLITS=4 GROUP=2` runs shard 2 of 4; a bare `make test-shard`
-# runs the whole (non-live) suite as a single split, matching `make test`.
+# runs the whole offline suite as a single split; `make test` runs all four shards.
 SPLITS ?= 1
 GROUP ?= 1
 DURATIONS_FILE ?= .test_durations

backend/README.md

@@ -517,9 +517,12 @@ the only execution path, which keeps operational mistakes off the table. See
 ### Testing
 
 ```bash
-# Default offline backend suite (live external-API and blocking-I/O tests are excluded)
+# Default offline backend suite (four parallel shards; excludes live and blocking-I/O tests)
 make test
 
+# Run the same shards sequentially
+make test TEST_JOBS=1
+
 # Strict blocking-I/O suite
 make test-blocking-io
 

backend/tests/test_make_test_parallel.py

@@ -0,0 +1,110 @@
+"""Exercise the backend test scheduler with offline worker doubles."""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+BACKEND_ROOT = Path(__file__).resolve().parents[1]
+MAKE = shutil.which("make")
+pytestmark = pytest.mark.skipif(MAKE is None or os.name == "nt", reason="requires POSIX make")
+
+
+@pytest.fixture
+def test_project(tmp_path: Path) -> Path:
+    project = tmp_path / "backend"
+    project.mkdir()
+    shutil.copyfile(BACKEND_ROOT / "Makefile", project / "Makefile")
+    (project / ".test_durations").write_text("{}\n", encoding="utf-8")
+    bin_dir = project / "bin"
+    bin_dir.mkdir()
+    worker = project / "worker.py"
+    worker.write_text(
+        """import json
+import os
+import sys
+import time
+from pathlib import Path
+
+args = sys.argv[1:]
+group = args[args.index('--group') + 1] if '--group' in args else '0'
+Path(f'{group}.started').write_text(json.dumps(args), encoding='utf-8')
+if group == '0':
+    sys.exit(0)
+if os.environ.get('EXPECT_PARALLEL') == '1':
+    deadline = time.monotonic() + 10
+    while not all(Path(f'{other}.started').exists() for other in range(1, 5)):
+        if time.monotonic() > deadline:
+            sys.exit('workers did not start concurrently')
+        time.sleep(0.01)
+Path(f'{group}.finished').write_text('done', encoding='utf-8')
+sys.exit(7 if group == os.environ.get('FAIL_GROUP') else 0)
+""",
+        encoding="utf-8",
+    )
+    uv = bin_dir / "uv"
+    uv.write_text('#!/bin/sh\nexec "$TEST_WORKER_PYTHON" "$TEST_WORKER_SCRIPT" "$@"\n', encoding="utf-8")
+    uv.chmod(0o755)
+    return project
+
+
+def run_make(project: Path, *args: str, fail_group: str = "", parallel: bool = True) -> subprocess.CompletedProcess[str]:
+    env = {
+        **os.environ,
+        "PATH": str(project / "bin") + os.pathsep + os.environ.get("PATH", ""),
+        "TEST_WORKER_PYTHON": sys.executable,
+        "TEST_WORKER_SCRIPT": str(project / "worker.py"),
+        "EXPECT_PARALLEL": "1" if parallel else "0",
+        "FAIL_GROUP": fail_group,
+    }
+    env.pop("MAKEFLAGS", None)
+    env.pop("MFLAGS", None)
+    env.pop("MAKELEVEL", None)
+    return subprocess.run([MAKE, *args], cwd=project, env=env, capture_output=True, text=True, encoding="utf-8", check=False, timeout=30)
+
+
+@pytest.mark.parametrize("fail_group", ["", "1"])
+def test_make_test_runs_all_four_shards_and_reports_any_failure(test_project: Path, fail_group: str) -> None:
+    result = run_make(test_project, "test", fail_group=fail_group)
+
+    assert (result.returncode == 0) == (not fail_group), result.stdout + result.stderr
+    assert {path.stem for path in test_project.glob("*.finished")} == {"1", "2", "3", "4"}
+    for group in range(1, 5):
+        args = json.loads((test_project / f"{group}.started").read_text(encoding="utf-8"))
+        assert args[:3] == ["run", "pytest", "-m"]
+        assert args[args.index("-m") + 1] == "not live"
+        assert "--ignore=tests/blocking_io" in args
+        assert args[args.index("--splits") + 1] == "4"
+        assert args[args.index("--group") + 1] == str(group)
+        assert args[args.index("--splitting-algorithm") + 1] == "least_duration"
+        assert "--store-durations" not in args
+
+
+def test_ci_can_still_run_one_shard(test_project: Path) -> None:
+    result = run_make(test_project, "test-shard", "SPLITS=4", "GROUP=2", parallel=False)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert {path.stem for path in test_project.glob("*.finished")} == {"2"}
+
+
+def test_make_test_can_limit_concurrency(test_project: Path) -> None:
+    result = run_make(test_project, "test", "TEST_JOBS=1", parallel=False)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert {path.stem for path in test_project.glob("*.finished")} == {"1", "2", "3", "4"}
+
+
+def test_make_test_reports_missing_durations(test_project: Path) -> None:
+    (test_project / ".test_durations").unlink()
+
+    result = run_make(test_project, "test")
+
+    assert result.returncode != 0
+    assert "make test-shard-durations" in result.stdout + result.stderr
+    assert not list(test_project.glob("*.started"))

backend/tests/test_skill_review_core.py

@@ -529,13 +529,21 @@ def test_skillscan_scans_eval_fixture_files_other_than_skill_markdown(tmp_path,
     assert f"{fixture_dir}/SKILL.md" not in scanned_paths
 
 
+def test_case_sensitivity_probe_preserves_existing_files(tmp_path):
+    existing = tmp_path / "CaseProbe"
+    _write(existing, "keep this file")
+
+    _is_case_insensitive_directory(tmp_path)
+
+    assert existing.read_text(encoding="utf-8") == "keep this file"
+    assert list(tmp_path.iterdir()) == [existing]
+
+
 def _is_case_insensitive_directory(path: Path) -> bool:
-    probe = path / "CaseProbe"
-    probe.touch()
-    try:
-        return (path / "caseprobe").exists()
-    finally:
-        probe.unlink()
+    with tempfile.TemporaryDirectory(dir=path) as probe_dir:
+        directory = Path(probe_dir)
+        (directory / "CaseProbe").touch()
+        return (directory / "caseprobe").exists()
 
 
 @pytest.mark.parametrize(