ci: run public skill script tests in Skill Review CI (#7350)

.github/workflows/skill-review-ci.yml

@@ -5,6 +5,7 @@ on:
     branches: ["main", "2.1.x-dev"]
     paths:
       - "skills/public/**"
+      - "tests/skills/**"
       - "backend/packages/harness/deerflow/skills/review/**"
       - "contracts/skill_review/**"
       - "scripts/review_changed_public_skills.py"
@@ -17,6 +18,7 @@ on:
     types: [opened, synchronize, reopened, ready_for_review]
     paths:
       - "skills/public/**"
+      - "tests/skills/**"
       - "backend/packages/harness/deerflow/skills/review/**"
       - "contracts/skill_review/**"
       - "scripts/review_changed_public_skills.py"
@@ -34,6 +36,40 @@ permissions:
   contents: read
 
 jobs:
+  skill-tests:
+    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    env:
+      # Test imports must not leave .pyc files in the reviewed skill packages.
+      PYTHONDONTWRITEBYTECODE: "1"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          # Keep this aligned with backend/Dockerfile and the review job below.
+          version: "0.11.1"
+
+      - name: Install skill test dependencies
+        working-directory: backend
+        run: uv sync --locked --group dev
+
+      - name: Run public skill script tests
+        working-directory: backend
+        run: |
+          # Fail instead of silently skipping data-analysis tests on missing extras.
+          uv run --locked python -c "import duckdb, openpyxl, requests"
+          uv run --locked python -m pytest ../tests/skills -q -rs
+
   skill-review:
     if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
     runs-on: ubuntu-latest

AGENTS.md

@@ -216,8 +216,9 @@ Repo-wide conventions:
 - **Test-driven development** — features and bug fixes ship with tests. Backend tests live
   in `backend/tests/` (TDD is mandatory there; see [backend/AGENTS.md](backend/AGENTS.md));
   frontend tests live in `frontend/tests/`.
-  Public skill script regressions live in `tests/skills/`; exercise dependency fallbacks
-  with the optional import unavailable and mock HTTP transport to keep tests offline.
+  Public skill script regressions live in `tests/skills/` and run in Skill Review CI
+  independently of package review. Exercise dependency fallbacks with the optional
+  import unavailable and mock HTTP transport to keep tests offline.
 - **Before pushing** — backend `make format`, frontend `pnpm check`.
   Backend CI enforces `ruff format --check`.
 - **Skill text encoding** — use explicit UTF-8 for textual skill resources.

backend/tests/test_skill_script_ci.py

@@ -0,0 +1,59 @@
+"""Keep the root public-skill regression suite covered by Skill Review CI."""
+
+from __future__ import annotations
+
+import os
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def _workflow() -> dict:
+    # BaseLoader preserves GitHub Actions' `on` key rather than treating it as
+    # YAML 1.1's boolean True.
+    return yaml.load((REPO_ROOT / ".github/workflows/skill-review-ci.yml").read_text(encoding="utf-8"), Loader=yaml.BaseLoader)
+
+
+@pytest.mark.parametrize("event", ["push", "pull_request"])
+def test_skill_script_changes_trigger_skill_review_ci(event: str) -> None:
+    paths = _workflow()["on"][event]["paths"]
+    assert "tests/skills/**" in paths
+    assert "skills/public/**" in paths
+
+
+def test_skill_ci_collects_every_root_skill_test_module() -> None:
+    workflow = _workflow()
+    job = workflow["jobs"].get("skill-tests")
+    assert job is not None, "Skill Review CI must run public-skill script tests independently of package review"
+    assert not job.get("needs"), "package review failures must not skip the script tests"
+    assert job["if"] == workflow["jobs"]["skill-review"]["if"]
+    step = next(step for step in job["steps"] if "python -m pytest" in step.get("run", ""))
+    command = next(shlex.split(line) for line in step["run"].splitlines() if "python -m pytest" in line)
+    assert command[:6] == ["uv", "run", "--locked", "python", "-m", "pytest"]
+    working_directory = REPO_ROOT / step.get("working-directory", job.get("defaults", {}).get("run", {}).get("working-directory", "."))
+    selection = [argument for argument in command[6:] if not argument.startswith("-")]
+    assert len(selection) == 1
+    assert (working_directory / selection[0]).resolve() == REPO_ROOT / "tests/skills"
+
+    env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", *command[6:], "--collect-only", "-p", "no:cacheprovider"],
+        cwd=working_directory,
+        env=env,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        check=False,
+    )
+    output = result.stdout + result.stderr
+    assert result.returncode == 0, output
+    collected = {Path(line.split("::", 1)[0]).name for line in result.stdout.splitlines() if "::" in line}
+    expected = {path.name for path in (REPO_ROOT / "tests/skills").glob("test_*.py")}
+    assert expected
+    assert collected == expected, output