From 5cdcc40582fab8f3107a44289dd2806bc1a2803e Mon Sep 17 00:00:00 2001 From: Willem Jiang Date: Fri, 23 Oct 2026 16:55:18 +0800 Subject: [PATCH] ci: run public skill script tests in Skill Review CI (#7350) --- .github/workflows/skill-review-ci.yml | 36 ++++++++++++++++ AGENTS.md | 5 ++- backend/tests/test_skill_script_ci.py | 59 +++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 backend/tests/test_skill_script_ci.py diff --git a/.github/workflows/skill-review-ci.yml b/.github/workflows/skill-review-ci.yml index c52d71cf4c..aae8460d6d 100644 --- a/.github/workflows/skill-review-ci.yml +++ b/.github/workflows/skill-review-ci.yml @@ -5,6 +5,7 @@ on: branches: ["main", "2.1.x-dev"] paths: - "skills/public/**" + - "tests/skills/**" - "backend/packages/harness/deerflow/skills/review/**" - "contracts/skill_review/**" - "scripts/review_changed_public_skills.py" @@ -17,6 +18,7 @@ on: types: [opened, synchronize, reopened, ready_for_review] paths: - "skills/public/**" + - "tests/skills/**" - "backend/packages/harness/deerflow/skills/review/**" - "contracts/skill_review/**" - "scripts/review_changed_public_skills.py" @@ -34,6 +36,40 @@ permissions: contents: read jobs: + skill-tests: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + # Test imports must not leave .pyc files in the reviewed skill packages. + PYTHONDONTWRITEBYTECODE: "1" + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + # Keep this aligned with backend/Dockerfile and the review job below. + version: "0.11.1" + + - name: Install skill test dependencies + working-directory: backend + run: uv sync --locked --group dev + + - name: Run public skill script tests + working-directory: backend + run: | + # Fail instead of silently skipping data-analysis tests on missing extras. + uv run --locked python -c "import duckdb, openpyxl, requests" + uv run --locked python -m pytest ../tests/skills -q -rs + skill-review: if: github.event_name != 'pull_request' || github.event.pull_request.draft == false runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index c78639d50d..8ac766ef50 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -216,8 +216,9 @@ Repo-wide conventions: - **Test-driven development** — features and bug fixes ship with tests. Backend tests live in `backend/tests/` (TDD is mandatory there; see [backend/AGENTS.md](backend/AGENTS.md)); frontend tests live in `frontend/tests/`. - Public skill script regressions live in `tests/skills/`; exercise dependency fallbacks - with the optional import unavailable and mock HTTP transport to keep tests offline. + Public skill script regressions live in `tests/skills/` and run in Skill Review CI + independently of package review. Exercise dependency fallbacks with the optional + import unavailable and mock HTTP transport to keep tests offline. - **Before pushing** — backend `make format`, frontend `pnpm check`. Backend CI enforces `ruff format --check`. - **Skill text encoding** — use explicit UTF-8 for textual skill resources. diff --git a/backend/tests/test_skill_script_ci.py b/backend/tests/test_skill_script_ci.py new file mode 100644 index 0000000000..ef1d4a5988 --- /dev/null +++ b/backend/tests/test_skill_script_ci.py @@ -0,0 +1,59 @@ +"""Keep the root public-skill regression suite covered by Skill Review CI.""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _workflow() -> dict: + # BaseLoader preserves GitHub Actions' `on` key rather than treating it as + # YAML 1.1's boolean True. + return yaml.load((REPO_ROOT / ".github/workflows/skill-review-ci.yml").read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + + +@pytest.mark.parametrize("event", ["push", "pull_request"]) +def test_skill_script_changes_trigger_skill_review_ci(event: str) -> None: + paths = _workflow()["on"][event]["paths"] + assert "tests/skills/**" in paths + assert "skills/public/**" in paths + + +def test_skill_ci_collects_every_root_skill_test_module() -> None: + workflow = _workflow() + job = workflow["jobs"].get("skill-tests") + assert job is not None, "Skill Review CI must run public-skill script tests independently of package review" + assert not job.get("needs"), "package review failures must not skip the script tests" + assert job["if"] == workflow["jobs"]["skill-review"]["if"] + step = next(step for step in job["steps"] if "python -m pytest" in step.get("run", "")) + command = next(shlex.split(line) for line in step["run"].splitlines() if "python -m pytest" in line) + assert command[:6] == ["uv", "run", "--locked", "python", "-m", "pytest"] + working_directory = REPO_ROOT / step.get("working-directory", job.get("defaults", {}).get("run", {}).get("working-directory", ".")) + selection = [argument for argument in command[6:] if not argument.startswith("-")] + assert len(selection) == 1 + assert (working_directory / selection[0]).resolve() == REPO_ROOT / "tests/skills" + + env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"} + result = subprocess.run( + [sys.executable, "-m", "pytest", *command[6:], "--collect-only", "-p", "no:cacheprovider"], + cwd=working_directory, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, output + collected = {Path(line.split("::", 1)[0]).name for line in result.stdout.splitlines() if "::" in line} + expected = {path.name for path in (REPO_ROOT / "tests/skills").glob("test_*.py")} + assert expected + assert collected == expected, output