From 04de86f9393263f896fdd318e8e7a59ca711e00f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 29 Oct 2026 15:06:43 +0000 Subject: [PATCH] :sparkles: Add a build task to the local CI orchestrator `./scripts/ci --build` compiles each module the way production does, without going through `manage.sh build-bundle`, which wipes node_modules and reinstalls dependencies. It runs the release commands already defined per module and reuses existing caches, so a broken release build (shadow-cljs optimizations, the wasm targets, the backend uberjar, tsc/typedoc) shows up before a bundle is built. The task is opt-in and runs last, after lint, test and fmt, so the default flow keeps its duration. `common` has no build of its own and reports as skipped. `media-processor` joins as the ninth module, with its lint skipped and `tsc --noEmit` folded into its build. All build outputs are gitignored. `scripts/test_ci.py` covers the flag wiring and one command per module through `--dry-run`; it never runs a real build. AI-assisted-by: deepseek-v4.1-flash --- .agents/skills/local-ci/SKILL.md | 22 ++++- .serena/memories/scripts/ci.md | 72 +++++++++++--- scripts/ci | 38 +++++++- scripts/test_ci.py | 159 +++++++++++++++++++++++++++++++ 4 files changed, 273 insertions(+), 18 deletions(-) create mode 100644 scripts/test_ci.py diff --git a/.agents/skills/local-ci/SKILL.md b/.agents/skills/local-ci/SKILL.md index 5cda6acf89..d04f581dc1 100644 --- a/.agents/skills/local-ci/SKILL.md +++ b/.agents/skills/local-ci/SKILL.md @@ -1,6 +1,6 @@ --- name: local-ci -description: Run local CI-style checks with ./scripts/ci (lint, tests, format) per monorepo module. Use when verifying changes before declaring work done, running lint or tests locally, fixing formatting, or repairing Clojure delimiter errors. +description: Run local CI-style checks with ./scripts/ci (lint, tests, format, and on request production builds) per monorepo module. Use when verifying changes before declaring work done, running lint or tests locally, fixing formatting, repairing Clojure delimiter errors, or checking that a module still compiles. --- # Local CI @@ -29,10 +29,12 @@ Run from the repo root: ``` Modules: `frontend` `backend` `common` `render-wasm` `exporter` `mcp` -`plugins` `library`, or `--all` for every module. +`plugins` `library` `media-processor`, or `--all` for every module. With no task flags it runs three tasks per module, in order: **lint**, -**test**, **fmt** (format check; `--fix` formats files instead). +**test**, **fmt** (format check; `--fix` formats files instead). **build** +is opt-in: it compiles the module the way production does and is the slow +one, so it never runs unless you ask for it. | Flag | Effect | |------|--------| @@ -41,6 +43,7 @@ With no task flags it runs three tasks per module, in order: **lint**, | `--lint` / `--no-lint` | Run only lint / drop lint | | `--test` / `--no-test` | Run only tests / drop tests | | `--fmt` / `--no-fmt` | Run only format check / drop it | +| `--build` / `--no-build` | Run only the production build / drop it | | `--fix` | Format files instead of checking (other tasks unaffected) | | `--paren-repair` | Fix delimiter errors in Clojure/CLJS files | | `--fail-fast` | Stop at the first failure | @@ -72,6 +75,12 @@ log path. # Format the module without running the test suite ./scripts/ci --fix --no-test frontend +# Check that the module still compiles (slow: minutes) +./scripts/ci --build frontend + +# Compile every module (8 builds, ~4 min with warm caches) +./scripts/ci --build --all + # Broke delimiters in Clojure/CLJS files: repair first, then lint ./scripts/ci --paren-repair frontend ./scripts/ci --lint frontend @@ -89,7 +98,12 @@ log path. - Test tasks are long-running (backend runs `clojure -M:dev:test`); give the bash call a generous timeout (10–20 minutes) instead of letting it time out mid-run. -- `mcp` has no lint task — it shows as skipped, not failed. +- `mcp` and `media-processor` have no lint task — they show as skipped, + not failed. `common` has no build task for the same reason. +- Build runs are the slow ones (frontend ~2 min even with warm caches; + cold caches roughly double it). Give the shell a generous timeout and + do not run two builds at once in the same checkout: `render-wasm` takes + a build lock. - `--paren-repair` only fixes delimiters; run lint afterwards to catch what remains. See `mem:scripts/paren-repair`. - What to run and how to read test results: `mem:testing`. diff --git a/.serena/memories/scripts/ci.md b/.serena/memories/scripts/ci.md index e3ee25c614..d236bad383 100644 --- a/.serena/memories/scripts/ci.md +++ b/.serena/memories/scripts/ci.md @@ -1,8 +1,9 @@ # CI (scripts/ci) -`scripts/ci` runs CI-style checks — lint, tests, format — for one or more -monorepo modules and prints a per-task summary. It is the local equivalent -of CI; use it to verify changes before declaring work done. +`scripts/ci` runs CI-style checks — lint, tests, format — and, on request, +the production build of each module, for one or more monorepo modules, and +prints a per-task summary. It is the local equivalent of CI; use it to +verify changes before declaring work done. ## When to use @@ -19,6 +20,7 @@ Run from the repo root: ```bash ./scripts/ci MODULE... # lint + test + fmt per module +./scripts/ci --build MODULE # production build only ./scripts/ci --all --no-test # lint + fmt on all modules ./scripts/ci --lint frontend # lint only ./scripts/ci --fix --no-test frontend # format files, skip tests @@ -26,14 +28,15 @@ Run from the repo root: ./scripts/ci --dry-run --all # preview what would run ``` -Modules: `frontend backend common render-wasm exporter mcp plugins library`. +Modules: `frontend backend common render-wasm exporter mcp plugins library +media-processor`. Flags: - Default tasks: `lint`, `test`, `fmt` (format check; `--fix` formats - instead). -- `--lint` / `--test` / `--fmt` run one task only; `--no-lint` / - `--no-test` / `--no-fmt` drop one task from the default set. + instead). Build is **not** in the default set. +- `--lint` / `--test` / `--fmt` / `--build` run one task only; `--no-lint` / + `--no-test` / `--no-fmt` / `--no-build` drop one task from the default set. - `--paren-repair` runs only the delimiter repair — it wraps `scripts/paren-repair` over each module's Clojure/CLJS sources; see `mem:scripts/paren-repair`. @@ -42,6 +45,49 @@ Flags: output; `--dry-run` prints commands without running; `--clean` removes the log directory. +## Build task + +`--build` compiles each module the way production does, reusing caches +(`.shadow-cljs`, `target/`, `node_modules`) instead of clearing them. It +does not package bundles: `manage.sh build-bundle` remains the way to +produce deployable artifacts. + +| Module | Command | Output | +|---|---|---| +| `frontend` | `NODE_ENV=production pnpm run build:app:main && pnpm run build:app:libs && pnpm run build:app:assets` | `frontend/resources/public/`, `frontend/target/` | +| `backend` | `clojure -T:build jar` | `backend/target/penpot.jar` | +| `common` | *(none; skipped)* | — | +| `render-wasm` | `./build` (targets `frontend` and `export`) | `render-wasm/target/` | +| `exporter` | `NODE_ENV=production pnpm run build:app` | `exporter/target/` | +| `mcp` | `pnpm run build` | `mcp/*/dist` | +| `plugins` | `pnpm run build:runtime && pnpm run build:plugins && pnpm run build:styles-example && pnpm run build:doc` | `plugins/dist/` | +| `library` | `NODE_ENV=production clojure -M:dev:shadow-cljs release library` | `library/target/library/` | +| `media-processor` | `pnpm run types:check && pnpm run build` | `media-processor/dist/` | + +- Every output path above is gitignored, so a build leaves `git status` + clean. +- The module `build` scripts in each `scripts/build` are **not** used: they + delete `node_modules`, wipe `resources/public` and run `pnpm install`, + which belongs to the bundle pipeline, not to a local check. +- `frontend` and `library` have a `build` script that clears the shadow-cljs + cache first; the task calls the inner commands to stay incremental. + `exporter` uses `build:app` for the same reason. +- `common` has no artifact of its own: its compilation is exercised by its + `test` task and by the builds of `frontend`, `backend` and `exporter`. +- `media-processor` is the ninth module. It has no lint, so that task shows + as skipped; its build runs `tsc --noEmit` before esbuild because esbuild + does not type-check. +- Cold-cache cost measured on the devenv: `library` ~120 s, `frontend` ~120 s, + `exporter` ~80 s, `backend` ~35 s. With warm caches the whole set (8 + builds) takes ~4 min. The first run of `clojure -T:build jar` downloads + `tools.build` if it is not in `~/.m2`. +- `pnpm run` may trigger a dependency check/install when `node_modules` + drifts from the lockfile (`verify-deps-before-run`), so the first run + after a dependency change can need network. +- `render-wasm/build` takes a lock (`_build_lock`): never run two builds at + once in the same checkout. + + ## Logs and exit codes - Full output of every task: `.ci-logs/-.log`. @@ -53,12 +99,16 @@ Flags: ## Notes -- `mcp` has no lint task (shows as skipped). `render-wasm` uses `./lint`, - `./test`, and `cargo fmt`. +- `mcp` has no lint task and `media-processor` has no lint task; both show + as skipped. `render-wasm` uses `./lint`, `./test`, and `cargo fmt`. +- `scripts/test_ci.py` (unittest, `python3 scripts/test_ci.py`) covers the + flag wiring, one build command per module and the help text through + `--dry-run`; it executes no real build. - Exports `PENPOT_FLAGS` with `enable-backend-asserts` appended to any existing value, so backend test runs compile in and exercise precondition asserts; see `mem:backend/testing`. -- Test tasks are long-running (backend: `clojure -M:dev:test`); use a - generous timeout when calling it from an agent shell. +- Test and build tasks are long-running (backend: `clojure -M:dev:test`; + frontend build: ~2 min warm); use a generous timeout when calling it from + an agent shell. - Skill entry point: `.agents/skills/local-ci/SKILL.md`. - Testing principles and output discipline: `mem:testing`. diff --git a/scripts/ci b/scripts/ci index 4428b673f9..db58292ae9 100755 --- a/scripts/ci +++ b/scripts/ci @@ -24,7 +24,7 @@ BOLD='\033[1m' NC='\033[0m' # Available modules -ALL_MODULES=("frontend" "backend" "common" "render-wasm" "exporter" "mcp" "plugins" "library") +ALL_MODULES=("frontend" "backend" "common" "render-wasm" "exporter" "mcp" "plugins" "library" "media-processor") # Module commands declare -A LINT_CMD=( @@ -36,6 +36,7 @@ declare -A LINT_CMD=( [mcp]="" [plugins]="pnpm run lint" [library]="pnpm run lint" + [media-processor]="" ) declare -A TEST_CMD=( @@ -47,6 +48,7 @@ declare -A TEST_CMD=( [mcp]="pnpm run test" [plugins]="pnpm run test" [library]="pnpm run test" + [media-processor]="pnpm run test" ) declare -A FMT_CHECK_CMD=( @@ -58,6 +60,7 @@ declare -A FMT_CHECK_CMD=( [mcp]="pnpm run fmt:check" [plugins]="pnpm run format:check" [library]="pnpm run check-fmt:clj" + [media-processor]="pnpm run fmt:check" ) declare -A FMT_FIX_CMD=( @@ -69,6 +72,19 @@ declare -A FMT_FIX_CMD=( [mcp]="pnpm run fmt" [plugins]="pnpm run format" [library]="pnpm run fmt:clj" + [media-processor]="pnpm run fmt" +) + +declare -A BUILD_CMD=( + [frontend]="NODE_ENV=production pnpm run build:app:main && pnpm run build:app:libs && pnpm run build:app:assets" + [backend]="clojure -T:build jar" + [common]="" + [render-wasm]="./build" + [exporter]="NODE_ENV=production pnpm run build:app" + [mcp]="pnpm run build" + [plugins]="pnpm run build:runtime && pnpm run build:plugins && pnpm run build:styles-example && pnpm run build:doc" + [library]="NODE_ENV=production clojure -M:dev:shadow-cljs release library" + [media-processor]="pnpm run types:check && pnpm run build" ) declare -A PAREN_REPAIR_CMD=( @@ -80,6 +96,7 @@ declare -A PAREN_REPAIR_CMD=( [mcp]="" [plugins]="" [library]="find src test -name '*.clj' -o -name '*.cljs' -o -name '*.cljc' | xargs $PROJECT_ROOT/scripts/paren-repair" + [media-processor]="" ) # Default options @@ -197,6 +214,9 @@ run_module() { cmd="${FMT_CHECK_CMD[$module]:-}" fi ;; + build) + cmd="${BUILD_CMD[$module]:-}" + ;; paren-repair) cmd="${PAREN_REPAIR_CMD[$module]:-}" ;; @@ -282,8 +302,8 @@ Usage: $(basename "$0") [OPTIONS] [MODULES...] CI orchestration script for Penpot monorepo. MODULES: - frontend, backend, common, render-wasm, exporter, mcp, plugins, library - Use --all to run all modules. + frontend, backend, common, render-wasm, exporter, mcp, plugins, library, + media-processor. Use --all to run all modules. OPTIONS: --all Run all modules @@ -294,6 +314,8 @@ OPTIONS: --no-test Skip tests --fmt Run format check only --no-fmt Skip format check + --build Run production build only + --no-build Skip build --fix Run format fix instead of check --paren-repair Fix delimiter errors in Clojure/CLJS files --fail-fast Stop on first failure @@ -307,6 +329,8 @@ EXAMPLES: $(basename "$0") frontend backend # Run lint, test, fmt on frontend and backend $(basename "$0") --all --no-test # Run lint and fmt on all modules $(basename "$0") --lint common # Run only lint on common + $(basename "$0") --build frontend # Run only the production build of frontend + $(basename "$0") --build --all # Build every module $(basename "$0") --fix frontend # Fix formatting in frontend $(basename "$0") --paren-repair frontend # Fix delimiter errors in frontend $(basename "$0") --paren-repair --all # Fix delimiter errors in all Clojure modules @@ -353,6 +377,14 @@ main() { TASKS=("${TASKS[@]/fmt/}") shift ;; + --build) + TASKS=("build") + shift + ;; + --no-build) + TASKS=("${TASKS[@]/build/}") + shift + ;; --paren-repair) TASKS=("paren-repair") shift diff --git a/scripts/test_ci.py b/scripts/test_ci.py new file mode 100644 index 0000000000..405f385d8e --- /dev/null +++ b/scripts/test_ci.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Tests for scripts/ci. + +Run with: + + python3 scripts/test_ci.py + +Covers the wiring of the orchestrator around the `build` task and the +`media-processor` module: the flag set, one build command per module, the +help text, the dry-run mode (which must execute and write nothing), and the +default task list staying free of build. +""" + +import pathlib +import re +import subprocess +import sys +import unittest + +sys.dont_write_bytecode = True + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +CI = REPO_ROOT / "scripts" / "ci" +LOG_DIR = REPO_ROOT / ".ci-logs" + +ANSI = re.compile(r"\x1b\[[0-9;]*m") + +# Modules with a production build. `common` has no build of its own: its +# compilation happens in its test task and in the modules consuming it. +BUILD_MODULES = ( + "frontend", + "backend", + "render-wasm", + "exporter", + "mcp", + "plugins", + "library", + "media-processor", +) + +ALL_MODULES = BUILD_MODULES + ("common",) + + +def run_ci(*args, cwd=REPO_ROOT): + """Run scripts/ci and return (exit_code, output without ANSI escapes).""" + proc = subprocess.run( + [str(CI), *args], + cwd=cwd, + capture_output=True, + text=True, + ) + return proc.returncode, ANSI.sub("", proc.stdout + proc.stderr) + + +def log_snapshot(): + """Names and mtimes of every entry under .ci-logs, or None when absent.""" + if not LOG_DIR.exists(): + return None + return sorted((p.name, p.stat().st_mtime_ns) for p in LOG_DIR.iterdir()) + + +class BuildTaskTests(unittest.TestCase): + def test_build_all_has_one_command_per_module_without_common(self): + code, out = run_ci("--build", "--dry-run", "--all") + + self.assertEqual(code, 0, out) + self.assertEqual(out.count("build: would run"), len(BUILD_MODULES), out) + self.assertIn("build: not defined for common, skipping", out) + for module in BUILD_MODULES: + self.assertIn(f" in {module}/", out) + + def test_build_is_not_part_of_the_default_tasks(self): + code, out = run_ci("--dry-run", "--all") + + self.assertEqual(code, 0, out) + self.assertNotIn("build:", out) + for task in ("lint", "test", "fmt"): + self.assertIn(f"{task}: would run", out) + + def test_build_flag_runs_build_only(self): + code, out = run_ci("--build", "--dry-run", "frontend") + + self.assertEqual(code, 0, out) + self.assertIn("build: would run", out) + for task in ("lint", "test", "fmt"): + self.assertNotIn(f"{task}:", out) + + def test_no_build_flag_is_accepted(self): + code, out = run_ci("--no-build", "--dry-run", "frontend") + + self.assertEqual(code, 0, out) + self.assertIn("lint: would run", out) + self.assertNotIn("build:", out) + + def test_dry_run_executes_nothing(self): + before = log_snapshot() + + run_ci("--build", "--dry-run", "mcp") + + self.assertEqual(before, log_snapshot()) + + +class MediaProcessorModuleTests(unittest.TestCase): + def test_module_is_accepted_and_lint_is_skipped(self): + code, out = run_ci("--dry-run", "media-processor") + + self.assertEqual(code, 0, out) + self.assertIn( + "lint: not defined for media-processor, skipping", + out, + ) + for task in ("test", "fmt"): + self.assertIn(f"{task}: would run", out) + + def test_module_is_included_in_all(self): + code, out = run_ci("--dry-run", "--all") + + self.assertEqual(code, 0, out) + self.assertIn("media-processor", out) + self.assertIn("media-processor:test", out) + + def test_build_runs_types_check_before_esbuild(self): + code, out = run_ci("--build", "--dry-run", "media-processor") + + self.assertEqual(code, 0, out) + self.assertIn( + "pnpm run types:check && pnpm run build", + out, + ) + + +class HelpTests(unittest.TestCase): + def help_options(self, out): + """Option tokens from the OPTIONS section, e.g. '--build'.""" + options = [] + for line in out.splitlines(): + stripped = line.strip() + if stripped.startswith("--"): + options.append(stripped.split()[0]) + return options + + def test_help_documents_the_build_flags(self): + code, out = run_ci("--help") + + self.assertEqual(code, 0, out) + options = self.help_options(out) + self.assertIn("--build", options) + self.assertIn("--no-build", options) + + def test_help_lists_every_module(self): + code, out = run_ci("--help") + + self.assertEqual(code, 0, out) + for module in ALL_MODULES: + self.assertIn(module, out) + + +if __name__ == "__main__": + unittest.main()