✨ Add test coverage reports to CI and pull requests (#13156)

Measure the coverage of the backend, common, frontend and render-wasm
test suites in CI, and show it on the workflow run pages and in one
"Coverage Report" comment on each pull request.

- Backend and common run kaocha with the cloverage plugin through a new
  `:coverage` alias, which writes lcov, HTML and an EMMA report. The PR
  report shows lines and Clojure forms (as statements); cloverage
  measures no functions or branches. Plain local runs are unchanged.
- Frontend runs the compiled test bundle with V8 coverage
  (`pnpm run test:coverage`, `frontend/scripts/coverage.js`) and maps it
  back to the ClojureScript sources through the source maps of the test
  build. UI namespaces (`app/main/ui`) are left out.
- Render-wasm runs its native Rust tests under cargo-llvm-cov when
  `RENDER_WASM_COVERAGE` is set. The devenv image lacks the tool, so the
  workflow downloads a pinned release and checks its sha256.
- `scripts/coverage-summary.mjs` renders an lcov file as a markdown table
  by source area for `$GITHUB_STEP_SUMMARY`; each workflow also uploads
  the HTML report as an artifact.
- `scripts/coverage-comment.mjs` keeps a single PR comment, laid out like
  the Vitest Coverage Report Action, with a section per project and the
  coverage of the changed files. Each job updates its own section; the
  data of every project lives in a hidden block of the comment, and a
  job writes again when a concurrent one dropped its data.
- Lines have targets that only colour the report: 70% backend and
  common, 50% frontend, 40% render-wasm.
- The comment step only runs for PRs from branches of this repo: fork
  PRs get a read-only token and keep the summary on the run page.

AI-assisted-by: claude-opus-5-5
This commit is contained in:
Alonso Torres 2026-11-18 17:53:32 +01:00 committed by GitHub
parent eddb824ca7
commit 15f9994022
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 917 additions and 7 deletions

View File

@ -34,6 +34,9 @@ permissions:
jobs:
test-backend:
if: ${{ !github.event.pull_request.draft }}
permissions:
contents: read
pull-requests: write # coverage comment on the PR
name: "Backend Tests"
runs-on: penpot-extended-runner
container:
@ -91,4 +94,39 @@ jobs:
run: |
mkdir -p /tmp/penpot;
clojure -M:dev:test
clojure -M:dev:test:coverage
- name: Coverage summary
if: ${{ !cancelled() && hashFiles('backend/target/coverage/lcov.info') != '' }}
run: |
node scripts/coverage-summary.mjs \
--title "Backend test coverage" \
--note "Namespaces under \`backend/src/app\`." \
--lcov backend/target/coverage/lcov.info \
> backend/target/coverage/summary.md
cat backend/target/coverage/summary.md >> "$GITHUB_STEP_SUMMARY"
- name: Coverage comment
# Fork PRs get a read-only token, so only branches of this repo comment.
if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && hashFiles('backend/target/coverage/lcov.info') != '' }}
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
node scripts/coverage-comment.mjs \
--project Backend \
--lcov backend/target/coverage/lcov.info \
--emma-xml backend/target/coverage/coverage.xml \
--prefix backend/
- name: Upload coverage report
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: ${{ !cancelled() }}
with:
name: backend-coverage
path: backend/target/coverage/
overwrite: true
if-no-files-found: ignore
retention-days: 7

View File

@ -32,6 +32,9 @@ permissions:
jobs:
test-common:
if: ${{ !github.event.pull_request.draft }}
permissions:
contents: read
pull-requests: write # coverage comment on the PR
name: "Common Tests"
runs-on: penpot-extended-runner
container:
@ -58,3 +61,38 @@ jobs:
working-directory: ./common
run: |
./scripts/test
- name: Coverage summary
if: ${{ !cancelled() && hashFiles('common/target/coverage/lcov.info') != '' }}
run: |
node scripts/coverage-summary.mjs \
--title "Common JVM test coverage" \
--note "Namespaces under \`common/src/app\`, measured by the JVM tests." \
--lcov common/target/coverage/lcov.info \
> common/target/coverage/summary.md
cat common/target/coverage/summary.md >> "$GITHUB_STEP_SUMMARY"
- name: Coverage comment
# Fork PRs get a read-only token, so only branches of this repo comment.
if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && hashFiles('common/target/coverage/lcov.info') != '' }}
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
node scripts/coverage-comment.mjs \
--project Common \
--lcov common/target/coverage/lcov.info \
--emma-xml common/target/coverage/coverage.xml \
--prefix common/
- name: Upload coverage report
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: ${{ !cancelled() }}
with:
name: common-coverage
path: common/target/coverage/
overwrite: true
if-no-files-found: ignore
retention-days: 7

View File

@ -36,6 +36,9 @@ permissions:
jobs:
test-frontend:
if: ${{ !github.event.pull_request.draft }}
permissions:
contents: read
pull-requests: write # coverage comment on the PR
name: "Frontend Tests"
runs-on: penpot-extended-runner
container:
@ -66,6 +69,42 @@ jobs:
run: |
./scripts/test
- name: Coverage summary
if: ${{ !cancelled() && hashFiles('frontend/coverage/lcov.info') != '' }}
run: |
node scripts/coverage-summary.mjs \
--title "Frontend unit test coverage" \
--note "Logic namespaces under \`frontend/src/app\`, without \`app/main/ui\`." \
--lcov frontend/coverage/lcov.info \
--unloaded frontend/coverage/unloaded.txt \
> frontend/coverage/summary.md
cat frontend/coverage/summary.md >> "$GITHUB_STEP_SUMMARY"
- name: Coverage comment
# Fork PRs get a read-only token, so only branches of this repo comment.
if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && hashFiles('frontend/coverage/lcov.info') != '' }}
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
node scripts/coverage-comment.mjs \
--project Frontend \
--lcov frontend/coverage/lcov.info \
--json-summary frontend/coverage/coverage-summary.json \
--unloaded frontend/coverage/unloaded.txt
- name: Upload coverage report
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: ${{ !cancelled() }}
with:
name: frontend-coverage
path: frontend/coverage/
overwrite: true
if-no-files-found: ignore
retention-days: 7
- name: Component Tests
working-directory: ./frontend
env:

View File

@ -32,6 +32,9 @@ permissions:
jobs:
test-render-wasm:
if: ${{ !github.event.pull_request.draft }}
permissions:
contents: read
pull-requests: write # coverage comment on the PR
name: "Render WASM Tests"
runs-on: penpot-extended-runner
container:
@ -56,7 +59,64 @@ jobs:
run: |
./lint
- name: Install cargo-llvm-cov
env:
CARGO_LLVM_COV_VERSION: "0.9.1"
run: |
case "$(uname -m)" in
aarch64|arm64)
arch="aarch64"
sha256="abf5f13c1520f8756d2192bfaaeadb0208f5b7eaa8cadc15bf847befa42b7360"
;;
*)
arch="x86_64"
sha256="b3f68e625481fed9b16444174f3fa5ebcdbde4a1878803a35eabe2dcefcdc41a"
;;
esac
archive="cargo-llvm-cov-${arch}-unknown-linux-gnu.tar.gz"
curl -fsSL --retry 3 -o "$RUNNER_TEMP/$archive" \
"https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${CARGO_LLVM_COV_VERSION}/${archive}"
echo "${sha256} $RUNNER_TEMP/$archive" | sha256sum -c -
tar -xzf "$RUNNER_TEMP/$archive" -C /usr/local/bin cargo-llvm-cov
cargo llvm-cov --version
- name: Test
working-directory: ./render-wasm
env:
RENDER_WASM_COVERAGE: "true"
run: |
./test
- name: Coverage summary
if: ${{ !cancelled() && hashFiles('render-wasm/target/coverage/lcov.info') != '' }}
run: |
node scripts/coverage-summary.mjs \
--title "Render WASM test coverage" \
--note "Rust sources under \`render-wasm/src\`, measured by cargo-llvm-cov on the native test build." \
--lcov render-wasm/target/coverage/lcov.info \
> render-wasm/target/coverage/summary.md
cat render-wasm/target/coverage/summary.md >> "$GITHUB_STEP_SUMMARY"
- name: Coverage comment
# Fork PRs get a read-only token, so only branches of this repo comment.
if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && hashFiles('render-wasm/target/coverage/lcov.info') != '' }}
continue-on-error: true
env:
GITHUB_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
node scripts/coverage-comment.mjs \
--project "Render WASM" \
--lcov render-wasm/target/coverage/lcov.info \
--llvm-json render-wasm/target/coverage/summary.json
- name: Upload coverage report
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: ${{ !cancelled() }}
with:
name: render-wasm-coverage
path: render-wasm/target/coverage/
overwrite: true
if-no-files-found: ignore
retention-days: 7

1
.gitignore vendored
View File

@ -68,6 +68,7 @@ opencode.json
/frontend/playwright-report/
/frontend/playwright/ui/visual-specs/
/frontend/text-editor/src/wasm/
/frontend/coverage/
/frontend/dist/
/frontend/npm-debug.log
/frontend/out/

View File

@ -13,6 +13,7 @@ JVM `clojure.test` (kaocha runner) under `backend/test/backend_tests/`.
- Isolated run: `clojure -M:dev:test --focus backend-tests.my-ns-test` for a specific test namespace, or `clojure -M:dev:test --focus backend-tests.my-ns-test/my-test-var` for a specific test var.
- Multiple focus symbols are supported by REPEATING the flag (kaocha accumulates them, last-wins does not apply): `clojure -M:dev:test --focus backend-tests.ns-one --focus backend-tests.ns-two` — preferred over a shell loop of isolated runs.
- Regression run: `clojure -M:dev:test` to ensure no regressions in related functional areas.
- Line coverage: append the `:coverage` alias (`clojure -M:dev:test:coverage`, kaocha-cloverage). It instruments every `app.*` namespace under `src/` at load, before any test runs, so even a `--focus` run instruments them all; options live under `:cloverage/opts` in `tests.edn`, output in `target/coverage/` (lcov, HTML, and EMMA `coverage.xml` whose form counts the PR report shows as statements; cloverage measures no functions or branches) (`lcov.info`, HTML). About 2x slower and noisy with reflection warnings (instrumented forms lose type hints): never use it for normal runs. `tests-backend.yml` runs it and renders the step summary with `scripts/coverage-summary.mjs`.
- If you need to filter output, tee to a temp file first: `clojure -M:dev:test 2>&1 | tee /tmp/penpot-test-output.txt`.
- RPC test helpers `command!`/`management-command!` split the data map: qualified keys become server params, unqualified keys become request body params. To inject request-level context (headers, `:app.http/auth-key-id`, ip), pass a map under `:app.http/request` metadata; non-map `IRequest` stubs fall back to a dummy request.
- The test system is `main/system-config` + `main/worker-config` with the

View File

@ -15,6 +15,7 @@ From `common/`:
- Full JS test run (always builds, build output visible): `pnpm run test`
- Focus a JVM test namespace: `clojure -M:dev:test --focus common-tests.logic.variants-switch-test`
- Focus a JVM test var: `clojure -M:dev:test --focus common-tests.logic.variants-switch-test/test-basic-switch`
- JVM line coverage: `clojure -M:dev:test:coverage` (kaocha-cloverage, about 2x slower; `:cloverage/opts` in `tests.edn`, which excludes `app.common.test-helpers.*`; output in `target/coverage/` (lcov, HTML, and EMMA `coverage.xml` whose form counts the PR report shows as statements; cloverage measures no functions or branches)). `scripts/test` (CI) uses it; the JS side has no coverage.
- Focus a JS test namespace: `pnpm run test:quiet -- --focus common-tests.logic.comp-sync-test`
- Focus a JS test var: `pnpm run test:quiet -- --focus common-tests.logic.comp-sync-test/test-sync-when-changing-attribute`
- Quiet logging during a JS run: append `--log-level warn` (or `trace|debug|info|warn|error`)

View File

@ -61,6 +61,8 @@ The memory is structured in a way that you can get the critical information abou
- `scripts/psql` — PostgreSQL client wrapper with devenv defaults. Companion: `scripts/db-schema` for DDL dumps. See `mem:scripts/psql`.
- `scripts/taiga.py` — Fetch public issues, user stories, and tasks from the Penpot Taiga project without authentication. See `mem:scripts/taiga`.
- `scripts/gh.py` — GitHub operations helper: list milestone issues, fetch PR details, compare against CHANGES.md. Requires `gh` CLI. See `mem:scripts/gh`.
- `scripts/coverage-summary.mjs` — Renders an lcov file as a markdown table grouped by source area, for `$GITHUB_STEP_SUMMARY` (`--title`, `--lcov`, optional `--note`, `--unloaded`). Used by the backend, common and frontend test workflows; module coverage commands are in `mem:backend/testing`, `mem:common/testing`, `mem:frontend/testing`.
- `scripts/coverage-comment.mjs` — Keeps one PR comment (`<!-- coverage-report -->`) with the coverage table of Backend, Common, Frontend and Render WASM, in the format of the Vitest Coverage Report Action (status, metric, %, covered/total, changed-file coverage, run/commit footer). Each test job updates its own project: the data of all three lives base64-encoded in a hidden `coverage-data` block, and the job re-reads after `SETTLE_MS` and rewrites if a concurrent job dropped its data; extra report comments from simultaneous creates are merged and deleted. Targets (`THRESHOLDS`, per project and metric, status colour only, never fail CI) are on lines only: 70 Backend, 70 Common, 50 Frontend, 40 Render WASM; other metrics get the blue dot. Statements are Clojure forms for Backend/Common (`--emma-xml`) and llvm regions for Render WASM (`--llvm-json`). Shared lcov parsing: `scripts/coverage-lcov.mjs`. The test workflows run it only for PRs from branches of this repo (job-level `pull-requests: write`, `continue-on-error`): fork PRs get a read-only token, so they only get the run-page summary. A fork-safe `workflow_run` commenter would run only from the default branch's copy of the workflow.
- `scripts/project-anomalies.py` — Main-board anomaly check for a milestone (`check` writes `tmp/<MILESTONE>-ANOMALIES.md`). Bulk resolution via `scripts/gh.py`. See `mem:scripts/project-anomalies`.
- `scripts/changelog.py` — Changelog checks for `mem:workflow/updating-changelog`: `boom` (generate the `:boom:` section from `breaking change` issues), `check-merged` (PR merge status), `cross-ref` (milestone PRs vs changelog section), `report` (anomaly report to CHANGES-ISSUES.md). Calls `scripts/gh.py` via subprocess. See `mem:scripts/changelog`.
- `scripts/error-reports.mjs` — Query error reports via RPC API with token authentication. Supports list/get operations with filtering and pagination. See `mem:scripts/error-reports`.

View File

@ -51,6 +51,7 @@ From `frontend/`:
- Build test target only (no run): `pnpm run build:test`.
- After `build:test` has been run, run the compiled runner directly: `node target/tests/test.js [--focus ...] [--log-level ...]`.
- Watch tests: `pnpm run watch:test`.
- Coverage: `pnpm run test:coverage` (builds, runs the suite under `NODE_V8_COVERAGE`, keeps its exit code). `scripts/coverage.js` maps V8 data back to `.cljs`/`.cljc` through the test build's source maps (`ast-v8-to-istanbul` + acorn) and writes `coverage/` (`lcov.info`, `coverage-summary.json`, `html/`, `unloaded.txt`); `tests-frontend.yml` renders the step summary with `scripts/coverage-summary.mjs` and uploads `coverage/` as the `frontend-coverage` artifact. `scripts/test` (CI) runs `test:coverage`, not `test`. Scope: `frontend/src/app/**` minus `app/main/ui/` (`EXCLUDED_DIRS`); namespaces no test loads are listed apart, never counted. Never pass `ignoreSourceCode` to `convert`: CLJS maps point past source lines and it throws. Trust line % over branch % (branches are of the generated JS).
New frontend test namespaces must be required/listed in `frontend_tests/runner.cljs`; new vars in existing namespaces need no runner change.

View File

@ -34,6 +34,7 @@ From `render-wasm/`:
- Clean one target: `./clean [frontend|export]`; never run root `cargo clean` on the shared `target/`.
- Watch rebuild: `./watch [frontend|export]`; its initial and change-triggered builds use `./build`.
- Rust tests: `./test` or `cargo test <name>`.
- Rust coverage: `RENDER_WASM_COVERAGE=1 ./test` runs the same tests under `cargo llvm-cov` (needs `cargo-llvm-cov` on PATH; the script adds `llvm-tools-preview`) and writes `target/coverage/` (`lcov.info` with absolute paths, `summary.json` llvm export totals, `html/`). Native build only: code behind wasm-only `cfg` is not measured. `tests-wasm.yml` downloads a pinned, checksum-verified `cargo-llvm-cov` release because the devenv image lacks it.
- Cross-cutting testing principles and anti-patterns: `mem:testing`.
- Lint: `./lint`.
- Format check: `cargo fmt --check`.

View File

@ -95,6 +95,11 @@
:jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"]
:extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
;; Use after :test (`-M:dev:test:coverage`) to run the suite with cloverage.
:coverage
{:main-opts ["-m" "kaocha.runner" "--plugin" "cloverage"]
:extra-deps {lambdaisland/kaocha-cloverage {:mvn/version "1.1.89"}}}
:outdated
{:extra-deps {com.github.liquidz/antq {:mvn/version "2.11.1276"}}
:main-opts ["-m" "antq.core"]}

View File

@ -3,4 +3,16 @@
[{:id :unit
:test-paths ["test" "src"]
:ns-patterns [".*-test$"]
:kaocha/reporter [kaocha.report/dots]}]}
:kaocha/reporter [kaocha.report/dots]}]
;; Only read when the cloverage plugin is on (`:coverage` alias).
:cloverage/opts {:src-ns-path ["src"]
:ns-regex ["^app\\..*"]
:output "target/coverage"
:lcov? true
;; EMMA XML carries the form counts the PR report shows as statements.
:emma-xml? true
:html? true
:summary? true
:codecov? false
:fail-threshold 0}}

View File

@ -76,6 +76,11 @@
{:main-opts ["-m" "kaocha.runner"]
:extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}}
;; Use after :test (`-M:dev:test:coverage`) to run the suite with cloverage.
:coverage
{:main-opts ["-m" "kaocha.runner" "--plugin" "cloverage"]
:extra-deps {lambdaisland/kaocha-cloverage {:mvn/version "1.1.89"}}}
:shadow-cljs
{:main-opts ["-m" "shadow.cljs.devtools.cli"]
:jvm-opts ["--sun-misc-unsafe-memory-access=allow"]}

View File

@ -3,4 +3,4 @@
set -ex
pnpm install;
pnpm run test;
clojure -M:dev:test;
clojure -M:dev:test:coverage;

View File

@ -1,4 +1,17 @@
#kaocha/v1
{:tests [{:id :unit
:test-paths ["test"]}]
:kaocha/reporter [kaocha.report/dots]}
:kaocha/reporter [kaocha.report/dots]
;; Only read when the cloverage plugin is on (`:coverage` alias).
:cloverage/opts {:src-ns-path ["src"]
:ns-regex ["^app\\..*"]
:ns-exclude-regex ["^app\\.common\\.test-helpers\\..*"]
:output "target/coverage"
:lcov? true
;; EMMA XML carries the form counts the PR report shows as statements.
:emma-xml? true
:html? true
:summary? true
:codecov? false
:fail-threshold 0}}

View File

@ -36,6 +36,7 @@
"build:test": "pnpm run build:wasm && clojure -M:dev:shadow-cljs compile test",
"test": "pnpm run build:test && node target/tests/test.js",
"test:quiet": "node ./scripts/test-quiet.js",
"test:coverage": "pnpm run build:test && node ./scripts/coverage.js",
"test:storybook": "vitest run --project=storybook",
"watch:test": "mkdir -p target/tests && concurrently \"clojure -M:dev:shadow-cljs watch test\" \"nodemon -C -d 2 -w target/tests --exec 'node target/tests/test.js'\"",
"test:e2e": "playwright test --project default",
@ -67,7 +68,11 @@
"@vitest/browser": "5.0.3",
"@vitest/browser-playwright": "5.0.3",
"@vitest/coverage-v8": "5.0.3",
"@vitest/istanbul-lib-coverage": "1.0.2",
"@vitest/istanbul-lib-report": "1.0.2",
"@zip.js/zip.js": "2.8.34",
"acorn": "8.19.0",
"ast-v8-to-istanbul": "1.0.7",
"autoprefixer": "^10.6.1",
"compression": "^1.8.2",
"concurrently": "^10.0.5",

View File

@ -252,9 +252,21 @@ importers:
'@vitest/coverage-v8':
specifier: 5.0.3
version: 5.0.3(@vitest/browser@5.0.3)(vitest@5.0.3)
'@vitest/istanbul-lib-coverage':
specifier: 1.0.2
version: 1.0.2
'@vitest/istanbul-lib-report':
specifier: 1.0.2
version: 1.0.2
'@zip.js/zip.js':
specifier: 2.8.34
version: 2.8.34(patch_hash=7b556bbd426f152eb086f0126a53900e369a95cf64357c380b7c8d8e940c3d95)
acorn:
specifier: 8.19.0
version: 8.19.0
ast-v8-to-istanbul:
specifier: 1.0.7
version: 1.0.7
autoprefixer:
specifier: ^10.6.1
version: 10.6.1(postcss@8.5.29)

View File

@ -0,0 +1,139 @@
// Runs the compiled unit test bundle with V8 coverage enabled and writes
// Istanbul reports mapped back to the ClojureScript sources through the
// source maps of the shadow-cljs test build.
//
// Expects the test bundle to be built (`pnpm run build:test`). Extra
// arguments are passed to the test runner. Exits with the status of the
// tests. Reports land in `coverage/`, with `unloaded.txt` listing the
// namespaces in scope that no test loads (`scripts/coverage-summary.mjs`
// renders both as markdown).
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { parse } from "acorn";
import { convert } from "ast-v8-to-istanbul";
import { createCoverageMap } from "@vitest/istanbul-lib-coverage";
import { create, createContext } from "@vitest/istanbul-lib-report";
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const REPO_ROOT = path.resolve(ROOT, "..");
const TEST_BUNDLE = path.join(ROOT, "target/tests/test.js");
const RUNTIME_DIR = path.join(ROOT, "target/tests/cljs-runtime");
const RAW_DIR = path.join(ROOT, "target/coverage-raw");
const REPORT_DIR = path.join(ROOT, "coverage");
const SOURCE_DIR = path.join(ROOT, "src");
// UI components are left out: coverage measures the logic namespaces.
const EXCLUDED_DIRS = ["app/main/ui/"];
function inScope(relPath) {
return (
/^app\/.*\.clj[sc]$/.test(relPath) &&
!EXCLUDED_DIRS.some((dir) => relPath.startsWith(dir))
);
}
function runTests(args) {
fs.rmSync(RAW_DIR, { recursive: true, force: true });
const result = spawnSync("node", [TEST_BUNDLE, ...args], {
stdio: "inherit",
env: { ...process.env, NODE_V8_COVERAGE: RAW_DIR },
});
return result.status ?? 1;
}
// Resolves the sources of a source map to absolute paths under `src/`;
// returns null when the map has no source in scope.
function resolveSources(sourceMap) {
let anyInScope = false;
const sources = sourceMap.sources.map((source) => {
const relPath = path.join(sourceMap.sourceRoot ?? "", source);
if (inScope(relPath) && fs.existsSync(path.join(SOURCE_DIR, relPath))) {
anyInScope = true;
return path.join(SOURCE_DIR, relPath);
}
return source;
});
return anyInScope ? { ...sourceMap, sources, sourceRoot: "" } : null;
}
async function collectCoverage() {
const coverageMap = createCoverageMap({});
const runtimeUrl = pathToFileURL(RUNTIME_DIR).href + "/";
for (const rawFile of fs.readdirSync(RAW_DIR)) {
const raw = JSON.parse(
fs.readFileSync(path.join(RAW_DIR, rawFile), "utf8"),
);
for (const script of raw.result) {
if (!script.url.startsWith(runtimeUrl)) continue;
const file = fileURLToPath(script.url);
const mapFile = file + ".map";
if (!fs.existsSync(mapFile)) continue;
const sourceMap = resolveSources(
JSON.parse(fs.readFileSync(mapFile, "utf8")),
);
if (!sourceMap) continue;
const code = fs.readFileSync(file, "utf8");
const data = await convert({
ast: parse(code, { ecmaVersion: "latest", sourceType: "module" }),
code,
wrapperLength: 0,
coverage: { url: script.url, functions: script.functions },
sourceMap,
});
for (const [sourcePath, fileCoverage] of Object.entries(data)) {
if (inScope(path.relative(SOURCE_DIR, sourcePath))) {
coverageMap.merge({ [sourcePath]: fileCoverage });
}
}
}
}
return coverageMap;
}
function listScopeFiles(dir = path.join(SOURCE_DIR, "app")) {
return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) return listScopeFiles(fullPath);
return inScope(path.relative(SOURCE_DIR, fullPath)) ? [fullPath] : [];
});
}
function writeReports(coverageMap) {
fs.rmSync(REPORT_DIR, { recursive: true, force: true });
const context = createContext({
dir: REPORT_DIR,
coverageMap,
defaultSummarizer: "nested",
});
create("lcovonly", { projectRoot: REPO_ROOT }).execute(context);
create("json-summary").execute(context);
create("html", { subdir: "html" }).execute(context);
create("text-summary").execute(context);
}
const testStatus = runTests(process.argv.slice(2));
const coverageMap = await collectCoverage();
const loaded = new Set(coverageMap.files());
const unloaded = listScopeFiles()
.filter((file) => !loaded.has(file))
.sort();
writeReports(coverageMap);
fs.writeFileSync(
path.join(REPORT_DIR, "unloaded.txt"),
unloaded.map((file) => path.relative(REPO_ROOT, file) + "\n").join(""),
);
fs.rmSync(RAW_DIR, { recursive: true, force: true });
process.exit(testStatus);

View File

@ -9,7 +9,7 @@ pushd ../render-wasm;
popd
pnpm run lint:scss;
pnpm run test;
pnpm run test:coverage;
# Mousetrap JS unit tests (node:test, zero deps)
pushd packages/mousetrap;

View File

@ -17,9 +17,25 @@ pushd "$_SCRIPT_DIR" > /dev/null
node --test scripts/cargo-build.test.cjs || exit $?
setup
cargo test --bin render_wasm -- --show-output
RESULT=$?
# With RENDER_WASM_COVERAGE set, the same tests run under cargo-llvm-cov
# (must be on PATH) and leave lcov, a JSON summary and HTML in
# target/coverage/.
if [ -n "${RENDER_WASM_COVERAGE:-}" ]; then
rustup component add llvm-tools-preview --toolchain "$RUST_VERSION"
_COV_ARGS=(--target "$CARGO_BUILD_TARGET")
cargo llvm-cov clean --workspace
# Keep going on failure so the reports still get written.
RESULT=0
cargo llvm-cov --no-report "${_COV_ARGS[@]}" --bin render_wasm -- --show-output || RESULT=$?
mkdir -p target/coverage
cargo llvm-cov report "${_COV_ARGS[@]}" --lcov --output-path target/coverage/lcov.info
cargo llvm-cov report "${_COV_ARGS[@]}" --json --summary-only --output-path target/coverage/summary.json
cargo llvm-cov report "${_COV_ARGS[@]}" --html --output-dir target/coverage
else
cargo test --bin render_wasm -- --show-output
RESULT=$?
fi
popd > /dev/null
exit $RESULT

View File

@ -0,0 +1,373 @@
#!/usr/bin/env node
// Keeps one pull request comment with the coverage report of every project
// (backend, common, frontend, render-wasm). Each test job runs it with its own
// lcov file.
// The comment stores the data of every project in a hidden block, so a run
// updates its own project and renders the whole report again.
//
// Usage:
// node scripts/coverage-comment.mjs --project <name> --lcov <file>
// [--prefix <dir/>] [--json-summary <file>] [--emma-xml <file>]
// [--llvm-json <file>]
// [--unloaded <file>]
//
// --prefix prepended to lcov paths relative to the project directory
// --json-summary Istanbul json-summary whose totals replace the lcov ones
// (it adds statements)
// --emma-xml cloverage EMMA report; its form counts become statements
// --llvm-json `cargo llvm-cov --json --summary-only` output whose totals
// replace the lcov ones (regions become statements)
// --unloaded repo-relative sources no test loads, one per line
//
// Env: GITHUB_TOKEN, GITHUB_REPOSITORY, PR_NUMBER and HEAD_SHA; also
// GITHUB_API_URL, GITHUB_SERVER_URL, GITHUB_RUN_ID and GITHUB_RUN_NUMBER,
// which Actions sets.
import fs from "node:fs";
import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import { parseArgs } from "node:util";
import { addTotals, emptyTotals, parseLcov } from "./coverage-lcov.mjs";
const PROJECTS = ["Backend", "Common", "Frontend", "Render WASM"];
const METRICS = [
["lines", "Lines"],
["statements", "Statements"],
["functions", "Functions"],
["branches", "Branches"],
];
// Minimum percentage per project and metric; a metric without one gets the
// blue status.
const THRESHOLDS = {
Backend: { lines: 70 },
Common: { lines: 70 },
Frontend: { lines: 50 },
"Render WASM": { lines: 40 },
};
const MAX_CHANGED_FILES = 100;
const MAX_ATTEMPTS = 3;
// Time to let a concurrent update of another project land before checking
// that this project's data survived it.
const SETTLE_MS = Number(process.env.COVERAGE_COMMENT_SETTLE_MS ?? 10000);
const MARKER = "<!-- coverage-report -->";
const DATA_RE = /<!-- coverage-data ([A-Za-z0-9+/=]+) -->/;
const { values: opts } = parseArgs({
options: {
project: { type: "string" },
lcov: { type: "string" },
prefix: { type: "string", default: "" },
"json-summary": { type: "string" },
"emma-xml": { type: "string" },
"llvm-json": { type: "string" },
unloaded: { type: "string" },
},
});
const env = process.env;
for (const [name, value] of Object.entries({
"--project": PROJECTS.includes(opts.project),
"--lcov": opts.lcov,
GITHUB_TOKEN: env.GITHUB_TOKEN,
GITHUB_REPOSITORY: env.GITHUB_REPOSITORY,
PR_NUMBER: env.PR_NUMBER,
HEAD_SHA: env.HEAD_SHA,
})) {
if (!value) {
console.error(
`Missing or invalid ${name}. Usage: coverage-comment.mjs ` +
`--project <${PROJECTS.join("|")}> --lcov <file> [--prefix <dir/>] ` +
"[--json-summary <file>] [--emma-xml <file>] [--llvm-json <file>] " +
"[--unloaded <file>]",
);
process.exit(2);
}
}
const API = `${env.GITHUB_API_URL ?? "https://api.github.com"}/repos/${env.GITHUB_REPOSITORY}`;
const SERVER = env.GITHUB_SERVER_URL ?? "https://github.com";
class NotFound extends Error {}
async function request(method, url, body) {
const response = await fetch(url, {
method,
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${env.GITHUB_TOKEN}`,
"X-GitHub-Api-Version": "2022-11-28",
...(body ? { "Content-Type": "application/json" } : {}),
},
body: body ? JSON.stringify(body) : undefined,
});
if (response.status === 404) throw new NotFound(`${method} ${url}: 404`);
if (!response.ok) {
throw new Error(
`${method} ${url}: ${response.status} ${await response.text()}`,
);
}
return response.status === 204 ? null : response.json();
}
async function getAll(path) {
const items = [];
for (let page = 1; ; page++) {
const batch = await request(
"GET",
`${API}${path}?per_page=100&page=${page}`,
);
items.push(...batch);
if (batch.length < 100) return items;
}
}
// --- Coverage of this project -------------------------------------------
function readList(file) {
return file && fs.existsSync(file)
? fs.readFileSync(file, "utf8").split("\n").filter(Boolean)
: [];
}
function totalsOf(records) {
const totals = emptyTotals();
records.forEach((record) => addTotals(totals, record));
const result = {};
for (const [key] of METRICS) {
if (totals[key]?.total > 0) result[key] = totals[key];
}
if (opts["json-summary"]) {
const summary = JSON.parse(fs.readFileSync(opts["json-summary"], "utf8"));
for (const [key] of METRICS) {
const { covered, total } = summary.total[key] ?? {};
if (total > 0) result[key] = { covered, total };
}
}
if (opts["llvm-json"]) {
Object.assign(
result,
llvmTotals(JSON.parse(fs.readFileSync(opts["llvm-json"], "utf8"))),
);
}
if (opts["emma-xml"]) {
const forms = emmaForms(fs.readFileSync(opts["emma-xml"], "utf8"));
if (forms?.total > 0) result.statements = forms;
}
return result;
}
// Form counts of the whole report, from the `block, %` entry of the `total`
// element: `<coverage type="block, %" value="67% (1234/5678)"/>`.
function emmaForms(xml) {
const match = xml.match(
/<all name="total">[\s\S]*?<coverage type="block, %" value="[^"(]*\((\d+)\/(\d+)\)"/,
);
return match ? { covered: Number(match[1]), total: Number(match[2]) } : null;
}
// Totals of an llvm-cov JSON export; regions stand for statements.
function llvmTotals(json) {
const totals = json.data?.[0]?.totals ?? {};
const result = {};
for (const [key, source] of [
["lines", "lines"],
["statements", "regions"],
["functions", "functions"],
["branches", "branches"],
]) {
const { count, covered } = totals[source] ?? {};
if (count > 0) result[key] = { covered, total: count };
}
return result;
}
// llvm-cov writes absolute paths; the other tools write paths relative to the
// repository or to the project directory.
function repoPath(lcovPath) {
if (path.isAbsolute(lcovPath)) return path.relative(process.cwd(), lcovPath);
return lcovPath.startsWith(opts.prefix) ? lcovPath : opts.prefix + lcovPath;
}
async function projectData() {
const records = parseLcov(fs.readFileSync(opts.lcov, "utf8"));
const byPath = new Map(records.map((r) => [repoPath(r.path), r]));
const unloaded = new Set(readList(opts.unloaded));
const prFiles = await getAll(`/pulls/${env.PR_NUMBER}/files`);
const changed = prFiles
.filter((f) => f.status !== "removed")
.map((f) => f.filename)
.filter((file) => byPath.has(file) || unloaded.has(file))
.sort()
.map((file) =>
byPath.has(file)
? { file, lines: byPath.get(file).lines }
: { file, unloaded: true },
);
return {
sha: env.HEAD_SHA,
runId: Number(env.GITHUB_RUN_ID ?? 0),
runNumber: env.GITHUB_RUN_NUMBER ?? env.GITHUB_RUN_ID ?? "",
totals: totalsOf(records),
changed,
};
}
// --- Rendering ------------------------------------------------------------
function pct({ covered, total }) {
return `${Math.round((10000 * covered) / total) / 100}%`;
}
function statusOf(threshold, cov) {
if (threshold === undefined) return "🔵";
return (100 * cov.covered) / cov.total >= threshold ? "🟢" : "🔴";
}
function metricRow(threshold, label, cov) {
const target = threshold === undefined ? "" : ` (🎯 ${threshold}%)`;
return (
`<tr> <td align="center">${statusOf(threshold, cov)}</td> <td align="left">${label}</td> ` +
`<td align="right">${pct(cov)}${target}</td> ` +
`<td align="right">${cov.covered} / ${cov.total}</td> </tr>`
);
}
function fileCoverage(changed) {
if (changed.length === 0) return "No changed files found.";
const shown = changed.slice(0, MAX_CHANGED_FILES);
const rows = shown.map(({ file, lines, unloaded }) =>
unloaded
? `<tr> <td align="left"><code>${file}</code></td> <td align="right" colspan="2">no test loads it</td> </tr>`
: `<tr> <td align="left"><code>${file}</code></td> <td align="right">${lines.total ? pct(lines) : "-"}</td> <td align="right">${lines.covered} / ${lines.total}</td> </tr>`,
);
const more =
changed.length > shown.length
? `<p>${changed.length - shown.length} more files not shown.</p>`
: "";
return (
`<table> <thead> <tr> <th align="left">File</th> <th align="right">Lines</th> ` +
`<th align="right">Covered / Total</th> </tr> </thead> <tbody> ${rows.join(" ")} </tbody> </table>${more}`
);
}
function renderProject(name, data) {
if (!data) {
return `<h3>${name}</h3>\n<p><em>Its tests have not run on this pull request.</em></p>`;
}
const rows = METRICS.filter(([key]) => data.totals[key]).map(([key, label]) =>
metricRow(THRESHOLDS[name]?.[key], label, data.totals[key]),
);
const runUrl = `${SERVER}/${env.GITHUB_REPOSITORY}/actions/runs/${data.runId}`;
const commitUrl = `${SERVER}/${env.GITHUB_REPOSITORY}/commit/${data.sha}`;
return [
`<h3>${name}</h3>`,
`<table> <thead> <tr> <th align="center">Status</th> <th align="left">Category</th> ` +
`<th align="right">Percentage</th> <th align="right">Covered / Total</th> </tr> </thead> ` +
`<tbody> ${rows.join(" ")} </tbody> </table>`,
`<details><summary>File Coverage</summary>${fileCoverage(data.changed)}</details>`,
`<em>Generated in workflow <a href="${runUrl}">#${data.runNumber}</a> ` +
`for commit <a href="${commitUrl}">${data.sha.slice(0, 7)}</a></em>`,
].join("\n");
}
function render(all) {
const encoded = Buffer.from(JSON.stringify(all)).toString("base64");
return [
MARKER,
"<h2>Coverage Report</h2>",
...PROJECTS.map((name) => renderProject(name, all[name])),
"<sub>Backend and Common are measured by cloverage, which counts lines " +
"and Clojure forms (shown as statements) but not functions or branches. " +
"Render WASM is measured by cargo-llvm-cov on the native test build; " +
"its statements are llvm regions.</sub>",
`<!-- coverage-data ${encoded} -->`,
"",
].join("\n\n");
}
// --- Comment update -------------------------------------------------------
function dataOf(comment) {
const match = comment.body?.match(DATA_RE);
if (!match) return {};
try {
return JSON.parse(Buffer.from(match[1], "base64").toString("utf8"));
} catch {
return {};
}
}
async function reportComments() {
const comments = await getAll(`/issues/${env.PR_NUMBER}/comments`);
return comments
.filter((c) => c.user?.type === "Bot" && c.body?.startsWith(MARKER))
.sort((a, b) => a.id - b.id);
}
// Merges the data of every report comment, keeping the latest run of each
// project.
function mergeData(comments) {
const merged = {};
for (const comment of comments) {
for (const [name, data] of Object.entries(dataOf(comment))) {
if (PROJECTS.includes(name) && (merged[name]?.runId ?? -1) < data.runId) {
merged[name] = data;
}
}
}
return merged;
}
// Writes the report on the oldest report comment and drops the others, which
// appear when two jobs create the comment at the same time.
async function write(comments, all) {
const body = render(all);
if (comments.length === 0) {
await request("POST", `${API}/issues/${env.PR_NUMBER}/comments`, { body });
return;
}
const [primary, ...extra] = comments;
await request("PATCH", `${API}/issues/comments/${primary.id}`, { body });
for (const comment of extra) {
// Another job may have dropped it already.
await request("DELETE", `${API}/issues/comments/${comment.id}`).catch(
(error) => {
if (!(error instanceof NotFound)) throw error;
},
);
}
}
const mine = await projectData();
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
const comments = await reportComments();
const all = mergeData(comments);
if ((all[opts.project]?.runId ?? -1) > mine.runId) {
console.log(`A newer run already reported ${opts.project}.`);
break;
}
all[opts.project] = mine;
try {
await write(comments, all);
} catch (error) {
// The comment vanished under a concurrent update; the check below retries.
if (!(error instanceof NotFound)) throw error;
}
await sleep(SETTLE_MS * (1 + Math.random() / 2));
const after = await reportComments();
const saved = mergeData(after)[opts.project];
if (after.length === 1 && (saved?.runId ?? -1) >= mine.runId) {
console.log(`Coverage report updated for ${opts.project}.`);
break;
}
if (attempt === MAX_ATTEMPTS) {
throw new Error(`Could not keep the ${opts.project} coverage report.`);
}
console.log(`Concurrent update detected; writing ${opts.project} again.`);
}

49
scripts/coverage-lcov.mjs Normal file
View File

@ -0,0 +1,49 @@
// lcov parsing shared by the coverage scripts.
const COUNTERS = {
LF: "lines",
LH: "lines",
FNF: "functions",
FNH: "functions",
BRF: "branches",
BRH: "branches",
};
export function emptyTotals() {
return {
files: 0,
lines: { covered: 0, total: 0 },
functions: { covered: 0, total: 0 },
branches: { covered: 0, total: 0 },
};
}
// One record per source file: `{path, lines, functions, branches}`, each
// metric as `{covered, total}`.
export function parseLcov(text) {
const records = [];
let current = null;
for (const line of text.split("\n")) {
const sep = line.indexOf(":");
const tag = sep === -1 ? line.trim() : line.slice(0, sep);
const value = line.slice(sep + 1).trim();
if (tag === "SF") {
current = { path: value, ...emptyTotals() };
} else if (tag === "end_of_record" && current) {
records.push(current);
current = null;
} else if (current && tag in COUNTERS) {
const key = tag.endsWith("F") ? "total" : "covered";
current[COUNTERS[tag]][key] = Number(value);
}
}
return records;
}
export function addTotals(totals, record) {
totals.files += 1;
for (const key of ["lines", "functions", "branches"]) {
totals[key].covered += record[key].covered;
totals[key].total += record[key].total;
}
}

View File

@ -0,0 +1,99 @@
#!/usr/bin/env node
// Prints a markdown coverage summary of an lcov file, grouped by source area.
//
// Usage:
// node scripts/coverage-summary.mjs --title <title> --lcov <file>
// [--note <text>] [--unloaded <file>]
//
// `--unloaded` names a file listing, one per line, the sources no test loaded;
// they are shown apart and never counted. Meant to be appended to
// `$GITHUB_STEP_SUMMARY`.
import fs from "node:fs";
import { parseArgs } from "node:util";
import { addTotals, emptyTotals, parseLcov } from "./coverage-lcov.mjs";
const { values: opts } = parseArgs({
options: {
title: { type: "string" },
lcov: { type: "string" },
note: { type: "string" },
unloaded: { type: "string" },
},
});
if (!opts.title || !opts.lcov) {
console.error(
"Usage: coverage-summary.mjs --title <title> --lcov <file> [--note <text>] [--unloaded <file>]",
);
process.exit(2);
}
// Path from `app/` on: `app/main/<dir>` and `app/common/<dir>` group one
// level deeper than the rest, which group by `app/<dir>`.
function areaOf(filePath) {
const parts = filePath.replace(/^.*?\bsrc\//, "").split("/");
const depth =
["main", "common"].includes(parts[1]) && parts.length > 3 ? 3 : 2;
return parts.slice(0, Math.min(depth, parts.length - 1)).join("/") || "src";
}
function percent({ covered, total }) {
return total === 0 ? "-" : `${((100 * covered) / total).toFixed(1)}%`;
}
function row(name, t, metrics) {
const cells = metrics.map((key) => percent(t[key]));
return `| ${name} | ${t.files} | ${cells.join(" | ")} |`;
}
const records = parseLcov(fs.readFileSync(opts.lcov, "utf8"));
const areas = new Map();
const total = emptyTotals();
for (const record of records) {
const area = areaOf(record.path);
if (!areas.has(area)) areas.set(area, emptyTotals());
addTotals(areas.get(area), record);
addTotals(total, record);
}
// Some tools (cloverage) report lines only; metrics with no data are left out.
const metrics = ["lines", "functions", "branches"].filter(
(key) => total[key].total > 0,
);
const headers = {
lines: "Lines",
functions: "Functions",
branches: "Branches",
};
const unloaded =
opts.unloaded && fs.existsSync(opts.unloaded)
? fs.readFileSync(opts.unloaded, "utf8").split("\n").filter(Boolean)
: [];
const out = [`## ${opts.title}`, ""];
if (opts.note) out.push(opts.note, "");
out.push(
`**${percent(total.lines)}** of lines covered (${total.lines.covered} of ${total.lines.total}).`,
"",
`| Area | Files | ${metrics.map((key) => headers[key]).join(" | ")} |`,
`| --- | ---: | ${metrics.map(() => "---:").join(" | ")} |`,
...[...areas.keys()]
.sort()
.map((area) => row(area, areas.get(area), metrics)),
row("**Total**", total, metrics),
"",
);
if (unloaded.length > 0) {
out.push(
`<details><summary>${unloaded.length} sources no test loads (not counted above)</summary>`,
"",
...unloaded.map((file) => `- \`${file}\``),
"",
"</details>",
"",
);
}
console.log(out.join("\n"));