diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index e1883ae4d4..67ea6d4994 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -39,7 +39,7 @@ "^(?:backend|frontend|exporter|common)/deps.edn" "^(?:backend|frontend|exporter|common)/scripts/" "^(?:backend|frontend|exporter|common)/dev/" - "^(?:backend|frontend|exporter|common)/test/(?!benches/|frontend_tests/benches/)"] + "^(?:backend|frontend|exporter|common)/test/(?!performance_tests/)"] :linter-name true} diff --git a/frontend/deps.edn b/frontend/deps.edn index 1aaab63117..299e74018d 100644 --- a/frontend/deps.edn +++ b/frontend/deps.edn @@ -1,4 +1,4 @@ -{:paths ["src" "vendor" "resources" "test"] +{:paths ["src" "vendor" "resources" "test" "performance"] :deps {;; Carries `app.common.render-wasm.*`, shared with the headless exporter. penpot/common @@ -37,8 +37,7 @@ instaparse/instaparse {:mvn/version "1.5.0"} garden/garden {:git/url "https://github.com/noprompt/garden" - :git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"} - } + :git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"}} :aliases {:outdated @@ -70,5 +69,4 @@ :renderer-bench {:main-opts ["-m" "shadow.cljs.devtools.cli"] - :jvm-opts ["-Dpenpot.wasm.profile-marks=false"]} - }} + :jvm-opts ["-Dpenpot.wasm.profile-marks=false"]}}} diff --git a/frontend/package.json b/frontend/package.json index 6e7ba80f58..bb3cbe09c6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,12 +14,6 @@ "url": "https://github.com/penpot/penpot" }, "scripts": { - "benchmark:renderer": "node scripts/renderer-benchmarks.js", - "test:renderer-benchmarks": "node --test scripts/renderer-benchmarks/tests/*.test.js", - "verify:renderer-benchmarks": "node scripts/renderer-benchmarks/verify.js", - "verify:renderer-benchmark-failures": "node scripts/renderer-benchmarks/verify-failures.js", - "verify:renderer-benchmark-cli": "node scripts/renderer-benchmarks/verify-cli.js", - "verify:renderer-playground": "node scripts/renderer-benchmarks/verify-pages.js", "build:app:assets": "node ./scripts/build-app-assets.js", "build:fonts-preview": "node ./scripts/build-fonts-preview.js", "build:storybook": "(cd packages/ui && pnpm run build) && pnpm run build:storybook:assets && pnpm run build:storybook:cljs && storybook build", @@ -32,19 +26,21 @@ "build:app:main": "clojure -M:dev:shadow-cljs release main worker", "build:app:worker": "clojure -M:dev:shadow-cljs release worker", "build:app": "pnpm run clear:shadow-cache && pnpm run build:app:main && pnpm run build:app:libs", - "check-fmt:clj": "cljfmt check --parallel=true src/ test/", + "check-fmt:clj": "cljfmt check --parallel=true src/ performance/ test/", "check-fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js", "check-fmt:scss": "prettier -c resources/styles -c src/**/*.scss", - "fmt:clj": "cljfmt fix --parallel=true src/ test/", + "fmt:clj": "cljfmt fix --parallel=true src/ performance/ test/", "fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js -w", "fmt:scss": "prettier -c resources/styles -c src/**/*.scss -w", - "lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/ test/benches/ test/frontend_tests/benches/", + "lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/ performance/ test/performance_tests/render_wasm/", "lint:js": "exit 0", - "lint:renderer-benchmarks": "clj-kondo --config-dir ../.clj-kondo --lint test/benches/ test/frontend_tests/benches/", + "lint:performance": "clj-kondo --config-dir ../.clj-kondo --lint performance/ test/performance_tests/render_wasm/", "lint:scss": "pnpm exec stylelint '{src,resources}/**/*.scss'", "build:test": "pnpm run build:wasm && clojure -M:dev:shadow-cljs compile test", + "build:performance-tests": "clojure -M:dev:shadow-cljs compile performance-tests", "test": "pnpm run build:test && node target/tests/test.js", "test:quiet": "node ./scripts/test-quiet.js", + "test:performance-tests": "pnpm run build:performance-tests && node target/performance-tests/test.js", "test:storybook": "vitest run --project=storybook", "watch:test": "mkdir -p target/tests && concurrently \"clojure -M:dev:shadow-cljs watch test\" \"nodemon -C -d 2 -w target/tests --exec 'node target/tests/test.js'\"", "test:e2e": "playwright test --project default", diff --git a/frontend/test/benches/render_wasm/browser/bridge.cljs b/frontend/performance/benches/render_wasm/browser/bridge.cljs similarity index 100% rename from frontend/test/benches/render_wasm/browser/bridge.cljs rename to frontend/performance/benches/render_wasm/browser/bridge.cljs diff --git a/frontend/test/benches/render_wasm/browser/pilot.html b/frontend/performance/benches/render_wasm/browser/pilot.html similarity index 100% rename from frontend/test/benches/render_wasm/browser/pilot.html rename to frontend/performance/benches/render_wasm/browser/pilot.html diff --git a/frontend/test/benches/render_wasm/builder.clj b/frontend/performance/benches/render_wasm/builder.clj similarity index 100% rename from frontend/test/benches/render_wasm/builder.clj rename to frontend/performance/benches/render_wasm/builder.clj diff --git a/frontend/test/benches/render_wasm/builder.cljs b/frontend/performance/benches/render_wasm/builder.cljs similarity index 100% rename from frontend/test/benches/render_wasm/builder.cljs rename to frontend/performance/benches/render_wasm/builder.cljs diff --git a/frontend/test/benches/render_wasm/cases.cljs b/frontend/performance/benches/render_wasm/cases.cljs similarity index 100% rename from frontend/test/benches/render_wasm/cases.cljs rename to frontend/performance/benches/render_wasm/cases.cljs diff --git a/frontend/test/benches/render_wasm/codec.cljs b/frontend/performance/benches/render_wasm/codec.cljs similarity index 100% rename from frontend/test/benches/render_wasm/codec.cljs rename to frontend/performance/benches/render_wasm/codec.cljs diff --git a/frontend/test/benches/render_wasm/declarations.clj b/frontend/performance/benches/render_wasm/declarations.clj similarity index 100% rename from frontend/test/benches/render_wasm/declarations.clj rename to frontend/performance/benches/render_wasm/declarations.clj diff --git a/frontend/test/benches/render_wasm/declarations.cljs b/frontend/performance/benches/render_wasm/declarations.cljs similarity index 100% rename from frontend/test/benches/render_wasm/declarations.cljs rename to frontend/performance/benches/render_wasm/declarations.cljs diff --git a/frontend/test/benches/render_wasm/failures.cljs b/frontend/performance/benches/render_wasm/failures.cljs similarity index 100% rename from frontend/test/benches/render_wasm/failures.cljs rename to frontend/performance/benches/render_wasm/failures.cljs diff --git a/frontend/test/benches/render_wasm/fingerprint.cljs b/frontend/performance/benches/render_wasm/fingerprint.cljs similarity index 100% rename from frontend/test/benches/render_wasm/fingerprint.cljs rename to frontend/performance/benches/render_wasm/fingerprint.cljs diff --git a/frontend/test/benches/render_wasm/measurement.cljs b/frontend/performance/benches/render_wasm/measurement.cljs similarity index 100% rename from frontend/test/benches/render_wasm/measurement.cljs rename to frontend/performance/benches/render_wasm/measurement.cljs diff --git a/frontend/test/benches/render_wasm/node/pilot.cljs b/frontend/performance/benches/render_wasm/node/pilot.cljs similarity index 97% rename from frontend/test/benches/render_wasm/node/pilot.cljs rename to frontend/performance/benches/render_wasm/node/pilot.cljs index 476af4a6cf..d882a26851 100644 --- a/frontend/test/benches/render_wasm/node/pilot.cljs +++ b/frontend/performance/benches/render_wasm/node/pilot.cljs @@ -32,7 +32,7 @@ Follow README.md's prepared renderer release commands first. clojure -M:dev:renderer-bench release bench-render-wasm-browser pnpm run build:renderer-benchmarks:pilot - node target/renderer-benchmarks/pilot.cjs --seed 42 --screenshot /tmp/pilot-rects.png + node target/performance/render-wasm/pilot.cjs --seed 42 --screenshot /tmp/pilot-rects.png The pilot serves the release browser bundle, so compile `bench-render-wasm-browser` first (after the prepared renderer build @@ -62,10 +62,10 @@ (defn- frontend-root "Resolves the frontend checkout root from the compiled pilot location - (`target/renderer-benchmarks/pilot.cjs`, next to `runner.cjs`). Ticket 10 + (`target/performance/render-wasm/pilot.cjs`, next to `runner.cjs`). Ticket 10 owns command cwd resolution; the pilot pins its own roots instead." [] - (path/resolve js/__dirname ".." "..")) + (path/resolve js/__dirname ".." ".." "..")) (def ^:private mime {".html" "text/html" @@ -209,9 +209,9 @@ (trace "pilot: parsing args") (let [opts (parse-args (or argv [])) root (frontend-root) - bundle (path/join root "target" "renderer-benchmarks" "browser") + bundle (path/join root "target" "performance" "render-wasm" "browser") js-dir (path/join root "resources" "public" "js") - html (path/join root "test" "benches" "render_wasm" "browser" "pilot.html") + html (path/join root "performance" "bench" "render_wasm" "browser" "pilot.html") seed (:seed opts) selected (some #(when (= (:id %) (:case-id opts)) %) (cases/collect-cases {:master-seed seed})) diff --git a/frontend/test/benches/render_wasm/node/runner.cljs b/frontend/performance/benches/render_wasm/node/runner.cljs similarity index 84% rename from frontend/test/benches/render_wasm/node/runner.cljs rename to frontend/performance/benches/render_wasm/node/runner.cljs index 0721154a33..3ab15e716a 100644 --- a/frontend/test/benches/render_wasm/node/runner.cljs +++ b/frontend/performance/benches/render_wasm/node/runner.cljs @@ -30,7 +30,7 @@ (defn- usage [summary] - (str "Usage: node target/renderer-benchmarks/runner.cjs [options]\n\n" + (str "Usage: node target/performance/render-wasm/runner.cjs [options]\n\n" "Options:\n" summary "\n\n" "No command is implemented yet. Upcoming tickets add:\n" @@ -57,9 +57,9 @@ (seq errors) (fail! (str/join "\n" errors)) - ;; TODO(mem:render-wasm/performance/cljs-rewrite/10-process-build-and-server, - ;; mem:render-wasm/performance/cljs-rewrite/11-runner-and-failure-accounting, - ;; mem:render-wasm/performance/cljs-rewrite/14-custom-measured-operations): + ;; TODO(mem:render-wasm/performance_tests/cljs-rewrite/10-process-build-and-server, + ;; mem:render-wasm/performance_tests/cljs-rewrite/11-runner-and-failure-accounting, + ;; mem:render-wasm/performance_tests/cljs-rewrite/14-custom-measured-operations): ;; dispatch `run`, `ab` and `compare` instead of failing. (seq arguments) (fail! (str "Unknown command: " (str/join " " arguments) "\nTry --help")) diff --git a/frontend/test/benches/render_wasm/random.cljs b/frontend/performance/benches/render_wasm/random.cljs similarity index 100% rename from frontend/test/benches/render_wasm/random.cljs rename to frontend/performance/benches/render_wasm/random.cljs diff --git a/frontend/test/benches/render_wasm/report/analysis.cljs b/frontend/performance/benches/render_wasm/report/analysis.cljs similarity index 100% rename from frontend/test/benches/render_wasm/report/analysis.cljs rename to frontend/performance/benches/render_wasm/report/analysis.cljs diff --git a/frontend/test/benches/render_wasm/report/compare.cljs b/frontend/performance/benches/render_wasm/report/compare.cljs similarity index 100% rename from frontend/test/benches/render_wasm/report/compare.cljs rename to frontend/performance/benches/render_wasm/report/compare.cljs diff --git a/frontend/test/benches/render_wasm/report/format.cljs b/frontend/performance/benches/render_wasm/report/format.cljs similarity index 100% rename from frontend/test/benches/render_wasm/report/format.cljs rename to frontend/performance/benches/render_wasm/report/format.cljs diff --git a/frontend/test/benches/render_wasm/report/summarize.cljs b/frontend/performance/benches/render_wasm/report/summarize.cljs similarity index 100% rename from frontend/test/benches/render_wasm/report/summarize.cljs rename to frontend/performance/benches/render_wasm/report/summarize.cljs diff --git a/frontend/test/benches/render_wasm/result.cljs b/frontend/performance/benches/render_wasm/result.cljs similarity index 100% rename from frontend/test/benches/render_wasm/result.cljs rename to frontend/performance/benches/render_wasm/result.cljs diff --git a/frontend/test/benches/render_wasm/runtime/camera.cljs b/frontend/performance/benches/render_wasm/runtime/camera.cljs similarity index 100% rename from frontend/test/benches/render_wasm/runtime/camera.cljs rename to frontend/performance/benches/render_wasm/runtime/camera.cljs diff --git a/frontend/test/benches/render_wasm/runtime/protocol.cljs b/frontend/performance/benches/render_wasm/runtime/protocol.cljs similarity index 100% rename from frontend/test/benches/render_wasm/runtime/protocol.cljs rename to frontend/performance/benches/render_wasm/runtime/protocol.cljs diff --git a/frontend/test/benches/render_wasm/scenes/masks.cljs b/frontend/performance/benches/render_wasm/scenes/masks.cljs similarity index 100% rename from frontend/test/benches/render_wasm/scenes/masks.cljs rename to frontend/performance/benches/render_wasm/scenes/masks.cljs diff --git a/frontend/test/benches/render_wasm/scenes/paths.cljs b/frontend/performance/benches/render_wasm/scenes/paths.cljs similarity index 100% rename from frontend/test/benches/render_wasm/scenes/paths.cljs rename to frontend/performance/benches/render_wasm/scenes/paths.cljs diff --git a/frontend/test/benches/render_wasm/scenes/plus.cljs b/frontend/performance/benches/render_wasm/scenes/plus.cljs similarity index 100% rename from frontend/test/benches/render_wasm/scenes/plus.cljs rename to frontend/performance/benches/render_wasm/scenes/plus.cljs diff --git a/frontend/test/benches/render_wasm/scenes/rects.cljs b/frontend/performance/benches/render_wasm/scenes/rects.cljs similarity index 100% rename from frontend/test/benches/render_wasm/scenes/rects.cljs rename to frontend/performance/benches/render_wasm/scenes/rects.cljs diff --git a/frontend/test/benches/render_wasm/snapshot.cljs b/frontend/performance/benches/render_wasm/snapshot.cljs similarity index 100% rename from frontend/test/benches/render_wasm/snapshot.cljs rename to frontend/performance/benches/render_wasm/snapshot.cljs diff --git a/frontend/resources/wasm-playground/BENCHMARKS.md b/frontend/resources/wasm-playground/BENCHMARKS.md deleted file mode 100644 index 464de2db5f..0000000000 --- a/frontend/resources/wasm-playground/BENCHMARKS.md +++ /dev/null @@ -1,206 +0,0 @@ -# Renderer benchmarks - -Run from `frontend/` in the prepared Penpot development container, as the normal -`penpot` user. Dependencies, the matching Playwright Chromium installation, and -the static server must already exist. Start the server separately with -`node scripts/e2e-server.js`. The runner never installs packages or browsers, -starts services, or cleans Cargo targets. - -Use the Cargo cache prepared for that user. In this container, noninteractive -shells inherit `CARGO_HOME=/opt/cargo`, while the populated cache is -`/home/penpot/.cargo`. For those shells, prefix the command with -`CARGO_HOME=/home/penpot/.cargo`. The runner preserves the supplied environment; -it does not move caches or download missing crates. - -```sh -pnpm run benchmark:renderer run --output /tmp/baseline.json -pnpm run benchmark:renderer run --filter texts --seed 42 --output /tmp/texts.json -pnpm run benchmark:renderer run --headed --filter rects/default/zoom -pnpm run benchmark:renderer run --no-build --filter rects/default/pan -pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json -pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json --diagnostic -pnpm run test:renderer-benchmarks -``` - -Each run builds the ordinary optimized frontend renderer once using -`PENPOT_WASM_PREPARED=1`, offline Cargo, and the existing build/copy path. Setup is -skipped, package-manager network fallback is disabled, and the emitted module is -named `render-wasm-benchmark`. Playground assets are refreshed before launch. A -per-run marker checks that the server points at this checkout. Concurrent builds -or benchmark commands against the same checkout are unsupported. `run --no-build` -skips the build, the Cargo manifest read, and the Emscripten/esbuild preflight -checks, then measures the existing `render-wasm-benchmark.js`/`.wasm` pair; it -still refreshes playground assets and checks the marker. It rejects `--features` -and never applies to `ab`, and records `metadata.build.built: false` with a -warning that the artifact was not rebuilt from this revision. - -The default server URL is `http://localhost:3000`; override it with `--base-url`. -Default JSON output goes to a unique file under the system temporary directory. -Use `--warmups`, `--repetitions`, and `--timeout-ms` to override the provisional -defaults of 3, 10, and 30000. Counts are fixed, not adaptive; failures are never -silently replaced. A failed case stops its remaining attempts and records how -many were unattempted, then later independent cases can continue. A lost browser -ends the run and retains earlier results. Exit status reports execution validity, -not a performance regression. - -## Workloads and state - -Seven versioned scenes use a seeded generator: 1000 rectangles, stars, lines, or -shadowed rectangles; 100 text shapes; and fixed mask/clip scenes. Scene generation -and buffer construction happen before upload timing. Text uses the renderer's -embedded fallback font, not production font fetching. The JavaScript fixtures -share their implementation with the interactive playground pages; `?seed=42` is -repeatable, and numeric shape-count overrides remain available on those pages. - -All cases run serially in one Chromium process with a 1920x1080 CSS viewport, -DPR 2, and product WebGL2 context options. Effective sizes, context attributes, -browser/platform identity, seeds, and workload parameters appear in JSON. -Software WebGL is allowed and warns; a GPU launch flag does not prove hardware -acceleration. Fresh pages do not clear browser-process, compilation, or driver -caches. - -- **Fresh module/context:** each attempt gets a new page/context and module. - Module initialization starts at the Emscripten factory and includes WASM - fetching/instantiation; the module's JavaScript import precedes timing. -- **Context initialization:** canvas/context registration, native initialization, - render/browser options, and viewport sizing. -- **Upload:** direct FFI upload, pending text layouts, background, initial view, - and initial tile-index setup. This is not the frontend's serialization or - batched asynchronous upload path. -- **First render:** request to `ViewportReady` and final `Full`, including rAF - scheduling. An immediate `Full` supplies both boundaries. -- **Warm pan/zoom:** separate case-owned instances; restore the starting view and - finish preparation rendering before each attempt. Caches evolve within each - case. Restoring the view is not a cache reset. - -Interactions use declared frame paths, cached rendering, a declared 100 ms -settling interval, and product finalization/continuation flags. The interval -models a settling case, not every possible editor gesture. Finalization time -starts before `_set_view_end`, and that call's duration also appears separately. -Last-input-to-`Full` additionally includes the delay. Raw render/cached calls, -timestamps, flags, frame types, and zero-duration slices remain available. - -## Interpretation - -These are renderer submission timings. They do not measure GPU completion, -displayed pixels, input delivery, whole-editor responsiveness, network behavior, -or visual correctness. WASM buffer size is context, not attributed CPU/GPU memory. - -Reports retain means, medians, ranges, sample deviation, and unscaled median -absolute deviation. Nominal 95% bootstrap intervals require 10 valid attempts; -p95 requires 100 and p99 requires 500. Intervals are exploratory and conditional -on these runs: serial correlation, cache history, system load, and thermal drift -remain. Confirm important findings with repeated complete runs. No p-values or -automated regression thresholds are provided. - -Offline comparison checks scenario/version, seeds, resolved workloads, -preparation, environment, and metric semantics. Source/build changes are expected. -`--diagnostic` lists incompatibilities and permits inspection without claiming -equivalence. Dirty worktrees warn and continue, recording only SHA and dirty -state, not diffs or artifact hashes. - -## Separate fixture validation - -```sh -pnpm run verify:renderer-benchmarks -pnpm run verify:renderer-benchmarks --screenshots /tmp/renderer-diagnostics -pnpm run verify:renderer-benchmark-failures -pnpm run verify:renderer-benchmark-cli -pnpm run verify:renderer-playground --filter rects -``` - -This unscored smoke uses the already-built benchmark artifact and Playwright -request routing; it starts no server. It checks every workload and interaction -through `Full`. Optional screenshots support independent manual fixture checks. -They never enter scored runs, and smoke durations are not benchmark evidence. - -The failure checks inject context loss, cancellation, stalled work, and crashes. -The CLI checks use the existing server and prepared build path, saving each -failure's JSON and log under a printed temporary directory. The playground check -loads the actual legacy pages and exercises pointer drag and wheel zoom. Run -these commands separately from scored runs. - -The runner uses the manifest's default Cargo features and records their local -feature expansion. It rejects diagnostic defaults and disables the optional -function-name profiling switch for scored builds. - -## Feature A/B comparisons - -`run --features a,b` builds and runs one configuration with explicit Cargo -features and records it. `ab --features a,b` builds and runs every on/off -configuration of the requested features (2^N for N features, default cap 3, -`--max-features` 1..6), then compares the Boolean-lattice edges: pairs that -differ in exactly one feature. Build order follows a binary-reflected Gray -code so consecutive builds differ by one feature bit; this only minimizes -rebuild wall time. Build durations are provenance -(`metadata.build.durationMs`), never metrics, but non-adjacent edges are -measured further apart in wall time, so repeated whole runs confirm findings -that matter. - -```sh -pnpm run benchmark:renderer run --features branch-b --filter rects/default/pan --output /tmp/renderer-one.json -pnpm run benchmark:renderer ab --features branch-b --filter rects/default/pan --output /tmp/renderer-ab-branch-b -pnpm run benchmark:renderer ab --features branch-b --dry-run -``` - -`ab` requires empty manifest defaults and aborts when a requested feature -overlaps the enabled defaults or names a `stats*`/`profile*` diagnostic; -`run --features` records defaults and enforces only the overlap abort. -`--dry-run` prints the plan and builds nothing. Configurations beyond 8, or an -empty `--filter`, require `--confirm`. Each configuration launches its own -browser; the printed plan states this cost. `--output` for `ab` is a directory -root that must not exist yet; `configs/.json`, -`comparisons/__.json`, and `matrix.json` land beneath it. - -Run JSON is `schemaVersion` 2: `metadata.build.built` is `false` when the run -reused an existing artifact, and the build-derived fields below are absent, so -comparing it with a built run fails compatibility. `metadata.build.features` is -the explicit sorted set (the A/B identity axis), `defaultFeatures` the resolved -manifest defaults, plus the runner-controlled `env` map and the allowlisted -`ambientEnv` (`RUSTFLAGS`, `RUSTC_WRAPPER`, `CARGO_TARGET_DIR`, `CARGO_HOME`, -`CC`, `CFLAGS`, `CXXFLAGS`, `LDFLAGS`, `PENPOT_WASM_FUNCTION_NAMES`, -`CARGO_PROFILE_*`, `EMSDK*`). Comparison of the six build fields is always -diagnostic; the expected `features` mismatch on every edge is tagged, not -suppressed. An edge is marked `controlled: true` when the intended toggle is -the only mismatch: the comparison measures exactly what the matrix was built -for, and the diagnostic banner reads as single-run evidence, not as a -verdict. Edges with other mismatches (environment, seeds, manifest defaults -shifting mid-matrix) keep the strong diagnostic-only reading and print a -terminal warning. `branch-a` has no observable effect because the scenes never -call `set_modifiers`. Exit 0 means every edge has a comparison, never a -performance verdict. - -## Current validation limits - -The September 17 software-rendered pilot took 25 minutes 43 seconds excluding -the build at 3 warm-ups and 10 measured attempts. Eighteen cases completed all -attempts; all three shadow cases failed. A later one-attempt, zero-warm-up run -took 2 minutes 36 seconds; shadow pan and zoom still lost their WebGL contexts. -These runs used dirty source trees and are workflow evidence, not a comparison -of renderer changes. Shadow visual validation and default sampling remain open. -Keep the full workloads and provisional defaults until these limits are resolved. - -Isolated September 17 probes reproduce the shadow pattern without changing -workloads or defaults. `verify.js --filter shadows/default/load` passes through -`Full` on SwiftShader; `--filter shadows/default/pan` and -`--filter shadows/default/zoom` fail with `WebGL context lost` in the warm -interaction loop that calls `_render_from_cache` per frame. Load success records -renderer submission only, not displayed pixels. The fixture keeps 1000 shapes -with two drop shadows and one inner shadow, seven FFI arguments per shadow, -style 0 for drop and 1 for inner, and a boolean hidden flag, matching -`render-wasm/src/wasm/shadows.rs`. No fixture defect, workload reduction, or -renderer change is claimed. - -Sampling stays provisional at 3 warm-ups, 10 measured attempts, and 30s timeout. -The full pilot ran 240 attempts in 1543337 ms, about 6.4 s per attempt; the -one-attempt workflow run took 156466 ms for 21 attempts, about 7.4 s per -attempt. The 2-5 minute goal allows roughly 18-46 attempts in total, or one to -two per case across 21 cases, below the 10 valid attempts needed for median -intervals. Use zero-warm-up single attempts with `--filter` for workflow -checks; use the full provisional counts for intervals. Final calibration needs -a stable graphics environment with passing shadows. - -The implementation roadmap, pilot evidence, and remaining calibration work live -in Serena memory `render-wasm/performance/core` and its numbered tickets. The -2-5 minute full-suite goal excludes build time and is a target, not a guarantee -on software-rendered systems. diff --git a/frontend/resources/wasm-playground/benchmark.html b/frontend/resources/wasm-playground/benchmark.html deleted file mode 100644 index f68f56ebe0..0000000000 --- a/frontend/resources/wasm-playground/benchmark.html +++ /dev/null @@ -1,31 +0,0 @@ - - - - - Renderer benchmarks - - - - - - - - diff --git a/frontend/resources/wasm-playground/benchmarks/browser.js b/frontend/resources/wasm-playground/benchmarks/browser.js deleted file mode 100644 index 6f5634948d..0000000000 --- a/frontend/resources/wasm-playground/benchmarks/browser.js +++ /dev/null @@ -1,246 +0,0 @@ -import { scenarios } from "../scenes/index.js"; -import { drain, interact, restore } from "./protocol.js"; -import { WEBGL_OPTIONS, validateMetrics } from "./contract.js"; -import { summarizeRun, formatSummary } from "./statistics.js"; - -let session; -const canvas = document.querySelector("canvas"); - -function check(owner = session) { - if (!owner || owner.canceled) throw new Error("Benchmark canceled"); - if (owner !== session) throw new Error("Stale benchmark operation"); - if (owner.lost || owner.gl?.isContextLost()) - throw new Error("WebGL context lost"); - if (performance.now() > owner.deadline) throw new Error("Operation timeout"); -} - -function runtime() { - const owner = session; - return { - module: owner.module, - now: () => performance.now(), - recordSlice: (slice) => owner.slices.push(slice), - recordCached: (slice) => owner.cachedSlices.push(slice), - check: () => { - if (owner !== session) throw new Error("Stale benchmark operation"); - check(); - }, - frame: () => - new Promise((resolve, reject) => { - owner.cancelPending = reject; - owner.raf = requestAnimationFrame((timestamp) => { - owner.cancelPending = null; - owner.raf = null; - resolve(timestamp); - }); - }), - sleep: (ms) => - new Promise((resolve, reject) => { - owner.cancelPending = reject; - owner.timer = setTimeout(() => { - owner.cancelPending = null; - resolve(); - }, ms); - }), - }; -} - -function webglInfo(gl) { - const extension = gl.getExtension("WEBGL_debug_renderer_info"); - const vendor = gl.getParameter(extension?.UNMASKED_VENDOR_WEBGL ?? gl.VENDOR); - const renderer = gl.getParameter( - extension?.UNMASKED_RENDERER_WEBGL ?? gl.RENDERER, - ); - return { - vendor, - renderer, - version: gl.getParameter(gl.VERSION), - unmasked: Boolean(extension), - acceleration: /swiftshader|llvmpipe|software|softpipe/i.test( - `${vendor} ${renderer}`, - ) - ? "software" - : "unknown", - options: gl.getContextAttributes(), - drawingBuffer: { - width: gl.drawingBufferWidth, - height: gl.drawingBufferHeight, - }, - }; -} - -async function initialize(benchmarkCase, options) { - const scenario = scenarios.find( - (item) => item.id === benchmarkCase.scenarioId, - ); - if (!scenario) - throw new Error(`Unknown scenario: ${benchmarkCase.scenarioId}`); - // Construction and module script import deliberately precede initialization timing. - const scene = scenario.createScene( - benchmarkCase.params, - benchmarkCase.sceneSeed, - ); - const owner = { - slices: [], - cachedSlices: [], - canceled: false, - lost: false, - deadline: performance.now() + options.timeoutMs, - options, - benchmarkCase, - }; - session = owner; - const { default: createModule } = await import(options.moduleUrl); - check(owner); - const moduleStart = performance.now(); - owner.module = await createModule(); - const moduleInitMs = performance.now() - moduleStart; - check(owner); - const contextStart = performance.now(); - canvas.style.width = `${options.viewport.width}px`; - canvas.style.height = `${options.viewport.height}px`; - canvas.width = Math.floor(options.viewport.width * options.dpr); - canvas.height = Math.floor(options.viewport.height * options.dpr); - const gl = canvas.getContext("webgl2", WEBGL_OPTIONS); - if (!gl) throw new Error("WebGL2 is unavailable"); - owner.gl = gl; - // Native initialization queries the unmasked renderer identity. - gl.getExtension("WEBGL_debug_renderer_info"); - owner.onLost = (event) => { - event.preventDefault(); - owner.lost = true; - }; - canvas.addEventListener("webglcontextlost", owner.onLost); - owner.handle = owner.module.GL.registerContext(gl, { majorVersion: 2 }); - owner.module.GL.makeContextCurrent(owner.handle); - owner.module._init(options.viewport.width, options.viewport.height); - owner.initialized = true; - owner.module._set_render_options(0, options.dpr); - owner.module._set_browser(1); // Rust Browser::Chrome - owner.module._resize_viewbox(options.viewport.width, options.viewport.height); - const contextInitMs = performance.now() - contextStart; - // Diagnostic queries are outside renderer and upload timing. - const environment = { - webgl: webglInfo(gl), - canvas: { - cssWidth: canvas.clientWidth, - cssHeight: canvas.clientHeight, - width: canvas.width, - height: canvas.height, - }, - devicePixelRatio: globalThis.devicePixelRatio, - }; - owner.environment = environment; - const uploadStart = performance.now(); - scenario.upload(owner.module, scene); - owner.module._set_canvas_background(0xfffabada); - const view = benchmarkCase.initialView; - owner.module._set_view(view.scale, view.x, view.y); - owner.module._set_view_end(); // Bulk upload needs its initial tile index. - const uploadMs = performance.now() - uploadStart; - const first = await drain(runtime(), { flags: 0 }); - return { - environment, - metrics: { - moduleInitMs, - contextInitMs, - uploadMs, - timeToViewportReadyMs: first.viewportReadyMs, - timeToFullMs: first.fullMs, - }, - slices: first.slices, - cachedSlices: [], - }; -} - -function finish(result) { - const invalidMetrics = validateMetrics(result.metrics); - const slices = result.slices ?? []; - const cachedSlices = result.cachedSlices ?? []; - const timestamps = cachedSlices.map((slice) => slice.timestamp); - return { - ...result, - status: Object.keys(invalidMetrics).length ? "invalid" : "ok", - invalidMetrics, - frameCounts: { - Partial: slices.filter((s) => s.frameType === 1).length, - ViewportReady: slices.filter((s) => s.frameType === 3).length, - Full: slices.filter((s) => s.frameType === 2).length, - }, - rafGapsMs: timestamps.slice(1).map((time, i) => time - timestamps[i]), - renderRafGapsMs: slices - .slice(1) - .map((slice, i) => slice.timestamp - slices[i].timestamp), - zeroSlices: [...slices, ...cachedSlices].filter((s) => s.durationMs === 0) - .length, - wasmMemoryBytes: session.module.HEAPU8.buffer.byteLength, - }; -} - -async function runAttempt(benchmarkCase, options) { - try { - let result; - if (!session) { - result = await initialize(benchmarkCase, options); - if (benchmarkCase.group === "fresh-module-context") return finish(result); - } - session.deadline = performance.now() + options.timeoutMs; - await restore(runtime(), benchmarkCase.initialView); - session.slices = []; - session.cachedSlices = []; - session.deadline = performance.now() + options.timeoutMs; - result = await interact(runtime(), benchmarkCase.interaction); - return finish({ - ...result, - environment: { ...session.environment, webgl: webglInfo(session.gl) }, - }); - } catch (error) { - return { - status: "failed", - metrics: {}, - invalidMetrics: {}, - slices: session?.slices ?? [], - cachedSlices: session?.cachedSlices ?? [], - environment: session?.environment, - error: String(error?.stack ?? error), - }; - } -} - -function dispose() { - if (!session) return; - const owner = session; - owner.canceled = true; - owner.cancelPending?.(new Error("Benchmark canceled")); - cancelAnimationFrame(owner.raf); - clearTimeout(owner.timer); - canvas.removeEventListener("webglcontextlost", owner.onLost); - try { - if (owner.initialized && !owner.lost) { - owner.module._free_gpu_resources?.(); - owner.module._clean_up(); - } - } finally { - if (owner.handle !== undefined) owner.module.GL.deleteContext(owner.handle); - owner.gl?.getExtension("WEBGL_lose_context")?.loseContext(); - session = null; - } -} - -globalThis.rendererBenchmark = { - runAttempt, - dispose, - showResult(result) { - globalThis.rendererBenchmarkResult = result; - const report = document.querySelector("pre"); - report.textContent = [ - result.scope, - ...(result.warnings ?? []), - formatSummary(result.summary ?? summarizeRun(result)), - "Run settings:", - JSON.stringify(result.metadata, null, 2), - ].join("\n\n"); - report.hidden = false; - }, -}; -globalThis.rendererBenchmarkReady = true; diff --git a/frontend/resources/wasm-playground/benchmarks/contract.js b/frontend/resources/wasm-playground/benchmarks/contract.js deleted file mode 100644 index 98a8f27331..0000000000 --- a/frontend/resources/wasm-playground/benchmarks/contract.js +++ /dev/null @@ -1,124 +0,0 @@ -export const WEBGL_OPTIONS = Object.freeze({ - antialias: false, - depth: true, - stencil: true, - alpha: true, - preserveDrawingBuffer: true, -}); - -export function deriveSeed(masterSeed, caseId) { - let seed = 2166136261; - for (const character of `${masterSeed}\0${caseId}`) { - seed = Math.imul(seed ^ character.charCodeAt(0), 16777619) >>> 0; - } - return seed; -} - -function serializable(value, path = "case") { - if (value === null || typeof value === "string" || typeof value === "boolean") - return; - if (typeof value === "number" && Number.isFinite(value)) return; - if (Array.isArray(value)) { - value.forEach((v, i) => serializable(v, `${path}[${i}]`)); - return; - } - if (value && Object.getPrototypeOf(value) === Object.prototype) { - for (const [key, v] of Object.entries(value)) - serializable(v, `${path}.${key}`); - return; - } - throw new Error(`Non-serializable value: ${path}`); -} - -function viewValid(view) { - return ( - view && - Number.isFinite(view.scale) && - view.scale > 0 && - Number.isFinite(view.x) && - Number.isFinite(view.y) - ); -} - -export function collectCases(scenarios, masterSeed, filter = "") { - const cases = []; - const seen = new Set(); - const scenarioIds = new Set(); - for (const scenario of scenarios) { - if ( - !scenario.id || - scenarioIds.has(scenario.id) || - !Number.isInteger(scenario.version) || - typeof scenario.createScene !== "function" || - typeof scenario.upload !== "function" || - !Array.isArray(scenario.cases) || - !scenario.cases.length - ) - throw new Error("Invalid or duplicate scenario"); - scenarioIds.add(scenario.id); - for (const c of scenario.cases) { - serializable(c); - const baseId = `${scenario.id}/${c.id}`; - if (!c.id || seen.has(baseId)) - throw new Error(`Missing or duplicate case ID: ${baseId}`); - seen.add(baseId); - if (!c.params || !viewValid(c.initialView)) - throw new Error(`Invalid parameters/view: ${baseId}`); - for (const kind of ["pan", "zoom"]) { - const interaction = c.interactions?.[kind]; - if ( - !interaction || - !Array.isArray(interaction.frames) || - !interaction.frames.length || - !interaction.frames.every(viewValid) || - !Number.isFinite(interaction.settleMs) || - interaction.settleMs < 0 - ) - throw new Error(`Incomplete ${kind} interaction: ${baseId}`); - } - // Validate static workloads before any build or browser measurement. Browser - // construction still owns its descriptors and remains outside timed spans. - scenario.createScene(c.params, deriveSeed(masterSeed, baseId)); - for (const kind of ["load", "pan", "zoom"]) { - const id = `${baseId}/${kind}`; - if (filter && !id.includes(filter)) continue; - cases.push({ - id, - scenarioId: scenario.id, - scenarioVersion: scenario.version, - description: scenario.description, - sceneSeed: deriveSeed(masterSeed, baseId), - params: c.params, - initialView: c.initialView, - interaction: kind === "load" ? null : c.interactions[kind], - group: kind === "load" ? "fresh-module-context" : `warm-${kind}`, - preparation: - kind === "load" - ? "new-context-module; upload; set-view-end; full" - : "case-owned-context; upload; full; restore-view-and-full-before-each-attempt; evolving-caches", - status: "unattempted", - attempts: [], - }); - } - } - } - if (!cases.length) throw new Error(`No matching cases: ${filter}`); - return cases; -} - -export function validateMetrics(metrics) { - return Object.fromEntries( - Object.entries(metrics) - .filter( - ([key, value]) => - !Number.isFinite(value) || - value < 0 || - (value === 0 && - !["setViewEndMs", "settlingRequestedMs"].includes(key)), - ) - .map(([key, value]) => [ - key, - Number.isFinite(value) ? "below-timer-resolution" : "non-finite", - ]), - ); -} diff --git a/frontend/resources/wasm-playground/benchmarks/protocol.js b/frontend/resources/wasm-playground/benchmarks/protocol.js deleted file mode 100644 index 2d3929525b..0000000000 --- a/frontend/resources/wasm-playground/benchmarks/protocol.js +++ /dev/null @@ -1,77 +0,0 @@ -// Browser-independent protocol; the runtime owns the clock and cancellation. -export async function drain( - rt, - { flags = 0, origin = rt.now(), immediate = false } = {}, -) { - const slices = []; - let viewportReadyMs; - do { - rt.check(); - const timestamp = immediate ? rt.now() : await rt.frame(); - immediate = false; - rt.check(); - const start = rt.now(); - const frameType = rt.module._render(timestamp, flags); - const end = rt.now(); - slices.push({ timestamp, flags, frameType, durationMs: end - start }); - rt.recordSlice?.(slices.at(-1)); - rt.check(); - if (![1, 2, 3].includes(frameType)) - throw new Error(`Unexpected frame type: ${frameType}`); - if (frameType === 3 || frameType === 2) viewportReadyMs ??= end - origin; - if (frameType === 2) - return { slices, viewportReadyMs, fullMs: end - origin }; - // The production scheduler passes the previous FrameType as flags. - flags = frameType; - } while (true); -} - -export async function restore(rt, view) { - rt.check(); - rt.module._set_view_start(); - rt.module._set_view(view.scale, view.x, view.y); - rt.module._set_view_end(); - return drain(rt, { flags: 4, immediate: true }); -} - -export async function interact(rt, interaction) { - rt.check(); - const start = rt.now(); - rt.module._set_view_start(); - const cachedSlices = []; - let lastInput; - for (const view of interaction.frames) { - const timestamp = await rt.frame(); - rt.check(); - lastInput = rt.now(); - rt.module._set_view(view.scale, view.x, view.y); - const cacheStart = rt.now(); - rt.module._render_from_cache(0); - cachedSlices.push({ timestamp, durationMs: rt.now() - cacheStart }); - rt.recordCached?.(cachedSlices.at(-1)); - } - const activeEnd = rt.now(); - await rt.sleep(interaction.settleMs); - rt.check(); - const finalizationStart = rt.now(); - rt.module._set_view_end(); - const setViewEndMs = rt.now() - finalizationStart; - const result = await drain(rt, { - flags: 4, - origin: finalizationStart, - immediate: true, - }); - return { - metrics: { - activeInteractionMs: activeEnd - start, - settlingRequestedMs: interaction.settleMs, - settlingActualMs: finalizationStart - activeEnd, - setViewEndMs, - timeToViewportReadyMs: result.viewportReadyMs, - timeToFullMs: result.fullMs, - lastInputToFullMs: rt.now() - lastInput, - }, - slices: result.slices, - cachedSlices, - }; -} diff --git a/frontend/resources/wasm-playground/benchmarks/statistics.js b/frontend/resources/wasm-playground/benchmarks/statistics.js deleted file mode 100644 index 286b775ca4..0000000000 --- a/frontend/resources/wasm-playground/benchmarks/statistics.js +++ /dev/null @@ -1,274 +0,0 @@ -// Shared by the benchmark page and offline Node reports. No browser work or I/O. -export const STATISTICAL_METHOD = Object.freeze({ - version: 1, - name: "percentile-bootstrap-of-medians", - confidence: 0.95, - resamples: 2000, - seed: 0x70656e70, - minimumIntervalSamples: 10, - minimumP95Samples: 100, - minimumP99Samples: 500, -}); - -export const STATISTICAL_WARNINGS = Object.freeze([ - "Intervals are exploratory, nominal 95%, and conditional on these observed runs; attempts can be correlated.", - "Independent runs can drift with system load and thermal state. Confirm important changes with repeated complete runs.", -]); - -export function statisticalMethod(options = {}) { - const method = { ...STATISTICAL_METHOD, ...options }; - if ( - !Number.isInteger(method.resamples) || - method.resamples < 100 || - method.resamples > 10000 - ) { - throw new Error("resamples must be an integer between 100 and 10000"); - } - if ( - !Number.isInteger(method.seed) || - method.seed < 0 || - method.seed > 0xffffffff - ) { - throw new Error("seed must be a uint32 integer"); - } - // Only the resampling count and seed are configurable; support thresholds and - // interval interpretation form part of the versioned statistical method. - return { - ...STATISTICAL_METHOD, - resamples: method.resamples, - seed: method.seed, - }; -} - -function randomGenerator(seed) { - let state = seed >>> 0; - return () => { - state = (state + 0x6d2b79f5) >>> 0; - let value = Math.imul(state ^ (state >>> 15), 1 | state); - value ^= value + Math.imul(value ^ (value >>> 7), 61 | value); - return ((value ^ (value >>> 14)) >>> 0) / 4294967296; - }; -} - -function quantile(sorted, probability) { - if (!sorted.length) return null; - const position = (sorted.length - 1) * probability; - const lower = Math.floor(position); - const upper = Math.ceil(position); - return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower); -} - -function sorted(values) { - return [...values].sort((a, b) => a - b); -} - -function interval(values) { - const ordered = sorted(values); - return { low: quantile(ordered, 0.025), high: quantile(ordered, 0.975) }; -} - -function resampleMedian(values, random) { - const sample = Array.from( - { length: values.length }, - () => values[Math.floor(random() * values.length)], - ); - return quantile(sorted(sample), 0.5); -} - -export function summarize(values, options = {}) { - const method = statisticalMethod(options); - const ordered = sorted( - values.filter((value) => Number.isFinite(value) && value >= 0), - ); - const n = ordered.length; - const median = quantile(ordered, 0.5); - const mean = n ? ordered.reduce((sum, value) => sum + value / n, 0) : null; - const summary = { - n, - invalid: values.length - n, - mean, - median, - min: n ? ordered[0] : null, - max: n ? ordered[n - 1] : null, - standardDeviation: - n > 1 - ? Math.sqrt( - ordered.reduce( - (sum, value) => sum + (value - mean) ** 2 / (n - 1), - 0, - ), - ) - : null, - mad: n - ? quantile(sorted(ordered.map((value) => Math.abs(value - median))), 0.5) - : null, - medianInterval: null, - p95: null, - p99: null, - suppressed: {}, - }; - if (n >= method.minimumIntervalSamples) { - const random = randomGenerator(method.seed); - summary.medianInterval = interval( - Array.from({ length: method.resamples }, () => - resampleMedian(ordered, random), - ), - ); - } else { - summary.suppressed.medianInterval = `Requires at least ${method.minimumIntervalSamples} valid attempts.`; - } - for (const [name, threshold, probability] of [ - ["p95", method.minimumP95Samples, 0.95], - ["p99", method.minimumP99Samples, 0.99], - ]) { - if (n >= threshold) summary[name] = quantile(ordered, probability); - else - summary.suppressed[name] = - `Requires at least ${threshold} valid attempts.`; - } - return summary; -} - -export function metricNames(benchmarkCase) { - return [ - ...new Set( - benchmarkCase.attempts.flatMap((attempt) => [ - ...Object.keys(attempt.metrics ?? {}), - ...Object.keys(attempt.invalidMetrics ?? {}), - ]), - ), - ].sort(); -} - -export function observations(benchmarkCase, metric) { - return benchmarkCase.attempts - .filter((attempt) => !attempt.warmup) - .map((attempt) => { - const invalid = attempt.invalidMetrics ?? {}; - if ( - attempt.status === "failed" || - (attempt.status === "invalid" && !Object.keys(invalid).length) || - Object.hasOwn(invalid, metric) - ) - return NaN; - return attempt.metrics?.[metric] ?? NaN; - }); -} - -export function summarizeRun(result, options = {}) { - return { - method: statisticalMethod(options), - warnings: [...STATISTICAL_WARNINGS], - cases: result.cases.map((benchmarkCase) => { - const measured = benchmarkCase.attempts.filter( - (attempt) => !attempt.warmup, - ); - return { - id: benchmarkCase.id, - status: benchmarkCase.status, - unattempted: benchmarkCase.unattempted ?? 0, - attempts: { - measured: measured.length, - warmup: benchmarkCase.attempts.length - measured.length, - ok: measured.filter((attempt) => attempt.status === "ok").length, - invalid: measured.filter((attempt) => attempt.status === "invalid") - .length, - failed: measured.filter((attempt) => attempt.status === "failed") - .length, - }, - metrics: Object.fromEntries( - metricNames(benchmarkCase).map((metric) => [ - metric, - summarize(observations(benchmarkCase, metric), options), - ]), - ), - }; - }), - }; -} - -export function compareObservations( - baselineValues, - candidateValues, - options = {}, -) { - const method = statisticalMethod(options); - const baseline = summarize(baselineValues, options); - const candidate = summarize(candidateValues, options); - const result = { - baseline, - candidate, - absoluteChange: null, - relativeChangePercent: null, - absoluteInterval: null, - relativeIntervalPercent: null, - suppressed: {}, - }; - if (!baseline.n || !candidate.n) { - result.suppressed.comparison = "Both runs require valid observations."; - return result; - } - result.absoluteChange = candidate.median - baseline.median; - if (baseline.median === 0) - result.suppressed.relativeChangePercent = "Baseline median is zero."; - else - result.relativeChangePercent = - (100 * result.absoluteChange) / baseline.median; - if ( - baseline.n < method.minimumIntervalSamples || - candidate.n < method.minimumIntervalSamples - ) { - result.suppressed.intervals = `Each run requires at least ${method.minimumIntervalSamples} valid attempts.`; - return result; - } - const before = baselineValues.filter( - (value) => Number.isFinite(value) && value >= 0, - ); - const after = candidateValues.filter( - (value) => Number.isFinite(value) && value >= 0, - ); - const random = randomGenerator(method.seed); - const differences = []; - const relative = []; - for (let index = 0; index < method.resamples; index++) { - // Independent draws preserve the unpaired nature of separate runs. - const first = resampleMedian(before, random); - const second = resampleMedian(after, random); - differences.push(second - first); - if (first !== 0) relative.push((100 * (second - first)) / first); - } - result.absoluteInterval = interval(differences); - if (baseline.median !== 0 && relative.length === method.resamples) - result.relativeIntervalPercent = interval(relative); - else - result.suppressed.relativeIntervalPercent = - "Baseline median or a resampled baseline median is zero."; - return result; -} - -function display(value) { - return value === null ? "n/a" : value.toFixed(3); -} - -export function formatSummary(summary) { - const lines = [ - `Statistics: ${summary.method.name} v${summary.method.version}; ${summary.method.resamples} resamples, seed ${summary.method.seed}.`, - ...summary.warnings, - `Median interval: Requires at least ${summary.method.minimumIntervalSamples} valid attempts; p95 requires ${summary.method.minimumP95Samples}, p99 requires ${summary.method.minimumP99Samples}. Unsupported values are n/a.`, - ]; - for (const benchmarkCase of summary.cases) { - const counts = benchmarkCase.attempts; - lines.push( - `${benchmarkCase.id} (${benchmarkCase.status}): ${counts.warmup} warm-up, ${counts.ok} valid, ${counts.invalid} invalid, ${counts.failed} failed, ${benchmarkCase.unattempted} unattempted`, - ); - for (const [name, metric] of Object.entries(benchmarkCase.metrics)) { - lines.push( - ` ${name}: median ${display(metric.median)}, mean ${display(metric.mean)}, range ${display(metric.min)}–${display(metric.max)}, SD ${display(metric.standardDeviation)}, MAD ${display(metric.mad)}; ${metric.n} valid, ${metric.invalid} invalid`, - ); - lines.push( - ` median interval: ${metric.medianInterval ? `[${display(metric.medianInterval.low)}, ${display(metric.medianInterval.high)}]` : "n/a"}; p95: ${display(metric.p95)}; p99: ${display(metric.p99)}`, - ); - } - } - return lines.join("\n"); -} diff --git a/frontend/scripts/renderer-benchmarks.js b/frontend/scripts/renderer-benchmarks.js deleted file mode 100644 index a23f5aae19..0000000000 --- a/frontend/scripts/renderer-benchmarks.js +++ /dev/null @@ -1,965 +0,0 @@ -#!/usr/bin/env node -import fs from "node:fs/promises"; -import path from "node:path"; -import os from "node:os"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { randomUUID } from "node:crypto"; -import { spawn, execFileSync } from "node:child_process"; -import { parseArgs } from "node:util"; -import { scenarios } from "../resources/wasm-playground/scenes/index.js"; -import { - collectCases, - WEBGL_OPTIONS, -} from "../resources/wasm-playground/benchmarks/contract.js"; -import { - summarizeRun, - formatSummary, -} from "../resources/wasm-playground/benchmarks/statistics.js"; -import { - compareRuns, - formatComparison, - tagExpectedFeatureDiff, -} from "./renderer-benchmarks/compare.js"; - -const frontend = fileURLToPath(new URL("../", import.meta.url)); -const root = path.resolve(frontend, ".."); -const LIMITS = - "Renderer submission timings only: not GPU completion, displayed pixels, whole-editor responsiveness, or visual correctness."; - -// Runner-controlled build overrides shared by run and ab. VERSION stays out: -// every run stamps its own runId as VERSION in the child environment. -export const BUILD_ENV = { - PENPOT_WASM_PREPARED: "1", - PENPOT_WASM_FUNCTION_NAMES: "0", - BUILD_MODE: "release", - NODE_ENV: "production", - RENDER_TARGET: "frontend", - BUILD_NAME: "render-wasm-benchmark", - CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", - COREPACK_ENABLE_NETWORK: "0", - CARGO_NET_OFFLINE: "true", -}; - -export function readOptions(argv) { - const { values, positionals } = parseArgs({ - args: argv, - allowPositionals: true, - options: { - help: { type: "boolean" }, - headed: { type: "boolean", default: false }, - diagnostic: { type: "boolean", default: false }, - "base-url": { type: "string", default: "http://localhost:3000" }, - output: { type: "string" }, - filter: { type: "string", default: "" }, - seed: { type: "string", default: "42" }, - warmups: { type: "string", default: "3" }, - repetitions: { type: "string", default: "10" }, - "timeout-ms": { type: "string", default: "30000" }, - features: { type: "string", default: "" }, - "max-features": { type: "string", default: "3" }, - confirm: { type: "boolean", default: false }, - "dry-run": { type: "boolean", default: false }, - "no-build": { type: "boolean", default: false }, - }, - }); - const command = positionals[0] ?? "run"; - if (!["run", "compare", "ab"].includes(command)) - throw new Error(`Unknown command: ${command}`); - const number = (name, minimum, maximum) => { - const value = Number(values[name]); - if (!Number.isSafeInteger(value) || value < minimum || value > maximum) - throw new Error(`Invalid --${name}`); - return value; - }; - if ( - !values.help && - (command === "compare" ? positionals.length !== 3 : positionals.length > 1) - ) - throw new Error( - "Use run [options], ab --features a,b [options], or compare baseline.json candidate.json [options]", - ); - const featureList = values.features - .split(",") - .map((token) => token.trim()) - .filter(Boolean); - for (const token of featureList) { - if (!/^[a-z0-9-_]+$/.test(token)) - throw new Error(`Invalid --features token: ${token}`); - } - const features = [...new Set(featureList)].sort(); - const maxFeatures = number("max-features", 1, 6); - if (command !== "ab" && values["dry-run"]) - throw new Error("--dry-run is only valid for ab"); - if (values["no-build"] && command !== "run") - throw new Error("--no-build is only valid for run"); - if (values["no-build"] && features.length) - throw new Error("--no-build skips the build, so --features cannot apply"); - if (command === "ab" && !values.help) { - if (features.some((name) => /^(stats|profile)(-|$)/.test(name))) - throw new Error( - "ab rejects renderer diagnostic features; record them with run --features instead", - ); - if (features.length < 1) - throw new Error("ab requires --features with at least one feature"); - if (features.length > maxFeatures) - throw new Error( - `ab requests ${features.length} features but --max-features is ${maxFeatures}`, - ); - } - const baseUrl = new URL(values["base-url"]); - if ( - !["http:", "https:"].includes(baseUrl.protocol) || - baseUrl.username || - baseUrl.password - ) - throw new Error( - "--base-url must be an HTTP(S) server URL without credentials", - ); - return { - command, - files: positionals.slice(1), - help: values.help, - headed: values.headed, - diagnostic: values.diagnostic, - baseUrl: baseUrl.href, - output: values.output, - filter: values.filter, - seed: number("seed", 0, 0xffffffff), - warmups: number("warmups", 0, 1000), - repetitions: number("repetitions", 1, 10000), - timeoutMs: number("timeout-ms", 100, 600000), - features, - maxFeatures, - confirm: values.confirm, - dryRun: values["dry-run"], - noBuild: values["no-build"], - }; -} - -// All on/off configurations of sorted unique feature names in binary-reflected -// Gray order, plus the Boolean-lattice edges (pairs differing in exactly one -// feature). features[0] is the most significant bit. Pure: no I/O. -export function planFeatureMatrix(features) { - const configurations = []; - for (let index = 0; index < 2 ** features.length; index++) { - const gray = index ^ (index >> 1); - const slug = features - .map((_, bit) => (gray & (1 << (features.length - 1 - bit)) ? "1" : "0")) - .join(""); - configurations.push({ - slug, - features: features.filter((_, bit) => slug[bit] === "1"), - }); - } - const edges = []; - configurations.forEach((config) => { - features.forEach((name, bit) => { - if (config.slug[bit] === "0") { - edges.push({ - from: config.slug, - to: config.slug.slice(0, bit) + "1" + config.slug.slice(bit + 1), - toggledFeature: name, - }); - } - }); - }); - return { configurations, edges }; -} - -export async function bounded(operation, milliseconds, signal) { - let timer; - let onAbort; - try { - return await Promise.race([ - operation, - new Promise((_, reject) => { - timer = setTimeout( - () => - reject(new Error(`Node watchdog expired after ${milliseconds} ms`)), - milliseconds, - ); - }), - new Promise((_, reject) => { - onAbort = () => reject(signal.reason ?? new Error("Interrupted")); - if (signal?.aborted) onAbort(); - else signal?.addEventListener("abort", onAbort, { once: true }); - }), - ]); - } finally { - clearTimeout(timer); - signal?.removeEventListener("abort", onAbort); - } -} - -async function save(output, result) { - await fs.mkdir(path.dirname(output), { recursive: true }); - const temporary = `${output}.tmp`; - await fs.writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`); - await fs.rename(temporary, output); -} - -function execute(command, args, options = {}) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { stdio: "inherit", ...options }); - child.once("error", reject); - child.once("exit", (code, signal) => - code === 0 - ? resolve() - : reject(new Error(`${command} failed (${signal ?? code})`)), - ); - }); -} - -async function requireFile(filename, instruction) { - try { - await fs.access(filename); - } catch { - throw new Error(instruction); - } -} - -// Cargo metadata --no-deps reads the manifest without requiring unused targets' -// dependencies to be cached. Only local feature aliases need expansion here. -export function defaultFeatures(definitions) { - const enabled = new Set(); - const visit = (name) => { - if (enabled.has(name)) return; - enabled.add(name); - for (const child of definitions[name] ?? []) visit(child); - }; - visit("default"); - if ([...enabled].some((name) => /^(stats|profile)(-|$)/.test(name))) - throw new Error( - "Default Cargo features enable renderer diagnostics; use a plain release manifest", - ); - return [...enabled].sort(); -} - -// Reads the renderer package's feature definitions from the Cargo manifest -// without requiring unused targets' dependencies to be cached. -export function readRendererManifest() { - const manifest = JSON.parse( - execFileSync( - "cargo", - [ - "metadata", - "--offline", - "--locked", - "--no-deps", - "--format-version", - "1", - ], - { cwd: path.join(root, "render-wasm"), encoding: "utf8" }, - ), - ); - const renderer = manifest.packages.find( - (item) => item.manifest_path === path.join(root, "render-wasm/Cargo.toml"), - ); - if (!renderer) - throw new Error("Renderer package missing from Cargo metadata"); - return renderer.features; -} - -// Validates requested explicit features against the manifest definitions. -// Returns the resolved manifest defaults (synthetic "default" node included). -// Pure: no I/O. -export function validateFeatureSelection(definitions, features, command) { - const resolved = defaultFeatures(definitions); - const enabled = resolved.filter((name) => name !== "default"); - for (const name of features) { - if (!Object.hasOwn(definitions, name)) - throw new Error(`Unknown Cargo feature: ${name}`); - } - const overlap = features.filter((name) => enabled.includes(name)); - if (overlap.length) - throw new Error( - `Requested features already enabled by default: ${overlap.join(",")}`, - ); - if (command === "ab" && enabled.length) - throw new Error( - `ab requires an empty default feature set; found: ${enabled.join(",")}`, - ); - return resolved; -} - -const AMBIENT_ENV_EXACT = new Set([ - "RUSTFLAGS", - "RUSTC_WRAPPER", - "CARGO_TARGET_DIR", - "CARGO_HOME", - "CC", - "CFLAGS", - "CXXFLAGS", - "LDFLAGS", - "PENPOT_WASM_FUNCTION_NAMES", -]); - -// Records the ambient build environment allowlist. Never the full process -// environment. Pure: no I/O. -export function recordAmbientEnv(env = process.env) { - const recorded = {}; - for (const [key, value] of Object.entries(env)) { - if ( - AMBIENT_ENV_EXACT.has(key) || - key.startsWith("CARGO_PROFILE_") || - key.startsWith("EMSDK") - ) { - recorded[key] = String(value); - } - } - return recorded; -} - -async function preflight(options) { - if (options.noBuild) { - for (const artifact of [ - "render-wasm-benchmark.js", - "render-wasm-benchmark.wasm", - ]) - await requireFile( - path.join(frontend, "resources/public/js", artifact), - "No existing renderer benchmark build; run once without --no-build first.", - ); - } else { - await requireFile( - "/opt/emsdk/emsdk_env.sh", - "Run inside the prepared Penpot development container (Emscripten is missing).", - ); - await requireFile( - path.join(root, "render-wasm/node_modules/esbuild/package.json"), - "Prepare renderer dependencies with pnpm install in render-wasm first.", - ); - } - const { chromium } = await import("playwright"); - await requireFile( - chromium.executablePath(), - "Chromium is missing; run the frontend scripts/setup preparation command first.", - ); - try { - const response = await fetch(options.baseUrl, { - signal: AbortSignal.timeout(3000), - }); - // A static root may legitimately be absent before assets have been copied. - if (response.status >= 500) throw new Error(`HTTP ${response.status}`); - } catch (error) { - throw new Error( - `Static server unavailable at ${options.baseUrl}; run node scripts/e2e-server.js from frontend first. ${error.message}`, - ); - } - return chromium; -} - -export function recordAttempt(benchmarkCase, attempt, index, warmups) { - if (attempt.environment) { - benchmarkCase.environment ??= attempt.environment; - if ( - JSON.stringify(benchmarkCase.environment) !== - JSON.stringify(attempt.environment) - ) { - attempt.status = "failed"; - attempt.error = "Effective renderer environment changed within the case"; - } - } - benchmarkCase.attempts.push({ index, warmup: index < warmups, ...attempt }); - if (attempt.status !== "ok") benchmarkCase.status = "failed"; -} - -export function finishCases(cases, expected, status) { - for (const c of cases) { - c.unattempted = expected - c.attempts.length; - if (c.status === "running") c.status = status; - } -} - -export async function run(options) { - const abort = new AbortController(); - const interrupted = () => abort.abort(new Error("Benchmark interrupted")); - process.once("SIGINT", interrupted); - process.once("SIGTERM", interrupted); - const runId = randomUUID(); - const output = path.resolve( - options.output ?? - path.join(os.tmpdir(), `renderer-benchmarks-${runId}.json`), - ); - const browserArgs = ["--enable-gpu"]; - const configuration = { - viewport: { width: 1920, height: 1080 }, - dpr: 2, - webglOptions: WEBGL_OPTIONS, - rendererOptions: { debug: 0, browser: 1, backgroundArgb: 0xfffabada }, - warmups: options.warmups, - timeoutMs: options.timeoutMs, - }; - const git = (args) => - execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); - const result = { - schemaVersion: 2, - runId, - timestamp: new Date().toISOString(), - status: "running", - scope: LIMITS, - warnings: [], - metadata: { - git: { - sha: git(["rev-parse", "HEAD"]), - dirty: Boolean(git(["status", "--porcelain"])), - }, - build: { - mode: "release", - built: !options.noBuild, - featurePolicy: "manifest-defaults", - features: [], - target: "frontend", - prepared: true, - }, - environment: { - node: process.version, - os: os.platform(), - osRelease: os.release(), - arch: os.arch(), - cpu: os.cpus()[0]?.model ?? "unknown", - host: os.hostname(), - headless: !options.headed, - browserArgs, - }, - configuration, - baseUrl: options.baseUrl, - serverReachable: false, - masterSeed: options.seed, - repetitions: options.repetitions, - metricSemantics: - "renderer-boundaries-v1;module-factory-includes-wasm-fetch;upload-includes-initial-view-index;finalization-includes-set-view-end", - }, - cases: collectCases(scenarios, options.seed, options.filter), - }; - let server; - let browser; - let context; - let page; - let pageErrors = []; - let suiteStart; - const closeContext = async () => { - if (!context) return; - const previous = context; - const previousPage = page; - context = null; - page = null; - try { - if (previousPage) - await bounded( - previousPage.evaluate(() => globalThis.rendererBenchmark?.dispose()), - 2000, - ); - } catch { - /* A failed or blocked page still has to be closed. */ - } - await bounded(previous.close(), 5000); - }; - const openPage = async () => { - context = await browser.newContext({ - viewport: configuration.viewport, - deviceScaleFactor: configuration.dpr, - serviceWorkers: "block", - }); - page = await context.newPage(); - pageErrors = []; - page.on("pageerror", (error) => pageErrors.push(error.message)); - page.on("crash", () => pageErrors.push("Browser page crashed")); - await page.goto( - new URL(`wasm-playground/benchmark.html?run=${runId}`, options.baseUrl) - .href, - { timeout: options.timeoutMs }, - ); - await page.waitForFunction( - () => globalThis.rendererBenchmarkReady === true, - null, - { timeout: options.timeoutMs }, - ); - }; - try { - if (result.metadata.git.dirty) { - result.warnings.push( - "Dirty worktree: local development evidence, not a reproducible canonical record.", - ); - console.warn(result.warnings.at(-1)); - } - const chromium = await preflight(options); - result.metadata.serverReachable = true; - const packageJson = JSON.parse( - await fs.readFile( - new URL("../node_modules/playwright/package.json", import.meta.url), - "utf8", - ), - ); - result.metadata.environment.playwright = packageJson.version; - if (options.noBuild) { - const warning = - "Build skipped (--no-build): the measured artifact was not rebuilt from this revision."; - result.warnings.push(warning); - console.warn(warning); - } else { - const requested = [...(options.features ?? [])].sort(); - const resolved = validateFeatureSelection( - readRendererManifest(), - requested, - options.command, - ); - result.metadata.build.features = requested; - result.metadata.build.defaultFeatures = resolved; - if (requested.length) result.metadata.build.featurePolicy = "explicit"; - result.metadata.build.env = { ...BUILD_ENV }; - result.metadata.build.ambientEnv = recordAmbientEnv(); - // Disallow package-manager network fallback in a prepared run. - const buildStart = performance.now(); - await execute( - "./build", - [ - "frontend", - "--offline", - "--locked", - ...(requested.length ? ["--features", requested.join(",")] : []), - ], - { - cwd: path.join(root, "render-wasm"), - signal: abort.signal, - env: { ...process.env, ...BUILD_ENV, VERSION: runId }, - }, - ); - result.metadata.build.durationMs = performance.now() - buildStart; - } - const { copyWasmPlayground } = await import("./_helpers.js"); - const previousCwd = process.cwd(); - try { - process.chdir(frontend); - await copyWasmPlayground(); - } finally { - process.chdir(previousCwd); - } - const markerName = "benchmark-run.json"; - await fs.writeFile( - path.join(frontend, "resources/public/wasm-playground", markerName), - JSON.stringify({ runId }), - ); - const marker = await fetch( - new URL(`wasm-playground/${markerName}?run=${runId}`, options.baseUrl), - { signal: AbortSignal.timeout(3000), cache: "no-store" }, - ); - if (!marker.ok || (await marker.json()).runId !== runId) - throw new Error( - "The static server does not serve this workspace's current benchmark assets", - ); - suiteStart = performance.now(); - server = await chromium.launchServer({ - headless: !options.headed, - args: browserArgs, - }); - browser = await chromium.connect(server.wsEndpoint()); - result.metadata.environment.chromium = browser.version(); - const attemptOptions = { - ...configuration, - moduleUrl: new URL( - `js/render-wasm-benchmark.js?run=${runId}`, - options.baseUrl, - ).href, - }; - for (const benchmarkCase of result.cases) { - if (abort.signal.aborted) throw abort.signal.reason; - console.log(`Running ${benchmarkCase.id}`); - benchmarkCase.status = "running"; - for ( - let index = 0; - index < options.warmups + options.repetitions; - index++ - ) { - let attempt; - try { - if (!page) await bounded(openPage(), options.timeoutMs, abort.signal); - attempt = await bounded( - page.evaluate( - ([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o), - [benchmarkCase, attemptOptions], - ), - options.timeoutMs * 2 + 5000, - abort.signal, - ); - } catch (error) { - attempt = { - status: "failed", - metrics: {}, - invalidMetrics: {}, - error: String(error.stack ?? error), - slices: [], - cachedSlices: [], - }; - } - if (pageErrors.length) { - attempt.status = "failed"; - attempt.error = [attempt.error, ...pageErrors] - .filter(Boolean) - .join("\n"); - } - recordAttempt(benchmarkCase, attempt, index, options.warmups); - if (attempt.environment) { - if (attempt.environment.webgl.acceleration === "software") { - const warning = - "Software WebGL: compare only equivalent renderer environments."; - if (!result.warnings.includes(warning)) { - result.warnings.push(warning); - console.warn(warning); - } - } - } - if (attempt.status !== "ok") benchmarkCase.status = "failed"; - // Persist only outside browser timing; every failed attempt remains visible. - await save(output, result); - if (abort.signal.aborted) { - benchmarkCase.status = "interrupted"; - throw abort.signal.reason; - } - if (!browser.isConnected()) - throw new Error( - "Browser disconnected; remaining cases were not attempted", - ); - if (attempt.status === "failed") { - // Cache history cannot be restored after failure: stop this case, then - // continue independent cases. Do not silently replace failed samples. - benchmarkCase.unattempted = - options.warmups + options.repetitions - index - 1; - await closeContext(); - break; - } - if (benchmarkCase.group === "fresh-module-context") - await closeContext(); - } - await closeContext(); - if (benchmarkCase.status === "running") benchmarkCase.status = "complete"; - } - result.status = result.cases.every((c) => c.status === "complete") - ? "complete" - : "failed"; - } catch (error) { - result.status = abort.signal.aborted ? "interrupted" : "failed"; - result.error = String(error.stack ?? error); - console.error(result.error); - } finally { - finishCases( - result.cases, - options.warmups + options.repetitions, - result.status, - ); - if (suiteStart !== undefined) - result.metadata.suiteDurationMs = performance.now() - suiteStart; - result.summary = summarizeRun(result); - // Render a report once, after every scored case. Report work never separates - // attempts or cases. The same data also goes to JSON and the terminal. - if (browser?.isConnected() && !abort.signal.aborted) { - try { - if (!page) await bounded(openPage(), 5000); - await bounded( - page.evaluate( - (value) => globalThis.rendererBenchmark.showResult(value), - result, - ), - 5000, - ); - } catch (error) { - result.warnings.push(`Page report unavailable: ${error.message}`); - } - } - try { - await closeContext(); - } catch { - /* Browser cleanup below also closes blocked pages. */ - } - // The browser server exposes its child process for a bounded hard cleanup. - try { - if (browser) await bounded(browser.close(), 5000); - } catch { - server?.process()?.kill("SIGKILL"); - } - try { - if (server) await bounded(server.close(), 5000); - } catch { - server?.process()?.kill("SIGKILL"); - } - process.removeListener("SIGINT", interrupted); - process.removeListener("SIGTERM", interrupted); - result.finishedAt = new Date().toISOString(); - await save(output, result); - } - console.log(LIMITS); - console.log(formatSummary(result.summary)); - console.log(`Result: ${output}`); - return result.status === "complete" ? 0 : 1; -} - -// Classifies one configuration output for the feature-matrix loop: whether the -// matrix continues with the next configuration or stops and marks the rest -// unattempted. Takes the parsed config JSON, or null when the file is missing -// or unparseable. Pure: no I/O. -export function classifyConfigResult(result) { - if (result === null || result === undefined) - return { - outcome: "stop", - status: "unattempted", - error: "configuration output is missing or invalid", - }; - if (result.status === "interrupted") - return { outcome: "stop", status: "interrupted" }; - if (typeof result.metadata?.build?.durationMs !== "number") - return { - outcome: "stop", - status: "failed", - error: "build or preflight failed before any suite ran", - }; - const attempts = (result.cases ?? []).reduce( - (sum, benchmarkCase) => sum + (benchmarkCase.attempts?.length ?? 0), - 0, - ); - if (attempts === 0) - return { - outcome: "stop", - status: "failed", - error: "suite recorded no attempts", - }; - return { - outcome: "continue", - status: result.status === "complete" ? "complete" : "failed", - }; -} - -// Refuses an existing feature-matrix output root: it must be absent, or an -// empty directory. Never deletes or overwrites previous matrix output. -async function ensureFreshAbOutputRoot(outputRoot) { - let stat; - try { - stat = await fs.stat(outputRoot); - } catch (error) { - if (error.code === "ENOENT") return; - throw error; - } - if (!stat.isDirectory()) - throw new Error(`ab output root already exists: ${outputRoot}`); - if ((await fs.readdir(outputRoot)).length) - throw new Error( - `ab output root already exists and is not empty: ${outputRoot}`, - ); -} - -// Builds and runs every on/off configuration of the requested features, then -// compares the Boolean-lattice edges. Each configuration runs in Gray order -// with its own build and browser. Diagnostic tooling only: exit 0 reports -// that every edge has a comparison, never a performance verdict. An -// interruption between configurations or edges still writes matrix.json and -// exits 1. Callers may inject an AbortSignal; otherwise ab watches SIGINT and -// SIGTERM itself. -export async function ab( - options, - { - runFn = run, - compareFn = compareRuns, - manifestFn = readRendererManifest, - signal = null, - } = {}, -) { - const abId = randomUUID(); - const outputRoot = options.output - ? path.resolve(options.output) - : path.join( - os.tmpdir(), - `renderer-ab-${new Date().toISOString().replaceAll(":", "-")}-${abId.slice(0, 8)}`, - ); - await ensureFreshAbOutputRoot(outputRoot); - const definitions = await manifestFn(); - validateFeatureSelection(definitions, options.features, "ab"); - const { configurations, edges } = planFeatureMatrix(options.features); - const cases = collectCases(scenarios, options.seed, options.filter); - const attemptsPerCase = options.warmups + options.repetitions; - const plan = [ - `Feature matrix: ${options.features.join(",")}`, - ...configurations.map( - (config) => ` ${config.slug}: [${config.features.join(",")}]`, - ), - ...edges.map( - (edge) => ` ${edge.from}__${edge.to}: ${edge.toggledFeature}`, - ), - `${cases.length} cases, ${attemptsPerCase} attempts per case, one browser per configuration.`, - ].join("\n"); - console.log(plan); - if (options.dryRun) return 0; - if ((configurations.length > 8 || options.filter === "") && !options.confirm) - throw new Error( - "ab plans more work than confirmed; rerun with --confirm to proceed", - ); - let watched = signal; - let releaseSignal = null; - if (watched === null) { - const abort = new AbortController(); - watched = abort.signal; - const interrupt = () => abort.abort(new Error("Benchmark interrupted")); - process.once("SIGINT", interrupt); - process.once("SIGTERM", interrupt); - releaseSignal = () => { - process.removeListener("SIGINT", interrupt); - process.removeListener("SIGTERM", interrupt); - }; - } - await fs.mkdir(path.join(outputRoot, "configs"), { recursive: true }); - await fs.mkdir(path.join(outputRoot, "comparisons"), { recursive: true }); - const parsed = new Map(); - const configurationRecords = []; - let stopped = false; - const unattemptedRecord = (config) => ({ - slug: config.slug, - features: config.features, - status: "unattempted", - file: path.join(outputRoot, "configs", `${config.slug}.json`), - runId: null, - runStatus: "unattempted", - }); - for (const config of configurations) { - if (watched.aborted) { - configurationRecords.push({ - ...unattemptedRecord(config), - error: "Benchmark interrupted", - }); - continue; - } - if (stopped) { - configurationRecords.push(unattemptedRecord(config)); - continue; - } - const configPath = path.join(outputRoot, "configs", `${config.slug}.json`); - await runFn({ - ...options, - command: "ab", - features: config.features, - output: configPath, - }); - let result = null; - try { - result = JSON.parse(await fs.readFile(configPath, "utf8")); - } catch { - result = null; - } - const verdict = classifyConfigResult(result); - if (result !== null) parsed.set(config.slug, result); - configurationRecords.push({ - slug: config.slug, - features: config.features, - status: verdict.status, - file: configPath, - runId: result?.runId ?? null, - runStatus: result?.status ?? "unattempted", - ...(verdict.error ? { error: verdict.error } : {}), - }); - console.log(`${config.slug}: ${verdict.status}`); - if (verdict.outcome === "stop") stopped = true; - } - const comparisonRecords = []; - for (const edge of edges) { - if (watched.aborted) { - comparisonRecords.push({ ...edge, status: "not-attempted", file: null }); - continue; - } - const from = parsed.get(edge.from); - const to = parsed.get(edge.to); - if (!from || !to) { - comparisonRecords.push({ ...edge, status: "not-attempted", file: null }); - continue; - } - const comparison = tagExpectedFeatureDiff( - await compareFn(from, to, { diagnostic: true }), - edge.toggledFeature, - ); - // An edge is controlled when the intended feature toggle is the only - // mismatch. Anything else (environment, seeds, manifest defaults shifting - // mid-matrix) keeps the strong diagnostic-only reading. - const controlled = - comparison.mismatches?.every((entry) => entry.expected === true) ?? false; - const comparisonPath = path.join( - outputRoot, - "comparisons", - `${edge.from}__${edge.to}.json`, - ); - await save(comparisonPath, { ...comparison, edge, controlled }); - console.log(`${edge.from}__${edge.to}: ${comparisonPath}`); - if (!controlled) - console.warn( - `${edge.from}__${edge.to}: unexpected mismatches beyond the intended feature toggle`, - ); - comparisonRecords.push({ - ...edge, - status: "compared", - file: comparisonPath, - compatible: comparison.compatible, - diagnostic: comparison.diagnostic, - controlled, - }); - } - const status = - configurationRecords.every((record) => - ["complete", "failed"].includes(record.status), - ) && comparisonRecords.every((record) => record.status === "compared") - ? "complete" - : "failed"; - await save(path.join(outputRoot, "matrix.json"), { - schemaVersion: 1, - abId, - timestamp: new Date().toISOString(), - features: options.features, - maxFeatures: options.maxFeatures, - filter: options.filter, - outputRoot, - configurations: configurationRecords, - comparisons: comparisonRecords, - status, - warnings: [LIMITS], - }); - console.log(`Matrix: ${path.join(outputRoot, "matrix.json")}`); - // ab runs once per process; release before the single return. An unexpected - // throw past this point exits through main's handler with the listeners. - releaseSignal?.(); - return status === "complete" ? 0 : 1; -} - -export async function main(argv = process.argv.slice(2)) { - const options = readOptions(argv); - if (options.help) { - console.log( - "Renderer benchmarks\n node scripts/renderer-benchmarks.js run [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--features a,b] [--output FILE] [--no-build]\n node scripts/renderer-benchmarks.js ab --features a,b [--max-features N] [--confirm] [--dry-run] [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--output DIR]\n node scripts/renderer-benchmarks.js compare BASELINE CANDIDATE [--diagnostic] [--output FILE]\nRun requires a prepared development container and an existing static server on port 3000. --no-build reuses the existing renderer artifact instead of building and cannot be combined with --features or ab. Defaults: 3 warmups, 10 measured attempts, 30s timeout; provisional pending calibration.\n" + - LIMITS, - ); - return 0; - } - if (options.command === "compare") { - const results = await Promise.all( - options.files.map(async (file) => - JSON.parse(await fs.readFile(file, "utf8")), - ), - ); - const comparison = compareRuns(...results, { - diagnostic: options.diagnostic, - }); - console.log(LIMITS); - console.log(formatComparison(comparison)); - if (options.output) await save(path.resolve(options.output), comparison); - return 0; - } - if (options.command === "ab") return ab(options); - return run(options); -} - -if ( - process.argv[1] && - import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href -) { - main() - .then((code) => { - process.exitCode = code; - }) - .catch((error) => { - console.error(error.message); - process.exitCode = 1; - }); -} diff --git a/frontend/scripts/renderer-benchmarks/compare.js b/frontend/scripts/renderer-benchmarks/compare.js deleted file mode 100644 index 0741a7ce9f..0000000000 --- a/frontend/scripts/renderer-benchmarks/compare.js +++ /dev/null @@ -1,323 +0,0 @@ -import { - compareObservations, - metricNames, - observations, - statisticalMethod, - STATISTICAL_WARNINGS, -} from "../../resources/wasm-playground/benchmarks/statistics.js"; - -function record(value) { - return value !== null && typeof value === "object" && !Array.isArray(value); -} - -function requireValid(condition, message) { - if (!condition) throw new Error(`Invalid benchmark result: ${message}`); -} - -export function validateResult(result) { - requireValid( - record(result) && result.schemaVersion === 2, - "expected schemaVersion 2", - ); - requireValid(record(result.metadata), "metadata is required"); - for (const field of ["environment", "configuration"]) - requireValid( - record(result.metadata[field]), - `metadata.${field} is required`, - ); - requireValid(Array.isArray(result.cases), "cases must be an array"); - const ids = new Set(); - for (const benchmarkCase of result.cases) { - requireValid(record(benchmarkCase), "case must be an object"); - requireValid( - typeof benchmarkCase.id === "string" && - benchmarkCase.id.length && - !ids.has(benchmarkCase.id), - "case IDs must be unique nonempty strings", - ); - ids.add(benchmarkCase.id); - for (const field of ["scenarioId", "group", "preparation"]) - requireValid( - typeof benchmarkCase[field] === "string" && benchmarkCase[field].length, - `${benchmarkCase.id}.${field} is required`, - ); - requireValid( - Number.isInteger(benchmarkCase.scenarioVersion) && - benchmarkCase.scenarioVersion >= 1, - "scenarioVersion must be a positive integer", - ); - requireValid( - Number.isInteger(benchmarkCase.sceneSeed) && - benchmarkCase.sceneSeed >= 0 && - benchmarkCase.sceneSeed <= 0xffffffff, - "sceneSeed must be a uint32", - ); - for (const field of ["params", "initialView"]) - requireValid( - record(benchmarkCase[field]), - `${benchmarkCase.id}.${field} is required`, - ); - requireValid( - benchmarkCase.interaction === null || record(benchmarkCase.interaction), - "interaction must be an object or null", - ); - requireValid( - ["complete", "failed", "interrupted", "unattempted"].includes( - benchmarkCase.status, - ), - "unknown case status", - ); - requireValid( - Array.isArray(benchmarkCase.attempts), - "attempts must be an array", - ); - for (const attempt of benchmarkCase.attempts) { - requireValid( - record(attempt) && ["ok", "invalid", "failed"].includes(attempt.status), - "unknown attempt status", - ); - requireValid( - typeof attempt.warmup === "boolean", - "attempt.warmup must be boolean", - ); - requireValid( - record(attempt.metrics), - "attempt.metrics must be an object", - ); - requireValid( - attempt.invalidMetrics === undefined || record(attempt.invalidMetrics), - "invalidMetrics must be an object", - ); - for (const [name, value] of Object.entries(attempt.metrics)) - requireValid( - (value === null && - Object.hasOwn(attempt.invalidMetrics ?? {}, name)) || - (Number.isFinite(value) && value >= 0), - `metric ${name} must be finite and nonnegative, or explicitly invalid null`, - ); - for (const reason of Object.values(attempt.invalidMetrics ?? {})) - requireValid( - typeof reason === "string" && reason.length, - "invalid metric reasons must be nonempty strings", - ); - } - } -} - -function differences(baseline, candidate, path, mismatches) { - if (record(baseline) && record(candidate)) { - for (const key of [ - ...new Set([...Object.keys(baseline), ...Object.keys(candidate)]), - ].sort()) - differences(baseline[key], candidate[key], `${path}.${key}`, mismatches); - } else if (Array.isArray(baseline) && Array.isArray(candidate)) { - if (baseline.length !== candidate.length) - mismatches.push({ path, baseline, candidate }); - else - baseline.forEach((value, index) => - differences(value, candidate[index], `${path}[${index}]`, mismatches), - ); - } else if (baseline !== candidate) - mismatches.push({ - path, - baseline: baseline ?? null, - candidate: candidate ?? null, - }); -} - -export class CompatibilityError extends Error { - constructor(mismatches) { - super( - `Incompatible benchmark runs: ${mismatches.map((entry) => entry.path).join(", ")}. Use an explicit diagnostic comparison to inspect mismatches.`, - ); - this.name = "CompatibilityError"; - this.mismatches = mismatches; - } -} - -// Tags the expected per-edge feature difference of a feature-matrix comparison -// without suppressing it: the `metadata.build.features` mismatch stays listed -// but carries the toggling feature. Mutates the comparison's mismatch entries -// in place; no I/O. -export function tagExpectedFeatureDiff(comparison, toggledFeature) { - for (const entry of comparison.mismatches) { - if (entry.path === "metadata.build.features") { - entry.expected = true; - entry.toggledFeature = toggledFeature; - } - } - return comparison; -} - -export function compareRuns( - baseline, - candidate, - { diagnostic = false, ...options } = {}, -) { - validateResult(baseline); - validateResult(candidate); - const mismatches = []; - // Compare renderer revisions, but never silently mix debug/profiling builds - // with ordinary release measurements. - for (const field of [ - "mode", - "features", - "target", - "defaultFeatures", - "env", - "ambientEnv", - ]) { - differences( - baseline.metadata.build?.[field], - candidate.metadata.build?.[field], - `metadata.build.${field}`, - mismatches, - ); - } - for (const field of ["environment", "configuration", "metricSemantics"]) - differences( - baseline.metadata[field], - candidate.metadata[field], - `metadata.${field}`, - mismatches, - ); - const firstCases = new Map( - baseline.cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]), - ); - const secondCases = new Map( - candidate.cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]), - ); - const caseIds = [ - ...new Set([...firstCases.keys(), ...secondCases.keys()]), - ].sort(); - for (const id of caseIds) { - const first = firstCases.get(id); - const second = secondCases.get(id); - if (!first || !second) { - mismatches.push({ - path: `cases.${id}`, - baseline: Boolean(first), - candidate: Boolean(second), - }); - continue; - } - for (const field of [ - "scenarioId", - "scenarioVersion", - "sceneSeed", - "params", - "initialView", - "interaction", - "group", - "preparation", - "environment", - "metricSemantics", - ]) - differences( - first[field], - second[field], - `cases.${id}.${field}`, - mismatches, - ); - // A failed attempt may lack metrics. Only compare present metric sets when - // both cases completed; failures still produce honest sample counts. - if (first.status === "complete" && second.status === "complete") - differences( - metricNames(first), - metricNames(second), - `cases.${id}.metrics`, - mismatches, - ); - } - if (mismatches.length && !diagnostic) - throw new CompatibilityError(mismatches); - const runs = Object.fromEntries( - [ - ["baseline", baseline], - ["candidate", candidate], - ].map(([name, run]) => [ - name, - { - runId: run.runId ?? null, - status: run.status ?? "unknown", - timestamp: run.timestamp ?? null, - git: run.metadata.git ?? null, - }, - ]), - ); - return { - schemaVersion: 1, - runs, - compatible: mismatches.length === 0, - diagnostic, - mismatches, - method: statisticalMethod(options), - warnings: [ - ...Object.entries(runs) - .filter(([, run]) => run.status !== "complete") - .map( - ([name, run]) => - `The ${name} run is ${run.status}; this comparison covers only available valid observations.`, - ), - ...(mismatches.length - ? [ - "DIAGNOSTIC ONLY: these runs are incompatible; differences cannot establish a performance change.", - ] - : []), - ...STATISTICAL_WARNINGS, - ], - cases: caseIds - .filter((id) => firstCases.has(id) && secondCases.has(id)) - .map((id) => { - const first = firstCases.get(id); - const second = secondCases.get(id); - const metrics = [ - ...new Set([...metricNames(first), ...metricNames(second)]), - ].sort(); - return { - id, - metrics: Object.fromEntries( - metrics.map((metric) => [ - metric, - compareObservations( - observations(first, metric), - observations(second, metric), - options, - ), - ]), - ), - }; - }), - }; -} - -export function formatComparison(comparison) { - const lines = [ - `Statistics: ${comparison.method.name} v${comparison.method.version}; ${comparison.method.resamples} resamples, seed ${comparison.method.seed}.`, - ...comparison.warnings, - ]; - for (const mismatch of comparison.mismatches) - lines.push( - `MISMATCH ${mismatch.path}: ${JSON.stringify(mismatch.baseline)} -> ${JSON.stringify(mismatch.candidate)}`, - ); - const display = (value) => (value === null ? "n/a" : value.toFixed(3)); - for (const benchmarkCase of comparison.cases) { - lines.push(benchmarkCase.id); - for (const [name, metric] of Object.entries(benchmarkCase.metrics)) { - lines.push( - ` ${name}: median ${display(metric.baseline.median)} -> ${display(metric.candidate.median)}, change ${display(metric.absoluteChange)} (${display(metric.relativeChangePercent)}%); n=${metric.baseline.n}/${metric.candidate.n}`, - ); - if (metric.absoluteInterval) - lines.push( - ` change interval: [${display(metric.absoluteInterval.low)}, ${display(metric.absoluteInterval.high)}]`, - ); - if (metric.relativeIntervalPercent) - lines.push( - ` relative interval: [${display(metric.relativeIntervalPercent.low)}, ${display(metric.relativeIntervalPercent.high)}]%`, - ); - for (const [field, reason] of Object.entries(metric.suppressed)) - lines.push(` ${field}: ${reason}`); - } - } - return lines.join("\n"); -} diff --git a/frontend/scripts/renderer-benchmarks/tests/adapter.test.js b/frontend/scripts/renderer-benchmarks/tests/adapter.test.js deleted file mode 100644 index 2827756be9..0000000000 --- a/frontend/scripts/renderer-benchmarks/tests/adapter.test.js +++ /dev/null @@ -1,252 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; -import { createRendererAdapter } from "../../../resources/wasm-playground/js/lib.js"; - -function fixture() { - let next = 0; - const frames = new Map(); - const timers = new Map(); - const scheduler = { - requestAnimationFrame(callback) { - frames.set(++next, callback); - return next; - }, - cancelAnimationFrame(id) { - frames.delete(id); - }, - setTimeout(callback) { - timers.set(++next, callback); - return next; - }, - clearTimeout(id) { - timers.delete(id); - }, - }; - const state = { rendered: 0, cleaned: false, deleted: false, lost: false }; - const calls = { views: [], cached: 0, start: 0, end: 0 }; - const gl = { - getExtension: (name) => - name === "WEBGL_lose_context" - ? { - loseContext() { - state.lost = true; - }, - } - : null, - isContextLost: () => state.lost, - }; - const canvas = new EventTarget(); - Object.assign(canvas, { width: 1920, height: 1080, getContext: () => gl }); - const module = { - GL: { - registerContext: () => 1, - makeContextCurrent() {}, - deleteContext() { - state.deleted = true; - }, - }, - _init() {}, - _set_render_options() {}, - _resize_viewbox() {}, - _set_browser() {}, - _set_view_start() { - calls.start++; - }, - _set_view(...args) { - calls.views.push(args); - }, - _set_view_end() { - calls.end++; - }, - _render_from_cache() { - calls.cached++; - }, - _render() { - state.rendered++; - return 2; - }, - _clean_up() { - state.cleaned = true; - }, - }; - const adapter = createRendererAdapter(module, scheduler); - adapter.assignCanvas(canvas); - adapter.setupInteraction(canvas); - return { adapter, state, calls, canvas, frames, timers, module }; -} - -for (const dpr of [1, 1.25, 2]) { - test(`adapter initializes the full drawing buffer at DPR ${dpr}`, () => { - const size = { width: 0, height: 0 }; - let rendererDpr = 1; - const module = { - GL: { - registerContext: () => 1, - makeContextCurrent() {}, - }, - _init(width, height) { - Object.assign(size, { width, height }); - }, - _set_render_options(_flags, value) { - rendererDpr = value; - }, - _resize_viewbox(width, height) { - // The renderer applies DPR when resizing, not when setting options. - Object.assign(size, { - width: width * rendererDpr, - height: height * rendererDpr, - }); - }, - }; - const canvas = { - width: 1920 * dpr, - height: 1080 * dpr, - getContext: () => ({ getExtension() {} }), - }; - - createRendererAdapter(module).assignCanvas(canvas, dpr); - - assert.deepEqual(size, { width: canvas.width, height: canvas.height }); - }); -} - -test("wheel zoom flushes once per animation frame", () => { - const f = fixture(); - const wheel = (deltaY, offsetX, offsetY) => - f.canvas.dispatchEvent( - Object.assign(new Event("wheel"), { deltaY, offsetX, offsetY }), - ); - const runFrame = () => { - const [id] = f.frames.keys(); - const callback = f.frames.get(id); - f.frames.delete(id); - callback(16); - }; - - wheel(-1, 10, 20); - wheel(-1, 30, 40); - wheel(-1, 50, 60); - - assert.equal(f.frames.size, 1); - assert.equal(f.calls.start, 1); - assert.equal(f.calls.views.length, 0); - assert.equal(f.calls.cached, 0); - - runFrame(); - - assert.equal(f.calls.views.length, 1); - assert.equal(f.calls.cached, 1); - const [scale, x, y] = f.calls.views[0]; - assert.ok(Math.abs(scale - 1.1 ** 3) < 1e-9, `unexpected scale ${scale}`); - assert.ok( - Math.abs(x - (50 / 1.1 ** 3 - 50)) < 1e-9, - `unexpected x anchor ${x}`, - ); - assert.ok( - Math.abs(y - (60 / 1.1 ** 3 - 60)) < 1e-9, - `unexpected y anchor ${y}`, - ); - assert.equal(f.timers.size, 1); - - // The flush releases the frame slot and resets the accumulated factor. - wheel(-1, 10, 20); - assert.equal(f.frames.size, 1); - assert.equal(f.calls.cached, 1); - - runFrame(); - - assert.equal(f.calls.views.length, 2); - assert.equal(f.calls.cached, 2); - assert.ok( - Math.abs(f.calls.views[1][0] - 1.1 ** 4) < 1e-9, - `unexpected scale ${f.calls.views[1][0]}`, - ); - f.adapter.dispose(); -}); - -test("wheel zoom clamps the accumulated factor to the scale bounds", () => { - const f = fixture(); - const wheel = (deltaY) => - f.canvas.dispatchEvent( - Object.assign(new Event("wheel"), { deltaY, offsetX: 10, offsetY: 20 }), - ); - const runFrame = () => { - const [id] = f.frames.keys(); - const callback = f.frames.get(id); - f.frames.delete(id); - callback(16); - }; - - for (let index = 0; index < 100; index++) wheel(-1); - runFrame(); - assert.equal(f.calls.views[0][0], 100); - - for (let index = 0; index < 200; index++) wheel(1); - runFrame(); - assert.equal(f.calls.views[1][0], 0.01); - f.adapter.dispose(); -}); - -test("pointer pan still renders on every move", () => { - const f = fixture(); - f.canvas.dispatchEvent( - Object.assign(new Event("pointerdown"), { - clientX: 0, - clientY: 0, - pointerId: 1, - }), - ); - f.canvas.dispatchEvent( - Object.assign(new Event("pointermove"), { clientX: 10, clientY: 20 }), - ); - f.canvas.dispatchEvent( - Object.assign(new Event("pointermove"), { clientX: 30, clientY: 40 }), - ); - - assert.equal(f.calls.views.length, 2); - assert.equal(f.calls.cached, 2); - assert.equal(f.frames.size, 0); - f.adapter.dispose(); -}); - -test("disposing an adapter settles pending work and isolates late callbacks", async () => { - const first = fixture(); - const second = fixture(); - const pending = first.adapter.renderFull(); - const lateRender = [...first.frames.values()][0]; - first.canvas.dispatchEvent( - Object.assign(new Event("wheel"), { deltaY: 1, offsetX: 10, offsetY: 20 }), - ); - assert.equal(first.frames.size, 1); - const lateZoom = [...first.frames.values()][0]; - first.adapter.dispose(); - first.adapter.dispose(); - lateRender(16); - lateZoom(16); - assert.equal(await pending, false); - assert.equal(first.frames.size + first.timers.size, 0); - assert.deepEqual(first.state, { - rendered: 0, - cleaned: true, - deleted: true, - lost: true, - }); - assert.equal(first.calls.cached, 0); - first.canvas.dispatchEvent(new Event("wheel")); - assert.equal(first.frames.size + first.timers.size, 0); - const completed = second.adapter.renderFull(); - [...second.frames.values()][0](32); - assert.equal(await completed, true); - assert.equal(second.state.cleaned, false); - second.adapter.dispose(); -}); - -test("adapter deletes its context even when native cleanup throws", () => { - const f = fixture(); - f.module._clean_up = () => { - throw new Error("cleanup failed"); - }; - assert.throws(() => f.adapter.dispose(), /cleanup failed/); - assert.equal(f.state.deleted, true); - assert.equal(f.state.lost, true); -}); diff --git a/frontend/scripts/renderer-benchmarks/tests/compare.test.js b/frontend/scripts/renderer-benchmarks/tests/compare.test.js deleted file mode 100644 index 063275b0b3..0000000000 --- a/frontend/scripts/renderer-benchmarks/tests/compare.test.js +++ /dev/null @@ -1,240 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { compareRuns, formatComparison } from "../compare.js"; -import { tagExpectedFeatureDiff } from "../compare.js"; - -function run(value = 10) { - return { - schemaVersion: 2, - status: "complete", - metadata: { - git: { sha: "before", dirty: false }, - build: { - mode: "release", - featurePolicy: "manifest-defaults", - features: [], - defaultFeatures: ["default"], - target: "frontend", - env: { BUILD_MODE: "release" }, - ambientEnv: { CARGO_HOME: "/home/penpot/.cargo" }, - }, - environment: { chromium: "123", headless: true }, - configuration: { viewport: { width: 1920, height: 1080 }, dpr: 2 }, - }, - cases: [ - { - id: "rects/default/load", - scenarioId: "rects", - scenarioVersion: 1, - sceneSeed: 123, - params: { count: 1000 }, - initialView: { x: 0, y: 0, zoom: 1 }, - interaction: null, - group: "fresh-module-context", - preparation: "fresh", - environment: { webgl: { renderer: "GPU" } }, - status: "complete", - attempts: Array.from({ length: 10 }, (_, index) => ({ - index, - warmup: false, - status: "ok", - metrics: { render: value }, - invalidMetrics: {}, - })), - }, - ], - }; -} - -test("offline comparison allows intentional source and build changes", () => { - const candidate = run(12); - candidate.metadata.git.sha = "after"; - candidate.metadata.build.revision = "candidate"; - const result = compareRuns(run(), candidate); - const metric = result.cases[0].metrics.render; - assert.equal(result.compatible, true); - assert.equal(metric.absoluteChange, 2); - assert.equal(metric.relativeChangePercent, 20); - assert.deepEqual(metric.absoluteInterval, { low: 2, high: 2 }); - assert.deepEqual(result, compareRuns(run(), candidate)); -}); - -for (const [name, change] of [ - [ - "profiling features", - (r) => { - r.metadata.build.features = ["profile"]; - }, - ], - [ - "build mode", - (r) => { - r.metadata.build.mode = "debug"; - }, - ], - [ - "DPR", - (r) => { - r.metadata.configuration.dpr = 1; - }, - ], - [ - "browser", - (r) => { - r.metadata.environment.chromium = "124"; - }, - ], - [ - "GPU", - (r) => { - r.cases[0].environment.webgl.renderer = "software"; - }, - ], - [ - "seed", - (r) => { - r.cases[0].sceneSeed = 456; - }, - ], - [ - "version", - (r) => { - r.cases[0].scenarioVersion = 2; - }, - ], - [ - "workload", - (r) => { - r.cases[0].params.count = 2000; - }, - ], - [ - "preparation", - (r) => { - r.cases[0].preparation = "warm"; - }, - ], - [ - "case set", - (r) => { - r.cases[0].id = "different"; - }, - ], -]) { - test(`comparison refuses incompatible ${name} unless diagnostic override is explicit`, () => { - const candidate = run(); - change(candidate); - assert.throws(() => compareRuns(run(), candidate), { - name: "CompatibilityError", - }); - const result = compareRuns(run(), candidate, { diagnostic: true }); - assert.equal(result.compatible, false); - assert.ok(result.mismatches.length > 0); - assert.match(result.warnings.join(" "), /incompatible/i); - }); -} - -test("object key order is immaterial to compatibility", () => { - const candidate = run(); - candidate.metadata.configuration.viewport = { height: 1080, width: 1920 }; - assert.equal(compareRuns(run(), candidate).compatible, true); -}); - -test("relative change is unsupported when baseline is zero", () => { - const metric = compareRuns(run(0), run(1)).cases[0].metrics.render; - assert.equal(metric.relativeChangePercent, null); - assert.equal(metric.relativeIntervalPercent, null); - assert.match(metric.suppressed.relativeChangePercent, /zero/); -}); - -test("failed attempts never enter comparison observations", () => { - const candidate = run(12); - candidate.cases[0].attempts[0].status = "failed"; - const metric = compareRuns(run(), candidate).cases[0].metrics.render; - assert.equal(metric.candidate.n, 9); - assert.equal(metric.candidate.invalid, 1); - assert.equal(metric.absoluteInterval, null); -}); - -test("comparisons warn when a run failed even if available metrics are compatible", () => { - const candidate = run(12); - candidate.status = "failed"; - candidate.cases[0].status = "failed"; - const result = compareRuns(run(), candidate); - assert.equal(result.compatible, true); - assert.equal(result.runs.candidate.status, "failed"); - assert.match(formatComparison(result), /candidate run is failed/i); -}); - -test("offline comparison rejects malformed results without diagnostic bypass", () => { - const missingEnvironment = run(); - delete missingEnvironment.metadata.environment; - const duplicateCase = run(); - duplicateCase.cases.push(structuredClone(duplicateCase.cases[0])); - const invalidMetric = run(); - invalidMetric.cases[0].attempts[0].metrics.render = "12"; - for (const invalid of [ - null, - {}, - { schemaVersion: 1 }, - { schemaVersion: 3 }, - missingEnvironment, - duplicateCase, - invalidMetric, - ]) { - assert.throws( - () => compareRuns(invalid, run(), { diagnostic: true }), - /Invalid benchmark result/, - ); - } -}); - -test("text comparisons display changes, intervals and incompatibilities", () => { - const candidate = run(12); - candidate.metadata.configuration.dpr = 1; - const report = formatComparison( - compareRuns(run(), candidate, { diagnostic: true }), - ); - assert.match(report, /DIAGNOSTIC ONLY/); - assert.match(report, /MISMATCH metadata.configuration.dpr: 2 -> 1/); - assert.match(report, /change 2\.000 \(20\.000%\)/); - assert.match(report, /change interval: \[2\.000, 2\.000\]/); -}); - -for (const [name, change] of [ - ["default features", (r) => (r.metadata.build.defaultFeatures = [])], - ["build env", (r) => (r.metadata.build.env.BUILD_MODE = "debug")], - ["ambient env", (r) => (r.metadata.build.ambientEnv.CARGO_HOME = "/other")], -]) { - test(`comparison refuses changed ${name} unless diagnostic override is explicit`, () => { - const candidate = run(); - change(candidate); - assert.throws(() => compareRuns(run(), candidate), { - name: "CompatibilityError", - }); - const result = compareRuns(run(), candidate, { diagnostic: true }); - assert.equal(result.compatible, false); - assert.ok(result.mismatches.length > 0); - }); -} - -test("ab edge tags the expected feature mismatch without suppressing it", () => { - const candidate = run(); - candidate.metadata.build.features = ["branch-b"]; - candidate.metadata.build.featurePolicy = "explicit"; - assert.throws(() => compareRuns(run(), candidate), { - name: "CompatibilityError", - }); - const compared = compareRuns(run(), candidate, { diagnostic: true }); - assert.equal(compared.compatible, false); - tagExpectedFeatureDiff(compared, "branch-b"); - const tagged = compared.mismatches.filter((entry) => entry.expected); - assert.equal(tagged.length, 1); - assert.deepEqual(tagged[0], { - path: "metadata.build.features", - baseline: [], - candidate: ["branch-b"], - expected: true, - toggledFeature: "branch-b", - }); -}); diff --git a/frontend/scripts/renderer-benchmarks/tests/protocol.test.js b/frontend/scripts/renderer-benchmarks/tests/protocol.test.js deleted file mode 100644 index 625d6ec91b..0000000000 --- a/frontend/scripts/renderer-benchmarks/tests/protocol.test.js +++ /dev/null @@ -1,97 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; -import { - drain, - interact, - restore, -} from "../../../resources/wasm-playground/benchmarks/protocol.js"; - -function runtime(results) { - let time = 0; - const flags = []; - return { - flags, - now: () => time, - frame: async () => (time += 16), - sleep: async (ms) => { - time += ms; - }, - check: () => {}, - module: { - _render: (_timestamp, flag) => { - flags.push(flag); - time += 2; - return results.shift(); - }, - _set_view_start: () => { - time += 1; - }, - _set_view: () => { - time += 1; - }, - _render_from_cache: () => { - time += 2; - }, - _set_view_end: () => { - time += 5; - }, - }, - }; -} - -test("progressive rendering retains returned states and stops at Full", async () => { - const rt = runtime([1, 3, 2]); - const result = await drain(rt, { flags: 0 }); - assert.deepEqual(rt.flags, [0, 1, 3]); - assert.equal(result.slices.length, 3); - assert.ok(result.viewportReadyMs < result.fullMs); -}); - -test("immediate Full supplies both completion boundaries", async () => { - const result = await drain(runtime([2])); - assert.equal(result.viewportReadyMs, result.fullMs); -}); - -test("Partial can finish directly at Full without ViewportReady", async () => { - const rt = runtime([1, 2]); - const result = await drain(rt); - assert.deepEqual(rt.flags, [0, 1]); - assert.equal(result.viewportReadyMs, result.fullMs); -}); - -test("preparation drains to Full and applies SyncTiles only initially", async () => { - const rt = runtime([1, 3, 2]); - const result = await restore(rt, { scale: 1, x: 0, y: 0 }); - assert.deepEqual(rt.flags, [4, 1, 3]); - assert.equal(result.slices.at(-1).frameType, 2); -}); - -test("None and unknown frame types fail explicitly", async () => { - for (const frame of [0, 7, NaN]) { - await assert.rejects(drain(runtime([frame])), /frame type/); - } -}); - -test("finalization includes set_view_end but separates the settling delay", async () => { - const rt = runtime([3, 2]); - const result = await interact(rt, { - frames: [{ scale: 1, x: 3, y: 4 }], - settleMs: 100, - }); - assert.equal(result.metrics.setViewEndMs, 5); - assert.equal(result.metrics.settlingActualMs, 100); - assert.ok(result.metrics.timeToViewportReadyMs >= 7); - assert.ok( - result.metrics.lastInputToFullMs >= 100 + result.metrics.timeToFullMs, - ); - assert.deepEqual(rt.flags, [4, 3]); -}); - -test("cancellation stops further progressive work", async () => { - const rt = runtime([1, 2]); - rt.check = () => { - if (rt.flags.length) throw new Error("Canceled"); - }; - await assert.rejects(drain(rt), /Canceled/); - assert.equal(rt.flags.length, 1); -}); diff --git a/frontend/scripts/renderer-benchmarks/tests/report.test.js b/frontend/scripts/renderer-benchmarks/tests/report.test.js deleted file mode 100644 index 7b4874718a..0000000000 --- a/frontend/scripts/renderer-benchmarks/tests/report.test.js +++ /dev/null @@ -1,39 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; - -test("page reports keep raw results and show the shared readable summary", async () => { - const previous = globalThis.document; - const report = { hidden: true, textContent: "" }; - globalThis.document = { - querySelector: (selector) => (selector === "pre" ? report : {}), - }; - try { - await import("../../../resources/wasm-playground/benchmarks/browser.js"); - const result = { - scope: "Renderer submission", - metadata: {}, - warnings: [], - cases: [ - { - id: "rects/load", - status: "complete", - unattempted: 0, - attempts: [ - { warmup: false, status: "ok", metrics: { timeToFullMs: 2 } }, - ], - }, - ], - }; - globalThis.rendererBenchmark.showResult(result); - assert.equal(globalThis.rendererBenchmarkResult, result); - assert.equal(report.hidden, false); - assert.match(report.textContent, /median 2\.000/); - assert.match(report.textContent, /1 valid, 0 invalid/); - assert.match(report.textContent, /Renderer submission/); - } finally { - globalThis.document = previous; - delete globalThis.rendererBenchmark; - delete globalThis.rendererBenchmarkReady; - delete globalThis.rendererBenchmarkResult; - } -}); diff --git a/frontend/scripts/renderer-benchmarks/tests/runner.test.js b/frontend/scripts/renderer-benchmarks/tests/runner.test.js deleted file mode 100644 index 86320f30bc..0000000000 --- a/frontend/scripts/renderer-benchmarks/tests/runner.test.js +++ /dev/null @@ -1,717 +0,0 @@ -import test from "node:test"; -import assert from "node:assert/strict"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { - readOptions, - bounded, - main, - recordAttempt, - finishCases, - defaultFeatures, - planFeatureMatrix, - validateFeatureSelection, - recordAmbientEnv, - BUILD_ENV, - classifyConfigResult, - ab, -} from "../../renderer-benchmarks.js"; - -test("build provenance expands the renderer's declared default features", () => { - assert.deepEqual(defaultFeatures({ default: [] }), ["default"]); - assert.deepEqual( - defaultFeatures({ default: ["fast"], fast: ["shared"], shared: ["fast"] }), - ["default", "fast", "shared"], - ); - assert.throws( - () => defaultFeatures({ default: ["profile"], profile: [] }), - /diagnostic/, - ); -}); -import { - collectCases, - deriveSeed, - validateMetrics, -} from "../../../resources/wasm-playground/benchmarks/contract.js"; -import { scenarios } from "../../../resources/wasm-playground/scenes/index.js"; - -test("CLI validates bounded numbers, commands and URLs before execution", () => { - assert.equal(readOptions([]).repetitions, 10); - for (const args of [ - ["--repetitions", "NaN"], - ["--warmups", "-1"], - ["--seed", "4294967296"], - ["--base-url", "file:///tmp"], - ["--unknown"], - ["compare", "one.json"], - ]) { - assert.throws(() => readOptions(args)); - } - assert.equal( - readOptions(["compare", "before.json", "after.json", "--diagnostic"]) - .diagnostic, - true, - ); -}); - -test("feature matrix plans Gray-ordered configurations and lattice edges", () => { - assert.deepEqual(planFeatureMatrix(["branch-b"]), { - configurations: [ - { slug: "0", features: [] }, - { slug: "1", features: ["branch-b"] }, - ], - edges: [{ from: "0", to: "1", toggledFeature: "branch-b" }], - }); - const two = planFeatureMatrix(["a", "b"]); - assert.deepEqual( - two.configurations.map((c) => c.slug), - ["00", "01", "11", "10"], - ); - assert.deepEqual( - two.configurations.map((c) => c.features), - [[], ["b"], ["a", "b"], ["a"]], - ); - assert.deepEqual( - two.edges.map((e) => `${e.from}__${e.to}:${e.toggledFeature}`).sort(), - ["00__01:b", "00__10:a", "01__11:a", "10__11:b"], - ); - const three = planFeatureMatrix(["a", "b", "c"]); - assert.equal(three.configurations.length, 8); - assert.equal(three.edges.length, 12); - assert.equal(new Set(three.edges.map((e) => `${e.from}__${e.to}`)).size, 12); - for (const config of three.configurations) { - assert.equal( - three.edges.filter((e) => e.from === config.slug || e.to === config.slug) - .length, - 3, - ); - } -}); - -test("ab help needs no features and dry run is ab-only", async () => { - assert.equal(readOptions(["ab", "--help"]).command, "ab"); - assert.equal(await main(["ab", "--help"]), 0); - for (const args of [ - ["run", "--dry-run"], - ["compare", "a.json", "b.json", "--dry-run"], - ]) { - assert.throws(() => readOptions(args), /dry-run/); - } -}); - -test("CLI parses feature selection for run and ab", () => { - assert.deepEqual(readOptions(["run"]).features, []); - assert.deepEqual(readOptions(["run", "--features", "b,a"]).features, [ - "a", - "b", - ]); - assert.deepEqual(readOptions(["run", "--features", "a,a,, b "]).features, [ - "a", - "b", - ]); - assert.deepEqual(readOptions(["run", "--features", "profile"]).features, [ - "profile", - ]); - const ab = readOptions(["ab", "--features", "branch-b"]); - assert.deepEqual(ab.features, ["branch-b"]); - assert.equal(ab.maxFeatures, 3); - assert.equal(ab.confirm, false); - assert.equal(ab.dryRun, false); - assert.equal( - readOptions(["ab", "--features", "a", "--confirm", "--dry-run"]).confirm, - true, - ); - for (const args of [ - ["ab"], - ["ab", "--features", ""], - ["ab", "--features", "a", "--max-features", "0"], - ["ab", "--features", "a", "--max-features", "7"], - ["ab", "--features", "Bad_Name"], - ["ab", "--features", "profile"], - ["ab", "--features", "stats-foo"], - ["ab", "--features", "a", "extra.json"], - ["run", "--features", "a", "extra.json"], - ]) { - assert.throws(() => readOptions(args)); - } -}); - -test("--no-build skips the renderer build only for run", () => { - assert.equal(readOptions(["run"]).noBuild, false); - assert.equal(readOptions(["run", "--no-build"]).noBuild, true); - for (const args of [ - ["ab", "--features", "branch-b", "--no-build"], - ["compare", "a.json", "b.json", "--no-build"], - ["run", "--no-build", "--features", "branch-b"], - ]) { - assert.throws(() => readOptions(args), /--no-build/); - } -}); - -test("feature selection validates against the renderer manifest", () => { - const definitions = { default: [], "branch-a": [], "branch-b": [] }; - assert.deepEqual(validateFeatureSelection(definitions, ["branch-b"], "ab"), [ - "default", - ]); - assert.deepEqual(validateFeatureSelection(definitions, [], "run"), [ - "default", - ]); - assert.throws( - () => validateFeatureSelection({ default: ["branch-b"] }, [], "ab"), - /empty default feature set/, - ); - assert.deepEqual( - validateFeatureSelection({ default: ["branch-b"] }, [], "run"), - ["branch-b", "default"], - ); - assert.throws( - () => validateFeatureSelection(definitions, ["nope"], "run"), - /Unknown Cargo feature/, - ); - for (const command of ["run", "ab"]) { - assert.throws( - () => - validateFeatureSelection( - { default: ["shared"], shared: [] }, - ["shared"], - command, - ), - /already enabled by default/, - ); - } -}); - -test("build environment recording excludes volatile values", () => { - assert.equal("VERSION" in BUILD_ENV, false); - assert.equal(BUILD_ENV.PENPOT_WASM_FUNCTION_NAMES, "0"); - assert.deepEqual( - recordAmbientEnv({ - RUSTFLAGS: "--deny warnings", - CARGO_HOME: "/home/penpot/.cargo", - CARGO_PROFILE_RELEASE_DEBUG: "true", - EMSDK: "/opt/emsdk", - EMSDK_CLANG: "clang", - PATH: "/usr/bin", - VERSION: "volatile", - PENPOT_WASM_PREPARED: "1", - }), - { - RUSTFLAGS: "--deny warnings", - CARGO_HOME: "/home/penpot/.cargo", - CARGO_PROFILE_RELEASE_DEBUG: "true", - EMSDK: "/opt/emsdk", - EMSDK_CLANG: "clang", - }, - ); -}); - -test("classifier separates continuation from matrix stops", () => { - const complete = { - status: "complete", - metadata: { build: { durationMs: 5 } }, - cases: [{ attempts: [{}] }], - }; - assert.deepEqual(classifyConfigResult(complete), { - outcome: "continue", - status: "complete", - }); - assert.deepEqual(classifyConfigResult({ ...complete, status: "failed" }), { - outcome: "continue", - status: "failed", - }); - assert.deepEqual(classifyConfigResult(null), { - outcome: "stop", - status: "unattempted", - error: "configuration output is missing or invalid", - }); - assert.deepEqual(classifyConfigResult({ status: "interrupted" }), { - outcome: "stop", - status: "interrupted", - }); - assert.deepEqual( - classifyConfigResult({ - status: "failed", - metadata: { build: {} }, - cases: [], - }), - { - outcome: "stop", - status: "failed", - error: "build or preflight failed before any suite ran", - }, - ); - assert.deepEqual( - classifyConfigResult({ - status: "failed", - metadata: { build: { durationMs: 5 } }, - cases: [], - }), - { - outcome: "stop", - status: "failed", - error: "suite recorded no attempts", - }, - ); -}); - -const abDefinitions = { default: [], "branch-a": [], "branch-b": [] }; - -async function abTempRoot() { - const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "renderer-ab-test-")); - return path.join(tmp, "matrix"); -} - -function abOptions(argv) { - return readOptions(argv); -} - -function stubManifest(definitions = abDefinitions) { - return async () => definitions; -} - -function stubRun(failSlugs = new Set()) { - const calls = []; - const runFn = async (options) => { - calls.push(options); - const slug = path.basename(options.output, ".json"); - const failed = failSlugs.has(slug); - const result = failed - ? { - schemaVersion: 2, - runId: `run-${slug}`, - status: "failed", - error: "./build failed (1)", - metadata: { build: { mode: "release", features: [] } }, - cases: [], - } - : { - schemaVersion: 2, - runId: `run-${slug}`, - status: "complete", - metadata: { - build: { - mode: "release", - features: options.features, - durationMs: 5, - }, - }, - cases: [ - { - id: "rects/default/pan", - status: "complete", - attempts: [{ index: 0, warmup: false, status: "ok" }], - }, - ], - }; - await fs.mkdir(path.dirname(options.output), { recursive: true }); - await fs.writeFile(options.output, JSON.stringify(result)); - return failed ? 1 : 0; - }; - return { calls, runFn }; -} - -function stubCompare(extra = []) { - const calls = []; - const compareFn = async (baseline, candidate, { diagnostic } = {}) => { - calls.push({ baseline, candidate, diagnostic }); - return { - schemaVersion: 1, - compatible: false, - diagnostic: diagnostic ?? false, - mismatches: [ - { - path: "metadata.build.features", - baseline: baseline.metadata.build.features, - candidate: candidate.metadata.build.features, - }, - ...extra, - ], - warnings: [], - cases: [], - }; - }; - return { calls, compareFn }; -} - -test("ab refuses an existing output root before any preflight", async () => { - const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "renderer-ab-test-")); - const nonEmpty = path.join(tmp, "full"); - await fs.mkdir(nonEmpty); - await fs.writeFile(path.join(nonEmpty, "x"), "x"); - const filePath = path.join(tmp, "file.json"); - await fs.writeFile(filePath, "{}"); - const mustNotRun = () => { - throw new Error("must not be called"); - }; - for (const output of [nonEmpty, filePath]) { - await assert.rejects( - ab(abOptions(["ab", "--features", "branch-b", "--output", output]), { - runFn: mustNotRun, - compareFn: mustNotRun, - manifestFn: mustNotRun, - }), - /already exists|non-empty/, - ); - } - await fs.rm(tmp, { recursive: true, force: true }); -}); - -test("ab dry run prints the plan without building", async () => { - const output = await abTempRoot(); - const mustNotRun = () => { - throw new Error("must not be called"); - }; - const code = await ab( - abOptions([ - "ab", - "--features", - "branch-b", - "--filter", - "rects/default/pan", - "--dry-run", - "--output", - output, - ]), - { runFn: mustNotRun, compareFn: mustNotRun, manifestFn: stubManifest() }, - ); - assert.equal(code, 0); - await assert.rejects(fs.access(output), /ENOENT/); - await fs.rm(path.dirname(output), { recursive: true, force: true }); -}); - -test("ab confirm gate blocks wide matrices and empty filters", async () => { - const runFn = () => { - throw new Error("must not be called"); - }; - const wide = await abTempRoot(); - const narrow = await abTempRoot(); - await assert.rejects( - ab( - abOptions([ - "ab", - "--features", - "f1,f2,f3,f4", - "--max-features", - "4", - "--filter", - "rects", - "--output", - wide, - ]), - { - runFn, - manifestFn: stubManifest({ - default: [], - f1: [], - f2: [], - f3: [], - f4: [], - }), - }, - ), - /confirm/, - ); - await assert.rejects( - ab(abOptions(["ab", "--features", "branch-b", "--output", narrow]), { - runFn, - manifestFn: stubManifest(), - }), - /confirm/, - ); - await fs.rm(path.dirname(wide), { recursive: true, force: true }); - await fs.rm(path.dirname(narrow), { recursive: true, force: true }); -}); - -test("ab runs the matrix and writes configs, comparisons and index", async () => { - const output = await abTempRoot(); - const { calls: runCalls, runFn } = stubRun(); - const { calls: compareCalls, compareFn } = stubCompare(); - const before = process.listenerCount("SIGINT"); - const code = await ab( - abOptions([ - "ab", - "--features", - "branch-b", - "--filter", - "rects/default/pan", - "--output", - output, - ]), - { runFn, compareFn, manifestFn: stubManifest() }, - ); - assert.equal(code, 0); - assert.equal(process.listenerCount("SIGINT"), before); - assert.deepEqual( - runCalls.map((options) => options.features), - [[], ["branch-b"]], - ); - assert.equal(compareCalls.length, 1); - assert.equal(compareCalls[0].diagnostic, true); - const first = JSON.parse( - await fs.readFile(path.join(output, "configs", "0.json"), "utf8"), - ); - assert.equal(first.runId, "run-0"); - const comparison = JSON.parse( - await fs.readFile(path.join(output, "comparisons", "0__1.json"), "utf8"), - ); - assert.deepEqual(comparison.edge, { - from: "0", - to: "1", - toggledFeature: "branch-b", - }); - assert.deepEqual( - comparison.mismatches.filter((entry) => entry.expected), - [ - { - path: "metadata.build.features", - baseline: [], - candidate: ["branch-b"], - expected: true, - toggledFeature: "branch-b", - }, - ], - ); - const matrix = JSON.parse( - await fs.readFile(path.join(output, "matrix.json"), "utf8"), - ); - assert.equal(matrix.schemaVersion, 1); - assert.deepEqual(matrix.features, ["branch-b"]); - assert.deepEqual( - matrix.configurations.map((c) => c.status), - ["complete", "complete"], - ); - assert.deepEqual( - matrix.comparisons.map((c) => [c.from, c.to, c.status]), - [["0", "1", "compared"]], - ); - assert.equal(matrix.status, "complete"); - assert.equal(comparison.controlled, true); - assert.equal(matrix.comparisons[0].controlled, true); - await fs.rm(path.dirname(output), { recursive: true, force: true }); -}); - -test("ab marks edges with unexpected diffs uncontrolled", async () => { - const output = await abTempRoot(); - const { runFn } = stubRun(); - const { compareFn } = stubCompare([ - { path: "metadata.build.env", baseline: {}, candidate: {} }, - ]); - const code = await ab( - abOptions([ - "ab", - "--features", - "branch-b", - "--filter", - "rects/default/pan", - "--output", - output, - ]), - { runFn, compareFn, manifestFn: stubManifest() }, - ); - assert.equal(code, 0); - const comparison = JSON.parse( - await fs.readFile(path.join(output, "comparisons", "0__1.json"), "utf8"), - ); - assert.equal(comparison.controlled, false); - const matrix = JSON.parse( - await fs.readFile(path.join(output, "matrix.json"), "utf8"), - ); - assert.equal(matrix.comparisons[0].controlled, false); - await fs.rm(path.dirname(output), { recursive: true, force: true }); -}); - -test("ab interruption between configurations still writes the matrix", async () => { - const output = await abTempRoot(); - const controller = new AbortController(); - const { runFn } = stubRun(); - const interruptAfterFirst = async (options) => { - const code = await runFn(options); - controller.abort(new Error("stopped")); - return code; - }; - const { compareFn } = stubCompare(); - const code = await ab( - abOptions([ - "ab", - "--features", - "a,b", - "--max-features", - "2", - "--filter", - "rects", - "--confirm", - "--output", - output, - ]), - { - runFn: interruptAfterFirst, - compareFn, - manifestFn: stubManifest({ default: [], a: [], b: [] }), - signal: controller.signal, - }, - ); - assert.equal(code, 1); - const matrix = JSON.parse( - await fs.readFile(path.join(output, "matrix.json"), "utf8"), - ); - assert.deepEqual( - matrix.configurations.map((c) => [c.slug, c.status]), - [ - ["00", "complete"], - ["01", "unattempted"], - ["11", "unattempted"], - ["10", "unattempted"], - ], - ); - assert.ok(matrix.comparisons.every((c) => c.status === "not-attempted")); - assert.equal(matrix.status, "failed"); - await fs.rm(path.dirname(output), { recursive: true, force: true }); -}); - -test("ab stops on build failure and marks the rest unattempted", async () => { - const output = await abTempRoot(); - const { runFn } = stubRun(new Set(["01"])); - const { compareFn } = stubCompare(); - const code = await ab( - abOptions([ - "ab", - "--features", - "b,a", - "--max-features", - "2", - "--filter", - "rects", - "--confirm", - "--output", - output, - ]), - { - runFn, - compareFn, - manifestFn: stubManifest({ default: [], a: [], b: [] }), - }, - ); - assert.equal(code, 1); - const matrix = JSON.parse( - await fs.readFile(path.join(output, "matrix.json"), "utf8"), - ); - assert.deepEqual( - matrix.configurations.map((c) => [c.slug, c.status]), - [ - ["00", "complete"], - ["01", "failed"], - ["11", "unattempted"], - ["10", "unattempted"], - ], - ); - assert.deepEqual( - matrix.comparisons.map((c) => [c.from, c.to, c.status]), - [ - ["00", "10", "not-attempted"], - ["00", "01", "compared"], - ["01", "11", "not-attempted"], - ["10", "11", "not-attempted"], - ], - ); - assert.equal(matrix.status, "failed"); - await fs.rm(path.dirname(output), { recursive: true, force: true }); -}); - -test("case collector retains a single scene seed across lifecycle and interaction groups", () => { - const cases = collectCases(scenarios, 42); - assert.equal(cases.length, 21); - const rectangles = cases.filter((c) => c.scenarioId === "rects"); - assert.equal(new Set(rectangles.map((c) => c.sceneSeed)).size, 1); - assert.equal(rectangles[0].sceneSeed, deriveSeed(42, "rects/default")); - assert.throws(() => collectCases(scenarios, 42, "missing"), /No matching/); - assert.throws( - () => collectCases([scenarios[0], scenarios[0]], 42), - /duplicate/, - ); -}); - -test("case collector rejects missing interactions and non-JSON case values", () => { - const scenario = scenarios[0]; - const invalid = { - ...scenario, - cases: [{ ...scenario.cases[0], interactions: {} }], - }; - assert.throws(() => collectCases([invalid], 42), /Incomplete/); - assert.throws( - () => - collectCases( - [ - { - ...scenario, - cases: [{ ...scenario.cases[0], params: { count: Infinity } }], - }, - ], - 42, - ), - /Non-serializable/, - ); -}); - -test("watchdog bounds unresolved work and propagates cancellation", async () => { - await assert.rejects(bounded(new Promise(() => {}), 10), /watchdog/); - const abort = new AbortController(); - const operation = bounded(new Promise(() => {}), 10000, abort.signal); - abort.abort(new Error("stopped")); - await assert.rejects(operation, /stopped/); - assert.equal(await bounded(Promise.resolve(42), 100), 42); -}); - -test("invalid aggregates retain an explicit reason", () => { - assert.deepEqual( - validateMetrics({ uploadMs: 0, timeToFullMs: NaN, activeInteractionMs: 2 }), - { - uploadMs: "below-timer-resolution", - timeToFullMs: "non-finite", - }, - ); -}); - -test("help does not need a renderer build or browser", async () => { - assert.equal(await main(["--help"]), 0); -}); - -test("changed environment is stored as a failed observation", () => { - const c = { - status: "running", - attempts: [], - environment: { webgl: { renderer: "A" } }, - }; - recordAttempt( - c, - { - status: "ok", - metrics: { uploadMs: 1 }, - environment: { webgl: { renderer: "B" } }, - }, - 0, - 0, - ); - assert.equal(c.attempts[0].status, "failed"); - assert.match(c.attempts[0].error, /environment changed/); - assert.equal(c.status, "failed"); -}); - -test("fatal failure and interruption preserve partial and unattempted counts", () => { - for (const status of ["failed", "interrupted"]) { - const cases = [ - { status: "complete", attempts: [{ status: "ok" }, { status: "ok" }] }, - { status: "running", attempts: [{ status: "ok" }] }, - { status: "unattempted", attempts: [] }, - ]; - finishCases(cases, 2, status); - assert.deepEqual( - cases.map((c) => [c.status, c.unattempted]), - [ - ["complete", 0], - [status, 1], - ["unattempted", 2], - ], - ); - } -}); diff --git a/frontend/scripts/renderer-benchmarks/tests/scenes.test.js b/frontend/scripts/renderer-benchmarks/tests/scenes.test.js deleted file mode 100644 index a9a47c1e0f..0000000000 --- a/frontend/scripts/renderer-benchmarks/tests/scenes.test.js +++ /dev/null @@ -1,123 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { scenarios } from "../../../resources/wasm-playground/scenes/index.js"; -import { - serializeText, - validateCount, -} from "../../../resources/wasm-playground/scenes/shared.js"; - -test("all seven workloads reproduce the same exact scene for a seed", () => { - assert.deepEqual( - scenarios.map((scenario) => scenario.id), - ["rects", "paths", "plus", "texts", "masks", "clips", "shadows"], - ); - for (const scenario of scenarios) { - const params = scenario.cases[0].params; - assert.deepEqual( - scenario.createScene(params, 42), - scenario.createScene(params, 42), - ); - } -}); - -test("shape counts are integers before allocation", () => { - assert.equal(validateCount("1000"), 1000); - for (const value of ["", "2x", -1, 1.5, Infinity, true, null, 0, 100001]) { - assert.throws(() => validateCount(value), /count/i); - } -}); - -test("scenes reject ignored parameters and malformed text dimensions", () => { - for (const scenario of scenarios) { - assert.throws( - () => scenario.createScene({ ...scenario.cases[0].params, typo: 1 }, 42), - /Unsupported parameter/, - ); - } - const texts = scenarios.find((s) => s.id === "texts"); - for (const override of [ - { width: -1 }, - { words: [] }, - { maxLines: 1.5 }, - { minFontSize: 100 }, - ]) { - assert.throws(() => - texts.createScene({ ...texts.cases[0].params, ...override }, 42), - ); - } -}); - -test("every shape is reachable exactly once and agrees with its parent", () => { - for (const scenario of scenarios) { - const scene = scenario.createScene(scenario.cases[0].params, 123); - const key = (id) => id.join(":"); - const shapes = new Map(scene.shapes.map((shape) => [key(shape.id), shape])); - assert.equal(shapes.size, scene.shapes.length); - const seen = new Set(); - function visit(parent, children) { - for (const id of children) { - const child = shapes.get(key(id)); - assert.ok(child, `${scenario.id}: missing child`); - assert.deepEqual(child.parent, parent); - assert.ok(!seen.has(key(id)), `${scenario.id}: duplicate child`); - seen.add(key(id)); - visit(id, child.children ?? []); - } - } - visit([0, 0, 0, 0], scene.children); - assert.equal(seen.size, scene.shapes.length, scenario.id); - } -}); - -test("text payload follows the current Rust paragraph and span ABI", () => { - const bytes = serializeText("Penpot café", 24, 0xff112233); - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - const encoded = new TextEncoder().encode("Penpot café"); - assert.equal(bytes.length, 16 + 1344 + encoded.length); - assert.equal(view.getUint32(0, true), 1); - assert.equal(view.getUint8(4), 0); // Left, not Center. - assert.equal(view.getFloat32(16 + 4, true), 24); - assert.equal(view.getInt32(16 + 16, true), 400); - assert.equal(view.getUint32(16 + 56, true), encoded.length); - assert.equal(view.getUint32(16 + 60, true), 1); - assert.equal(view.getUint32(16 + 64 + 4, true), 0xff112233); - assert.deepEqual(bytes.subarray(16 + 1344), encoded); -}); - -test("clip effects use valid enum values and actual booleans", () => { - const scenario = scenarios.find(({ id }) => id === "clips"); - const scene = scenario.createScene(scenario.cases[0].params, 42); - assert.equal(scene.shapes.length, 2); - for (const shape of scene.shapes) { - if (shape.blur) assert.equal(shape.blur[1], false); - for (const shadow of shape.shadows ?? []) { - assert.ok([0, 1].includes(shadow[5])); - assert.equal(typeof shadow[6], "boolean"); - } - } -}); - -test("shadow workload retains intended effects with complete valid FFI calls", () => { - const scenario = scenarios.find(({ id }) => id === "shadows"); - const params = scenario.cases[0].params; - assert.equal(params.count, 1000); - assert.equal(params.shadows.length, 3); - assert.deepEqual( - params.shadows.map((shadow) => shadow[5]), - [0, 0, 1], - ); - const scene = scenario.createScene(params, 42); - assert.equal(scene.shapes.length, 1000); - for (const shape of scene.shapes) { - assert.deepEqual(shape.shadows, params.shadows); - for (const shadow of shape.shadows) { - assert.equal(shadow.length, 7); - assert.ok(Number.isInteger(shadow[0])); - for (const value of shadow.slice(1, 5)) { - assert.ok(Number.isFinite(value)); - } - assert.ok([0, 1].includes(shadow[5])); - assert.equal(shadow[6], false); - } - } -}); diff --git a/frontend/scripts/renderer-benchmarks/tests/statistics.test.js b/frontend/scripts/renderer-benchmarks/tests/statistics.test.js deleted file mode 100644 index 5821c595b6..0000000000 --- a/frontend/scripts/renderer-benchmarks/tests/statistics.test.js +++ /dev/null @@ -1,129 +0,0 @@ -import assert from "node:assert/strict"; -import test from "node:test"; -import { - summarize, - summarizeRun, - formatSummary, -} from "../../../resources/wasm-playground/benchmarks/statistics.js"; - -test("summaries use sample deviation and unscaled median absolute deviation", () => { - const summary = summarize([1, 2, 3, 4, 5]); - assert.equal(summary.mean, 3); - assert.equal(summary.median, 3); - assert.equal(summary.standardDeviation, Math.sqrt(2.5)); - assert.equal(summary.mad, 1); - assert.equal(summary.min, 1); - assert.equal(summary.max, 5); -}); - -test("invalid numbers are counted and zero observations survive", () => { - const summary = summarize([0, 2, NaN, Infinity, "3", null, -1]); - assert.equal(summary.n, 2); - assert.equal(summary.invalid, 5); - assert.equal(summary.median, 1); - assert.equal(summary.medianInterval, null); - assert.match(summary.suppressed.medianInterval, /10/); -}); - -test("empty and singleton observations have no invented spread", () => { - assert.equal(summarize([]).median, null); - assert.equal(summarize([]).standardDeviation, null); - assert.equal(summarize([2]).standardDeviation, null); - assert.equal(summarize([2]).mad, 0); -}); - -test("bootstrap intervals are deterministic and tail support is explicit", () => { - const values = Array.from({ length: 100 }, (_, index) => index + 1); - const first = summarize(values); - assert.deepEqual(first, summarize(values)); - assert.ok(first.medianInterval.low < first.median); - assert.ok(first.medianInterval.high > first.median); - assert.equal(first.p95, 95.05); - assert.equal(first.p99, null); - assert.match(first.suppressed.p99, /500/); - assert.equal(summarize(Array(500).fill(3)).p99, 3); -}); - -test("run summaries exclude warmups, failed attempts and invalid metrics", () => { - const result = summarizeRun({ - cases: [ - { - id: "rects/load", - status: "complete", - attempts: [ - { warmup: true, status: "ok", metrics: { render: 1000 } }, - { warmup: false, status: "ok", metrics: { render: 10, upload: 0 } }, - { - warmup: false, - status: "invalid", - metrics: { render: 0, upload: 2 }, - invalidMetrics: { render: "below resolution" }, - }, - { warmup: false, status: "failed", metrics: { render: 300 } }, - ], - }, - ], - }); - const summary = result.cases[0]; - assert.deepEqual(summary.attempts, { - measured: 3, - warmup: 1, - ok: 1, - invalid: 1, - failed: 1, - }); - assert.equal(summary.metrics.render.median, 10); - assert.equal(summary.metrics.render.invalid, 2); - assert.equal(summary.metrics.upload.median, 1); - assert.equal(summary.metrics.upload.invalid, 1); - assert.match(result.warnings.join(" "), /conditional/); -}); - -test("statistics reject unsafe bootstrap settings", () => { - assert.throws(() => summarize([1], { resamples: Infinity }), /resamples/); - assert.throws(() => summarize([1], { seed: NaN }), /seed/); -}); - -test("text summaries display the same observations and support limits as JSON", () => { - const summary = summarizeRun({ - cases: [ - { - id: "rects/load", - status: "complete", - attempts: [{ warmup: false, status: "ok", metrics: { render: 2 } }], - }, - ], - }); - const report = formatSummary(summary); - assert.match(report, /median 2\.000/); - assert.match(report, /1 valid, 0 invalid/); - assert.match(report, /Requires at least 10 valid attempts/); - assert.match(report, /2000 resamples/); -}); - -test("reports expose missing attempts and explain support limits once", () => { - const summary = summarizeRun({ - cases: [ - { - id: "rects/load", - status: "failed", - unattempted: 9, - attempts: [ - { - warmup: false, - status: "failed", - metrics: { render: 3, upload: 2 }, - }, - ], - }, - ], - }); - assert.equal(summary.cases[0].unattempted, 9); - const report = formatSummary(summary); - assert.match(report, /9 unattempted/); - assert.match(report, /1 failed/); - assert.equal( - report.split("Requires at least 10 valid attempts").length - 1, - 1, - ); -}); diff --git a/frontend/scripts/renderer-benchmarks/verify-cli.js b/frontend/scripts/renderer-benchmarks/verify-cli.js deleted file mode 100644 index 4b3d1ba9f9..0000000000 --- a/frontend/scripts/renderer-benchmarks/verify-cli.js +++ /dev/null @@ -1,163 +0,0 @@ -// Unscored end-to-end failure checks. Each child uses the real prepared build, -// server, browser, persistence and cleanup paths. No service is started here. -import assert from "node:assert/strict"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { spawn } from "node:child_process"; -import { fileURLToPath } from "node:url"; -import { bounded, main } from "../renderer-benchmarks.js"; - -const filename = fileURLToPath(import.meta.url); - -if (process.argv[2] === "--child") { - const [, , , fault, output] = process.argv; - const { chromium } = await import("playwright"); - const connect = chromium.connect.bind(chromium); - let checkpoints = 0; - chromium.connect = async (...args) => { - const browser = await connect(...args); - const newContext = browser.newContext.bind(browser); - browser.newContext = async (...contextArgs) => { - const context = await newContext(...contextArgs); - await context.exposeBinding( - "benchmarkFaultCheckpoint", - async ({ page }) => { - if (++checkpoints !== 2) return; - if (fault === "interrupt") process.kill(process.pid, "SIGTERM"); - if (fault === "browser-disconnect") - void browser.close().catch(() => {}); - if (fault === "page-crash") { - const cdp = await context.newCDPSession(page); - void cdp.send("Page.crash").catch(() => {}); - } - }, - ); - await context.addInitScript(() => { - let api; - Object.defineProperty(globalThis, "rendererBenchmark", { - get: () => api, - set(value) { - api = value; - const attempt = value.runAttempt; - value.runAttempt = async (...args) => { - await globalThis.benchmarkFaultCheckpoint(); - return attempt(...args); - }; - }, - }); - }); - return context; - }; - return browser; - }; - const args = [ - "--filter", - "rects", - "--warmups", - "0", - "--repetitions", - "2", - "--output", - output, - ]; - if (fault === "missing-server") - args.push("--base-url", "http://127.0.0.1:1/"); - if (fault === "wrong-server") - args.push("--base-url", "http://localhost:3000/not-this-workspace/"); - process.exitCode = await main(args); -} else { - const directory = await fs.mkdtemp( - path.join(os.tmpdir(), "renderer-cli-faults-"), - ); - console.log(`Diagnostic results: ${directory}`); - for (const fault of [ - "missing-browser", - "missing-server", - "wrong-server", - "page-crash", - "browser-disconnect", - "interrupt", - ]) { - const output = path.join(directory, `${fault}.json`); - const log = await fs.open(path.join(directory, `${fault}.log`), "w"); - const child = spawn( - process.execPath, - [filename, "--child", fault, output], - { - detached: true, - stdio: ["ignore", log.fd, log.fd], - env: { - ...process.env, - ...(fault === "missing-browser" - ? { - PLAYWRIGHT_BROWSERS_PATH: path.join( - directory, - "missing-browser", - ), - } - : {}), - }, - }, - ); - try { - const exit = await bounded( - new Promise((resolve, reject) => { - child.once("error", reject); - child.once("exit", (code, signal) => resolve({ code, signal })); - }), - 180000, - ); - assert.deepEqual( - exit, - { code: 1, signal: null }, - `${fault}: must exit with an execution failure`, - ); - const result = JSON.parse(await fs.readFile(output, "utf8")); - assert.ok(result.finishedAt); - assert.equal( - result.status, - fault === "interrupt" ? "interrupted" : "failed", - ); - assert.ok( - result.cases.every((c) => c.attempts.length + c.unattempted === 2), - ); - if ( - ["missing-browser", "missing-server", "wrong-server"].includes(fault) - ) { - assert.ok(result.cases.every((c) => c.status === "unattempted")); - assert.match( - result.error, - fault === "missing-browser" - ? /Chromium is missing/ - : fault === "missing-server" - ? /Static server unavailable/ - : /current benchmark assets/, - ); - } else { - assert.equal( - result.cases[0].attempts[0].status, - "ok", - "retain the successful attempt before injection", - ); - assert.equal(result.cases[0].attempts[1].status, "failed"); - if (fault === "page-crash") { - assert.ok( - result.cases.slice(1).every((c) => c.status === "complete"), - "independent cases recover in fresh contexts", - ); - } else { - assert.ok( - result.cases.slice(1).every((c) => c.status === "unattempted"), - ); - } - } - console.log(`PASS ${fault}: partial results and bounded exit (unscored)`); - } finally { - // Kill only this diagnostic child's process group if its watchdog expired. - if (child.exitCode === null && child.signalCode === null) - process.kill(-child.pid, "SIGKILL"); - await log.close(); - } - } -} diff --git a/frontend/scripts/renderer-benchmarks/verify-failures.js b/frontend/scripts/renderer-benchmarks/verify-failures.js deleted file mode 100644 index 1ee0169bc1..0000000000 --- a/frontend/scripts/renderer-benchmarks/verify-failures.js +++ /dev/null @@ -1,184 +0,0 @@ -// Unscored browser fault checks. Uses prepared assets; manages no services. -import assert from "node:assert/strict"; -import { chromium } from "playwright"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { bounded } from "../renderer-benchmarks.js"; -import { scenarios } from "../../resources/wasm-playground/scenes/index.js"; -import { collectCases } from "../../resources/wasm-playground/benchmarks/contract.js"; - -const frontend = fileURLToPath(new URL("../../", import.meta.url)); -const server = await chromium.launchServer({ - headless: true, - args: ["--enable-gpu"], -}); -const process = server.process(); -const browser = await chromium.connect(server.wsEndpoint()); -const benchmarkCase = collectCases(scenarios, 42, "rects/default/pan")[0]; -const options = { - viewport: { width: 1920, height: 1080 }, - dpr: 2, - timeoutMs: 30000, - moduleUrl: "http://benchmark.invalid/fault-module.js", -}; -try { - for (const fault of [ - "unknown-frame", - "context-loss", - "timeout", - "cancel-init", - "cancel-frame", - "blocked-wasm", - "page-crash", - "report", - ]) { - const context = await browser.newContext({ - viewport: options.viewport, - deviceScaleFactor: options.dpr, - }); - try { - await context.route("http://benchmark.invalid/**", async (route) => { - const pathname = new URL(route.request().url()).pathname; - if (pathname === "/fault-module.js") { - await route.fulfill({ - contentType: "text/javascript", - body: ` - import createModule from '/js/render-wasm-benchmark.js'; - export default async function() { - const module = await createModule(); - if (${JSON.stringify(fault)} === 'cancel-init') await new Promise(resolve => { globalThis.releaseModule = resolve; }); - const render = module._render; - module._render = (...args) => { - if (${JSON.stringify(fault)} === 'unknown-frame') return 0; - if (${JSON.stringify(fault)} === 'blocked-wasm') { while (true) {} } - if (${JSON.stringify(fault)} === 'context-loss') document.querySelector('canvas').getContext('webgl2').getExtension('WEBGL_lose_context').loseContext(); - return render(...args); - }; - return module; - }`, - }); - return; - } - const base = path.join( - frontend, - pathname.startsWith("/wasm-playground/") - ? "resources" - : "resources/public", - ); - const target = path.resolve(base, `.${pathname}`); - if (!target.startsWith(`${base}/`)) return route.abort(); - await route.fulfill({ path: target }); - }); - const page = await context.newPage(); - await page.goto( - "http://benchmark.invalid/wasm-playground/benchmark.html", - ); - await page.waitForFunction(() => globalThis.rendererBenchmarkReady); - if (fault === "timeout") - await page.evaluate(() => { - globalThis.requestAnimationFrame = () => 1; - }); - if (fault === "cancel-frame") - await page.evaluate(() => { - globalThis.requestAnimationFrame = () => { - globalThis.framePending = true; - return 1; - }; - }); - if (fault === "report") { - const report = await page.evaluate((c) => { - const value = { - scope: "Diagnostic report", - metadata: {}, - warnings: [], - cases: [ - { - ...c, - status: "complete", - unattempted: 0, - attempts: [ - { warmup: false, status: "ok", metrics: { timeToFullMs: 2 } }, - ], - }, - ], - }; - globalThis.rendererBenchmark.showResult(value); - return { - result: globalThis.rendererBenchmarkResult, - text: document.querySelector("pre").textContent, - }; - }, benchmarkCase); - assert.equal( - report.result.cases[0].attempts[0].metrics.timeToFullMs, - 2, - ); - assert.match(report.text, /median 2\.000/); - assert.match(report.text, /1 valid, 0 invalid/); - } else if (fault === "page-crash") { - const crashed = page.waitForEvent("crash"); - const cdp = await context.newCDPSession(page); - void cdp.send("Page.crash").catch(() => {}); - await bounded(crashed, 5000); - await assert.rejects( - page.evaluate(() => 1), - /crash|closed/i, - ); - } else { - // Attach rejection handling immediately; the watchdog may finish before injection. - const outcome = bounded( - page.evaluate( - ([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o), - [benchmarkCase, options], - ), - fault === "blocked-wasm" || fault === "timeout" ? 2500 : 10000, - ).then( - (value) => ({ value }), - (error) => ({ error }), - ); - if (fault === "cancel-init" || fault === "cancel-frame") { - await page.waitForFunction( - (key) => Boolean(globalThis[key]), - fault === "cancel-init" ? "releaseModule" : "framePending", - ); - await page.evaluate(() => { - globalThis.rendererBenchmark.dispose(); - globalThis.releaseModule?.(); - }); - } - const result = await outcome; - if (fault === "blocked-wasm" || fault === "timeout") - assert.match(result.error?.message ?? "", /watchdog/); - else { - assert.equal(result.error, undefined); - assert.equal(result.value.status, "failed"); - assert.match( - result.value.error, - fault === "unknown-frame" - ? /Unexpected frame type/ - : fault === "context-loss" - ? /context lost/ - : /canceled|Stale/, - ); - } - } - console.log(`PASS ${fault} (unscored)`); - } finally { - await bounded(context.close(), 5000); - } - } -} finally { - try { - await bounded(browser.close(), 5000); - } catch { - process.kill("SIGKILL"); - } - try { - await bounded(server.close(), 5000); - } catch { - process.kill("SIGKILL"); - } - assert.ok( - process.exitCode !== null || process.signalCode !== null, - "browser process must exit", - ); -} diff --git a/frontend/scripts/renderer-benchmarks/verify-pages.js b/frontend/scripts/renderer-benchmarks/verify-pages.js deleted file mode 100644 index e2f23f0471..0000000000 --- a/frontend/scripts/renderer-benchmarks/verify-pages.js +++ /dev/null @@ -1,163 +0,0 @@ -// Unscored checks of the actual playground pages, including real pointer/wheel -// events. Expose the mounted adapter only in this diagnostic routing layer. -import assert from "node:assert/strict"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { parseArgs } from "node:util"; -import { chromium } from "playwright"; -import { bounded } from "../renderer-benchmarks.js"; -import { scenarios } from "../../resources/wasm-playground/scenes/index.js"; - -const { values } = parseArgs({ - options: { - filter: { type: "string", default: "" }, - headed: { type: "boolean", default: false }, - }, -}); -const frontend = fileURLToPath(new URL("../../", import.meta.url)); -const selected = scenarios.filter((s) => s.id.includes(values.filter)); -assert.ok(selected.length, "No matching scenes"); -const server = await chromium.launchServer({ - headless: !values.headed, - args: ["--enable-gpu"], -}); -const browser = await chromium.connect(server.wsEndpoint()); -try { - for (const scenario of selected) { - const context = await browser.newContext({ - viewport: { width: 1920, height: 1080 }, - deviceScaleFactor: 2, - }); - try { - await bounded( - (async () => { - await context.route("http://benchmark.invalid/**", async (route) => { - const pathname = new URL(route.request().url()).pathname; - const base = path.join( - frontend, - pathname.startsWith("/wasm-playground/") - ? "resources" - : "resources/public", - ); - const target = path.resolve(base, `.${pathname}`); - if (!target.startsWith(`${base}/`)) return route.abort(); - if (pathname === "/wasm-playground/js/lib.js") { - const source = await fs.readFile(target, "utf8"); - await route.fulfill({ - contentType: "text/javascript", - body: - source.replace( - "export async function mountScenario(", - "async function mountOriginalScenario(", - ) + - ` - export async function mountScenario(...args) { - const adapter = await mountOriginalScenario(...args); - const render = adapter.module._render; - const view = adapter.module._set_view; - globalThis.playgroundProbe = { full: 0, views: [] }; - adapter.module._render = (...args) => { - const result = render(...args); - if (result === 2) globalThis.playgroundProbe.full++; - return result; - }; - adapter.module._set_view = (...args) => { - globalThis.playgroundProbe.views.push(args); - return view(...args); - }; - globalThis.playgroundAdapter = adapter; - return adapter; - }`, - }); - } else { - try { - await route.fulfill({ path: target }); - } catch { - await route.fulfill({ - status: 404, - body: "Missing diagnostic asset", - }); - } - } - }); - const page = await context.newPage(); - const errors = []; - page.on("pageerror", (error) => errors.push(error.message)); - page.on("console", (message) => { - if (message.type() === "error") errors.push(message.text()); - }); - await page.goto( - `http://benchmark.invalid/wasm-playground/${scenario.id}.html?seed=42`, - ); - await page.waitForFunction( - () => - globalThis.playgroundAdapter || document.querySelector("#error"), - null, - { timeout: 30000 }, - ); - assert.equal( - await page.locator("#error").count(), - 0, - "page must mount successfully", - ); - await page.mouse.move(800, 400); - await page.mouse.down(); - await page.mouse.move(860, 430, { steps: 3 }); - await page.mouse.up(); - await page.waitForFunction( - () => globalThis.playgroundProbe.full >= 1, - ); - await page.evaluate(() => { - globalThis.playgroundProbe.full = 0; - }); - await page.mouse.wheel(0, -100); - await page.waitForFunction( - () => globalThis.playgroundProbe.full >= 1, - ); - const state = await page.evaluate(() => ({ - views: globalThis.playgroundProbe.views, - lost: document - .querySelector("canvas") - .getContext("webgl2") - .isContextLost(), - })); - assert.equal(state.lost, false); - assert.ok( - state.views.some(([scale, x]) => scale === 1 && x > 0), - "pointer drag must pan", - ); - assert.ok( - state.views.some(([scale]) => scale > 1), - "wheel must zoom", - ); - assert.deepEqual(errors, []); - await bounded( - page.evaluate(() => globalThis.playgroundAdapter.dispose()), - 5000, - ); - console.log( - `PASS ${scenario.id}: legacy mount, pan, zoom and disposal (unscored)`, - ); - })(), - 65000, - ); - } catch (error) { - process.exitCode = 1; - console.error(`FAIL ${scenario.id}: ${error.stack ?? error}`); - } finally { - await bounded(context.close(), 5000); - } - } -} finally { - try { - await bounded(browser.close(), 5000); - } catch { - server.process()?.kill("SIGKILL"); - } - try { - await bounded(server.close(), 5000); - } catch { - server.process()?.kill("SIGKILL"); - } -} diff --git a/frontend/scripts/renderer-benchmarks/verify.js b/frontend/scripts/renderer-benchmarks/verify.js deleted file mode 100644 index 0e571c9796..0000000000 --- a/frontend/scripts/renderer-benchmarks/verify.js +++ /dev/null @@ -1,96 +0,0 @@ -// Unscored correctness smoke. Request routing serves local files without a service. -// Optional screenshots belong only to this diagnostic process, never scored runs. -import { chromium } from "playwright"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { parseArgs } from "node:util"; -import { scenarios } from "../../resources/wasm-playground/scenes/index.js"; -import { collectCases } from "../../resources/wasm-playground/benchmarks/contract.js"; -import { bounded } from "../renderer-benchmarks.js"; - -const { values } = parseArgs({ - options: { - screenshots: { type: "string" }, - headed: { type: "boolean", default: false }, - filter: { type: "string", default: "" }, - }, -}); -const frontend = fileURLToPath(new URL("../../", import.meta.url)); -const cases = collectCases(scenarios, 42, values.filter); -const server = await chromium.launchServer({ - headless: !values.headed, - args: ["--enable-gpu"], -}); -const browser = await chromium.connect(server.wsEndpoint()); -try { - if (values.screenshots) - await fs.mkdir(values.screenshots, { recursive: true }); - for (const c of cases) { - const context = await browser.newContext({ - viewport: { width: 1920, height: 1080 }, - deviceScaleFactor: 2, - }); - await context.route("http://benchmark.invalid/**", async (route) => { - const pathname = new URL(route.request().url()).pathname; - const base = pathname.startsWith("/wasm-playground/") - ? path.join(frontend, "resources") - : path.join(frontend, "resources/public"); - const target = path.resolve(base, `.${pathname}`); - if (!target.startsWith(`${base}/`)) return route.abort(); - try { - await route.fulfill({ path: target }); - } catch { - await route.fulfill({ status: 404, body: "Missing diagnostic asset" }); - } - }); - const page = await context.newPage(); - const errors = []; - page.on("pageerror", (error) => errors.push(error.message)); - await page.goto("http://benchmark.invalid/wasm-playground/benchmark.html"); - await page.waitForFunction( - () => globalThis.rendererBenchmarkReady === true, - null, - { timeout: 30000 }, - ); - const attempt = await bounded( - page.evaluate( - ([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o), - [ - c, - { - viewport: { width: 1920, height: 1080 }, - dpr: 2, - timeoutMs: 30000, - moduleUrl: "http://benchmark.invalid/js/render-wasm-benchmark.js", - }, - ], - ), - 65000, - ); - if (attempt.status === "failed" || errors.length) - throw new Error(`${c.id}: ${attempt.error ?? errors.join("; ")}`); - if (attempt.slices.at(-1)?.frameType !== 2) - throw new Error(`${c.id}: no Full completion`); - if (values.screenshots && c.group === "fresh-module-context") - await page.screenshot({ - path: path.join(values.screenshots, `${c.scenarioId}.png`), - }); - console.log( - `PASS ${c.id}; ${attempt.environment.webgl.renderer}; ${attempt.slices.length} render calls (unscored)`, - ); - await page.evaluate(() => globalThis.rendererBenchmark.dispose()); - await bounded(context.close(), 5000); - } -} finally { - try { - await bounded(browser.close(), 5000); - } catch { - server.process()?.kill("SIGKILL"); - } - try { - await bounded(server.close(), 5000); - } catch { - server.process()?.kill("SIGKILL"); - } -} diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index 7d6b84b46e..306ec6fe72 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -177,10 +177,27 @@ :elide-asserts true :anon-fn-naming-policy :off}}} + :performance-tests + {:target :esm + :output-dir "target/performance-tests" + :runtime :custom + :js-options {:js-provider :import} + + + :modules + {:test {:init-fn performance-tests.render-wasm.runner/init + :prepend-js ";if (typeof globalThis.navigator?.userAgent === 'undefined') { globalThis.navigator = {userAgent: ''}; };"}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + :bench-render-wasm-runner {:target :node-script - :output-to "target/renderer-benchmarks/runner.cjs" - :output-dir "target/renderer-benchmarks/runner/" + :output-to "target/performance/render-wasm/runner.cjs" + :output-dir "target/performance/render-wasm/runner/" :main benches.render-wasm.node.runner/-main :devtools {:autoload false} @@ -197,8 +214,8 @@ :bench-render-wasm-pilot {:target :node-script - :output-to "target/renderer-benchmarks/pilot.cjs" - :output-dir "target/renderer-benchmarks/pilot/" + :output-to "target/performance/render-wasm/pilot.cjs" + :output-dir "target/performance/render-wasm/pilot/" :main benches.render-wasm.node.pilot/-main :devtools {:autoload false} @@ -215,7 +232,7 @@ :bench-render-wasm-browser {:target :esm - :output-dir "target/renderer-benchmarks/browser/" + :output-dir "target/performance/render-wasm/browser/" ;; Bundle npm deps (date-fns via app.common): the pilot serves plain ;; static files with no import-map resolver. TODO(10): revisit when ;; ticket 10 owns staging; artifacts must stay self-contained. diff --git a/frontend/test/frontend_tests/benches/boolean_test.cljs b/frontend/test/performance_tests/render_wasm/boolean_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/boolean_test.cljs rename to frontend/test/performance_tests/render_wasm/boolean_test.cljs index 1679663a4a..87a3889f2e 100644 --- a/frontend/test/frontend_tests/benches/boolean_test.cljs +++ b/frontend/test/performance_tests/render_wasm/boolean_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.boolean-test +(ns performance-tests.render-wasm.boolean-test (:require [app.common.geom.point :as gpt] [app.common.geom.shapes.intersect :as gint] @@ -14,7 +14,7 @@ [benches.render-wasm.builder :as b :include-macros true] [benches.render-wasm.snapshot :as snapshot] [cljs.test :as t :include-macros true] - [frontend-tests.benches.test-helpers :as helpers])) + [performance-tests.render-wasm.test-helpers :as helpers])) (t/deftest difference-of-overlapping-rects-keeps-hierarchy-and-bounds (let [red {:fill-color "#ff0000" :fill-opacity 1} diff --git a/frontend/test/frontend_tests/benches/builder_test.cljs b/frontend/test/performance_tests/render_wasm/builder_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/builder_test.cljs rename to frontend/test/performance_tests/render_wasm/builder_test.cljs index 828801e0fe..dbb8e49cac 100644 --- a/frontend/test/frontend_tests/benches/builder_test.cljs +++ b/frontend/test/performance_tests/render_wasm/builder_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.builder-test +(ns performance-tests.render-wasm.builder-test "Tests canonical scene construction and the shared seeded number generator. Fixed random sequences preserve scene draws during generator extraction." (:require @@ -15,7 +15,7 @@ [benches.render-wasm.scenes.rects] [benches.render-wasm.snapshot :as snapshot] [cljs.test :as t :include-macros true] - [frontend-tests.benches.test-helpers :as helpers])) + [performance-tests.render-wasm.test-helpers :as helpers])) (defn- build ([] (build {})) diff --git a/frontend/test/frontend_tests/benches/containers_test.cljs b/frontend/test/performance_tests/render_wasm/containers_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/containers_test.cljs rename to frontend/test/performance_tests/render_wasm/containers_test.cljs index 3d9f842f33..cf495409d3 100644 --- a/frontend/test/frontend_tests/benches/containers_test.cljs +++ b/frontend/test/performance_tests/render_wasm/containers_test.cljs @@ -4,12 +4,12 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.containers-test +(ns performance-tests.render-wasm.containers-test (:require [app.common.uuid :as uuid] [benches.render-wasm.builder :as builder :include-macros true] [cljs.test :as t :include-macros true] - [frontend-tests.benches.test-helpers :as helpers])) + [performance-tests.render-wasm.test-helpers :as helpers])) (t/deftest frame-attaches-children-in-order (let [instance (builder/scene {:seed 1} diff --git a/frontend/test/frontend_tests/benches/contracts_test.cljs b/frontend/test/performance_tests/render_wasm/contracts_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/contracts_test.cljs rename to frontend/test/performance_tests/render_wasm/contracts_test.cljs index 80dc366be7..f2fea4ed07 100644 --- a/frontend/test/frontend_tests/benches/contracts_test.cljs +++ b/frontend/test/performance_tests/render_wasm/contracts_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.contracts-test +(ns performance-tests.render-wasm.contracts-test (:require [app.common.schema :as sm] [app.common.transit :as transit] @@ -425,7 +425,7 @@ (t/is (decl/transit-round-trips? (first collected)))) (t/is (empty? @body-calls)) (let [entry (decl/registered-case :contracts-scene/case)] - (t/is (= "frontend-tests.benches.contracts-test" (:ns entry))) + (t/is (= "performance-tests.render-wasm.contracts-test" (:ns entry))) (t/is (fn? (:run! entry))) (t/is (= :runtime ((:run! entry) :runtime))) (t/is (= [:touched] @body-calls)))) diff --git a/frontend/test/frontend_tests/benches/fingerprint_test.cljs b/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/fingerprint_test.cljs rename to frontend/test/performance_tests/render_wasm/fingerprint_test.cljs index daeaff24b9..151063bf12 100644 --- a/frontend/test/frontend_tests/benches/fingerprint_test.cljs +++ b/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.fingerprint-test +(ns performance-tests.render-wasm.fingerprint-test "Tests content identity with production records, paths and collection types. Validated scene snapshots retain child order even when maps reorder." (:require diff --git a/frontend/test/frontend_tests/benches/geometry_scenes_test.cljs b/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs similarity index 97% rename from frontend/test/frontend_tests/benches/geometry_scenes_test.cljs rename to frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs index a47697fe9d..0457391dfd 100644 --- a/frontend/test/frontend_tests/benches/geometry_scenes_test.cljs +++ b/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.geometry-scenes-test +(ns performance-tests.render-wasm.geometry-scenes-test (:require [app.common.types.path :as path] [app.common.uuid :as uuid] @@ -16,7 +16,7 @@ [benches.render-wasm.scenes.plus] [benches.render-wasm.snapshot :as snapshot] [cljs.test :as t :include-macros true] - [frontend-tests.benches.test-helpers :as helpers])) + [performance-tests.render-wasm.test-helpers :as helpers])) (t/deftest curved-paths-use-canonical-content-and-stable-order (let [params {:count 12 :width 1920 :height 1080 :seed 42} diff --git a/frontend/test/frontend_tests/benches/lifecycle_test.cljs b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs similarity index 97% rename from frontend/test/frontend_tests/benches/lifecycle_test.cljs rename to frontend/test/performance_tests/render_wasm/lifecycle_test.cljs index 80fb45a28b..d2afaf37db 100644 --- a/frontend/test/frontend_tests/benches/lifecycle_test.cljs +++ b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.lifecycle-test +(ns performance-tests.render-wasm.lifecycle-test "Tests the browser bridge with fake DOM and renderer boundaries. It covers bridge input, ownership, disposal and `load-scene` with fake @@ -253,6 +253,8 @@ prev-window (unchecked-get js/globalThis "window") prev-import (unchecked-get js/globalThis "dynamicImport") prev-performance (unchecked-get js/globalThis "performance") + prev-now-fn (unchecked-get prev-performance "now") + prev-now-desc (js/Object.getOwnPropertyDescriptor prev-performance "now") prev-now (unchecked-get js/globalThis "__benchNow") prev-script (unchecked-get js/globalThis "__benchRenderScript") prev-calls (unchecked-get js/globalThis "__benchCalls") @@ -283,19 +285,25 @@ (cb timestamp)) (cb 0))) nil)} - performance #js {:now (fn [] - (if-some [clock (unchecked-get js/globalThis "__benchNow")] - clock - (.now prev-performance)))} + fake-now (fn [] + (if-some [clock (unchecked-get js/globalThis "__benchNow")] + clock + (.call prev-now-fn prev-performance))) dynamic-import (fn [url] (js/eval (str "import(" (pr-str url) ")")))] (unchecked-set js/globalThis "document" document) (unchecked-set js/globalThis "window" window) - (unchecked-set js/globalThis "performance" performance) + ;; Patch `now` on the real object: shadow hangs the compiled + ;; `performance.*` namespaces off the global `performance` object, so + ;; replacing that global hides every `benches.render-wasm.*` var. + (js/Object.defineProperty prev-performance "now" + #js {:value fake-now :configurable true :writable true}) (unchecked-set js/globalThis "dynamicImport" dynamic-import) (fn restore! [] (unchecked-set js/globalThis "document" prev-document) (unchecked-set js/globalThis "window" prev-window) - (unchecked-set js/globalThis "performance" prev-performance) + (if (undefined? prev-now-desc) + (js-delete prev-performance "now") + (js/Object.defineProperty prev-performance "now" prev-now-desc)) (unchecked-set js/globalThis "__benchNow" prev-now) (unchecked-set js/globalThis "__benchRenderScript" prev-script) (unchecked-set js/globalThis "__benchCalls" prev-calls) diff --git a/frontend/test/frontend_tests/benches/protocol_test.cljs b/frontend/test/performance_tests/render_wasm/protocol_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/protocol_test.cljs rename to frontend/test/performance_tests/render_wasm/protocol_test.cljs index 670d91a71a..9e3ba8d0f1 100644 --- a/frontend/test/frontend_tests/benches/protocol_test.cljs +++ b/frontend/test/performance_tests/render_wasm/protocol_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.protocol-test +(ns performance-tests.render-wasm.protocol-test "Tests drain, restore and interact with an injected clock and renderer. Browser entry tests cover load and warm pan with a fake module." (:require diff --git a/frontend/test/frontend_tests/benches/report_test.cljs b/frontend/test/performance_tests/render_wasm/report_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/report_test.cljs rename to frontend/test/performance_tests/render_wasm/report_test.cljs index 89530ade26..d2254c8c40 100644 --- a/frontend/test/frontend_tests/benches/report_test.cljs +++ b/frontend/test/performance_tests/render_wasm/report_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.report-test +(ns performance-tests.render-wasm.report-test "Pure tests for raw records, exploratory statistics and saved-run comparisons. Fixtures use saved case definitions without importing a registry or renderer." (:require diff --git a/frontend/test/performance_tests/render_wasm/runner.cljs b/frontend/test/performance_tests/render_wasm/runner.cljs new file mode 100644 index 0000000000..0f1968202a --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/runner.cljs @@ -0,0 +1,195 @@ +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.runner + (:require + [app.common.logging :as l] + [cljs.test :as t] + [clojure.string :as str] + [clojure.tools.cli :refer [parse-opts]] + [goog.object :as gobj] + [performance-tests.render-wasm.boolean-test] + [performance-tests.render-wasm.builder-test] + [performance-tests.render-wasm.containers-test] + [performance-tests.render-wasm.contracts-test] + [performance-tests.render-wasm.fingerprint-test] + [performance-tests.render-wasm.geometry-scenes-test] + [performance-tests.render-wasm.lifecycle-test] + [performance-tests.render-wasm.protocol-test] + [performance-tests.render-wasm.report-test] + [performance-tests.render-wasm.scene-snapshot-test])) + +(enable-console-print!) + +(defmethod t/report [:cljs.test/default :begin-test-var] [m] + (let [v (:var m)] + (println (str " ▸ " (:ns (meta v)) "/" (:name (meta v)))))) + +(defmethod t/report [:cljs.test/default :end-run-tests] [m] + (if (cljs.test/successful? m) + (.exit js/process 0) + (.exit js/process 1))) + +(def test-namespaces + ['performance-tests.render-wasm.boolean-test + 'performance-tests.render-wasm.builder-test + 'performance-tests.render-wasm.containers-test + 'performance-tests.render-wasm.contracts-test + 'performance-tests.render-wasm.fingerprint-test + 'performance-tests.render-wasm.geometry-scenes-test + 'performance-tests.render-wasm.lifecycle-test + 'performance-tests.render-wasm.protocol-test + 'performance-tests.render-wasm.report-test + 'performance-tests.render-wasm.scene-snapshot-test]) + +(assert (every? find-ns-obj test-namespaces) + "test-namespaces contains a namespace that isn't required in runner.cljs") + +;; This runner mirrors frontend-tests.runner for the bench-only +;; :performance-tests target. Both runners need forwarded CLI args, focused +;; namespace/var execution, fixture preservation, and app log-level +;; setup. The logic stays local to avoid cross-target test classpath +;; coupling. +(def ^:private log-levels + #{:trace :debug :info :warn :error}) + +(def cli-options + [["-f" "--focus FOCUS" "Run one test namespace or one test var, e.g. performance-tests.render-wasm.contracts-test"] + ["-l" "--log-level LEVEL" "Set app logger level: trace|debug|info|warn|error" + :parse-fn keyword + :validate [log-levels "must be one of trace, debug, info, warn, error"]] + ["-h" "--help"]]) + +(defn- argv + [] + (let [args (->> (.-argv js/process) + (array-seq) + (drop 2))] + ;; `pnpm run test:performance-tests -- --focus ...` forwards the separator to + ;; the node process, so drop one leading `--` before handing args to + ;; tools.cli. + (cond-> args + (= "--" (first args)) rest))) + +(defn- usage + [summary] + (str "Usage: pnpm run test:performance-tests -- [options]\n\n" + "Options:\n" + summary "\n\n" + "Focus examples:\n" + " pnpm run test:performance-tests -- --focus performance-tests.render-wasm.contracts-test\n" + " pnpm run test:performance-tests -- --focus performance-tests.render-wasm.contracts-test/some-test-var\n\n" + "Log level example (quiets app logging during the run):\n" + " pnpm run test:performance-tests -- --focus performance-tests.render-wasm.contracts-test --log-level warn")) + +(defn- fail! + [message] + (js/console.error message) + (.exit js/process 1)) + +(defn- parse-focus + [focus] + (let [[ns-name test-name & extra] (str/split focus #"/")] + (cond + (or (str/blank? ns-name) (seq extra)) + (fail! (str "Invalid --focus value: " focus)) + + (some? test-name) + {:ns (symbol ns-name) :test test-name} + + :else + {:ns (symbol ns-name)}))) + +(defn- fixture-value + [ns-obj fixture-name] + (let [value (gobj/get ns-obj (munge fixture-name))] + (when-not (undefined? value) + value))) + +(defn- ns-test-vars + [ns-sym] + (when-let [ns-obj (find-ns-obj ns-sym)] + (->> (js-keys ns-obj) + (keep (fn [key] + (some-> (gobj/get ns-obj key) + (.-cljs$lang$var)))) + (filter (comp :test meta)) + (sort-by (comp :line meta))))) + +(defn- ns-fixtures + [ns-sym vars] + (when-let [ns-obj (find-ns-obj ns-sym)] + (let [ns-key (or (some-> vars first meta :ns) ns-sym) + once-fixtures (fixture-value ns-obj "cljs-test-once-fixtures") + each-fixtures (fixture-value ns-obj "cljs-test-each-fixtures")] + {:once (when once-fixtures {ns-key once-fixtures}) + :each (when each-fixtures {ns-key each-fixtures})}))) + +(defn- selected-tests + [{:keys [ns test]}] + (when-not (some #{ns} test-namespaces) + (fail! (str "Unknown test namespace: " ns))) + (let [vars (vec (ns-test-vars ns))] + (when (empty? vars) + (fail! (str "No tests found in namespace: " ns))) + (if test + (let [test-sym (symbol test) + test-var (some #(when (= test-sym (:name (meta %))) %) vars)] + (if test-var + {:vars [test-var] + :fixtures (ns-fixtures ns [test-var])} + (fail! (str "Unknown test var: " ns "/" test)))) + {:vars vars + :fixtures (ns-fixtures ns vars)}))) + +(defn- merge-fixtures + [fixtures] + {:once (apply merge (keep :once fixtures)) + :each (apply merge (keep :each fixtures))}) + +(defn- run-test-vars! + [tests] + (let [vars (vec (mapcat :vars tests)) + fixtures (merge-fixtures (map :fixtures tests)) + env (assoc (t/empty-env) + :once-fixtures (:once fixtures) + :each-fixtures (:each fixtures)) + summary (volatile! {:test 0 :pass 0 :fail 0 :error 0 :type :summary})] + + (t/set-env! env) + + (t/run-block + (concat (t/test-vars-block vars) + [(fn [] + (vswap! summary + (partial merge-with +) + (:report-counters (t/get-current-env)))) + (fn [] + (t/report @summary) + (t/report (assoc @summary :type :end-run-tests)))])))) + +(defn- run-focused-test! + [focus] + (run-test-vars! [(selected-tests (parse-focus focus))])) + +(defn init + [] + (let [{:keys [options errors summary]} (parse-opts (argv) cli-options)] + (cond + (seq errors) + (fail! (str/join "\n" errors)) + + (:help options) + (do + (println (usage summary)) + (.exit js/process 0)) + + :else + (do + (l/setup! {:app (or (:log-level options) :warn)}) + + (if (:focus options) + (run-focused-test! (:focus options)) + (run-test-vars! (map #(selected-tests {:ns %}) test-namespaces))))))) diff --git a/frontend/test/frontend_tests/benches/scene_snapshot_test.cljs b/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs similarity index 99% rename from frontend/test/frontend_tests/benches/scene_snapshot_test.cljs rename to frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs index 06b7d05175..08f6070de3 100644 --- a/frontend/test/frontend_tests/benches/scene_snapshot_test.cljs +++ b/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL -(ns frontend-tests.benches.scene-snapshot-test +(ns performance-tests.render-wasm.scene-snapshot-test (:require [app.common.schema :as sm] [app.common.types.shape :as cts] diff --git a/frontend/test/frontend_tests/benches/test_helpers.cljs b/frontend/test/performance_tests/render_wasm/test_helpers.cljs similarity index 96% rename from frontend/test/frontend_tests/benches/test_helpers.cljs rename to frontend/test/performance_tests/render_wasm/test_helpers.cljs index 5eec3e4bb9..4609e8f079 100644 --- a/frontend/test/frontend_tests/benches/test_helpers.cljs +++ b/frontend/test/performance_tests/render_wasm/test_helpers.cljs @@ -1,4 +1,4 @@ -(ns frontend-tests.benches.test-helpers +(ns performance-tests.render-wasm.test-helpers "Shared helpers for renderer benchmark scene tests. These helpers assert nothing about shapes or the renderer. They only