mirror of
https://github.com/penpot/penpot.git
synced 2026-10-28 21:40:34 +00:00
🚧 WIP 24 Relocate all code, delete old benchmarks
This commit is contained in:
Miguel de Benito Delgado
parent
35cd79cb03
commit
36d696c345
@ -39,7 +39,7 @@
|
||||
"^(?:backend|frontend|exporter|common)/deps.edn"
|
||||
"^(?:backend|frontend|exporter|common)/scripts/"
|
||||
"^(?:backend|frontend|exporter|common)/dev/"
|
||||
"^(?:backend|frontend|exporter|common)/test/(?!benches/|frontend_tests/benches/)"]
|
||||
"^(?:backend|frontend|exporter|common)/test/(?!performance_tests/)"]
|
||||
|
||||
:linter-name true}
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
{:paths ["src" "vendor" "resources" "test"]
|
||||
{:paths ["src" "vendor" "resources" "test" "performance"]
|
||||
:deps
|
||||
{;; Carries `app.common.render-wasm.*`, shared with the headless exporter.
|
||||
penpot/common
|
||||
@ -37,8 +37,7 @@
|
||||
|
||||
instaparse/instaparse {:mvn/version "1.5.0"}
|
||||
garden/garden {:git/url "https://github.com/noprompt/garden"
|
||||
:git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"}
|
||||
}
|
||||
:git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"}}
|
||||
|
||||
:aliases
|
||||
{:outdated
|
||||
@ -70,5 +69,4 @@
|
||||
|
||||
:renderer-bench
|
||||
{:main-opts ["-m" "shadow.cljs.devtools.cli"]
|
||||
:jvm-opts ["-Dpenpot.wasm.profile-marks=false"]}
|
||||
}}
|
||||
:jvm-opts ["-Dpenpot.wasm.profile-marks=false"]}}}
|
||||
|
||||
@ -14,12 +14,6 @@
|
||||
"url": "https://github.com/penpot/penpot"
|
||||
},
|
||||
"scripts": {
|
||||
"benchmark:renderer": "node scripts/renderer-benchmarks.js",
|
||||
"test:renderer-benchmarks": "node --test scripts/renderer-benchmarks/tests/*.test.js",
|
||||
"verify:renderer-benchmarks": "node scripts/renderer-benchmarks/verify.js",
|
||||
"verify:renderer-benchmark-failures": "node scripts/renderer-benchmarks/verify-failures.js",
|
||||
"verify:renderer-benchmark-cli": "node scripts/renderer-benchmarks/verify-cli.js",
|
||||
"verify:renderer-playground": "node scripts/renderer-benchmarks/verify-pages.js",
|
||||
"build:app:assets": "node ./scripts/build-app-assets.js",
|
||||
"build:fonts-preview": "node ./scripts/build-fonts-preview.js",
|
||||
"build:storybook": "(cd packages/ui && pnpm run build) && pnpm run build:storybook:assets && pnpm run build:storybook:cljs && storybook build",
|
||||
@ -32,19 +26,21 @@
|
||||
"build:app:main": "clojure -M:dev:shadow-cljs release main worker",
|
||||
"build:app:worker": "clojure -M:dev:shadow-cljs release worker",
|
||||
"build:app": "pnpm run clear:shadow-cache && pnpm run build:app:main && pnpm run build:app:libs",
|
||||
"check-fmt:clj": "cljfmt check --parallel=true src/ test/",
|
||||
"check-fmt:clj": "cljfmt check --parallel=true src/ performance/ test/",
|
||||
"check-fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js",
|
||||
"check-fmt:scss": "prettier -c resources/styles -c src/**/*.scss",
|
||||
"fmt:clj": "cljfmt fix --parallel=true src/ test/",
|
||||
"fmt:clj": "cljfmt fix --parallel=true src/ performance/ test/",
|
||||
"fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js -w",
|
||||
"fmt:scss": "prettier -c resources/styles -c src/**/*.scss -w",
|
||||
"lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/ test/benches/ test/frontend_tests/benches/",
|
||||
"lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/ performance/ test/performance_tests/render_wasm/",
|
||||
"lint:js": "exit 0",
|
||||
"lint:renderer-benchmarks": "clj-kondo --config-dir ../.clj-kondo --lint test/benches/ test/frontend_tests/benches/",
|
||||
"lint:performance": "clj-kondo --config-dir ../.clj-kondo --lint performance/ test/performance_tests/render_wasm/",
|
||||
"lint:scss": "pnpm exec stylelint '{src,resources}/**/*.scss'",
|
||||
"build:test": "pnpm run build:wasm && clojure -M:dev:shadow-cljs compile test",
|
||||
"build:performance-tests": "clojure -M:dev:shadow-cljs compile performance-tests",
|
||||
"test": "pnpm run build:test && node target/tests/test.js",
|
||||
"test:quiet": "node ./scripts/test-quiet.js",
|
||||
"test:performance-tests": "pnpm run build:performance-tests && node target/performance-tests/test.js",
|
||||
"test:storybook": "vitest run --project=storybook",
|
||||
"watch:test": "mkdir -p target/tests && concurrently \"clojure -M:dev:shadow-cljs watch test\" \"nodemon -C -d 2 -w target/tests --exec 'node target/tests/test.js'\"",
|
||||
"test:e2e": "playwright test --project default",
|
||||
|
||||
@ -32,7 +32,7 @@
|
||||
Follow README.md's prepared renderer release commands first.
|
||||
clojure -M:dev:renderer-bench release bench-render-wasm-browser
|
||||
pnpm run build:renderer-benchmarks:pilot
|
||||
node target/renderer-benchmarks/pilot.cjs --seed 42 --screenshot /tmp/pilot-rects.png
|
||||
node target/performance/render-wasm/pilot.cjs --seed 42 --screenshot /tmp/pilot-rects.png
|
||||
|
||||
The pilot serves the release browser bundle, so compile
|
||||
`bench-render-wasm-browser` first (after the prepared renderer build
|
||||
@ -62,10 +62,10 @@
|
||||
|
||||
(defn- frontend-root
|
||||
"Resolves the frontend checkout root from the compiled pilot location
|
||||
(`target/renderer-benchmarks/pilot.cjs`, next to `runner.cjs`). Ticket 10
|
||||
(`target/performance/render-wasm/pilot.cjs`, next to `runner.cjs`). Ticket 10
|
||||
owns command cwd resolution; the pilot pins its own roots instead."
|
||||
[]
|
||||
(path/resolve js/__dirname ".." ".."))
|
||||
(path/resolve js/__dirname ".." ".." ".."))
|
||||
|
||||
(def ^:private mime
|
||||
{".html" "text/html"
|
||||
@ -209,9 +209,9 @@
|
||||
(trace "pilot: parsing args")
|
||||
(let [opts (parse-args (or argv []))
|
||||
root (frontend-root)
|
||||
bundle (path/join root "target" "renderer-benchmarks" "browser")
|
||||
bundle (path/join root "target" "performance" "render-wasm" "browser")
|
||||
js-dir (path/join root "resources" "public" "js")
|
||||
html (path/join root "test" "benches" "render_wasm" "browser" "pilot.html")
|
||||
html (path/join root "performance" "bench" "render_wasm" "browser" "pilot.html")
|
||||
seed (:seed opts)
|
||||
selected (some #(when (= (:id %) (:case-id opts)) %)
|
||||
(cases/collect-cases {:master-seed seed}))
|
||||
@ -30,7 +30,7 @@
|
||||
|
||||
(defn- usage
|
||||
[summary]
|
||||
(str "Usage: node target/renderer-benchmarks/runner.cjs [options]\n\n"
|
||||
(str "Usage: node target/performance/render-wasm/runner.cjs [options]\n\n"
|
||||
"Options:\n"
|
||||
summary "\n\n"
|
||||
"No command is implemented yet. Upcoming tickets add:\n"
|
||||
@ -57,9 +57,9 @@
|
||||
(seq errors)
|
||||
(fail! (str/join "\n" errors))
|
||||
|
||||
;; TODO(mem:render-wasm/performance/cljs-rewrite/10-process-build-and-server,
|
||||
;; mem:render-wasm/performance/cljs-rewrite/11-runner-and-failure-accounting,
|
||||
;; mem:render-wasm/performance/cljs-rewrite/14-custom-measured-operations):
|
||||
;; TODO(mem:render-wasm/performance_tests/cljs-rewrite/10-process-build-and-server,
|
||||
;; mem:render-wasm/performance_tests/cljs-rewrite/11-runner-and-failure-accounting,
|
||||
;; mem:render-wasm/performance_tests/cljs-rewrite/14-custom-measured-operations):
|
||||
;; dispatch `run`, `ab` and `compare` instead of failing.
|
||||
(seq arguments)
|
||||
(fail! (str "Unknown command: " (str/join " " arguments) "\nTry --help"))
|
||||
@ -1,206 +0,0 @@
|
||||
# Renderer benchmarks
|
||||
|
||||
Run from `frontend/` in the prepared Penpot development container, as the normal
|
||||
`penpot` user. Dependencies, the matching Playwright Chromium installation, and
|
||||
the static server must already exist. Start the server separately with
|
||||
`node scripts/e2e-server.js`. The runner never installs packages or browsers,
|
||||
starts services, or cleans Cargo targets.
|
||||
|
||||
Use the Cargo cache prepared for that user. In this container, noninteractive
|
||||
shells inherit `CARGO_HOME=/opt/cargo`, while the populated cache is
|
||||
`/home/penpot/.cargo`. For those shells, prefix the command with
|
||||
`CARGO_HOME=/home/penpot/.cargo`. The runner preserves the supplied environment;
|
||||
it does not move caches or download missing crates.
|
||||
|
||||
```sh
|
||||
pnpm run benchmark:renderer run --output /tmp/baseline.json
|
||||
pnpm run benchmark:renderer run --filter texts --seed 42 --output /tmp/texts.json
|
||||
pnpm run benchmark:renderer run --headed --filter rects/default/zoom
|
||||
pnpm run benchmark:renderer run --no-build --filter rects/default/pan
|
||||
pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json
|
||||
pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json --diagnostic
|
||||
pnpm run test:renderer-benchmarks
|
||||
```
|
||||
|
||||
Each run builds the ordinary optimized frontend renderer once using
|
||||
`PENPOT_WASM_PREPARED=1`, offline Cargo, and the existing build/copy path. Setup is
|
||||
skipped, package-manager network fallback is disabled, and the emitted module is
|
||||
named `render-wasm-benchmark`. Playground assets are refreshed before launch. A
|
||||
per-run marker checks that the server points at this checkout. Concurrent builds
|
||||
or benchmark commands against the same checkout are unsupported. `run --no-build`
|
||||
skips the build, the Cargo manifest read, and the Emscripten/esbuild preflight
|
||||
checks, then measures the existing `render-wasm-benchmark.js`/`.wasm` pair; it
|
||||
still refreshes playground assets and checks the marker. It rejects `--features`
|
||||
and never applies to `ab`, and records `metadata.build.built: false` with a
|
||||
warning that the artifact was not rebuilt from this revision.
|
||||
|
||||
The default server URL is `http://localhost:3000`; override it with `--base-url`.
|
||||
Default JSON output goes to a unique file under the system temporary directory.
|
||||
Use `--warmups`, `--repetitions`, and `--timeout-ms` to override the provisional
|
||||
defaults of 3, 10, and 30000. Counts are fixed, not adaptive; failures are never
|
||||
silently replaced. A failed case stops its remaining attempts and records how
|
||||
many were unattempted, then later independent cases can continue. A lost browser
|
||||
ends the run and retains earlier results. Exit status reports execution validity,
|
||||
not a performance regression.
|
||||
|
||||
## Workloads and state
|
||||
|
||||
Seven versioned scenes use a seeded generator: 1000 rectangles, stars, lines, or
|
||||
shadowed rectangles; 100 text shapes; and fixed mask/clip scenes. Scene generation
|
||||
and buffer construction happen before upload timing. Text uses the renderer's
|
||||
embedded fallback font, not production font fetching. The JavaScript fixtures
|
||||
share their implementation with the interactive playground pages; `?seed=42` is
|
||||
repeatable, and numeric shape-count overrides remain available on those pages.
|
||||
|
||||
All cases run serially in one Chromium process with a 1920x1080 CSS viewport,
|
||||
DPR 2, and product WebGL2 context options. Effective sizes, context attributes,
|
||||
browser/platform identity, seeds, and workload parameters appear in JSON.
|
||||
Software WebGL is allowed and warns; a GPU launch flag does not prove hardware
|
||||
acceleration. Fresh pages do not clear browser-process, compilation, or driver
|
||||
caches.
|
||||
|
||||
- **Fresh module/context:** each attempt gets a new page/context and module.
|
||||
Module initialization starts at the Emscripten factory and includes WASM
|
||||
fetching/instantiation; the module's JavaScript import precedes timing.
|
||||
- **Context initialization:** canvas/context registration, native initialization,
|
||||
render/browser options, and viewport sizing.
|
||||
- **Upload:** direct FFI upload, pending text layouts, background, initial view,
|
||||
and initial tile-index setup. This is not the frontend's serialization or
|
||||
batched asynchronous upload path.
|
||||
- **First render:** request to `ViewportReady` and final `Full`, including rAF
|
||||
scheduling. An immediate `Full` supplies both boundaries.
|
||||
- **Warm pan/zoom:** separate case-owned instances; restore the starting view and
|
||||
finish preparation rendering before each attempt. Caches evolve within each
|
||||
case. Restoring the view is not a cache reset.
|
||||
|
||||
Interactions use declared frame paths, cached rendering, a declared 100 ms
|
||||
settling interval, and product finalization/continuation flags. The interval
|
||||
models a settling case, not every possible editor gesture. Finalization time
|
||||
starts before `_set_view_end`, and that call's duration also appears separately.
|
||||
Last-input-to-`Full` additionally includes the delay. Raw render/cached calls,
|
||||
timestamps, flags, frame types, and zero-duration slices remain available.
|
||||
|
||||
## Interpretation
|
||||
|
||||
These are renderer submission timings. They do not measure GPU completion,
|
||||
displayed pixels, input delivery, whole-editor responsiveness, network behavior,
|
||||
or visual correctness. WASM buffer size is context, not attributed CPU/GPU memory.
|
||||
|
||||
Reports retain means, medians, ranges, sample deviation, and unscaled median
|
||||
absolute deviation. Nominal 95% bootstrap intervals require 10 valid attempts;
|
||||
p95 requires 100 and p99 requires 500. Intervals are exploratory and conditional
|
||||
on these runs: serial correlation, cache history, system load, and thermal drift
|
||||
remain. Confirm important findings with repeated complete runs. No p-values or
|
||||
automated regression thresholds are provided.
|
||||
|
||||
Offline comparison checks scenario/version, seeds, resolved workloads,
|
||||
preparation, environment, and metric semantics. Source/build changes are expected.
|
||||
`--diagnostic` lists incompatibilities and permits inspection without claiming
|
||||
equivalence. Dirty worktrees warn and continue, recording only SHA and dirty
|
||||
state, not diffs or artifact hashes.
|
||||
|
||||
## Separate fixture validation
|
||||
|
||||
```sh
|
||||
pnpm run verify:renderer-benchmarks
|
||||
pnpm run verify:renderer-benchmarks --screenshots /tmp/renderer-diagnostics
|
||||
pnpm run verify:renderer-benchmark-failures
|
||||
pnpm run verify:renderer-benchmark-cli
|
||||
pnpm run verify:renderer-playground --filter rects
|
||||
```
|
||||
|
||||
This unscored smoke uses the already-built benchmark artifact and Playwright
|
||||
request routing; it starts no server. It checks every workload and interaction
|
||||
through `Full`. Optional screenshots support independent manual fixture checks.
|
||||
They never enter scored runs, and smoke durations are not benchmark evidence.
|
||||
|
||||
The failure checks inject context loss, cancellation, stalled work, and crashes.
|
||||
The CLI checks use the existing server and prepared build path, saving each
|
||||
failure's JSON and log under a printed temporary directory. The playground check
|
||||
loads the actual legacy pages and exercises pointer drag and wheel zoom. Run
|
||||
these commands separately from scored runs.
|
||||
|
||||
The runner uses the manifest's default Cargo features and records their local
|
||||
feature expansion. It rejects diagnostic defaults and disables the optional
|
||||
function-name profiling switch for scored builds.
|
||||
|
||||
## Feature A/B comparisons
|
||||
|
||||
`run --features a,b` builds and runs one configuration with explicit Cargo
|
||||
features and records it. `ab --features a,b` builds and runs every on/off
|
||||
configuration of the requested features (2^N for N features, default cap 3,
|
||||
`--max-features` 1..6), then compares the Boolean-lattice edges: pairs that
|
||||
differ in exactly one feature. Build order follows a binary-reflected Gray
|
||||
code so consecutive builds differ by one feature bit; this only minimizes
|
||||
rebuild wall time. Build durations are provenance
|
||||
(`metadata.build.durationMs`), never metrics, but non-adjacent edges are
|
||||
measured further apart in wall time, so repeated whole runs confirm findings
|
||||
that matter.
|
||||
|
||||
```sh
|
||||
pnpm run benchmark:renderer run --features branch-b --filter rects/default/pan --output /tmp/renderer-one.json
|
||||
pnpm run benchmark:renderer ab --features branch-b --filter rects/default/pan --output /tmp/renderer-ab-branch-b
|
||||
pnpm run benchmark:renderer ab --features branch-b --dry-run
|
||||
```
|
||||
|
||||
`ab` requires empty manifest defaults and aborts when a requested feature
|
||||
overlaps the enabled defaults or names a `stats*`/`profile*` diagnostic;
|
||||
`run --features` records defaults and enforces only the overlap abort.
|
||||
`--dry-run` prints the plan and builds nothing. Configurations beyond 8, or an
|
||||
empty `--filter`, require `--confirm`. Each configuration launches its own
|
||||
browser; the printed plan states this cost. `--output` for `ab` is a directory
|
||||
root that must not exist yet; `configs/<slug>.json`,
|
||||
`comparisons/<from>__<to>.json`, and `matrix.json` land beneath it.
|
||||
|
||||
Run JSON is `schemaVersion` 2: `metadata.build.built` is `false` when the run
|
||||
reused an existing artifact, and the build-derived fields below are absent, so
|
||||
comparing it with a built run fails compatibility. `metadata.build.features` is
|
||||
the explicit sorted set (the A/B identity axis), `defaultFeatures` the resolved
|
||||
manifest defaults, plus the runner-controlled `env` map and the allowlisted
|
||||
`ambientEnv` (`RUSTFLAGS`, `RUSTC_WRAPPER`, `CARGO_TARGET_DIR`, `CARGO_HOME`,
|
||||
`CC`, `CFLAGS`, `CXXFLAGS`, `LDFLAGS`, `PENPOT_WASM_FUNCTION_NAMES`,
|
||||
`CARGO_PROFILE_*`, `EMSDK*`). Comparison of the six build fields is always
|
||||
diagnostic; the expected `features` mismatch on every edge is tagged, not
|
||||
suppressed. An edge is marked `controlled: true` when the intended toggle is
|
||||
the only mismatch: the comparison measures exactly what the matrix was built
|
||||
for, and the diagnostic banner reads as single-run evidence, not as a
|
||||
verdict. Edges with other mismatches (environment, seeds, manifest defaults
|
||||
shifting mid-matrix) keep the strong diagnostic-only reading and print a
|
||||
terminal warning. `branch-a` has no observable effect because the scenes never
|
||||
call `set_modifiers`. Exit 0 means every edge has a comparison, never a
|
||||
performance verdict.
|
||||
|
||||
## Current validation limits
|
||||
|
||||
The September 17 software-rendered pilot took 25 minutes 43 seconds excluding
|
||||
the build at 3 warm-ups and 10 measured attempts. Eighteen cases completed all
|
||||
attempts; all three shadow cases failed. A later one-attempt, zero-warm-up run
|
||||
took 2 minutes 36 seconds; shadow pan and zoom still lost their WebGL contexts.
|
||||
These runs used dirty source trees and are workflow evidence, not a comparison
|
||||
of renderer changes. Shadow visual validation and default sampling remain open.
|
||||
Keep the full workloads and provisional defaults until these limits are resolved.
|
||||
|
||||
Isolated September 17 probes reproduce the shadow pattern without changing
|
||||
workloads or defaults. `verify.js --filter shadows/default/load` passes through
|
||||
`Full` on SwiftShader; `--filter shadows/default/pan` and
|
||||
`--filter shadows/default/zoom` fail with `WebGL context lost` in the warm
|
||||
interaction loop that calls `_render_from_cache` per frame. Load success records
|
||||
renderer submission only, not displayed pixels. The fixture keeps 1000 shapes
|
||||
with two drop shadows and one inner shadow, seven FFI arguments per shadow,
|
||||
style 0 for drop and 1 for inner, and a boolean hidden flag, matching
|
||||
`render-wasm/src/wasm/shadows.rs`. No fixture defect, workload reduction, or
|
||||
renderer change is claimed.
|
||||
|
||||
Sampling stays provisional at 3 warm-ups, 10 measured attempts, and 30s timeout.
|
||||
The full pilot ran 240 attempts in 1543337 ms, about 6.4 s per attempt; the
|
||||
one-attempt workflow run took 156466 ms for 21 attempts, about 7.4 s per
|
||||
attempt. The 2-5 minute goal allows roughly 18-46 attempts in total, or one to
|
||||
two per case across 21 cases, below the 10 valid attempts needed for median
|
||||
intervals. Use zero-warm-up single attempts with `--filter` for workflow
|
||||
checks; use the full provisional counts for intervals. Final calibration needs
|
||||
a stable graphics environment with passing shadows.
|
||||
|
||||
The implementation roadmap, pilot evidence, and remaining calibration work live
|
||||
in Serena memory `render-wasm/performance/core` and its numbered tickets. The
|
||||
2-5 minute full-suite goal excludes build time and is a target, not a guarantee
|
||||
on software-rendered systems.
|
||||
@ -1,31 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Renderer benchmarks</title>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
canvas {
|
||||
display: block;
|
||||
}
|
||||
pre {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
margin: 0;
|
||||
padding: 16px;
|
||||
overflow: auto;
|
||||
background: white;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<canvas aria-label="Benchmark scene"></canvas>
|
||||
<pre hidden aria-label="Benchmark result"></pre>
|
||||
<script type="module" src="./benchmarks/browser.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,246 +0,0 @@
|
||||
import { scenarios } from "../scenes/index.js";
|
||||
import { drain, interact, restore } from "./protocol.js";
|
||||
import { WEBGL_OPTIONS, validateMetrics } from "./contract.js";
|
||||
import { summarizeRun, formatSummary } from "./statistics.js";
|
||||
|
||||
let session;
|
||||
const canvas = document.querySelector("canvas");
|
||||
|
||||
function check(owner = session) {
|
||||
if (!owner || owner.canceled) throw new Error("Benchmark canceled");
|
||||
if (owner !== session) throw new Error("Stale benchmark operation");
|
||||
if (owner.lost || owner.gl?.isContextLost())
|
||||
throw new Error("WebGL context lost");
|
||||
if (performance.now() > owner.deadline) throw new Error("Operation timeout");
|
||||
}
|
||||
|
||||
function runtime() {
|
||||
const owner = session;
|
||||
return {
|
||||
module: owner.module,
|
||||
now: () => performance.now(),
|
||||
recordSlice: (slice) => owner.slices.push(slice),
|
||||
recordCached: (slice) => owner.cachedSlices.push(slice),
|
||||
check: () => {
|
||||
if (owner !== session) throw new Error("Stale benchmark operation");
|
||||
check();
|
||||
},
|
||||
frame: () =>
|
||||
new Promise((resolve, reject) => {
|
||||
owner.cancelPending = reject;
|
||||
owner.raf = requestAnimationFrame((timestamp) => {
|
||||
owner.cancelPending = null;
|
||||
owner.raf = null;
|
||||
resolve(timestamp);
|
||||
});
|
||||
}),
|
||||
sleep: (ms) =>
|
||||
new Promise((resolve, reject) => {
|
||||
owner.cancelPending = reject;
|
||||
owner.timer = setTimeout(() => {
|
||||
owner.cancelPending = null;
|
||||
resolve();
|
||||
}, ms);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function webglInfo(gl) {
|
||||
const extension = gl.getExtension("WEBGL_debug_renderer_info");
|
||||
const vendor = gl.getParameter(extension?.UNMASKED_VENDOR_WEBGL ?? gl.VENDOR);
|
||||
const renderer = gl.getParameter(
|
||||
extension?.UNMASKED_RENDERER_WEBGL ?? gl.RENDERER,
|
||||
);
|
||||
return {
|
||||
vendor,
|
||||
renderer,
|
||||
version: gl.getParameter(gl.VERSION),
|
||||
unmasked: Boolean(extension),
|
||||
acceleration: /swiftshader|llvmpipe|software|softpipe/i.test(
|
||||
`${vendor} ${renderer}`,
|
||||
)
|
||||
? "software"
|
||||
: "unknown",
|
||||
options: gl.getContextAttributes(),
|
||||
drawingBuffer: {
|
||||
width: gl.drawingBufferWidth,
|
||||
height: gl.drawingBufferHeight,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function initialize(benchmarkCase, options) {
|
||||
const scenario = scenarios.find(
|
||||
(item) => item.id === benchmarkCase.scenarioId,
|
||||
);
|
||||
if (!scenario)
|
||||
throw new Error(`Unknown scenario: ${benchmarkCase.scenarioId}`);
|
||||
// Construction and module script import deliberately precede initialization timing.
|
||||
const scene = scenario.createScene(
|
||||
benchmarkCase.params,
|
||||
benchmarkCase.sceneSeed,
|
||||
);
|
||||
const owner = {
|
||||
slices: [],
|
||||
cachedSlices: [],
|
||||
canceled: false,
|
||||
lost: false,
|
||||
deadline: performance.now() + options.timeoutMs,
|
||||
options,
|
||||
benchmarkCase,
|
||||
};
|
||||
session = owner;
|
||||
const { default: createModule } = await import(options.moduleUrl);
|
||||
check(owner);
|
||||
const moduleStart = performance.now();
|
||||
owner.module = await createModule();
|
||||
const moduleInitMs = performance.now() - moduleStart;
|
||||
check(owner);
|
||||
const contextStart = performance.now();
|
||||
canvas.style.width = `${options.viewport.width}px`;
|
||||
canvas.style.height = `${options.viewport.height}px`;
|
||||
canvas.width = Math.floor(options.viewport.width * options.dpr);
|
||||
canvas.height = Math.floor(options.viewport.height * options.dpr);
|
||||
const gl = canvas.getContext("webgl2", WEBGL_OPTIONS);
|
||||
if (!gl) throw new Error("WebGL2 is unavailable");
|
||||
owner.gl = gl;
|
||||
// Native initialization queries the unmasked renderer identity.
|
||||
gl.getExtension("WEBGL_debug_renderer_info");
|
||||
owner.onLost = (event) => {
|
||||
event.preventDefault();
|
||||
owner.lost = true;
|
||||
};
|
||||
canvas.addEventListener("webglcontextlost", owner.onLost);
|
||||
owner.handle = owner.module.GL.registerContext(gl, { majorVersion: 2 });
|
||||
owner.module.GL.makeContextCurrent(owner.handle);
|
||||
owner.module._init(options.viewport.width, options.viewport.height);
|
||||
owner.initialized = true;
|
||||
owner.module._set_render_options(0, options.dpr);
|
||||
owner.module._set_browser(1); // Rust Browser::Chrome
|
||||
owner.module._resize_viewbox(options.viewport.width, options.viewport.height);
|
||||
const contextInitMs = performance.now() - contextStart;
|
||||
// Diagnostic queries are outside renderer and upload timing.
|
||||
const environment = {
|
||||
webgl: webglInfo(gl),
|
||||
canvas: {
|
||||
cssWidth: canvas.clientWidth,
|
||||
cssHeight: canvas.clientHeight,
|
||||
width: canvas.width,
|
||||
height: canvas.height,
|
||||
},
|
||||
devicePixelRatio: globalThis.devicePixelRatio,
|
||||
};
|
||||
owner.environment = environment;
|
||||
const uploadStart = performance.now();
|
||||
scenario.upload(owner.module, scene);
|
||||
owner.module._set_canvas_background(0xfffabada);
|
||||
const view = benchmarkCase.initialView;
|
||||
owner.module._set_view(view.scale, view.x, view.y);
|
||||
owner.module._set_view_end(); // Bulk upload needs its initial tile index.
|
||||
const uploadMs = performance.now() - uploadStart;
|
||||
const first = await drain(runtime(), { flags: 0 });
|
||||
return {
|
||||
environment,
|
||||
metrics: {
|
||||
moduleInitMs,
|
||||
contextInitMs,
|
||||
uploadMs,
|
||||
timeToViewportReadyMs: first.viewportReadyMs,
|
||||
timeToFullMs: first.fullMs,
|
||||
},
|
||||
slices: first.slices,
|
||||
cachedSlices: [],
|
||||
};
|
||||
}
|
||||
|
||||
function finish(result) {
|
||||
const invalidMetrics = validateMetrics(result.metrics);
|
||||
const slices = result.slices ?? [];
|
||||
const cachedSlices = result.cachedSlices ?? [];
|
||||
const timestamps = cachedSlices.map((slice) => slice.timestamp);
|
||||
return {
|
||||
...result,
|
||||
status: Object.keys(invalidMetrics).length ? "invalid" : "ok",
|
||||
invalidMetrics,
|
||||
frameCounts: {
|
||||
Partial: slices.filter((s) => s.frameType === 1).length,
|
||||
ViewportReady: slices.filter((s) => s.frameType === 3).length,
|
||||
Full: slices.filter((s) => s.frameType === 2).length,
|
||||
},
|
||||
rafGapsMs: timestamps.slice(1).map((time, i) => time - timestamps[i]),
|
||||
renderRafGapsMs: slices
|
||||
.slice(1)
|
||||
.map((slice, i) => slice.timestamp - slices[i].timestamp),
|
||||
zeroSlices: [...slices, ...cachedSlices].filter((s) => s.durationMs === 0)
|
||||
.length,
|
||||
wasmMemoryBytes: session.module.HEAPU8.buffer.byteLength,
|
||||
};
|
||||
}
|
||||
|
||||
async function runAttempt(benchmarkCase, options) {
|
||||
try {
|
||||
let result;
|
||||
if (!session) {
|
||||
result = await initialize(benchmarkCase, options);
|
||||
if (benchmarkCase.group === "fresh-module-context") return finish(result);
|
||||
}
|
||||
session.deadline = performance.now() + options.timeoutMs;
|
||||
await restore(runtime(), benchmarkCase.initialView);
|
||||
session.slices = [];
|
||||
session.cachedSlices = [];
|
||||
session.deadline = performance.now() + options.timeoutMs;
|
||||
result = await interact(runtime(), benchmarkCase.interaction);
|
||||
return finish({
|
||||
...result,
|
||||
environment: { ...session.environment, webgl: webglInfo(session.gl) },
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
status: "failed",
|
||||
metrics: {},
|
||||
invalidMetrics: {},
|
||||
slices: session?.slices ?? [],
|
||||
cachedSlices: session?.cachedSlices ?? [],
|
||||
environment: session?.environment,
|
||||
error: String(error?.stack ?? error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function dispose() {
|
||||
if (!session) return;
|
||||
const owner = session;
|
||||
owner.canceled = true;
|
||||
owner.cancelPending?.(new Error("Benchmark canceled"));
|
||||
cancelAnimationFrame(owner.raf);
|
||||
clearTimeout(owner.timer);
|
||||
canvas.removeEventListener("webglcontextlost", owner.onLost);
|
||||
try {
|
||||
if (owner.initialized && !owner.lost) {
|
||||
owner.module._free_gpu_resources?.();
|
||||
owner.module._clean_up();
|
||||
}
|
||||
} finally {
|
||||
if (owner.handle !== undefined) owner.module.GL.deleteContext(owner.handle);
|
||||
owner.gl?.getExtension("WEBGL_lose_context")?.loseContext();
|
||||
session = null;
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.rendererBenchmark = {
|
||||
runAttempt,
|
||||
dispose,
|
||||
showResult(result) {
|
||||
globalThis.rendererBenchmarkResult = result;
|
||||
const report = document.querySelector("pre");
|
||||
report.textContent = [
|
||||
result.scope,
|
||||
...(result.warnings ?? []),
|
||||
formatSummary(result.summary ?? summarizeRun(result)),
|
||||
"Run settings:",
|
||||
JSON.stringify(result.metadata, null, 2),
|
||||
].join("\n\n");
|
||||
report.hidden = false;
|
||||
},
|
||||
};
|
||||
globalThis.rendererBenchmarkReady = true;
|
||||
@ -1,124 +0,0 @@
|
||||
export const WEBGL_OPTIONS = Object.freeze({
|
||||
antialias: false,
|
||||
depth: true,
|
||||
stencil: true,
|
||||
alpha: true,
|
||||
preserveDrawingBuffer: true,
|
||||
});
|
||||
|
||||
export function deriveSeed(masterSeed, caseId) {
|
||||
let seed = 2166136261;
|
||||
for (const character of `${masterSeed}\0${caseId}`) {
|
||||
seed = Math.imul(seed ^ character.charCodeAt(0), 16777619) >>> 0;
|
||||
}
|
||||
return seed;
|
||||
}
|
||||
|
||||
function serializable(value, path = "case") {
|
||||
if (value === null || typeof value === "string" || typeof value === "boolean")
|
||||
return;
|
||||
if (typeof value === "number" && Number.isFinite(value)) return;
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((v, i) => serializable(v, `${path}[${i}]`));
|
||||
return;
|
||||
}
|
||||
if (value && Object.getPrototypeOf(value) === Object.prototype) {
|
||||
for (const [key, v] of Object.entries(value))
|
||||
serializable(v, `${path}.${key}`);
|
||||
return;
|
||||
}
|
||||
throw new Error(`Non-serializable value: ${path}`);
|
||||
}
|
||||
|
||||
function viewValid(view) {
|
||||
return (
|
||||
view &&
|
||||
Number.isFinite(view.scale) &&
|
||||
view.scale > 0 &&
|
||||
Number.isFinite(view.x) &&
|
||||
Number.isFinite(view.y)
|
||||
);
|
||||
}
|
||||
|
||||
export function collectCases(scenarios, masterSeed, filter = "") {
|
||||
const cases = [];
|
||||
const seen = new Set();
|
||||
const scenarioIds = new Set();
|
||||
for (const scenario of scenarios) {
|
||||
if (
|
||||
!scenario.id ||
|
||||
scenarioIds.has(scenario.id) ||
|
||||
!Number.isInteger(scenario.version) ||
|
||||
typeof scenario.createScene !== "function" ||
|
||||
typeof scenario.upload !== "function" ||
|
||||
!Array.isArray(scenario.cases) ||
|
||||
!scenario.cases.length
|
||||
)
|
||||
throw new Error("Invalid or duplicate scenario");
|
||||
scenarioIds.add(scenario.id);
|
||||
for (const c of scenario.cases) {
|
||||
serializable(c);
|
||||
const baseId = `${scenario.id}/${c.id}`;
|
||||
if (!c.id || seen.has(baseId))
|
||||
throw new Error(`Missing or duplicate case ID: ${baseId}`);
|
||||
seen.add(baseId);
|
||||
if (!c.params || !viewValid(c.initialView))
|
||||
throw new Error(`Invalid parameters/view: ${baseId}`);
|
||||
for (const kind of ["pan", "zoom"]) {
|
||||
const interaction = c.interactions?.[kind];
|
||||
if (
|
||||
!interaction ||
|
||||
!Array.isArray(interaction.frames) ||
|
||||
!interaction.frames.length ||
|
||||
!interaction.frames.every(viewValid) ||
|
||||
!Number.isFinite(interaction.settleMs) ||
|
||||
interaction.settleMs < 0
|
||||
)
|
||||
throw new Error(`Incomplete ${kind} interaction: ${baseId}`);
|
||||
}
|
||||
// Validate static workloads before any build or browser measurement. Browser
|
||||
// construction still owns its descriptors and remains outside timed spans.
|
||||
scenario.createScene(c.params, deriveSeed(masterSeed, baseId));
|
||||
for (const kind of ["load", "pan", "zoom"]) {
|
||||
const id = `${baseId}/${kind}`;
|
||||
if (filter && !id.includes(filter)) continue;
|
||||
cases.push({
|
||||
id,
|
||||
scenarioId: scenario.id,
|
||||
scenarioVersion: scenario.version,
|
||||
description: scenario.description,
|
||||
sceneSeed: deriveSeed(masterSeed, baseId),
|
||||
params: c.params,
|
||||
initialView: c.initialView,
|
||||
interaction: kind === "load" ? null : c.interactions[kind],
|
||||
group: kind === "load" ? "fresh-module-context" : `warm-${kind}`,
|
||||
preparation:
|
||||
kind === "load"
|
||||
? "new-context-module; upload; set-view-end; full"
|
||||
: "case-owned-context; upload; full; restore-view-and-full-before-each-attempt; evolving-caches",
|
||||
status: "unattempted",
|
||||
attempts: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!cases.length) throw new Error(`No matching cases: ${filter}`);
|
||||
return cases;
|
||||
}
|
||||
|
||||
export function validateMetrics(metrics) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(metrics)
|
||||
.filter(
|
||||
([key, value]) =>
|
||||
!Number.isFinite(value) ||
|
||||
value < 0 ||
|
||||
(value === 0 &&
|
||||
!["setViewEndMs", "settlingRequestedMs"].includes(key)),
|
||||
)
|
||||
.map(([key, value]) => [
|
||||
key,
|
||||
Number.isFinite(value) ? "below-timer-resolution" : "non-finite",
|
||||
]),
|
||||
);
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
// Browser-independent protocol; the runtime owns the clock and cancellation.
|
||||
export async function drain(
|
||||
rt,
|
||||
{ flags = 0, origin = rt.now(), immediate = false } = {},
|
||||
) {
|
||||
const slices = [];
|
||||
let viewportReadyMs;
|
||||
do {
|
||||
rt.check();
|
||||
const timestamp = immediate ? rt.now() : await rt.frame();
|
||||
immediate = false;
|
||||
rt.check();
|
||||
const start = rt.now();
|
||||
const frameType = rt.module._render(timestamp, flags);
|
||||
const end = rt.now();
|
||||
slices.push({ timestamp, flags, frameType, durationMs: end - start });
|
||||
rt.recordSlice?.(slices.at(-1));
|
||||
rt.check();
|
||||
if (![1, 2, 3].includes(frameType))
|
||||
throw new Error(`Unexpected frame type: ${frameType}`);
|
||||
if (frameType === 3 || frameType === 2) viewportReadyMs ??= end - origin;
|
||||
if (frameType === 2)
|
||||
return { slices, viewportReadyMs, fullMs: end - origin };
|
||||
// The production scheduler passes the previous FrameType as flags.
|
||||
flags = frameType;
|
||||
} while (true);
|
||||
}
|
||||
|
||||
export async function restore(rt, view) {
|
||||
rt.check();
|
||||
rt.module._set_view_start();
|
||||
rt.module._set_view(view.scale, view.x, view.y);
|
||||
rt.module._set_view_end();
|
||||
return drain(rt, { flags: 4, immediate: true });
|
||||
}
|
||||
|
||||
export async function interact(rt, interaction) {
|
||||
rt.check();
|
||||
const start = rt.now();
|
||||
rt.module._set_view_start();
|
||||
const cachedSlices = [];
|
||||
let lastInput;
|
||||
for (const view of interaction.frames) {
|
||||
const timestamp = await rt.frame();
|
||||
rt.check();
|
||||
lastInput = rt.now();
|
||||
rt.module._set_view(view.scale, view.x, view.y);
|
||||
const cacheStart = rt.now();
|
||||
rt.module._render_from_cache(0);
|
||||
cachedSlices.push({ timestamp, durationMs: rt.now() - cacheStart });
|
||||
rt.recordCached?.(cachedSlices.at(-1));
|
||||
}
|
||||
const activeEnd = rt.now();
|
||||
await rt.sleep(interaction.settleMs);
|
||||
rt.check();
|
||||
const finalizationStart = rt.now();
|
||||
rt.module._set_view_end();
|
||||
const setViewEndMs = rt.now() - finalizationStart;
|
||||
const result = await drain(rt, {
|
||||
flags: 4,
|
||||
origin: finalizationStart,
|
||||
immediate: true,
|
||||
});
|
||||
return {
|
||||
metrics: {
|
||||
activeInteractionMs: activeEnd - start,
|
||||
settlingRequestedMs: interaction.settleMs,
|
||||
settlingActualMs: finalizationStart - activeEnd,
|
||||
setViewEndMs,
|
||||
timeToViewportReadyMs: result.viewportReadyMs,
|
||||
timeToFullMs: result.fullMs,
|
||||
lastInputToFullMs: rt.now() - lastInput,
|
||||
},
|
||||
slices: result.slices,
|
||||
cachedSlices,
|
||||
};
|
||||
}
|
||||
@ -1,274 +0,0 @@
|
||||
// Shared by the benchmark page and offline Node reports. No browser work or I/O.
|
||||
export const STATISTICAL_METHOD = Object.freeze({
|
||||
version: 1,
|
||||
name: "percentile-bootstrap-of-medians",
|
||||
confidence: 0.95,
|
||||
resamples: 2000,
|
||||
seed: 0x70656e70,
|
||||
minimumIntervalSamples: 10,
|
||||
minimumP95Samples: 100,
|
||||
minimumP99Samples: 500,
|
||||
});
|
||||
|
||||
export const STATISTICAL_WARNINGS = Object.freeze([
|
||||
"Intervals are exploratory, nominal 95%, and conditional on these observed runs; attempts can be correlated.",
|
||||
"Independent runs can drift with system load and thermal state. Confirm important changes with repeated complete runs.",
|
||||
]);
|
||||
|
||||
export function statisticalMethod(options = {}) {
|
||||
const method = { ...STATISTICAL_METHOD, ...options };
|
||||
if (
|
||||
!Number.isInteger(method.resamples) ||
|
||||
method.resamples < 100 ||
|
||||
method.resamples > 10000
|
||||
) {
|
||||
throw new Error("resamples must be an integer between 100 and 10000");
|
||||
}
|
||||
if (
|
||||
!Number.isInteger(method.seed) ||
|
||||
method.seed < 0 ||
|
||||
method.seed > 0xffffffff
|
||||
) {
|
||||
throw new Error("seed must be a uint32 integer");
|
||||
}
|
||||
// Only the resampling count and seed are configurable; support thresholds and
|
||||
// interval interpretation form part of the versioned statistical method.
|
||||
return {
|
||||
...STATISTICAL_METHOD,
|
||||
resamples: method.resamples,
|
||||
seed: method.seed,
|
||||
};
|
||||
}
|
||||
|
||||
function randomGenerator(seed) {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state = (state + 0x6d2b79f5) >>> 0;
|
||||
let value = Math.imul(state ^ (state >>> 15), 1 | state);
|
||||
value ^= value + Math.imul(value ^ (value >>> 7), 61 | value);
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
function quantile(sorted, probability) {
|
||||
if (!sorted.length) return null;
|
||||
const position = (sorted.length - 1) * probability;
|
||||
const lower = Math.floor(position);
|
||||
const upper = Math.ceil(position);
|
||||
return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower);
|
||||
}
|
||||
|
||||
function sorted(values) {
|
||||
return [...values].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function interval(values) {
|
||||
const ordered = sorted(values);
|
||||
return { low: quantile(ordered, 0.025), high: quantile(ordered, 0.975) };
|
||||
}
|
||||
|
||||
function resampleMedian(values, random) {
|
||||
const sample = Array.from(
|
||||
{ length: values.length },
|
||||
() => values[Math.floor(random() * values.length)],
|
||||
);
|
||||
return quantile(sorted(sample), 0.5);
|
||||
}
|
||||
|
||||
export function summarize(values, options = {}) {
|
||||
const method = statisticalMethod(options);
|
||||
const ordered = sorted(
|
||||
values.filter((value) => Number.isFinite(value) && value >= 0),
|
||||
);
|
||||
const n = ordered.length;
|
||||
const median = quantile(ordered, 0.5);
|
||||
const mean = n ? ordered.reduce((sum, value) => sum + value / n, 0) : null;
|
||||
const summary = {
|
||||
n,
|
||||
invalid: values.length - n,
|
||||
mean,
|
||||
median,
|
||||
min: n ? ordered[0] : null,
|
||||
max: n ? ordered[n - 1] : null,
|
||||
standardDeviation:
|
||||
n > 1
|
||||
? Math.sqrt(
|
||||
ordered.reduce(
|
||||
(sum, value) => sum + (value - mean) ** 2 / (n - 1),
|
||||
0,
|
||||
),
|
||||
)
|
||||
: null,
|
||||
mad: n
|
||||
? quantile(sorted(ordered.map((value) => Math.abs(value - median))), 0.5)
|
||||
: null,
|
||||
medianInterval: null,
|
||||
p95: null,
|
||||
p99: null,
|
||||
suppressed: {},
|
||||
};
|
||||
if (n >= method.minimumIntervalSamples) {
|
||||
const random = randomGenerator(method.seed);
|
||||
summary.medianInterval = interval(
|
||||
Array.from({ length: method.resamples }, () =>
|
||||
resampleMedian(ordered, random),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
summary.suppressed.medianInterval = `Requires at least ${method.minimumIntervalSamples} valid attempts.`;
|
||||
}
|
||||
for (const [name, threshold, probability] of [
|
||||
["p95", method.minimumP95Samples, 0.95],
|
||||
["p99", method.minimumP99Samples, 0.99],
|
||||
]) {
|
||||
if (n >= threshold) summary[name] = quantile(ordered, probability);
|
||||
else
|
||||
summary.suppressed[name] =
|
||||
`Requires at least ${threshold} valid attempts.`;
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
export function metricNames(benchmarkCase) {
|
||||
return [
|
||||
...new Set(
|
||||
benchmarkCase.attempts.flatMap((attempt) => [
|
||||
...Object.keys(attempt.metrics ?? {}),
|
||||
...Object.keys(attempt.invalidMetrics ?? {}),
|
||||
]),
|
||||
),
|
||||
].sort();
|
||||
}
|
||||
|
||||
export function observations(benchmarkCase, metric) {
|
||||
return benchmarkCase.attempts
|
||||
.filter((attempt) => !attempt.warmup)
|
||||
.map((attempt) => {
|
||||
const invalid = attempt.invalidMetrics ?? {};
|
||||
if (
|
||||
attempt.status === "failed" ||
|
||||
(attempt.status === "invalid" && !Object.keys(invalid).length) ||
|
||||
Object.hasOwn(invalid, metric)
|
||||
)
|
||||
return NaN;
|
||||
return attempt.metrics?.[metric] ?? NaN;
|
||||
});
|
||||
}
|
||||
|
||||
export function summarizeRun(result, options = {}) {
|
||||
return {
|
||||
method: statisticalMethod(options),
|
||||
warnings: [...STATISTICAL_WARNINGS],
|
||||
cases: result.cases.map((benchmarkCase) => {
|
||||
const measured = benchmarkCase.attempts.filter(
|
||||
(attempt) => !attempt.warmup,
|
||||
);
|
||||
return {
|
||||
id: benchmarkCase.id,
|
||||
status: benchmarkCase.status,
|
||||
unattempted: benchmarkCase.unattempted ?? 0,
|
||||
attempts: {
|
||||
measured: measured.length,
|
||||
warmup: benchmarkCase.attempts.length - measured.length,
|
||||
ok: measured.filter((attempt) => attempt.status === "ok").length,
|
||||
invalid: measured.filter((attempt) => attempt.status === "invalid")
|
||||
.length,
|
||||
failed: measured.filter((attempt) => attempt.status === "failed")
|
||||
.length,
|
||||
},
|
||||
metrics: Object.fromEntries(
|
||||
metricNames(benchmarkCase).map((metric) => [
|
||||
metric,
|
||||
summarize(observations(benchmarkCase, metric), options),
|
||||
]),
|
||||
),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function compareObservations(
|
||||
baselineValues,
|
||||
candidateValues,
|
||||
options = {},
|
||||
) {
|
||||
const method = statisticalMethod(options);
|
||||
const baseline = summarize(baselineValues, options);
|
||||
const candidate = summarize(candidateValues, options);
|
||||
const result = {
|
||||
baseline,
|
||||
candidate,
|
||||
absoluteChange: null,
|
||||
relativeChangePercent: null,
|
||||
absoluteInterval: null,
|
||||
relativeIntervalPercent: null,
|
||||
suppressed: {},
|
||||
};
|
||||
if (!baseline.n || !candidate.n) {
|
||||
result.suppressed.comparison = "Both runs require valid observations.";
|
||||
return result;
|
||||
}
|
||||
result.absoluteChange = candidate.median - baseline.median;
|
||||
if (baseline.median === 0)
|
||||
result.suppressed.relativeChangePercent = "Baseline median is zero.";
|
||||
else
|
||||
result.relativeChangePercent =
|
||||
(100 * result.absoluteChange) / baseline.median;
|
||||
if (
|
||||
baseline.n < method.minimumIntervalSamples ||
|
||||
candidate.n < method.minimumIntervalSamples
|
||||
) {
|
||||
result.suppressed.intervals = `Each run requires at least ${method.minimumIntervalSamples} valid attempts.`;
|
||||
return result;
|
||||
}
|
||||
const before = baselineValues.filter(
|
||||
(value) => Number.isFinite(value) && value >= 0,
|
||||
);
|
||||
const after = candidateValues.filter(
|
||||
(value) => Number.isFinite(value) && value >= 0,
|
||||
);
|
||||
const random = randomGenerator(method.seed);
|
||||
const differences = [];
|
||||
const relative = [];
|
||||
for (let index = 0; index < method.resamples; index++) {
|
||||
// Independent draws preserve the unpaired nature of separate runs.
|
||||
const first = resampleMedian(before, random);
|
||||
const second = resampleMedian(after, random);
|
||||
differences.push(second - first);
|
||||
if (first !== 0) relative.push((100 * (second - first)) / first);
|
||||
}
|
||||
result.absoluteInterval = interval(differences);
|
||||
if (baseline.median !== 0 && relative.length === method.resamples)
|
||||
result.relativeIntervalPercent = interval(relative);
|
||||
else
|
||||
result.suppressed.relativeIntervalPercent =
|
||||
"Baseline median or a resampled baseline median is zero.";
|
||||
return result;
|
||||
}
|
||||
|
||||
function display(value) {
|
||||
return value === null ? "n/a" : value.toFixed(3);
|
||||
}
|
||||
|
||||
export function formatSummary(summary) {
|
||||
const lines = [
|
||||
`Statistics: ${summary.method.name} v${summary.method.version}; ${summary.method.resamples} resamples, seed ${summary.method.seed}.`,
|
||||
...summary.warnings,
|
||||
`Median interval: Requires at least ${summary.method.minimumIntervalSamples} valid attempts; p95 requires ${summary.method.minimumP95Samples}, p99 requires ${summary.method.minimumP99Samples}. Unsupported values are n/a.`,
|
||||
];
|
||||
for (const benchmarkCase of summary.cases) {
|
||||
const counts = benchmarkCase.attempts;
|
||||
lines.push(
|
||||
`${benchmarkCase.id} (${benchmarkCase.status}): ${counts.warmup} warm-up, ${counts.ok} valid, ${counts.invalid} invalid, ${counts.failed} failed, ${benchmarkCase.unattempted} unattempted`,
|
||||
);
|
||||
for (const [name, metric] of Object.entries(benchmarkCase.metrics)) {
|
||||
lines.push(
|
||||
` ${name}: median ${display(metric.median)}, mean ${display(metric.mean)}, range ${display(metric.min)}–${display(metric.max)}, SD ${display(metric.standardDeviation)}, MAD ${display(metric.mad)}; ${metric.n} valid, ${metric.invalid} invalid`,
|
||||
);
|
||||
lines.push(
|
||||
` median interval: ${metric.medianInterval ? `[${display(metric.medianInterval.low)}, ${display(metric.medianInterval.high)}]` : "n/a"}; p95: ${display(metric.p95)}; p99: ${display(metric.p99)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
@ -1,965 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { spawn, execFileSync } from "node:child_process";
|
||||
import { parseArgs } from "node:util";
|
||||
import { scenarios } from "../resources/wasm-playground/scenes/index.js";
|
||||
import {
|
||||
collectCases,
|
||||
WEBGL_OPTIONS,
|
||||
} from "../resources/wasm-playground/benchmarks/contract.js";
|
||||
import {
|
||||
summarizeRun,
|
||||
formatSummary,
|
||||
} from "../resources/wasm-playground/benchmarks/statistics.js";
|
||||
import {
|
||||
compareRuns,
|
||||
formatComparison,
|
||||
tagExpectedFeatureDiff,
|
||||
} from "./renderer-benchmarks/compare.js";
|
||||
|
||||
const frontend = fileURLToPath(new URL("../", import.meta.url));
|
||||
const root = path.resolve(frontend, "..");
|
||||
const LIMITS =
|
||||
"Renderer submission timings only: not GPU completion, displayed pixels, whole-editor responsiveness, or visual correctness.";
|
||||
|
||||
// Runner-controlled build overrides shared by run and ab. VERSION stays out:
|
||||
// every run stamps its own runId as VERSION in the child environment.
|
||||
export const BUILD_ENV = {
|
||||
PENPOT_WASM_PREPARED: "1",
|
||||
PENPOT_WASM_FUNCTION_NAMES: "0",
|
||||
BUILD_MODE: "release",
|
||||
NODE_ENV: "production",
|
||||
RENDER_TARGET: "frontend",
|
||||
BUILD_NAME: "render-wasm-benchmark",
|
||||
CARGO_BUILD_TARGET: "wasm32-unknown-emscripten",
|
||||
COREPACK_ENABLE_NETWORK: "0",
|
||||
CARGO_NET_OFFLINE: "true",
|
||||
};
|
||||
|
||||
export function readOptions(argv) {
|
||||
const { values, positionals } = parseArgs({
|
||||
args: argv,
|
||||
allowPositionals: true,
|
||||
options: {
|
||||
help: { type: "boolean" },
|
||||
headed: { type: "boolean", default: false },
|
||||
diagnostic: { type: "boolean", default: false },
|
||||
"base-url": { type: "string", default: "http://localhost:3000" },
|
||||
output: { type: "string" },
|
||||
filter: { type: "string", default: "" },
|
||||
seed: { type: "string", default: "42" },
|
||||
warmups: { type: "string", default: "3" },
|
||||
repetitions: { type: "string", default: "10" },
|
||||
"timeout-ms": { type: "string", default: "30000" },
|
||||
features: { type: "string", default: "" },
|
||||
"max-features": { type: "string", default: "3" },
|
||||
confirm: { type: "boolean", default: false },
|
||||
"dry-run": { type: "boolean", default: false },
|
||||
"no-build": { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
const command = positionals[0] ?? "run";
|
||||
if (!["run", "compare", "ab"].includes(command))
|
||||
throw new Error(`Unknown command: ${command}`);
|
||||
const number = (name, minimum, maximum) => {
|
||||
const value = Number(values[name]);
|
||||
if (!Number.isSafeInteger(value) || value < minimum || value > maximum)
|
||||
throw new Error(`Invalid --${name}`);
|
||||
return value;
|
||||
};
|
||||
if (
|
||||
!values.help &&
|
||||
(command === "compare" ? positionals.length !== 3 : positionals.length > 1)
|
||||
)
|
||||
throw new Error(
|
||||
"Use run [options], ab --features a,b [options], or compare baseline.json candidate.json [options]",
|
||||
);
|
||||
const featureList = values.features
|
||||
.split(",")
|
||||
.map((token) => token.trim())
|
||||
.filter(Boolean);
|
||||
for (const token of featureList) {
|
||||
if (!/^[a-z0-9-_]+$/.test(token))
|
||||
throw new Error(`Invalid --features token: ${token}`);
|
||||
}
|
||||
const features = [...new Set(featureList)].sort();
|
||||
const maxFeatures = number("max-features", 1, 6);
|
||||
if (command !== "ab" && values["dry-run"])
|
||||
throw new Error("--dry-run is only valid for ab");
|
||||
if (values["no-build"] && command !== "run")
|
||||
throw new Error("--no-build is only valid for run");
|
||||
if (values["no-build"] && features.length)
|
||||
throw new Error("--no-build skips the build, so --features cannot apply");
|
||||
if (command === "ab" && !values.help) {
|
||||
if (features.some((name) => /^(stats|profile)(-|$)/.test(name)))
|
||||
throw new Error(
|
||||
"ab rejects renderer diagnostic features; record them with run --features instead",
|
||||
);
|
||||
if (features.length < 1)
|
||||
throw new Error("ab requires --features with at least one feature");
|
||||
if (features.length > maxFeatures)
|
||||
throw new Error(
|
||||
`ab requests ${features.length} features but --max-features is ${maxFeatures}`,
|
||||
);
|
||||
}
|
||||
const baseUrl = new URL(values["base-url"]);
|
||||
if (
|
||||
!["http:", "https:"].includes(baseUrl.protocol) ||
|
||||
baseUrl.username ||
|
||||
baseUrl.password
|
||||
)
|
||||
throw new Error(
|
||||
"--base-url must be an HTTP(S) server URL without credentials",
|
||||
);
|
||||
return {
|
||||
command,
|
||||
files: positionals.slice(1),
|
||||
help: values.help,
|
||||
headed: values.headed,
|
||||
diagnostic: values.diagnostic,
|
||||
baseUrl: baseUrl.href,
|
||||
output: values.output,
|
||||
filter: values.filter,
|
||||
seed: number("seed", 0, 0xffffffff),
|
||||
warmups: number("warmups", 0, 1000),
|
||||
repetitions: number("repetitions", 1, 10000),
|
||||
timeoutMs: number("timeout-ms", 100, 600000),
|
||||
features,
|
||||
maxFeatures,
|
||||
confirm: values.confirm,
|
||||
dryRun: values["dry-run"],
|
||||
noBuild: values["no-build"],
|
||||
};
|
||||
}
|
||||
|
||||
// All on/off configurations of sorted unique feature names in binary-reflected
|
||||
// Gray order, plus the Boolean-lattice edges (pairs differing in exactly one
|
||||
// feature). features[0] is the most significant bit. Pure: no I/O.
|
||||
export function planFeatureMatrix(features) {
|
||||
const configurations = [];
|
||||
for (let index = 0; index < 2 ** features.length; index++) {
|
||||
const gray = index ^ (index >> 1);
|
||||
const slug = features
|
||||
.map((_, bit) => (gray & (1 << (features.length - 1 - bit)) ? "1" : "0"))
|
||||
.join("");
|
||||
configurations.push({
|
||||
slug,
|
||||
features: features.filter((_, bit) => slug[bit] === "1"),
|
||||
});
|
||||
}
|
||||
const edges = [];
|
||||
configurations.forEach((config) => {
|
||||
features.forEach((name, bit) => {
|
||||
if (config.slug[bit] === "0") {
|
||||
edges.push({
|
||||
from: config.slug,
|
||||
to: config.slug.slice(0, bit) + "1" + config.slug.slice(bit + 1),
|
||||
toggledFeature: name,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
return { configurations, edges };
|
||||
}
|
||||
|
||||
export async function bounded(operation, milliseconds, signal) {
|
||||
let timer;
|
||||
let onAbort;
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation,
|
||||
new Promise((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() =>
|
||||
reject(new Error(`Node watchdog expired after ${milliseconds} ms`)),
|
||||
milliseconds,
|
||||
);
|
||||
}),
|
||||
new Promise((_, reject) => {
|
||||
onAbort = () => reject(signal.reason ?? new Error("Interrupted"));
|
||||
if (signal?.aborted) onAbort();
|
||||
else signal?.addEventListener("abort", onAbort, { once: true });
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
async function save(output, result) {
|
||||
await fs.mkdir(path.dirname(output), { recursive: true });
|
||||
const temporary = `${output}.tmp`;
|
||||
await fs.writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`);
|
||||
await fs.rename(temporary, output);
|
||||
}
|
||||
|
||||
function execute(command, args, options = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: "inherit", ...options });
|
||||
child.once("error", reject);
|
||||
child.once("exit", (code, signal) =>
|
||||
code === 0
|
||||
? resolve()
|
||||
: reject(new Error(`${command} failed (${signal ?? code})`)),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function requireFile(filename, instruction) {
|
||||
try {
|
||||
await fs.access(filename);
|
||||
} catch {
|
||||
throw new Error(instruction);
|
||||
}
|
||||
}
|
||||
|
||||
// Cargo metadata --no-deps reads the manifest without requiring unused targets'
|
||||
// dependencies to be cached. Only local feature aliases need expansion here.
|
||||
export function defaultFeatures(definitions) {
|
||||
const enabled = new Set();
|
||||
const visit = (name) => {
|
||||
if (enabled.has(name)) return;
|
||||
enabled.add(name);
|
||||
for (const child of definitions[name] ?? []) visit(child);
|
||||
};
|
||||
visit("default");
|
||||
if ([...enabled].some((name) => /^(stats|profile)(-|$)/.test(name)))
|
||||
throw new Error(
|
||||
"Default Cargo features enable renderer diagnostics; use a plain release manifest",
|
||||
);
|
||||
return [...enabled].sort();
|
||||
}
|
||||
|
||||
// Reads the renderer package's feature definitions from the Cargo manifest
|
||||
// without requiring unused targets' dependencies to be cached.
|
||||
export function readRendererManifest() {
|
||||
const manifest = JSON.parse(
|
||||
execFileSync(
|
||||
"cargo",
|
||||
[
|
||||
"metadata",
|
||||
"--offline",
|
||||
"--locked",
|
||||
"--no-deps",
|
||||
"--format-version",
|
||||
"1",
|
||||
],
|
||||
{ cwd: path.join(root, "render-wasm"), encoding: "utf8" },
|
||||
),
|
||||
);
|
||||
const renderer = manifest.packages.find(
|
||||
(item) => item.manifest_path === path.join(root, "render-wasm/Cargo.toml"),
|
||||
);
|
||||
if (!renderer)
|
||||
throw new Error("Renderer package missing from Cargo metadata");
|
||||
return renderer.features;
|
||||
}
|
||||
|
||||
// Validates requested explicit features against the manifest definitions.
|
||||
// Returns the resolved manifest defaults (synthetic "default" node included).
|
||||
// Pure: no I/O.
|
||||
export function validateFeatureSelection(definitions, features, command) {
|
||||
const resolved = defaultFeatures(definitions);
|
||||
const enabled = resolved.filter((name) => name !== "default");
|
||||
for (const name of features) {
|
||||
if (!Object.hasOwn(definitions, name))
|
||||
throw new Error(`Unknown Cargo feature: ${name}`);
|
||||
}
|
||||
const overlap = features.filter((name) => enabled.includes(name));
|
||||
if (overlap.length)
|
||||
throw new Error(
|
||||
`Requested features already enabled by default: ${overlap.join(",")}`,
|
||||
);
|
||||
if (command === "ab" && enabled.length)
|
||||
throw new Error(
|
||||
`ab requires an empty default feature set; found: ${enabled.join(",")}`,
|
||||
);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
const AMBIENT_ENV_EXACT = new Set([
|
||||
"RUSTFLAGS",
|
||||
"RUSTC_WRAPPER",
|
||||
"CARGO_TARGET_DIR",
|
||||
"CARGO_HOME",
|
||||
"CC",
|
||||
"CFLAGS",
|
||||
"CXXFLAGS",
|
||||
"LDFLAGS",
|
||||
"PENPOT_WASM_FUNCTION_NAMES",
|
||||
]);
|
||||
|
||||
// Records the ambient build environment allowlist. Never the full process
|
||||
// environment. Pure: no I/O.
|
||||
export function recordAmbientEnv(env = process.env) {
|
||||
const recorded = {};
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
if (
|
||||
AMBIENT_ENV_EXACT.has(key) ||
|
||||
key.startsWith("CARGO_PROFILE_") ||
|
||||
key.startsWith("EMSDK")
|
||||
) {
|
||||
recorded[key] = String(value);
|
||||
}
|
||||
}
|
||||
return recorded;
|
||||
}
|
||||
|
||||
async function preflight(options) {
|
||||
if (options.noBuild) {
|
||||
for (const artifact of [
|
||||
"render-wasm-benchmark.js",
|
||||
"render-wasm-benchmark.wasm",
|
||||
])
|
||||
await requireFile(
|
||||
path.join(frontend, "resources/public/js", artifact),
|
||||
"No existing renderer benchmark build; run once without --no-build first.",
|
||||
);
|
||||
} else {
|
||||
await requireFile(
|
||||
"/opt/emsdk/emsdk_env.sh",
|
||||
"Run inside the prepared Penpot development container (Emscripten is missing).",
|
||||
);
|
||||
await requireFile(
|
||||
path.join(root, "render-wasm/node_modules/esbuild/package.json"),
|
||||
"Prepare renderer dependencies with pnpm install in render-wasm first.",
|
||||
);
|
||||
}
|
||||
const { chromium } = await import("playwright");
|
||||
await requireFile(
|
||||
chromium.executablePath(),
|
||||
"Chromium is missing; run the frontend scripts/setup preparation command first.",
|
||||
);
|
||||
try {
|
||||
const response = await fetch(options.baseUrl, {
|
||||
signal: AbortSignal.timeout(3000),
|
||||
});
|
||||
// A static root may legitimately be absent before assets have been copied.
|
||||
if (response.status >= 500) throw new Error(`HTTP ${response.status}`);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Static server unavailable at ${options.baseUrl}; run node scripts/e2e-server.js from frontend first. ${error.message}`,
|
||||
);
|
||||
}
|
||||
return chromium;
|
||||
}
|
||||
|
||||
export function recordAttempt(benchmarkCase, attempt, index, warmups) {
|
||||
if (attempt.environment) {
|
||||
benchmarkCase.environment ??= attempt.environment;
|
||||
if (
|
||||
JSON.stringify(benchmarkCase.environment) !==
|
||||
JSON.stringify(attempt.environment)
|
||||
) {
|
||||
attempt.status = "failed";
|
||||
attempt.error = "Effective renderer environment changed within the case";
|
||||
}
|
||||
}
|
||||
benchmarkCase.attempts.push({ index, warmup: index < warmups, ...attempt });
|
||||
if (attempt.status !== "ok") benchmarkCase.status = "failed";
|
||||
}
|
||||
|
||||
export function finishCases(cases, expected, status) {
|
||||
for (const c of cases) {
|
||||
c.unattempted = expected - c.attempts.length;
|
||||
if (c.status === "running") c.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export async function run(options) {
|
||||
const abort = new AbortController();
|
||||
const interrupted = () => abort.abort(new Error("Benchmark interrupted"));
|
||||
process.once("SIGINT", interrupted);
|
||||
process.once("SIGTERM", interrupted);
|
||||
const runId = randomUUID();
|
||||
const output = path.resolve(
|
||||
options.output ??
|
||||
path.join(os.tmpdir(), `renderer-benchmarks-${runId}.json`),
|
||||
);
|
||||
const browserArgs = ["--enable-gpu"];
|
||||
const configuration = {
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
dpr: 2,
|
||||
webglOptions: WEBGL_OPTIONS,
|
||||
rendererOptions: { debug: 0, browser: 1, backgroundArgb: 0xfffabada },
|
||||
warmups: options.warmups,
|
||||
timeoutMs: options.timeoutMs,
|
||||
};
|
||||
const git = (args) =>
|
||||
execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
|
||||
const result = {
|
||||
schemaVersion: 2,
|
||||
runId,
|
||||
timestamp: new Date().toISOString(),
|
||||
status: "running",
|
||||
scope: LIMITS,
|
||||
warnings: [],
|
||||
metadata: {
|
||||
git: {
|
||||
sha: git(["rev-parse", "HEAD"]),
|
||||
dirty: Boolean(git(["status", "--porcelain"])),
|
||||
},
|
||||
build: {
|
||||
mode: "release",
|
||||
built: !options.noBuild,
|
||||
featurePolicy: "manifest-defaults",
|
||||
features: [],
|
||||
target: "frontend",
|
||||
prepared: true,
|
||||
},
|
||||
environment: {
|
||||
node: process.version,
|
||||
os: os.platform(),
|
||||
osRelease: os.release(),
|
||||
arch: os.arch(),
|
||||
cpu: os.cpus()[0]?.model ?? "unknown",
|
||||
host: os.hostname(),
|
||||
headless: !options.headed,
|
||||
browserArgs,
|
||||
},
|
||||
configuration,
|
||||
baseUrl: options.baseUrl,
|
||||
serverReachable: false,
|
||||
masterSeed: options.seed,
|
||||
repetitions: options.repetitions,
|
||||
metricSemantics:
|
||||
"renderer-boundaries-v1;module-factory-includes-wasm-fetch;upload-includes-initial-view-index;finalization-includes-set-view-end",
|
||||
},
|
||||
cases: collectCases(scenarios, options.seed, options.filter),
|
||||
};
|
||||
let server;
|
||||
let browser;
|
||||
let context;
|
||||
let page;
|
||||
let pageErrors = [];
|
||||
let suiteStart;
|
||||
const closeContext = async () => {
|
||||
if (!context) return;
|
||||
const previous = context;
|
||||
const previousPage = page;
|
||||
context = null;
|
||||
page = null;
|
||||
try {
|
||||
if (previousPage)
|
||||
await bounded(
|
||||
previousPage.evaluate(() => globalThis.rendererBenchmark?.dispose()),
|
||||
2000,
|
||||
);
|
||||
} catch {
|
||||
/* A failed or blocked page still has to be closed. */
|
||||
}
|
||||
await bounded(previous.close(), 5000);
|
||||
};
|
||||
const openPage = async () => {
|
||||
context = await browser.newContext({
|
||||
viewport: configuration.viewport,
|
||||
deviceScaleFactor: configuration.dpr,
|
||||
serviceWorkers: "block",
|
||||
});
|
||||
page = await context.newPage();
|
||||
pageErrors = [];
|
||||
page.on("pageerror", (error) => pageErrors.push(error.message));
|
||||
page.on("crash", () => pageErrors.push("Browser page crashed"));
|
||||
await page.goto(
|
||||
new URL(`wasm-playground/benchmark.html?run=${runId}`, options.baseUrl)
|
||||
.href,
|
||||
{ timeout: options.timeoutMs },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() => globalThis.rendererBenchmarkReady === true,
|
||||
null,
|
||||
{ timeout: options.timeoutMs },
|
||||
);
|
||||
};
|
||||
try {
|
||||
if (result.metadata.git.dirty) {
|
||||
result.warnings.push(
|
||||
"Dirty worktree: local development evidence, not a reproducible canonical record.",
|
||||
);
|
||||
console.warn(result.warnings.at(-1));
|
||||
}
|
||||
const chromium = await preflight(options);
|
||||
result.metadata.serverReachable = true;
|
||||
const packageJson = JSON.parse(
|
||||
await fs.readFile(
|
||||
new URL("../node_modules/playwright/package.json", import.meta.url),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
result.metadata.environment.playwright = packageJson.version;
|
||||
if (options.noBuild) {
|
||||
const warning =
|
||||
"Build skipped (--no-build): the measured artifact was not rebuilt from this revision.";
|
||||
result.warnings.push(warning);
|
||||
console.warn(warning);
|
||||
} else {
|
||||
const requested = [...(options.features ?? [])].sort();
|
||||
const resolved = validateFeatureSelection(
|
||||
readRendererManifest(),
|
||||
requested,
|
||||
options.command,
|
||||
);
|
||||
result.metadata.build.features = requested;
|
||||
result.metadata.build.defaultFeatures = resolved;
|
||||
if (requested.length) result.metadata.build.featurePolicy = "explicit";
|
||||
result.metadata.build.env = { ...BUILD_ENV };
|
||||
result.metadata.build.ambientEnv = recordAmbientEnv();
|
||||
// Disallow package-manager network fallback in a prepared run.
|
||||
const buildStart = performance.now();
|
||||
await execute(
|
||||
"./build",
|
||||
[
|
||||
"frontend",
|
||||
"--offline",
|
||||
"--locked",
|
||||
...(requested.length ? ["--features", requested.join(",")] : []),
|
||||
],
|
||||
{
|
||||
cwd: path.join(root, "render-wasm"),
|
||||
signal: abort.signal,
|
||||
env: { ...process.env, ...BUILD_ENV, VERSION: runId },
|
||||
},
|
||||
);
|
||||
result.metadata.build.durationMs = performance.now() - buildStart;
|
||||
}
|
||||
const { copyWasmPlayground } = await import("./_helpers.js");
|
||||
const previousCwd = process.cwd();
|
||||
try {
|
||||
process.chdir(frontend);
|
||||
await copyWasmPlayground();
|
||||
} finally {
|
||||
process.chdir(previousCwd);
|
||||
}
|
||||
const markerName = "benchmark-run.json";
|
||||
await fs.writeFile(
|
||||
path.join(frontend, "resources/public/wasm-playground", markerName),
|
||||
JSON.stringify({ runId }),
|
||||
);
|
||||
const marker = await fetch(
|
||||
new URL(`wasm-playground/${markerName}?run=${runId}`, options.baseUrl),
|
||||
{ signal: AbortSignal.timeout(3000), cache: "no-store" },
|
||||
);
|
||||
if (!marker.ok || (await marker.json()).runId !== runId)
|
||||
throw new Error(
|
||||
"The static server does not serve this workspace's current benchmark assets",
|
||||
);
|
||||
suiteStart = performance.now();
|
||||
server = await chromium.launchServer({
|
||||
headless: !options.headed,
|
||||
args: browserArgs,
|
||||
});
|
||||
browser = await chromium.connect(server.wsEndpoint());
|
||||
result.metadata.environment.chromium = browser.version();
|
||||
const attemptOptions = {
|
||||
...configuration,
|
||||
moduleUrl: new URL(
|
||||
`js/render-wasm-benchmark.js?run=${runId}`,
|
||||
options.baseUrl,
|
||||
).href,
|
||||
};
|
||||
for (const benchmarkCase of result.cases) {
|
||||
if (abort.signal.aborted) throw abort.signal.reason;
|
||||
console.log(`Running ${benchmarkCase.id}`);
|
||||
benchmarkCase.status = "running";
|
||||
for (
|
||||
let index = 0;
|
||||
index < options.warmups + options.repetitions;
|
||||
index++
|
||||
) {
|
||||
let attempt;
|
||||
try {
|
||||
if (!page) await bounded(openPage(), options.timeoutMs, abort.signal);
|
||||
attempt = await bounded(
|
||||
page.evaluate(
|
||||
([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o),
|
||||
[benchmarkCase, attemptOptions],
|
||||
),
|
||||
options.timeoutMs * 2 + 5000,
|
||||
abort.signal,
|
||||
);
|
||||
} catch (error) {
|
||||
attempt = {
|
||||
status: "failed",
|
||||
metrics: {},
|
||||
invalidMetrics: {},
|
||||
error: String(error.stack ?? error),
|
||||
slices: [],
|
||||
cachedSlices: [],
|
||||
};
|
||||
}
|
||||
if (pageErrors.length) {
|
||||
attempt.status = "failed";
|
||||
attempt.error = [attempt.error, ...pageErrors]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
recordAttempt(benchmarkCase, attempt, index, options.warmups);
|
||||
if (attempt.environment) {
|
||||
if (attempt.environment.webgl.acceleration === "software") {
|
||||
const warning =
|
||||
"Software WebGL: compare only equivalent renderer environments.";
|
||||
if (!result.warnings.includes(warning)) {
|
||||
result.warnings.push(warning);
|
||||
console.warn(warning);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (attempt.status !== "ok") benchmarkCase.status = "failed";
|
||||
// Persist only outside browser timing; every failed attempt remains visible.
|
||||
await save(output, result);
|
||||
if (abort.signal.aborted) {
|
||||
benchmarkCase.status = "interrupted";
|
||||
throw abort.signal.reason;
|
||||
}
|
||||
if (!browser.isConnected())
|
||||
throw new Error(
|
||||
"Browser disconnected; remaining cases were not attempted",
|
||||
);
|
||||
if (attempt.status === "failed") {
|
||||
// Cache history cannot be restored after failure: stop this case, then
|
||||
// continue independent cases. Do not silently replace failed samples.
|
||||
benchmarkCase.unattempted =
|
||||
options.warmups + options.repetitions - index - 1;
|
||||
await closeContext();
|
||||
break;
|
||||
}
|
||||
if (benchmarkCase.group === "fresh-module-context")
|
||||
await closeContext();
|
||||
}
|
||||
await closeContext();
|
||||
if (benchmarkCase.status === "running") benchmarkCase.status = "complete";
|
||||
}
|
||||
result.status = result.cases.every((c) => c.status === "complete")
|
||||
? "complete"
|
||||
: "failed";
|
||||
} catch (error) {
|
||||
result.status = abort.signal.aborted ? "interrupted" : "failed";
|
||||
result.error = String(error.stack ?? error);
|
||||
console.error(result.error);
|
||||
} finally {
|
||||
finishCases(
|
||||
result.cases,
|
||||
options.warmups + options.repetitions,
|
||||
result.status,
|
||||
);
|
||||
if (suiteStart !== undefined)
|
||||
result.metadata.suiteDurationMs = performance.now() - suiteStart;
|
||||
result.summary = summarizeRun(result);
|
||||
// Render a report once, after every scored case. Report work never separates
|
||||
// attempts or cases. The same data also goes to JSON and the terminal.
|
||||
if (browser?.isConnected() && !abort.signal.aborted) {
|
||||
try {
|
||||
if (!page) await bounded(openPage(), 5000);
|
||||
await bounded(
|
||||
page.evaluate(
|
||||
(value) => globalThis.rendererBenchmark.showResult(value),
|
||||
result,
|
||||
),
|
||||
5000,
|
||||
);
|
||||
} catch (error) {
|
||||
result.warnings.push(`Page report unavailable: ${error.message}`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await closeContext();
|
||||
} catch {
|
||||
/* Browser cleanup below also closes blocked pages. */
|
||||
}
|
||||
// The browser server exposes its child process for a bounded hard cleanup.
|
||||
try {
|
||||
if (browser) await bounded(browser.close(), 5000);
|
||||
} catch {
|
||||
server?.process()?.kill("SIGKILL");
|
||||
}
|
||||
try {
|
||||
if (server) await bounded(server.close(), 5000);
|
||||
} catch {
|
||||
server?.process()?.kill("SIGKILL");
|
||||
}
|
||||
process.removeListener("SIGINT", interrupted);
|
||||
process.removeListener("SIGTERM", interrupted);
|
||||
result.finishedAt = new Date().toISOString();
|
||||
await save(output, result);
|
||||
}
|
||||
console.log(LIMITS);
|
||||
console.log(formatSummary(result.summary));
|
||||
console.log(`Result: ${output}`);
|
||||
return result.status === "complete" ? 0 : 1;
|
||||
}
|
||||
|
||||
// Classifies one configuration output for the feature-matrix loop: whether the
|
||||
// matrix continues with the next configuration or stops and marks the rest
|
||||
// unattempted. Takes the parsed config JSON, or null when the file is missing
|
||||
// or unparseable. Pure: no I/O.
|
||||
export function classifyConfigResult(result) {
|
||||
if (result === null || result === undefined)
|
||||
return {
|
||||
outcome: "stop",
|
||||
status: "unattempted",
|
||||
error: "configuration output is missing or invalid",
|
||||
};
|
||||
if (result.status === "interrupted")
|
||||
return { outcome: "stop", status: "interrupted" };
|
||||
if (typeof result.metadata?.build?.durationMs !== "number")
|
||||
return {
|
||||
outcome: "stop",
|
||||
status: "failed",
|
||||
error: "build or preflight failed before any suite ran",
|
||||
};
|
||||
const attempts = (result.cases ?? []).reduce(
|
||||
(sum, benchmarkCase) => sum + (benchmarkCase.attempts?.length ?? 0),
|
||||
0,
|
||||
);
|
||||
if (attempts === 0)
|
||||
return {
|
||||
outcome: "stop",
|
||||
status: "failed",
|
||||
error: "suite recorded no attempts",
|
||||
};
|
||||
return {
|
||||
outcome: "continue",
|
||||
status: result.status === "complete" ? "complete" : "failed",
|
||||
};
|
||||
}
|
||||
|
||||
// Refuses an existing feature-matrix output root: it must be absent, or an
|
||||
// empty directory. Never deletes or overwrites previous matrix output.
|
||||
async function ensureFreshAbOutputRoot(outputRoot) {
|
||||
let stat;
|
||||
try {
|
||||
stat = await fs.stat(outputRoot);
|
||||
} catch (error) {
|
||||
if (error.code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
if (!stat.isDirectory())
|
||||
throw new Error(`ab output root already exists: ${outputRoot}`);
|
||||
if ((await fs.readdir(outputRoot)).length)
|
||||
throw new Error(
|
||||
`ab output root already exists and is not empty: ${outputRoot}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Builds and runs every on/off configuration of the requested features, then
|
||||
// compares the Boolean-lattice edges. Each configuration runs in Gray order
|
||||
// with its own build and browser. Diagnostic tooling only: exit 0 reports
|
||||
// that every edge has a comparison, never a performance verdict. An
|
||||
// interruption between configurations or edges still writes matrix.json and
|
||||
// exits 1. Callers may inject an AbortSignal; otherwise ab watches SIGINT and
|
||||
// SIGTERM itself.
|
||||
export async function ab(
|
||||
options,
|
||||
{
|
||||
runFn = run,
|
||||
compareFn = compareRuns,
|
||||
manifestFn = readRendererManifest,
|
||||
signal = null,
|
||||
} = {},
|
||||
) {
|
||||
const abId = randomUUID();
|
||||
const outputRoot = options.output
|
||||
? path.resolve(options.output)
|
||||
: path.join(
|
||||
os.tmpdir(),
|
||||
`renderer-ab-${new Date().toISOString().replaceAll(":", "-")}-${abId.slice(0, 8)}`,
|
||||
);
|
||||
await ensureFreshAbOutputRoot(outputRoot);
|
||||
const definitions = await manifestFn();
|
||||
validateFeatureSelection(definitions, options.features, "ab");
|
||||
const { configurations, edges } = planFeatureMatrix(options.features);
|
||||
const cases = collectCases(scenarios, options.seed, options.filter);
|
||||
const attemptsPerCase = options.warmups + options.repetitions;
|
||||
const plan = [
|
||||
`Feature matrix: ${options.features.join(",")}`,
|
||||
...configurations.map(
|
||||
(config) => ` ${config.slug}: [${config.features.join(",")}]`,
|
||||
),
|
||||
...edges.map(
|
||||
(edge) => ` ${edge.from}__${edge.to}: ${edge.toggledFeature}`,
|
||||
),
|
||||
`${cases.length} cases, ${attemptsPerCase} attempts per case, one browser per configuration.`,
|
||||
].join("\n");
|
||||
console.log(plan);
|
||||
if (options.dryRun) return 0;
|
||||
if ((configurations.length > 8 || options.filter === "") && !options.confirm)
|
||||
throw new Error(
|
||||
"ab plans more work than confirmed; rerun with --confirm to proceed",
|
||||
);
|
||||
let watched = signal;
|
||||
let releaseSignal = null;
|
||||
if (watched === null) {
|
||||
const abort = new AbortController();
|
||||
watched = abort.signal;
|
||||
const interrupt = () => abort.abort(new Error("Benchmark interrupted"));
|
||||
process.once("SIGINT", interrupt);
|
||||
process.once("SIGTERM", interrupt);
|
||||
releaseSignal = () => {
|
||||
process.removeListener("SIGINT", interrupt);
|
||||
process.removeListener("SIGTERM", interrupt);
|
||||
};
|
||||
}
|
||||
await fs.mkdir(path.join(outputRoot, "configs"), { recursive: true });
|
||||
await fs.mkdir(path.join(outputRoot, "comparisons"), { recursive: true });
|
||||
const parsed = new Map();
|
||||
const configurationRecords = [];
|
||||
let stopped = false;
|
||||
const unattemptedRecord = (config) => ({
|
||||
slug: config.slug,
|
||||
features: config.features,
|
||||
status: "unattempted",
|
||||
file: path.join(outputRoot, "configs", `${config.slug}.json`),
|
||||
runId: null,
|
||||
runStatus: "unattempted",
|
||||
});
|
||||
for (const config of configurations) {
|
||||
if (watched.aborted) {
|
||||
configurationRecords.push({
|
||||
...unattemptedRecord(config),
|
||||
error: "Benchmark interrupted",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (stopped) {
|
||||
configurationRecords.push(unattemptedRecord(config));
|
||||
continue;
|
||||
}
|
||||
const configPath = path.join(outputRoot, "configs", `${config.slug}.json`);
|
||||
await runFn({
|
||||
...options,
|
||||
command: "ab",
|
||||
features: config.features,
|
||||
output: configPath,
|
||||
});
|
||||
let result = null;
|
||||
try {
|
||||
result = JSON.parse(await fs.readFile(configPath, "utf8"));
|
||||
} catch {
|
||||
result = null;
|
||||
}
|
||||
const verdict = classifyConfigResult(result);
|
||||
if (result !== null) parsed.set(config.slug, result);
|
||||
configurationRecords.push({
|
||||
slug: config.slug,
|
||||
features: config.features,
|
||||
status: verdict.status,
|
||||
file: configPath,
|
||||
runId: result?.runId ?? null,
|
||||
runStatus: result?.status ?? "unattempted",
|
||||
...(verdict.error ? { error: verdict.error } : {}),
|
||||
});
|
||||
console.log(`${config.slug}: ${verdict.status}`);
|
||||
if (verdict.outcome === "stop") stopped = true;
|
||||
}
|
||||
const comparisonRecords = [];
|
||||
for (const edge of edges) {
|
||||
if (watched.aborted) {
|
||||
comparisonRecords.push({ ...edge, status: "not-attempted", file: null });
|
||||
continue;
|
||||
}
|
||||
const from = parsed.get(edge.from);
|
||||
const to = parsed.get(edge.to);
|
||||
if (!from || !to) {
|
||||
comparisonRecords.push({ ...edge, status: "not-attempted", file: null });
|
||||
continue;
|
||||
}
|
||||
const comparison = tagExpectedFeatureDiff(
|
||||
await compareFn(from, to, { diagnostic: true }),
|
||||
edge.toggledFeature,
|
||||
);
|
||||
// An edge is controlled when the intended feature toggle is the only
|
||||
// mismatch. Anything else (environment, seeds, manifest defaults shifting
|
||||
// mid-matrix) keeps the strong diagnostic-only reading.
|
||||
const controlled =
|
||||
comparison.mismatches?.every((entry) => entry.expected === true) ?? false;
|
||||
const comparisonPath = path.join(
|
||||
outputRoot,
|
||||
"comparisons",
|
||||
`${edge.from}__${edge.to}.json`,
|
||||
);
|
||||
await save(comparisonPath, { ...comparison, edge, controlled });
|
||||
console.log(`${edge.from}__${edge.to}: ${comparisonPath}`);
|
||||
if (!controlled)
|
||||
console.warn(
|
||||
`${edge.from}__${edge.to}: unexpected mismatches beyond the intended feature toggle`,
|
||||
);
|
||||
comparisonRecords.push({
|
||||
...edge,
|
||||
status: "compared",
|
||||
file: comparisonPath,
|
||||
compatible: comparison.compatible,
|
||||
diagnostic: comparison.diagnostic,
|
||||
controlled,
|
||||
});
|
||||
}
|
||||
const status =
|
||||
configurationRecords.every((record) =>
|
||||
["complete", "failed"].includes(record.status),
|
||||
) && comparisonRecords.every((record) => record.status === "compared")
|
||||
? "complete"
|
||||
: "failed";
|
||||
await save(path.join(outputRoot, "matrix.json"), {
|
||||
schemaVersion: 1,
|
||||
abId,
|
||||
timestamp: new Date().toISOString(),
|
||||
features: options.features,
|
||||
maxFeatures: options.maxFeatures,
|
||||
filter: options.filter,
|
||||
outputRoot,
|
||||
configurations: configurationRecords,
|
||||
comparisons: comparisonRecords,
|
||||
status,
|
||||
warnings: [LIMITS],
|
||||
});
|
||||
console.log(`Matrix: ${path.join(outputRoot, "matrix.json")}`);
|
||||
// ab runs once per process; release before the single return. An unexpected
|
||||
// throw past this point exits through main's handler with the listeners.
|
||||
releaseSignal?.();
|
||||
return status === "complete" ? 0 : 1;
|
||||
}
|
||||
|
||||
export async function main(argv = process.argv.slice(2)) {
|
||||
const options = readOptions(argv);
|
||||
if (options.help) {
|
||||
console.log(
|
||||
"Renderer benchmarks\n node scripts/renderer-benchmarks.js run [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--features a,b] [--output FILE] [--no-build]\n node scripts/renderer-benchmarks.js ab --features a,b [--max-features N] [--confirm] [--dry-run] [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--output DIR]\n node scripts/renderer-benchmarks.js compare BASELINE CANDIDATE [--diagnostic] [--output FILE]\nRun requires a prepared development container and an existing static server on port 3000. --no-build reuses the existing renderer artifact instead of building and cannot be combined with --features or ab. Defaults: 3 warmups, 10 measured attempts, 30s timeout; provisional pending calibration.\n" +
|
||||
LIMITS,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
if (options.command === "compare") {
|
||||
const results = await Promise.all(
|
||||
options.files.map(async (file) =>
|
||||
JSON.parse(await fs.readFile(file, "utf8")),
|
||||
),
|
||||
);
|
||||
const comparison = compareRuns(...results, {
|
||||
diagnostic: options.diagnostic,
|
||||
});
|
||||
console.log(LIMITS);
|
||||
console.log(formatComparison(comparison));
|
||||
if (options.output) await save(path.resolve(options.output), comparison);
|
||||
return 0;
|
||||
}
|
||||
if (options.command === "ab") return ab(options);
|
||||
return run(options);
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
|
||||
) {
|
||||
main()
|
||||
.then((code) => {
|
||||
process.exitCode = code;
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@ -1,323 +0,0 @@
|
||||
import {
|
||||
compareObservations,
|
||||
metricNames,
|
||||
observations,
|
||||
statisticalMethod,
|
||||
STATISTICAL_WARNINGS,
|
||||
} from "../../resources/wasm-playground/benchmarks/statistics.js";
|
||||
|
||||
function record(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function requireValid(condition, message) {
|
||||
if (!condition) throw new Error(`Invalid benchmark result: ${message}`);
|
||||
}
|
||||
|
||||
export function validateResult(result) {
|
||||
requireValid(
|
||||
record(result) && result.schemaVersion === 2,
|
||||
"expected schemaVersion 2",
|
||||
);
|
||||
requireValid(record(result.metadata), "metadata is required");
|
||||
for (const field of ["environment", "configuration"])
|
||||
requireValid(
|
||||
record(result.metadata[field]),
|
||||
`metadata.${field} is required`,
|
||||
);
|
||||
requireValid(Array.isArray(result.cases), "cases must be an array");
|
||||
const ids = new Set();
|
||||
for (const benchmarkCase of result.cases) {
|
||||
requireValid(record(benchmarkCase), "case must be an object");
|
||||
requireValid(
|
||||
typeof benchmarkCase.id === "string" &&
|
||||
benchmarkCase.id.length &&
|
||||
!ids.has(benchmarkCase.id),
|
||||
"case IDs must be unique nonempty strings",
|
||||
);
|
||||
ids.add(benchmarkCase.id);
|
||||
for (const field of ["scenarioId", "group", "preparation"])
|
||||
requireValid(
|
||||
typeof benchmarkCase[field] === "string" && benchmarkCase[field].length,
|
||||
`${benchmarkCase.id}.${field} is required`,
|
||||
);
|
||||
requireValid(
|
||||
Number.isInteger(benchmarkCase.scenarioVersion) &&
|
||||
benchmarkCase.scenarioVersion >= 1,
|
||||
"scenarioVersion must be a positive integer",
|
||||
);
|
||||
requireValid(
|
||||
Number.isInteger(benchmarkCase.sceneSeed) &&
|
||||
benchmarkCase.sceneSeed >= 0 &&
|
||||
benchmarkCase.sceneSeed <= 0xffffffff,
|
||||
"sceneSeed must be a uint32",
|
||||
);
|
||||
for (const field of ["params", "initialView"])
|
||||
requireValid(
|
||||
record(benchmarkCase[field]),
|
||||
`${benchmarkCase.id}.${field} is required`,
|
||||
);
|
||||
requireValid(
|
||||
benchmarkCase.interaction === null || record(benchmarkCase.interaction),
|
||||
"interaction must be an object or null",
|
||||
);
|
||||
requireValid(
|
||||
["complete", "failed", "interrupted", "unattempted"].includes(
|
||||
benchmarkCase.status,
|
||||
),
|
||||
"unknown case status",
|
||||
);
|
||||
requireValid(
|
||||
Array.isArray(benchmarkCase.attempts),
|
||||
"attempts must be an array",
|
||||
);
|
||||
for (const attempt of benchmarkCase.attempts) {
|
||||
requireValid(
|
||||
record(attempt) && ["ok", "invalid", "failed"].includes(attempt.status),
|
||||
"unknown attempt status",
|
||||
);
|
||||
requireValid(
|
||||
typeof attempt.warmup === "boolean",
|
||||
"attempt.warmup must be boolean",
|
||||
);
|
||||
requireValid(
|
||||
record(attempt.metrics),
|
||||
"attempt.metrics must be an object",
|
||||
);
|
||||
requireValid(
|
||||
attempt.invalidMetrics === undefined || record(attempt.invalidMetrics),
|
||||
"invalidMetrics must be an object",
|
||||
);
|
||||
for (const [name, value] of Object.entries(attempt.metrics))
|
||||
requireValid(
|
||||
(value === null &&
|
||||
Object.hasOwn(attempt.invalidMetrics ?? {}, name)) ||
|
||||
(Number.isFinite(value) && value >= 0),
|
||||
`metric ${name} must be finite and nonnegative, or explicitly invalid null`,
|
||||
);
|
||||
for (const reason of Object.values(attempt.invalidMetrics ?? {}))
|
||||
requireValid(
|
||||
typeof reason === "string" && reason.length,
|
||||
"invalid metric reasons must be nonempty strings",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function differences(baseline, candidate, path, mismatches) {
|
||||
if (record(baseline) && record(candidate)) {
|
||||
for (const key of [
|
||||
...new Set([...Object.keys(baseline), ...Object.keys(candidate)]),
|
||||
].sort())
|
||||
differences(baseline[key], candidate[key], `${path}.${key}`, mismatches);
|
||||
} else if (Array.isArray(baseline) && Array.isArray(candidate)) {
|
||||
if (baseline.length !== candidate.length)
|
||||
mismatches.push({ path, baseline, candidate });
|
||||
else
|
||||
baseline.forEach((value, index) =>
|
||||
differences(value, candidate[index], `${path}[${index}]`, mismatches),
|
||||
);
|
||||
} else if (baseline !== candidate)
|
||||
mismatches.push({
|
||||
path,
|
||||
baseline: baseline ?? null,
|
||||
candidate: candidate ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
export class CompatibilityError extends Error {
|
||||
constructor(mismatches) {
|
||||
super(
|
||||
`Incompatible benchmark runs: ${mismatches.map((entry) => entry.path).join(", ")}. Use an explicit diagnostic comparison to inspect mismatches.`,
|
||||
);
|
||||
this.name = "CompatibilityError";
|
||||
this.mismatches = mismatches;
|
||||
}
|
||||
}
|
||||
|
||||
// Tags the expected per-edge feature difference of a feature-matrix comparison
|
||||
// without suppressing it: the `metadata.build.features` mismatch stays listed
|
||||
// but carries the toggling feature. Mutates the comparison's mismatch entries
|
||||
// in place; no I/O.
|
||||
export function tagExpectedFeatureDiff(comparison, toggledFeature) {
|
||||
for (const entry of comparison.mismatches) {
|
||||
if (entry.path === "metadata.build.features") {
|
||||
entry.expected = true;
|
||||
entry.toggledFeature = toggledFeature;
|
||||
}
|
||||
}
|
||||
return comparison;
|
||||
}
|
||||
|
||||
export function compareRuns(
|
||||
baseline,
|
||||
candidate,
|
||||
{ diagnostic = false, ...options } = {},
|
||||
) {
|
||||
validateResult(baseline);
|
||||
validateResult(candidate);
|
||||
const mismatches = [];
|
||||
// Compare renderer revisions, but never silently mix debug/profiling builds
|
||||
// with ordinary release measurements.
|
||||
for (const field of [
|
||||
"mode",
|
||||
"features",
|
||||
"target",
|
||||
"defaultFeatures",
|
||||
"env",
|
||||
"ambientEnv",
|
||||
]) {
|
||||
differences(
|
||||
baseline.metadata.build?.[field],
|
||||
candidate.metadata.build?.[field],
|
||||
`metadata.build.${field}`,
|
||||
mismatches,
|
||||
);
|
||||
}
|
||||
for (const field of ["environment", "configuration", "metricSemantics"])
|
||||
differences(
|
||||
baseline.metadata[field],
|
||||
candidate.metadata[field],
|
||||
`metadata.${field}`,
|
||||
mismatches,
|
||||
);
|
||||
const firstCases = new Map(
|
||||
baseline.cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]),
|
||||
);
|
||||
const secondCases = new Map(
|
||||
candidate.cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]),
|
||||
);
|
||||
const caseIds = [
|
||||
...new Set([...firstCases.keys(), ...secondCases.keys()]),
|
||||
].sort();
|
||||
for (const id of caseIds) {
|
||||
const first = firstCases.get(id);
|
||||
const second = secondCases.get(id);
|
||||
if (!first || !second) {
|
||||
mismatches.push({
|
||||
path: `cases.${id}`,
|
||||
baseline: Boolean(first),
|
||||
candidate: Boolean(second),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
for (const field of [
|
||||
"scenarioId",
|
||||
"scenarioVersion",
|
||||
"sceneSeed",
|
||||
"params",
|
||||
"initialView",
|
||||
"interaction",
|
||||
"group",
|
||||
"preparation",
|
||||
"environment",
|
||||
"metricSemantics",
|
||||
])
|
||||
differences(
|
||||
first[field],
|
||||
second[field],
|
||||
`cases.${id}.${field}`,
|
||||
mismatches,
|
||||
);
|
||||
// A failed attempt may lack metrics. Only compare present metric sets when
|
||||
// both cases completed; failures still produce honest sample counts.
|
||||
if (first.status === "complete" && second.status === "complete")
|
||||
differences(
|
||||
metricNames(first),
|
||||
metricNames(second),
|
||||
`cases.${id}.metrics`,
|
||||
mismatches,
|
||||
);
|
||||
}
|
||||
if (mismatches.length && !diagnostic)
|
||||
throw new CompatibilityError(mismatches);
|
||||
const runs = Object.fromEntries(
|
||||
[
|
||||
["baseline", baseline],
|
||||
["candidate", candidate],
|
||||
].map(([name, run]) => [
|
||||
name,
|
||||
{
|
||||
runId: run.runId ?? null,
|
||||
status: run.status ?? "unknown",
|
||||
timestamp: run.timestamp ?? null,
|
||||
git: run.metadata.git ?? null,
|
||||
},
|
||||
]),
|
||||
);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runs,
|
||||
compatible: mismatches.length === 0,
|
||||
diagnostic,
|
||||
mismatches,
|
||||
method: statisticalMethod(options),
|
||||
warnings: [
|
||||
...Object.entries(runs)
|
||||
.filter(([, run]) => run.status !== "complete")
|
||||
.map(
|
||||
([name, run]) =>
|
||||
`The ${name} run is ${run.status}; this comparison covers only available valid observations.`,
|
||||
),
|
||||
...(mismatches.length
|
||||
? [
|
||||
"DIAGNOSTIC ONLY: these runs are incompatible; differences cannot establish a performance change.",
|
||||
]
|
||||
: []),
|
||||
...STATISTICAL_WARNINGS,
|
||||
],
|
||||
cases: caseIds
|
||||
.filter((id) => firstCases.has(id) && secondCases.has(id))
|
||||
.map((id) => {
|
||||
const first = firstCases.get(id);
|
||||
const second = secondCases.get(id);
|
||||
const metrics = [
|
||||
...new Set([...metricNames(first), ...metricNames(second)]),
|
||||
].sort();
|
||||
return {
|
||||
id,
|
||||
metrics: Object.fromEntries(
|
||||
metrics.map((metric) => [
|
||||
metric,
|
||||
compareObservations(
|
||||
observations(first, metric),
|
||||
observations(second, metric),
|
||||
options,
|
||||
),
|
||||
]),
|
||||
),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatComparison(comparison) {
|
||||
const lines = [
|
||||
`Statistics: ${comparison.method.name} v${comparison.method.version}; ${comparison.method.resamples} resamples, seed ${comparison.method.seed}.`,
|
||||
...comparison.warnings,
|
||||
];
|
||||
for (const mismatch of comparison.mismatches)
|
||||
lines.push(
|
||||
`MISMATCH ${mismatch.path}: ${JSON.stringify(mismatch.baseline)} -> ${JSON.stringify(mismatch.candidate)}`,
|
||||
);
|
||||
const display = (value) => (value === null ? "n/a" : value.toFixed(3));
|
||||
for (const benchmarkCase of comparison.cases) {
|
||||
lines.push(benchmarkCase.id);
|
||||
for (const [name, metric] of Object.entries(benchmarkCase.metrics)) {
|
||||
lines.push(
|
||||
` ${name}: median ${display(metric.baseline.median)} -> ${display(metric.candidate.median)}, change ${display(metric.absoluteChange)} (${display(metric.relativeChangePercent)}%); n=${metric.baseline.n}/${metric.candidate.n}`,
|
||||
);
|
||||
if (metric.absoluteInterval)
|
||||
lines.push(
|
||||
` change interval: [${display(metric.absoluteInterval.low)}, ${display(metric.absoluteInterval.high)}]`,
|
||||
);
|
||||
if (metric.relativeIntervalPercent)
|
||||
lines.push(
|
||||
` relative interval: [${display(metric.relativeIntervalPercent.low)}, ${display(metric.relativeIntervalPercent.high)}]%`,
|
||||
);
|
||||
for (const [field, reason] of Object.entries(metric.suppressed))
|
||||
lines.push(` ${field}: ${reason}`);
|
||||
}
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
@ -1,252 +0,0 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createRendererAdapter } from "../../../resources/wasm-playground/js/lib.js";
|
||||
|
||||
function fixture() {
|
||||
let next = 0;
|
||||
const frames = new Map();
|
||||
const timers = new Map();
|
||||
const scheduler = {
|
||||
requestAnimationFrame(callback) {
|
||||
frames.set(++next, callback);
|
||||
return next;
|
||||
},
|
||||
cancelAnimationFrame(id) {
|
||||
frames.delete(id);
|
||||
},
|
||||
setTimeout(callback) {
|
||||
timers.set(++next, callback);
|
||||
return next;
|
||||
},
|
||||
clearTimeout(id) {
|
||||
timers.delete(id);
|
||||
},
|
||||
};
|
||||
const state = { rendered: 0, cleaned: false, deleted: false, lost: false };
|
||||
const calls = { views: [], cached: 0, start: 0, end: 0 };
|
||||
const gl = {
|
||||
getExtension: (name) =>
|
||||
name === "WEBGL_lose_context"
|
||||
? {
|
||||
loseContext() {
|
||||
state.lost = true;
|
||||
},
|
||||
}
|
||||
: null,
|
||||
isContextLost: () => state.lost,
|
||||
};
|
||||
const canvas = new EventTarget();
|
||||
Object.assign(canvas, { width: 1920, height: 1080, getContext: () => gl });
|
||||
const module = {
|
||||
GL: {
|
||||
registerContext: () => 1,
|
||||
makeContextCurrent() {},
|
||||
deleteContext() {
|
||||
state.deleted = true;
|
||||
},
|
||||
},
|
||||
_init() {},
|
||||
_set_render_options() {},
|
||||
_resize_viewbox() {},
|
||||
_set_browser() {},
|
||||
_set_view_start() {
|
||||
calls.start++;
|
||||
},
|
||||
_set_view(...args) {
|
||||
calls.views.push(args);
|
||||
},
|
||||
_set_view_end() {
|
||||
calls.end++;
|
||||
},
|
||||
_render_from_cache() {
|
||||
calls.cached++;
|
||||
},
|
||||
_render() {
|
||||
state.rendered++;
|
||||
return 2;
|
||||
},
|
||||
_clean_up() {
|
||||
state.cleaned = true;
|
||||
},
|
||||
};
|
||||
const adapter = createRendererAdapter(module, scheduler);
|
||||
adapter.assignCanvas(canvas);
|
||||
adapter.setupInteraction(canvas);
|
||||
return { adapter, state, calls, canvas, frames, timers, module };
|
||||
}
|
||||
|
||||
for (const dpr of [1, 1.25, 2]) {
|
||||
test(`adapter initializes the full drawing buffer at DPR ${dpr}`, () => {
|
||||
const size = { width: 0, height: 0 };
|
||||
let rendererDpr = 1;
|
||||
const module = {
|
||||
GL: {
|
||||
registerContext: () => 1,
|
||||
makeContextCurrent() {},
|
||||
},
|
||||
_init(width, height) {
|
||||
Object.assign(size, { width, height });
|
||||
},
|
||||
_set_render_options(_flags, value) {
|
||||
rendererDpr = value;
|
||||
},
|
||||
_resize_viewbox(width, height) {
|
||||
// The renderer applies DPR when resizing, not when setting options.
|
||||
Object.assign(size, {
|
||||
width: width * rendererDpr,
|
||||
height: height * rendererDpr,
|
||||
});
|
||||
},
|
||||
};
|
||||
const canvas = {
|
||||
width: 1920 * dpr,
|
||||
height: 1080 * dpr,
|
||||
getContext: () => ({ getExtension() {} }),
|
||||
};
|
||||
|
||||
createRendererAdapter(module).assignCanvas(canvas, dpr);
|
||||
|
||||
assert.deepEqual(size, { width: canvas.width, height: canvas.height });
|
||||
});
|
||||
}
|
||||
|
||||
test("wheel zoom flushes once per animation frame", () => {
|
||||
const f = fixture();
|
||||
const wheel = (deltaY, offsetX, offsetY) =>
|
||||
f.canvas.dispatchEvent(
|
||||
Object.assign(new Event("wheel"), { deltaY, offsetX, offsetY }),
|
||||
);
|
||||
const runFrame = () => {
|
||||
const [id] = f.frames.keys();
|
||||
const callback = f.frames.get(id);
|
||||
f.frames.delete(id);
|
||||
callback(16);
|
||||
};
|
||||
|
||||
wheel(-1, 10, 20);
|
||||
wheel(-1, 30, 40);
|
||||
wheel(-1, 50, 60);
|
||||
|
||||
assert.equal(f.frames.size, 1);
|
||||
assert.equal(f.calls.start, 1);
|
||||
assert.equal(f.calls.views.length, 0);
|
||||
assert.equal(f.calls.cached, 0);
|
||||
|
||||
runFrame();
|
||||
|
||||
assert.equal(f.calls.views.length, 1);
|
||||
assert.equal(f.calls.cached, 1);
|
||||
const [scale, x, y] = f.calls.views[0];
|
||||
assert.ok(Math.abs(scale - 1.1 ** 3) < 1e-9, `unexpected scale ${scale}`);
|
||||
assert.ok(
|
||||
Math.abs(x - (50 / 1.1 ** 3 - 50)) < 1e-9,
|
||||
`unexpected x anchor ${x}`,
|
||||
);
|
||||
assert.ok(
|
||||
Math.abs(y - (60 / 1.1 ** 3 - 60)) < 1e-9,
|
||||
`unexpected y anchor ${y}`,
|
||||
);
|
||||
assert.equal(f.timers.size, 1);
|
||||
|
||||
// The flush releases the frame slot and resets the accumulated factor.
|
||||
wheel(-1, 10, 20);
|
||||
assert.equal(f.frames.size, 1);
|
||||
assert.equal(f.calls.cached, 1);
|
||||
|
||||
runFrame();
|
||||
|
||||
assert.equal(f.calls.views.length, 2);
|
||||
assert.equal(f.calls.cached, 2);
|
||||
assert.ok(
|
||||
Math.abs(f.calls.views[1][0] - 1.1 ** 4) < 1e-9,
|
||||
`unexpected scale ${f.calls.views[1][0]}`,
|
||||
);
|
||||
f.adapter.dispose();
|
||||
});
|
||||
|
||||
test("wheel zoom clamps the accumulated factor to the scale bounds", () => {
|
||||
const f = fixture();
|
||||
const wheel = (deltaY) =>
|
||||
f.canvas.dispatchEvent(
|
||||
Object.assign(new Event("wheel"), { deltaY, offsetX: 10, offsetY: 20 }),
|
||||
);
|
||||
const runFrame = () => {
|
||||
const [id] = f.frames.keys();
|
||||
const callback = f.frames.get(id);
|
||||
f.frames.delete(id);
|
||||
callback(16);
|
||||
};
|
||||
|
||||
for (let index = 0; index < 100; index++) wheel(-1);
|
||||
runFrame();
|
||||
assert.equal(f.calls.views[0][0], 100);
|
||||
|
||||
for (let index = 0; index < 200; index++) wheel(1);
|
||||
runFrame();
|
||||
assert.equal(f.calls.views[1][0], 0.01);
|
||||
f.adapter.dispose();
|
||||
});
|
||||
|
||||
test("pointer pan still renders on every move", () => {
|
||||
const f = fixture();
|
||||
f.canvas.dispatchEvent(
|
||||
Object.assign(new Event("pointerdown"), {
|
||||
clientX: 0,
|
||||
clientY: 0,
|
||||
pointerId: 1,
|
||||
}),
|
||||
);
|
||||
f.canvas.dispatchEvent(
|
||||
Object.assign(new Event("pointermove"), { clientX: 10, clientY: 20 }),
|
||||
);
|
||||
f.canvas.dispatchEvent(
|
||||
Object.assign(new Event("pointermove"), { clientX: 30, clientY: 40 }),
|
||||
);
|
||||
|
||||
assert.equal(f.calls.views.length, 2);
|
||||
assert.equal(f.calls.cached, 2);
|
||||
assert.equal(f.frames.size, 0);
|
||||
f.adapter.dispose();
|
||||
});
|
||||
|
||||
test("disposing an adapter settles pending work and isolates late callbacks", async () => {
|
||||
const first = fixture();
|
||||
const second = fixture();
|
||||
const pending = first.adapter.renderFull();
|
||||
const lateRender = [...first.frames.values()][0];
|
||||
first.canvas.dispatchEvent(
|
||||
Object.assign(new Event("wheel"), { deltaY: 1, offsetX: 10, offsetY: 20 }),
|
||||
);
|
||||
assert.equal(first.frames.size, 1);
|
||||
const lateZoom = [...first.frames.values()][0];
|
||||
first.adapter.dispose();
|
||||
first.adapter.dispose();
|
||||
lateRender(16);
|
||||
lateZoom(16);
|
||||
assert.equal(await pending, false);
|
||||
assert.equal(first.frames.size + first.timers.size, 0);
|
||||
assert.deepEqual(first.state, {
|
||||
rendered: 0,
|
||||
cleaned: true,
|
||||
deleted: true,
|
||||
lost: true,
|
||||
});
|
||||
assert.equal(first.calls.cached, 0);
|
||||
first.canvas.dispatchEvent(new Event("wheel"));
|
||||
assert.equal(first.frames.size + first.timers.size, 0);
|
||||
const completed = second.adapter.renderFull();
|
||||
[...second.frames.values()][0](32);
|
||||
assert.equal(await completed, true);
|
||||
assert.equal(second.state.cleaned, false);
|
||||
second.adapter.dispose();
|
||||
});
|
||||
|
||||
test("adapter deletes its context even when native cleanup throws", () => {
|
||||
const f = fixture();
|
||||
f.module._clean_up = () => {
|
||||
throw new Error("cleanup failed");
|
||||
};
|
||||
assert.throws(() => f.adapter.dispose(), /cleanup failed/);
|
||||
assert.equal(f.state.deleted, true);
|
||||
assert.equal(f.state.lost, true);
|
||||
});
|
||||
@ -1,240 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { compareRuns, formatComparison } from "../compare.js";
|
||||
import { tagExpectedFeatureDiff } from "../compare.js";
|
||||
|
||||
function run(value = 10) {
|
||||
return {
|
||||
schemaVersion: 2,
|
||||
status: "complete",
|
||||
metadata: {
|
||||
git: { sha: "before", dirty: false },
|
||||
build: {
|
||||
mode: "release",
|
||||
featurePolicy: "manifest-defaults",
|
||||
features: [],
|
||||
defaultFeatures: ["default"],
|
||||
target: "frontend",
|
||||
env: { BUILD_MODE: "release" },
|
||||
ambientEnv: { CARGO_HOME: "/home/penpot/.cargo" },
|
||||
},
|
||||
environment: { chromium: "123", headless: true },
|
||||
configuration: { viewport: { width: 1920, height: 1080 }, dpr: 2 },
|
||||
},
|
||||
cases: [
|
||||
{
|
||||
id: "rects/default/load",
|
||||
scenarioId: "rects",
|
||||
scenarioVersion: 1,
|
||||
sceneSeed: 123,
|
||||
params: { count: 1000 },
|
||||
initialView: { x: 0, y: 0, zoom: 1 },
|
||||
interaction: null,
|
||||
group: "fresh-module-context",
|
||||
preparation: "fresh",
|
||||
environment: { webgl: { renderer: "GPU" } },
|
||||
status: "complete",
|
||||
attempts: Array.from({ length: 10 }, (_, index) => ({
|
||||
index,
|
||||
warmup: false,
|
||||
status: "ok",
|
||||
metrics: { render: value },
|
||||
invalidMetrics: {},
|
||||
})),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test("offline comparison allows intentional source and build changes", () => {
|
||||
const candidate = run(12);
|
||||
candidate.metadata.git.sha = "after";
|
||||
candidate.metadata.build.revision = "candidate";
|
||||
const result = compareRuns(run(), candidate);
|
||||
const metric = result.cases[0].metrics.render;
|
||||
assert.equal(result.compatible, true);
|
||||
assert.equal(metric.absoluteChange, 2);
|
||||
assert.equal(metric.relativeChangePercent, 20);
|
||||
assert.deepEqual(metric.absoluteInterval, { low: 2, high: 2 });
|
||||
assert.deepEqual(result, compareRuns(run(), candidate));
|
||||
});
|
||||
|
||||
for (const [name, change] of [
|
||||
[
|
||||
"profiling features",
|
||||
(r) => {
|
||||
r.metadata.build.features = ["profile"];
|
||||
},
|
||||
],
|
||||
[
|
||||
"build mode",
|
||||
(r) => {
|
||||
r.metadata.build.mode = "debug";
|
||||
},
|
||||
],
|
||||
[
|
||||
"DPR",
|
||||
(r) => {
|
||||
r.metadata.configuration.dpr = 1;
|
||||
},
|
||||
],
|
||||
[
|
||||
"browser",
|
||||
(r) => {
|
||||
r.metadata.environment.chromium = "124";
|
||||
},
|
||||
],
|
||||
[
|
||||
"GPU",
|
||||
(r) => {
|
||||
r.cases[0].environment.webgl.renderer = "software";
|
||||
},
|
||||
],
|
||||
[
|
||||
"seed",
|
||||
(r) => {
|
||||
r.cases[0].sceneSeed = 456;
|
||||
},
|
||||
],
|
||||
[
|
||||
"version",
|
||||
(r) => {
|
||||
r.cases[0].scenarioVersion = 2;
|
||||
},
|
||||
],
|
||||
[
|
||||
"workload",
|
||||
(r) => {
|
||||
r.cases[0].params.count = 2000;
|
||||
},
|
||||
],
|
||||
[
|
||||
"preparation",
|
||||
(r) => {
|
||||
r.cases[0].preparation = "warm";
|
||||
},
|
||||
],
|
||||
[
|
||||
"case set",
|
||||
(r) => {
|
||||
r.cases[0].id = "different";
|
||||
},
|
||||
],
|
||||
]) {
|
||||
test(`comparison refuses incompatible ${name} unless diagnostic override is explicit`, () => {
|
||||
const candidate = run();
|
||||
change(candidate);
|
||||
assert.throws(() => compareRuns(run(), candidate), {
|
||||
name: "CompatibilityError",
|
||||
});
|
||||
const result = compareRuns(run(), candidate, { diagnostic: true });
|
||||
assert.equal(result.compatible, false);
|
||||
assert.ok(result.mismatches.length > 0);
|
||||
assert.match(result.warnings.join(" "), /incompatible/i);
|
||||
});
|
||||
}
|
||||
|
||||
test("object key order is immaterial to compatibility", () => {
|
||||
const candidate = run();
|
||||
candidate.metadata.configuration.viewport = { height: 1080, width: 1920 };
|
||||
assert.equal(compareRuns(run(), candidate).compatible, true);
|
||||
});
|
||||
|
||||
test("relative change is unsupported when baseline is zero", () => {
|
||||
const metric = compareRuns(run(0), run(1)).cases[0].metrics.render;
|
||||
assert.equal(metric.relativeChangePercent, null);
|
||||
assert.equal(metric.relativeIntervalPercent, null);
|
||||
assert.match(metric.suppressed.relativeChangePercent, /zero/);
|
||||
});
|
||||
|
||||
test("failed attempts never enter comparison observations", () => {
|
||||
const candidate = run(12);
|
||||
candidate.cases[0].attempts[0].status = "failed";
|
||||
const metric = compareRuns(run(), candidate).cases[0].metrics.render;
|
||||
assert.equal(metric.candidate.n, 9);
|
||||
assert.equal(metric.candidate.invalid, 1);
|
||||
assert.equal(metric.absoluteInterval, null);
|
||||
});
|
||||
|
||||
test("comparisons warn when a run failed even if available metrics are compatible", () => {
|
||||
const candidate = run(12);
|
||||
candidate.status = "failed";
|
||||
candidate.cases[0].status = "failed";
|
||||
const result = compareRuns(run(), candidate);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.equal(result.runs.candidate.status, "failed");
|
||||
assert.match(formatComparison(result), /candidate run is failed/i);
|
||||
});
|
||||
|
||||
test("offline comparison rejects malformed results without diagnostic bypass", () => {
|
||||
const missingEnvironment = run();
|
||||
delete missingEnvironment.metadata.environment;
|
||||
const duplicateCase = run();
|
||||
duplicateCase.cases.push(structuredClone(duplicateCase.cases[0]));
|
||||
const invalidMetric = run();
|
||||
invalidMetric.cases[0].attempts[0].metrics.render = "12";
|
||||
for (const invalid of [
|
||||
null,
|
||||
{},
|
||||
{ schemaVersion: 1 },
|
||||
{ schemaVersion: 3 },
|
||||
missingEnvironment,
|
||||
duplicateCase,
|
||||
invalidMetric,
|
||||
]) {
|
||||
assert.throws(
|
||||
() => compareRuns(invalid, run(), { diagnostic: true }),
|
||||
/Invalid benchmark result/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("text comparisons display changes, intervals and incompatibilities", () => {
|
||||
const candidate = run(12);
|
||||
candidate.metadata.configuration.dpr = 1;
|
||||
const report = formatComparison(
|
||||
compareRuns(run(), candidate, { diagnostic: true }),
|
||||
);
|
||||
assert.match(report, /DIAGNOSTIC ONLY/);
|
||||
assert.match(report, /MISMATCH metadata.configuration.dpr: 2 -> 1/);
|
||||
assert.match(report, /change 2\.000 \(20\.000%\)/);
|
||||
assert.match(report, /change interval: \[2\.000, 2\.000\]/);
|
||||
});
|
||||
|
||||
for (const [name, change] of [
|
||||
["default features", (r) => (r.metadata.build.defaultFeatures = [])],
|
||||
["build env", (r) => (r.metadata.build.env.BUILD_MODE = "debug")],
|
||||
["ambient env", (r) => (r.metadata.build.ambientEnv.CARGO_HOME = "/other")],
|
||||
]) {
|
||||
test(`comparison refuses changed ${name} unless diagnostic override is explicit`, () => {
|
||||
const candidate = run();
|
||||
change(candidate);
|
||||
assert.throws(() => compareRuns(run(), candidate), {
|
||||
name: "CompatibilityError",
|
||||
});
|
||||
const result = compareRuns(run(), candidate, { diagnostic: true });
|
||||
assert.equal(result.compatible, false);
|
||||
assert.ok(result.mismatches.length > 0);
|
||||
});
|
||||
}
|
||||
|
||||
test("ab edge tags the expected feature mismatch without suppressing it", () => {
|
||||
const candidate = run();
|
||||
candidate.metadata.build.features = ["branch-b"];
|
||||
candidate.metadata.build.featurePolicy = "explicit";
|
||||
assert.throws(() => compareRuns(run(), candidate), {
|
||||
name: "CompatibilityError",
|
||||
});
|
||||
const compared = compareRuns(run(), candidate, { diagnostic: true });
|
||||
assert.equal(compared.compatible, false);
|
||||
tagExpectedFeatureDiff(compared, "branch-b");
|
||||
const tagged = compared.mismatches.filter((entry) => entry.expected);
|
||||
assert.equal(tagged.length, 1);
|
||||
assert.deepEqual(tagged[0], {
|
||||
path: "metadata.build.features",
|
||||
baseline: [],
|
||||
candidate: ["branch-b"],
|
||||
expected: true,
|
||||
toggledFeature: "branch-b",
|
||||
});
|
||||
});
|
||||
@ -1,97 +0,0 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
drain,
|
||||
interact,
|
||||
restore,
|
||||
} from "../../../resources/wasm-playground/benchmarks/protocol.js";
|
||||
|
||||
function runtime(results) {
|
||||
let time = 0;
|
||||
const flags = [];
|
||||
return {
|
||||
flags,
|
||||
now: () => time,
|
||||
frame: async () => (time += 16),
|
||||
sleep: async (ms) => {
|
||||
time += ms;
|
||||
},
|
||||
check: () => {},
|
||||
module: {
|
||||
_render: (_timestamp, flag) => {
|
||||
flags.push(flag);
|
||||
time += 2;
|
||||
return results.shift();
|
||||
},
|
||||
_set_view_start: () => {
|
||||
time += 1;
|
||||
},
|
||||
_set_view: () => {
|
||||
time += 1;
|
||||
},
|
||||
_render_from_cache: () => {
|
||||
time += 2;
|
||||
},
|
||||
_set_view_end: () => {
|
||||
time += 5;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("progressive rendering retains returned states and stops at Full", async () => {
|
||||
const rt = runtime([1, 3, 2]);
|
||||
const result = await drain(rt, { flags: 0 });
|
||||
assert.deepEqual(rt.flags, [0, 1, 3]);
|
||||
assert.equal(result.slices.length, 3);
|
||||
assert.ok(result.viewportReadyMs < result.fullMs);
|
||||
});
|
||||
|
||||
test("immediate Full supplies both completion boundaries", async () => {
|
||||
const result = await drain(runtime([2]));
|
||||
assert.equal(result.viewportReadyMs, result.fullMs);
|
||||
});
|
||||
|
||||
test("Partial can finish directly at Full without ViewportReady", async () => {
|
||||
const rt = runtime([1, 2]);
|
||||
const result = await drain(rt);
|
||||
assert.deepEqual(rt.flags, [0, 1]);
|
||||
assert.equal(result.viewportReadyMs, result.fullMs);
|
||||
});
|
||||
|
||||
test("preparation drains to Full and applies SyncTiles only initially", async () => {
|
||||
const rt = runtime([1, 3, 2]);
|
||||
const result = await restore(rt, { scale: 1, x: 0, y: 0 });
|
||||
assert.deepEqual(rt.flags, [4, 1, 3]);
|
||||
assert.equal(result.slices.at(-1).frameType, 2);
|
||||
});
|
||||
|
||||
test("None and unknown frame types fail explicitly", async () => {
|
||||
for (const frame of [0, 7, NaN]) {
|
||||
await assert.rejects(drain(runtime([frame])), /frame type/);
|
||||
}
|
||||
});
|
||||
|
||||
test("finalization includes set_view_end but separates the settling delay", async () => {
|
||||
const rt = runtime([3, 2]);
|
||||
const result = await interact(rt, {
|
||||
frames: [{ scale: 1, x: 3, y: 4 }],
|
||||
settleMs: 100,
|
||||
});
|
||||
assert.equal(result.metrics.setViewEndMs, 5);
|
||||
assert.equal(result.metrics.settlingActualMs, 100);
|
||||
assert.ok(result.metrics.timeToViewportReadyMs >= 7);
|
||||
assert.ok(
|
||||
result.metrics.lastInputToFullMs >= 100 + result.metrics.timeToFullMs,
|
||||
);
|
||||
assert.deepEqual(rt.flags, [4, 3]);
|
||||
});
|
||||
|
||||
test("cancellation stops further progressive work", async () => {
|
||||
const rt = runtime([1, 2]);
|
||||
rt.check = () => {
|
||||
if (rt.flags.length) throw new Error("Canceled");
|
||||
};
|
||||
await assert.rejects(drain(rt), /Canceled/);
|
||||
assert.equal(rt.flags.length, 1);
|
||||
});
|
||||
@ -1,39 +0,0 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
test("page reports keep raw results and show the shared readable summary", async () => {
|
||||
const previous = globalThis.document;
|
||||
const report = { hidden: true, textContent: "" };
|
||||
globalThis.document = {
|
||||
querySelector: (selector) => (selector === "pre" ? report : {}),
|
||||
};
|
||||
try {
|
||||
await import("../../../resources/wasm-playground/benchmarks/browser.js");
|
||||
const result = {
|
||||
scope: "Renderer submission",
|
||||
metadata: {},
|
||||
warnings: [],
|
||||
cases: [
|
||||
{
|
||||
id: "rects/load",
|
||||
status: "complete",
|
||||
unattempted: 0,
|
||||
attempts: [
|
||||
{ warmup: false, status: "ok", metrics: { timeToFullMs: 2 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
globalThis.rendererBenchmark.showResult(result);
|
||||
assert.equal(globalThis.rendererBenchmarkResult, result);
|
||||
assert.equal(report.hidden, false);
|
||||
assert.match(report.textContent, /median 2\.000/);
|
||||
assert.match(report.textContent, /1 valid, 0 invalid/);
|
||||
assert.match(report.textContent, /Renderer submission/);
|
||||
} finally {
|
||||
globalThis.document = previous;
|
||||
delete globalThis.rendererBenchmark;
|
||||
delete globalThis.rendererBenchmarkReady;
|
||||
delete globalThis.rendererBenchmarkResult;
|
||||
}
|
||||
});
|
||||
@ -1,717 +0,0 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
readOptions,
|
||||
bounded,
|
||||
main,
|
||||
recordAttempt,
|
||||
finishCases,
|
||||
defaultFeatures,
|
||||
planFeatureMatrix,
|
||||
validateFeatureSelection,
|
||||
recordAmbientEnv,
|
||||
BUILD_ENV,
|
||||
classifyConfigResult,
|
||||
ab,
|
||||
} from "../../renderer-benchmarks.js";
|
||||
|
||||
test("build provenance expands the renderer's declared default features", () => {
|
||||
assert.deepEqual(defaultFeatures({ default: [] }), ["default"]);
|
||||
assert.deepEqual(
|
||||
defaultFeatures({ default: ["fast"], fast: ["shared"], shared: ["fast"] }),
|
||||
["default", "fast", "shared"],
|
||||
);
|
||||
assert.throws(
|
||||
() => defaultFeatures({ default: ["profile"], profile: [] }),
|
||||
/diagnostic/,
|
||||
);
|
||||
});
|
||||
import {
|
||||
collectCases,
|
||||
deriveSeed,
|
||||
validateMetrics,
|
||||
} from "../../../resources/wasm-playground/benchmarks/contract.js";
|
||||
import { scenarios } from "../../../resources/wasm-playground/scenes/index.js";
|
||||
|
||||
test("CLI validates bounded numbers, commands and URLs before execution", () => {
|
||||
assert.equal(readOptions([]).repetitions, 10);
|
||||
for (const args of [
|
||||
["--repetitions", "NaN"],
|
||||
["--warmups", "-1"],
|
||||
["--seed", "4294967296"],
|
||||
["--base-url", "file:///tmp"],
|
||||
["--unknown"],
|
||||
["compare", "one.json"],
|
||||
]) {
|
||||
assert.throws(() => readOptions(args));
|
||||
}
|
||||
assert.equal(
|
||||
readOptions(["compare", "before.json", "after.json", "--diagnostic"])
|
||||
.diagnostic,
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("feature matrix plans Gray-ordered configurations and lattice edges", () => {
|
||||
assert.deepEqual(planFeatureMatrix(["branch-b"]), {
|
||||
configurations: [
|
||||
{ slug: "0", features: [] },
|
||||
{ slug: "1", features: ["branch-b"] },
|
||||
],
|
||||
edges: [{ from: "0", to: "1", toggledFeature: "branch-b" }],
|
||||
});
|
||||
const two = planFeatureMatrix(["a", "b"]);
|
||||
assert.deepEqual(
|
||||
two.configurations.map((c) => c.slug),
|
||||
["00", "01", "11", "10"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
two.configurations.map((c) => c.features),
|
||||
[[], ["b"], ["a", "b"], ["a"]],
|
||||
);
|
||||
assert.deepEqual(
|
||||
two.edges.map((e) => `${e.from}__${e.to}:${e.toggledFeature}`).sort(),
|
||||
["00__01:b", "00__10:a", "01__11:a", "10__11:b"],
|
||||
);
|
||||
const three = planFeatureMatrix(["a", "b", "c"]);
|
||||
assert.equal(three.configurations.length, 8);
|
||||
assert.equal(three.edges.length, 12);
|
||||
assert.equal(new Set(three.edges.map((e) => `${e.from}__${e.to}`)).size, 12);
|
||||
for (const config of three.configurations) {
|
||||
assert.equal(
|
||||
three.edges.filter((e) => e.from === config.slug || e.to === config.slug)
|
||||
.length,
|
||||
3,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("ab help needs no features and dry run is ab-only", async () => {
|
||||
assert.equal(readOptions(["ab", "--help"]).command, "ab");
|
||||
assert.equal(await main(["ab", "--help"]), 0);
|
||||
for (const args of [
|
||||
["run", "--dry-run"],
|
||||
["compare", "a.json", "b.json", "--dry-run"],
|
||||
]) {
|
||||
assert.throws(() => readOptions(args), /dry-run/);
|
||||
}
|
||||
});
|
||||
|
||||
test("CLI parses feature selection for run and ab", () => {
|
||||
assert.deepEqual(readOptions(["run"]).features, []);
|
||||
assert.deepEqual(readOptions(["run", "--features", "b,a"]).features, [
|
||||
"a",
|
||||
"b",
|
||||
]);
|
||||
assert.deepEqual(readOptions(["run", "--features", "a,a,, b "]).features, [
|
||||
"a",
|
||||
"b",
|
||||
]);
|
||||
assert.deepEqual(readOptions(["run", "--features", "profile"]).features, [
|
||||
"profile",
|
||||
]);
|
||||
const ab = readOptions(["ab", "--features", "branch-b"]);
|
||||
assert.deepEqual(ab.features, ["branch-b"]);
|
||||
assert.equal(ab.maxFeatures, 3);
|
||||
assert.equal(ab.confirm, false);
|
||||
assert.equal(ab.dryRun, false);
|
||||
assert.equal(
|
||||
readOptions(["ab", "--features", "a", "--confirm", "--dry-run"]).confirm,
|
||||
true,
|
||||
);
|
||||
for (const args of [
|
||||
["ab"],
|
||||
["ab", "--features", ""],
|
||||
["ab", "--features", "a", "--max-features", "0"],
|
||||
["ab", "--features", "a", "--max-features", "7"],
|
||||
["ab", "--features", "Bad_Name"],
|
||||
["ab", "--features", "profile"],
|
||||
["ab", "--features", "stats-foo"],
|
||||
["ab", "--features", "a", "extra.json"],
|
||||
["run", "--features", "a", "extra.json"],
|
||||
]) {
|
||||
assert.throws(() => readOptions(args));
|
||||
}
|
||||
});
|
||||
|
||||
test("--no-build skips the renderer build only for run", () => {
|
||||
assert.equal(readOptions(["run"]).noBuild, false);
|
||||
assert.equal(readOptions(["run", "--no-build"]).noBuild, true);
|
||||
for (const args of [
|
||||
["ab", "--features", "branch-b", "--no-build"],
|
||||
["compare", "a.json", "b.json", "--no-build"],
|
||||
["run", "--no-build", "--features", "branch-b"],
|
||||
]) {
|
||||
assert.throws(() => readOptions(args), /--no-build/);
|
||||
}
|
||||
});
|
||||
|
||||
test("feature selection validates against the renderer manifest", () => {
|
||||
const definitions = { default: [], "branch-a": [], "branch-b": [] };
|
||||
assert.deepEqual(validateFeatureSelection(definitions, ["branch-b"], "ab"), [
|
||||
"default",
|
||||
]);
|
||||
assert.deepEqual(validateFeatureSelection(definitions, [], "run"), [
|
||||
"default",
|
||||
]);
|
||||
assert.throws(
|
||||
() => validateFeatureSelection({ default: ["branch-b"] }, [], "ab"),
|
||||
/empty default feature set/,
|
||||
);
|
||||
assert.deepEqual(
|
||||
validateFeatureSelection({ default: ["branch-b"] }, [], "run"),
|
||||
["branch-b", "default"],
|
||||
);
|
||||
assert.throws(
|
||||
() => validateFeatureSelection(definitions, ["nope"], "run"),
|
||||
/Unknown Cargo feature/,
|
||||
);
|
||||
for (const command of ["run", "ab"]) {
|
||||
assert.throws(
|
||||
() =>
|
||||
validateFeatureSelection(
|
||||
{ default: ["shared"], shared: [] },
|
||||
["shared"],
|
||||
command,
|
||||
),
|
||||
/already enabled by default/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("build environment recording excludes volatile values", () => {
|
||||
assert.equal("VERSION" in BUILD_ENV, false);
|
||||
assert.equal(BUILD_ENV.PENPOT_WASM_FUNCTION_NAMES, "0");
|
||||
assert.deepEqual(
|
||||
recordAmbientEnv({
|
||||
RUSTFLAGS: "--deny warnings",
|
||||
CARGO_HOME: "/home/penpot/.cargo",
|
||||
CARGO_PROFILE_RELEASE_DEBUG: "true",
|
||||
EMSDK: "/opt/emsdk",
|
||||
EMSDK_CLANG: "clang",
|
||||
PATH: "/usr/bin",
|
||||
VERSION: "volatile",
|
||||
PENPOT_WASM_PREPARED: "1",
|
||||
}),
|
||||
{
|
||||
RUSTFLAGS: "--deny warnings",
|
||||
CARGO_HOME: "/home/penpot/.cargo",
|
||||
CARGO_PROFILE_RELEASE_DEBUG: "true",
|
||||
EMSDK: "/opt/emsdk",
|
||||
EMSDK_CLANG: "clang",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("classifier separates continuation from matrix stops", () => {
|
||||
const complete = {
|
||||
status: "complete",
|
||||
metadata: { build: { durationMs: 5 } },
|
||||
cases: [{ attempts: [{}] }],
|
||||
};
|
||||
assert.deepEqual(classifyConfigResult(complete), {
|
||||
outcome: "continue",
|
||||
status: "complete",
|
||||
});
|
||||
assert.deepEqual(classifyConfigResult({ ...complete, status: "failed" }), {
|
||||
outcome: "continue",
|
||||
status: "failed",
|
||||
});
|
||||
assert.deepEqual(classifyConfigResult(null), {
|
||||
outcome: "stop",
|
||||
status: "unattempted",
|
||||
error: "configuration output is missing or invalid",
|
||||
});
|
||||
assert.deepEqual(classifyConfigResult({ status: "interrupted" }), {
|
||||
outcome: "stop",
|
||||
status: "interrupted",
|
||||
});
|
||||
assert.deepEqual(
|
||||
classifyConfigResult({
|
||||
status: "failed",
|
||||
metadata: { build: {} },
|
||||
cases: [],
|
||||
}),
|
||||
{
|
||||
outcome: "stop",
|
||||
status: "failed",
|
||||
error: "build or preflight failed before any suite ran",
|
||||
},
|
||||
);
|
||||
assert.deepEqual(
|
||||
classifyConfigResult({
|
||||
status: "failed",
|
||||
metadata: { build: { durationMs: 5 } },
|
||||
cases: [],
|
||||
}),
|
||||
{
|
||||
outcome: "stop",
|
||||
status: "failed",
|
||||
error: "suite recorded no attempts",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
const abDefinitions = { default: [], "branch-a": [], "branch-b": [] };
|
||||
|
||||
async function abTempRoot() {
|
||||
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "renderer-ab-test-"));
|
||||
return path.join(tmp, "matrix");
|
||||
}
|
||||
|
||||
function abOptions(argv) {
|
||||
return readOptions(argv);
|
||||
}
|
||||
|
||||
function stubManifest(definitions = abDefinitions) {
|
||||
return async () => definitions;
|
||||
}
|
||||
|
||||
function stubRun(failSlugs = new Set()) {
|
||||
const calls = [];
|
||||
const runFn = async (options) => {
|
||||
calls.push(options);
|
||||
const slug = path.basename(options.output, ".json");
|
||||
const failed = failSlugs.has(slug);
|
||||
const result = failed
|
||||
? {
|
||||
schemaVersion: 2,
|
||||
runId: `run-${slug}`,
|
||||
status: "failed",
|
||||
error: "./build failed (1)",
|
||||
metadata: { build: { mode: "release", features: [] } },
|
||||
cases: [],
|
||||
}
|
||||
: {
|
||||
schemaVersion: 2,
|
||||
runId: `run-${slug}`,
|
||||
status: "complete",
|
||||
metadata: {
|
||||
build: {
|
||||
mode: "release",
|
||||
features: options.features,
|
||||
durationMs: 5,
|
||||
},
|
||||
},
|
||||
cases: [
|
||||
{
|
||||
id: "rects/default/pan",
|
||||
status: "complete",
|
||||
attempts: [{ index: 0, warmup: false, status: "ok" }],
|
||||
},
|
||||
],
|
||||
};
|
||||
await fs.mkdir(path.dirname(options.output), { recursive: true });
|
||||
await fs.writeFile(options.output, JSON.stringify(result));
|
||||
return failed ? 1 : 0;
|
||||
};
|
||||
return { calls, runFn };
|
||||
}
|
||||
|
||||
function stubCompare(extra = []) {
|
||||
const calls = [];
|
||||
const compareFn = async (baseline, candidate, { diagnostic } = {}) => {
|
||||
calls.push({ baseline, candidate, diagnostic });
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
compatible: false,
|
||||
diagnostic: diagnostic ?? false,
|
||||
mismatches: [
|
||||
{
|
||||
path: "metadata.build.features",
|
||||
baseline: baseline.metadata.build.features,
|
||||
candidate: candidate.metadata.build.features,
|
||||
},
|
||||
...extra,
|
||||
],
|
||||
warnings: [],
|
||||
cases: [],
|
||||
};
|
||||
};
|
||||
return { calls, compareFn };
|
||||
}
|
||||
|
||||
test("ab refuses an existing output root before any preflight", async () => {
|
||||
const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "renderer-ab-test-"));
|
||||
const nonEmpty = path.join(tmp, "full");
|
||||
await fs.mkdir(nonEmpty);
|
||||
await fs.writeFile(path.join(nonEmpty, "x"), "x");
|
||||
const filePath = path.join(tmp, "file.json");
|
||||
await fs.writeFile(filePath, "{}");
|
||||
const mustNotRun = () => {
|
||||
throw new Error("must not be called");
|
||||
};
|
||||
for (const output of [nonEmpty, filePath]) {
|
||||
await assert.rejects(
|
||||
ab(abOptions(["ab", "--features", "branch-b", "--output", output]), {
|
||||
runFn: mustNotRun,
|
||||
compareFn: mustNotRun,
|
||||
manifestFn: mustNotRun,
|
||||
}),
|
||||
/already exists|non-empty/,
|
||||
);
|
||||
}
|
||||
await fs.rm(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("ab dry run prints the plan without building", async () => {
|
||||
const output = await abTempRoot();
|
||||
const mustNotRun = () => {
|
||||
throw new Error("must not be called");
|
||||
};
|
||||
const code = await ab(
|
||||
abOptions([
|
||||
"ab",
|
||||
"--features",
|
||||
"branch-b",
|
||||
"--filter",
|
||||
"rects/default/pan",
|
||||
"--dry-run",
|
||||
"--output",
|
||||
output,
|
||||
]),
|
||||
{ runFn: mustNotRun, compareFn: mustNotRun, manifestFn: stubManifest() },
|
||||
);
|
||||
assert.equal(code, 0);
|
||||
await assert.rejects(fs.access(output), /ENOENT/);
|
||||
await fs.rm(path.dirname(output), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("ab confirm gate blocks wide matrices and empty filters", async () => {
|
||||
const runFn = () => {
|
||||
throw new Error("must not be called");
|
||||
};
|
||||
const wide = await abTempRoot();
|
||||
const narrow = await abTempRoot();
|
||||
await assert.rejects(
|
||||
ab(
|
||||
abOptions([
|
||||
"ab",
|
||||
"--features",
|
||||
"f1,f2,f3,f4",
|
||||
"--max-features",
|
||||
"4",
|
||||
"--filter",
|
||||
"rects",
|
||||
"--output",
|
||||
wide,
|
||||
]),
|
||||
{
|
||||
runFn,
|
||||
manifestFn: stubManifest({
|
||||
default: [],
|
||||
f1: [],
|
||||
f2: [],
|
||||
f3: [],
|
||||
f4: [],
|
||||
}),
|
||||
},
|
||||
),
|
||||
/confirm/,
|
||||
);
|
||||
await assert.rejects(
|
||||
ab(abOptions(["ab", "--features", "branch-b", "--output", narrow]), {
|
||||
runFn,
|
||||
manifestFn: stubManifest(),
|
||||
}),
|
||||
/confirm/,
|
||||
);
|
||||
await fs.rm(path.dirname(wide), { recursive: true, force: true });
|
||||
await fs.rm(path.dirname(narrow), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("ab runs the matrix and writes configs, comparisons and index", async () => {
|
||||
const output = await abTempRoot();
|
||||
const { calls: runCalls, runFn } = stubRun();
|
||||
const { calls: compareCalls, compareFn } = stubCompare();
|
||||
const before = process.listenerCount("SIGINT");
|
||||
const code = await ab(
|
||||
abOptions([
|
||||
"ab",
|
||||
"--features",
|
||||
"branch-b",
|
||||
"--filter",
|
||||
"rects/default/pan",
|
||||
"--output",
|
||||
output,
|
||||
]),
|
||||
{ runFn, compareFn, manifestFn: stubManifest() },
|
||||
);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(process.listenerCount("SIGINT"), before);
|
||||
assert.deepEqual(
|
||||
runCalls.map((options) => options.features),
|
||||
[[], ["branch-b"]],
|
||||
);
|
||||
assert.equal(compareCalls.length, 1);
|
||||
assert.equal(compareCalls[0].diagnostic, true);
|
||||
const first = JSON.parse(
|
||||
await fs.readFile(path.join(output, "configs", "0.json"), "utf8"),
|
||||
);
|
||||
assert.equal(first.runId, "run-0");
|
||||
const comparison = JSON.parse(
|
||||
await fs.readFile(path.join(output, "comparisons", "0__1.json"), "utf8"),
|
||||
);
|
||||
assert.deepEqual(comparison.edge, {
|
||||
from: "0",
|
||||
to: "1",
|
||||
toggledFeature: "branch-b",
|
||||
});
|
||||
assert.deepEqual(
|
||||
comparison.mismatches.filter((entry) => entry.expected),
|
||||
[
|
||||
{
|
||||
path: "metadata.build.features",
|
||||
baseline: [],
|
||||
candidate: ["branch-b"],
|
||||
expected: true,
|
||||
toggledFeature: "branch-b",
|
||||
},
|
||||
],
|
||||
);
|
||||
const matrix = JSON.parse(
|
||||
await fs.readFile(path.join(output, "matrix.json"), "utf8"),
|
||||
);
|
||||
assert.equal(matrix.schemaVersion, 1);
|
||||
assert.deepEqual(matrix.features, ["branch-b"]);
|
||||
assert.deepEqual(
|
||||
matrix.configurations.map((c) => c.status),
|
||||
["complete", "complete"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
matrix.comparisons.map((c) => [c.from, c.to, c.status]),
|
||||
[["0", "1", "compared"]],
|
||||
);
|
||||
assert.equal(matrix.status, "complete");
|
||||
assert.equal(comparison.controlled, true);
|
||||
assert.equal(matrix.comparisons[0].controlled, true);
|
||||
await fs.rm(path.dirname(output), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("ab marks edges with unexpected diffs uncontrolled", async () => {
|
||||
const output = await abTempRoot();
|
||||
const { runFn } = stubRun();
|
||||
const { compareFn } = stubCompare([
|
||||
{ path: "metadata.build.env", baseline: {}, candidate: {} },
|
||||
]);
|
||||
const code = await ab(
|
||||
abOptions([
|
||||
"ab",
|
||||
"--features",
|
||||
"branch-b",
|
||||
"--filter",
|
||||
"rects/default/pan",
|
||||
"--output",
|
||||
output,
|
||||
]),
|
||||
{ runFn, compareFn, manifestFn: stubManifest() },
|
||||
);
|
||||
assert.equal(code, 0);
|
||||
const comparison = JSON.parse(
|
||||
await fs.readFile(path.join(output, "comparisons", "0__1.json"), "utf8"),
|
||||
);
|
||||
assert.equal(comparison.controlled, false);
|
||||
const matrix = JSON.parse(
|
||||
await fs.readFile(path.join(output, "matrix.json"), "utf8"),
|
||||
);
|
||||
assert.equal(matrix.comparisons[0].controlled, false);
|
||||
await fs.rm(path.dirname(output), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("ab interruption between configurations still writes the matrix", async () => {
|
||||
const output = await abTempRoot();
|
||||
const controller = new AbortController();
|
||||
const { runFn } = stubRun();
|
||||
const interruptAfterFirst = async (options) => {
|
||||
const code = await runFn(options);
|
||||
controller.abort(new Error("stopped"));
|
||||
return code;
|
||||
};
|
||||
const { compareFn } = stubCompare();
|
||||
const code = await ab(
|
||||
abOptions([
|
||||
"ab",
|
||||
"--features",
|
||||
"a,b",
|
||||
"--max-features",
|
||||
"2",
|
||||
"--filter",
|
||||
"rects",
|
||||
"--confirm",
|
||||
"--output",
|
||||
output,
|
||||
]),
|
||||
{
|
||||
runFn: interruptAfterFirst,
|
||||
compareFn,
|
||||
manifestFn: stubManifest({ default: [], a: [], b: [] }),
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
assert.equal(code, 1);
|
||||
const matrix = JSON.parse(
|
||||
await fs.readFile(path.join(output, "matrix.json"), "utf8"),
|
||||
);
|
||||
assert.deepEqual(
|
||||
matrix.configurations.map((c) => [c.slug, c.status]),
|
||||
[
|
||||
["00", "complete"],
|
||||
["01", "unattempted"],
|
||||
["11", "unattempted"],
|
||||
["10", "unattempted"],
|
||||
],
|
||||
);
|
||||
assert.ok(matrix.comparisons.every((c) => c.status === "not-attempted"));
|
||||
assert.equal(matrix.status, "failed");
|
||||
await fs.rm(path.dirname(output), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("ab stops on build failure and marks the rest unattempted", async () => {
|
||||
const output = await abTempRoot();
|
||||
const { runFn } = stubRun(new Set(["01"]));
|
||||
const { compareFn } = stubCompare();
|
||||
const code = await ab(
|
||||
abOptions([
|
||||
"ab",
|
||||
"--features",
|
||||
"b,a",
|
||||
"--max-features",
|
||||
"2",
|
||||
"--filter",
|
||||
"rects",
|
||||
"--confirm",
|
||||
"--output",
|
||||
output,
|
||||
]),
|
||||
{
|
||||
runFn,
|
||||
compareFn,
|
||||
manifestFn: stubManifest({ default: [], a: [], b: [] }),
|
||||
},
|
||||
);
|
||||
assert.equal(code, 1);
|
||||
const matrix = JSON.parse(
|
||||
await fs.readFile(path.join(output, "matrix.json"), "utf8"),
|
||||
);
|
||||
assert.deepEqual(
|
||||
matrix.configurations.map((c) => [c.slug, c.status]),
|
||||
[
|
||||
["00", "complete"],
|
||||
["01", "failed"],
|
||||
["11", "unattempted"],
|
||||
["10", "unattempted"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
matrix.comparisons.map((c) => [c.from, c.to, c.status]),
|
||||
[
|
||||
["00", "10", "not-attempted"],
|
||||
["00", "01", "compared"],
|
||||
["01", "11", "not-attempted"],
|
||||
["10", "11", "not-attempted"],
|
||||
],
|
||||
);
|
||||
assert.equal(matrix.status, "failed");
|
||||
await fs.rm(path.dirname(output), { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("case collector retains a single scene seed across lifecycle and interaction groups", () => {
|
||||
const cases = collectCases(scenarios, 42);
|
||||
assert.equal(cases.length, 21);
|
||||
const rectangles = cases.filter((c) => c.scenarioId === "rects");
|
||||
assert.equal(new Set(rectangles.map((c) => c.sceneSeed)).size, 1);
|
||||
assert.equal(rectangles[0].sceneSeed, deriveSeed(42, "rects/default"));
|
||||
assert.throws(() => collectCases(scenarios, 42, "missing"), /No matching/);
|
||||
assert.throws(
|
||||
() => collectCases([scenarios[0], scenarios[0]], 42),
|
||||
/duplicate/,
|
||||
);
|
||||
});
|
||||
|
||||
test("case collector rejects missing interactions and non-JSON case values", () => {
|
||||
const scenario = scenarios[0];
|
||||
const invalid = {
|
||||
...scenario,
|
||||
cases: [{ ...scenario.cases[0], interactions: {} }],
|
||||
};
|
||||
assert.throws(() => collectCases([invalid], 42), /Incomplete/);
|
||||
assert.throws(
|
||||
() =>
|
||||
collectCases(
|
||||
[
|
||||
{
|
||||
...scenario,
|
||||
cases: [{ ...scenario.cases[0], params: { count: Infinity } }],
|
||||
},
|
||||
],
|
||||
42,
|
||||
),
|
||||
/Non-serializable/,
|
||||
);
|
||||
});
|
||||
|
||||
test("watchdog bounds unresolved work and propagates cancellation", async () => {
|
||||
await assert.rejects(bounded(new Promise(() => {}), 10), /watchdog/);
|
||||
const abort = new AbortController();
|
||||
const operation = bounded(new Promise(() => {}), 10000, abort.signal);
|
||||
abort.abort(new Error("stopped"));
|
||||
await assert.rejects(operation, /stopped/);
|
||||
assert.equal(await bounded(Promise.resolve(42), 100), 42);
|
||||
});
|
||||
|
||||
test("invalid aggregates retain an explicit reason", () => {
|
||||
assert.deepEqual(
|
||||
validateMetrics({ uploadMs: 0, timeToFullMs: NaN, activeInteractionMs: 2 }),
|
||||
{
|
||||
uploadMs: "below-timer-resolution",
|
||||
timeToFullMs: "non-finite",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("help does not need a renderer build or browser", async () => {
|
||||
assert.equal(await main(["--help"]), 0);
|
||||
});
|
||||
|
||||
test("changed environment is stored as a failed observation", () => {
|
||||
const c = {
|
||||
status: "running",
|
||||
attempts: [],
|
||||
environment: { webgl: { renderer: "A" } },
|
||||
};
|
||||
recordAttempt(
|
||||
c,
|
||||
{
|
||||
status: "ok",
|
||||
metrics: { uploadMs: 1 },
|
||||
environment: { webgl: { renderer: "B" } },
|
||||
},
|
||||
0,
|
||||
0,
|
||||
);
|
||||
assert.equal(c.attempts[0].status, "failed");
|
||||
assert.match(c.attempts[0].error, /environment changed/);
|
||||
assert.equal(c.status, "failed");
|
||||
});
|
||||
|
||||
test("fatal failure and interruption preserve partial and unattempted counts", () => {
|
||||
for (const status of ["failed", "interrupted"]) {
|
||||
const cases = [
|
||||
{ status: "complete", attempts: [{ status: "ok" }, { status: "ok" }] },
|
||||
{ status: "running", attempts: [{ status: "ok" }] },
|
||||
{ status: "unattempted", attempts: [] },
|
||||
];
|
||||
finishCases(cases, 2, status);
|
||||
assert.deepEqual(
|
||||
cases.map((c) => [c.status, c.unattempted]),
|
||||
[
|
||||
["complete", 0],
|
||||
[status, 1],
|
||||
["unattempted", 2],
|
||||
],
|
||||
);
|
||||
}
|
||||
});
|
||||
@ -1,123 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { scenarios } from "../../../resources/wasm-playground/scenes/index.js";
|
||||
import {
|
||||
serializeText,
|
||||
validateCount,
|
||||
} from "../../../resources/wasm-playground/scenes/shared.js";
|
||||
|
||||
test("all seven workloads reproduce the same exact scene for a seed", () => {
|
||||
assert.deepEqual(
|
||||
scenarios.map((scenario) => scenario.id),
|
||||
["rects", "paths", "plus", "texts", "masks", "clips", "shadows"],
|
||||
);
|
||||
for (const scenario of scenarios) {
|
||||
const params = scenario.cases[0].params;
|
||||
assert.deepEqual(
|
||||
scenario.createScene(params, 42),
|
||||
scenario.createScene(params, 42),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("shape counts are integers before allocation", () => {
|
||||
assert.equal(validateCount("1000"), 1000);
|
||||
for (const value of ["", "2x", -1, 1.5, Infinity, true, null, 0, 100001]) {
|
||||
assert.throws(() => validateCount(value), /count/i);
|
||||
}
|
||||
});
|
||||
|
||||
test("scenes reject ignored parameters and malformed text dimensions", () => {
|
||||
for (const scenario of scenarios) {
|
||||
assert.throws(
|
||||
() => scenario.createScene({ ...scenario.cases[0].params, typo: 1 }, 42),
|
||||
/Unsupported parameter/,
|
||||
);
|
||||
}
|
||||
const texts = scenarios.find((s) => s.id === "texts");
|
||||
for (const override of [
|
||||
{ width: -1 },
|
||||
{ words: [] },
|
||||
{ maxLines: 1.5 },
|
||||
{ minFontSize: 100 },
|
||||
]) {
|
||||
assert.throws(() =>
|
||||
texts.createScene({ ...texts.cases[0].params, ...override }, 42),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("every shape is reachable exactly once and agrees with its parent", () => {
|
||||
for (const scenario of scenarios) {
|
||||
const scene = scenario.createScene(scenario.cases[0].params, 123);
|
||||
const key = (id) => id.join(":");
|
||||
const shapes = new Map(scene.shapes.map((shape) => [key(shape.id), shape]));
|
||||
assert.equal(shapes.size, scene.shapes.length);
|
||||
const seen = new Set();
|
||||
function visit(parent, children) {
|
||||
for (const id of children) {
|
||||
const child = shapes.get(key(id));
|
||||
assert.ok(child, `${scenario.id}: missing child`);
|
||||
assert.deepEqual(child.parent, parent);
|
||||
assert.ok(!seen.has(key(id)), `${scenario.id}: duplicate child`);
|
||||
seen.add(key(id));
|
||||
visit(id, child.children ?? []);
|
||||
}
|
||||
}
|
||||
visit([0, 0, 0, 0], scene.children);
|
||||
assert.equal(seen.size, scene.shapes.length, scenario.id);
|
||||
}
|
||||
});
|
||||
|
||||
test("text payload follows the current Rust paragraph and span ABI", () => {
|
||||
const bytes = serializeText("Penpot café", 24, 0xff112233);
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
const encoded = new TextEncoder().encode("Penpot café");
|
||||
assert.equal(bytes.length, 16 + 1344 + encoded.length);
|
||||
assert.equal(view.getUint32(0, true), 1);
|
||||
assert.equal(view.getUint8(4), 0); // Left, not Center.
|
||||
assert.equal(view.getFloat32(16 + 4, true), 24);
|
||||
assert.equal(view.getInt32(16 + 16, true), 400);
|
||||
assert.equal(view.getUint32(16 + 56, true), encoded.length);
|
||||
assert.equal(view.getUint32(16 + 60, true), 1);
|
||||
assert.equal(view.getUint32(16 + 64 + 4, true), 0xff112233);
|
||||
assert.deepEqual(bytes.subarray(16 + 1344), encoded);
|
||||
});
|
||||
|
||||
test("clip effects use valid enum values and actual booleans", () => {
|
||||
const scenario = scenarios.find(({ id }) => id === "clips");
|
||||
const scene = scenario.createScene(scenario.cases[0].params, 42);
|
||||
assert.equal(scene.shapes.length, 2);
|
||||
for (const shape of scene.shapes) {
|
||||
if (shape.blur) assert.equal(shape.blur[1], false);
|
||||
for (const shadow of shape.shadows ?? []) {
|
||||
assert.ok([0, 1].includes(shadow[5]));
|
||||
assert.equal(typeof shadow[6], "boolean");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("shadow workload retains intended effects with complete valid FFI calls", () => {
|
||||
const scenario = scenarios.find(({ id }) => id === "shadows");
|
||||
const params = scenario.cases[0].params;
|
||||
assert.equal(params.count, 1000);
|
||||
assert.equal(params.shadows.length, 3);
|
||||
assert.deepEqual(
|
||||
params.shadows.map((shadow) => shadow[5]),
|
||||
[0, 0, 1],
|
||||
);
|
||||
const scene = scenario.createScene(params, 42);
|
||||
assert.equal(scene.shapes.length, 1000);
|
||||
for (const shape of scene.shapes) {
|
||||
assert.deepEqual(shape.shadows, params.shadows);
|
||||
for (const shadow of shape.shadows) {
|
||||
assert.equal(shadow.length, 7);
|
||||
assert.ok(Number.isInteger(shadow[0]));
|
||||
for (const value of shadow.slice(1, 5)) {
|
||||
assert.ok(Number.isFinite(value));
|
||||
}
|
||||
assert.ok([0, 1].includes(shadow[5]));
|
||||
assert.equal(shadow[6], false);
|
||||
}
|
||||
}
|
||||
});
|
||||
@ -1,129 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import {
|
||||
summarize,
|
||||
summarizeRun,
|
||||
formatSummary,
|
||||
} from "../../../resources/wasm-playground/benchmarks/statistics.js";
|
||||
|
||||
test("summaries use sample deviation and unscaled median absolute deviation", () => {
|
||||
const summary = summarize([1, 2, 3, 4, 5]);
|
||||
assert.equal(summary.mean, 3);
|
||||
assert.equal(summary.median, 3);
|
||||
assert.equal(summary.standardDeviation, Math.sqrt(2.5));
|
||||
assert.equal(summary.mad, 1);
|
||||
assert.equal(summary.min, 1);
|
||||
assert.equal(summary.max, 5);
|
||||
});
|
||||
|
||||
test("invalid numbers are counted and zero observations survive", () => {
|
||||
const summary = summarize([0, 2, NaN, Infinity, "3", null, -1]);
|
||||
assert.equal(summary.n, 2);
|
||||
assert.equal(summary.invalid, 5);
|
||||
assert.equal(summary.median, 1);
|
||||
assert.equal(summary.medianInterval, null);
|
||||
assert.match(summary.suppressed.medianInterval, /10/);
|
||||
});
|
||||
|
||||
test("empty and singleton observations have no invented spread", () => {
|
||||
assert.equal(summarize([]).median, null);
|
||||
assert.equal(summarize([]).standardDeviation, null);
|
||||
assert.equal(summarize([2]).standardDeviation, null);
|
||||
assert.equal(summarize([2]).mad, 0);
|
||||
});
|
||||
|
||||
test("bootstrap intervals are deterministic and tail support is explicit", () => {
|
||||
const values = Array.from({ length: 100 }, (_, index) => index + 1);
|
||||
const first = summarize(values);
|
||||
assert.deepEqual(first, summarize(values));
|
||||
assert.ok(first.medianInterval.low < first.median);
|
||||
assert.ok(first.medianInterval.high > first.median);
|
||||
assert.equal(first.p95, 95.05);
|
||||
assert.equal(first.p99, null);
|
||||
assert.match(first.suppressed.p99, /500/);
|
||||
assert.equal(summarize(Array(500).fill(3)).p99, 3);
|
||||
});
|
||||
|
||||
test("run summaries exclude warmups, failed attempts and invalid metrics", () => {
|
||||
const result = summarizeRun({
|
||||
cases: [
|
||||
{
|
||||
id: "rects/load",
|
||||
status: "complete",
|
||||
attempts: [
|
||||
{ warmup: true, status: "ok", metrics: { render: 1000 } },
|
||||
{ warmup: false, status: "ok", metrics: { render: 10, upload: 0 } },
|
||||
{
|
||||
warmup: false,
|
||||
status: "invalid",
|
||||
metrics: { render: 0, upload: 2 },
|
||||
invalidMetrics: { render: "below resolution" },
|
||||
},
|
||||
{ warmup: false, status: "failed", metrics: { render: 300 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const summary = result.cases[0];
|
||||
assert.deepEqual(summary.attempts, {
|
||||
measured: 3,
|
||||
warmup: 1,
|
||||
ok: 1,
|
||||
invalid: 1,
|
||||
failed: 1,
|
||||
});
|
||||
assert.equal(summary.metrics.render.median, 10);
|
||||
assert.equal(summary.metrics.render.invalid, 2);
|
||||
assert.equal(summary.metrics.upload.median, 1);
|
||||
assert.equal(summary.metrics.upload.invalid, 1);
|
||||
assert.match(result.warnings.join(" "), /conditional/);
|
||||
});
|
||||
|
||||
test("statistics reject unsafe bootstrap settings", () => {
|
||||
assert.throws(() => summarize([1], { resamples: Infinity }), /resamples/);
|
||||
assert.throws(() => summarize([1], { seed: NaN }), /seed/);
|
||||
});
|
||||
|
||||
test("text summaries display the same observations and support limits as JSON", () => {
|
||||
const summary = summarizeRun({
|
||||
cases: [
|
||||
{
|
||||
id: "rects/load",
|
||||
status: "complete",
|
||||
attempts: [{ warmup: false, status: "ok", metrics: { render: 2 } }],
|
||||
},
|
||||
],
|
||||
});
|
||||
const report = formatSummary(summary);
|
||||
assert.match(report, /median 2\.000/);
|
||||
assert.match(report, /1 valid, 0 invalid/);
|
||||
assert.match(report, /Requires at least 10 valid attempts/);
|
||||
assert.match(report, /2000 resamples/);
|
||||
});
|
||||
|
||||
test("reports expose missing attempts and explain support limits once", () => {
|
||||
const summary = summarizeRun({
|
||||
cases: [
|
||||
{
|
||||
id: "rects/load",
|
||||
status: "failed",
|
||||
unattempted: 9,
|
||||
attempts: [
|
||||
{
|
||||
warmup: false,
|
||||
status: "failed",
|
||||
metrics: { render: 3, upload: 2 },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(summary.cases[0].unattempted, 9);
|
||||
const report = formatSummary(summary);
|
||||
assert.match(report, /9 unattempted/);
|
||||
assert.match(report, /1 failed/);
|
||||
assert.equal(
|
||||
report.split("Requires at least 10 valid attempts").length - 1,
|
||||
1,
|
||||
);
|
||||
});
|
||||
@ -1,163 +0,0 @@
|
||||
// Unscored end-to-end failure checks. Each child uses the real prepared build,
|
||||
// server, browser, persistence and cleanup paths. No service is started here.
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { bounded, main } from "../renderer-benchmarks.js";
|
||||
|
||||
const filename = fileURLToPath(import.meta.url);
|
||||
|
||||
if (process.argv[2] === "--child") {
|
||||
const [, , , fault, output] = process.argv;
|
||||
const { chromium } = await import("playwright");
|
||||
const connect = chromium.connect.bind(chromium);
|
||||
let checkpoints = 0;
|
||||
chromium.connect = async (...args) => {
|
||||
const browser = await connect(...args);
|
||||
const newContext = browser.newContext.bind(browser);
|
||||
browser.newContext = async (...contextArgs) => {
|
||||
const context = await newContext(...contextArgs);
|
||||
await context.exposeBinding(
|
||||
"benchmarkFaultCheckpoint",
|
||||
async ({ page }) => {
|
||||
if (++checkpoints !== 2) return;
|
||||
if (fault === "interrupt") process.kill(process.pid, "SIGTERM");
|
||||
if (fault === "browser-disconnect")
|
||||
void browser.close().catch(() => {});
|
||||
if (fault === "page-crash") {
|
||||
const cdp = await context.newCDPSession(page);
|
||||
void cdp.send("Page.crash").catch(() => {});
|
||||
}
|
||||
},
|
||||
);
|
||||
await context.addInitScript(() => {
|
||||
let api;
|
||||
Object.defineProperty(globalThis, "rendererBenchmark", {
|
||||
get: () => api,
|
||||
set(value) {
|
||||
api = value;
|
||||
const attempt = value.runAttempt;
|
||||
value.runAttempt = async (...args) => {
|
||||
await globalThis.benchmarkFaultCheckpoint();
|
||||
return attempt(...args);
|
||||
};
|
||||
},
|
||||
});
|
||||
});
|
||||
return context;
|
||||
};
|
||||
return browser;
|
||||
};
|
||||
const args = [
|
||||
"--filter",
|
||||
"rects",
|
||||
"--warmups",
|
||||
"0",
|
||||
"--repetitions",
|
||||
"2",
|
||||
"--output",
|
||||
output,
|
||||
];
|
||||
if (fault === "missing-server")
|
||||
args.push("--base-url", "http://127.0.0.1:1/");
|
||||
if (fault === "wrong-server")
|
||||
args.push("--base-url", "http://localhost:3000/not-this-workspace/");
|
||||
process.exitCode = await main(args);
|
||||
} else {
|
||||
const directory = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), "renderer-cli-faults-"),
|
||||
);
|
||||
console.log(`Diagnostic results: ${directory}`);
|
||||
for (const fault of [
|
||||
"missing-browser",
|
||||
"missing-server",
|
||||
"wrong-server",
|
||||
"page-crash",
|
||||
"browser-disconnect",
|
||||
"interrupt",
|
||||
]) {
|
||||
const output = path.join(directory, `${fault}.json`);
|
||||
const log = await fs.open(path.join(directory, `${fault}.log`), "w");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[filename, "--child", fault, output],
|
||||
{
|
||||
detached: true,
|
||||
stdio: ["ignore", log.fd, log.fd],
|
||||
env: {
|
||||
...process.env,
|
||||
...(fault === "missing-browser"
|
||||
? {
|
||||
PLAYWRIGHT_BROWSERS_PATH: path.join(
|
||||
directory,
|
||||
"missing-browser",
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
try {
|
||||
const exit = await bounded(
|
||||
new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("exit", (code, signal) => resolve({ code, signal }));
|
||||
}),
|
||||
180000,
|
||||
);
|
||||
assert.deepEqual(
|
||||
exit,
|
||||
{ code: 1, signal: null },
|
||||
`${fault}: must exit with an execution failure`,
|
||||
);
|
||||
const result = JSON.parse(await fs.readFile(output, "utf8"));
|
||||
assert.ok(result.finishedAt);
|
||||
assert.equal(
|
||||
result.status,
|
||||
fault === "interrupt" ? "interrupted" : "failed",
|
||||
);
|
||||
assert.ok(
|
||||
result.cases.every((c) => c.attempts.length + c.unattempted === 2),
|
||||
);
|
||||
if (
|
||||
["missing-browser", "missing-server", "wrong-server"].includes(fault)
|
||||
) {
|
||||
assert.ok(result.cases.every((c) => c.status === "unattempted"));
|
||||
assert.match(
|
||||
result.error,
|
||||
fault === "missing-browser"
|
||||
? /Chromium is missing/
|
||||
: fault === "missing-server"
|
||||
? /Static server unavailable/
|
||||
: /current benchmark assets/,
|
||||
);
|
||||
} else {
|
||||
assert.equal(
|
||||
result.cases[0].attempts[0].status,
|
||||
"ok",
|
||||
"retain the successful attempt before injection",
|
||||
);
|
||||
assert.equal(result.cases[0].attempts[1].status, "failed");
|
||||
if (fault === "page-crash") {
|
||||
assert.ok(
|
||||
result.cases.slice(1).every((c) => c.status === "complete"),
|
||||
"independent cases recover in fresh contexts",
|
||||
);
|
||||
} else {
|
||||
assert.ok(
|
||||
result.cases.slice(1).every((c) => c.status === "unattempted"),
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(`PASS ${fault}: partial results and bounded exit (unscored)`);
|
||||
} finally {
|
||||
// Kill only this diagnostic child's process group if its watchdog expired.
|
||||
if (child.exitCode === null && child.signalCode === null)
|
||||
process.kill(-child.pid, "SIGKILL");
|
||||
await log.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,184 +0,0 @@
|
||||
// Unscored browser fault checks. Uses prepared assets; manages no services.
|
||||
import assert from "node:assert/strict";
|
||||
import { chromium } from "playwright";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { bounded } from "../renderer-benchmarks.js";
|
||||
import { scenarios } from "../../resources/wasm-playground/scenes/index.js";
|
||||
import { collectCases } from "../../resources/wasm-playground/benchmarks/contract.js";
|
||||
|
||||
const frontend = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const server = await chromium.launchServer({
|
||||
headless: true,
|
||||
args: ["--enable-gpu"],
|
||||
});
|
||||
const process = server.process();
|
||||
const browser = await chromium.connect(server.wsEndpoint());
|
||||
const benchmarkCase = collectCases(scenarios, 42, "rects/default/pan")[0];
|
||||
const options = {
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
dpr: 2,
|
||||
timeoutMs: 30000,
|
||||
moduleUrl: "http://benchmark.invalid/fault-module.js",
|
||||
};
|
||||
try {
|
||||
for (const fault of [
|
||||
"unknown-frame",
|
||||
"context-loss",
|
||||
"timeout",
|
||||
"cancel-init",
|
||||
"cancel-frame",
|
||||
"blocked-wasm",
|
||||
"page-crash",
|
||||
"report",
|
||||
]) {
|
||||
const context = await browser.newContext({
|
||||
viewport: options.viewport,
|
||||
deviceScaleFactor: options.dpr,
|
||||
});
|
||||
try {
|
||||
await context.route("http://benchmark.invalid/**", async (route) => {
|
||||
const pathname = new URL(route.request().url()).pathname;
|
||||
if (pathname === "/fault-module.js") {
|
||||
await route.fulfill({
|
||||
contentType: "text/javascript",
|
||||
body: `
|
||||
import createModule from '/js/render-wasm-benchmark.js';
|
||||
export default async function() {
|
||||
const module = await createModule();
|
||||
if (${JSON.stringify(fault)} === 'cancel-init') await new Promise(resolve => { globalThis.releaseModule = resolve; });
|
||||
const render = module._render;
|
||||
module._render = (...args) => {
|
||||
if (${JSON.stringify(fault)} === 'unknown-frame') return 0;
|
||||
if (${JSON.stringify(fault)} === 'blocked-wasm') { while (true) {} }
|
||||
if (${JSON.stringify(fault)} === 'context-loss') document.querySelector('canvas').getContext('webgl2').getExtension('WEBGL_lose_context').loseContext();
|
||||
return render(...args);
|
||||
};
|
||||
return module;
|
||||
}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const base = path.join(
|
||||
frontend,
|
||||
pathname.startsWith("/wasm-playground/")
|
||||
? "resources"
|
||||
: "resources/public",
|
||||
);
|
||||
const target = path.resolve(base, `.${pathname}`);
|
||||
if (!target.startsWith(`${base}/`)) return route.abort();
|
||||
await route.fulfill({ path: target });
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.goto(
|
||||
"http://benchmark.invalid/wasm-playground/benchmark.html",
|
||||
);
|
||||
await page.waitForFunction(() => globalThis.rendererBenchmarkReady);
|
||||
if (fault === "timeout")
|
||||
await page.evaluate(() => {
|
||||
globalThis.requestAnimationFrame = () => 1;
|
||||
});
|
||||
if (fault === "cancel-frame")
|
||||
await page.evaluate(() => {
|
||||
globalThis.requestAnimationFrame = () => {
|
||||
globalThis.framePending = true;
|
||||
return 1;
|
||||
};
|
||||
});
|
||||
if (fault === "report") {
|
||||
const report = await page.evaluate((c) => {
|
||||
const value = {
|
||||
scope: "Diagnostic report",
|
||||
metadata: {},
|
||||
warnings: [],
|
||||
cases: [
|
||||
{
|
||||
...c,
|
||||
status: "complete",
|
||||
unattempted: 0,
|
||||
attempts: [
|
||||
{ warmup: false, status: "ok", metrics: { timeToFullMs: 2 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
globalThis.rendererBenchmark.showResult(value);
|
||||
return {
|
||||
result: globalThis.rendererBenchmarkResult,
|
||||
text: document.querySelector("pre").textContent,
|
||||
};
|
||||
}, benchmarkCase);
|
||||
assert.equal(
|
||||
report.result.cases[0].attempts[0].metrics.timeToFullMs,
|
||||
2,
|
||||
);
|
||||
assert.match(report.text, /median 2\.000/);
|
||||
assert.match(report.text, /1 valid, 0 invalid/);
|
||||
} else if (fault === "page-crash") {
|
||||
const crashed = page.waitForEvent("crash");
|
||||
const cdp = await context.newCDPSession(page);
|
||||
void cdp.send("Page.crash").catch(() => {});
|
||||
await bounded(crashed, 5000);
|
||||
await assert.rejects(
|
||||
page.evaluate(() => 1),
|
||||
/crash|closed/i,
|
||||
);
|
||||
} else {
|
||||
// Attach rejection handling immediately; the watchdog may finish before injection.
|
||||
const outcome = bounded(
|
||||
page.evaluate(
|
||||
([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o),
|
||||
[benchmarkCase, options],
|
||||
),
|
||||
fault === "blocked-wasm" || fault === "timeout" ? 2500 : 10000,
|
||||
).then(
|
||||
(value) => ({ value }),
|
||||
(error) => ({ error }),
|
||||
);
|
||||
if (fault === "cancel-init" || fault === "cancel-frame") {
|
||||
await page.waitForFunction(
|
||||
(key) => Boolean(globalThis[key]),
|
||||
fault === "cancel-init" ? "releaseModule" : "framePending",
|
||||
);
|
||||
await page.evaluate(() => {
|
||||
globalThis.rendererBenchmark.dispose();
|
||||
globalThis.releaseModule?.();
|
||||
});
|
||||
}
|
||||
const result = await outcome;
|
||||
if (fault === "blocked-wasm" || fault === "timeout")
|
||||
assert.match(result.error?.message ?? "", /watchdog/);
|
||||
else {
|
||||
assert.equal(result.error, undefined);
|
||||
assert.equal(result.value.status, "failed");
|
||||
assert.match(
|
||||
result.value.error,
|
||||
fault === "unknown-frame"
|
||||
? /Unexpected frame type/
|
||||
: fault === "context-loss"
|
||||
? /context lost/
|
||||
: /canceled|Stale/,
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(`PASS ${fault} (unscored)`);
|
||||
} finally {
|
||||
await bounded(context.close(), 5000);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await bounded(browser.close(), 5000);
|
||||
} catch {
|
||||
process.kill("SIGKILL");
|
||||
}
|
||||
try {
|
||||
await bounded(server.close(), 5000);
|
||||
} catch {
|
||||
process.kill("SIGKILL");
|
||||
}
|
||||
assert.ok(
|
||||
process.exitCode !== null || process.signalCode !== null,
|
||||
"browser process must exit",
|
||||
);
|
||||
}
|
||||
@ -1,163 +0,0 @@
|
||||
// Unscored checks of the actual playground pages, including real pointer/wheel
|
||||
// events. Expose the mounted adapter only in this diagnostic routing layer.
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import { chromium } from "playwright";
|
||||
import { bounded } from "../renderer-benchmarks.js";
|
||||
import { scenarios } from "../../resources/wasm-playground/scenes/index.js";
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
filter: { type: "string", default: "" },
|
||||
headed: { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
const frontend = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const selected = scenarios.filter((s) => s.id.includes(values.filter));
|
||||
assert.ok(selected.length, "No matching scenes");
|
||||
const server = await chromium.launchServer({
|
||||
headless: !values.headed,
|
||||
args: ["--enable-gpu"],
|
||||
});
|
||||
const browser = await chromium.connect(server.wsEndpoint());
|
||||
try {
|
||||
for (const scenario of selected) {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
deviceScaleFactor: 2,
|
||||
});
|
||||
try {
|
||||
await bounded(
|
||||
(async () => {
|
||||
await context.route("http://benchmark.invalid/**", async (route) => {
|
||||
const pathname = new URL(route.request().url()).pathname;
|
||||
const base = path.join(
|
||||
frontend,
|
||||
pathname.startsWith("/wasm-playground/")
|
||||
? "resources"
|
||||
: "resources/public",
|
||||
);
|
||||
const target = path.resolve(base, `.${pathname}`);
|
||||
if (!target.startsWith(`${base}/`)) return route.abort();
|
||||
if (pathname === "/wasm-playground/js/lib.js") {
|
||||
const source = await fs.readFile(target, "utf8");
|
||||
await route.fulfill({
|
||||
contentType: "text/javascript",
|
||||
body:
|
||||
source.replace(
|
||||
"export async function mountScenario(",
|
||||
"async function mountOriginalScenario(",
|
||||
) +
|
||||
`
|
||||
export async function mountScenario(...args) {
|
||||
const adapter = await mountOriginalScenario(...args);
|
||||
const render = adapter.module._render;
|
||||
const view = adapter.module._set_view;
|
||||
globalThis.playgroundProbe = { full: 0, views: [] };
|
||||
adapter.module._render = (...args) => {
|
||||
const result = render(...args);
|
||||
if (result === 2) globalThis.playgroundProbe.full++;
|
||||
return result;
|
||||
};
|
||||
adapter.module._set_view = (...args) => {
|
||||
globalThis.playgroundProbe.views.push(args);
|
||||
return view(...args);
|
||||
};
|
||||
globalThis.playgroundAdapter = adapter;
|
||||
return adapter;
|
||||
}`,
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
await route.fulfill({ path: target });
|
||||
} catch {
|
||||
await route.fulfill({
|
||||
status: 404,
|
||||
body: "Missing diagnostic asset",
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const errors = [];
|
||||
page.on("pageerror", (error) => errors.push(error.message));
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error") errors.push(message.text());
|
||||
});
|
||||
await page.goto(
|
||||
`http://benchmark.invalid/wasm-playground/${scenario.id}.html?seed=42`,
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
globalThis.playgroundAdapter || document.querySelector("#error"),
|
||||
null,
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
assert.equal(
|
||||
await page.locator("#error").count(),
|
||||
0,
|
||||
"page must mount successfully",
|
||||
);
|
||||
await page.mouse.move(800, 400);
|
||||
await page.mouse.down();
|
||||
await page.mouse.move(860, 430, { steps: 3 });
|
||||
await page.mouse.up();
|
||||
await page.waitForFunction(
|
||||
() => globalThis.playgroundProbe.full >= 1,
|
||||
);
|
||||
await page.evaluate(() => {
|
||||
globalThis.playgroundProbe.full = 0;
|
||||
});
|
||||
await page.mouse.wheel(0, -100);
|
||||
await page.waitForFunction(
|
||||
() => globalThis.playgroundProbe.full >= 1,
|
||||
);
|
||||
const state = await page.evaluate(() => ({
|
||||
views: globalThis.playgroundProbe.views,
|
||||
lost: document
|
||||
.querySelector("canvas")
|
||||
.getContext("webgl2")
|
||||
.isContextLost(),
|
||||
}));
|
||||
assert.equal(state.lost, false);
|
||||
assert.ok(
|
||||
state.views.some(([scale, x]) => scale === 1 && x > 0),
|
||||
"pointer drag must pan",
|
||||
);
|
||||
assert.ok(
|
||||
state.views.some(([scale]) => scale > 1),
|
||||
"wheel must zoom",
|
||||
);
|
||||
assert.deepEqual(errors, []);
|
||||
await bounded(
|
||||
page.evaluate(() => globalThis.playgroundAdapter.dispose()),
|
||||
5000,
|
||||
);
|
||||
console.log(
|
||||
`PASS ${scenario.id}: legacy mount, pan, zoom and disposal (unscored)`,
|
||||
);
|
||||
})(),
|
||||
65000,
|
||||
);
|
||||
} catch (error) {
|
||||
process.exitCode = 1;
|
||||
console.error(`FAIL ${scenario.id}: ${error.stack ?? error}`);
|
||||
} finally {
|
||||
await bounded(context.close(), 5000);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await bounded(browser.close(), 5000);
|
||||
} catch {
|
||||
server.process()?.kill("SIGKILL");
|
||||
}
|
||||
try {
|
||||
await bounded(server.close(), 5000);
|
||||
} catch {
|
||||
server.process()?.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
@ -1,96 +0,0 @@
|
||||
// Unscored correctness smoke. Request routing serves local files without a service.
|
||||
// Optional screenshots belong only to this diagnostic process, never scored runs.
|
||||
import { chromium } from "playwright";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import { scenarios } from "../../resources/wasm-playground/scenes/index.js";
|
||||
import { collectCases } from "../../resources/wasm-playground/benchmarks/contract.js";
|
||||
import { bounded } from "../renderer-benchmarks.js";
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
screenshots: { type: "string" },
|
||||
headed: { type: "boolean", default: false },
|
||||
filter: { type: "string", default: "" },
|
||||
},
|
||||
});
|
||||
const frontend = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const cases = collectCases(scenarios, 42, values.filter);
|
||||
const server = await chromium.launchServer({
|
||||
headless: !values.headed,
|
||||
args: ["--enable-gpu"],
|
||||
});
|
||||
const browser = await chromium.connect(server.wsEndpoint());
|
||||
try {
|
||||
if (values.screenshots)
|
||||
await fs.mkdir(values.screenshots, { recursive: true });
|
||||
for (const c of cases) {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
deviceScaleFactor: 2,
|
||||
});
|
||||
await context.route("http://benchmark.invalid/**", async (route) => {
|
||||
const pathname = new URL(route.request().url()).pathname;
|
||||
const base = pathname.startsWith("/wasm-playground/")
|
||||
? path.join(frontend, "resources")
|
||||
: path.join(frontend, "resources/public");
|
||||
const target = path.resolve(base, `.${pathname}`);
|
||||
if (!target.startsWith(`${base}/`)) return route.abort();
|
||||
try {
|
||||
await route.fulfill({ path: target });
|
||||
} catch {
|
||||
await route.fulfill({ status: 404, body: "Missing diagnostic asset" });
|
||||
}
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const errors = [];
|
||||
page.on("pageerror", (error) => errors.push(error.message));
|
||||
await page.goto("http://benchmark.invalid/wasm-playground/benchmark.html");
|
||||
await page.waitForFunction(
|
||||
() => globalThis.rendererBenchmarkReady === true,
|
||||
null,
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
const attempt = await bounded(
|
||||
page.evaluate(
|
||||
([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o),
|
||||
[
|
||||
c,
|
||||
{
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
dpr: 2,
|
||||
timeoutMs: 30000,
|
||||
moduleUrl: "http://benchmark.invalid/js/render-wasm-benchmark.js",
|
||||
},
|
||||
],
|
||||
),
|
||||
65000,
|
||||
);
|
||||
if (attempt.status === "failed" || errors.length)
|
||||
throw new Error(`${c.id}: ${attempt.error ?? errors.join("; ")}`);
|
||||
if (attempt.slices.at(-1)?.frameType !== 2)
|
||||
throw new Error(`${c.id}: no Full completion`);
|
||||
if (values.screenshots && c.group === "fresh-module-context")
|
||||
await page.screenshot({
|
||||
path: path.join(values.screenshots, `${c.scenarioId}.png`),
|
||||
});
|
||||
console.log(
|
||||
`PASS ${c.id}; ${attempt.environment.webgl.renderer}; ${attempt.slices.length} render calls (unscored)`,
|
||||
);
|
||||
await page.evaluate(() => globalThis.rendererBenchmark.dispose());
|
||||
await bounded(context.close(), 5000);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await bounded(browser.close(), 5000);
|
||||
} catch {
|
||||
server.process()?.kill("SIGKILL");
|
||||
}
|
||||
try {
|
||||
await bounded(server.close(), 5000);
|
||||
} catch {
|
||||
server.process()?.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
@ -177,10 +177,27 @@
|
||||
:elide-asserts true
|
||||
:anon-fn-naming-policy :off}}}
|
||||
|
||||
:performance-tests
|
||||
{:target :esm
|
||||
:output-dir "target/performance-tests"
|
||||
:runtime :custom
|
||||
:js-options {:js-provider :import}
|
||||
|
||||
|
||||
:modules
|
||||
{:test {:init-fn performance-tests.render-wasm.runner/init
|
||||
:prepend-js ";if (typeof globalThis.navigator?.userAgent === 'undefined') { globalThis.navigator = {userAgent: ''}; };"}}
|
||||
|
||||
:release
|
||||
{:compiler-options
|
||||
{:fn-invoke-direct true
|
||||
:elide-asserts true
|
||||
:anon-fn-naming-policy :off}}}
|
||||
|
||||
:bench-render-wasm-runner
|
||||
{:target :node-script
|
||||
:output-to "target/renderer-benchmarks/runner.cjs"
|
||||
:output-dir "target/renderer-benchmarks/runner/"
|
||||
:output-to "target/performance/render-wasm/runner.cjs"
|
||||
:output-dir "target/performance/render-wasm/runner/"
|
||||
:main benches.render-wasm.node.runner/-main
|
||||
:devtools {:autoload false}
|
||||
|
||||
@ -197,8 +214,8 @@
|
||||
|
||||
:bench-render-wasm-pilot
|
||||
{:target :node-script
|
||||
:output-to "target/renderer-benchmarks/pilot.cjs"
|
||||
:output-dir "target/renderer-benchmarks/pilot/"
|
||||
:output-to "target/performance/render-wasm/pilot.cjs"
|
||||
:output-dir "target/performance/render-wasm/pilot/"
|
||||
:main benches.render-wasm.node.pilot/-main
|
||||
:devtools {:autoload false}
|
||||
|
||||
@ -215,7 +232,7 @@
|
||||
|
||||
:bench-render-wasm-browser
|
||||
{:target :esm
|
||||
:output-dir "target/renderer-benchmarks/browser/"
|
||||
:output-dir "target/performance/render-wasm/browser/"
|
||||
;; Bundle npm deps (date-fns via app.common): the pilot serves plain
|
||||
;; static files with no import-map resolver. TODO(10): revisit when
|
||||
;; ticket 10 owns staging; artifacts must stay self-contained.
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.boolean-test
|
||||
(ns performance-tests.render-wasm.boolean-test
|
||||
(:require
|
||||
[app.common.geom.point :as gpt]
|
||||
[app.common.geom.shapes.intersect :as gint]
|
||||
@ -14,7 +14,7 @@
|
||||
[benches.render-wasm.builder :as b :include-macros true]
|
||||
[benches.render-wasm.snapshot :as snapshot]
|
||||
[cljs.test :as t :include-macros true]
|
||||
[frontend-tests.benches.test-helpers :as helpers]))
|
||||
[performance-tests.render-wasm.test-helpers :as helpers]))
|
||||
|
||||
(t/deftest difference-of-overlapping-rects-keeps-hierarchy-and-bounds
|
||||
(let [red {:fill-color "#ff0000" :fill-opacity 1}
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.builder-test
|
||||
(ns performance-tests.render-wasm.builder-test
|
||||
"Tests canonical scene construction and the shared seeded number generator.
|
||||
Fixed random sequences preserve scene draws during generator extraction."
|
||||
(:require
|
||||
@ -15,7 +15,7 @@
|
||||
[benches.render-wasm.scenes.rects]
|
||||
[benches.render-wasm.snapshot :as snapshot]
|
||||
[cljs.test :as t :include-macros true]
|
||||
[frontend-tests.benches.test-helpers :as helpers]))
|
||||
[performance-tests.render-wasm.test-helpers :as helpers]))
|
||||
|
||||
(defn- build
|
||||
([] (build {}))
|
||||
@ -4,12 +4,12 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.containers-test
|
||||
(ns performance-tests.render-wasm.containers-test
|
||||
(:require
|
||||
[app.common.uuid :as uuid]
|
||||
[benches.render-wasm.builder :as builder :include-macros true]
|
||||
[cljs.test :as t :include-macros true]
|
||||
[frontend-tests.benches.test-helpers :as helpers]))
|
||||
[performance-tests.render-wasm.test-helpers :as helpers]))
|
||||
|
||||
(t/deftest frame-attaches-children-in-order
|
||||
(let [instance (builder/scene {:seed 1}
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.contracts-test
|
||||
(ns performance-tests.render-wasm.contracts-test
|
||||
(:require
|
||||
[app.common.schema :as sm]
|
||||
[app.common.transit :as transit]
|
||||
@ -425,7 +425,7 @@
|
||||
(t/is (decl/transit-round-trips? (first collected))))
|
||||
(t/is (empty? @body-calls))
|
||||
(let [entry (decl/registered-case :contracts-scene/case)]
|
||||
(t/is (= "frontend-tests.benches.contracts-test" (:ns entry)))
|
||||
(t/is (= "performance-tests.render-wasm.contracts-test" (:ns entry)))
|
||||
(t/is (fn? (:run! entry)))
|
||||
(t/is (= :runtime ((:run! entry) :runtime)))
|
||||
(t/is (= [:touched] @body-calls))))
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.fingerprint-test
|
||||
(ns performance-tests.render-wasm.fingerprint-test
|
||||
"Tests content identity with production records, paths and collection types.
|
||||
Validated scene snapshots retain child order even when maps reorder."
|
||||
(:require
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.geometry-scenes-test
|
||||
(ns performance-tests.render-wasm.geometry-scenes-test
|
||||
(:require
|
||||
[app.common.types.path :as path]
|
||||
[app.common.uuid :as uuid]
|
||||
@ -16,7 +16,7 @@
|
||||
[benches.render-wasm.scenes.plus]
|
||||
[benches.render-wasm.snapshot :as snapshot]
|
||||
[cljs.test :as t :include-macros true]
|
||||
[frontend-tests.benches.test-helpers :as helpers]))
|
||||
[performance-tests.render-wasm.test-helpers :as helpers]))
|
||||
|
||||
(t/deftest curved-paths-use-canonical-content-and-stable-order
|
||||
(let [params {:count 12 :width 1920 :height 1080 :seed 42}
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.lifecycle-test
|
||||
(ns performance-tests.render-wasm.lifecycle-test
|
||||
"Tests the browser bridge with fake DOM and renderer boundaries.
|
||||
|
||||
It covers bridge input, ownership, disposal and `load-scene` with fake
|
||||
@ -253,6 +253,8 @@
|
||||
prev-window (unchecked-get js/globalThis "window")
|
||||
prev-import (unchecked-get js/globalThis "dynamicImport")
|
||||
prev-performance (unchecked-get js/globalThis "performance")
|
||||
prev-now-fn (unchecked-get prev-performance "now")
|
||||
prev-now-desc (js/Object.getOwnPropertyDescriptor prev-performance "now")
|
||||
prev-now (unchecked-get js/globalThis "__benchNow")
|
||||
prev-script (unchecked-get js/globalThis "__benchRenderScript")
|
||||
prev-calls (unchecked-get js/globalThis "__benchCalls")
|
||||
@ -283,19 +285,25 @@
|
||||
(cb timestamp))
|
||||
(cb 0)))
|
||||
nil)}
|
||||
performance #js {:now (fn []
|
||||
(if-some [clock (unchecked-get js/globalThis "__benchNow")]
|
||||
clock
|
||||
(.now prev-performance)))}
|
||||
fake-now (fn []
|
||||
(if-some [clock (unchecked-get js/globalThis "__benchNow")]
|
||||
clock
|
||||
(.call prev-now-fn prev-performance)))
|
||||
dynamic-import (fn [url] (js/eval (str "import(" (pr-str url) ")")))]
|
||||
(unchecked-set js/globalThis "document" document)
|
||||
(unchecked-set js/globalThis "window" window)
|
||||
(unchecked-set js/globalThis "performance" performance)
|
||||
;; Patch `now` on the real object: shadow hangs the compiled
|
||||
;; `performance.*` namespaces off the global `performance` object, so
|
||||
;; replacing that global hides every `benches.render-wasm.*` var.
|
||||
(js/Object.defineProperty prev-performance "now"
|
||||
#js {:value fake-now :configurable true :writable true})
|
||||
(unchecked-set js/globalThis "dynamicImport" dynamic-import)
|
||||
(fn restore! []
|
||||
(unchecked-set js/globalThis "document" prev-document)
|
||||
(unchecked-set js/globalThis "window" prev-window)
|
||||
(unchecked-set js/globalThis "performance" prev-performance)
|
||||
(if (undefined? prev-now-desc)
|
||||
(js-delete prev-performance "now")
|
||||
(js/Object.defineProperty prev-performance "now" prev-now-desc))
|
||||
(unchecked-set js/globalThis "__benchNow" prev-now)
|
||||
(unchecked-set js/globalThis "__benchRenderScript" prev-script)
|
||||
(unchecked-set js/globalThis "__benchCalls" prev-calls)
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.protocol-test
|
||||
(ns performance-tests.render-wasm.protocol-test
|
||||
"Tests drain, restore and interact with an injected clock and renderer.
|
||||
Browser entry tests cover load and warm pan with a fake module."
|
||||
(:require
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.report-test
|
||||
(ns performance-tests.render-wasm.report-test
|
||||
"Pure tests for raw records, exploratory statistics and saved-run comparisons.
|
||||
Fixtures use saved case definitions without importing a registry or renderer."
|
||||
(:require
|
||||
195
frontend/test/performance_tests/render_wasm/runner.cljs
Normal file
195
frontend/test/performance_tests/render_wasm/runner.cljs
Normal file
@ -0,0 +1,195 @@
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns performance-tests.render-wasm.runner
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[cljs.test :as t]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.cli :refer [parse-opts]]
|
||||
[goog.object :as gobj]
|
||||
[performance-tests.render-wasm.boolean-test]
|
||||
[performance-tests.render-wasm.builder-test]
|
||||
[performance-tests.render-wasm.containers-test]
|
||||
[performance-tests.render-wasm.contracts-test]
|
||||
[performance-tests.render-wasm.fingerprint-test]
|
||||
[performance-tests.render-wasm.geometry-scenes-test]
|
||||
[performance-tests.render-wasm.lifecycle-test]
|
||||
[performance-tests.render-wasm.protocol-test]
|
||||
[performance-tests.render-wasm.report-test]
|
||||
[performance-tests.render-wasm.scene-snapshot-test]))
|
||||
|
||||
(enable-console-print!)
|
||||
|
||||
(defmethod t/report [:cljs.test/default :begin-test-var] [m]
|
||||
(let [v (:var m)]
|
||||
(println (str " ▸ " (:ns (meta v)) "/" (:name (meta v))))))
|
||||
|
||||
(defmethod t/report [:cljs.test/default :end-run-tests] [m]
|
||||
(if (cljs.test/successful? m)
|
||||
(.exit js/process 0)
|
||||
(.exit js/process 1)))
|
||||
|
||||
(def test-namespaces
|
||||
['performance-tests.render-wasm.boolean-test
|
||||
'performance-tests.render-wasm.builder-test
|
||||
'performance-tests.render-wasm.containers-test
|
||||
'performance-tests.render-wasm.contracts-test
|
||||
'performance-tests.render-wasm.fingerprint-test
|
||||
'performance-tests.render-wasm.geometry-scenes-test
|
||||
'performance-tests.render-wasm.lifecycle-test
|
||||
'performance-tests.render-wasm.protocol-test
|
||||
'performance-tests.render-wasm.report-test
|
||||
'performance-tests.render-wasm.scene-snapshot-test])
|
||||
|
||||
(assert (every? find-ns-obj test-namespaces)
|
||||
"test-namespaces contains a namespace that isn't required in runner.cljs")
|
||||
|
||||
;; This runner mirrors frontend-tests.runner for the bench-only
|
||||
;; :performance-tests target. Both runners need forwarded CLI args, focused
|
||||
;; namespace/var execution, fixture preservation, and app log-level
|
||||
;; setup. The logic stays local to avoid cross-target test classpath
|
||||
;; coupling.
|
||||
(def ^:private log-levels
|
||||
#{:trace :debug :info :warn :error})
|
||||
|
||||
(def cli-options
|
||||
[["-f" "--focus FOCUS" "Run one test namespace or one test var, e.g. performance-tests.render-wasm.contracts-test"]
|
||||
["-l" "--log-level LEVEL" "Set app logger level: trace|debug|info|warn|error"
|
||||
:parse-fn keyword
|
||||
:validate [log-levels "must be one of trace, debug, info, warn, error"]]
|
||||
["-h" "--help"]])
|
||||
|
||||
(defn- argv
|
||||
[]
|
||||
(let [args (->> (.-argv js/process)
|
||||
(array-seq)
|
||||
(drop 2))]
|
||||
;; `pnpm run test:performance-tests -- --focus ...` forwards the separator to
|
||||
;; the node process, so drop one leading `--` before handing args to
|
||||
;; tools.cli.
|
||||
(cond-> args
|
||||
(= "--" (first args)) rest)))
|
||||
|
||||
(defn- usage
|
||||
[summary]
|
||||
(str "Usage: pnpm run test:performance-tests -- [options]\n\n"
|
||||
"Options:\n"
|
||||
summary "\n\n"
|
||||
"Focus examples:\n"
|
||||
" pnpm run test:performance-tests -- --focus performance-tests.render-wasm.contracts-test\n"
|
||||
" pnpm run test:performance-tests -- --focus performance-tests.render-wasm.contracts-test/some-test-var\n\n"
|
||||
"Log level example (quiets app logging during the run):\n"
|
||||
" pnpm run test:performance-tests -- --focus performance-tests.render-wasm.contracts-test --log-level warn"))
|
||||
|
||||
(defn- fail!
|
||||
[message]
|
||||
(js/console.error message)
|
||||
(.exit js/process 1))
|
||||
|
||||
(defn- parse-focus
|
||||
[focus]
|
||||
(let [[ns-name test-name & extra] (str/split focus #"/")]
|
||||
(cond
|
||||
(or (str/blank? ns-name) (seq extra))
|
||||
(fail! (str "Invalid --focus value: " focus))
|
||||
|
||||
(some? test-name)
|
||||
{:ns (symbol ns-name) :test test-name}
|
||||
|
||||
:else
|
||||
{:ns (symbol ns-name)})))
|
||||
|
||||
(defn- fixture-value
|
||||
[ns-obj fixture-name]
|
||||
(let [value (gobj/get ns-obj (munge fixture-name))]
|
||||
(when-not (undefined? value)
|
||||
value)))
|
||||
|
||||
(defn- ns-test-vars
|
||||
[ns-sym]
|
||||
(when-let [ns-obj (find-ns-obj ns-sym)]
|
||||
(->> (js-keys ns-obj)
|
||||
(keep (fn [key]
|
||||
(some-> (gobj/get ns-obj key)
|
||||
(.-cljs$lang$var))))
|
||||
(filter (comp :test meta))
|
||||
(sort-by (comp :line meta)))))
|
||||
|
||||
(defn- ns-fixtures
|
||||
[ns-sym vars]
|
||||
(when-let [ns-obj (find-ns-obj ns-sym)]
|
||||
(let [ns-key (or (some-> vars first meta :ns) ns-sym)
|
||||
once-fixtures (fixture-value ns-obj "cljs-test-once-fixtures")
|
||||
each-fixtures (fixture-value ns-obj "cljs-test-each-fixtures")]
|
||||
{:once (when once-fixtures {ns-key once-fixtures})
|
||||
:each (when each-fixtures {ns-key each-fixtures})})))
|
||||
|
||||
(defn- selected-tests
|
||||
[{:keys [ns test]}]
|
||||
(when-not (some #{ns} test-namespaces)
|
||||
(fail! (str "Unknown test namespace: " ns)))
|
||||
(let [vars (vec (ns-test-vars ns))]
|
||||
(when (empty? vars)
|
||||
(fail! (str "No tests found in namespace: " ns)))
|
||||
(if test
|
||||
(let [test-sym (symbol test)
|
||||
test-var (some #(when (= test-sym (:name (meta %))) %) vars)]
|
||||
(if test-var
|
||||
{:vars [test-var]
|
||||
:fixtures (ns-fixtures ns [test-var])}
|
||||
(fail! (str "Unknown test var: " ns "/" test))))
|
||||
{:vars vars
|
||||
:fixtures (ns-fixtures ns vars)})))
|
||||
|
||||
(defn- merge-fixtures
|
||||
[fixtures]
|
||||
{:once (apply merge (keep :once fixtures))
|
||||
:each (apply merge (keep :each fixtures))})
|
||||
|
||||
(defn- run-test-vars!
|
||||
[tests]
|
||||
(let [vars (vec (mapcat :vars tests))
|
||||
fixtures (merge-fixtures (map :fixtures tests))
|
||||
env (assoc (t/empty-env)
|
||||
:once-fixtures (:once fixtures)
|
||||
:each-fixtures (:each fixtures))
|
||||
summary (volatile! {:test 0 :pass 0 :fail 0 :error 0 :type :summary})]
|
||||
|
||||
(t/set-env! env)
|
||||
|
||||
(t/run-block
|
||||
(concat (t/test-vars-block vars)
|
||||
[(fn []
|
||||
(vswap! summary
|
||||
(partial merge-with +)
|
||||
(:report-counters (t/get-current-env))))
|
||||
(fn []
|
||||
(t/report @summary)
|
||||
(t/report (assoc @summary :type :end-run-tests)))]))))
|
||||
|
||||
(defn- run-focused-test!
|
||||
[focus]
|
||||
(run-test-vars! [(selected-tests (parse-focus focus))]))
|
||||
|
||||
(defn init
|
||||
[]
|
||||
(let [{:keys [options errors summary]} (parse-opts (argv) cli-options)]
|
||||
(cond
|
||||
(seq errors)
|
||||
(fail! (str/join "\n" errors))
|
||||
|
||||
(:help options)
|
||||
(do
|
||||
(println (usage summary))
|
||||
(.exit js/process 0))
|
||||
|
||||
:else
|
||||
(do
|
||||
(l/setup! {:app (or (:log-level options) :warn)})
|
||||
|
||||
(if (:focus options)
|
||||
(run-focused-test! (:focus options))
|
||||
(run-test-vars! (map #(selected-tests {:ns %}) test-namespaces)))))))
|
||||
@ -4,7 +4,7 @@
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns frontend-tests.benches.scene-snapshot-test
|
||||
(ns performance-tests.render-wasm.scene-snapshot-test
|
||||
(:require
|
||||
[app.common.schema :as sm]
|
||||
[app.common.types.shape :as cts]
|
||||
@ -1,4 +1,4 @@
|
||||
(ns frontend-tests.benches.test-helpers
|
||||
(ns performance-tests.render-wasm.test-helpers
|
||||
"Shared helpers for renderer benchmark scene tests.
|
||||
|
||||
These helpers assert nothing about shapes or the renderer. They only
|
||||
Loading…
x
Reference in New Issue
Block a user