From 731bcc566486cd0457f6b9f94b870cd03f1dc0df Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Thu, 17 Sep 2026 16:51:12 +0200 Subject: [PATCH] :tada: Add playground renderer benchmark suite Share seven deterministic fixtures with the playground pages and measure renderer lifecycle, pan, and zoom through a prepared Playwright runner. Retain raw attempts and failures with offline statistical comparisons. Add focused tests and separate browser fault diagnostics. Full-suite calibration and shadow GPU validation remain tracked follow-up work. AI-assisted-by: GPT-6 --- frontend/package.json | 4 + .../resources/wasm-playground/BENCHMARKS.md | 111 ++++ .../resources/wasm-playground/benchmark.html | 31 + .../wasm-playground/benchmarks/browser.js | 249 ++++++++ .../wasm-playground/benchmarks/contract.js | 124 ++++ .../wasm-playground/benchmarks/protocol.js | 77 +++ .../wasm-playground/benchmarks/statistics.js | 269 +++++++++ frontend/resources/wasm-playground/clips.html | 134 ++--- frontend/resources/wasm-playground/index.html | 57 +- frontend/resources/wasm-playground/js/lib.js | 528 ++++++++--------- frontend/resources/wasm-playground/masks.html | 129 ++--- frontend/resources/wasm-playground/paths.html | 126 ++--- frontend/resources/wasm-playground/plus.html | 150 ++--- frontend/resources/wasm-playground/rects.html | 123 ++-- .../resources/wasm-playground/scenes/clips.js | 43 ++ .../resources/wasm-playground/scenes/index.js | 9 + .../resources/wasm-playground/scenes/masks.js | 44 ++ .../resources/wasm-playground/scenes/paths.js | 15 + .../resources/wasm-playground/scenes/plus.js | 15 + .../resources/wasm-playground/scenes/rects.js | 15 + .../wasm-playground/scenes/shadows.js | 26 + .../wasm-playground/scenes/shared.js | 266 +++++++++ .../resources/wasm-playground/scenes/texts.js | 123 ++++ .../resources/wasm-playground/shadows.html | 131 ++--- frontend/resources/wasm-playground/texts.html | 144 ++--- frontend/scripts/renderer-benchmarks.js | 530 ++++++++++++++++++ .../scripts/renderer-benchmarks/compare.js | 281 ++++++++++ .../renderer-benchmarks/tests/adapter.test.js | 106 ++++ .../renderer-benchmarks/tests/compare.test.js | 181 ++++++ .../tests/protocol.test.js | 97 ++++ .../renderer-benchmarks/tests/runner.test.js | 132 +++++ .../renderer-benchmarks/tests/scenes.test.js | 98 ++++ .../tests/statistics.test.js | 102 ++++ .../renderer-benchmarks/verify-failures.js | 154 +++++ .../scripts/renderer-benchmarks/verify.js | 96 ++++ render-wasm/build | 11 +- 36 files changed, 3789 insertions(+), 942 deletions(-) create mode 100644 frontend/resources/wasm-playground/BENCHMARKS.md create mode 100644 frontend/resources/wasm-playground/benchmark.html create mode 100644 frontend/resources/wasm-playground/benchmarks/browser.js create mode 100644 frontend/resources/wasm-playground/benchmarks/contract.js create mode 100644 frontend/resources/wasm-playground/benchmarks/protocol.js create mode 100644 frontend/resources/wasm-playground/benchmarks/statistics.js create mode 100644 frontend/resources/wasm-playground/scenes/clips.js create mode 100644 frontend/resources/wasm-playground/scenes/index.js create mode 100644 frontend/resources/wasm-playground/scenes/masks.js create mode 100644 frontend/resources/wasm-playground/scenes/paths.js create mode 100644 frontend/resources/wasm-playground/scenes/plus.js create mode 100644 frontend/resources/wasm-playground/scenes/rects.js create mode 100644 frontend/resources/wasm-playground/scenes/shadows.js create mode 100644 frontend/resources/wasm-playground/scenes/shared.js create mode 100644 frontend/resources/wasm-playground/scenes/texts.js create mode 100644 frontend/scripts/renderer-benchmarks.js create mode 100644 frontend/scripts/renderer-benchmarks/compare.js create mode 100644 frontend/scripts/renderer-benchmarks/tests/adapter.test.js create mode 100644 frontend/scripts/renderer-benchmarks/tests/compare.test.js create mode 100644 frontend/scripts/renderer-benchmarks/tests/protocol.test.js create mode 100644 frontend/scripts/renderer-benchmarks/tests/runner.test.js create mode 100644 frontend/scripts/renderer-benchmarks/tests/scenes.test.js create mode 100644 frontend/scripts/renderer-benchmarks/tests/statistics.test.js create mode 100644 frontend/scripts/renderer-benchmarks/verify-failures.js create mode 100644 frontend/scripts/renderer-benchmarks/verify.js diff --git a/frontend/package.json b/frontend/package.json index 4e1433e206..37cf86285a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,10 @@ "url": "https://github.com/penpot/penpot" }, "scripts": { + "benchmark:renderer": "node scripts/renderer-benchmarks.js", + "test:renderer-benchmarks": "node --test scripts/renderer-benchmarks/tests/*.test.js", + "verify:renderer-benchmarks": "node scripts/renderer-benchmarks/verify.js", + "verify:renderer-benchmark-failures": "node scripts/renderer-benchmarks/verify-failures.js", "build:app:assets": "node ./scripts/build-app-assets.js", "build:fonts-preview": "node ./scripts/build-fonts-preview.js", "build:storybook": "(cd packages/ui && pnpm run build) && pnpm run build:storybook:assets && pnpm run build:storybook:cljs && storybook build", diff --git a/frontend/resources/wasm-playground/BENCHMARKS.md b/frontend/resources/wasm-playground/BENCHMARKS.md new file mode 100644 index 0000000000..3df5a6901d --- /dev/null +++ b/frontend/resources/wasm-playground/BENCHMARKS.md @@ -0,0 +1,111 @@ +# Renderer benchmarks + +Run from `frontend/` in the prepared Penpot development container, as the normal +`penpot` user. Dependencies, the matching Playwright Chromium installation, and +the static server must already exist. Start the server separately with +`node scripts/e2e-server.js`. The runner never installs packages or browsers, +starts services, or cleans Cargo targets. + +Use the Cargo cache prepared for that user. In this container, noninteractive +shells inherit `CARGO_HOME=/opt/cargo`, while the populated cache is +`/home/penpot/.cargo`. For those shells, prefix the command with +`CARGO_HOME=/home/penpot/.cargo`. The runner preserves the supplied environment; +it does not move caches or download missing crates. + +```sh +pnpm run benchmark:renderer run --output /tmp/baseline.json +pnpm run benchmark:renderer run --filter texts --seed 42 --output /tmp/texts.json +pnpm run benchmark:renderer run --headed --filter rects/default/zoom +pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json +pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json --diagnostic +pnpm run test:renderer-benchmarks +``` + +Each run builds the ordinary optimized frontend renderer once using +`PENPOT_WASM_PREPARED=1`, offline Cargo, and the existing build/copy path. Setup is +skipped, package-manager network fallback is disabled, and the emitted module is +named `render-wasm-benchmark`. Playground assets are refreshed before launch. A +per-run marker checks that the server points at this checkout. Concurrent builds +or benchmark commands against the same checkout are unsupported. + +The default server URL is `http://localhost:3000`; override it with `--base-url`. +Default JSON output goes to a unique file under the system temporary directory. +Use `--warmups`, `--repetitions`, and `--timeout-ms` to override the provisional +defaults of 3, 10, and 30000. Counts are fixed, not adaptive; failures are never +silently replaced. A failed case stops its remaining attempts and records how +many were unattempted, then later independent cases can continue. A lost browser +ends the run and retains earlier results. Exit status reports execution validity, +not a performance regression. + +## Workloads and state + +Seven versioned scenes use a seeded generator: 1000 rectangles, stars, lines, or +shadowed rectangles; 100 text shapes; and fixed mask/clip scenes. Scene generation +and buffer construction happen before upload timing. Text uses the renderer's +embedded fallback font, not production font fetching. The JavaScript fixtures +share their implementation with the interactive playground pages; `?seed=42` is +repeatable, and numeric shape-count overrides remain available on those pages. + +All cases run serially in one Chromium process with a 1920x1080 CSS viewport, +DPR 2, and product WebGL2 context options. Effective sizes, context attributes, +browser/platform identity, seeds, and workload parameters appear in JSON. +Software WebGL is allowed and warns; a GPU launch flag does not prove hardware +acceleration. Fresh pages do not clear browser-process, compilation, or driver +caches. + +- **Fresh module/context:** each attempt gets a new page/context and module. + Module initialization starts at the Emscripten factory and includes WASM + fetching/instantiation; the module's JavaScript import precedes timing. +- **Context initialization:** canvas/context registration, native initialization, + render/browser options, and viewport sizing. +- **Upload:** direct FFI upload, pending text layouts, background, initial view, + and initial tile-index setup. This is not the frontend's serialization or + batched asynchronous upload path. +- **First render:** request to `ViewportReady` and final `Full`, including rAF + scheduling. An immediate `Full` supplies both boundaries. +- **Warm pan/zoom:** separate case-owned instances; restore the starting view and + finish preparation rendering before each attempt. Caches evolve within each + case. Restoring the view is not a cache reset. + +Interactions use declared frame paths, cached rendering, a declared 100 ms +settling interval, and product finalization/continuation flags. The interval +models a settling case, not every possible editor gesture. Finalization time +starts before `_set_view_end`, and that call's duration also appears separately. +Last-input-to-`Full` additionally includes the delay. Raw render/cached calls, +timestamps, flags, frame types, and zero-duration slices remain available. + +## Interpretation + +These are renderer submission timings. They do not measure GPU completion, +displayed pixels, input delivery, whole-editor responsiveness, network behavior, +or visual correctness. WASM buffer size is context, not attributed CPU/GPU memory. + +Reports retain means, medians, ranges, sample deviation, and unscaled median +absolute deviation. Nominal 95% bootstrap intervals require 10 valid attempts; +p95 requires 100 and p99 requires 500. Intervals are exploratory and conditional +on these runs: serial correlation, cache history, system load, and thermal drift +remain. Confirm important findings with repeated complete runs. No p-values or +automated regression thresholds are provided. + +Offline comparison checks scenario/version, seeds, resolved workloads, +preparation, environment, and metric semantics. Source/build changes are expected. +`--diagnostic` lists incompatibilities and permits inspection without claiming +equivalence. Dirty worktrees warn and continue, recording only SHA and dirty +state, not diffs or artifact hashes. + +## Separate fixture validation + +```sh +pnpm run verify:renderer-benchmarks +pnpm run verify:renderer-benchmarks --screenshots /tmp/renderer-diagnostics +``` + +This unscored smoke uses the already-built benchmark artifact and Playwright +request routing; it starts no server. It checks every workload and interaction +through `Full`. Optional screenshots support independent manual fixture checks. +They never enter scored runs, and smoke durations are not benchmark evidence. + +The implementation roadmap, pilot evidence, and remaining calibration work live +in Serena memory `render-wasm/performance/core` and its numbered tickets. The +2-5 minute full-suite goal excludes build time and is a target, not a guarantee +on software-rendered systems. diff --git a/frontend/resources/wasm-playground/benchmark.html b/frontend/resources/wasm-playground/benchmark.html new file mode 100644 index 0000000000..f68f56ebe0 --- /dev/null +++ b/frontend/resources/wasm-playground/benchmark.html @@ -0,0 +1,31 @@ + + + + + Renderer benchmarks + + + + + + + + diff --git a/frontend/resources/wasm-playground/benchmarks/browser.js b/frontend/resources/wasm-playground/benchmarks/browser.js new file mode 100644 index 0000000000..1b53953163 --- /dev/null +++ b/frontend/resources/wasm-playground/benchmarks/browser.js @@ -0,0 +1,249 @@ +import { scenarios } from "../scenes/index.js"; +import { drain, interact, restore } from "./protocol.js"; +import { WEBGL_OPTIONS, validateMetrics } from "./contract.js"; +import { summarizeRun } from "./statistics.js"; + +let session; +const canvas = document.querySelector("canvas"); + +function check(owner = session) { + if (!owner || owner.canceled) throw new Error("Benchmark canceled"); + if (owner !== session) throw new Error("Stale benchmark operation"); + if (owner.lost || owner.gl?.isContextLost()) + throw new Error("WebGL context lost"); + if (performance.now() > owner.deadline) throw new Error("Operation timeout"); +} + +function runtime() { + const owner = session; + return { + module: owner.module, + now: () => performance.now(), + recordSlice: (slice) => owner.slices.push(slice), + recordCached: (slice) => owner.cachedSlices.push(slice), + check: () => { + if (owner !== session) throw new Error("Stale benchmark operation"); + check(); + }, + frame: () => + new Promise((resolve, reject) => { + owner.cancelPending = reject; + owner.raf = requestAnimationFrame((timestamp) => { + owner.cancelPending = null; + owner.raf = null; + resolve(timestamp); + }); + }), + sleep: (ms) => + new Promise((resolve, reject) => { + owner.cancelPending = reject; + owner.timer = setTimeout(() => { + owner.cancelPending = null; + resolve(); + }, ms); + }), + }; +} + +function webglInfo(gl) { + const extension = gl.getExtension("WEBGL_debug_renderer_info"); + const vendor = gl.getParameter(extension?.UNMASKED_VENDOR_WEBGL ?? gl.VENDOR); + const renderer = gl.getParameter( + extension?.UNMASKED_RENDERER_WEBGL ?? gl.RENDERER, + ); + return { + vendor, + renderer, + version: gl.getParameter(gl.VERSION), + unmasked: Boolean(extension), + acceleration: /swiftshader|llvmpipe|software|softpipe/i.test( + `${vendor} ${renderer}`, + ) + ? "software" + : "unknown", + options: gl.getContextAttributes(), + drawingBuffer: { + width: gl.drawingBufferWidth, + height: gl.drawingBufferHeight, + }, + }; +} + +async function initialize(benchmarkCase, options) { + const scenario = scenarios.find( + (item) => item.id === benchmarkCase.scenarioId, + ); + if (!scenario) + throw new Error(`Unknown scenario: ${benchmarkCase.scenarioId}`); + // Construction and module script import deliberately precede initialization timing. + const scene = scenario.createScene( + benchmarkCase.params, + benchmarkCase.sceneSeed, + ); + const owner = { + slices: [], + cachedSlices: [], + canceled: false, + lost: false, + deadline: performance.now() + options.timeoutMs, + options, + benchmarkCase, + }; + session = owner; + const { default: createModule } = await import(options.moduleUrl); + check(owner); + const moduleStart = performance.now(); + owner.module = await createModule(); + const moduleInitMs = performance.now() - moduleStart; + check(owner); + const contextStart = performance.now(); + canvas.style.width = `${options.viewport.width}px`; + canvas.style.height = `${options.viewport.height}px`; + canvas.width = Math.floor(options.viewport.width * options.dpr); + canvas.height = Math.floor(options.viewport.height * options.dpr); + const gl = canvas.getContext("webgl2", WEBGL_OPTIONS); + if (!gl) throw new Error("WebGL2 is unavailable"); + owner.gl = gl; + // Native initialization queries the unmasked renderer identity. + gl.getExtension("WEBGL_debug_renderer_info"); + owner.onLost = (event) => { + event.preventDefault(); + owner.lost = true; + }; + canvas.addEventListener("webglcontextlost", owner.onLost); + owner.handle = owner.module.GL.registerContext(gl, { majorVersion: 2 }); + owner.module.GL.makeContextCurrent(owner.handle); + owner.module._init(options.viewport.width, options.viewport.height); + owner.initialized = true; + owner.module._set_render_options(0, options.dpr); + owner.module._set_browser(1); // Rust Browser::Chrome + owner.module._resize_viewbox(options.viewport.width, options.viewport.height); + const contextInitMs = performance.now() - contextStart; + // Diagnostic queries are outside renderer and upload timing. + const environment = { + webgl: webglInfo(gl), + canvas: { + cssWidth: canvas.clientWidth, + cssHeight: canvas.clientHeight, + width: canvas.width, + height: canvas.height, + }, + devicePixelRatio: globalThis.devicePixelRatio, + }; + owner.environment = environment; + const uploadStart = performance.now(); + scenario.upload(owner.module, scene); + owner.module._set_canvas_background(0xfffabada); + const view = benchmarkCase.initialView; + owner.module._set_view(view.scale, view.x, view.y); + owner.module._set_view_end(); // Bulk upload needs its initial tile index. + const uploadMs = performance.now() - uploadStart; + const first = await drain(runtime(), { flags: 0 }); + return { + environment, + metrics: { + moduleInitMs, + contextInitMs, + uploadMs, + timeToViewportReadyMs: first.viewportReadyMs, + timeToFullMs: first.fullMs, + }, + slices: first.slices, + cachedSlices: [], + }; +} + +function finish(result) { + const invalidMetrics = validateMetrics(result.metrics); + const slices = result.slices ?? []; + const cachedSlices = result.cachedSlices ?? []; + const timestamps = cachedSlices.map((slice) => slice.timestamp); + return { + ...result, + status: Object.keys(invalidMetrics).length ? "invalid" : "ok", + invalidMetrics, + frameCounts: { + Partial: slices.filter((s) => s.frameType === 1).length, + ViewportReady: slices.filter((s) => s.frameType === 3).length, + Full: slices.filter((s) => s.frameType === 2).length, + }, + rafGapsMs: timestamps.slice(1).map((time, i) => time - timestamps[i]), + renderRafGapsMs: slices + .slice(1) + .map((slice, i) => slice.timestamp - slices[i].timestamp), + zeroSlices: [...slices, ...cachedSlices].filter((s) => s.durationMs === 0) + .length, + wasmMemoryBytes: session.module.HEAPU8.buffer.byteLength, + }; +} + +async function runAttempt(benchmarkCase, options) { + try { + let result; + if (!session) { + result = await initialize(benchmarkCase, options); + if (benchmarkCase.group === "fresh-module-context") return finish(result); + } + session.deadline = performance.now() + options.timeoutMs; + await restore(runtime(), benchmarkCase.initialView); + session.slices = []; + session.cachedSlices = []; + session.deadline = performance.now() + options.timeoutMs; + result = await interact(runtime(), benchmarkCase.interaction); + return finish({ + ...result, + environment: { ...session.environment, webgl: webglInfo(session.gl) }, + }); + } catch (error) { + return { + status: "failed", + metrics: {}, + invalidMetrics: {}, + slices: session?.slices ?? [], + cachedSlices: session?.cachedSlices ?? [], + environment: session?.environment, + error: String(error?.stack ?? error), + }; + } +} + +function dispose() { + if (!session) return; + const owner = session; + owner.canceled = true; + owner.cancelPending?.(new Error("Benchmark canceled")); + cancelAnimationFrame(owner.raf); + clearTimeout(owner.timer); + canvas.removeEventListener("webglcontextlost", owner.onLost); + try { + if (owner.initialized && !owner.lost) { + owner.module._free_gpu_resources?.(); + owner.module._clean_up(); + } + } finally { + if (owner.handle !== undefined) owner.module.GL.deleteContext(owner.handle); + owner.gl?.getExtension("WEBGL_lose_context")?.loseContext(); + session = null; + } +} + +globalThis.rendererBenchmark = { + runAttempt, + dispose, + showResult(result) { + globalThis.rendererBenchmarkResult = result; + const report = document.querySelector("pre"); + report.textContent = JSON.stringify( + { + scope: result.scope, + metadata: result.metadata, + warnings: result.warnings, + summary: result.summary ?? summarizeRun(result), + }, + null, + 2, + ); + report.hidden = false; + }, +}; +globalThis.rendererBenchmarkReady = true; diff --git a/frontend/resources/wasm-playground/benchmarks/contract.js b/frontend/resources/wasm-playground/benchmarks/contract.js new file mode 100644 index 0000000000..98a8f27331 --- /dev/null +++ b/frontend/resources/wasm-playground/benchmarks/contract.js @@ -0,0 +1,124 @@ +export const WEBGL_OPTIONS = Object.freeze({ + antialias: false, + depth: true, + stencil: true, + alpha: true, + preserveDrawingBuffer: true, +}); + +export function deriveSeed(masterSeed, caseId) { + let seed = 2166136261; + for (const character of `${masterSeed}\0${caseId}`) { + seed = Math.imul(seed ^ character.charCodeAt(0), 16777619) >>> 0; + } + return seed; +} + +function serializable(value, path = "case") { + if (value === null || typeof value === "string" || typeof value === "boolean") + return; + if (typeof value === "number" && Number.isFinite(value)) return; + if (Array.isArray(value)) { + value.forEach((v, i) => serializable(v, `${path}[${i}]`)); + return; + } + if (value && Object.getPrototypeOf(value) === Object.prototype) { + for (const [key, v] of Object.entries(value)) + serializable(v, `${path}.${key}`); + return; + } + throw new Error(`Non-serializable value: ${path}`); +} + +function viewValid(view) { + return ( + view && + Number.isFinite(view.scale) && + view.scale > 0 && + Number.isFinite(view.x) && + Number.isFinite(view.y) + ); +} + +export function collectCases(scenarios, masterSeed, filter = "") { + const cases = []; + const seen = new Set(); + const scenarioIds = new Set(); + for (const scenario of scenarios) { + if ( + !scenario.id || + scenarioIds.has(scenario.id) || + !Number.isInteger(scenario.version) || + typeof scenario.createScene !== "function" || + typeof scenario.upload !== "function" || + !Array.isArray(scenario.cases) || + !scenario.cases.length + ) + throw new Error("Invalid or duplicate scenario"); + scenarioIds.add(scenario.id); + for (const c of scenario.cases) { + serializable(c); + const baseId = `${scenario.id}/${c.id}`; + if (!c.id || seen.has(baseId)) + throw new Error(`Missing or duplicate case ID: ${baseId}`); + seen.add(baseId); + if (!c.params || !viewValid(c.initialView)) + throw new Error(`Invalid parameters/view: ${baseId}`); + for (const kind of ["pan", "zoom"]) { + const interaction = c.interactions?.[kind]; + if ( + !interaction || + !Array.isArray(interaction.frames) || + !interaction.frames.length || + !interaction.frames.every(viewValid) || + !Number.isFinite(interaction.settleMs) || + interaction.settleMs < 0 + ) + throw new Error(`Incomplete ${kind} interaction: ${baseId}`); + } + // Validate static workloads before any build or browser measurement. Browser + // construction still owns its descriptors and remains outside timed spans. + scenario.createScene(c.params, deriveSeed(masterSeed, baseId)); + for (const kind of ["load", "pan", "zoom"]) { + const id = `${baseId}/${kind}`; + if (filter && !id.includes(filter)) continue; + cases.push({ + id, + scenarioId: scenario.id, + scenarioVersion: scenario.version, + description: scenario.description, + sceneSeed: deriveSeed(masterSeed, baseId), + params: c.params, + initialView: c.initialView, + interaction: kind === "load" ? null : c.interactions[kind], + group: kind === "load" ? "fresh-module-context" : `warm-${kind}`, + preparation: + kind === "load" + ? "new-context-module; upload; set-view-end; full" + : "case-owned-context; upload; full; restore-view-and-full-before-each-attempt; evolving-caches", + status: "unattempted", + attempts: [], + }); + } + } + } + if (!cases.length) throw new Error(`No matching cases: ${filter}`); + return cases; +} + +export function validateMetrics(metrics) { + return Object.fromEntries( + Object.entries(metrics) + .filter( + ([key, value]) => + !Number.isFinite(value) || + value < 0 || + (value === 0 && + !["setViewEndMs", "settlingRequestedMs"].includes(key)), + ) + .map(([key, value]) => [ + key, + Number.isFinite(value) ? "below-timer-resolution" : "non-finite", + ]), + ); +} diff --git a/frontend/resources/wasm-playground/benchmarks/protocol.js b/frontend/resources/wasm-playground/benchmarks/protocol.js new file mode 100644 index 0000000000..2d3929525b --- /dev/null +++ b/frontend/resources/wasm-playground/benchmarks/protocol.js @@ -0,0 +1,77 @@ +// Browser-independent protocol; the runtime owns the clock and cancellation. +export async function drain( + rt, + { flags = 0, origin = rt.now(), immediate = false } = {}, +) { + const slices = []; + let viewportReadyMs; + do { + rt.check(); + const timestamp = immediate ? rt.now() : await rt.frame(); + immediate = false; + rt.check(); + const start = rt.now(); + const frameType = rt.module._render(timestamp, flags); + const end = rt.now(); + slices.push({ timestamp, flags, frameType, durationMs: end - start }); + rt.recordSlice?.(slices.at(-1)); + rt.check(); + if (![1, 2, 3].includes(frameType)) + throw new Error(`Unexpected frame type: ${frameType}`); + if (frameType === 3 || frameType === 2) viewportReadyMs ??= end - origin; + if (frameType === 2) + return { slices, viewportReadyMs, fullMs: end - origin }; + // The production scheduler passes the previous FrameType as flags. + flags = frameType; + } while (true); +} + +export async function restore(rt, view) { + rt.check(); + rt.module._set_view_start(); + rt.module._set_view(view.scale, view.x, view.y); + rt.module._set_view_end(); + return drain(rt, { flags: 4, immediate: true }); +} + +export async function interact(rt, interaction) { + rt.check(); + const start = rt.now(); + rt.module._set_view_start(); + const cachedSlices = []; + let lastInput; + for (const view of interaction.frames) { + const timestamp = await rt.frame(); + rt.check(); + lastInput = rt.now(); + rt.module._set_view(view.scale, view.x, view.y); + const cacheStart = rt.now(); + rt.module._render_from_cache(0); + cachedSlices.push({ timestamp, durationMs: rt.now() - cacheStart }); + rt.recordCached?.(cachedSlices.at(-1)); + } + const activeEnd = rt.now(); + await rt.sleep(interaction.settleMs); + rt.check(); + const finalizationStart = rt.now(); + rt.module._set_view_end(); + const setViewEndMs = rt.now() - finalizationStart; + const result = await drain(rt, { + flags: 4, + origin: finalizationStart, + immediate: true, + }); + return { + metrics: { + activeInteractionMs: activeEnd - start, + settlingRequestedMs: interaction.settleMs, + settlingActualMs: finalizationStart - activeEnd, + setViewEndMs, + timeToViewportReadyMs: result.viewportReadyMs, + timeToFullMs: result.fullMs, + lastInputToFullMs: rt.now() - lastInput, + }, + slices: result.slices, + cachedSlices, + }; +} diff --git a/frontend/resources/wasm-playground/benchmarks/statistics.js b/frontend/resources/wasm-playground/benchmarks/statistics.js new file mode 100644 index 0000000000..62e3b4319c --- /dev/null +++ b/frontend/resources/wasm-playground/benchmarks/statistics.js @@ -0,0 +1,269 @@ +// Shared by the benchmark page and offline Node reports. No browser work or I/O. +export const STATISTICAL_METHOD = Object.freeze({ + version: 1, + name: "percentile-bootstrap-of-medians", + confidence: 0.95, + resamples: 2000, + seed: 0x70656e70, + minimumIntervalSamples: 10, + minimumP95Samples: 100, + minimumP99Samples: 500, +}); + +export const STATISTICAL_WARNINGS = Object.freeze([ + "Intervals are exploratory, nominal 95%, and conditional on these observed runs; attempts can be correlated.", + "Independent runs can drift with system load and thermal state. Confirm important changes with repeated complete runs.", +]); + +export function statisticalMethod(options = {}) { + const method = { ...STATISTICAL_METHOD, ...options }; + if ( + !Number.isInteger(method.resamples) || + method.resamples < 100 || + method.resamples > 10000 + ) { + throw new Error("resamples must be an integer between 100 and 10000"); + } + if ( + !Number.isInteger(method.seed) || + method.seed < 0 || + method.seed > 0xffffffff + ) { + throw new Error("seed must be a uint32 integer"); + } + // Only the resampling count and seed are configurable; support thresholds and + // interval interpretation form part of the versioned statistical method. + return { + ...STATISTICAL_METHOD, + resamples: method.resamples, + seed: method.seed, + }; +} + +function randomGenerator(seed) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = Math.imul(state ^ (state >>> 15), 1 | state); + value ^= value + Math.imul(value ^ (value >>> 7), 61 | value); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; +} + +function quantile(sorted, probability) { + if (!sorted.length) return null; + const position = (sorted.length - 1) * probability; + const lower = Math.floor(position); + const upper = Math.ceil(position); + return sorted[lower] + (sorted[upper] - sorted[lower]) * (position - lower); +} + +function sorted(values) { + return [...values].sort((a, b) => a - b); +} + +function interval(values) { + const ordered = sorted(values); + return { low: quantile(ordered, 0.025), high: quantile(ordered, 0.975) }; +} + +function resampleMedian(values, random) { + const sample = Array.from( + { length: values.length }, + () => values[Math.floor(random() * values.length)], + ); + return quantile(sorted(sample), 0.5); +} + +export function summarize(values, options = {}) { + const method = statisticalMethod(options); + const ordered = sorted( + values.filter((value) => Number.isFinite(value) && value >= 0), + ); + const n = ordered.length; + const median = quantile(ordered, 0.5); + const mean = n ? ordered.reduce((sum, value) => sum + value / n, 0) : null; + const summary = { + n, + invalid: values.length - n, + mean, + median, + min: n ? ordered[0] : null, + max: n ? ordered[n - 1] : null, + standardDeviation: + n > 1 + ? Math.sqrt( + ordered.reduce( + (sum, value) => sum + (value - mean) ** 2 / (n - 1), + 0, + ), + ) + : null, + mad: n + ? quantile(sorted(ordered.map((value) => Math.abs(value - median))), 0.5) + : null, + medianInterval: null, + p95: null, + p99: null, + suppressed: {}, + }; + if (n >= method.minimumIntervalSamples) { + const random = randomGenerator(method.seed); + summary.medianInterval = interval( + Array.from({ length: method.resamples }, () => + resampleMedian(ordered, random), + ), + ); + } else { + summary.suppressed.medianInterval = `Requires at least ${method.minimumIntervalSamples} valid attempts.`; + } + for (const [name, threshold, probability] of [ + ["p95", method.minimumP95Samples, 0.95], + ["p99", method.minimumP99Samples, 0.99], + ]) { + if (n >= threshold) summary[name] = quantile(ordered, probability); + else + summary.suppressed[name] = + `Requires at least ${threshold} valid attempts.`; + } + return summary; +} + +export function metricNames(benchmarkCase) { + return [ + ...new Set( + benchmarkCase.attempts.flatMap((attempt) => [ + ...Object.keys(attempt.metrics ?? {}), + ...Object.keys(attempt.invalidMetrics ?? {}), + ]), + ), + ].sort(); +} + +export function observations(benchmarkCase, metric) { + return benchmarkCase.attempts + .filter((attempt) => !attempt.warmup) + .map((attempt) => { + const invalid = attempt.invalidMetrics ?? {}; + if ( + attempt.status === "failed" || + (attempt.status === "invalid" && !Object.keys(invalid).length) || + Object.hasOwn(invalid, metric) + ) + return NaN; + return attempt.metrics?.[metric] ?? NaN; + }); +} + +export function summarizeRun(result, options = {}) { + return { + method: statisticalMethod(options), + warnings: [...STATISTICAL_WARNINGS], + cases: result.cases.map((benchmarkCase) => { + const measured = benchmarkCase.attempts.filter( + (attempt) => !attempt.warmup, + ); + return { + id: benchmarkCase.id, + status: benchmarkCase.status, + attempts: { + measured: measured.length, + warmup: benchmarkCase.attempts.length - measured.length, + ok: measured.filter((attempt) => attempt.status === "ok").length, + invalid: measured.filter((attempt) => attempt.status === "invalid") + .length, + failed: measured.filter((attempt) => attempt.status === "failed") + .length, + }, + metrics: Object.fromEntries( + metricNames(benchmarkCase).map((metric) => [ + metric, + summarize(observations(benchmarkCase, metric), options), + ]), + ), + }; + }), + }; +} + +export function compareObservations( + baselineValues, + candidateValues, + options = {}, +) { + const method = statisticalMethod(options); + const baseline = summarize(baselineValues, options); + const candidate = summarize(candidateValues, options); + const result = { + baseline, + candidate, + absoluteChange: null, + relativeChangePercent: null, + absoluteInterval: null, + relativeIntervalPercent: null, + suppressed: {}, + }; + if (!baseline.n || !candidate.n) { + result.suppressed.comparison = "Both runs require valid observations."; + return result; + } + result.absoluteChange = candidate.median - baseline.median; + if (baseline.median === 0) + result.suppressed.relativeChangePercent = "Baseline median is zero."; + else + result.relativeChangePercent = + (100 * result.absoluteChange) / baseline.median; + if ( + baseline.n < method.minimumIntervalSamples || + candidate.n < method.minimumIntervalSamples + ) { + result.suppressed.intervals = `Each run requires at least ${method.minimumIntervalSamples} valid attempts.`; + return result; + } + const before = baselineValues.filter( + (value) => Number.isFinite(value) && value >= 0, + ); + const after = candidateValues.filter( + (value) => Number.isFinite(value) && value >= 0, + ); + const random = randomGenerator(method.seed); + const differences = []; + const relative = []; + for (let index = 0; index < method.resamples; index++) { + // Independent draws preserve the unpaired nature of separate runs. + const first = resampleMedian(before, random); + const second = resampleMedian(after, random); + differences.push(second - first); + if (first !== 0) relative.push((100 * (second - first)) / first); + } + result.absoluteInterval = interval(differences); + if (baseline.median !== 0 && relative.length === method.resamples) + result.relativeIntervalPercent = interval(relative); + else + result.suppressed.relativeIntervalPercent = + "Baseline median or a resampled baseline median is zero."; + return result; +} + +function display(value) { + return value === null ? "n/a" : value.toFixed(3); +} + +export function formatSummary(summary) { + const lines = [ + `Statistics: ${summary.method.name} v${summary.method.version}; ${summary.method.resamples} resamples, seed ${summary.method.seed}.`, + ...summary.warnings, + ]; + for (const benchmarkCase of summary.cases) { + lines.push(`${benchmarkCase.id} (${benchmarkCase.status})`); + for (const [name, metric] of Object.entries(benchmarkCase.metrics)) { + lines.push( + ` ${name}: median ${display(metric.median)}, mean ${display(metric.mean)}, range ${display(metric.min)}–${display(metric.max)}, SD ${display(metric.standardDeviation)}, MAD ${display(metric.mad)}; ${metric.n} valid, ${metric.invalid} invalid`, + ); + lines.push( + ` median interval: ${metric.medianInterval ? `[${display(metric.medianInterval.low)}, ${display(metric.medianInterval.high)}]` : metric.suppressed.medianInterval}; p95: ${metric.p95 === null ? metric.suppressed.p95 : display(metric.p95)}; p99: ${metric.p99 === null ? metric.suppressed.p99 : display(metric.p99)}`, + ); + } + } + return lines.join("\n"); +} diff --git a/frontend/resources/wasm-playground/clips.html b/frontend/resources/wasm-playground/clips.html index ba1951e56a..6823632784 100644 --- a/frontend/resources/wasm-playground/clips.html +++ b/frontend/resources/wasm-playground/clips.html @@ -1,91 +1,47 @@ - - - - - WASM + WebGL2 Canvas - - - - - - + mountScenario( + scenario, + initModule, + document.getElementById("canvas"), + ).catch((error) => { + console.error(error); + const message = document.createElement("pre"); + message.id = "error"; + message.textContent = error.message; + document.body.append(message); + }); + + diff --git a/frontend/resources/wasm-playground/index.html b/frontend/resources/wasm-playground/index.html index 9f7dd64d0b..6d9d59f54b 100644 --- a/frontend/resources/wasm-playground/index.html +++ b/frontend/resources/wasm-playground/index.html @@ -1,27 +1,44 @@ - - - + + + WASM Playground Home - - + +

WASM Playground Home

- - \ No newline at end of file + + diff --git a/frontend/resources/wasm-playground/js/lib.js b/frontend/resources/wasm-playground/js/lib.js index b890e32ce4..8096afd5ca 100644 --- a/frontend/resources/wasm-playground/js/lib.js +++ b/frontend/resources/wasm-playground/js/lib.js @@ -1,306 +1,252 @@ -let Module = null; +import { + BACKGROUND, + serializeChildren, + solidFill, + uploadBytes, + validateCount, +} from "../scenes/shared.js"; -let scale = 1; -let offsetX = 0; -let offsetY = 0; +export const contextOptions = Object.freeze({ + antialias: false, + depth: true, + alpha: true, + stencil: true, + preserveDrawingBuffer: true, +}); -let isPanning = false; -let lastX = 0; -let lastY = 0; +// All callbacks close over this instance; disposal cannot affect another module. +export function createRendererAdapter(module, scheduler = globalThis) { + let disposed = false; + let handle; + let context; + let frame; + let timer; + let cancelRender; + let detach = () => {}; + const stopRender = () => { + if (frame !== undefined) scheduler.cancelAnimationFrame(frame); + frame = undefined; + cancelRender?.(); + cancelRender = undefined; + }; -export function init(moduleInstance) { - Module = moduleInstance; -} - -export function assignCanvas(canvas) { - const glModule = Module.GL; - const context = canvas.getContext("webgl2", { - antialias: true, - depth: true, - alpha: false, - stencil: true, - preserveDrawingBuffer: true, - }); - - const handle = glModule.registerContext(context, { majorVersion: 2 }); - glModule.makeContextCurrent(handle); - context.getExtension("WEBGL_debug_renderer_info"); - - Module._init(canvas.width, canvas.height); - Module._set_render_options(0, 1); -} - -export function hexToU32ARGB(hex, opacity = 1) { - const rgb = parseInt(hex.slice(1), 16); - const a = Math.floor(opacity * 0xff); - const argb = (a << 24) | rgb; - return argb >>> 0; -} - -export function getRandomInt(min, max) { - return Math.floor(Math.random() * (max - min)) + min; -} - -export function getRandomColor() { - const r = getRandomInt(0, 256).toString(16).padStart(2, "0"); - const g = getRandomInt(0, 256).toString(16).padStart(2, "0"); - const b = getRandomInt(0, 256).toString(16).padStart(2, "0"); - return `#${r}${g}${b}`; -} - -export function getRandomFloat(min, max) { - return Math.random() * (max - min) + min; -} - -function getU32(id) { - const hex = id.replace(/-/g, ""); - const buffer = new Uint32Array(4); - for (let i = 0; i < 4; i++) { - buffer[i] = parseInt(hex.slice(i * 8, (i + 1) * 8), 16); - } - return buffer; -} - -function heapU32SetUUID(id, heap, offset) { - const buffer = getU32(id); - heap.set(buffer, offset); - return buffer; -} - -function ptr8ToPtr32(ptr8) { - return ptr8 >>> 2; -} - -export function allocBytes(size) { - return Module._alloc_bytes(size); -} - -export function getHeapU32() { - return Module.HEAPU32; -} - -export function clearShapeFills() { - Module._clear_shape_fills(); -} - -export function addShapeSolidFill(argb) { - const ptr = allocBytes(160); - const heap = getHeapU32(); - const dv = new DataView(heap.buffer); - dv.setUint8(ptr, 0x00, true); - dv.setUint32(ptr + 4, argb, true); - Module._add_shape_fill(); -} - -export function addShapeSolidStrokeFill(argb) { - const ptr = allocBytes(160); - const heap = getHeapU32(); - const dv = new DataView(heap.buffer); - dv.setUint8(ptr, 0x00, true); - dv.setUint32(ptr + 4, argb, true); - Module._add_shape_stroke_fill(); -} - -function serializePathAttrs(svgAttrs) { - return Object.entries(svgAttrs).reduce((acc, [key, value]) => { - return acc + key + "\0" + value + "\0"; - }, ""); -} - -export function draw_star(x, y, width, height) { - const len = 11; // 1 MOVE + 9 LINE + 1 CLOSE - const ptr = allocBytes(len * 28); - const heap = getHeapU32(); - const dv = new DataView(heap.buffer); - - const cx = x + width / 2; - const cy = y + height / 2; - const outerRadius = Math.min(width, height) / 2; - const innerRadius = outerRadius * 0.4; - - const star = []; - for (let i = 0; i < 10; i++) { - const angle = (Math.PI / 5) * i - Math.PI / 2; - const r = i % 2 === 0 ? outerRadius : innerRadius; - const px = cx + r * Math.cos(angle); - const py = cy + r * Math.sin(angle); - star.push([px, py]); + function assignCanvas(canvas, dpr = 1) { + if (disposed) throw new Error("Renderer has been disposed"); + context = canvas.getContext("webgl2", contextOptions); + if (!context) throw new Error("WebGL2 is unavailable"); + handle = module.GL.registerContext(context, { majorVersion: 2 }); + module.GL.makeContextCurrent(handle); + context.getExtension("WEBGL_debug_renderer_info"); + module._init( + Math.round(canvas.width / dpr), + Math.round(canvas.height / dpr), + ); + module._set_render_options(0, dpr); + // RawBrowser::Chrome = 1; use the same default browser as the benchmark. + module._set_browser?.(1); } - let offset = 0; - - // MOVE to first point - dv.setUint16(ptr + offset + 0, 1, true); // MOVE - dv.setFloat32(ptr + offset + 20, star[0][0], true); - dv.setFloat32(ptr + offset + 24, star[0][1], true); - offset += 28; - - // LINE to remaining points - for (let i = 1; i < star.length; i++) { - dv.setUint16(ptr + offset + 0, 2, true); // LINE - dv.setFloat32(ptr + offset + 20, star[i][0], true); - dv.setFloat32(ptr + offset + 24, star[i][1], true); - offset += 28; + function renderFull(flags = 4) { + if (disposed) return Promise.resolve(false); + stopRender(); + return new Promise((resolve, reject) => { + cancelRender = () => resolve(false); + const step = (timestamp) => { + frame = undefined; + if (disposed) return resolve(false); + try { + const result = module._render(Math.trunc(timestamp), flags); + if (result === 2) { + cancelRender = undefined; + resolve(true); + } else if (result === 1 || result === 3) { + flags = result; + frame = scheduler.requestAnimationFrame(step); + } else { + throw new Error(`Unexpected renderer frame type: ${result}`); + } + } catch (error) { + cancelRender = undefined; + reject(error); + } + }; + frame = scheduler.requestAnimationFrame(step); + }); } - // CLOSE the path - dv.setUint16(ptr + offset + 0, 4, true); // CLOSE - - Module._set_shape_path_content(); -} - -export function setShapeChildren(shapeIds) { - const offset = allocBytes(shapeIds.length * 16); - const heap = getHeapU32(); - let currentOffset = offset; - for (const id of shapeIds) { - heapU32SetUUID(id, heap, ptr8ToPtr32(currentOffset)); - currentOffset += 16; + function setupInteraction(canvas, initialView = { scale: 1, x: 0, y: 0 }) { + detach(); + let view = { ...initialView }; + let panning = false; + let active = false; + let previous; + const listeners = []; + const listen = (type, callback, options) => { + canvas.addEventListener(type, callback, options); + listeners.push(() => canvas.removeEventListener(type, callback, options)); + }; + const finish = () => { + timer = undefined; + if (disposed || !active) return; + active = false; + module._set_view_end(); + renderFull().catch((error) => console.error(error)); + }; + const begin = () => { + scheduler.clearTimeout(timer); + stopRender(); + if (!active) module._set_view_start(); + active = true; + }; + const update = () => { + module._set_view(view.scale, view.x, view.y); + module._render_from_cache(); + scheduler.clearTimeout(timer); + timer = scheduler.setTimeout(finish, 100); + }; + listen( + "wheel", + (event) => { + event.preventDefault(); + begin(); + const nextScale = Math.max( + 0.01, + Math.min(100, view.scale * (event.deltaY < 0 ? 1.1 : 0.9)), + ); + view.x += event.offsetX / nextScale - event.offsetX / view.scale; + view.y += event.offsetY / nextScale - event.offsetY / view.scale; + view.scale = nextScale; + update(); + }, + { passive: false }, + ); + listen("pointerdown", (event) => { + panning = true; + previous = [event.clientX, event.clientY]; + canvas.setPointerCapture?.(event.pointerId); + }); + listen("pointermove", (event) => { + if (!panning) return; + begin(); + view.x += (event.clientX - previous[0]) / view.scale; + view.y += (event.clientY - previous[1]) / view.scale; + previous = [event.clientX, event.clientY]; + update(); + }); + const endPan = () => { + panning = false; + scheduler.clearTimeout(timer); + finish(); + }; + listen("pointerup", endPan); + listen("pointercancel", endPan); + detach = () => { + for (const remove of listeners) remove(); + scheduler.clearTimeout(timer); + panning = false; + active = false; + }; + return detach; } - return Module._set_children(); -} -export function useShape(id) { - const buffer = getU32(id); - Module._use_shape(...buffer); -} - -export function set_parent(id) { - const buffer = getU32(id); - Module._set_parent(...buffer); -} - -function debounce(fn, delay) { - let timeout; - return (...args) => { - clearTimeout(timeout); - timeout = setTimeout(() => fn(...args), delay); + return { + module, + assignCanvas, + renderFull, + setupInteraction, + useShape: (id) => module._use_shape(...uuidParts(id)), + setShapeChildren: (ids) => + uploadBytes( + module, + serializeChildren(ids.map(uuidParts)), + "_set_children", + ), + addShapeSolidFill: (color) => + uploadBytes(module, solidFill(color), "_add_shape_fill"), + dispose() { + if (disposed) return; + disposed = true; + detach(); + scheduler.clearTimeout(timer); + stopRender(); + try { + if (context && !context.isContextLost()) module._clean_up(); + } finally { + if (handle !== undefined) module.GL.deleteContext(handle); + context?.getExtension("WEBGL_lose_context")?.loseContext(); + } + }, }; } -const debouncedRender = debounce(() => { - Module._render(Date.now()); -}, 100); - -export function setupInteraction(canvas) { - canvas.addEventListener("wheel", (e) => { - e.preventDefault(); - const zoomFactor = e.deltaY < 0 ? 1.1 : 0.9; - scale *= zoomFactor; - const mouseX = e.offsetX; - const mouseY = e.offsetY; - offsetX -= (mouseX - offsetX) * (zoomFactor - 1); - offsetY -= (mouseY - offsetY) * (zoomFactor - 1); - Module._set_view(scale, offsetX, offsetY); - Module._render_from_cache(); - debouncedRender(); - }); - - canvas.addEventListener("mousedown", (e) => { - isPanning = true; - lastX = e.offsetX; - lastY = e.offsetY; - }); - - canvas.addEventListener("mousemove", (e) => { - if (isPanning) { - const dx = e.offsetX - lastX; - const dy = e.offsetY - lastY; - offsetX += dx; - offsetY += dy; - lastX = e.offsetX; - lastY = e.offsetY; - Module._set_view(scale, offsetX, offsetY); - Module._render_from_cache(); - debouncedRender(); - } - }); - - canvas.addEventListener("mouseup", () => { - isPanning = false; - }); - canvas.addEventListener("mouseout", () => { - isPanning = false; - }); +function uuidParts(id) { + if (Array.isArray(id)) return id; + const hex = id.replaceAll("-", ""); + return Array.from({ length: 4 }, (_, index) => + Number.parseInt(hex.slice(index * 8, (index + 1) * 8), 16), + ); } -export function addTextShape(x, y, fontSize, text) { - const numLeaves = 1; // Single text leaf for simplicity - const paragraphAttrSize = 48; - // const leafAttrSize = 56; - const singleFillSize = 160; - const textBuffer = new TextEncoder().encode(text); - const textSize = textBuffer.byteLength; - - const leafSize = 1340; // leaf attrs + 8 fills - - // Calculate fills - const fills = [ - { - type: "solid", - color: getRandomColor(), - opacity: getRandomFloat(0.5, 1.0), - }, - ]; - const totalSize = paragraphAttrSize + leafSize + textSize; - - // Allocate buffer - const bufferPtr = allocBytes(totalSize); - const heap = new Uint8Array(Module.HEAPU8.buffer, bufferPtr, totalSize); - const dview = new DataView(heap.buffer, bufferPtr, totalSize); - - // Set number of leaves - dview.setUint32(0, numLeaves, true); - - // Serialize paragraph attributes - dview.setUint8(4, 1); // text-align: left - dview.setUint8(5, 0); // text-direction: LTR - dview.setUint8(6, 0); // text-decoration: none - dview.setUint8(7, 0); // text-transform: none - dview.setFloat32(8, 1.2, true); // line-height - dview.setFloat32(12, 0, true); // letter-spacing - dview.setUint32(16, 0, true); // typography-ref-file (UUID part 1) - dview.setUint32(20, 0, true); // typography-ref-file (UUID part 2) - dview.setUint32(24, 0, true); // typography-ref-file (UUID part 3) - dview.setInt32(28, 0, true); // typography-ref-file (UUID part 4) - dview.setUint32(32, 0, true); // typography-ref-id (UUID part 1) - dview.setUint32(36, 0, true); // typography-ref-id (UUID part 2) - dview.setUint32(40, 0, true); // typography-ref-id (UUID part 3) - dview.setInt32(44, 0, true); // typography-ref-id (UUID part 4) - - // Serialize leaf attributes - const leafOffset = paragraphAttrSize; - dview.setUint8(leafOffset, 0); // font-style: normal - dview.setFloat32(leafOffset + 4, fontSize, true); // font-size - dview.setFloat32(leafOffset + 8, 0, true); // letter-spacing - dview.setUint32(leafOffset + 12, 400, true); // font-weight: normal - dview.setUint32(leafOffset + 16, 0, true); // font-id (UUID part 1) - dview.setUint32(leafOffset + 20, 0, true); // font-id (UUID part 2) - dview.setUint32(leafOffset + 24, 0, true); // font-id (UUID part 3) - dview.setInt32(leafOffset + 28, 0, true); // font-id (UUID part 4) - dview.setInt32(leafOffset + 32, 0, true); // font-family hash - dview.setUint32(leafOffset + 36, 0, true); // font-variant-id (UUID part 1) - dview.setUint32(leafOffset + 40, 0, true); // font-variant-id (UUID part 2) - dview.setUint32(leafOffset + 44, 0, true); // font-variant-id (UUID part 3) - dview.setInt32(leafOffset + 48, 0, true); // font-variant-id (UUID part 4) - dview.setInt32(leafOffset + 52, textSize, true); // text-length - dview.setInt32(leafOffset + 56, fills.length, true); // total fills count - - // Serialize fills - let fillOffset = leafOffset + 60; - fills.forEach((fill) => { - if (fill.type === "solid") { - const argb = hexToU32ARGB(fill.color, fill.opacity); - dview.setUint8(fillOffset, 0x00, true); // Fill type: solid - dview.setUint32(fillOffset + 4, argb, true); - fillOffset += singleFillSize; // Move to the next fill - } - }); - - // Add text content - const textOffset = leafSize; - heap.set(textBuffer, textOffset); - - // Call the WebAssembly function - Module._set_shape_text_content(); +export async function mountScenario( + scenario, + initModule, + canvas, + search = location.search, +) { + const query = new URLSearchParams(search); + const sourceCase = scenario.cases[0]; + const params = { ...sourceCase.params }; + const requestedCount = query.get( + scenario.id === "texts" ? "texts" : "shapes", + ); + if (requestedCount !== null) params.count = validateCount(requestedCount); + const seed = Number(query.get("seed") ?? 42); + // All random construction and serialization happen before the module exists. + const scene = scenario.createScene(params, seed); + const dpr = globalThis.devicePixelRatio || 1; + canvas.width = Math.round(globalThis.innerWidth * dpr); + canvas.height = Math.round(globalThis.innerHeight * dpr); + const module = await initModule(); + const adapter = createRendererAdapter(module); + try { + adapter.assignCanvas(canvas, dpr); + scenario.upload(module, scene); + module._set_canvas_background(BACKGROUND); + const { scale, x, y } = sourceCase.initialView; + module._set_view(scale, x, y); + module._set_view_end(); + await adapter.renderFull(); + adapter.setupInteraction(canvas, sourceCase.initialView); + } catch (error) { + adapter.dispose(); + throw error; + } + const dispose = () => { + globalThis.removeEventListener("pagehide", dispose); + adapter.dispose(); + }; + globalThis.addEventListener("pagehide", dispose, { once: true }); + return { ...adapter, dispose }; } + +// Compatibility for the older, explicitly noncanonical comparison.html page. +// Its sequential synchronous setup can retain these names; new callers own adapters. +let legacyAdapter; +export function init(module) { + legacyAdapter = createRendererAdapter(module); +} +export const assignCanvas = (...args) => legacyAdapter.assignCanvas(...args); +export const useShape = (...args) => legacyAdapter.useShape(...args); +export const setShapeChildren = (...args) => + legacyAdapter.setShapeChildren(...args); +export const addShapeSolidFill = (...args) => + legacyAdapter.addShapeSolidFill(...args); +export const setupInteraction = (...args) => + legacyAdapter.setupInteraction(...args); +export const getRandomInt = (min, max) => + Math.floor(Math.random() * (max - min)) + min; +export const getRandomFloat = (min, max) => Math.random() * (max - min) + min; +export const getRandomColor = () => + `#${getRandomInt(0, 0x1000000).toString(16).padStart(6, "0")}`; +export const hexToU32ARGB = (hex, opacity = 1) => + ((Math.floor(opacity * 255) << 24) | Number.parseInt(hex.slice(1), 16)) >>> 0; diff --git a/frontend/resources/wasm-playground/masks.html b/frontend/resources/wasm-playground/masks.html index 332a525df8..af89ad5aeb 100644 --- a/frontend/resources/wasm-playground/masks.html +++ b/frontend/resources/wasm-playground/masks.html @@ -1,86 +1,47 @@ - - - - - WASM + WebGL2 Canvas - - - - - - + mountScenario( + scenario, + initModule, + document.getElementById("canvas"), + ).catch((error) => { + console.error(error); + const message = document.createElement("pre"); + message.id = "error"; + message.textContent = error.message; + document.body.append(message); + }); + + diff --git a/frontend/resources/wasm-playground/paths.html b/frontend/resources/wasm-playground/paths.html index c783d8bc40..5d941d586a 100644 --- a/frontend/resources/wasm-playground/paths.html +++ b/frontend/resources/wasm-playground/paths.html @@ -1,85 +1,47 @@ - - - - - WASM + WebGL2 Canvas - - - - - - + mountScenario( + scenario, + initModule, + document.getElementById("canvas"), + ).catch((error) => { + console.error(error); + const message = document.createElement("pre"); + message.id = "error"; + message.textContent = error.message; + document.body.append(message); + }); + + diff --git a/frontend/resources/wasm-playground/plus.html b/frontend/resources/wasm-playground/plus.html index 3cd8c11f6f..64916798e6 100644 --- a/frontend/resources/wasm-playground/plus.html +++ b/frontend/resources/wasm-playground/plus.html @@ -1,109 +1,47 @@ - - - - - WASM + WebGL2 Canvas - - - - - - + mountScenario( + scenario, + initModule, + document.getElementById("canvas"), + ).catch((error) => { + console.error(error); + const message = document.createElement("pre"); + message.id = "error"; + message.textContent = error.message; + document.body.append(message); + }); + + diff --git a/frontend/resources/wasm-playground/rects.html b/frontend/resources/wasm-playground/rects.html index cee9351b91..930e12205f 100644 --- a/frontend/resources/wasm-playground/rects.html +++ b/frontend/resources/wasm-playground/rects.html @@ -1,82 +1,47 @@ - - - - - WASM + WebGL2 Canvas - - - - - - + mountScenario( + scenario, + initModule, + document.getElementById("canvas"), + ).catch((error) => { + console.error(error); + const message = document.createElement("pre"); + message.id = "error"; + message.textContent = error.message; + document.body.append(message); + }); + + diff --git a/frontend/resources/wasm-playground/scenes/clips.js b/frontend/resources/wasm-playground/scenes/clips.js new file mode 100644 index 0000000000..4e25db2263 --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/clips.js @@ -0,0 +1,43 @@ +import { + ROOT, + argb, + defaultCase, + finishScene, + randomSource, + rectangle, + stroke, + uploadScene, + validateParameters, +} from "./shared.js"; + +export default { + id: "clips", + version: 1, + description: + "Fixed clipped frame with rounded corners, blur, stroke and drop shadow", + cases: [ + defaultCase({ count: 2 }, { anchor: [320, 280], panDistance: [60, 30] }), + ], + createScene(params, seed) { + validateParameters(params, ["count"]); + if (params.count !== 2) throw new Error("The clips scene requires count=2"); + const rng = randomSource(seed); + const frame = { + ...rectangle(rng.id(), ROOT, [200, 200, 450, 450], 0xffee0d32), + type: 0, + clip: true, + corners: [50, 50, 50, 50], + strokes: [stroke(25, 0xff000000)], + blur: [1, false, 4], + shadows: [[argb(0x000000, 0.2), 4, 40, 80, 80, 0, false]], + }; + const child = { + ...rectangle(rng.id(), frame.id, [100, 100, 300, 300], 0xff003df7), + blur: [1, false, 40], + }; + // The old fixture's second rectangle was unreachable and is deliberately omitted. + frame.children = [child.id]; + return finishScene([frame, child]); + }, + upload: uploadScene, +}; diff --git a/frontend/resources/wasm-playground/scenes/index.js b/frontend/resources/wasm-playground/scenes/index.js new file mode 100644 index 0000000000..a42065f1b9 --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/index.js @@ -0,0 +1,9 @@ +import rects from "./rects.js"; +import paths from "./paths.js"; +import plus from "./plus.js"; +import texts from "./texts.js"; +import masks from "./masks.js"; +import clips from "./clips.js"; +import shadows from "./shadows.js"; + +export const scenarios = [rects, paths, plus, texts, masks, clips, shadows]; diff --git a/frontend/resources/wasm-playground/scenes/masks.js b/frontend/resources/wasm-playground/scenes/masks.js new file mode 100644 index 0000000000..cb01190cd5 --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/masks.js @@ -0,0 +1,44 @@ +import { + ROOT, + defaultCase, + finishScene, + randomSource, + rectangle, + uploadScene, + validateParameters, +} from "./shared.js"; + +export default { + id: "masks", + version: 1, + description: "Fixed masked group containing a rectangle and an ellipse", + cases: [ + defaultCase({ count: 4 }, { anchor: [320, 280], panDistance: [60, 30] }), + ], + createScene(params, seed) { + validateParameters(params, ["count"]); + if (params.count !== 4) throw new Error("The masks scene requires count=4"); + const rng = randomSource(seed); + const group = { + id: rng.id(), + parent: ROOT, + type: 1, + bounds: [319, 144, 544, 332], + masked: true, + }; + const outside = rectangle(rng.id(), ROOT, [100, 100, 200, 200], 0xffaabbcc); + const mask = rectangle( + rng.id(), + group.id, + [319, 144, 544, 332], + 0xff0c44ea, + ); + const ellipse = { + ...rectangle(rng.id(), group.id, [98, 214, 426, 475], 0xffb1b2b5), + type: 6, + }; + group.children = [mask.id, ellipse.id]; + return finishScene([group, outside, mask, ellipse]); + }, + upload: uploadScene, +}; diff --git a/frontend/resources/wasm-playground/scenes/paths.js b/frontend/resources/wasm-playground/scenes/paths.js new file mode 100644 index 0000000000..9f8d197622 --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/paths.js @@ -0,0 +1,15 @@ +import { defaultCase, randomShapes, uploadScene } from "./shared.js"; + +export default { + id: "paths", + version: 1, + description: "Seeded ten-point star paths with fills and centered strokes", + cases: [ + defaultCase( + { count: 1000, width: 1920, height: 1080, minSize: 20, maxSize: 200 }, + { anchor: [960, 540], panDistance: [120, 60] }, + ), + ], + createScene: (params, seed) => randomShapes("paths", params, seed), + upload: uploadScene, +}; diff --git a/frontend/resources/wasm-playground/scenes/plus.js b/frontend/resources/wasm-playground/scenes/plus.js new file mode 100644 index 0000000000..4975e4ed0d --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/plus.js @@ -0,0 +1,15 @@ +import { defaultCase, randomShapes, uploadScene } from "./shared.js"; + +export default { + id: "plus", + version: 1, + description: "Seeded horizontal and vertical stroked line paths", + cases: [ + defaultCase( + { count: 1000, width: 1920, height: 1080, minSize: 20, maxSize: 200 }, + { anchor: [960, 540], panDistance: [120, 60] }, + ), + ], + createScene: (params, seed) => randomShapes("plus", params, seed), + upload: uploadScene, +}; diff --git a/frontend/resources/wasm-playground/scenes/rects.js b/frontend/resources/wasm-playground/scenes/rects.js new file mode 100644 index 0000000000..fe82417032 --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/rects.js @@ -0,0 +1,15 @@ +import { defaultCase, randomShapes, uploadScene } from "./shared.js"; + +export default { + id: "rects", + version: 1, + description: "Seeded rectangles with translucent fills and centered strokes", + cases: [ + defaultCase( + { count: 1000, width: 1920, height: 1080, minSize: 20, maxSize: 100 }, + { anchor: [960, 540], panDistance: [120, 60] }, + ), + ], + createScene: (params, seed) => randomShapes("rects", params, seed), + upload: uploadScene, +}; diff --git a/frontend/resources/wasm-playground/scenes/shadows.js b/frontend/resources/wasm-playground/scenes/shadows.js new file mode 100644 index 0000000000..ed29b318be --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/shadows.js @@ -0,0 +1,26 @@ +import { argb, defaultCase, randomShapes, uploadScene } from "./shared.js"; + +export default { + id: "shadows", + version: 1, + description: "Seeded rectangles with two drop shadows and an inner shadow", + cases: [ + defaultCase( + { + count: 1000, + width: 1920, + height: 1080, + minSize: 20, + maxSize: 100, + shadows: [ + [argb(0xdedede, 0.33), 4, -2, 0, 2, 0, false], + [argb(0xdedede), 12, -8, 0, 12, 0, false], + [argb(0x002046, 0.12), 12, -8, 0, -4, 1, false], + ], + }, + { anchor: [960, 540], panDistance: [120, 60] }, + ), + ], + createScene: (params, seed) => randomShapes("shadows", params, seed), + upload: uploadScene, +}; diff --git a/frontend/resources/wasm-playground/scenes/shared.js b/frontend/resources/wasm-playground/scenes/shared.js new file mode 100644 index 0000000000..fcec75150f --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/shared.js @@ -0,0 +1,266 @@ +// FFI layouts are defined by render-wasm/src/wasm/{text,fills,strokes}.rs. +// Keep this serializer covered by ABI regression tests when those layouts change. +export const ROOT = Object.freeze([0, 0, 0, 0]); +export const BACKGROUND = 0xfffabada; + +export function validateParameters(params, allowed) { + for (const key of Object.keys(params)) { + if (!allowed.includes(key)) + throw new Error(`Unsupported parameter: ${key}`); + } +} + +export function validateCount(value) { + if (typeof value !== "number" && typeof value !== "string") { + throw new Error("Shape count must be a positive integer (maximum 100000)"); + } + const count = Number(value); + if (!Number.isSafeInteger(count) || count < 1 || count > 100000) { + throw new Error("Shape count must be a positive integer (maximum 100000)"); + } + return count; +} + +export function randomSource(seed) { + if (!Number.isInteger(seed) || seed < 0 || seed > 0xffffffff) { + throw new Error("Scene seed must be an unsigned 32-bit integer"); + } + let state = seed >>> 0; + const random = () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = Math.imul(state ^ (state >>> 15), state | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4294967296; + }; + return { + float: (min, max) => min + random() * (max - min), + int: (min, max) => Math.floor(min + random() * (max - min)), + id: () => [ + Math.floor(random() * 4294967296), + ((Math.floor(random() * 4294967296) & 0xffff0fff) | 0x4000) >>> 0, + ((Math.floor(random() * 4294967296) & 0x3fffffff) | 0x80000000) >>> 0, + Math.floor(random() * 4294967296), + ], + }; +} + +export function argb(rgb, opacity = 1) { + return ((Math.floor(opacity * 255) << 24) | rgb) >>> 0; +} + +export function solidFill(color) { + const bytes = new Uint8Array(160); + new DataView(bytes.buffer).setUint32(4, color, true); + return bytes; +} + +export function serializeText(text, fontSize, color) { + const encoded = new TextEncoder().encode(text); + // RawParagraphData=16; RawTextSpan=64+8*160=1344. + const bytes = new Uint8Array(16 + 1344 + encoded.length); + const view = new DataView(bytes.buffer); + view.setUint32(0, 1, true); + view.setFloat32(8, 1.2, true); + const span = 16; + view.setFloat32(span + 4, fontSize, true); + view.setFloat32(span + 8, 1.2, true); + view.setInt32(span + 16, 400, true); + view.setUint32(span + 56, encoded.length, true); + view.setUint32(span + 60, 1, true); + bytes.set(solidFill(color), span + 64); + bytes.set(encoded, span + 1344); + return bytes; +} + +export function serializePath(points, close = false) { + const bytes = new Uint8Array((points.length + Number(close)) * 28); + const view = new DataView(bytes.buffer); + for (let i = 0; i < points.length; i++) { + view.setUint16(i * 28, i === 0 ? 1 : 2, true); + view.setFloat32(i * 28 + 20, points[i][0], true); + view.setFloat32(i * 28 + 24, points[i][1], true); + } + if (close) view.setUint16(points.length * 28, 4, true); + return bytes; +} + +export function serializeChildren(children) { + const bytes = new Uint8Array(children.length * 16); + const view = new DataView(bytes.buffer); + children.forEach((id, i) => + id.forEach((part, j) => view.setUint32(i * 16 + j * 4, part, true)), + ); + return bytes; +} + +export function finishScene(shapes) { + for (const shape of shapes) { + if (shape.children) shape.childrenBytes = serializeChildren(shape.children); + } + const children = shapes + .filter((shape) => shape.parent.every((part) => part === 0)) + .map((shape) => shape.id); + return { shapes, children, childrenBytes: serializeChildren(children) }; +} + +export function rectangle(id, parent, bounds, color) { + return { id, parent, type: 3, bounds, fills: [solidFill(color)] }; +} + +export function stroke(width, color) { + return { args: [width, 0, 0, 0, -1, -1], fill: solidFill(color) }; +} + +export function uploadBytes(module, bytes, setter) { + const pointer = module._alloc_bytes(bytes.byteLength); + // Allocation can grow the WASM heap, so obtain HEAPU8 after allocating. + module.HEAPU8.set(bytes, pointer); + module[setter](); +} + +export function uploadScene(module, scene) { + module._init_shapes_pool(scene.shapes.length + 1); + module._begin_loading(); + try { + module._use_shape(...ROOT); + for (const shape of scene.shapes) { + module._use_shape(...shape.id); + module._set_parent(...shape.parent); + module._set_shape_type(shape.type); + module._set_shape_selrect(...shape.bounds); + if (shape.path) + uploadBytes(module, shape.path, "_set_shape_path_content"); + for (const fill of shape.fills ?? []) + uploadBytes(module, fill, "_add_shape_fill"); + for (const item of shape.strokes ?? []) { + module._add_shape_center_stroke(...item.args); + uploadBytes(module, item.fill, "_add_shape_stroke_fill"); + } + if (shape.corners) module._set_shape_corners(...shape.corners); + if (shape.blur) module._set_shape_blur(...shape.blur); + for (const shadow of shape.shadows ?? []) + module._add_shape_shadow(...shadow); + if (shape.masked) module._set_shape_masked_group(true); + if (shape.clip !== undefined) module._set_shape_clip_content(shape.clip); + if (shape.childrenBytes) + uploadBytes(module, shape.childrenBytes, "_set_children"); + if (shape.textBytes) + uploadBytes(module, shape.textBytes, "_set_shape_text_content"); + } + module._use_shape(...ROOT); + uploadBytes(module, scene.childrenBytes, "_set_children"); + } finally { + module._end_loading(); + } + // Like the product bridge, compute text layouts only after ending loading. + for (const shape of scene.shapes) { + if (shape.textBytes) module._update_shape_text_layout_for(...shape.id); + } +} + +export function defaultCase(params, { anchor, panDistance, frameCount = 20 }) { + return { + id: "default", + params, + initialView: { scale: 1, x: 0, y: 0 }, + interactions: { + pan: { + settleMs: 100, + frames: Array.from({ length: frameCount }, (_, i) => ({ + scale: 1, + x: (panDistance[0] * (i + 1)) / frameCount, + y: (panDistance[1] * (i + 1)) / frameCount, + })), + }, + zoom: { + settleMs: 100, + anchor, + frames: Array.from({ length: frameCount }, (_, i) => { + const scale = 1 + (i + 1) / frameCount; + return { + scale, + x: anchor[0] / scale - anchor[0], + y: anchor[1] / scale - anchor[1], + }; + }), + }, + }, + }; +} + +export function randomShapes(kind, params, seed) { + validateParameters(params, [ + "count", + "width", + "height", + "minSize", + "maxSize", + ...(kind === "shadows" ? ["shadows"] : []), + ]); + const count = validateCount(params.count); + for (const name of ["width", "height", "minSize", "maxSize"]) { + if (!Number.isFinite(params[name]) || params[name] <= 0) + throw new Error(`Invalid ${name}`); + } + if (params.minSize >= params.maxSize) throw new Error("Invalid size range"); + if ( + kind === "shadows" && + (!Array.isArray(params.shadows) || + !params.shadows.every( + (shadow) => + Array.isArray(shadow) && + shadow.length === 7 && + Number.isInteger(shadow[0]) && + shadow[0] >= 0 && + shadow[0] <= 0xffffffff && + shadow.slice(1, 5).every(Number.isFinite) && + shadow[1] >= 0 && + [0, 1].includes(shadow[5]) && + typeof shadow[6] === "boolean", + )) + ) { + throw new Error("Invalid shadows"); + } + const rng = randomSource(seed); + const shapes = []; + for (let i = 0; i < count; i++) { + const id = rng.id(); + const x = rng.int(0, params.width); + const y = rng.int(0, params.height); + const width = rng.int(params.minSize, params.maxSize); + const height = rng.int(params.minSize, params.maxSize); + const color = rng.int(0, 0x1000000); + const shape = rectangle( + id, + ROOT, + [x, y, x + width, y + height], + argb(color, rng.float(0.1, 1)), + ); + shape.strokes = [stroke(10, argb(color, rng.float(0.1, 1)))]; + if (kind === "paths") { + shape.type = 4; + const radius = Math.min(width, height) / 2; + const points = Array.from({ length: 10 }, (_, point) => { + const angle = (Math.PI / 5) * point - Math.PI / 2; + const r = point % 2 === 0 ? radius : radius * 0.4; + return [ + x + width / 2 + r * Math.cos(angle), + y + height / 2 + r * Math.sin(angle), + ]; + }); + shape.path = serializePath(points, true); + } else if (kind === "plus") { + shape.type = 4; + shape.fills = []; + // Stroke extents account for line thickness. Keep the original zero-area selrect. + const horizontal = rng.float(0, 1) < 0.5; + const end = horizontal ? [x + width, y] : [x, y + width]; + shape.bounds = [x, y, ...end]; + shape.path = serializePath([[x, y], end]); + } else if (kind === "shadows") { + shape.shadows = params.shadows; + } + shapes.push(shape); + } + return finishScene(shapes); +} diff --git a/frontend/resources/wasm-playground/scenes/texts.js b/frontend/resources/wasm-playground/scenes/texts.js new file mode 100644 index 0000000000..39b6a60e70 --- /dev/null +++ b/frontend/resources/wasm-playground/scenes/texts.js @@ -0,0 +1,123 @@ +import { + ROOT, + argb, + defaultCase, + finishScene, + randomSource, + serializeText, + stroke, + uploadScene, + validateCount, + validateParameters, +} from "./shared.js"; + +export default { + id: "texts", + version: 1, + description: + "Seeded multiline text using the renderer's embedded fallback font", + cases: [ + defaultCase( + { + count: 100, + width: 1920, + height: 1080, + minWidth: 20, + maxWidth: 500, + minHeight: 20, + maxHeight: 100, + minFontSize: 10, + maxFontSize: 60, + maxLines: 5, + maxWords: 10, + words: [ + "Hello", + "World", + "Penpot", + "Canvas", + "Text", + "Shape", + "Random", + "Line", + ], + }, + { anchor: [960, 540], panDistance: [120, 60] }, + ), + ], + createScene(params, seed) { + const dimensions = [ + "width", + "height", + "minWidth", + "maxWidth", + "minHeight", + "maxHeight", + "minFontSize", + "maxFontSize", + "maxLines", + "maxWords", + ]; + validateParameters(params, ["count", "words", ...dimensions]); + if ( + !dimensions.every( + (key) => Number.isFinite(params[key]) && params[key] > 0, + ) || + !Number.isSafeInteger(params.maxLines) || + !Number.isSafeInteger(params.maxWords) || + params.minWidth >= params.maxWidth || + params.minHeight >= params.maxHeight || + params.minFontSize >= params.maxFontSize || + !Array.isArray(params.words) || + !params.words.length || + !params.words.every((word) => typeof word === "string" && word.length) + ) { + throw new Error("Invalid text parameters"); + } + const count = validateCount(params.count); + const rng = randomSource(seed); + const shapes = []; + for (let i = 0; i < count; i++) { + const id = rng.id(); + const x = rng.int(0, params.width); + const y = rng.int(0, params.height); + const width = rng.int(params.minWidth, params.maxWidth); + const height = rng.int(params.minHeight, params.maxHeight); + const strokes = []; + if (rng.float(0, 1) < 0.3) { + const strokeCount = rng.int(1, 3); + for (let j = 0; j < strokeCount; j++) + strokes.push( + stroke( + rng.int(1, 10), + argb(rng.int(0, 0x1000000), rng.float(0.1, 1)), + ), + ); + } + const fontSize = rng.float(params.minFontSize, params.maxFontSize); + const lines = Array.from( + { length: rng.int(1, params.maxLines + 1) }, + () => + Array.from( + { length: rng.int(1, params.maxWords + 1) }, + () => params.words[rng.int(0, params.words.length)], + ).join(" "), + ); + const text = lines.join("\n"); + shapes.push({ + id, + parent: ROOT, + type: 5, + bounds: [x, y, x + width, y + height], + strokes, + text, + textBytes: serializeText( + text, + fontSize, + argb(rng.int(0, 0x1000000), rng.float(0.5, 1)), + ), + }); + } + return finishScene(shapes); + }, + upload: uploadScene, +}; diff --git a/frontend/resources/wasm-playground/shadows.html b/frontend/resources/wasm-playground/shadows.html index ea6b206633..db448d5ae1 100644 --- a/frontend/resources/wasm-playground/shadows.html +++ b/frontend/resources/wasm-playground/shadows.html @@ -1,90 +1,47 @@ - - - - - WASM + WebGL2 Canvas - - - - - - + mountScenario( + scenario, + initModule, + document.getElementById("canvas"), + ).catch((error) => { + console.error(error); + const message = document.createElement("pre"); + message.id = "error"; + message.textContent = error.message; + document.body.append(message); + }); + + diff --git a/frontend/resources/wasm-playground/texts.html b/frontend/resources/wasm-playground/texts.html index d977fdd59e..b6dbd24b0b 100644 --- a/frontend/resources/wasm-playground/texts.html +++ b/frontend/resources/wasm-playground/texts.html @@ -1,105 +1,47 @@ - - - - - WASM + WebGL2 Texts - - - - - - + mountScenario( + scenario, + initModule, + document.getElementById("canvas"), + ).catch((error) => { + console.error(error); + const message = document.createElement("pre"); + message.id = "error"; + message.textContent = error.message; + document.body.append(message); + }); + + diff --git a/frontend/scripts/renderer-benchmarks.js b/frontend/scripts/renderer-benchmarks.js new file mode 100644 index 0000000000..6115705006 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks.js @@ -0,0 +1,530 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { randomUUID } from "node:crypto"; +import { spawn, execFileSync } from "node:child_process"; +import { parseArgs } from "node:util"; +import { scenarios } from "../resources/wasm-playground/scenes/index.js"; +import { + collectCases, + WEBGL_OPTIONS, +} from "../resources/wasm-playground/benchmarks/contract.js"; +import { + summarizeRun, + formatSummary, +} from "../resources/wasm-playground/benchmarks/statistics.js"; +import { + compareRuns, + formatComparison, +} from "./renderer-benchmarks/compare.js"; + +const frontend = fileURLToPath(new URL("../", import.meta.url)); +const root = path.resolve(frontend, ".."); +const LIMITS = + "Renderer submission timings only: not GPU completion, displayed pixels, whole-editor responsiveness, or visual correctness."; + +export function readOptions(argv) { + const { values, positionals } = parseArgs({ + args: argv, + allowPositionals: true, + options: { + help: { type: "boolean" }, + headed: { type: "boolean", default: false }, + diagnostic: { type: "boolean", default: false }, + "base-url": { type: "string", default: "http://localhost:3000" }, + output: { type: "string" }, + filter: { type: "string", default: "" }, + seed: { type: "string", default: "42" }, + warmups: { type: "string", default: "3" }, + repetitions: { type: "string", default: "10" }, + "timeout-ms": { type: "string", default: "30000" }, + }, + }); + const command = positionals[0] ?? "run"; + if (!["run", "compare"].includes(command)) + throw new Error(`Unknown command: ${command}`); + const number = (name, minimum, maximum) => { + const value = Number(values[name]); + if (!Number.isSafeInteger(value) || value < minimum || value > maximum) + throw new Error(`Invalid --${name}`); + return value; + }; + if ( + !values.help && + (command === "run" ? positionals.length > 1 : positionals.length !== 3) + ) + throw new Error( + "Use run [options] or compare baseline.json candidate.json [options]", + ); + const baseUrl = new URL(values["base-url"]); + if ( + !["http:", "https:"].includes(baseUrl.protocol) || + baseUrl.username || + baseUrl.password + ) + throw new Error( + "--base-url must be an HTTP(S) server URL without credentials", + ); + return { + command, + files: positionals.slice(1), + help: values.help, + headed: values.headed, + diagnostic: values.diagnostic, + baseUrl: baseUrl.href, + output: values.output, + filter: values.filter, + seed: number("seed", 0, 0xffffffff), + warmups: number("warmups", 0, 1000), + repetitions: number("repetitions", 1, 10000), + timeoutMs: number("timeout-ms", 100, 600000), + }; +} + +export async function bounded(operation, milliseconds, signal) { + let timer; + let onAbort; + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timer = setTimeout( + () => + reject(new Error(`Node watchdog expired after ${milliseconds} ms`)), + milliseconds, + ); + }), + new Promise((_, reject) => { + onAbort = () => reject(signal.reason ?? new Error("Interrupted")); + if (signal?.aborted) onAbort(); + else signal?.addEventListener("abort", onAbort, { once: true }); + }), + ]); + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + } +} + +async function save(output, result) { + await fs.mkdir(path.dirname(output), { recursive: true }); + const temporary = `${output}.tmp`; + await fs.writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`); + await fs.rename(temporary, output); +} + +function execute(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: "inherit", ...options }); + child.once("error", reject); + child.once("exit", (code, signal) => + code === 0 + ? resolve() + : reject(new Error(`${command} failed (${signal ?? code})`)), + ); + }); +} + +async function requireFile(filename, instruction) { + try { + await fs.access(filename); + } catch { + throw new Error(instruction); + } +} + +async function preflight(options) { + await requireFile( + "/opt/emsdk/emsdk_env.sh", + "Run inside the prepared Penpot development container (Emscripten is missing).", + ); + await requireFile( + path.join(root, "render-wasm/node_modules/esbuild/package.json"), + "Prepare renderer dependencies with pnpm install in render-wasm first.", + ); + const { chromium } = await import("playwright"); + await requireFile( + chromium.executablePath(), + "Chromium is missing; run the frontend scripts/setup preparation command first.", + ); + try { + const response = await fetch(options.baseUrl, { + signal: AbortSignal.timeout(3000), + }); + // A static root may legitimately be absent before assets have been copied. + if (response.status >= 500) throw new Error(`HTTP ${response.status}`); + } catch (error) { + throw new Error( + `Static server unavailable at ${options.baseUrl}; run node scripts/e2e-server.js from frontend first. ${error.message}`, + ); + } + return chromium; +} + +export function recordAttempt(benchmarkCase, attempt, index, warmups) { + if (attempt.environment) { + benchmarkCase.environment ??= attempt.environment; + if ( + JSON.stringify(benchmarkCase.environment) !== + JSON.stringify(attempt.environment) + ) { + attempt.status = "failed"; + attempt.error = "Effective renderer environment changed within the case"; + } + } + benchmarkCase.attempts.push({ index, warmup: index < warmups, ...attempt }); + if (attempt.status !== "ok") benchmarkCase.status = "failed"; +} + +export function finishCases(cases, expected, status) { + for (const c of cases) { + c.unattempted = expected - c.attempts.length; + if (c.status === "running") c.status = status; + } +} + +export async function run(options) { + const abort = new AbortController(); + const interrupted = () => abort.abort(new Error("Benchmark interrupted")); + process.once("SIGINT", interrupted); + process.once("SIGTERM", interrupted); + const runId = randomUUID(); + const output = path.resolve( + options.output ?? + path.join(os.tmpdir(), `renderer-benchmarks-${runId}.json`), + ); + const browserArgs = ["--enable-gpu"]; + const configuration = { + viewport: { width: 1920, height: 1080 }, + dpr: 2, + webglOptions: WEBGL_OPTIONS, + rendererOptions: { debug: 0, browser: 1, backgroundArgb: 0xfffabada }, + warmups: options.warmups, + timeoutMs: options.timeoutMs, + }; + const git = (args) => + execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); + const result = { + schemaVersion: 1, + runId, + timestamp: new Date().toISOString(), + status: "running", + scope: LIMITS, + warnings: [], + metadata: { + git: { + sha: git(["rev-parse", "HEAD"]), + dirty: Boolean(git(["status", "--porcelain"])), + }, + build: { + mode: "release", + features: [], + target: "frontend", + prepared: true, + }, + environment: { + node: process.version, + os: os.platform(), + osRelease: os.release(), + arch: os.arch(), + cpu: os.cpus()[0]?.model ?? "unknown", + host: os.hostname(), + headless: !options.headed, + browserArgs, + }, + configuration, + baseUrl: options.baseUrl, + serverReachable: false, + masterSeed: options.seed, + repetitions: options.repetitions, + metricSemantics: + "renderer-boundaries-v1;module-factory-includes-wasm-fetch;upload-includes-initial-view-index;finalization-includes-set-view-end", + }, + cases: collectCases(scenarios, options.seed, options.filter), + }; + let server; + let browser; + let context; + let page; + let pageErrors = []; + let suiteStart; + const closeContext = async () => { + if (!context) return; + const previous = context; + const previousPage = page; + context = null; + page = null; + try { + if (previousPage) + await bounded( + previousPage.evaluate(() => globalThis.rendererBenchmark?.dispose()), + 2000, + ); + } catch { + /* A failed or blocked page still has to be closed. */ + } + await bounded(previous.close(), 5000); + }; + const openPage = async () => { + context = await browser.newContext({ + viewport: configuration.viewport, + deviceScaleFactor: configuration.dpr, + serviceWorkers: "block", + }); + page = await context.newPage(); + pageErrors = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + page.on("crash", () => pageErrors.push("Browser page crashed")); + await page.goto( + new URL(`wasm-playground/benchmark.html?run=${runId}`, options.baseUrl) + .href, + { timeout: options.timeoutMs }, + ); + await page.waitForFunction( + () => globalThis.rendererBenchmarkReady === true, + null, + { timeout: options.timeoutMs }, + ); + }; + try { + if (result.metadata.git.dirty) { + result.warnings.push( + "Dirty worktree: local development evidence, not a reproducible canonical record.", + ); + console.warn(result.warnings.at(-1)); + } + const chromium = await preflight(options); + result.metadata.serverReachable = true; + const packageJson = JSON.parse( + await fs.readFile( + new URL("../node_modules/playwright/package.json", import.meta.url), + "utf8", + ), + ); + result.metadata.environment.playwright = packageJson.version; + // Disallow package-manager network fallback in a prepared run. + const buildStart = performance.now(); + await execute("./build", ["frontend", "--offline", "--locked"], { + cwd: path.join(root, "render-wasm"), + signal: abort.signal, + env: { + ...process.env, + PENPOT_WASM_PREPARED: "1", + BUILD_MODE: "release", + NODE_ENV: "production", + RENDER_TARGET: "frontend", + BUILD_NAME: "render-wasm-benchmark", + CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", + COREPACK_ENABLE_NETWORK: "0", + CARGO_NET_OFFLINE: "true", + VERSION: runId, + }, + }); + result.metadata.build.durationMs = performance.now() - buildStart; + const { copyWasmPlayground } = await import("./_helpers.js"); + const previousCwd = process.cwd(); + try { + process.chdir(frontend); + await copyWasmPlayground(); + } finally { + process.chdir(previousCwd); + } + const markerName = "benchmark-run.json"; + await fs.writeFile( + path.join(frontend, "resources/public/wasm-playground", markerName), + JSON.stringify({ runId }), + ); + const marker = await fetch( + new URL(`wasm-playground/${markerName}?run=${runId}`, options.baseUrl), + { signal: AbortSignal.timeout(3000), cache: "no-store" }, + ); + if (!marker.ok || (await marker.json()).runId !== runId) + throw new Error( + "The static server does not serve this workspace's current benchmark assets", + ); + suiteStart = performance.now(); + server = await chromium.launchServer({ + headless: !options.headed, + args: browserArgs, + }); + browser = await chromium.connect(server.wsEndpoint()); + result.metadata.environment.chromium = browser.version(); + const attemptOptions = { + ...configuration, + moduleUrl: new URL( + `js/render-wasm-benchmark.js?run=${runId}`, + options.baseUrl, + ).href, + }; + for (const benchmarkCase of result.cases) { + if (abort.signal.aborted) throw abort.signal.reason; + console.log(`Running ${benchmarkCase.id}`); + benchmarkCase.status = "running"; + for ( + let index = 0; + index < options.warmups + options.repetitions; + index++ + ) { + let attempt; + try { + if (!page) await bounded(openPage(), options.timeoutMs, abort.signal); + attempt = await bounded( + page.evaluate( + ([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o), + [benchmarkCase, attemptOptions], + ), + options.timeoutMs * 2 + 5000, + abort.signal, + ); + } catch (error) { + attempt = { + status: "failed", + metrics: {}, + invalidMetrics: {}, + error: String(error.stack ?? error), + slices: [], + cachedSlices: [], + }; + } + if (pageErrors.length) { + attempt.status = "failed"; + attempt.error = [attempt.error, ...pageErrors] + .filter(Boolean) + .join("\n"); + } + recordAttempt(benchmarkCase, attempt, index, options.warmups); + if (attempt.environment) { + if (attempt.environment.webgl.acceleration === "software") { + const warning = + "Software WebGL: compare only equivalent renderer environments."; + if (!result.warnings.includes(warning)) { + result.warnings.push(warning); + console.warn(warning); + } + } + } + if (attempt.status !== "ok") benchmarkCase.status = "failed"; + // Persist only outside browser timing; every failed attempt remains visible. + await save(output, result); + if (abort.signal.aborted) { + benchmarkCase.status = "interrupted"; + throw abort.signal.reason; + } + if (!browser.isConnected()) + throw new Error( + "Browser disconnected; remaining cases were not attempted", + ); + if (attempt.status === "failed") { + // Cache history cannot be restored after failure: stop this case, then + // continue independent cases. Do not silently replace failed samples. + benchmarkCase.unattempted = + options.warmups + options.repetitions - index - 1; + await closeContext(); + break; + } + if (benchmarkCase.group === "fresh-module-context") + await closeContext(); + } + await closeContext(); + if (benchmarkCase.status === "running") benchmarkCase.status = "complete"; + } + result.status = result.cases.every((c) => c.status === "complete") + ? "complete" + : "failed"; + } catch (error) { + result.status = abort.signal.aborted ? "interrupted" : "failed"; + result.error = String(error.stack ?? error); + console.error(result.error); + } finally { + finishCases( + result.cases, + options.warmups + options.repetitions, + result.status, + ); + if (suiteStart !== undefined) + result.metadata.suiteDurationMs = performance.now() - suiteStart; + result.summary = summarizeRun(result); + // Render a report once, after every scored case. Report work never separates + // attempts or cases. The same data also goes to JSON and the terminal. + if (browser?.isConnected() && !abort.signal.aborted) { + try { + if (!page) await bounded(openPage(), 5000); + await bounded( + page.evaluate( + (value) => globalThis.rendererBenchmark.showResult(value), + result, + ), + 5000, + ); + } catch (error) { + result.warnings.push(`Page report unavailable: ${error.message}`); + } + } + try { + await closeContext(); + } catch { + /* Browser cleanup below also closes blocked pages. */ + } + // The browser server exposes its child process for a bounded hard cleanup. + try { + if (browser) await bounded(browser.close(), 5000); + } catch { + server?.process()?.kill("SIGKILL"); + } + try { + if (server) await bounded(server.close(), 5000); + } catch { + server?.process()?.kill("SIGKILL"); + } + process.removeListener("SIGINT", interrupted); + process.removeListener("SIGTERM", interrupted); + result.finishedAt = new Date().toISOString(); + await save(output, result); + } + console.log(LIMITS); + console.log(formatSummary(result.summary)); + console.log(`Result: ${output}`); + return result.status === "complete" ? 0 : 1; +} + +export async function main(argv = process.argv.slice(2)) { + const options = readOptions(argv); + if (options.help) { + console.log( + "Renderer benchmarks\n node scripts/renderer-benchmarks.js run [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--output FILE]\n node scripts/renderer-benchmarks.js compare BASELINE CANDIDATE [--diagnostic] [--output FILE]\nRun requires a prepared development container and an existing static server on port 3000. Defaults: 3 warmups, 10 measured attempts, 30s timeout; provisional pending calibration.\n" + + LIMITS, + ); + return 0; + } + if (options.command === "compare") { + const results = await Promise.all( + options.files.map(async (file) => + JSON.parse(await fs.readFile(file, "utf8")), + ), + ); + const comparison = compareRuns(...results, { + diagnostic: options.diagnostic, + }); + console.log(LIMITS); + console.log(formatComparison(comparison)); + if (options.output) await save(path.resolve(options.output), comparison); + return 0; + } + return run(options); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href +) { + main() + .then((code) => { + process.exitCode = code; + }) + .catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/frontend/scripts/renderer-benchmarks/compare.js b/frontend/scripts/renderer-benchmarks/compare.js new file mode 100644 index 0000000000..db1bc47736 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/compare.js @@ -0,0 +1,281 @@ +import { + compareObservations, + metricNames, + observations, + statisticalMethod, + STATISTICAL_WARNINGS, +} from "../../resources/wasm-playground/benchmarks/statistics.js"; + +function record(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function requireValid(condition, message) { + if (!condition) throw new Error(`Invalid benchmark result: ${message}`); +} + +export function validateResult(result) { + requireValid( + record(result) && result.schemaVersion === 1, + "expected schemaVersion 1", + ); + requireValid(record(result.metadata), "metadata is required"); + for (const field of ["environment", "configuration"]) + requireValid( + record(result.metadata[field]), + `metadata.${field} is required`, + ); + requireValid(Array.isArray(result.cases), "cases must be an array"); + const ids = new Set(); + for (const benchmarkCase of result.cases) { + requireValid(record(benchmarkCase), "case must be an object"); + requireValid( + typeof benchmarkCase.id === "string" && + benchmarkCase.id.length && + !ids.has(benchmarkCase.id), + "case IDs must be unique nonempty strings", + ); + ids.add(benchmarkCase.id); + for (const field of ["scenarioId", "group", "preparation"]) + requireValid( + typeof benchmarkCase[field] === "string" && benchmarkCase[field].length, + `${benchmarkCase.id}.${field} is required`, + ); + requireValid( + Number.isInteger(benchmarkCase.scenarioVersion) && + benchmarkCase.scenarioVersion >= 1, + "scenarioVersion must be a positive integer", + ); + requireValid( + Number.isInteger(benchmarkCase.sceneSeed) && + benchmarkCase.sceneSeed >= 0 && + benchmarkCase.sceneSeed <= 0xffffffff, + "sceneSeed must be a uint32", + ); + for (const field of ["params", "initialView"]) + requireValid( + record(benchmarkCase[field]), + `${benchmarkCase.id}.${field} is required`, + ); + requireValid( + benchmarkCase.interaction === null || record(benchmarkCase.interaction), + "interaction must be an object or null", + ); + requireValid( + ["complete", "failed", "interrupted", "unattempted"].includes( + benchmarkCase.status, + ), + "unknown case status", + ); + requireValid( + Array.isArray(benchmarkCase.attempts), + "attempts must be an array", + ); + for (const attempt of benchmarkCase.attempts) { + requireValid( + record(attempt) && ["ok", "invalid", "failed"].includes(attempt.status), + "unknown attempt status", + ); + requireValid( + typeof attempt.warmup === "boolean", + "attempt.warmup must be boolean", + ); + requireValid( + record(attempt.metrics), + "attempt.metrics must be an object", + ); + requireValid( + attempt.invalidMetrics === undefined || record(attempt.invalidMetrics), + "invalidMetrics must be an object", + ); + for (const [name, value] of Object.entries(attempt.metrics)) + requireValid( + (value === null && + Object.hasOwn(attempt.invalidMetrics ?? {}, name)) || + (Number.isFinite(value) && value >= 0), + `metric ${name} must be finite and nonnegative, or explicitly invalid null`, + ); + for (const reason of Object.values(attempt.invalidMetrics ?? {})) + requireValid( + typeof reason === "string" && reason.length, + "invalid metric reasons must be nonempty strings", + ); + } + } +} + +function differences(baseline, candidate, path, mismatches) { + if (record(baseline) && record(candidate)) { + for (const key of [ + ...new Set([...Object.keys(baseline), ...Object.keys(candidate)]), + ].sort()) + differences(baseline[key], candidate[key], `${path}.${key}`, mismatches); + } else if (Array.isArray(baseline) && Array.isArray(candidate)) { + if (baseline.length !== candidate.length) + mismatches.push({ path, baseline, candidate }); + else + baseline.forEach((value, index) => + differences(value, candidate[index], `${path}[${index}]`, mismatches), + ); + } else if (baseline !== candidate) + mismatches.push({ + path, + baseline: baseline ?? null, + candidate: candidate ?? null, + }); +} + +export class CompatibilityError extends Error { + constructor(mismatches) { + super( + `Incompatible benchmark runs: ${mismatches.map((entry) => entry.path).join(", ")}. Use an explicit diagnostic comparison to inspect mismatches.`, + ); + this.name = "CompatibilityError"; + this.mismatches = mismatches; + } +} + +export function compareRuns( + baseline, + candidate, + { diagnostic = false, ...options } = {}, +) { + validateResult(baseline); + validateResult(candidate); + const mismatches = []; + // Compare renderer revisions, but never silently mix debug/profiling builds + // with ordinary release measurements. + for (const field of ["mode", "features", "target"]) { + differences( + baseline.metadata.build?.[field], + candidate.metadata.build?.[field], + `metadata.build.${field}`, + mismatches, + ); + } + for (const field of ["environment", "configuration", "metricSemantics"]) + differences( + baseline.metadata[field], + candidate.metadata[field], + `metadata.${field}`, + mismatches, + ); + const firstCases = new Map( + baseline.cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]), + ); + const secondCases = new Map( + candidate.cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]), + ); + const caseIds = [ + ...new Set([...firstCases.keys(), ...secondCases.keys()]), + ].sort(); + for (const id of caseIds) { + const first = firstCases.get(id); + const second = secondCases.get(id); + if (!first || !second) { + mismatches.push({ + path: `cases.${id}`, + baseline: Boolean(first), + candidate: Boolean(second), + }); + continue; + } + for (const field of [ + "scenarioId", + "scenarioVersion", + "sceneSeed", + "params", + "initialView", + "interaction", + "group", + "preparation", + "environment", + "metricSemantics", + ]) + differences( + first[field], + second[field], + `cases.${id}.${field}`, + mismatches, + ); + // A failed attempt may lack metrics. Only compare present metric sets when + // both cases completed; failures still produce honest sample counts. + if (first.status === "complete" && second.status === "complete") + differences( + metricNames(first), + metricNames(second), + `cases.${id}.metrics`, + mismatches, + ); + } + if (mismatches.length && !diagnostic) + throw new CompatibilityError(mismatches); + return { + schemaVersion: 1, + compatible: mismatches.length === 0, + diagnostic, + mismatches, + method: statisticalMethod(options), + warnings: [ + ...(mismatches.length + ? [ + "DIAGNOSTIC ONLY: these runs are incompatible; differences cannot establish a performance change.", + ] + : []), + ...STATISTICAL_WARNINGS, + ], + cases: caseIds + .filter((id) => firstCases.has(id) && secondCases.has(id)) + .map((id) => { + const first = firstCases.get(id); + const second = secondCases.get(id); + const metrics = [ + ...new Set([...metricNames(first), ...metricNames(second)]), + ].sort(); + return { + id, + metrics: Object.fromEntries( + metrics.map((metric) => [ + metric, + compareObservations( + observations(first, metric), + observations(second, metric), + options, + ), + ]), + ), + }; + }), + }; +} + +export function formatComparison(comparison) { + const lines = [ + `Statistics: ${comparison.method.name} v${comparison.method.version}; ${comparison.method.resamples} resamples, seed ${comparison.method.seed}.`, + ...comparison.warnings, + ]; + for (const mismatch of comparison.mismatches) + lines.push( + `MISMATCH ${mismatch.path}: ${JSON.stringify(mismatch.baseline)} -> ${JSON.stringify(mismatch.candidate)}`, + ); + const display = (value) => (value === null ? "n/a" : value.toFixed(3)); + for (const benchmarkCase of comparison.cases) { + lines.push(benchmarkCase.id); + for (const [name, metric] of Object.entries(benchmarkCase.metrics)) { + lines.push( + ` ${name}: median ${display(metric.baseline.median)} -> ${display(metric.candidate.median)}, change ${display(metric.absoluteChange)} (${display(metric.relativeChangePercent)}%); n=${metric.baseline.n}/${metric.candidate.n}`, + ); + if (metric.absoluteInterval) + lines.push( + ` change interval: [${display(metric.absoluteInterval.low)}, ${display(metric.absoluteInterval.high)}]`, + ); + if (metric.relativeIntervalPercent) + lines.push( + ` relative interval: [${display(metric.relativeIntervalPercent.low)}, ${display(metric.relativeIntervalPercent.high)}]%`, + ); + for (const [field, reason] of Object.entries(metric.suppressed)) + lines.push(` ${field}: ${reason}`); + } + } + return lines.join("\n"); +} diff --git a/frontend/scripts/renderer-benchmarks/tests/adapter.test.js b/frontend/scripts/renderer-benchmarks/tests/adapter.test.js new file mode 100644 index 0000000000..e1014a6468 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/tests/adapter.test.js @@ -0,0 +1,106 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { createRendererAdapter } from "../../../resources/wasm-playground/js/lib.js"; + +function fixture() { + let next = 0; + const frames = new Map(); + const timers = new Map(); + const scheduler = { + requestAnimationFrame(callback) { + frames.set(++next, callback); + return next; + }, + cancelAnimationFrame(id) { + frames.delete(id); + }, + setTimeout(callback) { + timers.set(++next, callback); + return next; + }, + clearTimeout(id) { + timers.delete(id); + }, + }; + const state = { rendered: 0, cleaned: false, deleted: false, lost: false }; + const gl = { + getExtension: (name) => + name === "WEBGL_lose_context" + ? { + loseContext() { + state.lost = true; + }, + } + : null, + isContextLost: () => state.lost, + }; + const canvas = new EventTarget(); + Object.assign(canvas, { width: 1920, height: 1080, getContext: () => gl }); + const module = { + GL: { + registerContext: () => 1, + makeContextCurrent() {}, + deleteContext() { + state.deleted = true; + }, + }, + _init() {}, + _set_render_options() {}, + _set_browser() {}, + _set_view_start() {}, + _set_view() {}, + _set_view_end() {}, + _render_from_cache() {}, + _render() { + state.rendered++; + return 2; + }, + _clean_up() { + state.cleaned = true; + }, + }; + const adapter = createRendererAdapter(module, scheduler); + adapter.assignCanvas(canvas); + adapter.setupInteraction(canvas); + return { adapter, state, canvas, frames, timers, module }; +} + +test("disposing an adapter settles pending work and isolates late callbacks", async () => { + const first = fixture(); + const second = fixture(); + const pending = first.adapter.renderFull(); + const lateFrame = [...first.frames.values()][0]; + first.canvas.dispatchEvent( + Object.assign(new Event("wheel"), { deltaY: 1, offsetX: 10, offsetY: 20 }), + ); + const lateTimer = [...first.timers.values()][0]; + first.adapter.dispose(); + first.adapter.dispose(); + lateFrame(16); + lateTimer(); + assert.equal(await pending, false); + assert.equal(first.frames.size + first.timers.size, 0); + assert.deepEqual(first.state, { + rendered: 0, + cleaned: true, + deleted: true, + lost: true, + }); + first.canvas.dispatchEvent(new Event("wheel")); + assert.equal(first.timers.size, 0); + const completed = second.adapter.renderFull(); + [...second.frames.values()][0](32); + assert.equal(await completed, true); + assert.equal(second.state.cleaned, false); + second.adapter.dispose(); +}); + +test("adapter deletes its context even when native cleanup throws", () => { + const f = fixture(); + f.module._clean_up = () => { + throw new Error("cleanup failed"); + }; + assert.throws(() => f.adapter.dispose(), /cleanup failed/); + assert.equal(f.state.deleted, true); + assert.equal(f.state.lost, true); +}); diff --git a/frontend/scripts/renderer-benchmarks/tests/compare.test.js b/frontend/scripts/renderer-benchmarks/tests/compare.test.js new file mode 100644 index 0000000000..ffda4cf585 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/tests/compare.test.js @@ -0,0 +1,181 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { compareRuns, formatComparison } from "../compare.js"; + +function run(value = 10) { + return { + schemaVersion: 1, + metadata: { + git: { sha: "before", dirty: false }, + build: { mode: "release", features: [] }, + environment: { chromium: "123", headless: true }, + configuration: { viewport: { width: 1920, height: 1080 }, dpr: 2 }, + }, + cases: [ + { + id: "rects/default/load", + scenarioId: "rects", + scenarioVersion: 1, + sceneSeed: 123, + params: { count: 1000 }, + initialView: { x: 0, y: 0, zoom: 1 }, + interaction: null, + group: "fresh-module-context", + preparation: "fresh", + environment: { webgl: { renderer: "GPU" } }, + status: "complete", + attempts: Array.from({ length: 10 }, (_, index) => ({ + index, + warmup: false, + status: "ok", + metrics: { render: value }, + invalidMetrics: {}, + })), + }, + ], + }; +} + +test("offline comparison allows intentional source and build changes", () => { + const candidate = run(12); + candidate.metadata.git.sha = "after"; + candidate.metadata.build.revision = "candidate"; + const result = compareRuns(run(), candidate); + const metric = result.cases[0].metrics.render; + assert.equal(result.compatible, true); + assert.equal(metric.absoluteChange, 2); + assert.equal(metric.relativeChangePercent, 20); + assert.deepEqual(metric.absoluteInterval, { low: 2, high: 2 }); + assert.deepEqual(result, compareRuns(run(), candidate)); +}); + +for (const [name, change] of [ + [ + "profiling features", + (r) => { + r.metadata.build.features = ["profile"]; + }, + ], + [ + "build mode", + (r) => { + r.metadata.build.mode = "debug"; + }, + ], + [ + "DPR", + (r) => { + r.metadata.configuration.dpr = 1; + }, + ], + [ + "browser", + (r) => { + r.metadata.environment.chromium = "124"; + }, + ], + [ + "GPU", + (r) => { + r.cases[0].environment.webgl.renderer = "software"; + }, + ], + [ + "seed", + (r) => { + r.cases[0].sceneSeed = 456; + }, + ], + [ + "version", + (r) => { + r.cases[0].scenarioVersion = 2; + }, + ], + [ + "workload", + (r) => { + r.cases[0].params.count = 2000; + }, + ], + [ + "preparation", + (r) => { + r.cases[0].preparation = "warm"; + }, + ], + [ + "case set", + (r) => { + r.cases[0].id = "different"; + }, + ], +]) { + test(`comparison refuses incompatible ${name} unless diagnostic override is explicit`, () => { + const candidate = run(); + change(candidate); + assert.throws(() => compareRuns(run(), candidate), { + name: "CompatibilityError", + }); + const result = compareRuns(run(), candidate, { diagnostic: true }); + assert.equal(result.compatible, false); + assert.ok(result.mismatches.length > 0); + assert.match(result.warnings.join(" "), /incompatible/i); + }); +} + +test("object key order is immaterial to compatibility", () => { + const candidate = run(); + candidate.metadata.configuration.viewport = { height: 1080, width: 1920 }; + assert.equal(compareRuns(run(), candidate).compatible, true); +}); + +test("relative change is unsupported when baseline is zero", () => { + const metric = compareRuns(run(0), run(1)).cases[0].metrics.render; + assert.equal(metric.relativeChangePercent, null); + assert.equal(metric.relativeIntervalPercent, null); + assert.match(metric.suppressed.relativeChangePercent, /zero/); +}); + +test("failed attempts never enter comparison observations", () => { + const candidate = run(12); + candidate.cases[0].attempts[0].status = "failed"; + const metric = compareRuns(run(), candidate).cases[0].metrics.render; + assert.equal(metric.candidate.n, 9); + assert.equal(metric.candidate.invalid, 1); + assert.equal(metric.absoluteInterval, null); +}); + +test("offline comparison rejects malformed results without diagnostic bypass", () => { + const missingEnvironment = run(); + delete missingEnvironment.metadata.environment; + const duplicateCase = run(); + duplicateCase.cases.push(structuredClone(duplicateCase.cases[0])); + const invalidMetric = run(); + invalidMetric.cases[0].attempts[0].metrics.render = "12"; + for (const invalid of [ + null, + {}, + { schemaVersion: 2 }, + missingEnvironment, + duplicateCase, + invalidMetric, + ]) { + assert.throws( + () => compareRuns(invalid, run(), { diagnostic: true }), + /Invalid benchmark result/, + ); + } +}); + +test("text comparisons display changes, intervals and incompatibilities", () => { + const candidate = run(12); + candidate.metadata.configuration.dpr = 1; + const report = formatComparison( + compareRuns(run(), candidate, { diagnostic: true }), + ); + assert.match(report, /DIAGNOSTIC ONLY/); + assert.match(report, /MISMATCH metadata.configuration.dpr: 2 -> 1/); + assert.match(report, /change 2\.000 \(20\.000%\)/); + assert.match(report, /change interval: \[2\.000, 2\.000\]/); +}); diff --git a/frontend/scripts/renderer-benchmarks/tests/protocol.test.js b/frontend/scripts/renderer-benchmarks/tests/protocol.test.js new file mode 100644 index 0000000000..625d6ec91b --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/tests/protocol.test.js @@ -0,0 +1,97 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + drain, + interact, + restore, +} from "../../../resources/wasm-playground/benchmarks/protocol.js"; + +function runtime(results) { + let time = 0; + const flags = []; + return { + flags, + now: () => time, + frame: async () => (time += 16), + sleep: async (ms) => { + time += ms; + }, + check: () => {}, + module: { + _render: (_timestamp, flag) => { + flags.push(flag); + time += 2; + return results.shift(); + }, + _set_view_start: () => { + time += 1; + }, + _set_view: () => { + time += 1; + }, + _render_from_cache: () => { + time += 2; + }, + _set_view_end: () => { + time += 5; + }, + }, + }; +} + +test("progressive rendering retains returned states and stops at Full", async () => { + const rt = runtime([1, 3, 2]); + const result = await drain(rt, { flags: 0 }); + assert.deepEqual(rt.flags, [0, 1, 3]); + assert.equal(result.slices.length, 3); + assert.ok(result.viewportReadyMs < result.fullMs); +}); + +test("immediate Full supplies both completion boundaries", async () => { + const result = await drain(runtime([2])); + assert.equal(result.viewportReadyMs, result.fullMs); +}); + +test("Partial can finish directly at Full without ViewportReady", async () => { + const rt = runtime([1, 2]); + const result = await drain(rt); + assert.deepEqual(rt.flags, [0, 1]); + assert.equal(result.viewportReadyMs, result.fullMs); +}); + +test("preparation drains to Full and applies SyncTiles only initially", async () => { + const rt = runtime([1, 3, 2]); + const result = await restore(rt, { scale: 1, x: 0, y: 0 }); + assert.deepEqual(rt.flags, [4, 1, 3]); + assert.equal(result.slices.at(-1).frameType, 2); +}); + +test("None and unknown frame types fail explicitly", async () => { + for (const frame of [0, 7, NaN]) { + await assert.rejects(drain(runtime([frame])), /frame type/); + } +}); + +test("finalization includes set_view_end but separates the settling delay", async () => { + const rt = runtime([3, 2]); + const result = await interact(rt, { + frames: [{ scale: 1, x: 3, y: 4 }], + settleMs: 100, + }); + assert.equal(result.metrics.setViewEndMs, 5); + assert.equal(result.metrics.settlingActualMs, 100); + assert.ok(result.metrics.timeToViewportReadyMs >= 7); + assert.ok( + result.metrics.lastInputToFullMs >= 100 + result.metrics.timeToFullMs, + ); + assert.deepEqual(rt.flags, [4, 3]); +}); + +test("cancellation stops further progressive work", async () => { + const rt = runtime([1, 2]); + rt.check = () => { + if (rt.flags.length) throw new Error("Canceled"); + }; + await assert.rejects(drain(rt), /Canceled/); + assert.equal(rt.flags.length, 1); +}); diff --git a/frontend/scripts/renderer-benchmarks/tests/runner.test.js b/frontend/scripts/renderer-benchmarks/tests/runner.test.js new file mode 100644 index 0000000000..305d639256 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/tests/runner.test.js @@ -0,0 +1,132 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { + readOptions, + bounded, + main, + recordAttempt, + finishCases, +} from "../../renderer-benchmarks.js"; +import { + collectCases, + deriveSeed, + validateMetrics, +} from "../../../resources/wasm-playground/benchmarks/contract.js"; +import { scenarios } from "../../../resources/wasm-playground/scenes/index.js"; + +test("CLI validates bounded numbers, commands and URLs before execution", () => { + assert.equal(readOptions([]).repetitions, 10); + for (const args of [ + ["--repetitions", "NaN"], + ["--warmups", "-1"], + ["--seed", "4294967296"], + ["--base-url", "file:///tmp"], + ["--unknown"], + ["compare", "one.json"], + ]) { + assert.throws(() => readOptions(args)); + } + assert.equal( + readOptions(["compare", "before.json", "after.json", "--diagnostic"]) + .diagnostic, + true, + ); +}); + +test("case collector retains a single scene seed across lifecycle and interaction groups", () => { + const cases = collectCases(scenarios, 42); + assert.equal(cases.length, 21); + const rectangles = cases.filter((c) => c.scenarioId === "rects"); + assert.equal(new Set(rectangles.map((c) => c.sceneSeed)).size, 1); + assert.equal(rectangles[0].sceneSeed, deriveSeed(42, "rects/default")); + assert.throws(() => collectCases(scenarios, 42, "missing"), /No matching/); + assert.throws( + () => collectCases([scenarios[0], scenarios[0]], 42), + /duplicate/, + ); +}); + +test("case collector rejects missing interactions and non-JSON case values", () => { + const scenario = scenarios[0]; + const invalid = { + ...scenario, + cases: [{ ...scenario.cases[0], interactions: {} }], + }; + assert.throws(() => collectCases([invalid], 42), /Incomplete/); + assert.throws( + () => + collectCases( + [ + { + ...scenario, + cases: [{ ...scenario.cases[0], params: { count: Infinity } }], + }, + ], + 42, + ), + /Non-serializable/, + ); +}); + +test("watchdog bounds unresolved work and propagates cancellation", async () => { + await assert.rejects(bounded(new Promise(() => {}), 10), /watchdog/); + const abort = new AbortController(); + const operation = bounded(new Promise(() => {}), 10000, abort.signal); + abort.abort(new Error("stopped")); + await assert.rejects(operation, /stopped/); + assert.equal(await bounded(Promise.resolve(42), 100), 42); +}); + +test("invalid aggregates retain an explicit reason", () => { + assert.deepEqual( + validateMetrics({ uploadMs: 0, timeToFullMs: NaN, activeInteractionMs: 2 }), + { + uploadMs: "below-timer-resolution", + timeToFullMs: "non-finite", + }, + ); +}); + +test("help does not need a renderer build or browser", async () => { + assert.equal(await main(["--help"]), 0); +}); + +test("changed environment is stored as a failed observation", () => { + const c = { + status: "running", + attempts: [], + environment: { webgl: { renderer: "A" } }, + }; + recordAttempt( + c, + { + status: "ok", + metrics: { uploadMs: 1 }, + environment: { webgl: { renderer: "B" } }, + }, + 0, + 0, + ); + assert.equal(c.attempts[0].status, "failed"); + assert.match(c.attempts[0].error, /environment changed/); + assert.equal(c.status, "failed"); +}); + +test("fatal failure and interruption preserve partial and unattempted counts", () => { + for (const status of ["failed", "interrupted"]) { + const cases = [ + { status: "complete", attempts: [{ status: "ok" }, { status: "ok" }] }, + { status: "running", attempts: [{ status: "ok" }] }, + { status: "unattempted", attempts: [] }, + ]; + finishCases(cases, 2, status); + assert.deepEqual( + cases.map((c) => [c.status, c.unattempted]), + [ + ["complete", 0], + [status, 1], + ["unattempted", 2], + ], + ); + } +}); diff --git a/frontend/scripts/renderer-benchmarks/tests/scenes.test.js b/frontend/scripts/renderer-benchmarks/tests/scenes.test.js new file mode 100644 index 0000000000..b5a8f40681 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/tests/scenes.test.js @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { scenarios } from "../../../resources/wasm-playground/scenes/index.js"; +import { + serializeText, + validateCount, +} from "../../../resources/wasm-playground/scenes/shared.js"; + +test("all seven workloads reproduce the same exact scene for a seed", () => { + assert.deepEqual( + scenarios.map((scenario) => scenario.id), + ["rects", "paths", "plus", "texts", "masks", "clips", "shadows"], + ); + for (const scenario of scenarios) { + const params = scenario.cases[0].params; + assert.deepEqual( + scenario.createScene(params, 42), + scenario.createScene(params, 42), + ); + } +}); + +test("shape counts are integers before allocation", () => { + assert.equal(validateCount("1000"), 1000); + for (const value of ["", "2x", -1, 1.5, Infinity, true, null, 0, 100001]) { + assert.throws(() => validateCount(value), /count/i); + } +}); + +test("scenes reject ignored parameters and malformed text dimensions", () => { + for (const scenario of scenarios) { + assert.throws( + () => scenario.createScene({ ...scenario.cases[0].params, typo: 1 }, 42), + /Unsupported parameter/, + ); + } + const texts = scenarios.find((s) => s.id === "texts"); + for (const override of [ + { width: -1 }, + { words: [] }, + { maxLines: 1.5 }, + { minFontSize: 100 }, + ]) { + assert.throws(() => + texts.createScene({ ...texts.cases[0].params, ...override }, 42), + ); + } +}); + +test("every shape is reachable exactly once and agrees with its parent", () => { + for (const scenario of scenarios) { + const scene = scenario.createScene(scenario.cases[0].params, 123); + const key = (id) => id.join(":"); + const shapes = new Map(scene.shapes.map((shape) => [key(shape.id), shape])); + assert.equal(shapes.size, scene.shapes.length); + const seen = new Set(); + function visit(parent, children) { + for (const id of children) { + const child = shapes.get(key(id)); + assert.ok(child, `${scenario.id}: missing child`); + assert.deepEqual(child.parent, parent); + assert.ok(!seen.has(key(id)), `${scenario.id}: duplicate child`); + seen.add(key(id)); + visit(id, child.children ?? []); + } + } + visit([0, 0, 0, 0], scene.children); + assert.equal(seen.size, scene.shapes.length, scenario.id); + } +}); + +test("text payload follows the current Rust paragraph and span ABI", () => { + const bytes = serializeText("Penpot café", 24, 0xff112233); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const encoded = new TextEncoder().encode("Penpot café"); + assert.equal(bytes.length, 16 + 1344 + encoded.length); + assert.equal(view.getUint32(0, true), 1); + assert.equal(view.getUint8(4), 0); // Left, not Center. + assert.equal(view.getFloat32(16 + 4, true), 24); + assert.equal(view.getInt32(16 + 16, true), 400); + assert.equal(view.getUint32(16 + 56, true), encoded.length); + assert.equal(view.getUint32(16 + 60, true), 1); + assert.equal(view.getUint32(16 + 64 + 4, true), 0xff112233); + assert.deepEqual(bytes.subarray(16 + 1344), encoded); +}); + +test("clip effects use valid enum values and actual booleans", () => { + const scenario = scenarios.find(({ id }) => id === "clips"); + const scene = scenario.createScene(scenario.cases[0].params, 42); + assert.equal(scene.shapes.length, 2); + for (const shape of scene.shapes) { + if (shape.blur) assert.equal(shape.blur[1], false); + for (const shadow of shape.shadows ?? []) { + assert.ok([0, 1].includes(shadow[5])); + assert.equal(typeof shadow[6], "boolean"); + } + } +}); diff --git a/frontend/scripts/renderer-benchmarks/tests/statistics.test.js b/frontend/scripts/renderer-benchmarks/tests/statistics.test.js new file mode 100644 index 0000000000..5f53a30cec --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/tests/statistics.test.js @@ -0,0 +1,102 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + summarize, + summarizeRun, + formatSummary, +} from "../../../resources/wasm-playground/benchmarks/statistics.js"; + +test("summaries use sample deviation and unscaled median absolute deviation", () => { + const summary = summarize([1, 2, 3, 4, 5]); + assert.equal(summary.mean, 3); + assert.equal(summary.median, 3); + assert.equal(summary.standardDeviation, Math.sqrt(2.5)); + assert.equal(summary.mad, 1); + assert.equal(summary.min, 1); + assert.equal(summary.max, 5); +}); + +test("invalid numbers are counted and zero observations survive", () => { + const summary = summarize([0, 2, NaN, Infinity, "3", null, -1]); + assert.equal(summary.n, 2); + assert.equal(summary.invalid, 5); + assert.equal(summary.median, 1); + assert.equal(summary.medianInterval, null); + assert.match(summary.suppressed.medianInterval, /10/); +}); + +test("empty and singleton observations have no invented spread", () => { + assert.equal(summarize([]).median, null); + assert.equal(summarize([]).standardDeviation, null); + assert.equal(summarize([2]).standardDeviation, null); + assert.equal(summarize([2]).mad, 0); +}); + +test("bootstrap intervals are deterministic and tail support is explicit", () => { + const values = Array.from({ length: 100 }, (_, index) => index + 1); + const first = summarize(values); + assert.deepEqual(first, summarize(values)); + assert.ok(first.medianInterval.low < first.median); + assert.ok(first.medianInterval.high > first.median); + assert.equal(first.p95, 95.05); + assert.equal(first.p99, null); + assert.match(first.suppressed.p99, /500/); + assert.equal(summarize(Array(500).fill(3)).p99, 3); +}); + +test("run summaries exclude warmups, failed attempts and invalid metrics", () => { + const result = summarizeRun({ + cases: [ + { + id: "rects/load", + status: "complete", + attempts: [ + { warmup: true, status: "ok", metrics: { render: 1000 } }, + { warmup: false, status: "ok", metrics: { render: 10, upload: 0 } }, + { + warmup: false, + status: "invalid", + metrics: { render: 0, upload: 2 }, + invalidMetrics: { render: "below resolution" }, + }, + { warmup: false, status: "failed", metrics: { render: 300 } }, + ], + }, + ], + }); + const summary = result.cases[0]; + assert.deepEqual(summary.attempts, { + measured: 3, + warmup: 1, + ok: 1, + invalid: 1, + failed: 1, + }); + assert.equal(summary.metrics.render.median, 10); + assert.equal(summary.metrics.render.invalid, 2); + assert.equal(summary.metrics.upload.median, 1); + assert.equal(summary.metrics.upload.invalid, 1); + assert.match(result.warnings.join(" "), /conditional/); +}); + +test("statistics reject unsafe bootstrap settings", () => { + assert.throws(() => summarize([1], { resamples: Infinity }), /resamples/); + assert.throws(() => summarize([1], { seed: NaN }), /seed/); +}); + +test("text summaries display the same observations and support limits as JSON", () => { + const summary = summarizeRun({ + cases: [ + { + id: "rects/load", + status: "complete", + attempts: [{ warmup: false, status: "ok", metrics: { render: 2 } }], + }, + ], + }); + const report = formatSummary(summary); + assert.match(report, /median 2\.000/); + assert.match(report, /1 valid, 0 invalid/); + assert.match(report, /Requires at least 10 valid attempts/); + assert.match(report, /2000 resamples/); +}); diff --git a/frontend/scripts/renderer-benchmarks/verify-failures.js b/frontend/scripts/renderer-benchmarks/verify-failures.js new file mode 100644 index 0000000000..b94ad6e804 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/verify-failures.js @@ -0,0 +1,154 @@ +// Unscored browser fault checks. Uses prepared assets; manages no services. +import assert from "node:assert/strict"; +import { chromium } from "playwright"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { bounded } from "../renderer-benchmarks.js"; +import { scenarios } from "../../resources/wasm-playground/scenes/index.js"; +import { collectCases } from "../../resources/wasm-playground/benchmarks/contract.js"; + +const frontend = fileURLToPath(new URL("../../", import.meta.url)); +const server = await chromium.launchServer({ + headless: true, + args: ["--enable-gpu"], +}); +const process = server.process(); +const browser = await chromium.connect(server.wsEndpoint()); +const benchmarkCase = collectCases(scenarios, 42, "rects/default/pan")[0]; +const options = { + viewport: { width: 1920, height: 1080 }, + dpr: 2, + timeoutMs: 30000, + moduleUrl: "http://benchmark.invalid/fault-module.js", +}; +try { + for (const fault of [ + "unknown-frame", + "context-loss", + "timeout", + "cancel-init", + "cancel-frame", + "blocked-wasm", + "page-crash", + ]) { + const context = await browser.newContext({ + viewport: options.viewport, + deviceScaleFactor: options.dpr, + }); + try { + await context.route("http://benchmark.invalid/**", async (route) => { + const pathname = new URL(route.request().url()).pathname; + if (pathname === "/fault-module.js") { + await route.fulfill({ + contentType: "text/javascript", + body: ` + import createModule from '/js/render-wasm-benchmark.js'; + export default async function() { + const module = await createModule(); + if (${JSON.stringify(fault)} === 'cancel-init') await new Promise(resolve => { globalThis.releaseModule = resolve; }); + const render = module._render; + module._render = (...args) => { + if (${JSON.stringify(fault)} === 'unknown-frame') return 0; + if (${JSON.stringify(fault)} === 'blocked-wasm') { while (true) {} } + if (${JSON.stringify(fault)} === 'context-loss') document.querySelector('canvas').getContext('webgl2').getExtension('WEBGL_lose_context').loseContext(); + return render(...args); + }; + return module; + }`, + }); + return; + } + const base = path.join( + frontend, + pathname.startsWith("/wasm-playground/") + ? "resources" + : "resources/public", + ); + const target = path.resolve(base, `.${pathname}`); + if (!target.startsWith(`${base}/`)) return route.abort(); + await route.fulfill({ path: target }); + }); + const page = await context.newPage(); + await page.goto( + "http://benchmark.invalid/wasm-playground/benchmark.html", + ); + await page.waitForFunction(() => globalThis.rendererBenchmarkReady); + if (fault === "timeout") + await page.evaluate(() => { + globalThis.requestAnimationFrame = () => 1; + }); + if (fault === "cancel-frame") + await page.evaluate(() => { + globalThis.requestAnimationFrame = () => { + globalThis.framePending = true; + return 1; + }; + }); + if (fault === "page-crash") { + const crashed = page.waitForEvent("crash"); + const cdp = await context.newCDPSession(page); + void cdp.send("Page.crash").catch(() => {}); + await bounded(crashed, 5000); + await assert.rejects( + page.evaluate(() => 1), + /crash|closed/i, + ); + } else { + // Attach rejection handling immediately; the watchdog may finish before injection. + const outcome = bounded( + page.evaluate( + ([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o), + [benchmarkCase, options], + ), + fault === "blocked-wasm" || fault === "timeout" ? 2500 : 10000, + ).then( + (value) => ({ value }), + (error) => ({ error }), + ); + if (fault === "cancel-init" || fault === "cancel-frame") { + await page.waitForFunction( + (key) => Boolean(globalThis[key]), + fault === "cancel-init" ? "releaseModule" : "framePending", + ); + await page.evaluate(() => { + globalThis.rendererBenchmark.dispose(); + globalThis.releaseModule?.(); + }); + } + const result = await outcome; + if (fault === "blocked-wasm" || fault === "timeout") + assert.match(result.error?.message ?? "", /watchdog/); + else { + assert.equal(result.error, undefined); + assert.equal(result.value.status, "failed"); + assert.match( + result.value.error, + fault === "unknown-frame" + ? /Unexpected frame type/ + : fault === "context-loss" + ? /context lost/ + : /canceled|Stale/, + ); + } + } + console.log(`PASS ${fault} (unscored)`); + } finally { + await bounded(context.close(), 5000); + } + } +} finally { + try { + await bounded(browser.close(), 5000); + } catch { + process.kill("SIGKILL"); + } + try { + await bounded(server.close(), 5000); + } catch { + process.kill("SIGKILL"); + } + assert.ok( + process.exitCode !== null || process.signalCode !== null, + "browser process must exit", + ); +} diff --git a/frontend/scripts/renderer-benchmarks/verify.js b/frontend/scripts/renderer-benchmarks/verify.js new file mode 100644 index 0000000000..0e571c9796 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/verify.js @@ -0,0 +1,96 @@ +// Unscored correctness smoke. Request routing serves local files without a service. +// Optional screenshots belong only to this diagnostic process, never scored runs. +import { chromium } from "playwright"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { scenarios } from "../../resources/wasm-playground/scenes/index.js"; +import { collectCases } from "../../resources/wasm-playground/benchmarks/contract.js"; +import { bounded } from "../renderer-benchmarks.js"; + +const { values } = parseArgs({ + options: { + screenshots: { type: "string" }, + headed: { type: "boolean", default: false }, + filter: { type: "string", default: "" }, + }, +}); +const frontend = fileURLToPath(new URL("../../", import.meta.url)); +const cases = collectCases(scenarios, 42, values.filter); +const server = await chromium.launchServer({ + headless: !values.headed, + args: ["--enable-gpu"], +}); +const browser = await chromium.connect(server.wsEndpoint()); +try { + if (values.screenshots) + await fs.mkdir(values.screenshots, { recursive: true }); + for (const c of cases) { + const context = await browser.newContext({ + viewport: { width: 1920, height: 1080 }, + deviceScaleFactor: 2, + }); + await context.route("http://benchmark.invalid/**", async (route) => { + const pathname = new URL(route.request().url()).pathname; + const base = pathname.startsWith("/wasm-playground/") + ? path.join(frontend, "resources") + : path.join(frontend, "resources/public"); + const target = path.resolve(base, `.${pathname}`); + if (!target.startsWith(`${base}/`)) return route.abort(); + try { + await route.fulfill({ path: target }); + } catch { + await route.fulfill({ status: 404, body: "Missing diagnostic asset" }); + } + }); + const page = await context.newPage(); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.goto("http://benchmark.invalid/wasm-playground/benchmark.html"); + await page.waitForFunction( + () => globalThis.rendererBenchmarkReady === true, + null, + { timeout: 30000 }, + ); + const attempt = await bounded( + page.evaluate( + ([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o), + [ + c, + { + viewport: { width: 1920, height: 1080 }, + dpr: 2, + timeoutMs: 30000, + moduleUrl: "http://benchmark.invalid/js/render-wasm-benchmark.js", + }, + ], + ), + 65000, + ); + if (attempt.status === "failed" || errors.length) + throw new Error(`${c.id}: ${attempt.error ?? errors.join("; ")}`); + if (attempt.slices.at(-1)?.frameType !== 2) + throw new Error(`${c.id}: no Full completion`); + if (values.screenshots && c.group === "fresh-module-context") + await page.screenshot({ + path: path.join(values.screenshots, `${c.scenarioId}.png`), + }); + console.log( + `PASS ${c.id}; ${attempt.environment.webgl.renderer}; ${attempt.slices.length} render calls (unscored)`, + ); + await page.evaluate(() => globalThis.rendererBenchmark.dispose()); + await bounded(context.close(), 5000); + } +} finally { + try { + await bounded(browser.close(), 5000); + } catch { + server.process()?.kill("SIGKILL"); + } + try { + await bounded(server.close(), 5000); + } catch { + server.process()?.kill("SIGKILL"); + } +} diff --git a/render-wasm/build b/render-wasm/build index c4061068f2..1a471758fa 100755 --- a/render-wasm/build +++ b/render-wasm/build @@ -4,6 +4,7 @@ # # With no target, builds both. Set BUILD_MODE=release (or NODE_ENV=production) # for an optimized build. See `_build_env` for what each target changes. +# Set PENPOT_WASM_PREPARED=1 to skip dependency setup in a prepared environment. _SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) _RENDER_TARGET=${1:-} @@ -37,8 +38,12 @@ acquire_build_lock set -x -setup -build -copy_target_artifacts +if [ "${PENPOT_WASM_PREPARED:-0}" != "1" ]; then + setup; +else + command -v cargo >/dev/null || { echo "Prepare the Rust toolchain first" >&2; exit 1; }; + command -v pnpm >/dev/null || { echo "Prepare pnpm and renderer dependencies first" >&2; exit 1; }; + test -d node_modules/esbuild || { echo "Run pnpm install in render-wasm before benchmarking" >&2; exit 1; }; +fi popd > /dev/null