From 0ae78cf7eca5468dd95e15108e3adbcf418a36d5 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Thu, 17 Sep 2026 18:06:08 +0200 Subject: [PATCH] :sparkles: Verify benchmark failures and improve reports Record manifest-default features without adding A/B selection. Keep scored builds free of the optional function-name profiling switch. Exercise CLI persistence, interruption, browser failure and legacy page interactions through separate diagnostics. Share readable reports and warn when offline comparisons include failed runs. Validate with 50 focused tests, browser and CLI fault checks, two full corpus workflow runs, and headed rectangle cases. Shadow graphics and sampling calibration remain open. AI-assisted-by: GPT-6 --- frontend/package.json | 2 + .../resources/wasm-playground/BENCHMARKS.md | 24 +++ .../wasm-playground/benchmarks/browser.js | 19 +- .../wasm-playground/benchmarks/statistics.js | 9 +- frontend/scripts/renderer-benchmarks.js | 42 ++++- .../scripts/renderer-benchmarks/compare.js | 21 +++ .../renderer-benchmarks/tests/compare.test.js | 11 ++ .../renderer-benchmarks/tests/report.test.js | 39 +++++ .../renderer-benchmarks/tests/runner.test.js | 13 ++ .../tests/statistics.test.js | 27 +++ .../scripts/renderer-benchmarks/verify-cli.js | 163 ++++++++++++++++++ .../renderer-benchmarks/verify-failures.js | 32 +++- .../renderer-benchmarks/verify-pages.js | 163 ++++++++++++++++++ 13 files changed, 550 insertions(+), 15 deletions(-) create mode 100644 frontend/scripts/renderer-benchmarks/tests/report.test.js create mode 100644 frontend/scripts/renderer-benchmarks/verify-cli.js create mode 100644 frontend/scripts/renderer-benchmarks/verify-pages.js diff --git a/frontend/package.json b/frontend/package.json index 37cf86285a..c514e59a03 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,8 @@ "test:renderer-benchmarks": "node --test scripts/renderer-benchmarks/tests/*.test.js", "verify:renderer-benchmarks": "node scripts/renderer-benchmarks/verify.js", "verify:renderer-benchmark-failures": "node scripts/renderer-benchmarks/verify-failures.js", + "verify:renderer-benchmark-cli": "node scripts/renderer-benchmarks/verify-cli.js", + "verify:renderer-playground": "node scripts/renderer-benchmarks/verify-pages.js", "build:app:assets": "node ./scripts/build-app-assets.js", "build:fonts-preview": "node ./scripts/build-fonts-preview.js", "build:storybook": "(cd packages/ui && pnpm run build) && pnpm run build:storybook:assets && pnpm run build:storybook:cljs && storybook build", diff --git a/frontend/resources/wasm-playground/BENCHMARKS.md b/frontend/resources/wasm-playground/BENCHMARKS.md index 3df5a6901d..d209c3ede4 100644 --- a/frontend/resources/wasm-playground/BENCHMARKS.md +++ b/frontend/resources/wasm-playground/BENCHMARKS.md @@ -98,6 +98,9 @@ state, not diffs or artifact hashes. ```sh pnpm run verify:renderer-benchmarks pnpm run verify:renderer-benchmarks --screenshots /tmp/renderer-diagnostics +pnpm run verify:renderer-benchmark-failures +pnpm run verify:renderer-benchmark-cli +pnpm run verify:renderer-playground --filter rects ``` This unscored smoke uses the already-built benchmark artifact and Playwright @@ -105,6 +108,27 @@ request routing; it starts no server. It checks every workload and interaction through `Full`. Optional screenshots support independent manual fixture checks. They never enter scored runs, and smoke durations are not benchmark evidence. +The failure checks inject context loss, cancellation, stalled work, and crashes. +The CLI checks use the existing server and prepared build path, saving each +failure's JSON and log under a printed temporary directory. The playground check +loads the actual legacy pages and exercises pointer drag and wheel zoom. Run +these commands separately from scored runs. + +The runner uses the manifest's default Cargo features and records their local +feature expansion. It rejects diagnostic defaults and disables the optional +function-name profiling switch for scored builds. Feature selection and A/B +orchestration are separate follow-up work. + +## Current validation limits + +The September 17 software-rendered pilot took 25 minutes 43 seconds excluding +the build at 3 warm-ups and 10 measured attempts. Eighteen cases completed all +attempts; all three shadow cases failed. A later one-attempt, zero-warm-up run +took 2 minutes 36 seconds; shadow pan and zoom still lost their WebGL contexts. +These runs used dirty source trees and are workflow evidence, not a comparison +of renderer changes. Shadow visual validation and default sampling remain open. +Keep the full workloads and provisional defaults until these limits are resolved. + The implementation roadmap, pilot evidence, and remaining calibration work live in Serena memory `render-wasm/performance/core` and its numbered tickets. The 2-5 minute full-suite goal excludes build time and is a target, not a guarantee diff --git a/frontend/resources/wasm-playground/benchmarks/browser.js b/frontend/resources/wasm-playground/benchmarks/browser.js index 1b53953163..6f5634948d 100644 --- a/frontend/resources/wasm-playground/benchmarks/browser.js +++ b/frontend/resources/wasm-playground/benchmarks/browser.js @@ -1,7 +1,7 @@ import { scenarios } from "../scenes/index.js"; import { drain, interact, restore } from "./protocol.js"; import { WEBGL_OPTIONS, validateMetrics } from "./contract.js"; -import { summarizeRun } from "./statistics.js"; +import { summarizeRun, formatSummary } from "./statistics.js"; let session; const canvas = document.querySelector("canvas"); @@ -233,16 +233,13 @@ globalThis.rendererBenchmark = { showResult(result) { globalThis.rendererBenchmarkResult = result; const report = document.querySelector("pre"); - report.textContent = JSON.stringify( - { - scope: result.scope, - metadata: result.metadata, - warnings: result.warnings, - summary: result.summary ?? summarizeRun(result), - }, - null, - 2, - ); + report.textContent = [ + result.scope, + ...(result.warnings ?? []), + formatSummary(result.summary ?? summarizeRun(result)), + "Run settings:", + JSON.stringify(result.metadata, null, 2), + ].join("\n\n"); report.hidden = false; }, }; diff --git a/frontend/resources/wasm-playground/benchmarks/statistics.js b/frontend/resources/wasm-playground/benchmarks/statistics.js index 62e3b4319c..286b775ca4 100644 --- a/frontend/resources/wasm-playground/benchmarks/statistics.js +++ b/frontend/resources/wasm-playground/benchmarks/statistics.js @@ -166,6 +166,7 @@ export function summarizeRun(result, options = {}) { return { id: benchmarkCase.id, status: benchmarkCase.status, + unattempted: benchmarkCase.unattempted ?? 0, attempts: { measured: measured.length, warmup: benchmarkCase.attempts.length - measured.length, @@ -253,15 +254,19 @@ export function formatSummary(summary) { const lines = [ `Statistics: ${summary.method.name} v${summary.method.version}; ${summary.method.resamples} resamples, seed ${summary.method.seed}.`, ...summary.warnings, + `Median interval: Requires at least ${summary.method.minimumIntervalSamples} valid attempts; p95 requires ${summary.method.minimumP95Samples}, p99 requires ${summary.method.minimumP99Samples}. Unsupported values are n/a.`, ]; for (const benchmarkCase of summary.cases) { - lines.push(`${benchmarkCase.id} (${benchmarkCase.status})`); + const counts = benchmarkCase.attempts; + lines.push( + `${benchmarkCase.id} (${benchmarkCase.status}): ${counts.warmup} warm-up, ${counts.ok} valid, ${counts.invalid} invalid, ${counts.failed} failed, ${benchmarkCase.unattempted} unattempted`, + ); for (const [name, metric] of Object.entries(benchmarkCase.metrics)) { lines.push( ` ${name}: median ${display(metric.median)}, mean ${display(metric.mean)}, range ${display(metric.min)}–${display(metric.max)}, SD ${display(metric.standardDeviation)}, MAD ${display(metric.mad)}; ${metric.n} valid, ${metric.invalid} invalid`, ); lines.push( - ` median interval: ${metric.medianInterval ? `[${display(metric.medianInterval.low)}, ${display(metric.medianInterval.high)}]` : metric.suppressed.medianInterval}; p95: ${metric.p95 === null ? metric.suppressed.p95 : display(metric.p95)}; p99: ${metric.p99 === null ? metric.suppressed.p99 : display(metric.p99)}`, + ` median interval: ${metric.medianInterval ? `[${display(metric.medianInterval.low)}, ${display(metric.medianInterval.high)}]` : "n/a"}; p95: ${display(metric.p95)}; p99: ${display(metric.p99)}`, ); } } diff --git a/frontend/scripts/renderer-benchmarks.js b/frontend/scripts/renderer-benchmarks.js index 6115705006..8afce7b4df 100644 --- a/frontend/scripts/renderer-benchmarks.js +++ b/frontend/scripts/renderer-benchmarks.js @@ -135,6 +135,23 @@ async function requireFile(filename, instruction) { } } +// Cargo metadata --no-deps reads the manifest without requiring unused targets' +// dependencies to be cached. Only local feature aliases need expansion here. +export function defaultFeatures(definitions) { + const enabled = new Set(); + const visit = (name) => { + if (enabled.has(name)) return; + enabled.add(name); + for (const child of definitions[name] ?? []) visit(child); + }; + visit("default"); + if ([...enabled].some((name) => /^(stats|profile)(-|$)/.test(name))) + throw new Error( + "Default Cargo features enable renderer diagnostics; use a plain release manifest", + ); + return [...enabled].sort(); +} + async function preflight(options) { await requireFile( "/opt/emsdk/emsdk_env.sh", @@ -220,7 +237,8 @@ export async function run(options) { }, build: { mode: "release", - features: [], + featurePolicy: "manifest-defaults", + features: null, target: "frontend", prepared: true, }, @@ -304,6 +322,27 @@ export async function run(options) { ), ); result.metadata.environment.playwright = packageJson.version; + const manifest = JSON.parse( + execFileSync( + "cargo", + [ + "metadata", + "--offline", + "--locked", + "--no-deps", + "--format-version", + "1", + ], + { cwd: path.join(root, "render-wasm"), encoding: "utf8" }, + ), + ); + const renderer = manifest.packages.find( + (item) => + item.manifest_path === path.join(root, "render-wasm/Cargo.toml"), + ); + if (!renderer) + throw new Error("Renderer package missing from Cargo metadata"); + result.metadata.build.features = defaultFeatures(renderer.features); // Disallow package-manager network fallback in a prepared run. const buildStart = performance.now(); await execute("./build", ["frontend", "--offline", "--locked"], { @@ -312,6 +351,7 @@ export async function run(options) { env: { ...process.env, PENPOT_WASM_PREPARED: "1", + PENPOT_WASM_FUNCTION_NAMES: "0", BUILD_MODE: "release", NODE_ENV: "production", RENDER_TARGET: "frontend", diff --git a/frontend/scripts/renderer-benchmarks/compare.js b/frontend/scripts/renderer-benchmarks/compare.js index db1bc47736..53dbb98a08 100644 --- a/frontend/scripts/renderer-benchmarks/compare.js +++ b/frontend/scripts/renderer-benchmarks/compare.js @@ -210,13 +210,34 @@ export function compareRuns( } if (mismatches.length && !diagnostic) throw new CompatibilityError(mismatches); + const runs = Object.fromEntries( + [ + ["baseline", baseline], + ["candidate", candidate], + ].map(([name, run]) => [ + name, + { + runId: run.runId ?? null, + status: run.status ?? "unknown", + timestamp: run.timestamp ?? null, + git: run.metadata.git ?? null, + }, + ]), + ); return { schemaVersion: 1, + runs, compatible: mismatches.length === 0, diagnostic, mismatches, method: statisticalMethod(options), warnings: [ + ...Object.entries(runs) + .filter(([, run]) => run.status !== "complete") + .map( + ([name, run]) => + `The ${name} run is ${run.status}; this comparison covers only available valid observations.`, + ), ...(mismatches.length ? [ "DIAGNOSTIC ONLY: these runs are incompatible; differences cannot establish a performance change.", diff --git a/frontend/scripts/renderer-benchmarks/tests/compare.test.js b/frontend/scripts/renderer-benchmarks/tests/compare.test.js index ffda4cf585..fe6784dcf5 100644 --- a/frontend/scripts/renderer-benchmarks/tests/compare.test.js +++ b/frontend/scripts/renderer-benchmarks/tests/compare.test.js @@ -5,6 +5,7 @@ import { compareRuns, formatComparison } from "../compare.js"; function run(value = 10) { return { schemaVersion: 1, + status: "complete", metadata: { git: { sha: "before", dirty: false }, build: { mode: "release", features: [] }, @@ -146,6 +147,16 @@ test("failed attempts never enter comparison observations", () => { assert.equal(metric.absoluteInterval, null); }); +test("comparisons warn when a run failed even if available metrics are compatible", () => { + const candidate = run(12); + candidate.status = "failed"; + candidate.cases[0].status = "failed"; + const result = compareRuns(run(), candidate); + assert.equal(result.compatible, true); + assert.equal(result.runs.candidate.status, "failed"); + assert.match(formatComparison(result), /candidate run is failed/i); +}); + test("offline comparison rejects malformed results without diagnostic bypass", () => { const missingEnvironment = run(); delete missingEnvironment.metadata.environment; diff --git a/frontend/scripts/renderer-benchmarks/tests/report.test.js b/frontend/scripts/renderer-benchmarks/tests/report.test.js new file mode 100644 index 0000000000..7b4874718a --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/tests/report.test.js @@ -0,0 +1,39 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +test("page reports keep raw results and show the shared readable summary", async () => { + const previous = globalThis.document; + const report = { hidden: true, textContent: "" }; + globalThis.document = { + querySelector: (selector) => (selector === "pre" ? report : {}), + }; + try { + await import("../../../resources/wasm-playground/benchmarks/browser.js"); + const result = { + scope: "Renderer submission", + metadata: {}, + warnings: [], + cases: [ + { + id: "rects/load", + status: "complete", + unattempted: 0, + attempts: [ + { warmup: false, status: "ok", metrics: { timeToFullMs: 2 } }, + ], + }, + ], + }; + globalThis.rendererBenchmark.showResult(result); + assert.equal(globalThis.rendererBenchmarkResult, result); + assert.equal(report.hidden, false); + assert.match(report.textContent, /median 2\.000/); + assert.match(report.textContent, /1 valid, 0 invalid/); + assert.match(report.textContent, /Renderer submission/); + } finally { + globalThis.document = previous; + delete globalThis.rendererBenchmark; + delete globalThis.rendererBenchmarkReady; + delete globalThis.rendererBenchmarkResult; + } +}); diff --git a/frontend/scripts/renderer-benchmarks/tests/runner.test.js b/frontend/scripts/renderer-benchmarks/tests/runner.test.js index 305d639256..7e35e5ecf4 100644 --- a/frontend/scripts/renderer-benchmarks/tests/runner.test.js +++ b/frontend/scripts/renderer-benchmarks/tests/runner.test.js @@ -6,7 +6,20 @@ import { main, recordAttempt, finishCases, + defaultFeatures, } from "../../renderer-benchmarks.js"; + +test("build provenance expands the renderer's declared default features", () => { + assert.deepEqual(defaultFeatures({ default: [] }), ["default"]); + assert.deepEqual( + defaultFeatures({ default: ["fast"], fast: ["shared"], shared: ["fast"] }), + ["default", "fast", "shared"], + ); + assert.throws( + () => defaultFeatures({ default: ["profile"], profile: [] }), + /diagnostic/, + ); +}); import { collectCases, deriveSeed, diff --git a/frontend/scripts/renderer-benchmarks/tests/statistics.test.js b/frontend/scripts/renderer-benchmarks/tests/statistics.test.js index 5f53a30cec..5821c595b6 100644 --- a/frontend/scripts/renderer-benchmarks/tests/statistics.test.js +++ b/frontend/scripts/renderer-benchmarks/tests/statistics.test.js @@ -100,3 +100,30 @@ test("text summaries display the same observations and support limits as JSON", assert.match(report, /Requires at least 10 valid attempts/); assert.match(report, /2000 resamples/); }); + +test("reports expose missing attempts and explain support limits once", () => { + const summary = summarizeRun({ + cases: [ + { + id: "rects/load", + status: "failed", + unattempted: 9, + attempts: [ + { + warmup: false, + status: "failed", + metrics: { render: 3, upload: 2 }, + }, + ], + }, + ], + }); + assert.equal(summary.cases[0].unattempted, 9); + const report = formatSummary(summary); + assert.match(report, /9 unattempted/); + assert.match(report, /1 failed/); + assert.equal( + report.split("Requires at least 10 valid attempts").length - 1, + 1, + ); +}); diff --git a/frontend/scripts/renderer-benchmarks/verify-cli.js b/frontend/scripts/renderer-benchmarks/verify-cli.js new file mode 100644 index 0000000000..4b3d1ba9f9 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/verify-cli.js @@ -0,0 +1,163 @@ +// Unscored end-to-end failure checks. Each child uses the real prepared build, +// server, browser, persistence and cleanup paths. No service is started here. +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { bounded, main } from "../renderer-benchmarks.js"; + +const filename = fileURLToPath(import.meta.url); + +if (process.argv[2] === "--child") { + const [, , , fault, output] = process.argv; + const { chromium } = await import("playwright"); + const connect = chromium.connect.bind(chromium); + let checkpoints = 0; + chromium.connect = async (...args) => { + const browser = await connect(...args); + const newContext = browser.newContext.bind(browser); + browser.newContext = async (...contextArgs) => { + const context = await newContext(...contextArgs); + await context.exposeBinding( + "benchmarkFaultCheckpoint", + async ({ page }) => { + if (++checkpoints !== 2) return; + if (fault === "interrupt") process.kill(process.pid, "SIGTERM"); + if (fault === "browser-disconnect") + void browser.close().catch(() => {}); + if (fault === "page-crash") { + const cdp = await context.newCDPSession(page); + void cdp.send("Page.crash").catch(() => {}); + } + }, + ); + await context.addInitScript(() => { + let api; + Object.defineProperty(globalThis, "rendererBenchmark", { + get: () => api, + set(value) { + api = value; + const attempt = value.runAttempt; + value.runAttempt = async (...args) => { + await globalThis.benchmarkFaultCheckpoint(); + return attempt(...args); + }; + }, + }); + }); + return context; + }; + return browser; + }; + const args = [ + "--filter", + "rects", + "--warmups", + "0", + "--repetitions", + "2", + "--output", + output, + ]; + if (fault === "missing-server") + args.push("--base-url", "http://127.0.0.1:1/"); + if (fault === "wrong-server") + args.push("--base-url", "http://localhost:3000/not-this-workspace/"); + process.exitCode = await main(args); +} else { + const directory = await fs.mkdtemp( + path.join(os.tmpdir(), "renderer-cli-faults-"), + ); + console.log(`Diagnostic results: ${directory}`); + for (const fault of [ + "missing-browser", + "missing-server", + "wrong-server", + "page-crash", + "browser-disconnect", + "interrupt", + ]) { + const output = path.join(directory, `${fault}.json`); + const log = await fs.open(path.join(directory, `${fault}.log`), "w"); + const child = spawn( + process.execPath, + [filename, "--child", fault, output], + { + detached: true, + stdio: ["ignore", log.fd, log.fd], + env: { + ...process.env, + ...(fault === "missing-browser" + ? { + PLAYWRIGHT_BROWSERS_PATH: path.join( + directory, + "missing-browser", + ), + } + : {}), + }, + }, + ); + try { + const exit = await bounded( + new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolve({ code, signal })); + }), + 180000, + ); + assert.deepEqual( + exit, + { code: 1, signal: null }, + `${fault}: must exit with an execution failure`, + ); + const result = JSON.parse(await fs.readFile(output, "utf8")); + assert.ok(result.finishedAt); + assert.equal( + result.status, + fault === "interrupt" ? "interrupted" : "failed", + ); + assert.ok( + result.cases.every((c) => c.attempts.length + c.unattempted === 2), + ); + if ( + ["missing-browser", "missing-server", "wrong-server"].includes(fault) + ) { + assert.ok(result.cases.every((c) => c.status === "unattempted")); + assert.match( + result.error, + fault === "missing-browser" + ? /Chromium is missing/ + : fault === "missing-server" + ? /Static server unavailable/ + : /current benchmark assets/, + ); + } else { + assert.equal( + result.cases[0].attempts[0].status, + "ok", + "retain the successful attempt before injection", + ); + assert.equal(result.cases[0].attempts[1].status, "failed"); + if (fault === "page-crash") { + assert.ok( + result.cases.slice(1).every((c) => c.status === "complete"), + "independent cases recover in fresh contexts", + ); + } else { + assert.ok( + result.cases.slice(1).every((c) => c.status === "unattempted"), + ); + } + } + console.log(`PASS ${fault}: partial results and bounded exit (unscored)`); + } finally { + // Kill only this diagnostic child's process group if its watchdog expired. + if (child.exitCode === null && child.signalCode === null) + process.kill(-child.pid, "SIGKILL"); + await log.close(); + } + } +} diff --git a/frontend/scripts/renderer-benchmarks/verify-failures.js b/frontend/scripts/renderer-benchmarks/verify-failures.js index b94ad6e804..1ee0169bc1 100644 --- a/frontend/scripts/renderer-benchmarks/verify-failures.js +++ b/frontend/scripts/renderer-benchmarks/verify-failures.js @@ -30,6 +30,7 @@ try { "cancel-frame", "blocked-wasm", "page-crash", + "report", ]) { const context = await browser.newContext({ viewport: options.viewport, @@ -84,7 +85,36 @@ try { return 1; }; }); - if (fault === "page-crash") { + if (fault === "report") { + const report = await page.evaluate((c) => { + const value = { + scope: "Diagnostic report", + metadata: {}, + warnings: [], + cases: [ + { + ...c, + status: "complete", + unattempted: 0, + attempts: [ + { warmup: false, status: "ok", metrics: { timeToFullMs: 2 } }, + ], + }, + ], + }; + globalThis.rendererBenchmark.showResult(value); + return { + result: globalThis.rendererBenchmarkResult, + text: document.querySelector("pre").textContent, + }; + }, benchmarkCase); + assert.equal( + report.result.cases[0].attempts[0].metrics.timeToFullMs, + 2, + ); + assert.match(report.text, /median 2\.000/); + assert.match(report.text, /1 valid, 0 invalid/); + } else if (fault === "page-crash") { const crashed = page.waitForEvent("crash"); const cdp = await context.newCDPSession(page); void cdp.send("Page.crash").catch(() => {}); diff --git a/frontend/scripts/renderer-benchmarks/verify-pages.js b/frontend/scripts/renderer-benchmarks/verify-pages.js new file mode 100644 index 0000000000..e2f23f0471 --- /dev/null +++ b/frontend/scripts/renderer-benchmarks/verify-pages.js @@ -0,0 +1,163 @@ +// Unscored checks of the actual playground pages, including real pointer/wheel +// events. Expose the mounted adapter only in this diagnostic routing layer. +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { chromium } from "playwright"; +import { bounded } from "../renderer-benchmarks.js"; +import { scenarios } from "../../resources/wasm-playground/scenes/index.js"; + +const { values } = parseArgs({ + options: { + filter: { type: "string", default: "" }, + headed: { type: "boolean", default: false }, + }, +}); +const frontend = fileURLToPath(new URL("../../", import.meta.url)); +const selected = scenarios.filter((s) => s.id.includes(values.filter)); +assert.ok(selected.length, "No matching scenes"); +const server = await chromium.launchServer({ + headless: !values.headed, + args: ["--enable-gpu"], +}); +const browser = await chromium.connect(server.wsEndpoint()); +try { + for (const scenario of selected) { + const context = await browser.newContext({ + viewport: { width: 1920, height: 1080 }, + deviceScaleFactor: 2, + }); + try { + await bounded( + (async () => { + await context.route("http://benchmark.invalid/**", async (route) => { + const pathname = new URL(route.request().url()).pathname; + const base = path.join( + frontend, + pathname.startsWith("/wasm-playground/") + ? "resources" + : "resources/public", + ); + const target = path.resolve(base, `.${pathname}`); + if (!target.startsWith(`${base}/`)) return route.abort(); + if (pathname === "/wasm-playground/js/lib.js") { + const source = await fs.readFile(target, "utf8"); + await route.fulfill({ + contentType: "text/javascript", + body: + source.replace( + "export async function mountScenario(", + "async function mountOriginalScenario(", + ) + + ` + export async function mountScenario(...args) { + const adapter = await mountOriginalScenario(...args); + const render = adapter.module._render; + const view = adapter.module._set_view; + globalThis.playgroundProbe = { full: 0, views: [] }; + adapter.module._render = (...args) => { + const result = render(...args); + if (result === 2) globalThis.playgroundProbe.full++; + return result; + }; + adapter.module._set_view = (...args) => { + globalThis.playgroundProbe.views.push(args); + return view(...args); + }; + globalThis.playgroundAdapter = adapter; + return adapter; + }`, + }); + } else { + try { + await route.fulfill({ path: target }); + } catch { + await route.fulfill({ + status: 404, + body: "Missing diagnostic asset", + }); + } + } + }); + const page = await context.newPage(); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + page.on("console", (message) => { + if (message.type() === "error") errors.push(message.text()); + }); + await page.goto( + `http://benchmark.invalid/wasm-playground/${scenario.id}.html?seed=42`, + ); + await page.waitForFunction( + () => + globalThis.playgroundAdapter || document.querySelector("#error"), + null, + { timeout: 30000 }, + ); + assert.equal( + await page.locator("#error").count(), + 0, + "page must mount successfully", + ); + await page.mouse.move(800, 400); + await page.mouse.down(); + await page.mouse.move(860, 430, { steps: 3 }); + await page.mouse.up(); + await page.waitForFunction( + () => globalThis.playgroundProbe.full >= 1, + ); + await page.evaluate(() => { + globalThis.playgroundProbe.full = 0; + }); + await page.mouse.wheel(0, -100); + await page.waitForFunction( + () => globalThis.playgroundProbe.full >= 1, + ); + const state = await page.evaluate(() => ({ + views: globalThis.playgroundProbe.views, + lost: document + .querySelector("canvas") + .getContext("webgl2") + .isContextLost(), + })); + assert.equal(state.lost, false); + assert.ok( + state.views.some(([scale, x]) => scale === 1 && x > 0), + "pointer drag must pan", + ); + assert.ok( + state.views.some(([scale]) => scale > 1), + "wheel must zoom", + ); + assert.deepEqual(errors, []); + await bounded( + page.evaluate(() => globalThis.playgroundAdapter.dispose()), + 5000, + ); + console.log( + `PASS ${scenario.id}: legacy mount, pan, zoom and disposal (unscored)`, + ); + })(), + 65000, + ); + } catch (error) { + process.exitCode = 1; + console.error(`FAIL ${scenario.id}: ${error.stack ?? error}`); + } finally { + await bounded(context.close(), 5000); + } + } +} finally { + try { + await bounded(browser.close(), 5000); + } catch { + server.process()?.kill("SIGKILL"); + } + try { + await bounded(server.close(), 5000); + } catch { + server.process()?.kill("SIGKILL"); + } +}