mirror of
https://github.com/penpot/penpot.git
synced 2026-10-29 05:50:35 +00:00
✨ Verify benchmark failures and improve reports
Record manifest-default features without adding A/B selection. Keep scored builds free of the optional function-name profiling switch. Exercise CLI persistence, interruption, browser failure and legacy page interactions through separate diagnostics. Share readable reports and warn when offline comparisons include failed runs. Validate with 50 focused tests, browser and CLI fault checks, two full corpus workflow runs, and headed rectangle cases. Shadow graphics and sampling calibration remain open. AI-assisted-by: GPT-6
This commit is contained in:
Miguel de Benito Delgado
parent
731bcc5664
commit
0ae78cf7ec
@ -18,6 +18,8 @@
|
||||
"test:renderer-benchmarks": "node --test scripts/renderer-benchmarks/tests/*.test.js",
|
||||
"verify:renderer-benchmarks": "node scripts/renderer-benchmarks/verify.js",
|
||||
"verify:renderer-benchmark-failures": "node scripts/renderer-benchmarks/verify-failures.js",
|
||||
"verify:renderer-benchmark-cli": "node scripts/renderer-benchmarks/verify-cli.js",
|
||||
"verify:renderer-playground": "node scripts/renderer-benchmarks/verify-pages.js",
|
||||
"build:app:assets": "node ./scripts/build-app-assets.js",
|
||||
"build:fonts-preview": "node ./scripts/build-fonts-preview.js",
|
||||
"build:storybook": "(cd packages/ui && pnpm run build) && pnpm run build:storybook:assets && pnpm run build:storybook:cljs && storybook build",
|
||||
|
||||
@ -98,6 +98,9 @@ state, not diffs or artifact hashes.
|
||||
```sh
|
||||
pnpm run verify:renderer-benchmarks
|
||||
pnpm run verify:renderer-benchmarks --screenshots /tmp/renderer-diagnostics
|
||||
pnpm run verify:renderer-benchmark-failures
|
||||
pnpm run verify:renderer-benchmark-cli
|
||||
pnpm run verify:renderer-playground --filter rects
|
||||
```
|
||||
|
||||
This unscored smoke uses the already-built benchmark artifact and Playwright
|
||||
@ -105,6 +108,27 @@ request routing; it starts no server. It checks every workload and interaction
|
||||
through `Full`. Optional screenshots support independent manual fixture checks.
|
||||
They never enter scored runs, and smoke durations are not benchmark evidence.
|
||||
|
||||
The failure checks inject context loss, cancellation, stalled work, and crashes.
|
||||
The CLI checks use the existing server and prepared build path, saving each
|
||||
failure's JSON and log under a printed temporary directory. The playground check
|
||||
loads the actual legacy pages and exercises pointer drag and wheel zoom. Run
|
||||
these commands separately from scored runs.
|
||||
|
||||
The runner uses the manifest's default Cargo features and records their local
|
||||
feature expansion. It rejects diagnostic defaults and disables the optional
|
||||
function-name profiling switch for scored builds. Feature selection and A/B
|
||||
orchestration are separate follow-up work.
|
||||
|
||||
## Current validation limits
|
||||
|
||||
The September 17 software-rendered pilot took 25 minutes 43 seconds excluding
|
||||
the build at 3 warm-ups and 10 measured attempts. Eighteen cases completed all
|
||||
attempts; all three shadow cases failed. A later one-attempt, zero-warm-up run
|
||||
took 2 minutes 36 seconds; shadow pan and zoom still lost their WebGL contexts.
|
||||
These runs used dirty source trees and are workflow evidence, not a comparison
|
||||
of renderer changes. Shadow visual validation and default sampling remain open.
|
||||
Keep the full workloads and provisional defaults until these limits are resolved.
|
||||
|
||||
The implementation roadmap, pilot evidence, and remaining calibration work live
|
||||
in Serena memory `render-wasm/performance/core` and its numbered tickets. The
|
||||
2-5 minute full-suite goal excludes build time and is a target, not a guarantee
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { scenarios } from "../scenes/index.js";
|
||||
import { drain, interact, restore } from "./protocol.js";
|
||||
import { WEBGL_OPTIONS, validateMetrics } from "./contract.js";
|
||||
import { summarizeRun } from "./statistics.js";
|
||||
import { summarizeRun, formatSummary } from "./statistics.js";
|
||||
|
||||
let session;
|
||||
const canvas = document.querySelector("canvas");
|
||||
@ -233,16 +233,13 @@ globalThis.rendererBenchmark = {
|
||||
showResult(result) {
|
||||
globalThis.rendererBenchmarkResult = result;
|
||||
const report = document.querySelector("pre");
|
||||
report.textContent = JSON.stringify(
|
||||
{
|
||||
scope: result.scope,
|
||||
metadata: result.metadata,
|
||||
warnings: result.warnings,
|
||||
summary: result.summary ?? summarizeRun(result),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
report.textContent = [
|
||||
result.scope,
|
||||
...(result.warnings ?? []),
|
||||
formatSummary(result.summary ?? summarizeRun(result)),
|
||||
"Run settings:",
|
||||
JSON.stringify(result.metadata, null, 2),
|
||||
].join("\n\n");
|
||||
report.hidden = false;
|
||||
},
|
||||
};
|
||||
|
||||
@ -166,6 +166,7 @@ export function summarizeRun(result, options = {}) {
|
||||
return {
|
||||
id: benchmarkCase.id,
|
||||
status: benchmarkCase.status,
|
||||
unattempted: benchmarkCase.unattempted ?? 0,
|
||||
attempts: {
|
||||
measured: measured.length,
|
||||
warmup: benchmarkCase.attempts.length - measured.length,
|
||||
@ -253,15 +254,19 @@ export function formatSummary(summary) {
|
||||
const lines = [
|
||||
`Statistics: ${summary.method.name} v${summary.method.version}; ${summary.method.resamples} resamples, seed ${summary.method.seed}.`,
|
||||
...summary.warnings,
|
||||
`Median interval: Requires at least ${summary.method.minimumIntervalSamples} valid attempts; p95 requires ${summary.method.minimumP95Samples}, p99 requires ${summary.method.minimumP99Samples}. Unsupported values are n/a.`,
|
||||
];
|
||||
for (const benchmarkCase of summary.cases) {
|
||||
lines.push(`${benchmarkCase.id} (${benchmarkCase.status})`);
|
||||
const counts = benchmarkCase.attempts;
|
||||
lines.push(
|
||||
`${benchmarkCase.id} (${benchmarkCase.status}): ${counts.warmup} warm-up, ${counts.ok} valid, ${counts.invalid} invalid, ${counts.failed} failed, ${benchmarkCase.unattempted} unattempted`,
|
||||
);
|
||||
for (const [name, metric] of Object.entries(benchmarkCase.metrics)) {
|
||||
lines.push(
|
||||
` ${name}: median ${display(metric.median)}, mean ${display(metric.mean)}, range ${display(metric.min)}–${display(metric.max)}, SD ${display(metric.standardDeviation)}, MAD ${display(metric.mad)}; ${metric.n} valid, ${metric.invalid} invalid`,
|
||||
);
|
||||
lines.push(
|
||||
` median interval: ${metric.medianInterval ? `[${display(metric.medianInterval.low)}, ${display(metric.medianInterval.high)}]` : metric.suppressed.medianInterval}; p95: ${metric.p95 === null ? metric.suppressed.p95 : display(metric.p95)}; p99: ${metric.p99 === null ? metric.suppressed.p99 : display(metric.p99)}`,
|
||||
` median interval: ${metric.medianInterval ? `[${display(metric.medianInterval.low)}, ${display(metric.medianInterval.high)}]` : "n/a"}; p95: ${display(metric.p95)}; p99: ${display(metric.p99)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -135,6 +135,23 @@ async function requireFile(filename, instruction) {
|
||||
}
|
||||
}
|
||||
|
||||
// Cargo metadata --no-deps reads the manifest without requiring unused targets'
|
||||
// dependencies to be cached. Only local feature aliases need expansion here.
|
||||
export function defaultFeatures(definitions) {
|
||||
const enabled = new Set();
|
||||
const visit = (name) => {
|
||||
if (enabled.has(name)) return;
|
||||
enabled.add(name);
|
||||
for (const child of definitions[name] ?? []) visit(child);
|
||||
};
|
||||
visit("default");
|
||||
if ([...enabled].some((name) => /^(stats|profile)(-|$)/.test(name)))
|
||||
throw new Error(
|
||||
"Default Cargo features enable renderer diagnostics; use a plain release manifest",
|
||||
);
|
||||
return [...enabled].sort();
|
||||
}
|
||||
|
||||
async function preflight(options) {
|
||||
await requireFile(
|
||||
"/opt/emsdk/emsdk_env.sh",
|
||||
@ -220,7 +237,8 @@ export async function run(options) {
|
||||
},
|
||||
build: {
|
||||
mode: "release",
|
||||
features: [],
|
||||
featurePolicy: "manifest-defaults",
|
||||
features: null,
|
||||
target: "frontend",
|
||||
prepared: true,
|
||||
},
|
||||
@ -304,6 +322,27 @@ export async function run(options) {
|
||||
),
|
||||
);
|
||||
result.metadata.environment.playwright = packageJson.version;
|
||||
const manifest = JSON.parse(
|
||||
execFileSync(
|
||||
"cargo",
|
||||
[
|
||||
"metadata",
|
||||
"--offline",
|
||||
"--locked",
|
||||
"--no-deps",
|
||||
"--format-version",
|
||||
"1",
|
||||
],
|
||||
{ cwd: path.join(root, "render-wasm"), encoding: "utf8" },
|
||||
),
|
||||
);
|
||||
const renderer = manifest.packages.find(
|
||||
(item) =>
|
||||
item.manifest_path === path.join(root, "render-wasm/Cargo.toml"),
|
||||
);
|
||||
if (!renderer)
|
||||
throw new Error("Renderer package missing from Cargo metadata");
|
||||
result.metadata.build.features = defaultFeatures(renderer.features);
|
||||
// Disallow package-manager network fallback in a prepared run.
|
||||
const buildStart = performance.now();
|
||||
await execute("./build", ["frontend", "--offline", "--locked"], {
|
||||
@ -312,6 +351,7 @@ export async function run(options) {
|
||||
env: {
|
||||
...process.env,
|
||||
PENPOT_WASM_PREPARED: "1",
|
||||
PENPOT_WASM_FUNCTION_NAMES: "0",
|
||||
BUILD_MODE: "release",
|
||||
NODE_ENV: "production",
|
||||
RENDER_TARGET: "frontend",
|
||||
|
||||
@ -210,13 +210,34 @@ export function compareRuns(
|
||||
}
|
||||
if (mismatches.length && !diagnostic)
|
||||
throw new CompatibilityError(mismatches);
|
||||
const runs = Object.fromEntries(
|
||||
[
|
||||
["baseline", baseline],
|
||||
["candidate", candidate],
|
||||
].map(([name, run]) => [
|
||||
name,
|
||||
{
|
||||
runId: run.runId ?? null,
|
||||
status: run.status ?? "unknown",
|
||||
timestamp: run.timestamp ?? null,
|
||||
git: run.metadata.git ?? null,
|
||||
},
|
||||
]),
|
||||
);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
runs,
|
||||
compatible: mismatches.length === 0,
|
||||
diagnostic,
|
||||
mismatches,
|
||||
method: statisticalMethod(options),
|
||||
warnings: [
|
||||
...Object.entries(runs)
|
||||
.filter(([, run]) => run.status !== "complete")
|
||||
.map(
|
||||
([name, run]) =>
|
||||
`The ${name} run is ${run.status}; this comparison covers only available valid observations.`,
|
||||
),
|
||||
...(mismatches.length
|
||||
? [
|
||||
"DIAGNOSTIC ONLY: these runs are incompatible; differences cannot establish a performance change.",
|
||||
|
||||
@ -5,6 +5,7 @@ import { compareRuns, formatComparison } from "../compare.js";
|
||||
function run(value = 10) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
status: "complete",
|
||||
metadata: {
|
||||
git: { sha: "before", dirty: false },
|
||||
build: { mode: "release", features: [] },
|
||||
@ -146,6 +147,16 @@ test("failed attempts never enter comparison observations", () => {
|
||||
assert.equal(metric.absoluteInterval, null);
|
||||
});
|
||||
|
||||
test("comparisons warn when a run failed even if available metrics are compatible", () => {
|
||||
const candidate = run(12);
|
||||
candidate.status = "failed";
|
||||
candidate.cases[0].status = "failed";
|
||||
const result = compareRuns(run(), candidate);
|
||||
assert.equal(result.compatible, true);
|
||||
assert.equal(result.runs.candidate.status, "failed");
|
||||
assert.match(formatComparison(result), /candidate run is failed/i);
|
||||
});
|
||||
|
||||
test("offline comparison rejects malformed results without diagnostic bypass", () => {
|
||||
const missingEnvironment = run();
|
||||
delete missingEnvironment.metadata.environment;
|
||||
|
||||
39
frontend/scripts/renderer-benchmarks/tests/report.test.js
Normal file
39
frontend/scripts/renderer-benchmarks/tests/report.test.js
Normal file
@ -0,0 +1,39 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
test("page reports keep raw results and show the shared readable summary", async () => {
|
||||
const previous = globalThis.document;
|
||||
const report = { hidden: true, textContent: "" };
|
||||
globalThis.document = {
|
||||
querySelector: (selector) => (selector === "pre" ? report : {}),
|
||||
};
|
||||
try {
|
||||
await import("../../../resources/wasm-playground/benchmarks/browser.js");
|
||||
const result = {
|
||||
scope: "Renderer submission",
|
||||
metadata: {},
|
||||
warnings: [],
|
||||
cases: [
|
||||
{
|
||||
id: "rects/load",
|
||||
status: "complete",
|
||||
unattempted: 0,
|
||||
attempts: [
|
||||
{ warmup: false, status: "ok", metrics: { timeToFullMs: 2 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
globalThis.rendererBenchmark.showResult(result);
|
||||
assert.equal(globalThis.rendererBenchmarkResult, result);
|
||||
assert.equal(report.hidden, false);
|
||||
assert.match(report.textContent, /median 2\.000/);
|
||||
assert.match(report.textContent, /1 valid, 0 invalid/);
|
||||
assert.match(report.textContent, /Renderer submission/);
|
||||
} finally {
|
||||
globalThis.document = previous;
|
||||
delete globalThis.rendererBenchmark;
|
||||
delete globalThis.rendererBenchmarkReady;
|
||||
delete globalThis.rendererBenchmarkResult;
|
||||
}
|
||||
});
|
||||
@ -6,7 +6,20 @@ import {
|
||||
main,
|
||||
recordAttempt,
|
||||
finishCases,
|
||||
defaultFeatures,
|
||||
} from "../../renderer-benchmarks.js";
|
||||
|
||||
test("build provenance expands the renderer's declared default features", () => {
|
||||
assert.deepEqual(defaultFeatures({ default: [] }), ["default"]);
|
||||
assert.deepEqual(
|
||||
defaultFeatures({ default: ["fast"], fast: ["shared"], shared: ["fast"] }),
|
||||
["default", "fast", "shared"],
|
||||
);
|
||||
assert.throws(
|
||||
() => defaultFeatures({ default: ["profile"], profile: [] }),
|
||||
/diagnostic/,
|
||||
);
|
||||
});
|
||||
import {
|
||||
collectCases,
|
||||
deriveSeed,
|
||||
|
||||
@ -100,3 +100,30 @@ test("text summaries display the same observations and support limits as JSON",
|
||||
assert.match(report, /Requires at least 10 valid attempts/);
|
||||
assert.match(report, /2000 resamples/);
|
||||
});
|
||||
|
||||
test("reports expose missing attempts and explain support limits once", () => {
|
||||
const summary = summarizeRun({
|
||||
cases: [
|
||||
{
|
||||
id: "rects/load",
|
||||
status: "failed",
|
||||
unattempted: 9,
|
||||
attempts: [
|
||||
{
|
||||
warmup: false,
|
||||
status: "failed",
|
||||
metrics: { render: 3, upload: 2 },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
assert.equal(summary.cases[0].unattempted, 9);
|
||||
const report = formatSummary(summary);
|
||||
assert.match(report, /9 unattempted/);
|
||||
assert.match(report, /1 failed/);
|
||||
assert.equal(
|
||||
report.split("Requires at least 10 valid attempts").length - 1,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
163
frontend/scripts/renderer-benchmarks/verify-cli.js
Normal file
163
frontend/scripts/renderer-benchmarks/verify-cli.js
Normal file
@ -0,0 +1,163 @@
|
||||
// Unscored end-to-end failure checks. Each child uses the real prepared build,
|
||||
// server, browser, persistence and cleanup paths. No service is started here.
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { bounded, main } from "../renderer-benchmarks.js";
|
||||
|
||||
const filename = fileURLToPath(import.meta.url);
|
||||
|
||||
if (process.argv[2] === "--child") {
|
||||
const [, , , fault, output] = process.argv;
|
||||
const { chromium } = await import("playwright");
|
||||
const connect = chromium.connect.bind(chromium);
|
||||
let checkpoints = 0;
|
||||
chromium.connect = async (...args) => {
|
||||
const browser = await connect(...args);
|
||||
const newContext = browser.newContext.bind(browser);
|
||||
browser.newContext = async (...contextArgs) => {
|
||||
const context = await newContext(...contextArgs);
|
||||
await context.exposeBinding(
|
||||
"benchmarkFaultCheckpoint",
|
||||
async ({ page }) => {
|
||||
if (++checkpoints !== 2) return;
|
||||
if (fault === "interrupt") process.kill(process.pid, "SIGTERM");
|
||||
if (fault === "browser-disconnect")
|
||||
void browser.close().catch(() => {});
|
||||
if (fault === "page-crash") {
|
||||
const cdp = await context.newCDPSession(page);
|
||||
void cdp.send("Page.crash").catch(() => {});
|
||||
}
|
||||
},
|
||||
);
|
||||
await context.addInitScript(() => {
|
||||
let api;
|
||||
Object.defineProperty(globalThis, "rendererBenchmark", {
|
||||
get: () => api,
|
||||
set(value) {
|
||||
api = value;
|
||||
const attempt = value.runAttempt;
|
||||
value.runAttempt = async (...args) => {
|
||||
await globalThis.benchmarkFaultCheckpoint();
|
||||
return attempt(...args);
|
||||
};
|
||||
},
|
||||
});
|
||||
});
|
||||
return context;
|
||||
};
|
||||
return browser;
|
||||
};
|
||||
const args = [
|
||||
"--filter",
|
||||
"rects",
|
||||
"--warmups",
|
||||
"0",
|
||||
"--repetitions",
|
||||
"2",
|
||||
"--output",
|
||||
output,
|
||||
];
|
||||
if (fault === "missing-server")
|
||||
args.push("--base-url", "http://127.0.0.1:1/");
|
||||
if (fault === "wrong-server")
|
||||
args.push("--base-url", "http://localhost:3000/not-this-workspace/");
|
||||
process.exitCode = await main(args);
|
||||
} else {
|
||||
const directory = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), "renderer-cli-faults-"),
|
||||
);
|
||||
console.log(`Diagnostic results: ${directory}`);
|
||||
for (const fault of [
|
||||
"missing-browser",
|
||||
"missing-server",
|
||||
"wrong-server",
|
||||
"page-crash",
|
||||
"browser-disconnect",
|
||||
"interrupt",
|
||||
]) {
|
||||
const output = path.join(directory, `${fault}.json`);
|
||||
const log = await fs.open(path.join(directory, `${fault}.log`), "w");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[filename, "--child", fault, output],
|
||||
{
|
||||
detached: true,
|
||||
stdio: ["ignore", log.fd, log.fd],
|
||||
env: {
|
||||
...process.env,
|
||||
...(fault === "missing-browser"
|
||||
? {
|
||||
PLAYWRIGHT_BROWSERS_PATH: path.join(
|
||||
directory,
|
||||
"missing-browser",
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
);
|
||||
try {
|
||||
const exit = await bounded(
|
||||
new Promise((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("exit", (code, signal) => resolve({ code, signal }));
|
||||
}),
|
||||
180000,
|
||||
);
|
||||
assert.deepEqual(
|
||||
exit,
|
||||
{ code: 1, signal: null },
|
||||
`${fault}: must exit with an execution failure`,
|
||||
);
|
||||
const result = JSON.parse(await fs.readFile(output, "utf8"));
|
||||
assert.ok(result.finishedAt);
|
||||
assert.equal(
|
||||
result.status,
|
||||
fault === "interrupt" ? "interrupted" : "failed",
|
||||
);
|
||||
assert.ok(
|
||||
result.cases.every((c) => c.attempts.length + c.unattempted === 2),
|
||||
);
|
||||
if (
|
||||
["missing-browser", "missing-server", "wrong-server"].includes(fault)
|
||||
) {
|
||||
assert.ok(result.cases.every((c) => c.status === "unattempted"));
|
||||
assert.match(
|
||||
result.error,
|
||||
fault === "missing-browser"
|
||||
? /Chromium is missing/
|
||||
: fault === "missing-server"
|
||||
? /Static server unavailable/
|
||||
: /current benchmark assets/,
|
||||
);
|
||||
} else {
|
||||
assert.equal(
|
||||
result.cases[0].attempts[0].status,
|
||||
"ok",
|
||||
"retain the successful attempt before injection",
|
||||
);
|
||||
assert.equal(result.cases[0].attempts[1].status, "failed");
|
||||
if (fault === "page-crash") {
|
||||
assert.ok(
|
||||
result.cases.slice(1).every((c) => c.status === "complete"),
|
||||
"independent cases recover in fresh contexts",
|
||||
);
|
||||
} else {
|
||||
assert.ok(
|
||||
result.cases.slice(1).every((c) => c.status === "unattempted"),
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(`PASS ${fault}: partial results and bounded exit (unscored)`);
|
||||
} finally {
|
||||
// Kill only this diagnostic child's process group if its watchdog expired.
|
||||
if (child.exitCode === null && child.signalCode === null)
|
||||
process.kill(-child.pid, "SIGKILL");
|
||||
await log.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -30,6 +30,7 @@ try {
|
||||
"cancel-frame",
|
||||
"blocked-wasm",
|
||||
"page-crash",
|
||||
"report",
|
||||
]) {
|
||||
const context = await browser.newContext({
|
||||
viewport: options.viewport,
|
||||
@ -84,7 +85,36 @@ try {
|
||||
return 1;
|
||||
};
|
||||
});
|
||||
if (fault === "page-crash") {
|
||||
if (fault === "report") {
|
||||
const report = await page.evaluate((c) => {
|
||||
const value = {
|
||||
scope: "Diagnostic report",
|
||||
metadata: {},
|
||||
warnings: [],
|
||||
cases: [
|
||||
{
|
||||
...c,
|
||||
status: "complete",
|
||||
unattempted: 0,
|
||||
attempts: [
|
||||
{ warmup: false, status: "ok", metrics: { timeToFullMs: 2 } },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
globalThis.rendererBenchmark.showResult(value);
|
||||
return {
|
||||
result: globalThis.rendererBenchmarkResult,
|
||||
text: document.querySelector("pre").textContent,
|
||||
};
|
||||
}, benchmarkCase);
|
||||
assert.equal(
|
||||
report.result.cases[0].attempts[0].metrics.timeToFullMs,
|
||||
2,
|
||||
);
|
||||
assert.match(report.text, /median 2\.000/);
|
||||
assert.match(report.text, /1 valid, 0 invalid/);
|
||||
} else if (fault === "page-crash") {
|
||||
const crashed = page.waitForEvent("crash");
|
||||
const cdp = await context.newCDPSession(page);
|
||||
void cdp.send("Page.crash").catch(() => {});
|
||||
|
||||
163
frontend/scripts/renderer-benchmarks/verify-pages.js
Normal file
163
frontend/scripts/renderer-benchmarks/verify-pages.js
Normal file
@ -0,0 +1,163 @@
|
||||
// Unscored checks of the actual playground pages, including real pointer/wheel
|
||||
// events. Expose the mounted adapter only in this diagnostic routing layer.
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import { chromium } from "playwright";
|
||||
import { bounded } from "../renderer-benchmarks.js";
|
||||
import { scenarios } from "../../resources/wasm-playground/scenes/index.js";
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
filter: { type: "string", default: "" },
|
||||
headed: { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
const frontend = fileURLToPath(new URL("../../", import.meta.url));
|
||||
const selected = scenarios.filter((s) => s.id.includes(values.filter));
|
||||
assert.ok(selected.length, "No matching scenes");
|
||||
const server = await chromium.launchServer({
|
||||
headless: !values.headed,
|
||||
args: ["--enable-gpu"],
|
||||
});
|
||||
const browser = await chromium.connect(server.wsEndpoint());
|
||||
try {
|
||||
for (const scenario of selected) {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1920, height: 1080 },
|
||||
deviceScaleFactor: 2,
|
||||
});
|
||||
try {
|
||||
await bounded(
|
||||
(async () => {
|
||||
await context.route("http://benchmark.invalid/**", async (route) => {
|
||||
const pathname = new URL(route.request().url()).pathname;
|
||||
const base = path.join(
|
||||
frontend,
|
||||
pathname.startsWith("/wasm-playground/")
|
||||
? "resources"
|
||||
: "resources/public",
|
||||
);
|
||||
const target = path.resolve(base, `.${pathname}`);
|
||||
if (!target.startsWith(`${base}/`)) return route.abort();
|
||||
if (pathname === "/wasm-playground/js/lib.js") {
|
||||
const source = await fs.readFile(target, "utf8");
|
||||
await route.fulfill({
|
||||
contentType: "text/javascript",
|
||||
body:
|
||||
source.replace(
|
||||
"export async function mountScenario(",
|
||||
"async function mountOriginalScenario(",
|
||||
) +
|
||||
`
|
||||
export async function mountScenario(...args) {
|
||||
const adapter = await mountOriginalScenario(...args);
|
||||
const render = adapter.module._render;
|
||||
const view = adapter.module._set_view;
|
||||
globalThis.playgroundProbe = { full: 0, views: [] };
|
||||
adapter.module._render = (...args) => {
|
||||
const result = render(...args);
|
||||
if (result === 2) globalThis.playgroundProbe.full++;
|
||||
return result;
|
||||
};
|
||||
adapter.module._set_view = (...args) => {
|
||||
globalThis.playgroundProbe.views.push(args);
|
||||
return view(...args);
|
||||
};
|
||||
globalThis.playgroundAdapter = adapter;
|
||||
return adapter;
|
||||
}`,
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
await route.fulfill({ path: target });
|
||||
} catch {
|
||||
await route.fulfill({
|
||||
status: 404,
|
||||
body: "Missing diagnostic asset",
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
const page = await context.newPage();
|
||||
const errors = [];
|
||||
page.on("pageerror", (error) => errors.push(error.message));
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error") errors.push(message.text());
|
||||
});
|
||||
await page.goto(
|
||||
`http://benchmark.invalid/wasm-playground/${scenario.id}.html?seed=42`,
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
globalThis.playgroundAdapter || document.querySelector("#error"),
|
||||
null,
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
assert.equal(
|
||||
await page.locator("#error").count(),
|
||||
0,
|
||||
"page must mount successfully",
|
||||
);
|
||||
await page.mouse.move(800, 400);
|
||||
await page.mouse.down();
|
||||
await page.mouse.move(860, 430, { steps: 3 });
|
||||
await page.mouse.up();
|
||||
await page.waitForFunction(
|
||||
() => globalThis.playgroundProbe.full >= 1,
|
||||
);
|
||||
await page.evaluate(() => {
|
||||
globalThis.playgroundProbe.full = 0;
|
||||
});
|
||||
await page.mouse.wheel(0, -100);
|
||||
await page.waitForFunction(
|
||||
() => globalThis.playgroundProbe.full >= 1,
|
||||
);
|
||||
const state = await page.evaluate(() => ({
|
||||
views: globalThis.playgroundProbe.views,
|
||||
lost: document
|
||||
.querySelector("canvas")
|
||||
.getContext("webgl2")
|
||||
.isContextLost(),
|
||||
}));
|
||||
assert.equal(state.lost, false);
|
||||
assert.ok(
|
||||
state.views.some(([scale, x]) => scale === 1 && x > 0),
|
||||
"pointer drag must pan",
|
||||
);
|
||||
assert.ok(
|
||||
state.views.some(([scale]) => scale > 1),
|
||||
"wheel must zoom",
|
||||
);
|
||||
assert.deepEqual(errors, []);
|
||||
await bounded(
|
||||
page.evaluate(() => globalThis.playgroundAdapter.dispose()),
|
||||
5000,
|
||||
);
|
||||
console.log(
|
||||
`PASS ${scenario.id}: legacy mount, pan, zoom and disposal (unscored)`,
|
||||
);
|
||||
})(),
|
||||
65000,
|
||||
);
|
||||
} catch (error) {
|
||||
process.exitCode = 1;
|
||||
console.error(`FAIL ${scenario.id}: ${error.stack ?? error}`);
|
||||
} finally {
|
||||
await bounded(context.close(), 5000);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await bounded(browser.close(), 5000);
|
||||
} catch {
|
||||
server.process()?.kill("SIGKILL");
|
||||
}
|
||||
try {
|
||||
await bounded(server.close(), 5000);
|
||||
} catch {
|
||||
server.process()?.kill("SIGKILL");
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user