mirror of
https://github.com/penpot/penpot.git
synced 2026-10-29 22:10:34 +00:00
731bcc5664
Share seven deterministic fixtures with the playground pages and measure renderer lifecycle, pan, and zoom through a prepared Playwright runner. Retain raw attempts and failures with offline statistical comparisons. Add focused tests and separate browser fault diagnostics. Full-suite calibration and shadow GPU validation remain tracked follow-up work. AI-assisted-by: GPT-6
531 lines
17 KiB
JavaScript
531 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import os from "node:os";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
import { randomUUID } from "node:crypto";
|
|
import { spawn, execFileSync } from "node:child_process";
|
|
import { parseArgs } from "node:util";
|
|
import { scenarios } from "../resources/wasm-playground/scenes/index.js";
|
|
import {
|
|
collectCases,
|
|
WEBGL_OPTIONS,
|
|
} from "../resources/wasm-playground/benchmarks/contract.js";
|
|
import {
|
|
summarizeRun,
|
|
formatSummary,
|
|
} from "../resources/wasm-playground/benchmarks/statistics.js";
|
|
import {
|
|
compareRuns,
|
|
formatComparison,
|
|
} from "./renderer-benchmarks/compare.js";
|
|
|
|
const frontend = fileURLToPath(new URL("../", import.meta.url));
|
|
const root = path.resolve(frontend, "..");
|
|
const LIMITS =
|
|
"Renderer submission timings only: not GPU completion, displayed pixels, whole-editor responsiveness, or visual correctness.";
|
|
|
|
export function readOptions(argv) {
|
|
const { values, positionals } = parseArgs({
|
|
args: argv,
|
|
allowPositionals: true,
|
|
options: {
|
|
help: { type: "boolean" },
|
|
headed: { type: "boolean", default: false },
|
|
diagnostic: { type: "boolean", default: false },
|
|
"base-url": { type: "string", default: "http://localhost:3000" },
|
|
output: { type: "string" },
|
|
filter: { type: "string", default: "" },
|
|
seed: { type: "string", default: "42" },
|
|
warmups: { type: "string", default: "3" },
|
|
repetitions: { type: "string", default: "10" },
|
|
"timeout-ms": { type: "string", default: "30000" },
|
|
},
|
|
});
|
|
const command = positionals[0] ?? "run";
|
|
if (!["run", "compare"].includes(command))
|
|
throw new Error(`Unknown command: ${command}`);
|
|
const number = (name, minimum, maximum) => {
|
|
const value = Number(values[name]);
|
|
if (!Number.isSafeInteger(value) || value < minimum || value > maximum)
|
|
throw new Error(`Invalid --${name}`);
|
|
return value;
|
|
};
|
|
if (
|
|
!values.help &&
|
|
(command === "run" ? positionals.length > 1 : positionals.length !== 3)
|
|
)
|
|
throw new Error(
|
|
"Use run [options] or compare baseline.json candidate.json [options]",
|
|
);
|
|
const baseUrl = new URL(values["base-url"]);
|
|
if (
|
|
!["http:", "https:"].includes(baseUrl.protocol) ||
|
|
baseUrl.username ||
|
|
baseUrl.password
|
|
)
|
|
throw new Error(
|
|
"--base-url must be an HTTP(S) server URL without credentials",
|
|
);
|
|
return {
|
|
command,
|
|
files: positionals.slice(1),
|
|
help: values.help,
|
|
headed: values.headed,
|
|
diagnostic: values.diagnostic,
|
|
baseUrl: baseUrl.href,
|
|
output: values.output,
|
|
filter: values.filter,
|
|
seed: number("seed", 0, 0xffffffff),
|
|
warmups: number("warmups", 0, 1000),
|
|
repetitions: number("repetitions", 1, 10000),
|
|
timeoutMs: number("timeout-ms", 100, 600000),
|
|
};
|
|
}
|
|
|
|
export async function bounded(operation, milliseconds, signal) {
|
|
let timer;
|
|
let onAbort;
|
|
try {
|
|
return await Promise.race([
|
|
operation,
|
|
new Promise((_, reject) => {
|
|
timer = setTimeout(
|
|
() =>
|
|
reject(new Error(`Node watchdog expired after ${milliseconds} ms`)),
|
|
milliseconds,
|
|
);
|
|
}),
|
|
new Promise((_, reject) => {
|
|
onAbort = () => reject(signal.reason ?? new Error("Interrupted"));
|
|
if (signal?.aborted) onAbort();
|
|
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
}),
|
|
]);
|
|
} finally {
|
|
clearTimeout(timer);
|
|
signal?.removeEventListener("abort", onAbort);
|
|
}
|
|
}
|
|
|
|
async function save(output, result) {
|
|
await fs.mkdir(path.dirname(output), { recursive: true });
|
|
const temporary = `${output}.tmp`;
|
|
await fs.writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`);
|
|
await fs.rename(temporary, output);
|
|
}
|
|
|
|
function execute(command, args, options = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn(command, args, { stdio: "inherit", ...options });
|
|
child.once("error", reject);
|
|
child.once("exit", (code, signal) =>
|
|
code === 0
|
|
? resolve()
|
|
: reject(new Error(`${command} failed (${signal ?? code})`)),
|
|
);
|
|
});
|
|
}
|
|
|
|
async function requireFile(filename, instruction) {
|
|
try {
|
|
await fs.access(filename);
|
|
} catch {
|
|
throw new Error(instruction);
|
|
}
|
|
}
|
|
|
|
async function preflight(options) {
|
|
await requireFile(
|
|
"/opt/emsdk/emsdk_env.sh",
|
|
"Run inside the prepared Penpot development container (Emscripten is missing).",
|
|
);
|
|
await requireFile(
|
|
path.join(root, "render-wasm/node_modules/esbuild/package.json"),
|
|
"Prepare renderer dependencies with pnpm install in render-wasm first.",
|
|
);
|
|
const { chromium } = await import("playwright");
|
|
await requireFile(
|
|
chromium.executablePath(),
|
|
"Chromium is missing; run the frontend scripts/setup preparation command first.",
|
|
);
|
|
try {
|
|
const response = await fetch(options.baseUrl, {
|
|
signal: AbortSignal.timeout(3000),
|
|
});
|
|
// A static root may legitimately be absent before assets have been copied.
|
|
if (response.status >= 500) throw new Error(`HTTP ${response.status}`);
|
|
} catch (error) {
|
|
throw new Error(
|
|
`Static server unavailable at ${options.baseUrl}; run node scripts/e2e-server.js from frontend first. ${error.message}`,
|
|
);
|
|
}
|
|
return chromium;
|
|
}
|
|
|
|
export function recordAttempt(benchmarkCase, attempt, index, warmups) {
|
|
if (attempt.environment) {
|
|
benchmarkCase.environment ??= attempt.environment;
|
|
if (
|
|
JSON.stringify(benchmarkCase.environment) !==
|
|
JSON.stringify(attempt.environment)
|
|
) {
|
|
attempt.status = "failed";
|
|
attempt.error = "Effective renderer environment changed within the case";
|
|
}
|
|
}
|
|
benchmarkCase.attempts.push({ index, warmup: index < warmups, ...attempt });
|
|
if (attempt.status !== "ok") benchmarkCase.status = "failed";
|
|
}
|
|
|
|
export function finishCases(cases, expected, status) {
|
|
for (const c of cases) {
|
|
c.unattempted = expected - c.attempts.length;
|
|
if (c.status === "running") c.status = status;
|
|
}
|
|
}
|
|
|
|
export async function run(options) {
|
|
const abort = new AbortController();
|
|
const interrupted = () => abort.abort(new Error("Benchmark interrupted"));
|
|
process.once("SIGINT", interrupted);
|
|
process.once("SIGTERM", interrupted);
|
|
const runId = randomUUID();
|
|
const output = path.resolve(
|
|
options.output ??
|
|
path.join(os.tmpdir(), `renderer-benchmarks-${runId}.json`),
|
|
);
|
|
const browserArgs = ["--enable-gpu"];
|
|
const configuration = {
|
|
viewport: { width: 1920, height: 1080 },
|
|
dpr: 2,
|
|
webglOptions: WEBGL_OPTIONS,
|
|
rendererOptions: { debug: 0, browser: 1, backgroundArgb: 0xfffabada },
|
|
warmups: options.warmups,
|
|
timeoutMs: options.timeoutMs,
|
|
};
|
|
const git = (args) =>
|
|
execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
|
|
const result = {
|
|
schemaVersion: 1,
|
|
runId,
|
|
timestamp: new Date().toISOString(),
|
|
status: "running",
|
|
scope: LIMITS,
|
|
warnings: [],
|
|
metadata: {
|
|
git: {
|
|
sha: git(["rev-parse", "HEAD"]),
|
|
dirty: Boolean(git(["status", "--porcelain"])),
|
|
},
|
|
build: {
|
|
mode: "release",
|
|
features: [],
|
|
target: "frontend",
|
|
prepared: true,
|
|
},
|
|
environment: {
|
|
node: process.version,
|
|
os: os.platform(),
|
|
osRelease: os.release(),
|
|
arch: os.arch(),
|
|
cpu: os.cpus()[0]?.model ?? "unknown",
|
|
host: os.hostname(),
|
|
headless: !options.headed,
|
|
browserArgs,
|
|
},
|
|
configuration,
|
|
baseUrl: options.baseUrl,
|
|
serverReachable: false,
|
|
masterSeed: options.seed,
|
|
repetitions: options.repetitions,
|
|
metricSemantics:
|
|
"renderer-boundaries-v1;module-factory-includes-wasm-fetch;upload-includes-initial-view-index;finalization-includes-set-view-end",
|
|
},
|
|
cases: collectCases(scenarios, options.seed, options.filter),
|
|
};
|
|
let server;
|
|
let browser;
|
|
let context;
|
|
let page;
|
|
let pageErrors = [];
|
|
let suiteStart;
|
|
const closeContext = async () => {
|
|
if (!context) return;
|
|
const previous = context;
|
|
const previousPage = page;
|
|
context = null;
|
|
page = null;
|
|
try {
|
|
if (previousPage)
|
|
await bounded(
|
|
previousPage.evaluate(() => globalThis.rendererBenchmark?.dispose()),
|
|
2000,
|
|
);
|
|
} catch {
|
|
/* A failed or blocked page still has to be closed. */
|
|
}
|
|
await bounded(previous.close(), 5000);
|
|
};
|
|
const openPage = async () => {
|
|
context = await browser.newContext({
|
|
viewport: configuration.viewport,
|
|
deviceScaleFactor: configuration.dpr,
|
|
serviceWorkers: "block",
|
|
});
|
|
page = await context.newPage();
|
|
pageErrors = [];
|
|
page.on("pageerror", (error) => pageErrors.push(error.message));
|
|
page.on("crash", () => pageErrors.push("Browser page crashed"));
|
|
await page.goto(
|
|
new URL(`wasm-playground/benchmark.html?run=${runId}`, options.baseUrl)
|
|
.href,
|
|
{ timeout: options.timeoutMs },
|
|
);
|
|
await page.waitForFunction(
|
|
() => globalThis.rendererBenchmarkReady === true,
|
|
null,
|
|
{ timeout: options.timeoutMs },
|
|
);
|
|
};
|
|
try {
|
|
if (result.metadata.git.dirty) {
|
|
result.warnings.push(
|
|
"Dirty worktree: local development evidence, not a reproducible canonical record.",
|
|
);
|
|
console.warn(result.warnings.at(-1));
|
|
}
|
|
const chromium = await preflight(options);
|
|
result.metadata.serverReachable = true;
|
|
const packageJson = JSON.parse(
|
|
await fs.readFile(
|
|
new URL("../node_modules/playwright/package.json", import.meta.url),
|
|
"utf8",
|
|
),
|
|
);
|
|
result.metadata.environment.playwright = packageJson.version;
|
|
// Disallow package-manager network fallback in a prepared run.
|
|
const buildStart = performance.now();
|
|
await execute("./build", ["frontend", "--offline", "--locked"], {
|
|
cwd: path.join(root, "render-wasm"),
|
|
signal: abort.signal,
|
|
env: {
|
|
...process.env,
|
|
PENPOT_WASM_PREPARED: "1",
|
|
BUILD_MODE: "release",
|
|
NODE_ENV: "production",
|
|
RENDER_TARGET: "frontend",
|
|
BUILD_NAME: "render-wasm-benchmark",
|
|
CARGO_BUILD_TARGET: "wasm32-unknown-emscripten",
|
|
COREPACK_ENABLE_NETWORK: "0",
|
|
CARGO_NET_OFFLINE: "true",
|
|
VERSION: runId,
|
|
},
|
|
});
|
|
result.metadata.build.durationMs = performance.now() - buildStart;
|
|
const { copyWasmPlayground } = await import("./_helpers.js");
|
|
const previousCwd = process.cwd();
|
|
try {
|
|
process.chdir(frontend);
|
|
await copyWasmPlayground();
|
|
} finally {
|
|
process.chdir(previousCwd);
|
|
}
|
|
const markerName = "benchmark-run.json";
|
|
await fs.writeFile(
|
|
path.join(frontend, "resources/public/wasm-playground", markerName),
|
|
JSON.stringify({ runId }),
|
|
);
|
|
const marker = await fetch(
|
|
new URL(`wasm-playground/${markerName}?run=${runId}`, options.baseUrl),
|
|
{ signal: AbortSignal.timeout(3000), cache: "no-store" },
|
|
);
|
|
if (!marker.ok || (await marker.json()).runId !== runId)
|
|
throw new Error(
|
|
"The static server does not serve this workspace's current benchmark assets",
|
|
);
|
|
suiteStart = performance.now();
|
|
server = await chromium.launchServer({
|
|
headless: !options.headed,
|
|
args: browserArgs,
|
|
});
|
|
browser = await chromium.connect(server.wsEndpoint());
|
|
result.metadata.environment.chromium = browser.version();
|
|
const attemptOptions = {
|
|
...configuration,
|
|
moduleUrl: new URL(
|
|
`js/render-wasm-benchmark.js?run=${runId}`,
|
|
options.baseUrl,
|
|
).href,
|
|
};
|
|
for (const benchmarkCase of result.cases) {
|
|
if (abort.signal.aborted) throw abort.signal.reason;
|
|
console.log(`Running ${benchmarkCase.id}`);
|
|
benchmarkCase.status = "running";
|
|
for (
|
|
let index = 0;
|
|
index < options.warmups + options.repetitions;
|
|
index++
|
|
) {
|
|
let attempt;
|
|
try {
|
|
if (!page) await bounded(openPage(), options.timeoutMs, abort.signal);
|
|
attempt = await bounded(
|
|
page.evaluate(
|
|
([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o),
|
|
[benchmarkCase, attemptOptions],
|
|
),
|
|
options.timeoutMs * 2 + 5000,
|
|
abort.signal,
|
|
);
|
|
} catch (error) {
|
|
attempt = {
|
|
status: "failed",
|
|
metrics: {},
|
|
invalidMetrics: {},
|
|
error: String(error.stack ?? error),
|
|
slices: [],
|
|
cachedSlices: [],
|
|
};
|
|
}
|
|
if (pageErrors.length) {
|
|
attempt.status = "failed";
|
|
attempt.error = [attempt.error, ...pageErrors]
|
|
.filter(Boolean)
|
|
.join("\n");
|
|
}
|
|
recordAttempt(benchmarkCase, attempt, index, options.warmups);
|
|
if (attempt.environment) {
|
|
if (attempt.environment.webgl.acceleration === "software") {
|
|
const warning =
|
|
"Software WebGL: compare only equivalent renderer environments.";
|
|
if (!result.warnings.includes(warning)) {
|
|
result.warnings.push(warning);
|
|
console.warn(warning);
|
|
}
|
|
}
|
|
}
|
|
if (attempt.status !== "ok") benchmarkCase.status = "failed";
|
|
// Persist only outside browser timing; every failed attempt remains visible.
|
|
await save(output, result);
|
|
if (abort.signal.aborted) {
|
|
benchmarkCase.status = "interrupted";
|
|
throw abort.signal.reason;
|
|
}
|
|
if (!browser.isConnected())
|
|
throw new Error(
|
|
"Browser disconnected; remaining cases were not attempted",
|
|
);
|
|
if (attempt.status === "failed") {
|
|
// Cache history cannot be restored after failure: stop this case, then
|
|
// continue independent cases. Do not silently replace failed samples.
|
|
benchmarkCase.unattempted =
|
|
options.warmups + options.repetitions - index - 1;
|
|
await closeContext();
|
|
break;
|
|
}
|
|
if (benchmarkCase.group === "fresh-module-context")
|
|
await closeContext();
|
|
}
|
|
await closeContext();
|
|
if (benchmarkCase.status === "running") benchmarkCase.status = "complete";
|
|
}
|
|
result.status = result.cases.every((c) => c.status === "complete")
|
|
? "complete"
|
|
: "failed";
|
|
} catch (error) {
|
|
result.status = abort.signal.aborted ? "interrupted" : "failed";
|
|
result.error = String(error.stack ?? error);
|
|
console.error(result.error);
|
|
} finally {
|
|
finishCases(
|
|
result.cases,
|
|
options.warmups + options.repetitions,
|
|
result.status,
|
|
);
|
|
if (suiteStart !== undefined)
|
|
result.metadata.suiteDurationMs = performance.now() - suiteStart;
|
|
result.summary = summarizeRun(result);
|
|
// Render a report once, after every scored case. Report work never separates
|
|
// attempts or cases. The same data also goes to JSON and the terminal.
|
|
if (browser?.isConnected() && !abort.signal.aborted) {
|
|
try {
|
|
if (!page) await bounded(openPage(), 5000);
|
|
await bounded(
|
|
page.evaluate(
|
|
(value) => globalThis.rendererBenchmark.showResult(value),
|
|
result,
|
|
),
|
|
5000,
|
|
);
|
|
} catch (error) {
|
|
result.warnings.push(`Page report unavailable: ${error.message}`);
|
|
}
|
|
}
|
|
try {
|
|
await closeContext();
|
|
} catch {
|
|
/* Browser cleanup below also closes blocked pages. */
|
|
}
|
|
// The browser server exposes its child process for a bounded hard cleanup.
|
|
try {
|
|
if (browser) await bounded(browser.close(), 5000);
|
|
} catch {
|
|
server?.process()?.kill("SIGKILL");
|
|
}
|
|
try {
|
|
if (server) await bounded(server.close(), 5000);
|
|
} catch {
|
|
server?.process()?.kill("SIGKILL");
|
|
}
|
|
process.removeListener("SIGINT", interrupted);
|
|
process.removeListener("SIGTERM", interrupted);
|
|
result.finishedAt = new Date().toISOString();
|
|
await save(output, result);
|
|
}
|
|
console.log(LIMITS);
|
|
console.log(formatSummary(result.summary));
|
|
console.log(`Result: ${output}`);
|
|
return result.status === "complete" ? 0 : 1;
|
|
}
|
|
|
|
export async function main(argv = process.argv.slice(2)) {
|
|
const options = readOptions(argv);
|
|
if (options.help) {
|
|
console.log(
|
|
"Renderer benchmarks\n node scripts/renderer-benchmarks.js run [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--output FILE]\n node scripts/renderer-benchmarks.js compare BASELINE CANDIDATE [--diagnostic] [--output FILE]\nRun requires a prepared development container and an existing static server on port 3000. Defaults: 3 warmups, 10 measured attempts, 30s timeout; provisional pending calibration.\n" +
|
|
LIMITS,
|
|
);
|
|
return 0;
|
|
}
|
|
if (options.command === "compare") {
|
|
const results = await Promise.all(
|
|
options.files.map(async (file) =>
|
|
JSON.parse(await fs.readFile(file, "utf8")),
|
|
),
|
|
);
|
|
const comparison = compareRuns(...results, {
|
|
diagnostic: options.diagnostic,
|
|
});
|
|
console.log(LIMITS);
|
|
console.log(formatComparison(comparison));
|
|
if (options.output) await save(path.resolve(options.output), comparison);
|
|
return 0;
|
|
}
|
|
return run(options);
|
|
}
|
|
|
|
if (
|
|
process.argv[1] &&
|
|
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
|
|
) {
|
|
main()
|
|
.then((code) => {
|
|
process.exitCode = code;
|
|
})
|
|
.catch((error) => {
|
|
console.error(error.message);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|