mirror of
https://github.com/penpot/penpot.git
synced 2026-10-29 05:50:35 +00:00
0ae78cf7ec
Record manifest-default features without adding A/B selection. Keep scored builds free of the optional function-name profiling switch. Exercise CLI persistence, interruption, browser failure and legacy page interactions through separate diagnostics. Share readable reports and warn when offline comparisons include failed runs. Validate with 50 focused tests, browser and CLI fault checks, two full corpus workflow runs, and headed rectangle cases. Shadow graphics and sampling calibration remain open. AI-assisted-by: GPT-6
571 lines
18 KiB
JavaScript
571 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import os from "node:os";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
import { randomUUID } from "node:crypto";
|
|
import { spawn, execFileSync } from "node:child_process";
|
|
import { parseArgs } from "node:util";
|
|
import { scenarios } from "../resources/wasm-playground/scenes/index.js";
|
|
import {
|
|
collectCases,
|
|
WEBGL_OPTIONS,
|
|
} from "../resources/wasm-playground/benchmarks/contract.js";
|
|
import {
|
|
summarizeRun,
|
|
formatSummary,
|
|
} from "../resources/wasm-playground/benchmarks/statistics.js";
|
|
import {
|
|
compareRuns,
|
|
formatComparison,
|
|
} from "./renderer-benchmarks/compare.js";
|
|
|
|
const frontend = fileURLToPath(new URL("../", import.meta.url));
|
|
const root = path.resolve(frontend, "..");
|
|
const LIMITS =
|
|
"Renderer submission timings only: not GPU completion, displayed pixels, whole-editor responsiveness, or visual correctness.";
|
|
|
|
export function readOptions(argv) {
|
|
const { values, positionals } = parseArgs({
|
|
args: argv,
|
|
allowPositionals: true,
|
|
options: {
|
|
help: { type: "boolean" },
|
|
headed: { type: "boolean", default: false },
|
|
diagnostic: { type: "boolean", default: false },
|
|
"base-url": { type: "string", default: "http://localhost:3000" },
|
|
output: { type: "string" },
|
|
filter: { type: "string", default: "" },
|
|
seed: { type: "string", default: "42" },
|
|
warmups: { type: "string", default: "3" },
|
|
repetitions: { type: "string", default: "10" },
|
|
"timeout-ms": { type: "string", default: "30000" },
|
|
},
|
|
});
|
|
const command = positionals[0] ?? "run";
|
|
if (!["run", "compare"].includes(command))
|
|
throw new Error(`Unknown command: ${command}`);
|
|
const number = (name, minimum, maximum) => {
|
|
const value = Number(values[name]);
|
|
if (!Number.isSafeInteger(value) || value < minimum || value > maximum)
|
|
throw new Error(`Invalid --${name}`);
|
|
return value;
|
|
};
|
|
if (
|
|
!values.help &&
|
|
(command === "run" ? positionals.length > 1 : positionals.length !== 3)
|
|
)
|
|
throw new Error(
|
|
"Use run [options] or compare baseline.json candidate.json [options]",
|
|
);
|
|
const baseUrl = new URL(values["base-url"]);
|
|
if (
|
|
!["http:", "https:"].includes(baseUrl.protocol) ||
|
|
baseUrl.username ||
|
|
baseUrl.password
|
|
)
|
|
throw new Error(
|
|
"--base-url must be an HTTP(S) server URL without credentials",
|
|
);
|
|
return {
|
|
command,
|
|
files: positionals.slice(1),
|
|
help: values.help,
|
|
headed: values.headed,
|
|
diagnostic: values.diagnostic,
|
|
baseUrl: baseUrl.href,
|
|
output: values.output,
|
|
filter: values.filter,
|
|
seed: number("seed", 0, 0xffffffff),
|
|
warmups: number("warmups", 0, 1000),
|
|
repetitions: number("repetitions", 1, 10000),
|
|
timeoutMs: number("timeout-ms", 100, 600000),
|
|
};
|
|
}
|
|
|
|
export async function bounded(operation, milliseconds, signal) {
|
|
let timer;
|
|
let onAbort;
|
|
try {
|
|
return await Promise.race([
|
|
operation,
|
|
new Promise((_, reject) => {
|
|
timer = setTimeout(
|
|
() =>
|
|
reject(new Error(`Node watchdog expired after ${milliseconds} ms`)),
|
|
milliseconds,
|
|
);
|
|
}),
|
|
new Promise((_, reject) => {
|
|
onAbort = () => reject(signal.reason ?? new Error("Interrupted"));
|
|
if (signal?.aborted) onAbort();
|
|
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
}),
|
|
]);
|
|
} finally {
|
|
clearTimeout(timer);
|
|
signal?.removeEventListener("abort", onAbort);
|
|
}
|
|
}
|
|
|
|
async function save(output, result) {
|
|
await fs.mkdir(path.dirname(output), { recursive: true });
|
|
const temporary = `${output}.tmp`;
|
|
await fs.writeFile(temporary, `${JSON.stringify(result, null, 2)}\n`);
|
|
await fs.rename(temporary, output);
|
|
}
|
|
|
|
function execute(command, args, options = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn(command, args, { stdio: "inherit", ...options });
|
|
child.once("error", reject);
|
|
child.once("exit", (code, signal) =>
|
|
code === 0
|
|
? resolve()
|
|
: reject(new Error(`${command} failed (${signal ?? code})`)),
|
|
);
|
|
});
|
|
}
|
|
|
|
async function requireFile(filename, instruction) {
|
|
try {
|
|
await fs.access(filename);
|
|
} catch {
|
|
throw new Error(instruction);
|
|
}
|
|
}
|
|
|
|
// Cargo metadata --no-deps reads the manifest without requiring unused targets'
|
|
// dependencies to be cached. Only local feature aliases need expansion here.
|
|
export function defaultFeatures(definitions) {
|
|
const enabled = new Set();
|
|
const visit = (name) => {
|
|
if (enabled.has(name)) return;
|
|
enabled.add(name);
|
|
for (const child of definitions[name] ?? []) visit(child);
|
|
};
|
|
visit("default");
|
|
if ([...enabled].some((name) => /^(stats|profile)(-|$)/.test(name)))
|
|
throw new Error(
|
|
"Default Cargo features enable renderer diagnostics; use a plain release manifest",
|
|
);
|
|
return [...enabled].sort();
|
|
}
|
|
|
|
async function preflight(options) {
|
|
await requireFile(
|
|
"/opt/emsdk/emsdk_env.sh",
|
|
"Run inside the prepared Penpot development container (Emscripten is missing).",
|
|
);
|
|
await requireFile(
|
|
path.join(root, "render-wasm/node_modules/esbuild/package.json"),
|
|
"Prepare renderer dependencies with pnpm install in render-wasm first.",
|
|
);
|
|
const { chromium } = await import("playwright");
|
|
await requireFile(
|
|
chromium.executablePath(),
|
|
"Chromium is missing; run the frontend scripts/setup preparation command first.",
|
|
);
|
|
try {
|
|
const response = await fetch(options.baseUrl, {
|
|
signal: AbortSignal.timeout(3000),
|
|
});
|
|
// A static root may legitimately be absent before assets have been copied.
|
|
if (response.status >= 500) throw new Error(`HTTP ${response.status}`);
|
|
} catch (error) {
|
|
throw new Error(
|
|
`Static server unavailable at ${options.baseUrl}; run node scripts/e2e-server.js from frontend first. ${error.message}`,
|
|
);
|
|
}
|
|
return chromium;
|
|
}
|
|
|
|
export function recordAttempt(benchmarkCase, attempt, index, warmups) {
|
|
if (attempt.environment) {
|
|
benchmarkCase.environment ??= attempt.environment;
|
|
if (
|
|
JSON.stringify(benchmarkCase.environment) !==
|
|
JSON.stringify(attempt.environment)
|
|
) {
|
|
attempt.status = "failed";
|
|
attempt.error = "Effective renderer environment changed within the case";
|
|
}
|
|
}
|
|
benchmarkCase.attempts.push({ index, warmup: index < warmups, ...attempt });
|
|
if (attempt.status !== "ok") benchmarkCase.status = "failed";
|
|
}
|
|
|
|
export function finishCases(cases, expected, status) {
|
|
for (const c of cases) {
|
|
c.unattempted = expected - c.attempts.length;
|
|
if (c.status === "running") c.status = status;
|
|
}
|
|
}
|
|
|
|
export async function run(options) {
|
|
const abort = new AbortController();
|
|
const interrupted = () => abort.abort(new Error("Benchmark interrupted"));
|
|
process.once("SIGINT", interrupted);
|
|
process.once("SIGTERM", interrupted);
|
|
const runId = randomUUID();
|
|
const output = path.resolve(
|
|
options.output ??
|
|
path.join(os.tmpdir(), `renderer-benchmarks-${runId}.json`),
|
|
);
|
|
const browserArgs = ["--enable-gpu"];
|
|
const configuration = {
|
|
viewport: { width: 1920, height: 1080 },
|
|
dpr: 2,
|
|
webglOptions: WEBGL_OPTIONS,
|
|
rendererOptions: { debug: 0, browser: 1, backgroundArgb: 0xfffabada },
|
|
warmups: options.warmups,
|
|
timeoutMs: options.timeoutMs,
|
|
};
|
|
const git = (args) =>
|
|
execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
|
|
const result = {
|
|
schemaVersion: 1,
|
|
runId,
|
|
timestamp: new Date().toISOString(),
|
|
status: "running",
|
|
scope: LIMITS,
|
|
warnings: [],
|
|
metadata: {
|
|
git: {
|
|
sha: git(["rev-parse", "HEAD"]),
|
|
dirty: Boolean(git(["status", "--porcelain"])),
|
|
},
|
|
build: {
|
|
mode: "release",
|
|
featurePolicy: "manifest-defaults",
|
|
features: null,
|
|
target: "frontend",
|
|
prepared: true,
|
|
},
|
|
environment: {
|
|
node: process.version,
|
|
os: os.platform(),
|
|
osRelease: os.release(),
|
|
arch: os.arch(),
|
|
cpu: os.cpus()[0]?.model ?? "unknown",
|
|
host: os.hostname(),
|
|
headless: !options.headed,
|
|
browserArgs,
|
|
},
|
|
configuration,
|
|
baseUrl: options.baseUrl,
|
|
serverReachable: false,
|
|
masterSeed: options.seed,
|
|
repetitions: options.repetitions,
|
|
metricSemantics:
|
|
"renderer-boundaries-v1;module-factory-includes-wasm-fetch;upload-includes-initial-view-index;finalization-includes-set-view-end",
|
|
},
|
|
cases: collectCases(scenarios, options.seed, options.filter),
|
|
};
|
|
let server;
|
|
let browser;
|
|
let context;
|
|
let page;
|
|
let pageErrors = [];
|
|
let suiteStart;
|
|
const closeContext = async () => {
|
|
if (!context) return;
|
|
const previous = context;
|
|
const previousPage = page;
|
|
context = null;
|
|
page = null;
|
|
try {
|
|
if (previousPage)
|
|
await bounded(
|
|
previousPage.evaluate(() => globalThis.rendererBenchmark?.dispose()),
|
|
2000,
|
|
);
|
|
} catch {
|
|
/* A failed or blocked page still has to be closed. */
|
|
}
|
|
await bounded(previous.close(), 5000);
|
|
};
|
|
const openPage = async () => {
|
|
context = await browser.newContext({
|
|
viewport: configuration.viewport,
|
|
deviceScaleFactor: configuration.dpr,
|
|
serviceWorkers: "block",
|
|
});
|
|
page = await context.newPage();
|
|
pageErrors = [];
|
|
page.on("pageerror", (error) => pageErrors.push(error.message));
|
|
page.on("crash", () => pageErrors.push("Browser page crashed"));
|
|
await page.goto(
|
|
new URL(`wasm-playground/benchmark.html?run=${runId}`, options.baseUrl)
|
|
.href,
|
|
{ timeout: options.timeoutMs },
|
|
);
|
|
await page.waitForFunction(
|
|
() => globalThis.rendererBenchmarkReady === true,
|
|
null,
|
|
{ timeout: options.timeoutMs },
|
|
);
|
|
};
|
|
try {
|
|
if (result.metadata.git.dirty) {
|
|
result.warnings.push(
|
|
"Dirty worktree: local development evidence, not a reproducible canonical record.",
|
|
);
|
|
console.warn(result.warnings.at(-1));
|
|
}
|
|
const chromium = await preflight(options);
|
|
result.metadata.serverReachable = true;
|
|
const packageJson = JSON.parse(
|
|
await fs.readFile(
|
|
new URL("../node_modules/playwright/package.json", import.meta.url),
|
|
"utf8",
|
|
),
|
|
);
|
|
result.metadata.environment.playwright = packageJson.version;
|
|
const manifest = JSON.parse(
|
|
execFileSync(
|
|
"cargo",
|
|
[
|
|
"metadata",
|
|
"--offline",
|
|
"--locked",
|
|
"--no-deps",
|
|
"--format-version",
|
|
"1",
|
|
],
|
|
{ cwd: path.join(root, "render-wasm"), encoding: "utf8" },
|
|
),
|
|
);
|
|
const renderer = manifest.packages.find(
|
|
(item) =>
|
|
item.manifest_path === path.join(root, "render-wasm/Cargo.toml"),
|
|
);
|
|
if (!renderer)
|
|
throw new Error("Renderer package missing from Cargo metadata");
|
|
result.metadata.build.features = defaultFeatures(renderer.features);
|
|
// Disallow package-manager network fallback in a prepared run.
|
|
const buildStart = performance.now();
|
|
await execute("./build", ["frontend", "--offline", "--locked"], {
|
|
cwd: path.join(root, "render-wasm"),
|
|
signal: abort.signal,
|
|
env: {
|
|
...process.env,
|
|
PENPOT_WASM_PREPARED: "1",
|
|
PENPOT_WASM_FUNCTION_NAMES: "0",
|
|
BUILD_MODE: "release",
|
|
NODE_ENV: "production",
|
|
RENDER_TARGET: "frontend",
|
|
BUILD_NAME: "render-wasm-benchmark",
|
|
CARGO_BUILD_TARGET: "wasm32-unknown-emscripten",
|
|
COREPACK_ENABLE_NETWORK: "0",
|
|
CARGO_NET_OFFLINE: "true",
|
|
VERSION: runId,
|
|
},
|
|
});
|
|
result.metadata.build.durationMs = performance.now() - buildStart;
|
|
const { copyWasmPlayground } = await import("./_helpers.js");
|
|
const previousCwd = process.cwd();
|
|
try {
|
|
process.chdir(frontend);
|
|
await copyWasmPlayground();
|
|
} finally {
|
|
process.chdir(previousCwd);
|
|
}
|
|
const markerName = "benchmark-run.json";
|
|
await fs.writeFile(
|
|
path.join(frontend, "resources/public/wasm-playground", markerName),
|
|
JSON.stringify({ runId }),
|
|
);
|
|
const marker = await fetch(
|
|
new URL(`wasm-playground/${markerName}?run=${runId}`, options.baseUrl),
|
|
{ signal: AbortSignal.timeout(3000), cache: "no-store" },
|
|
);
|
|
if (!marker.ok || (await marker.json()).runId !== runId)
|
|
throw new Error(
|
|
"The static server does not serve this workspace's current benchmark assets",
|
|
);
|
|
suiteStart = performance.now();
|
|
server = await chromium.launchServer({
|
|
headless: !options.headed,
|
|
args: browserArgs,
|
|
});
|
|
browser = await chromium.connect(server.wsEndpoint());
|
|
result.metadata.environment.chromium = browser.version();
|
|
const attemptOptions = {
|
|
...configuration,
|
|
moduleUrl: new URL(
|
|
`js/render-wasm-benchmark.js?run=${runId}`,
|
|
options.baseUrl,
|
|
).href,
|
|
};
|
|
for (const benchmarkCase of result.cases) {
|
|
if (abort.signal.aborted) throw abort.signal.reason;
|
|
console.log(`Running ${benchmarkCase.id}`);
|
|
benchmarkCase.status = "running";
|
|
for (
|
|
let index = 0;
|
|
index < options.warmups + options.repetitions;
|
|
index++
|
|
) {
|
|
let attempt;
|
|
try {
|
|
if (!page) await bounded(openPage(), options.timeoutMs, abort.signal);
|
|
attempt = await bounded(
|
|
page.evaluate(
|
|
([c, o]) => globalThis.rendererBenchmark.runAttempt(c, o),
|
|
[benchmarkCase, attemptOptions],
|
|
),
|
|
options.timeoutMs * 2 + 5000,
|
|
abort.signal,
|
|
);
|
|
} catch (error) {
|
|
attempt = {
|
|
status: "failed",
|
|
metrics: {},
|
|
invalidMetrics: {},
|
|
error: String(error.stack ?? error),
|
|
slices: [],
|
|
cachedSlices: [],
|
|
};
|
|
}
|
|
if (pageErrors.length) {
|
|
attempt.status = "failed";
|
|
attempt.error = [attempt.error, ...pageErrors]
|
|
.filter(Boolean)
|
|
.join("\n");
|
|
}
|
|
recordAttempt(benchmarkCase, attempt, index, options.warmups);
|
|
if (attempt.environment) {
|
|
if (attempt.environment.webgl.acceleration === "software") {
|
|
const warning =
|
|
"Software WebGL: compare only equivalent renderer environments.";
|
|
if (!result.warnings.includes(warning)) {
|
|
result.warnings.push(warning);
|
|
console.warn(warning);
|
|
}
|
|
}
|
|
}
|
|
if (attempt.status !== "ok") benchmarkCase.status = "failed";
|
|
// Persist only outside browser timing; every failed attempt remains visible.
|
|
await save(output, result);
|
|
if (abort.signal.aborted) {
|
|
benchmarkCase.status = "interrupted";
|
|
throw abort.signal.reason;
|
|
}
|
|
if (!browser.isConnected())
|
|
throw new Error(
|
|
"Browser disconnected; remaining cases were not attempted",
|
|
);
|
|
if (attempt.status === "failed") {
|
|
// Cache history cannot be restored after failure: stop this case, then
|
|
// continue independent cases. Do not silently replace failed samples.
|
|
benchmarkCase.unattempted =
|
|
options.warmups + options.repetitions - index - 1;
|
|
await closeContext();
|
|
break;
|
|
}
|
|
if (benchmarkCase.group === "fresh-module-context")
|
|
await closeContext();
|
|
}
|
|
await closeContext();
|
|
if (benchmarkCase.status === "running") benchmarkCase.status = "complete";
|
|
}
|
|
result.status = result.cases.every((c) => c.status === "complete")
|
|
? "complete"
|
|
: "failed";
|
|
} catch (error) {
|
|
result.status = abort.signal.aborted ? "interrupted" : "failed";
|
|
result.error = String(error.stack ?? error);
|
|
console.error(result.error);
|
|
} finally {
|
|
finishCases(
|
|
result.cases,
|
|
options.warmups + options.repetitions,
|
|
result.status,
|
|
);
|
|
if (suiteStart !== undefined)
|
|
result.metadata.suiteDurationMs = performance.now() - suiteStart;
|
|
result.summary = summarizeRun(result);
|
|
// Render a report once, after every scored case. Report work never separates
|
|
// attempts or cases. The same data also goes to JSON and the terminal.
|
|
if (browser?.isConnected() && !abort.signal.aborted) {
|
|
try {
|
|
if (!page) await bounded(openPage(), 5000);
|
|
await bounded(
|
|
page.evaluate(
|
|
(value) => globalThis.rendererBenchmark.showResult(value),
|
|
result,
|
|
),
|
|
5000,
|
|
);
|
|
} catch (error) {
|
|
result.warnings.push(`Page report unavailable: ${error.message}`);
|
|
}
|
|
}
|
|
try {
|
|
await closeContext();
|
|
} catch {
|
|
/* Browser cleanup below also closes blocked pages. */
|
|
}
|
|
// The browser server exposes its child process for a bounded hard cleanup.
|
|
try {
|
|
if (browser) await bounded(browser.close(), 5000);
|
|
} catch {
|
|
server?.process()?.kill("SIGKILL");
|
|
}
|
|
try {
|
|
if (server) await bounded(server.close(), 5000);
|
|
} catch {
|
|
server?.process()?.kill("SIGKILL");
|
|
}
|
|
process.removeListener("SIGINT", interrupted);
|
|
process.removeListener("SIGTERM", interrupted);
|
|
result.finishedAt = new Date().toISOString();
|
|
await save(output, result);
|
|
}
|
|
console.log(LIMITS);
|
|
console.log(formatSummary(result.summary));
|
|
console.log(`Result: ${output}`);
|
|
return result.status === "complete" ? 0 : 1;
|
|
}
|
|
|
|
export async function main(argv = process.argv.slice(2)) {
|
|
const options = readOptions(argv);
|
|
if (options.help) {
|
|
console.log(
|
|
"Renderer benchmarks\n node scripts/renderer-benchmarks.js run [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--output FILE]\n node scripts/renderer-benchmarks.js compare BASELINE CANDIDATE [--diagnostic] [--output FILE]\nRun requires a prepared development container and an existing static server on port 3000. Defaults: 3 warmups, 10 measured attempts, 30s timeout; provisional pending calibration.\n" +
|
|
LIMITS,
|
|
);
|
|
return 0;
|
|
}
|
|
if (options.command === "compare") {
|
|
const results = await Promise.all(
|
|
options.files.map(async (file) =>
|
|
JSON.parse(await fs.readFile(file, "utf8")),
|
|
),
|
|
);
|
|
const comparison = compareRuns(...results, {
|
|
diagnostic: options.diagnostic,
|
|
});
|
|
console.log(LIMITS);
|
|
console.log(formatComparison(comparison));
|
|
if (options.output) await save(path.resolve(options.output), comparison);
|
|
return 0;
|
|
}
|
|
return run(options);
|
|
}
|
|
|
|
if (
|
|
process.argv[1] &&
|
|
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href
|
|
) {
|
|
main()
|
|
.then((code) => {
|
|
process.exitCode = code;
|
|
})
|
|
.catch((error) => {
|
|
console.error(error.message);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|