diff --git a/frontend/resources/wasm-playground/BENCHMARKS.md b/frontend/resources/wasm-playground/BENCHMARKS.md index 9d3ce5bce9..464de2db5f 100644 --- a/frontend/resources/wasm-playground/BENCHMARKS.md +++ b/frontend/resources/wasm-playground/BENCHMARKS.md @@ -16,6 +16,7 @@ it does not move caches or download missing crates. pnpm run benchmark:renderer run --output /tmp/baseline.json pnpm run benchmark:renderer run --filter texts --seed 42 --output /tmp/texts.json pnpm run benchmark:renderer run --headed --filter rects/default/zoom +pnpm run benchmark:renderer run --no-build --filter rects/default/pan pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json pnpm run benchmark:renderer compare /tmp/baseline.json /tmp/candidate.json --diagnostic pnpm run test:renderer-benchmarks @@ -26,7 +27,12 @@ Each run builds the ordinary optimized frontend renderer once using skipped, package-manager network fallback is disabled, and the emitted module is named `render-wasm-benchmark`. Playground assets are refreshed before launch. A per-run marker checks that the server points at this checkout. Concurrent builds -or benchmark commands against the same checkout are unsupported. +or benchmark commands against the same checkout are unsupported. `run --no-build` +skips the build, the Cargo manifest read, and the Emscripten/esbuild preflight +checks, then measures the existing `render-wasm-benchmark.js`/`.wasm` pair; it +still refreshes playground assets and checks the marker. It rejects `--features` +and never applies to `ab`, and records `metadata.build.built: false` with a +warning that the artifact was not rebuilt from this revision. The default server URL is `http://localhost:3000`; override it with `--base-url`. Default JSON output goes to a unique file under the system temporary directory. @@ -146,9 +152,11 @@ browser; the printed plan states this cost. `--output` for `ab` is a directory root that must not exist yet; `configs/.json`, `comparisons/__.json`, and `matrix.json` land beneath it. -Run JSON is `schemaVersion` 2: `metadata.build.features` is the explicit -sorted set (the A/B identity axis), `defaultFeatures` the resolved manifest -defaults, plus the runner-controlled `env` map and the allowlisted +Run JSON is `schemaVersion` 2: `metadata.build.built` is `false` when the run +reused an existing artifact, and the build-derived fields below are absent, so +comparing it with a built run fails compatibility. `metadata.build.features` is +the explicit sorted set (the A/B identity axis), `defaultFeatures` the resolved +manifest defaults, plus the runner-controlled `env` map and the allowlisted `ambientEnv` (`RUSTFLAGS`, `RUSTC_WRAPPER`, `CARGO_TARGET_DIR`, `CARGO_HOME`, `CC`, `CFLAGS`, `CXXFLAGS`, `LDFLAGS`, `PENPOT_WASM_FUNCTION_NAMES`, `CARGO_PROFILE_*`, `EMSDK*`). Comparison of the six build fields is always diff --git a/frontend/scripts/renderer-benchmarks.js b/frontend/scripts/renderer-benchmarks.js index 109820f518..a23f5aae19 100644 --- a/frontend/scripts/renderer-benchmarks.js +++ b/frontend/scripts/renderer-benchmarks.js @@ -59,6 +59,7 @@ export function readOptions(argv) { "max-features": { type: "string", default: "3" }, confirm: { type: "boolean", default: false }, "dry-run": { type: "boolean", default: false }, + "no-build": { type: "boolean", default: false }, }, }); const command = positionals[0] ?? "run"; @@ -89,6 +90,10 @@ export function readOptions(argv) { const maxFeatures = number("max-features", 1, 6); if (command !== "ab" && values["dry-run"]) throw new Error("--dry-run is only valid for ab"); + if (values["no-build"] && command !== "run") + throw new Error("--no-build is only valid for run"); + if (values["no-build"] && features.length) + throw new Error("--no-build skips the build, so --features cannot apply"); if (command === "ab" && !values.help) { if (features.some((name) => /^(stats|profile)(-|$)/.test(name))) throw new Error( @@ -127,6 +132,7 @@ export function readOptions(argv) { maxFeatures, confirm: values.confirm, dryRun: values["dry-run"], + noBuild: values["no-build"], }; } @@ -305,14 +311,25 @@ export function recordAmbientEnv(env = process.env) { } async function preflight(options) { - await requireFile( - "/opt/emsdk/emsdk_env.sh", - "Run inside the prepared Penpot development container (Emscripten is missing).", - ); - await requireFile( - path.join(root, "render-wasm/node_modules/esbuild/package.json"), - "Prepare renderer dependencies with pnpm install in render-wasm first.", - ); + if (options.noBuild) { + for (const artifact of [ + "render-wasm-benchmark.js", + "render-wasm-benchmark.wasm", + ]) + await requireFile( + path.join(frontend, "resources/public/js", artifact), + "No existing renderer benchmark build; run once without --no-build first.", + ); + } else { + await requireFile( + "/opt/emsdk/emsdk_env.sh", + "Run inside the prepared Penpot development container (Emscripten is missing).", + ); + await requireFile( + path.join(root, "render-wasm/node_modules/esbuild/package.json"), + "Prepare renderer dependencies with pnpm install in render-wasm first.", + ); + } const { chromium } = await import("playwright"); await requireFile( chromium.executablePath(), @@ -389,6 +406,7 @@ export async function run(options) { }, build: { mode: "release", + built: !options.noBuild, featurePolicy: "manifest-defaults", features: [], target: "frontend", @@ -474,34 +492,41 @@ export async function run(options) { ), ); result.metadata.environment.playwright = packageJson.version; - const requested = [...(options.features ?? [])].sort(); - const resolved = validateFeatureSelection( - readRendererManifest(), - requested, - options.command, - ); - result.metadata.build.features = requested; - result.metadata.build.defaultFeatures = resolved; - if (requested.length) result.metadata.build.featurePolicy = "explicit"; - result.metadata.build.env = { ...BUILD_ENV }; - result.metadata.build.ambientEnv = recordAmbientEnv(); - // Disallow package-manager network fallback in a prepared run. - const buildStart = performance.now(); - await execute( - "./build", - [ - "frontend", - "--offline", - "--locked", - ...(requested.length ? ["--features", requested.join(",")] : []), - ], - { - cwd: path.join(root, "render-wasm"), - signal: abort.signal, - env: { ...process.env, ...BUILD_ENV, VERSION: runId }, - }, - ); - result.metadata.build.durationMs = performance.now() - buildStart; + if (options.noBuild) { + const warning = + "Build skipped (--no-build): the measured artifact was not rebuilt from this revision."; + result.warnings.push(warning); + console.warn(warning); + } else { + const requested = [...(options.features ?? [])].sort(); + const resolved = validateFeatureSelection( + readRendererManifest(), + requested, + options.command, + ); + result.metadata.build.features = requested; + result.metadata.build.defaultFeatures = resolved; + if (requested.length) result.metadata.build.featurePolicy = "explicit"; + result.metadata.build.env = { ...BUILD_ENV }; + result.metadata.build.ambientEnv = recordAmbientEnv(); + // Disallow package-manager network fallback in a prepared run. + const buildStart = performance.now(); + await execute( + "./build", + [ + "frontend", + "--offline", + "--locked", + ...(requested.length ? ["--features", requested.join(",")] : []), + ], + { + cwd: path.join(root, "render-wasm"), + signal: abort.signal, + env: { ...process.env, ...BUILD_ENV, VERSION: runId }, + }, + ); + result.metadata.build.durationMs = performance.now() - buildStart; + } const { copyWasmPlayground } = await import("./_helpers.js"); const previousCwd = process.cwd(); try { @@ -902,7 +927,7 @@ export async function main(argv = process.argv.slice(2)) { const options = readOptions(argv); if (options.help) { console.log( - "Renderer benchmarks\n node scripts/renderer-benchmarks.js run [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--features a,b] [--output FILE]\n node scripts/renderer-benchmarks.js ab --features a,b [--max-features N] [--confirm] [--dry-run] [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--output DIR]\n node scripts/renderer-benchmarks.js compare BASELINE CANDIDATE [--diagnostic] [--output FILE]\nRun requires a prepared development container and an existing static server on port 3000. Defaults: 3 warmups, 10 measured attempts, 30s timeout; provisional pending calibration.\n" + + "Renderer benchmarks\n node scripts/renderer-benchmarks.js run [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--features a,b] [--output FILE] [--no-build]\n node scripts/renderer-benchmarks.js ab --features a,b [--max-features N] [--confirm] [--dry-run] [--filter ID] [--seed N] [--warmups N] [--repetitions N] [--timeout-ms N] [--headed] [--base-url URL] [--output DIR]\n node scripts/renderer-benchmarks.js compare BASELINE CANDIDATE [--diagnostic] [--output FILE]\nRun requires a prepared development container and an existing static server on port 3000. --no-build reuses the existing renderer artifact instead of building and cannot be combined with --features or ab. Defaults: 3 warmups, 10 measured attempts, 30s timeout; provisional pending calibration.\n" + LIMITS, ); return 0; diff --git a/frontend/scripts/renderer-benchmarks/tests/runner.test.js b/frontend/scripts/renderer-benchmarks/tests/runner.test.js index cdc8293d88..86320f30bc 100644 --- a/frontend/scripts/renderer-benchmarks/tests/runner.test.js +++ b/frontend/scripts/renderer-benchmarks/tests/runner.test.js @@ -137,6 +137,18 @@ test("CLI parses feature selection for run and ab", () => { } }); +test("--no-build skips the renderer build only for run", () => { + assert.equal(readOptions(["run"]).noBuild, false); + assert.equal(readOptions(["run", "--no-build"]).noBuild, true); + for (const args of [ + ["ab", "--features", "branch-b", "--no-build"], + ["compare", "a.json", "b.json", "--no-build"], + ["run", "--no-build", "--features", "branch-b"], + ]) { + assert.throws(() => readOptions(args), /--no-build/); + } +}); + test("feature selection validates against the renderer manifest", () => { const definitions = { default: [], "branch-a": [], "branch-b": [] }; assert.deepEqual(validateFeatureSelection(definitions, ["branch-b"], "ab"), [