From 756541939aa9a6027b76acb0526ef3dec7c1611a Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Thu, 12 Nov 2026 11:02:34 +0100 Subject: [PATCH] :wrench: Add renderer benchmark runner (#12756) Add a CLI under frontend/performance/benches/render_wasm that builds the renderer, runs seeded scenes (rects, plus, paths, masks) in headless Chromium and saves per-case timings. - build, run: prepare or reuse builds; measure cases with warmups, repetitions, deadlines - ab: build and compare Cargo feature configurations - compare, merge: median changes with bootstrap CIs between saved runs, guarded by scene, build and settings checks - purge: delete managed builds, runs or comparisons - GPU runs through `run-devenv --gpu` Lint, format and pnpm scripts now cover performance/ and its tests. AI-assisted-by: gpt-6.1-sol, deepseek-flash, muse-spark, glm-5.3, gpt6-astra, claude-5.5-sonnet, claude-haiku-5.5 --- .clj-kondo/config.edn | 15 +- .clj-kondo/hooks/benches.clj | 77 + .clj-kondo/hooks/export.clj | 6 + .cljfmt.edn | 4 +- frontend/deps.edn | 10 +- frontend/package.json | 11 +- .../performance/benches/render_wasm/README.md | 1514 ++++++++++++ .../benches/render_wasm/browser/bridge.cljs | 1255 ++++++++++ .../benches/render_wasm/browser/index.html | 33 + .../benches/render_wasm/builder.cljc | 512 ++++ .../benches/render_wasm/cases.cljs | 128 + .../benches/render_wasm/codec.cljs | 84 + .../benches/render_wasm/declarations.cljc | 373 +++ .../benches/render_wasm/errors.cljc | 500 ++++ .../benches/render_wasm/features/plan.cljs | 76 + .../render_wasm/features/selection.cljs | 29 + .../benches/render_wasm/fingerprint.cljs | 77 + .../benches/render_wasm/identity.cljc | 36 + .../benches/render_wasm/measurement.cljs | 85 + .../render_wasm/node/catalog/entry.cljs | 59 + .../render_wasm/node/catalog/loader.cljs | 47 + .../render_wasm/node/cli/commands/ab.cljs | 55 + .../render_wasm/node/cli/commands/build.cljs | 41 + .../node/cli/commands/compare.cljs | 50 + .../render_wasm/node/cli/commands/merge.cljs | 31 + .../render_wasm/node/cli/commands/purge.cljs | 32 + .../render_wasm/node/cli/commands/run.cljs | 69 + .../benches/render_wasm/node/cli/console.cljs | 40 + .../render_wasm/node/cli/deferred.cljs | 74 + .../render_wasm/node/cli/execution.cljs | 258 ++ .../benches/render_wasm/node/cli/help.cljs | 25 + .../benches/render_wasm/node/cli/main.cljs | 60 + .../render_wasm/node/cli/registry.cljs | 68 + .../benches/render_wasm/node/cli/request.cljs | 41 + .../benches/render_wasm/node/cli/resolve.cljs | 106 + .../benches/render_wasm/node/cli/shared.cljs | 109 + .../benches/render_wasm/node/files.cljs | 51 + .../benches/render_wasm/node/log.cljs | 181 ++ .../benches/render_wasm/node/outcome.cljs | 57 + .../benches/render_wasm/node/process.cljs | 604 +++++ .../render_wasm/node/report/compare.cljs | 46 + .../render_wasm/node/report/merge.cljs | 28 + .../render_wasm/node/runner/browser.cljs | 385 +++ .../render_wasm/node/runner/build.cljs | 216 ++ .../node/runner/feature_comparison.cljs | 880 +++++++ .../benches/render_wasm/node/runner/run.cljs | 916 +++++++ .../render_wasm/node/runner/server.cljs | 309 +++ .../benches/render_wasm/node/settings.cljs | 14 + .../render_wasm/node/storage/artifacts.cljs | 403 +++ .../render_wasm/node/storage/runs.cljs | 78 + .../benches/render_wasm/node/time.cljs | 12 + .../benches/render_wasm/quantities.cljs | 25 + .../benches/render_wasm/random.cljs | 73 + .../benches/render_wasm/report/analysis.cljs | 241 ++ .../benches/render_wasm/report/compare.cljs | 281 +++ .../benches/render_wasm/report/format.cljs | 357 +++ .../benches/render_wasm/report/summarize.cljs | 238 ++ .../benches/render_wasm/result.cljs | 696 ++++++ .../benches/render_wasm/runtime/camera.cljs | 151 ++ .../benches/render_wasm/runtime/commands.clj | 90 + .../benches/render_wasm/runtime/commands.cljs | 241 ++ .../benches/render_wasm/runtime/protocol.clj | 30 + .../benches/render_wasm/runtime/protocol.cljs | 424 ++++ .../benches/render_wasm/scenes/masks.cljs | 59 + .../benches/render_wasm/scenes/paths.cljs | 78 + .../benches/render_wasm/scenes/plus.cljs | 68 + .../benches/render_wasm/scenes/rects.cljs | 118 + .../benches/render_wasm/snapshot.cljs | 203 ++ frontend/shadow-cljs.edn | 90 +- .../render_wasm/serialization_test.cljs | 9 - .../render_wasm/artifacts_test.cljs | 566 +++++ .../render_wasm/boolean_test.cljs | 214 ++ .../render_wasm/browser_test.cljs | 328 +++ .../render_wasm/build_test.cljs | 306 +++ .../render_wasm/builder_test.cljs | 288 +++ .../render_wasm/cli_integration.cjs | 2197 +++++++++++++++++ .../render_wasm/cli_test.cljs | 278 +++ .../render_wasm/commands_test.cljs | 520 ++++ .../render_wasm/compare_test.cljs | 278 +++ .../render_wasm/containers_test.cljs | 251 ++ .../render_wasm/contracts_test.cljs | 445 ++++ .../render_wasm/errors_test.cljs | 248 ++ .../render_wasm/execution_test.cljs | 346 +++ .../render_wasm/feature_comparison_test.cljs | 1363 ++++++++++ .../render_wasm/fingerprint_test.cljs | 100 + .../render_wasm/geometry_scenes_test.cljs | 86 + .../render_wasm/lifecycle_test.cljs | 1142 +++++++++ .../render_wasm/log_test.cljs | 161 ++ .../render_wasm/merge_test.cljs | 219 ++ .../render_wasm/outcome_test.cljs | 47 + .../render_wasm/process_test.cljs | 615 +++++ .../render_wasm/protocol_test.cljs | 417 ++++ .../render_wasm/report_test.cljs | 624 +++++ .../render_wasm/request_test.cljs | 208 ++ .../render_wasm/run_test.cljs | 1483 +++++++++++ .../performance_tests/render_wasm/runner.cljs | 229 ++ .../render_wasm/runs_test.cljs | 165 ++ .../render_wasm/scene_snapshot_test.cljs | 316 +++ .../render_wasm/server_test.cljs | 233 ++ .../render_wasm/test_helpers.cljs | 49 + render-wasm/scripts/cargo-build.cjs | 33 + render-wasm/scripts/cargo-build.test.cjs | 125 +- 102 files changed, 27497 insertions(+), 25 deletions(-) create mode 100644 .clj-kondo/hooks/benches.clj create mode 100644 frontend/performance/benches/render_wasm/README.md create mode 100644 frontend/performance/benches/render_wasm/browser/bridge.cljs create mode 100644 frontend/performance/benches/render_wasm/browser/index.html create mode 100644 frontend/performance/benches/render_wasm/builder.cljc create mode 100644 frontend/performance/benches/render_wasm/cases.cljs create mode 100644 frontend/performance/benches/render_wasm/codec.cljs create mode 100644 frontend/performance/benches/render_wasm/declarations.cljc create mode 100644 frontend/performance/benches/render_wasm/errors.cljc create mode 100644 frontend/performance/benches/render_wasm/features/plan.cljs create mode 100644 frontend/performance/benches/render_wasm/features/selection.cljs create mode 100644 frontend/performance/benches/render_wasm/fingerprint.cljs create mode 100644 frontend/performance/benches/render_wasm/identity.cljc create mode 100644 frontend/performance/benches/render_wasm/measurement.cljs create mode 100644 frontend/performance/benches/render_wasm/node/catalog/entry.cljs create mode 100644 frontend/performance/benches/render_wasm/node/catalog/loader.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/commands/ab.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/commands/build.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/commands/compare.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/commands/merge.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/commands/purge.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/commands/run.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/console.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/deferred.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/execution.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/help.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/main.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/registry.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/request.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/resolve.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/shared.cljs create mode 100644 frontend/performance/benches/render_wasm/node/files.cljs create mode 100644 frontend/performance/benches/render_wasm/node/log.cljs create mode 100644 frontend/performance/benches/render_wasm/node/outcome.cljs create mode 100644 frontend/performance/benches/render_wasm/node/process.cljs create mode 100644 frontend/performance/benches/render_wasm/node/report/compare.cljs create mode 100644 frontend/performance/benches/render_wasm/node/report/merge.cljs create mode 100644 frontend/performance/benches/render_wasm/node/runner/browser.cljs create mode 100644 frontend/performance/benches/render_wasm/node/runner/build.cljs create mode 100644 frontend/performance/benches/render_wasm/node/runner/feature_comparison.cljs create mode 100644 frontend/performance/benches/render_wasm/node/runner/run.cljs create mode 100644 frontend/performance/benches/render_wasm/node/runner/server.cljs create mode 100644 frontend/performance/benches/render_wasm/node/settings.cljs create mode 100644 frontend/performance/benches/render_wasm/node/storage/artifacts.cljs create mode 100644 frontend/performance/benches/render_wasm/node/storage/runs.cljs create mode 100644 frontend/performance/benches/render_wasm/node/time.cljs create mode 100644 frontend/performance/benches/render_wasm/quantities.cljs create mode 100644 frontend/performance/benches/render_wasm/random.cljs create mode 100644 frontend/performance/benches/render_wasm/report/analysis.cljs create mode 100644 frontend/performance/benches/render_wasm/report/compare.cljs create mode 100644 frontend/performance/benches/render_wasm/report/format.cljs create mode 100644 frontend/performance/benches/render_wasm/report/summarize.cljs create mode 100644 frontend/performance/benches/render_wasm/result.cljs create mode 100644 frontend/performance/benches/render_wasm/runtime/camera.cljs create mode 100644 frontend/performance/benches/render_wasm/runtime/commands.clj create mode 100644 frontend/performance/benches/render_wasm/runtime/commands.cljs create mode 100644 frontend/performance/benches/render_wasm/runtime/protocol.clj create mode 100644 frontend/performance/benches/render_wasm/runtime/protocol.cljs create mode 100644 frontend/performance/benches/render_wasm/scenes/masks.cljs create mode 100644 frontend/performance/benches/render_wasm/scenes/paths.cljs create mode 100644 frontend/performance/benches/render_wasm/scenes/plus.cljs create mode 100644 frontend/performance/benches/render_wasm/scenes/rects.cljs create mode 100644 frontend/performance/benches/render_wasm/snapshot.cljs create mode 100644 frontend/test/performance_tests/render_wasm/artifacts_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/boolean_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/browser_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/build_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/builder_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/cli_integration.cjs create mode 100644 frontend/test/performance_tests/render_wasm/cli_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/commands_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/compare_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/containers_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/contracts_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/errors_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/execution_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/fingerprint_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/lifecycle_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/log_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/merge_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/outcome_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/process_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/protocol_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/report_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/request_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/run_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/runner.cljs create mode 100644 frontend/test/performance_tests/render_wasm/runs_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/server_test.cljs create mode 100644 frontend/test/performance_tests/render_wasm/test_helpers.cljs diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 0caf92fbc4..2a7f6ea61b 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -9,7 +9,8 @@ app.common.data.macros/get-in clojure.core/get-in app.common.data.macros/with-open clojure.core/with-open app.common.data.macros/select-keys clojure.core/select-keys - app.common.logging/with-context clojure.core/do} + app.common.logging/with-context clojure.core/do + benches.render-wasm.errors/attempt clojure.core/do} :hooks {:analyze-call @@ -24,7 +25,11 @@ shadow.lazy/loadable hooks.export/rumext-lazycomponent app.util.i18n/tr hooks.i18n/tr-dynamic app.common.i18n/tr hooks.i18n/tr-dynamic - }} + benches.render-wasm.declarations/defscene hooks.benches/defscene + benches.render-wasm.declarations/defcase hooks.benches/defcase + benches.render-wasm.runtime.commands/defcommand hooks.benches/defcommand + benches.render-wasm.runtime.protocol/resolve-or-reject hooks.benches/resolve-or-reject + benches.render-wasm.runtime.protocol/resolve-or-reject-with hooks.benches/resolve-or-reject-with}} :output {:exclude-files @@ -36,7 +41,7 @@ "^(?:backend|frontend|exporter|common)/deps.edn" "^(?:backend|frontend|exporter|common)/scripts/" "^(?:backend|frontend|exporter|common)/dev/" - "^(?:backend|frontend|exporter|common)/test/"] + "^(?:backend|frontend|exporter|common)/test/(?!performance_tests/)"] :linter-name true} @@ -101,6 +106,4 @@ :unused-binding {:exclude-destructured-as true - :exclude-destructured-keys-in-fn-args false - } - }} + :exclude-destructured-keys-in-fn-args false}}} diff --git a/.clj-kondo/hooks/benches.clj b/.clj-kondo/hooks/benches.clj new file mode 100644 index 0000000000..5a100be0d8 --- /dev/null +++ b/.clj-kondo/hooks/benches.clj @@ -0,0 +1,77 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns hooks.benches + (:require [clj-kondo.hooks-api :as api])) + +(defn defscene + "Analyzes scene options and its compiled builder body." + [{:keys [node]}] + (let [[_ _ opts argv & body] (:children node)] + (if (and argv (api/vector-node? argv)) + {:node (api/list-node [(api/token-node (quote do)) opts + (api/list-node (into [(api/token-node (quote fn)) argv] body))])} + {:node node}))) + +(defn defcase + "Analyzes case options and its compiled operation body." + [{:keys [node]}] + (let [[_ _ opts argv & body] (:children node)] + (if (and (seq body) argv (api/vector-node? argv)) + {:node (api/list-node [(api/token-node (quote do)) opts + (api/list-node (into [(api/token-node (quote fn)) argv] body))])} + {:node node}))) + +(defn defcommand + "Analyzes command options and both generated functions." + [{:keys [node]}] + (let [[_ command & forms] (:children node) + [doc forms] (if (string? (api/sexpr (first forms))) + [(first forms) (next forms)] + [nil forms]) + [options forms] (if (api/map-node? (first forms)) + [(first forms) (next forms)] + [nil forms]) + [argv & body] forms + [conditions body] (if (and (api/map-node? (first body)) (next body)) + [(first body) (next body)] + [nil body]) + handlers (:async (some-> options api/sexpr)) + core (api/list-node + (into [(api/token-node 'fn) + (api/vector-node (mapv api/token-node handlers))] + body)) + starred (api/token-node (symbol (str (api/sexpr command) "*"))) + measured (get (some-> options api/sexpr) :measured-variant true) + definition (api/list-node + (into [(api/token-node 'defn) command] + (concat (when doc [doc]) [argv] + (when conditions [conditions]) [core])))] + {:node (api/list-node + (into [(api/token-node 'do)] + (concat (when options [(api/coerce (dissoc (api/sexpr options) :async))]) + [definition] + (when measured + [(api/list-node [(api/token-node 'def) + starred command])]))))})) + +(defn resolve-or-reject + "Analyzes the body of a Promise-returning form." + [{:keys [node]}] + (let [[_ & body] (:children node)] + {:node (api/list-node (into [(api/token-node (quote do))] body))})) + +(defn resolve-or-reject-with + "Analyzes the cause handler and body. Binds `cause` in the handler only." + [{:keys [node]}] + (let [[_ cause on-error & body] (:children node)] + {:node (api/list-node + (into [(api/token-node (quote do)) + (api/list-node + [(api/token-node (quote let)) + (api/vector-node [cause (api/token-node nil)]) + on-error])] + body))})) diff --git a/.clj-kondo/hooks/export.clj b/.clj-kondo/hooks/export.clj index 50e617de55..f4c0ebcf0c 100644 --- a/.clj-kondo/hooks/export.clj +++ b/.clj-kondo/hooks/export.clj @@ -1,3 +1,9 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + (ns hooks.export (:require [clj-kondo.hooks-api :as api])) diff --git a/.cljfmt.edn b/.cljfmt.edn index ad56b8c70d..2b988af01d 100644 --- a/.cljfmt.edn +++ b/.cljfmt.edn @@ -10,5 +10,7 @@ :extra-indents {rumext.v2/fnc [[:inner 0]] cljs.test/async [[:inner 0]] promesa.exec/thread [[:inner 0]] - specify! [[:inner 0] [:inner 1]]} + specify! [[:inner 0] [:inner 1]] + benches.render-wasm.runtime.commands/defcommand [[:inner 0]] + benches.render-wasm.runtime.protocol/resolve-or-reject-with [[:inner 0]]} } diff --git a/frontend/deps.edn b/frontend/deps.edn index ab7197c661..3de812cb70 100644 --- a/frontend/deps.edn +++ b/frontend/deps.edn @@ -1,4 +1,4 @@ -{:paths ["src" "vendor" "resources" "test"] +{:paths ["src" "vendor" "resources" "test" "performance"] :deps {;; Carries `app.common.render-wasm.*`, shared with the headless exporter. penpot/common @@ -37,8 +37,7 @@ instaparse/instaparse {:mvn/version "1.5.0"} garden/garden {:git/url "https://github.com/noprompt/garden" - :git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"} - } + :git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"}} :aliases {:outdated @@ -67,4 +66,7 @@ "-Dpenpot.wasm.profile-marks=true" "-XX:+UnlockExperimentalVMOptions" "-XX:CompileCommand=blackhole,criterium.blackhole.Blackhole::consume"]} - }} + + :renderer-bench + {:main-opts ["-m" "shadow.cljs.devtools.cli"] + :jvm-opts ["-Dpenpot.wasm.profile-marks=false"]}}} diff --git a/frontend/package.json b/frontend/package.json index d5b917acd0..5b57c35ed2 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,22 +20,27 @@ "build:storybook:assets": "node ./scripts/build-storybook-assets.js", "build:storybook:cljs": "clojure -M:dev:shadow-cljs compile storybook", "build:wasm": "../render-wasm/build frontend", + "build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-commands bench-render-wasm-runner", + "render-bench": "sh -c 'w=; [ \"${PENPOT_GPU:-0}\" = 1 ] && w=\"xvfb-run -a\"; exec $w node target/performance/render-wasm/runner.cjs \"$@\"' render-bench", "build:app:libs": "node ./scripts/build-libs.js", "build:app:main": "clojure -M:dev:shadow-cljs release main worker", "build:app:worker": "clojure -M:dev:shadow-cljs release worker", "build:app": "pnpm run clear:shadow-cache && pnpm run build:app:main && pnpm run build:app:libs", - "check-fmt:clj": "cljfmt check --parallel=true src/ test/", + "check-fmt:clj": "cljfmt check --parallel=true src/ performance/ test/", "check-fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js", "check-fmt:scss": "prettier -c resources/styles -c src/**/*.scss", - "fmt:clj": "cljfmt fix --parallel=true src/ test/", + "fmt:clj": "cljfmt fix --parallel=true src/ performance/ test/", "fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js -w", "fmt:scss": "prettier -c resources/styles -c src/**/*.scss -w", - "lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/", + "lint:clj": "clj-kondo --fail-level error --config-dir ../.clj-kondo --lint ../common/src src/ performance/ test/performance_tests/render_wasm/", "lint:js": "exit 0", + "lint:renderer-benchmarks": "clj-kondo --config-dir ../.clj-kondo --lint performance/ test/performance_tests/render_wasm/", "lint:scss": "pnpm exec stylelint '{src,resources}/**/*.scss'", "build:test": "pnpm run build:wasm && clojure -M:dev:shadow-cljs compile test", + "build:renderer-benchmarks:tests": "clojure -M:dev:shadow-cljs compile performance-tests", "test": "pnpm run build:test && node target/tests/test.js", "test:quiet": "node ./scripts/test-quiet.js", + "test:renderer-benchmarks": "pnpm run build:renderer-benchmarks:tests && node target/performance-tests/test.js", "test:storybook": "vitest run --project=storybook", "watch:test": "mkdir -p target/tests && concurrently \"clojure -M:dev:shadow-cljs watch test\" \"nodemon -C -d 2 -w target/tests --exec 'node target/tests/test.js'\"", "test:e2e": "playwright test --project default", diff --git a/frontend/performance/benches/render_wasm/README.md b/frontend/performance/benches/render_wasm/README.md new file mode 100644 index 0000000000..2aca3fc7a1 --- /dev/null +++ b/frontend/performance/benches/render_wasm/README.md @@ -0,0 +1,1514 @@ +# WASM renderer benchmarks + +The renderer benchmark suite measures the Penpot WebAssembly (WASM) renderer in +Chromium. Scenes describe shapes; cases describe the work to perform on them. +The suite records setup, rendering, pan and zoom durations and compares runs +with different renderer builds. [How to run](#how-to-run) gives the commands. + + +## Contents + +- [Main concepts](#main-concepts) +- [Architecture](#architecture) +- [Workloads](#workloads) +- [Renderer commands](#renderer-commands) +- [Measurement](#measurement) +- [Results and reports](#results-and-reports) +- [A/B feature experiments](#ab-feature-experiments) +- [How to run](#how-to-run) +- [Unit tests](#unit-tests) +- [Scope and limitations](#scope-and-limitations) +- [Repository map](#repository-map) +- [Further reading](#further-reading) + +## Main concepts + +### Workload + +- **Renderer**: the Rust Skia renderer compiled to WebAssembly. It owns shape + storage, tiles, caches, and the WebGL2 graphics state for one canvas. +- **Module**: one instantiated copy of the renderer. The browser creates a + module per prepared page and releases it with the page. +- **Graphics context**: the WebGL2 context and renderer state bound to one + canvas. A benchmark _case_ can use either a fresh or a reused context. +- **Scene**: a recipe that produces a starting set of shapes. A scene has an + id, a closed parameter schema, and a builder function. This function is given + a random seed and returns a _snapshot_. Scenes can be **procedural**, or + **captured**. The former build shapes from code, parameters, and a random + seed, whereas the latter render a page exported from Penpot. +- **Scene snapshot**: the concrete shapes a run renders. This is only static + data, camera movement and other interactions belong to a **case**. +- **Master seed**: the initial seed for the run, supplied on the command line. + It defaults to 42 and accepts a 32-bit unsigned integer. +- **Seed**: a 32-bit integer derived from the master seed, scene id, and + canonical parameters. The seed makes a scene reproducible. Load, pan, and + zoom cases for the same scene and parameters share one seed. +- **Scene builder**: the construction API used inside a scene. It provides basic + shapes (`rect`, `path`, `circle`, `frame`, `group`, and `bool`) constructors, + and a (hopefully) self-explanatory API. +- **Shape reference**: a label that a scene attaches to a shape during + construction. A _case_ body uses references to find shapes it needs. +- **Case**: a scene plus its execution settings: parameters, base view, context + policy, optional preparation callback, optional reset function, and an + executable body. A case is the unit that is measured. +- **Case body**: the function declared by `defcase` that performs a case's work + during one attempt. It receives a command context with the case data, scene + snapshot, current view, and hooks. It can call `render-sync`, `load`, `pan`, + `zoom`, or `restore` and their measured variants. +- **View**: the camera's zoom `:scale` and offsets `:x` and `:y`. + Optional `:viewport` settings give the canvas width, height, and device pixel + ratio, the number of drawing-buffer pixels per CSS pixel along each dimension. +- **Hook**: a function supplied through `:hooks` for clock readings, + scheduling, view updates, rendering, or checking whether work may continue. +- **Case descriptor**: What gets sent to the browser. It contains the case id, + scene, scene description, derived seed, parameters, view, context policy, and + preparation settings. Functions stay in the local registry and are removed + from the descriptor; the browser resolves them by case id. +- **Catalogue**: the compiled list of case descriptors and scene declarations + for one build. Node reads the catalogue to select cases. The browser executor + contains the same declarations and looks up the case body by case id. The + current catalogue declares 14 cases. +- **Fingerprint**: SHA-256 digest of the validated starting snapshot. A + comparison checks it to establish equal starting scene content. + +### Execution + +- **Build**: one compiled renderer plus the matching catalogue and browser + executor. A build id ties the three outputs together, and the runner rejects + a mismatch. +- **Retained build**: a ready build that the suite stores under the managed + build directory for reuse in later runs. +- **Fresh case**: a case with `:context :fresh`. Each attempt gets a new page, + module, graphics context, and upload. +- **Warm case**: a case with `:context :reuse`. One page, module, graphics + context, snapshot, and upload serve all attempts of the case. +- **Preparation**: setup for one graphics context. It covers module + instantiation, WebGL2 context creation, scene build, snapshot validation, + fingerprinting, upload-order derivation, and upload. It then runs the optional + preparation callback and drains to `Full` unless the case declares + `:preparation {:render :none}`. One page owns one preparation. +- **Attempt**: one execution of a case against a preparation. It records an + `:attempt-id`, one outcome, main and optional reset phase blocks, and failure + or partial evidence. A failed attempt still consumes its allocated slot. +- **Warmup**: an attempt that runs the case before measured attempts to warm + caches and stabilize the page. The run records warmups but excludes them from + statistics. +- **Repetition**: a measured attempt. A case has a fixed number of repetitions + chosen before the run. +- **Run**: a set of attempts over selected cases. A run owns the browser + processes it starts, fixes warmup and repetition counts, records failures in + chronological order, and performs no hidden retries. +- **Run record**: the saved evidence for one run: plan, build and environment + provenance, quantity descriptions, preparations, attempts with their phase + histories, failures, terminal case outcomes, remaining slots, and termination. + The runner stores it as a data file validated by a schema. +- **Bridge**: the object that runs in the browser, receives plain requests from + Node, owns the page's renderer context, runs preparations and attempts, and + returns plain results. +- **Runner**: the Node command line interface that builds or reuses a build, + selects cases, serves the build, launches Chromium, runs attempts, saves + records, and prints reports. +- **Browser executor**: the compiled page and JavaScript that execute + preparations and case bodies. Its bridge exposes the requests that Node can + send. +- **Measurement**: a numeric result for a quantity such as elapsed duration. + The suite measures module start, graphics start, upload, first render, and + camera movement separately. +- **Metric**: the name of a measured quantity. A command event records its raw + numeric values in a vector. A saved run supplies a `:unit` and an optional + `:description` for each metric type. It stores no statistic or reduction. +- **Metric key**: `[phase scope metric-type]`. Scope is the command that recorded + the metric or `:global` for every command with that type in the phase. +- **Observation**: one eligible phase execution's sum for one metric key. Inner + calls, slices, and loop repetitions stay within that execution and do not add + observations. Warmups and failed attempts supply none. +- **Slice**: a trace entry for one renderer call: timestamp, flags, frame type, + and duration. Slices describe the work within an attempt. +- **Trace**: details of the work, including individual renderer calls, + cached previews, requested waits, and observed durations. +- **Clock reading**: a time in milliseconds from the clock's origin. Subtracting + two readings from that clock gives an elapsed duration. +- **Endpoint**: the clock reading at the end of the measured work. The renderer + endpoint is the reading immediately after a renderer call returns. +- **Render drain**: repeated renderer calls until one returns `Full`. Partial + completion schedules another call on the next animation frame. +- **Cached preview**: one render from cached tiles after a view update. +- **Camera session**: camera start, view updates with cached previews, a settling + wait, camera end, and the final render drain. +- **Active interval**: the part of a camera session from camera start through + the last cached preview and its guard check, or an explicit pause that follows + the preview. +- **Settling wait**: the requested pause after the active interval and before + ending the interaction. The observed wait includes scheduling delays. +- **Camera finalization**: the interval from immediately before ending the + interaction through the renderer call that returns `Full`. +- **Frame type**: the completion signal returned by one renderer call. `None` + means no frame; `Partial` means visible work remains; `Full` means the + renderer finished the current request; `ViewportReady` means the visible + viewport was submitted while work outside the viewport may remain. +- **Tile**: a unit of renderer work with a fixed size. A logical tile covers + 512 document units. The renderer caches rendered tiles in a texture atlas. +- **Atlas**: a GPU texture that stores many tiles or a document-space preview. + Atlases let warm pan and zoom present cached pixels before new tiles finish. + +### Command execution + +- **Renderer command**: a function that accepts a command context or its Promise + and returns a Promise of the next context. The base function uses the declared + name, such as `pan`. The measured function adds `*`, such as `pan*`, and records + an event for its work. +- **Command context**: the map with keys `:case`, `:scene`, `:view`, and `:hooks`. + It carries the case descriptor, prepared scene snapshot, current view, and + hooks. +- **Command body**: the work declared by `defcommand`. A synchronous body + returns the next context. An asynchronous body completes through callbacks + named by its declaration. +- **Composite command**: a command that combines calls to other commands. + A declaration with `:measured-variant false` defines its base function and + collects events from the measured commands it calls. +- **Command execution**: one call to `commands/execute`, which runs a function + from one command context to the next and records measured calls in completion + order. +- **Event**: the record for one measured command call. It contains the command + name, vectors of raw metric values, and an optional trace. +- **Guard**: the `:check` hook. A return authorizes work; an error rejects it. + The renderer owner is the execution authorized to use the renderer. The guard + can report cancellation, deadline expiry, context loss, or replacement of + that owner. +- **Completion callback**: `complete` or `fail` in an asynchronous command body. + The first call determines the outcome: `complete` supplies the next context + and optional measurements; `fail` supplies a cause. + +### Results + +- **Report**: the statistics and tables computed from one or more run records. + Reports run in Node or the browser and do not load the renderer. +- **Comparison**: an offline analysis of two saved run records. It pairs cases, + checks compatibility, and reports changes for each metric key with intervals. +- **A/B comparison**: a comparison of two runs that differ by one renderer + feature. The feature is off in one run and on in the other. +- **Permission**: In principle, only run records with equal builds and equal + scenes are comparable. Because one sometimes needs to override this strict + requirement, permissions allow build environment or scene differences in a + comparison. +- **Termination**: the final state of a run: completed, failed, incomplete, or + interrupted. + +### Feature experiments + +- **Feature**: a switch at compile time in the renderer's Cargo manifest. +- **Feature configuration**: the set of Cargo features enabled for one renderer + build. +- **Feature comparison**: all on and off combinations of the requested features. + $N$ requested features produce $2^N$ configurations. +- **Edge**: two configurations that differ in exactly one feature, compared from + the configuration with the feature off to the configuration with the feature + on. A comparison with $N$ features has $N 2^{N-1}$ edges. +- **Edge comparison**: the comparison record for one edge, stored with the + feature comparison. +- **Feature-comparison record**: the saved plan, configurations, runs, edges, + failures, and termination for one A/B experiment. + +## Architecture + +The suite has three components: runner, browser, and reports. They are all driven via the same CLI. + +### Node runner + +The Node runner owns everything outside the page and +controls the experiment. + +- it prepares or loads a build; +- it reads the compiled catalogue and selects cases; +- it serves the selected build from a server on `127.0.0.1`; +- it launches Chromium and creates pages; +- it schedules warmups and measured attempts; +- it records preparations, attempts, failures, and termination; +- it writes the run record as work progresses; +- it prints a final report. + +### Browser bridge + +The bridge owns everything inside the page that runs the benchmark: + +- it validates a request before assigning the page to that work; +- it imports the renderer module and creates the canvas and WebGL2 context; +- it builds, validates, fingerprints, orders, and uploads the scene; +- it carries each phase's returned context into the next phase or attempt; +- it runs the case body and records the measurements its commands select; +- it returns plain data and disposes the context when asked. + +### Reporting + +This code performs the analysis offline: + +- it normalizes run records into summaries; +- it computes statistics and intervals for each metric; +- it pairs saved cases and checks compatibility; +- it formats run summaries, comparisons, and feature-comparison edge reports; +- it imports no browser, server, or renderer code. + +### End-to-end path + +```mermaid +sequenceDiagram + participant N as Node runner + participant S as Owned server + participant B as Browser bridge + participant W as WASM renderer + + N->>N: select cases from the catalogue + N->>S: serve the selected build + N->>N: launch Chromium + N->>B: open a page + B->>W: instantiate the module + B->>W: create the WebGL2 context + B->>B: build, validate, fingerprint the snapshot + B->>W: upload shapes in tree order + B->>B: run preparation callback + B->>W: render Full unless disabled + B-->>N: preparation evidence and raw phase block + N->>N: persist the run record + N->>B: run main and reset + B-->>N: raw main and reset phase blocks + N->>N: persist the run record +``` + +The Node runner saves the initial plan before it starts a browser, and updates +the run record after each preparation and attempt, and at termination. Reports +use these saved records. + +### Builds and catalogues + +A build has three matching outputs: + +- the renderer module; +- the case catalogue compiled for Node; +- the browser executor and its page. + +Build preparation compiles the renderer in release mode with Cargo, then compiles +the catalogue and browser executor with the same build id. The runner checks +the id before case collection and when a page starts, and rejects a mismatch. + +An ordinary measurement build uses the renderer's default Cargo feature set. A +feature experiment may enable selected functional features. Both kinds of +measurement build should exclude diagnostic features such as `stats`, `profile`, +`profile-macros`, and `profile-raf`. The build checks that we are in release +mode, the optimization level, and disabled profiling marks. + +The build record captures a fixed list of environment values supplied to Cargo, +including compiler, target, and Emscripten settings. These values provide +provenance for interpreting a run. Reproducing a build also requires its other +environment settings and dependencies. + +Builds are saved under `target/performance/render-wasm/builds//`. The +manifest records the build id, status, functional settings, Git provenance, and +build facts. An experiment run can select a retained build with `--build `, +and `run` prepares a new build when it omits `--build`. + +`build` and a run that prepares its build accept `--features`, +`--[no-]default-features`, and `--build-timeout-ms`. A run with `--build` reuses +the retained build and rejects those preparation options. + +Run one build preparation at a time in a checkout and keep renderer watchers +stopped until it finishes. The renderer build lock covers Cargo compilation +and copying the renderer files. The build script releases that lock before +ClojureScript compilation. + +### Browser host + +The runner serves the selected build directory from `127.0.0.1` on an allocated +port. + +The runner launches headless Chromium with Playwright and owns the browser +process it starts. Cases run serially. A fresh case gets a new page and context +for each preparation. A warm case keeps one page and context for all of its +attempts. + +Before cases run, `--startup-timeout-ms` bounds Git provenance, server startup, +and the initial browser launch. Each case then gets one absolute `--timeout-ms` +deadline. The deadline covers the case's page setup, preparation, warmups, +repetitions, reset, and screenshot diagnostics. It does not restart for a phase, +attempt, or browser replacement. Work on a reused preparation keeps its original +case deadline; a request timeout can only tighten that deadline. A browser +launched after a failed case uses the next case's remaining deadline. + +When a page stalls past its deadline, Node stops the browser process and records +the failure. The browser bridge also checks for context loss, expired deadlines, +and work associated with a superseded renderer owner before and after +asynchronous work. A browser thread occupied by synchronous work relies on the +Node deadline to stop it. The runner persists each delivered preparation and +attempt. It does not stream browser-local records and cannot recover in-flight +data when a blocked browser cannot return; records delivered before the block +remain saved. + +Clean-up closes the page, then the browser, then the server. The runner deletes +a build selected with `--remove` only after the final run record is saved. + +## Workloads + +### Declaring a scene + +A scene declaration names an id, a description, a closed parameter schema, and a +builder function. The builder runs inside a seeded scope. Random values come +from one generator owned by that scope, so the same recipe, parameters, and seed +produce the same snapshot. + +```clojure +(defscene :rects + {:description "Seeded rectangles with translucent fills and centered strokes" + :params-schema schema:params} + [params] + (scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)} + :defaults {:rect {:x (gen-int 0 (:width params)) + :y (gen-int 0 (:height params)) + :width (gen-int 20 100) + :height (gen-int 20 100) + :fills (gen-vector (gen-fill))}}} + (doseq [_ (range (:count params))] + (rect)))) +``` + +Each scene has one construction scope. Starting another scene inside it raises +an error. Ordinary `let`, `doseq`, and functions defined outside the scope work +inside the body. Each shape constructor accepts an optional label +or a label plus attributes. Attributes override generated defaults by key. +Labels must be keywords or vectors and must be unique within the snapshot. + +Containers follow the same pattern: + +- `frame` keeps the supplied `:x`, `:y`, `:width`, and `:height`. +- `group` derives bounds from its children. A masked group uses its first child + as the mask. +- `bool` derives content and bounds from its children with the standard path + engine. It requires a boolean operation type. Frame children raise a + construction error. + +### Declaring a case + +For a scene named `:rects`, case ids start with `:rects/`. The `defcase` form +derives the scene from that keyword namespace and attaches the case body. + +```clojure +(defcase :rects/pan + {:params default-params + :view {:scale 1 :x 0 :y 0} + :context :reuse + :reset (fn [rtx] + (camera/restore rtx (get-in rtx [:case :view])))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 200 :dy 0 :settle-ms 100})) +``` + +The case declares: + +- `:params`: scene parameters. Collection validates them against the scene's + closed schema. +- `:view`: the starting camera, with an optional viewport override. The default + viewport is 1920 by 1080 at device pixel ratio 2. +- `:context`: `:fresh` creates a page, module, and context per attempt. + `:reuse` keeps one page, module, context, snapshot, and upload for all + attempts of the case. +- `:preparation`: an optional `:callback` function runs after upload and returns + the next context or its Promise. Preparation then renders Full by default. + `:render :none` completes preparation after the callback for cold-render cases. +- `:reset`: an optional function that receives the context after successful + main execution and returns the next context or its Promise. Reset runs after + every successful main, including warmups and the final attempt. + +The case body owns the command sequence, arguments, and any deliberate +repetition. The declaration supplies the starting settings; the body and the +commands it calls own the work. + +Commands receive workload arguments from the case body. The callback, main body, +and reset share the command context and can each call measured commands. + +### Case collection and seeds + +Node collects cases from the compiled catalogue. Collection loads every +declaration, validates every scene and case, derives the scene seed, selects +the case's data fields, and checks that the descriptor survives a serialization +round trip. Shape construction, browser startup, and renderer initialization +take place during execution. + +The local registry keeps the main, reset, and preparation callback functions. +The descriptor carries the declared preparation data. The browser resolves the +functions from its registry using the case id. + +The scene seed depends only on the master seed, the scene id, and the canonical +parameters. These inputs give load, pan, and zoom cases the same starting +content across case bodies, collection orders, attempt counts, and feature +configurations. They also give A/B configurations matching scene content. + +A filter selects cases by a substring of the case id. An empty selection is an +error. Declarations outside the filter are still validated. + +### Snapshots and upload order + +The snapshot contract requires the canonical root frame, shape ids that match +their object keys, consistent child and parent links, a tree that reaches every +shape from the root, and references that point at present shapes. Validation +runs before any timer starts. + +The browser derives upload order from the snapshot. The root comes first, then +descendants in the order of each shape's +child list. The browser computes the order and the fingerprint before the upload +clock starts. The upload uses the same batch serializer as the editor, with +structure, fill, and stroke data batched and path and SVG attributes handled per +shape. + +### Some example scenes + +| Scene | Content | Cases | +| ------- | ----------------------------------------------------------------------- | -------------------------------------- | +| `rects` | 1000 rectangles with random size, translucent fills, strokes, and radii | load, pan, zoom, blocking-full, mixed-pans | +| `paths` | 1000 curved paths with fills and strokes | load, pan, zoom | +| `plus` | 1000 plus signs made from two open line subpaths | load, pan, zoom | +| `masks` | 24 masked groups, each with a circle mask and 8 striped rectangles | load, pan, zoom | + +The current catalogue has 14 cases: load, pan, and zoom for each scene, plus +`rects/blocking-full` and `rects/mixed-pans`. Every load case is cold, with +`:context :fresh` and `:preparation {:render :none}`, and its body calls +`camera/load*`. The pan and zoom cases for each scene reuse one prepared context +and keep their declared offsets, steps, factor, and settling time; their reset +calls the bare `camera/restore` to return to the case view without recording an +event. `rects/blocking-full` is another fresh cold case and calls +`camera/render-sync*`. `rects/mixed-pans` runs two measured pans around one bare +pan and resets with measured `camera/restore*`. + +Scene recipes supply canonical geometry for rendering and camera workloads. +Camera sessions submit view updates and wait for settling before their final +render. Each case declares its sequence, arguments, and reset behavior in code. + +## Renderer commands + +A renderer command accepts a command context or its Promise and returns a +Promise of the next context. The command context has four fields: + +| Field | Meaning | +| --- | --- | +| `:case` | Case descriptor, including the case id. | +| `:scene` | Prepared starting snapshot with concrete shapes and references. | +| `:view` | Current zoom scale, camera offsets, and optional viewport settings. | +| `:hooks` | Functions for clocks, scheduling, rendering, view changes, and the execution guard. | + +The `:scene` field describes the starting snapshot. Commands return view changes +through `:view` and apply renderer work through hooks. Every command context +supplies `:now`, `:check`, `:frame`, `:sleep`, `:render`, `:render-sync`, +`:render-from-cache`, `:set-view`, `:set-view-start`, and `:set-view-end` as +functions. The `:now` hook returns a clock reading in milliseconds. +The `:check` guard authorizes work by returning and reports cancellation, +deadline expiry, context loss, or a superseded owner +by throwing an error. The owner is the execution authorized to use the renderer. +Returned contexts retain the execution's `:check` guard; replacing it rejects +completion. + +### Public commands + +The runtime provides five public commands. Each has a bare function and a +measured variant with `*`. Both forms accept a context or its Promise and return +a Promise of the next context. + +| Command | Work | Recorded metric types | +| --- | --- | --- | +| `render-sync` | Runs the blocking renderer path once. | `:command/elapsed` | +| `load` | Waits for an animation frame, then renders until `Full`. | `:command/elapsed`, `:renderer/call-elapsed`, `:renderer/prepared-scene-to-full` | +| `pan` | Moves by `:dx` and `:dy` through `:steps` cached previews, waits `:settle-ms`, ends the interaction, and renders until `Full`. | `:command/elapsed`, `:renderer/cache-call-elapsed`, `:renderer/call-elapsed`, `:camera/finalization-to-full` | +| `zoom` | Multiplies scale by `:factor` with the same camera session. | The same types as `pan`. | +| `restore` | Applies a view and renders immediately with `SyncTiles` until `Full`. | `:command/elapsed`, `:renderer/call-elapsed` | + +Bare commands update the context but record no event. A measured command records +one event. `load`, `pan`, `zoom`, and `restore` carry the final renderer return +captured by the protocol into `:end-ms`; their command elapsed ends at that +endpoint, not at a later Promise callback. `:renderer/call-elapsed` holds one +duration per renderer call in the final drain, in call order. `pan` and `zoom` +also record one `:renderer/cache-call-elapsed` value per cached preview and a +single `:camera/finalization-to-full` value. `load` adds +`:renderer/prepared-scene-to-full`, the drain's `:full-ms`. `render-sync` is +synchronous and records only its command elapsed. + +`restore` requires the target view's resolved viewport to equal the current +context's resolved viewport. A different width, height, or device pixel ratio +fails before the command changes the renderer state. + +### Declaring a command + +`defcommand` defines two functions from one body. The base function runs the +work; the function with a `*` suffix also records a measurement event. Measured +functions require an active command execution. Both functions resolve their +incoming context before running the body. A synchronous body +returns the next command context. + +```clojure +(ns example.renderer-commands + (:require + [benches.render-wasm.runtime.camera :as camera] + [benches.render-wasm.runtime.commands :as commands :refer [defcommand]] + [benches.render-wasm.runtime.protocol :as protocol])) + +(defcommand render-sync + "Renders the prepared scene synchronously." + [rtx] + ((get-in rtx [:hooks :render-sync])) + rtx) +``` + +`:pre` conditions validate arguments before work and timing in every build. +Input resolution, context validation, and argument validation precede the start +clock reading. Context validation checks the complete set of hooks. Output +validation and event recording follow the endpoint. + +The body duration is + +\[ +\Delta t = t_{\mathrm{end}} - t_{\mathrm{start}}, +\] + +where both clock readings use milliseconds from the same origin. A synchronous +command captures the endpoint immediately after its body returns. An +asynchronous command supplies an endpoint or captures it when `complete` runs. + +### Asynchronous completion + +The option `:async [complete fail]` supplies two completion callbacks to the +body. Their names belong to the declaration; the public command arguments stay +the same. The first callback call determines the outcome: + +- `(complete next-rtx)` succeeds with the next command context and captures the + endpoint at that call. +- `(complete next-rtx evidence)` succeeds with an explicit endpoint and raw + measurements when `evidence` supplies them. +- `(fail cause)` rejects the command with its cause and available partial trace. + +Successful completion requires a call to `complete`. A returned Promise forwards +its rejection to `fail`. The execution records diagnostics for repeated +completion calls while it is active. Its guard checks authorization at +completion, and the measured command records its event after output validation. + +The optional completion map has these fields: + +| Field | Meaning | +| --- | --- | +| `:end-ms` | Clock reading at the end of the work, using the same clock origin as the start. | +| `:metrics` | Namespaced metric keywords mapped to nonempty vectors of raw numbers. | +| `:trace` | Details of the work, such as traces of render calls and requested waits. | + +The command supplies `:command/elapsed`, its body duration in milliseconds. +`:metrics` supplies additional quantities. Command events preserve zero, +negative, `NaN`, and infinite numeric values as raw measurements. + +Camera completion illustrates the callback interface. The current helpers copy +the protocol's captured endpoint and renderer call durations into the completion +map: + +```clojure +(defn- render-evidence + [evidence] + {:end-ms (:end-ms evidence) + :metrics {:renderer/call-elapsed (mapv :duration-ms (:slices evidence))} + :trace evidence}) + +(defn- camera-evidence + [evidence] + (-> (render-evidence evidence) + (assoc-in [:metrics :renderer/cache-call-elapsed] + (mapv :duration-ms (:cached-slices evidence))) + (assoc-in [:metrics :camera/finalization-to-full] + [(:time-to-full-ms evidence)]))) + +(defcommand pan + "Moves the view by `:dx` and `:dy` through `:steps` cached previews. + Waits `:settle-ms` milliseconds, ends the interaction and renders until Full." + {:async [complete fail]} + [rtx {:keys [dx dy] :as options}] + {:pre [(valid-pan? options) (valid-view? (pan-view (:view rtx) dx dy))]} + (let [target (pan-view (:view rtx) dx dy)] + (-> (run-camera rtx target options) + (.then (fn [evidence] + (complete (assoc rtx :view target) + (camera-evidence evidence)))) + (.catch fail)))) +``` + +The final renderer call can return before the Promise callback runs. The +`:end-ms` field carries that renderer endpoint through the scheduling delay. +For example, renderer work lasting 7 ms followed by a 500 ms callback delay +still supplies a renderer duration of 7 ms. A clock reading in the later +callback would measure 507 ms. The public commands table lists the metric types +each measured command records. + +### Composition and execution + +A composite combines commands. `:measured-variant false` defines its base +function and selects measurements through the calls in its body: + +```clojure +(defcommand selected-pans + {:measured-variant false} + [rtx] + (-> rtx + (camera/pan* {:dx 10 :dy 0 :steps 2 :settle-ms 5}) + (camera/pan {:dx 0 :dy 5 :steps 2 :settle-ms 5}) + (camera/pan* {:dx -10 :dy 0 :steps 2 :settle-ms 5}))) +``` + +All three calls update the view and renderer. The first and third record +events. Command functions resolve incoming Promises, so threading passes each +updated context to the next command. `rects/mixed-pans` declares this pattern: +two measured pans and one bare pan in main, then a measured `camera/restore*` +in reset. + +A command execution runs a function from one command context to the next, +authorizes its work, and collects events in completion order. Given a prepared +command context `rtx`: + +```clojure +(commands/execute rtx selected-pans) +``` + +The returned Promise resolves to an outcome. A successful outcome contains +`:outcome :completed`, the next `:context`, and ordered `:events`. A failed +outcome contains `:outcome :failed`, `:error`, and the events recorded before +failure. Existing Errors retain their identity and partial trace. Other thrown +values become Errors with the original value in `:thrown`. An active execution +can also record repeated completion calls in `:diagnostics`. + +An event has this shape: + +```clojure +{:command :benches.render-wasm.runtime.camera/pan + :metrics {:command/elapsed [74] + :renderer/call-elapsed [7 7] + :renderer/cache-call-elapsed [2 2] + :camera/finalization-to-full [33]} + :trace {:end-ms 74}} +``` + +One event represents one measured command call. Its metric vectors describe +the individual values that call produced. The command execution owns this +history; its guard authorizes work for that execution. Pass the resulting +context to `commands/execute` to start another command execution. + +### Case phases + +The browser executes preparation, main, and reset through the command executor. +Preparation builds and uploads the scene, runs its optional callback once, and +renders Full unless the case declares `:preparation {:render :none}`. A fresh +context prepares once for its attempt; a reused context prepares once for its +case. Each phase passes its returned context to the next phase. + +A successful preparation response contains its raw block in +`[:preparation :phases]`. A failed preparation returns its available blocks in +top-level `:phases`. An attempt response contains main and any declared reset +in `:phases`: + +```clojure +{:phase :main + :preparation-id "preparation-1" + :outcome :completed + :events [{:command :benches.render-wasm.runtime.camera/render-sync + :metrics {:command/elapsed [7]}}]} +``` + +Each block retains ordered command events, raw numeric vectors, and any +completion diagnostics. A phase can complete with an empty event vector. +Initialization and upload durations stay in preparation `:diagnostics`. +Automatic initial rendering retains its raw trace under +`[:diagnostics :setup-render]`. Measured commands in the callback supply the +preparation block's selected events. + +Reset runs after successful main. Its returned context supplies the next reused +attempt. A phase failure releases its owned context and returns the failed +block with earlier events and available traces. Completed preparation retains +its own evidence when main or reset fails. Any preparation, main, or reset +failure stops the case; the runner attempts no later slot. It disposes the +case's browser and starts the next case in a new browser when the run continues. + +### Runtime code + +[`runtime/commands.clj`](runtime/commands.clj) declares commands; +[`runtime/commands.cljs`](runtime/commands.cljs) executes them. +[`runtime/camera.cljs`](runtime/camera.cljs) defines rendering and camera +commands. [`runtime/protocol.cljs`](runtime/protocol.cljs) defines render +drains, camera sessions, hook contracts, and trace fields. + +Case declarations such as [`scenes/rects.cljs`](scenes/rects.cljs) attach case +bodies through [`declarations.cljc`](declarations.cljc). Node loads these +declarations and functions during catalogue collection. Renderer commands run +through the supplied hooks when the functions execute in the browser. +[`browser/bridge.cljs`](browser/bridge.cljs) constructs the runtime context for +case bodies and prepares graphics contexts, snapshots, and uploads. Reused +cases carry their returned context into the next attempt; their reset function +controls restoration. + +## Measurement + +### Preparation and measurement boundaries + +Each duration has a defined start and endpoint. Setup and diagnostics run +around the interval whose work they prepare or describe. + +Work around those intervals includes: + +- scene build, snapshot validation, and fingerprinting; +- upload order derivation; +- request and result serialization; +- validation, logging, and screenshots; +- module script import; +- the automatic initial Full render during preparation. + +Preparation diagnostics time: + +- module instantiation; +- graphics context creation and initial size synchronization; +- shape batch upload, including the loading lifecycle and first view setup; +- the automatic render drain to `Full`. + +Measured commands record their own intervals during preparation, main, or reset. +For `pan` and `zoom`, this includes view updates, cached previews, settling, +ending the interaction, and the final drain. + +### Lifecycle of a fresh load + +```mermaid +flowchart TD + A[Import module script - untimed] --> B[Time module start] + B --> C[Time graphics start] + C --> D[Build and validate snapshot - untimed] + D --> E[Fingerprint and derive upload order - untimed] + E --> F[Time upload and first view setup] + F --> G[Run preparation callback] + G --> H[Render initial Full unless render is none] + H --> I[Run main] + I --> J[Run reset after successful main] + J --> K[Return raw phase records] + K --> L[Dispose page and context] +``` + +The module script import is untimed because the browser loads it once per page. +Module instantiation starts a new renderer state. Graphics start creates the +WebGL2 context and synchronizes the drawing buffer. Upload covers the loading +lifecycle, batch serialization, and tile preparation. Each measured command +starts its clock immediately before its work. + +### Warm interactions + +A warm case owns one prepared page. Its main body starts from the context +returned by preparation or the preceding attempt's reset. A `pan` or `zoom` +command runs this session: + +1. start the camera interaction; +2. for each view update, wait for the next animation frame, apply the view, and + present cached pixels; +3. wait for the requested settling time; +4. end the camera interaction; +5. render until `Full`. + +A pure pan can reuse cached textures while it updates which tiles are visible. +A zoom rebuilds and invalidates tile textures. The standard pan and zoom cases +request a 100 ms settling wait; `rects/mixed-pans` uses 50 ms for its three +pans. They record the requested and observed durations separately from renderer +calls. Camera finalization includes the call that ends the interaction; the run +also records that call's duration. + +### Slices and metrics + +The browser returns ordered events within each phase. Preparation diagnostics +retain `module-init-ms`, `graphics-init-ms`, and `upload-ms` separately. + +A render drain retains these trace fields: + +- `slices`: one record per renderer call with `timestamp`, `flags`, + `frame-type`, and `duration-ms`; +- `viewport-ready-ms`: the first `ViewportReady` or `Full` completion from the + drain origin; +- `full-ms`: the `Full` completion from the drain origin. + +A `pan` or `zoom` command's trace contains the camera session and final drain: + +- `cached-slices`: one record per gesture frame with `timestamp`, + `duration-ms` for the cached preview call, and `view-and-preview-ms` for the + view update plus preview call; +- `slices`: the final drain, with the same shape as a fresh drain; +- `interact-ms`: the whole camera session, from before the interaction starts + through the final `Full`; +- `active-ms`: from before the interaction starts to after the last cached + preview and its guard check, or after an explicit pause that follows it; +- `settling-requested-ms` and `settling-actual-ms`: the declared and observed + settling waits; +- `set-view-end-ms`: elapsed time during the call that ends the interaction; +- `time-to-viewport-ready-ms` and `time-to-full-ms`: durations from camera + finalization through the first `ViewportReady` or `Full`, and through `Full`, + respectively; +- `last-input-to-full-ms`: from before the last view update to the endpoint of + the renderer call that returns `Full`. + +Both drains and camera sessions return `end-ms`, the clock reading immediately +after the final renderer call returns `Full`. Completion durations end at that +reading. Promise callbacks can run later as the browser schedules queued work. + +A `load` or `restore` command uses the final drain trace directly. `load` also +records `:renderer/prepared-scene-to-full` from `:full-ms`. `restore` starts its +drain immediately with `SyncTiles` and records the same per-call durations as the +final drain. + +Measured command events keep each metric's raw values in a vector. Repeated +commands produce separate events, including when they name the same metric. +Raw values preserve zero, negative, and non-finite measurements in the saved +record. Reports sum one metric type within one phase execution before computing +statistics; saved records contain no aggregates. + +### Renderer completion + +The renderer finishes in tile budgets. One renderer call returns a frame type, +and the caller continues while the renderer reports `Partial` or +`ViewportReady`. The drain completes when a call returns `Full`. `None` or an +unknown value fails the attempt. + +| Value | Frame type | Meaning | +| ----: | --------------- | -------------------------------------------------------------- | +| 0 | `None` | no frame | +| 1 | `Partial` | visible work remains | +| 2 | `Full` | the renderer finished the current request | +| 3 | `ViewportReady` | the visible viewport was submitted; work outside it may remain | + +`Full` marks renderer submission of the completed request. Skia and WebGL queue +GPU work, and GPU execution and display follow their own schedules. A call +duration measures synchronous work on the calling thread, including any driver +stall. These clock readings describe work through the renderer return. + +For scheduled calls, the browser passes the timestamp of the animation frame. +For immediate calls, it reads the clock. Each slice records its timestamp to +describe the call sequence. + +### Graphics environment + +The benchmark uses WebGL2 with `antialias=false`, `depth=true`, `stencil=true`, +`alpha=true`, and `preserveDrawingBuffer=true`. The default viewport is 1920 by +1080 at device pixel ratio 2. The preparation records the effective CSS size, +drawing-buffer size, device pixel ratio, context attributes, renderer and vendor +strings, and a software rendering warning. + +Benchmarks run inside the Linux development container. Start it with +`./manage.sh run-devenv --gpu` to enable hardware access. This sets +`PENPOT_GPU=1`; the runner then selects ANGLE Vulkan, and `pnpm run render-bench` +starts Xvfb to supply the display that hardware rendering needs in the container. + +`--require-gpu` makes `run` and `ab` require hardware WebGL2. The command first +checks for `PENPOT_GPU=1`, before builds, catalogue checks, or output allocation. +After each preparation returns, the runner checks its existing +`:effective-graphics` metadata before executing attempts. It requires WebGL2, +usable vendor and renderer identities, and `:software false`. A rejected +preparation aborts the run with exit 1 and stops later A/B configurations. The +record retains the preparation and graphics failure evidence, and leaves pending +attempt slots unattempted. The failure names the observed renderer and the +commands needed to enable GPU access. `ab --dry-run` skips these GPU checks. + +Each preparation records the graphics identity of its own context. The run stores +the GPU policy and environment signal under diagnostics. Default runs allow +SwiftShader fallback and warn when they record software +rendering. Comparisons check effective graphics settings and resolved launch +arguments. Comparing hardware and software runs requires +`--allow-all-differences`. + +## Results and reports + +### Run record + +A run record contains: + +- the plan: selected case descriptors, warmup count, repetition count, and + timeout policy (`:case-ms`); +- metadata: Git revision and dirty flag, build facts, environment, and diagnostics; +- the resolved analysis settings; +- saved quantity descriptions; +- preparations in chronological order; +- attempts in chronological order, each with its case, preparation, + `:attempt-id`, warmup flag, outcome, main/reset phase blocks, and failure or + partial evidence; +- terminal outcomes in `:case-outcomes`; +- remaining slots for each case; +- the termination reason. + +A preparation records its case, warmup flag, setup diagnostics, phase history, +and any failure. A completed preparation also has its fingerprint and effective +graphics. An attempt records one outcome and its main and optional reset phase +blocks. A phase block retains ordered command events, raw metric vectors, and +optional completion diagnostics. Errors retain available phase evidence and +traces after a failure. + +Each case has a terminal entry in `:case-outcomes` with `:outcome` set to +`:completed`, `:failed`, `:timed-out`, or `:interrupted` and an optional +`:failure`. Recording a terminal outcome closes the case: later preparation or +attempt evidence for that case is rejected. + +### Failures and termination + +A failed attempt still consumes its allocated slot. A fresh preparation failure +records a failed attempt and consumes the current slot. A reuse preparation +failure records no main or reset execution and consumes no main slot. Any +preparation, main, or reset failure stops the case for fresh and reuse alike, +so later slots stay unattempted. The runner disposes the failed case's browser +and starts the next case in a new browser when the run continues. A browser or +page failure follows the same case-stop rule. A failure to clean up the browser +for a failed case aborts later cases. The run fixes its attempt count before +execution and uses each allocated slot at most once. + +A malformed delivered response follows the same slot rules. An attempt response +consumes the slot it invoked, keeps the valid phase prefix, and saves the raw +response as partial evidence. A malformed preparation response fails its +preparation; a fresh case still consumes the current slot. + +A screenshot failure is diagnostic when it is an ordinary error. A deadline +expiry or interruption during the screenshot stops the case, while earlier +completed attempts remain saved. + +A case that finishes all its slots without a measured command gets a failed +case outcome whose failure has `:reason :no-measurements`. The run then +terminates as failed or incomplete according to the remaining slots. + +A run terminates in one of four states: + +- **completed**: every allocated slot was attempted, every case outcome is + `:completed`, and every case has at least one measurement. +- **failed**: every allocated slot was attempted and at least one attempt failed + or one case outcome is not `:completed`. +- **incomplete**: slots remain because a case stopped early or a startup, + browser, or storage failure prevented execution. +- **interrupted**: `SIGINT` or `SIGTERM` stopped the run. + +The runner saves partial records before it exits. Use exit status to determine +whether the command completed successfully; saved records also describe work +that ended in failure. + +### Summaries + +A summary reports one entry per case and metric key +`[phase scope metric-type]`. The scope is the command that recorded the metric, +or `:global` for every command with that type in the phase. For each key, the +report sums the raw values of one metric type within each eligible phase +execution and computes statistics across those execution sums. It labels the +reduction `:statistic :sum`. + +For each metric key, the report includes: + +- the observed mean, median, minimum, maximum, sample standard deviation, and + median absolute deviation; +- a bootstrap confidence interval for the median. The interval comes from + repeated resampling of the observed execution sums; +- observed p95 and p99 values when the sample is large enough; +- counts for the valid, invalid, absent, failed, unattempted, and warmup + categories, plus command invocation and raw-value counts; +- the saved quantity unit and optional description, and any validity + diagnostics. + +Statistical `n` is the valid count. Invocations count command events and +raw-values count all recorded numbers across eligible executions; neither is the +number of observations. Warmups and failed attempts are excluded. A failed +attempt excludes both its main and reset blocks. A completed preparation remains +eligible after a later main or reset failure. A fresh warmup preparation is +excluded; one reuse preparation supplies one observation, not one per attempt. + +A finite zero is valid. Negative and non-finite raw values remain in the saved +record but invalidate the dependent execution sum and aggregate; independent +metric keys remain usable. Unknown or malformed quantity metadata appears as a +diagnostic and leaves that key without usable statistics. A phase can complete +with no events. Preparation-only and reset-only measured cases are valid. A +case that finishes all its slots without any measured command gets a failed +case outcome with `:reason :no-measurements`; the run is failed when every slot +was attempted and incomplete when slots remain. + +The case report also counts actual preparation, main, and reset executions. +Completed and failed counts use each execution's own outcome, and a separate +count records warmup executions. These phase counts do not filter by attempt +eligibility. A completed main block from an attempt whose reset failed still +appears in the main phase counts but supplies no observation. + +The default analysis settings are: + +| Setting | Default | +| ----------- | ------------ | +| Draws | 2000 | +| Seed | `0x70656e70` | +| Confidence | 0.95 | +| Minimum CI | 10 | +| Minimum p95 | 100 | +| Minimum p99 | 500 | + +The run records draw count, seed, confidence, and minimum counts, and a report +can override them at analysis time. An estimate that cannot be computed keeps a +reason and, when applicable, the minimum count it needs. + +The intervals are exploratory and conditional on available executions. +Incomplete and timed-out cases can select which observations survive. Attempts +within one run share evolving caches and can have dependent durations. Repeated +whole runs provide evidence about reproducibility across those cache conditions. + +### Comparisons + +A comparison pairs saved cases by identical case id unless `--cases A,B` +selects one explicit baseline, candidate pair. The CLI checks the selector +syntax before it loads either record, then resolves each selector by saved case +id or exact name. + +By default, a comparison requires equal: + +- scene identity, scene seed, and scene parameters; +- starting-snapshot fingerprints; +- view, context policy, and preparation settings; +- effective graphics settings and runtime environment; +- build mode, effective features, instrumentation settings, and recorded build + environment; +- warmup count, timeout policy (`:case-ms`), and saved quantity units. + +Comparison pairs metric keys `[phase scope metric-type]`. Quantity description +text does not affect compatibility; changed prose is not a difference. Unknown +or malformed quantity metadata appears in the per-side report diagnostics and +leaves that key without usable statistics. + +Git revision, dirty state, run ids, timestamps, and saved analysis defaults +provide provenance for interpreting the paired cases. + +The comparison reports absolute and percentage median changes between execution +sums, with a bootstrap interval for each change. A zero baseline suppresses a +percentage change. The comparison leaves a metric key with differing quantity +units or incomplete quantity evidence out unless the all-differences +permission allows its valid raw values. + +### Comparison permissions + +The comparison records every compatibility difference. A permission acknowledges +a class of differences and labels each acknowledged +mismatch with the rule that allowed it. + +- `--allow-build-differences` acknowledges recorded build settings, effective + features, instrumentation, and the recorded build environment. Runtime + environment, graphics, and saved quantity units stay strict. +- `--allow-scene-differences` acknowledges scene identity, scene seed, scene + parameters, and differences in fingerprint content. Execution settings stay + strict. Missing or varying fingerprint evidence still needs the + all-differences permission. +- `--allow-all-differences` acknowledges every compatibility difference, + including mixed quantity units. The report compares valid raw execution sums + in their recorded units. Record validation and measurement validity still + govern the analysis. + +Build and scene permissions combine by union. All-differences includes both. +A report shows the selected permissions, every recorded difference, and the +rule that acknowledged it. Exit status reports execution and compatibility +checks; the estimates describe the observed timing differences. + +## A/B feature experiments + +An A/B experiment answers one question: how do the selected cases behave when a +renderer feature is off versus on, with everything else held constant? + +The `ab` command takes the requested features, plans the configurations, builds +every configuration, runs every configuration, and compares every edge: a pair +of configurations that differ by one feature. + +### Planning + +A feature configuration is a set of enabled Cargo features. The planner: + +1. normalizes the requested names by trimming, deduplicating, and sorting them; +2. validates them against the renderer manifest with Cargo metadata; +3. requires an empty Cargo default feature set, so each configuration states + its features explicitly; +4. rejects diagnostic features used for profiling or statistics; +5. orders the configurations in reflected Gray order, so adjacent + configurations differ by one feature, with the first requested feature as the + most significant bit; +6. emits every pair of configurations that differ in exactly one feature, from + the configuration with that feature off to the configuration with it on. + +The command requires at least one requested feature. For $N$ features, the +comparison has $2^N$ configurations and $N 2^{N-1}$ edges. The planner +compares each edge once, in the direction from off to on. For features +`[a, b]`, the configurations are `00, 01, 11, 10` and the edges are: + +```text +00 -> 10 : a +00 -> 01 : b +01 -> 11 : a +10 -> 11 : b +``` + +`--max-features` accepts 1 through 6 and defaults to 3. When `--dry-run` +is absent, the command requires `--confirm` for more than 8 configurations +or a filter that is absent, empty, or contains only whitespace. `--dry-run` +checks the requested features using Cargo metadata, selects cases from their +declarations, and prints the features, configurations, edges, selected case +count, and time limits. It creates no comparison directory, compiles no build, +and launches no browser. + +### Running a feature comparison + +Every configuration gets its own build, catalogue, browser executor, server, +browser, and run. Each configuration runs and cleans up before the next one +starts. The configurations share the selected cases and their seeds, parameters, +views, context policies, preparation settings, and case deadlines. Collection +checks each configuration's catalogue identity and requires the same selected +case descriptors in the same order. + +```bash +pnpm run build:renderer-benchmarks:runner + +pnpm run render-bench ab \ + --features , \ + --filter rects/load \ + --warmups 3 --repetitions 10 +``` + +The feature names must exist in the renderer manifest. Placeholders appear here +because the manifest decides which features are available. + +After a configuration records attempts, scheduling continues with its recorded +failures and remaining slots. A build failure, a catalogue +identity failure, or a run with zero attempts stops scheduling later +configurations; those configurations are recorded as unattempted with a reason. +`SIGINT` and `SIGTERM` save the available state and return 130 or 143. + +### Feature-comparison output + +The default output directory is +`target/performance/render-wasm/feature-comparisons//`. It contains: + +```text +feature-comparison.transit plan, progress, edges, termination +feature-comparison.log comparison progress and diagnostics +configs//run.transit the configuration's run record +configs//run.log the configuration's log +configs//*.png optional screenshots +comparisons/__.transit one edge comparison +``` + +The feature-comparison record stores the normalized options, comparison +permissions, selected cases, configuration order, requested and effective +features, build and run references, edges, failures, and interpretation +warnings. The runner validates the record with a schema and writes it +atomically. It writes the initial plan before the first build and updates the +record after each configuration, after each edge, and at termination. The +command prints `Comparison `, `Output `, and `Status ` +on stdout at termination. When a failed outcome has an empty error list, the +command appends a `Failures` section with saved reasons and artefact references. +The section shows up to eight failed configurations and comparisons, then names +the saved record and log with the complete evidence. Progress and runtime +failures go to stderr. Each configuration has its own build and run log. + +### Edge comparisons + +An edge comparison uses the ordinary comparison path. It automatically +acknowledges one difference: the effective feature change that matches the +planner's evidence. The evidence records the requested feature, both effective +feature sets, and the added and removed features. It also requires the two +configurations' requested feature selections to differ only by the requested +toggle and to record the same default-feature policy. If the saved sets do not +match that evidence, the difference remains unacknowledged. + +The comparison record also carries the edge context: the source and target +configuration slugs and the expected feature facts. An edge is analysed when +both endpoint records exist, even if one run recorded failed attempts. A missing +endpoint or an analysis error produces an explicit edge outcome. An edge that +retains an unacknowledged difference keeps its saved report, is marked failed, +and makes the invocation exit 1. + +Other build differences require `--allow-build-differences`. Scene differences +require `--allow-scene-differences`. View, context, preparation, graphics, +runtime environment, and saved quantity-unit differences stay strict unless +`--allow-all-differences` is selected. If two configurations report the same effective feature set, the +comparison includes their edge and labels it with that fact. Effective feature +sets describe the code settings that accompany the observed timings. + +The command returns 0 when every configuration completes, every edge analysis +finishes, and every difference is acknowledged. It returns 1 for execution, +persistence, or analysis failures, including unacknowledged differences. Edge +reports show raw times and statistics for the observed differences. Establishing +a causal effect requires experimental control and repeated measurements. + +## How to run + +Run these commands from `frontend/` inside the development container with +dependencies and Playwright Chromium installed. Keep renderer watchers stopped +until preparation finishes. + +List the commands, or show one command's options and defaults: + +```bash +pnpm run render-benchhelp +pnpm run render-bench run --help +``` + +Command options select declared cases and control the build source, sampling, +output and hardware policy. Case declarations own workload bodies, scene +parameters, view, context policy, preparation, reset, and case lifecycle. +`--seed` is the explicit exception: it changes generated scene content +reproducibly, but it does not replace a case body, its command arguments, or its +lifecycle settings. + +Build the runner and its command module: + +```bash +pnpm run build:renderer-benchmarks:runner +``` + +Run one case with one measured attempt: + +```bash +pnpm run render-bench run \ + --filter paths/pan --seed 42 --warmups 0 --repetitions 1 --screenshots +``` + +Run the blocking renderer and mixed gesture proof cases: + +```bash +pnpm run render-bench run \ + --filter rects/blocking-full --seed 42 \ + --warmups 0 --repetitions 1 --screenshots + +pnpm run render-bench run \ + --filter rects/mixed-pans --seed 42 \ + --warmups 0 --repetitions 1 --screenshots +``` + +Prepare a build once and reuse it: + +```bash +pnpm run render-bench build +pnpm run render-bench run --build +``` + +Run a selection with the default sampling: + +```bash +pnpm run render-bench run --filter rects +``` + +Require hardware rendering for rectangle load, pan, and zoom in a GPU container: + +```bash +pnpm run render-bench run --filter rects --require-gpu +``` + +For a functional check, add `--warmups 0 --repetitions 1`. A successful +run renders the scenes through `Full` and saves the vendor and renderer strings +with `:software false` in each preparation's `:effective-graphics`. + +Choose a new path for `--output-dir`. The runner rejects an existing output +directory. A/B feature comparisons apply the same rule. + +Compare two saved runs offline: + +```bash +pnpm run render-bench compare \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit +``` + +Allow a difference in build configuration, or compare two given scenes, +irrespective of their differences: + +```bash +pnpm run render-bench compare \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit \ + --allow-build-differences + +pnpm run render-bench compare \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit \ + --cases rects/load,paths/load --allow-scene-differences +``` + +Merge an additional saved run into a base run: + +```bash +pnpm run render-bench merge \ + target/performance/render-wasm/runs//run.transit \ + target/performance/render-wasm/runs//run.transit \ + --output-dir target/performance/render-wasm/runs/ +``` + +The merged record gets a new run id and a new run directory. Both input records +stay unchanged. Both inputs must be completed and compatible: same cases, +timeout policy, analysis, build facts, runtime environment, and Git provenance. +The additional run must use the same warmup count. `merge` appends only the +additional measured attempts and uses the base run's warmups. The additional +warmup attempts stay in the additional record. The command starts no build, +server, or browser. + +Inspect and clean managed outputs: + +```bash +pnpm run render-bench purge builds +pnpm run render-bench purge runs +pnpm run render-bench purge comparisons +pnpm run render-bench purge all +``` + +`purge` requires exactly one target: `builds`, `runs`, `comparisons`, or `all`. +It removes known build, standalone-run, or feature-comparison outputs and keeps +directories and unrelated files. It follows directory symlinks and can remove +matching output files from their targets, including targets outside the managed +directories. `all` removes every category. A missing, unknown or multiple target +is an argument error. + +The commands write results to stdout: + +- `build`: `Build ` and `Output ` after build preparation + ends. +- `run`: `Run ` and `Output ` after the run ends. When a + report is available, a `Report` heading and the formatted summary follow. +- `ab`: `Comparison `, `Output `, and final + `Status `. A failed outcome with an empty error list also prints a + bounded `Failures` section from the saved comparison record. +- `compare`: the comparison report. +- `merge`: `Run ` and `Output `. +- `--help` and `COMMAND --help`: usage text with that command's options and + defaults. `purge` produces no output on success. + +Each CLI command produces an outcome with an exit code, errors, log path, and +optional report text. The runner prints the final output once and returns the +numeric status. + +Progress and diagnostics go to stderr. The default shows build stages, case +headings, and attempt progress. `-q` / `--quiet` shows results, warnings, and +errors. `-v` / `--verbose` also shows debug details and raw build output. Quiet +and verbose are mutually exclusive. These options apply to every command. + +`--color` forces terminal colours; `--no-color` disables them. Automatic colour +uses a terminal stream when `NO_COLOR` is empty or absent and `TERM` is not +`dumb`. Colour choices apply to terminal diagnostics. Result text and file logs +use plain text. The runner reads child output as lines and normalizes their +line endings. Verbose terminal output prefixes and dims these lines; file logs +store the lines as text. Terminal errors include a limited extract from the end +of the log when available. + +Files receive every log level, including debug output hidden from the console. +Each build writes `build.log`; each A/B comparison writes +`feature-comparison.log`. The default run directory is +`target/performance/render-wasm/runs//`, which contains `run.log`, +`run.transit`, and optional screenshots. + +Exit 0 means the command completed successfully: a ready build for `build`, +every allocated attempt for `run`, a valid comparison for `compare`, a +validated merged record for `merge`, every configuration and edge for `ab`, and +a clean purge for `purge`. Exit 1 means a failed preparation, a failed attempt, +an incomplete run, an invalid selection, incompatible merge inputs, an +unacknowledged comparison difference, or a harness failure. Exits 130 and 143 +mean interruption by `SIGINT` and `SIGTERM` for `build`, `run` and `ab`. +Interruption uses those statuses to report aborted work. Errors writing the +command's record or log also produce diagnostics. + +`--warmups` defaults to 3, `--repetitions` to 10, `--timeout-ms` (the case +deadline) to 30000, and `--startup-timeout-ms` (host and initial browser +readiness) to 30000. `build`, a run that prepares its build, and an A/B +experiment use a default `--build-timeout-ms` of 300000; a run with `--build` +uses the retained build's recorded provenance. The master seed defaults to 42 +and accepts a 32-bit unsigned integer. A filter matches a substring of the case +id. + +## Unit tests + +Run these commands from `frontend/` inside the development container with +dependencies and Playwright Chromium installed. The benchmark tests use the +`performance-tests` build target and the runner in +`test/performance_tests/render_wasm/runner.cljs`. + +Build and run the full suite: + +```bash +pnpm run test:renderer-benchmarks +``` + +Build once, then run the compiled tests: + +```bash +pnpm run build:renderer-benchmarks:tests +node target/performance-tests/test.js +``` + +Focus on one namespace or one test var: + +```bash +node target/performance-tests/test.js \ + --focus performance-tests.render-wasm.contracts-test + +node target/performance-tests/test.js \ + --focus performance-tests.render-wasm.contracts-test/defscene-registers-metadata-and-build +``` + +Rebuild the test bundle after changing source or tests. `--focus` accepts one +namespace or one `namespace/test-var`. The runner exits 0 when the selected tests +pass and 1 when they fail. + +## Scope and limitations + +The suite records durations for setup, render calls, and camera movement inside +Chromium. +Rendering durations end when the renderer returns after submitting work. +Memory, power, frame rate, frame drops, visual correctness, network latency, +and editor input-to-paint each require measurements for those quantities. + +A fresh case starts a new page, module, and WebGL2 graphics context. The browser +process persists across successful cases, and module script import precedes +module timing. These startup conditions define measurements for fresh cases. + +Attempts share evolving caches, so bootstrap intervals describe the observations +within a run. Repeat whole runs to assess reproducibility, and judge whether the +observed change matters for the workload. + + +## Repository map + +```text +frontend/performance/benches/render_wasm/ +├── browser/ page entry and browser bridge +├── features/ feature selection, configurations and comparison edges +├── node/ +│ ├── catalog/ compiled catalog entry and retained catalog loader +│ ├── cli/ CLI entry, resolution, requests and execution lifecycle +│ │ └── commands/ per-command schemas, help and resolution +│ ├── report/ saved-record comparison and merge +│ ├── runner/ build preparation, server, browser host and run orchestration +│ └── storage/ artifacts and saved runs +├── report/ summaries, comparisons, statistics, formatting +├── runtime/ renderer commands, render drains, and camera sessions +├── scenes/ scene recipes and their cases +├── builder.cljc scene construction +├── declarations.cljc scene and case registry, schemas, seed derivation +├── cases.cljs case discovery and collection +├── codec.cljs Transit round-trip validation +├── errors.cljc failure values and diagnostics +├── snapshot.cljs snapshot validation and upload order +├── fingerprint.cljs digest of the starting snapshot +├── identity.cljc build identity and bundle compatibility checks +├── measurement.cljs projection of phase evidence into attempts +├── random.cljs seeded random generators +├── result.cljs raw run, comparison and feature-comparison records +├── README.md this document +└── REPORTING.md raw records, metrics, and statistics + +frontend/test/performance_tests/render_wasm/ +├── runner.cljs test entry point, namespace registration, focus selection +├── *_test.cljs tests for benchmark modules +├── test_helpers.cljs shared scene-test helpers +└── cli_integration.cjs tests for the CLI through Node + +frontend/package.json build and test commands +frontend/shadow-cljs.edn benchmark and performance-tests build targets +frontend/src/app/render_wasm/ production renderer wrapper +render-wasm/ Rust renderer and build +``` + +## Further reading + +- [`REPORTING.md`](REPORTING.md) describes the record schemas, raw command + histories, quantity metadata, and statistical calculations in detail. This + guide is the source for the command line and the A/B workflow. +- [`render-wasm/README.md`](../../../../render-wasm/README.md) and the renderer + source describe the Rust side. +- [`frontend/src/app/render_wasm/`](../../../src/app/render_wasm/) contains the + production renderer orchestration whose renderer bindings and upload + serializers the benchmark shares. diff --git a/frontend/performance/benches/render_wasm/browser/bridge.cljs b/frontend/performance/benches/render_wasm/browser/bridge.cljs new file mode 100644 index 0000000000..64f2857a47 --- /dev/null +++ b/frontend/performance/benches/render_wasm/browser/bridge.cljs @@ -0,0 +1,1255 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.browser.bridge + "Prepares renderer scenes and executes benchmark case phases in the browser. + + A scene is a recipe that builds a snapshot. A snapshot contains the concrete + shapes in `:objects` and label references in `:refs`. A case pairs a scene + with parameters, a camera view and execution settings. A collected case + descriptor reaches the browser through Transit; this namespace resolves the + matching local registration, which supplies the `run!` function and optional + preparation callback and reset function. + + A preparation owns one renderer lifetime: the instantiated module, canvas, + WebGL2 context, uploaded snapshot and command context. It loads the module, + creates graphics, builds and uploads the scene, runs the preparation callback + once, and then renders an initial Full frame unless the case declares + `:preparation {:render :none}`. Fresh cases prepare once per attempt. Reused + cases prepare once and carry the returned view and command context into later + attempts. + + Execution has up to three phases: preparation, main and reset. Main runs the + case's `run!` function. Reset follows each successful main execution. A + phase's raw block contains `:phase`, `:preparation-id`, `:outcome`, + selected command `:events` and, when present, `:diagnostics`. A successful + preparation exposes its blocks at `[:preparation :phases]`. A failed + preparation places its available blocks at the top level. An attempt places + main and optional reset blocks at the top level and keeps the preparation + evidence captured for that attempt. + + A command context has exactly four fields: `:case`, the collected case + descriptor; `:scene`, the snapshot; `:view`, the camera scale and offsets; + and `:hooks`, the functions for time, scheduling, checks and renderer + operations. Every command receives the full hook map with its context. The + context, module, graphics handles, canvas and snapshot stay in the browser. + Only projected evidence, such as preparation data, phase blocks, identity + data and summary or comparison data, crosses Transit. + + An owner is the current page lifetime. Lifecycle guards compare a captured + owner epoch with the current epoch; a mismatch throws a top-level + `::errors/stale` marker. A newer load, disposal or WebGL context loss + increments the epoch and supersedes the old owner. Only a top-level + `::errors/stale` marker yields a stale result. A superseded operation that + rejects for another reason returns failed evidence for that operation and + leaves the newer owner in place; a failed attempt keeps its own preparation + and phase blocks. An ordinary failure in the current owner ends that owner + and returns its phase, cause and available partial evidence. + + Renderer-call measurements end when each call returns. Setup diagnostics + retain separate intervals for module initialization, graphics setup, upload + and initial rendering. GPU execution and display proceed asynchronously + after renderer submission. + + The exported `bridge` object maps the names used by the runner to this public + interface; the runner calls it through `page.evaluate`. Report formatting + entries return text; the other entries return Transit text or, for `ping` and + `dispose`, status objects." + (:require-macros + [app.common.exceptions :as ex]) + (:require + [app.common.render-wasm.helpers :as h] + [app.common.render-wasm.serialize-shape :as serialize-shape] + [app.common.render-wasm.serializers :as sr] + [app.common.render-wasm.wasm :as wasm] + [app.common.schema :as sm] + [app.common.transit :as t] + ;; bind the enum table from the prepared renderer build's `shared.js` so + ;; a missing `shared.js` fails the compile. A stale or mismatched one throws + ;; at load + [app.render-wasm.api.enums] + [app.render-wasm.api.webgl :as webgl] + [benches.render-wasm.cases] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.fingerprint :as fingerprint] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [benches.render-wasm.runtime.commands :as commands] + [benches.render-wasm.runtime.protocol :as protocol] + [benches.render-wasm.snapshot :as snapshot])) + +;; Forward decls +(declare now guard-current! read-graphics) + +(def schema:load-scene-args + "Decoded form of the Transit request accepted by `load-scene` and + `prepare-scene-request`. + + `:case` is a collected case descriptor. `:preparation-id` labels the + preparation and defaults to the owner epoch. `:timeout-ms` bounds setup and, + through the stored preparation, later attempts that omit their own timeout. + `:module-url` and `:wasm-url` override the renderer artifacts." + [:map {:closed true} + [:case decl/schema:collected-case] + [:preparation-id {:optional true} some?] + [:timeout-ms {:optional true} [:int {:min 1}]] + [:module-url {:optional true} string?] + [:wasm-url {:optional true} string?]]) + +(def ^:const ^:private case-ms-budget + "Default elapsed-time deadline for setup or one attempt, checked by + `browser-hooks`. Requests can supply a timeout. The outer process timeout + covers blocked pages." + 30000) + +(defn- browser-hooks + "Returns the command hooks for one owner epoch and deadline. + + Args: + epoch: owner epoch that `:check` compares with the current epoch. + deadline-ms: deadline in milliseconds on the `now` clock. + + Returns: + A map with `:now`, `:frame`, `:sleep`, `:check`, `:render`, + `:render-sync`, `:render-from-cache`, `:set-view`, `:set-view-start` and + `:set-view-end`. `:check` rejects work after the owner is superseded, the + WebGL context is lost or the deadline passes." + [epoch deadline-ms] + {:now now + ;; rAF scheduling + :frame #(js/Promise. + (fn [resolve] (.requestAnimationFrame js/window resolve))) + :sleep #(js/Promise. (fn [resolve] (js/setTimeout resolve %))) + ;; checks that this execution still owns the page, WebGL context hasn't been + ;; lost, and the deadline hasn't passed + :check (fn [] + (when @wasm/context-lost? + (throw (ex-info "WebGL context lost" + {:phase "context-lost"}))) + (guard-current! epoch "protocol") + (when (>= (now) deadline-ms) + (throw (ex-info "case deadline exceeded" + {:phase "deadline"})))) + ;; WASM calls + :render (fn [timestamp flags] + (h/call wasm/internal-module "_render" timestamp flags)) + :render-sync #(h/call wasm/internal-module "_render_sync") + :render-from-cache #(h/call wasm/internal-module "_render_from_cache" 0) + :set-view #(h/call wasm/internal-module "_set_view" + (:scale %) (- (:x %)) (- (:y %))) + :set-view-start #(h/call wasm/internal-module "_set_view_start") + :set-view-end #(h/call wasm/internal-module "_set_view_end")}) + + +;; Monotonic owner generation. Async continuations capture the current epoch +;; and compare it before acting; a mismatch marks their work stale. +(defonce ^:private owner-epoch* + (atom 0)) + +;; Cached Emscripten import. Import precedes module-init timing (untimed). +(defonce ^:private module-promise* + (atom nil)) + +;; Page-scoped canvas owned by the current load. :fresh cases load once per +;; page. :reuse cases keep one preparation across all attempts. +(defonce ^:private canvas* + (atom nil)) + +;; Current completed preparation for `run-prepared`: owner, case, reusable +;; command context and browser-local graphics. +(defonce ^:private prepared* + (atom nil)) + + +(defn- mark-live! + "Marks the current renderer context as initialized and available." + [] + (reset! wasm/context-lost? false) + (set! wasm/context-initialized? true)) + +(defn- now + "Returns the browser's monotonic clock in milliseconds." + [] + (.now js/performance)) + +(defn- ensure-module! + "Imports the Emscripten glue and instantiates it with the prepared `.wasm` + bytes. + + The import is cached per page and untimed: script import precedes + module-init timing. `locateFile` points at `wasm-url`. + + Args: + module-url: URL of the Emscripten glue module. + wasm-url: URL of the `.wasm` file passed to `locateFile`. + + Returns: + A Promise of `{:module instance :factory-ms ms}`, where `:factory-ms` is + the instantiation duration in milliseconds. A failed import or factory + call rejects the Promise." + [module-url wasm-url] + (when (nil? @module-promise*) + (reset! module-promise* + (-> (js/dynamicImport module-url) + (.then (fn [module] + (let [t0 (now) + default-fn (unchecked-get module "default")] + (-> (default-fn #js {:locateFile (constantly wasm-url)}) + (.then (fn [instance] + {:module instance + :factory-ms (- (now) t0)}))))))))) + @module-promise*) + +(defn- graphics-string + "Reads a WebGL string parameter. + + Args: + context: WebGL context to query. + parameter: property name on `WEBGL_debug_renderer_info`, such as + `\"UNMASKED_RENDERER_WEBGL\"`. + + Returns: + The parameter's string value, or `\"unknown\"` when the extension is + unavailable or the read throws." + [context parameter] + (or (ex/ignoring + (when-let [ext (.getExtension ^js context "WEBGL_debug_renderer_info")] + (str (.getParameter ^js context (unchecked-get ext parameter))))) + "unknown")) + +(defn- renderer-string + "Reads the unmasked WebGL renderer identity. + + Args: + context: WebGL context to query. + + Returns: + The unmasked renderer text, or `\"unknown\"` when the debug extension is + unavailable or the read throws." + [context] + (graphics-string context "UNMASKED_RENDERER_WEBGL")) + +(defn- software-renderer? + "Recognizes common software graphics implementations from their renderer text. + + Args: + s: renderer text to inspect; nil is treated as empty. + + Returns: + `true` when the text names SwiftShader, llvmpipe, generic software or a + basic render, ignoring case; `false` otherwise." + [s] + (boolean (re-find #"(?i)swiftshader|llvmpipe|software|basic render" (or s "")))) + +(defn- teardown-canvas! + "Releases the context after successful initialization, then removes its + canvas. + + The initializer owns cleanup until it returns a context. + + Args: + canvas: canvas to remove; nil skips both steps. + + Returns: + nil after the cleanup." + [canvas] + (when (some? canvas) + (when-let [context wasm/gl-context] + (webgl/release-context! wasm/internal-module context wasm/gl-context-handle)) + (ex/ignoring (.remove ^js canvas)))) + +(defn- release-owner-resources! + "Releases the current owner's canvas, GL handles, module binding and + stored preparation. + + Returns: + nil after releasing the resources." + [] + (teardown-canvas! @canvas*) + (wasm/reset-context-state!) + ;; reset-context-state! deliberately preserves loss state for the editor + ;; reload path; bench disposal owns it explicitly. + (reset! wasm/context-lost? false) + (set! wasm/internal-module nil) + (reset! canvas* nil) + (reset! prepared* nil)) + +(defn dispose! + "Closes the current owner and returns a JavaScript status object. + + It bumps the epoch, so running continuations go stale, releases renderer + resources and drops the cached module promise so a later load re-imports + cleanly. Idempotent. Stale callers must not clean up a newer owner; use + `dispose-if-current!` from async continuations. + + Returns: + A JavaScript object with status `\"disposed\"`." + [] + (swap! owner-epoch* inc) + (release-owner-resources!) + (reset! module-promise* nil) + #js {"status" "disposed"}) + +(defn- dispose-if-current! + "Disposes only when `epoch` is still current. + + Args: + epoch: owner epoch captured by the caller. + + Returns: + `true` after disposing, or nil for stale work. Stale work leaves the newer + owner's canvas, GL context and module cache unchanged." + [epoch] + (when (= epoch @owner-epoch*) + (dispose!) + true)) + + +(defn- read-graphics + "Records the effective graphics settings for a live WebGL context. + + The result separates the requested CSS size and device pixel ratio from the + observed drawing-buffer size and the context's actual attributes. Requested + defaults live in `webgl/default-context-options`; the drawing-buffer size is + the context's actual `drawingBufferWidth` and `drawingBufferHeight`, not the + canvas attributes. Granted attributes come from `getContextAttributes`. + + Args: + ctx: live WebGL context to inspect. + width: requested CSS width in pixels. + height: requested CSS height in pixels. + dpr: requested device pixel ratio. + renderer: unmasked renderer text to record. + + Returns: + A map with the requested `:css-width`, `:css-height` and `:dpr`; the + observed `:drawing-buffer-width` and `:drawing-buffer-height`; `:webgl2` + set to `true`; the granted `:antialias`, `:depth`, `:stencil`, `:alpha` + and `:preserve-drawing-buffer` attributes; and the `:renderer`, `:vendor` + and `:software` identity fields." + [ctx width height dpr renderer] + (let [attrs (ex/ignoring (.getContextAttributes ^js ctx)) + attr #(when (some? attrs) + (unchecked-get attrs %)) + vendor (graphics-string ctx "UNMASKED_VENDOR_WEBGL")] + {:css-width width + :css-height height + :drawing-buffer-width (.-drawingBufferWidth ^js ctx) + :drawing-buffer-height (.-drawingBufferHeight ^js ctx) + :dpr dpr + :webgl2 true + :antialias (attr "antialias") + :depth (attr "depth") + :stencil (attr "stencil") + :alpha (attr "alpha") + :preserve-drawing-buffer (attr "preserveDrawingBuffer") + :renderer renderer + :vendor vendor + :software (software-renderer? renderer)})) + +(defn- install-listeners! + "Wires context lost/restored handling for `canvas`, guarded by the epoch. + + A stale canvas leaves the live owner untouched. On loss the current owner + releases its canvas and GL resources, clears the module promise, marks the + context lost and supersedes its epoch immediately. A restore event + supersedes the epoch when its owner still matches. + + Args: + canvas: canvas that receives the listeners. + epoch: owner epoch captured when the listeners are installed. + + Returns: + nil after installing both listeners." + [canvas epoch] + (.addEventListener + canvas "webglcontextlost" + (let [installed epoch] + (fn [e] + (.preventDefault ^js e) + (when (= installed @owner-epoch*) + (release-owner-resources!) + (reset! module-promise* nil) + (reset! wasm/context-lost? true) + (swap! owner-epoch* inc))))) + (.addEventListener + canvas "webglcontextrestored" + (let [installed epoch] + (fn [_] + (when (= installed @owner-epoch*) + (swap! owner-epoch* inc)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Public interface +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- terminal-failure + "Returns the bridge result for a rejected asynchronous step. + + Only a top-level `::errors/stale` marker decides staleness. A stale cause + returns `{:status \"stale\"}` and leaves the newer owner in place. Any other + cause disposes the owner when `epoch` is still current and returns failed + evidence. The evidence uses the phase from the cause, or `fallback-phase` + when absent, and keeps the original failure as plain data under `:cause`. + The function attaches stored preparation evidence when that preparation + belongs to `epoch`. + + Args: + epoch: owner epoch that the failed step belongs to. + cause: rejected value from the step. + fallback-phase: phase name used when the cause has no `:phase`. + + Returns: + `{:status \"stale\"}` for a top-level stale cause; otherwise failed + evidence with status `\"failed\"`, `:phase`, `:message`, `:cause`, optional + `:partial` data and optional `:preparation` evidence." + [epoch cause fallback-phase] + (if (errors/stale? cause) + {:status "stale"} + (let [phase (or (ex/ignoring (:phase (ex-data cause))) fallback-phase "aborted") + prepared @prepared* + preparation (when (= epoch (get-in prepared [:owner :epoch])) + (get-in prepared [:graphics :preparation]))] + (dispose-if-current! epoch) + (cond-> (errors/fail-data (errors/evidence phase cause)) + preparation (assoc :preparation preparation))))) + +(defn- guard-current! + "Throws the stale marker when `epoch` no longer owns the page. + + Callers pass their phase so the terminal handler can tell superseded work + from genuine failures. The marker is `::errors/stale` in `ex-data`; message + text is for humans only. + + Args: + epoch: owner epoch captured by the caller. + phase: phase name recorded in the stale marker. + + Returns: + nil when `epoch` is current. + + Throws: + An exception when `epoch` is stale, with `::errors/stale true` and + `:phase` in its `ex-data`." + [epoch phase] + (when-not (= epoch @owner-epoch*) + (throw (ex-info "superseded" {::errors/stale true :phase phase})))) + +(defn- build-canvas! + "Creates the bench canvas with a `width*dpr` backing buffer and an explicit + CSS size, so the drawn scene fills exactly the reported viewport. + + Args: + width: viewport width in CSS pixels. + height: viewport height in CSS pixels. + dpr: device pixel ratio used for the backing buffer. + + Returns: + An unattached canvas element whose backing buffer is `width*dpr` by + `height*dpr` and whose CSS size is `width` by `height` pixels." + [width height dpr] + (let [canvas (.createElement js/document "canvas")] + (aset canvas "width" (* width dpr)) + (aset canvas "height" (* height dpr)) + (aset (.-style ^js canvas) "width" (str width "px")) + (aset (.-style ^js canvas) "height" (str height "px")) + canvas)) + +(defn- ^:async run-phase + "Executes one lifecycle phase through the command executor. + + `run` receives a command context and returns the next context or a Promise + of it. The raw block keeps `:phase`, `:preparation-id`, `:outcome`, + selected command `:events` and optional `:diagnostics`, and is appended to + `phases`. + + Args: + phase: lifecycle phase name, such as `:preparation`, `:main` or `:reset`. + preparation-id: identifier recorded in the raw block. + rtx: command context supplied to the command executor. + run: function from a command context to the next context or a Promise. + phases: volatile that receives each raw phase block. + + Returns: + A Promise of the next command context. A failed command outcome rejects + the Promise with a phase-labeled error after the block is recorded." + [phase preparation-id rtx run phases] + (let [outcome (await (commands/execute rtx run)) + block (assoc (select-keys outcome [:outcome :events :diagnostics]) + :phase phase :preparation-id preparation-id)] + (vswap! phases conj block) + (when (= :failed (:outcome outcome)) + (let [cause (:error outcome)] + (throw (errors/with-context cause (errors/message-of cause "phase failed") + {:phase (or (:phase (ex-data cause)) (name phase))})))) + (:context outcome))) + +(defn- ^:async run-case + "Runs main and the optional reset against one prepared command context. + + Main executes the local case's `run!` function. Reset executes only after a + successful main. The context returned by main, or by reset when present, is + stored for the next reused attempt, so camera changes carry over. + + Args: + owner: map with `:epoch`, the owner epoch, and `:hooks`, the hooks used by + main and reset. + case-desc: collected case descriptor. + local-case: local case registration with `run!` and optional `:reset`. + graphics: browser-local graphics map containing the preparation evidence + and snapshot. + context: prepared command context for main. + + Returns: + A Promise of status `\"ok\"` with `:preparation` evidence, `:phases` for + main and any reset, `:scene` with the shape count in `:shapes` and the + seed in `:seed`, and `:effective-graphics`. A non-stale failure resolves + to failed evidence with this attempt's preparation and blocks, even when + a newer owner exists. A top-level `::errors/stale` marker resolves to the + stale result." + [{:keys [epoch hooks]} case-desc local-case graphics context] + (let [phases (volatile! []) + preparation (:preparation graphics) + preparation-id (:preparation-id preparation)] + (try + (let [rtx (assoc context :hooks hooks) + rtx (await (run-phase :main preparation-id rtx (:run! local-case) phases)) + rtx (if-some [reset (:reset local-case)] + (await (run-phase :reset preparation-id rtx reset phases)) + rtx)] + (guard-current! epoch "main") + (when-not (wasm/live?) + (throw (ex-info "disposed or context lost during case" {:phase "main"}))) + (swap! prepared* assoc :context rtx) + {:status "ok" + :preparation preparation + :phases @phases + :scene {:shapes (count (:objects (:snapshot graphics))) + :seed (:scene-seed case-desc)} + :effective-graphics (:effective-graphics preparation)}) + (catch :default cause + (cond-> (terminal-failure epoch cause "main") + (not (errors/stale? cause)) (assoc :preparation preparation :phases @phases)))))) + +(defn- parse-load-request + "Decodes and validates a Transit scene request before claiming the page. + + Args: + request: Transit string containing the fields in + `schema:load-scene-args`. + + Returns: + A map with `:params`, `:case-desc`, `:local-case` and `:scene`, or the + validation Error. Failure leaves the current owner in place." + [request] + (attempt + (let [params (t/decode-str request)] + (when-not (sm/validate schema:load-scene-args params) + (throw (ex-info "load-scene needs a Transit request with case" + {:phase "invalid-args"}))) + (let [case-desc (:case params) + scene (decl/registered-scene (:scene case-desc)) + local (decl/registered-case (:id case-desc))] + (when (nil? scene) + (throw (ex-info (str "unknown scene: " (:scene case-desc)) + {:phase "invalid-args"}))) + (when-not (and local + (fn? (:run! local)) + (= (:scene local) (:scene case-desc))) + (throw (ex-info (str "missing case body for " (:id case-desc)) + {:phase "invalid-args"}))) + (decl/check-registered-case! local) + {:params params + :case-desc case-desc + :local-case local + :scene scene})))) + +(defn- begin-load! + "Claims the page, releases the prior owner and resolves the viewport. + + It increments the owner epoch, releases the prior owner's canvas, graphics + resources and preparation, and clears the module promise so the next load + imports fresh. The caller supplies hooks and a deadline. + + Args: + case-desc: collected case descriptor whose `:view` supplies viewport + overrides. + + Returns: + A map with `:epoch`, the new owner epoch, and `:dims`, the resolved + viewport with `:width`, `:height` and `:dpr`." + [case-desc] + (let [epoch (swap! owner-epoch* inc)] + (release-owner-resources!) + (reset! module-promise* nil) + (let [viewport (decl/resolve-viewport (:view case-desc))] + {:epoch epoch + :dims viewport}))) + +(defn- phase-error + [phase message cause] + (ex-info message {:phase phase} cause)) + +(defn- load-module + "Loads and instantiates the renderer. + + Args: + epoch: owner epoch checked after the module resolves. + params: decoded request map; `:module-url` and `:wasm-url` override the + renderer artifact URLs. + + Returns: + A Promise of the module map with `:module` and `:factory-ms`. A failed + import or factory call rejects with the module-init phase and its cause + for bridge accounting; a superseded owner rejects with the stale marker." + [epoch params] + (-> (ensure-module! (or (:module-url params) "./render-wasm.js") + (or (:wasm-url params) "./render-wasm.wasm")) + (.catch #(throw (phase-error "module-init" + (str "module init failed: " (or (ex-message %) %)) + %))) + (.then (fn [installed] + (guard-current! epoch "module-init") + installed)))) + +(defn- initialize-graphics! + "Creates the canvas and WebGL context and binds them to the renderer. + + Args: + epoch: owner epoch used by the context-loss listeners. + viewport: map with `:width`, `:height` and `:dpr`. + installed: module map with `:module` and `:factory-ms`. + + Returns: + A map with `:module-ms`, the factory instantiation duration in + milliseconds; `:graphics-ms`, the context creation, renderer binding, + viewbox resize and listener installation duration in milliseconds; + `:renderer`, the unmasked renderer text; and `:ctx`, the graphics + resources kept in the browser. + + Throws: + The stale marker when `epoch` is no longer current. Context creation or + viewbox resize failures throw an exception with phase `graphics-init`." + [epoch {:keys [width height dpr]} {:keys [module factory-ms]}] + (guard-current! epoch "graphics-init") + (set! wasm/internal-module module) + (let [canvas (build-canvas! width height dpr)] + (.appendChild (.-body js/document) canvas) + (reset! canvas* canvas) + (let [g0 (now) + ctx (try + (webgl/init-context + canvas + {:module module + :context-id "webgl2" + :css-width width + :css-height height + :dpr dpr + :flags 0 + :browser (sr/translate-browser :chrome) + :params {}}) + (catch :default cause + (throw (phase-error "graphics-init" "graphics init failed" cause))))] + (when (nil? ctx) + (throw (phase-error "graphics-init" "WebGL2 context unavailable" nil))) + (set! wasm/gl-context-handle (:handle ctx)) + (set! wasm/gl-context (:context ctx)) + (set! wasm/canvas canvas) + (mark-live!) + (try + (h/call wasm/internal-module "_resize_viewbox" width height) + (catch :default cause + (throw (phase-error "graphics-init" "resize viewbox failed" cause)))) + (install-listeners! canvas epoch) + {:module-ms factory-ms + :graphics-ms (- (now) g0) + :renderer (renderer-string (:context ctx)) + :ctx ctx}))) + +(defn- prepare-scene + "Builds the registered scene, then fingerprints and orders its shapes. + + The scene builder returns a validated snapshot. Fingerprinting and upload + order derivation finish before the upload timer begins. + + Args: + epoch: owner epoch checked before the scene is built. + scene: registered scene declaration with a `:build` function. + case-desc: collected case descriptor whose `:params` are passed to the + builder. + seed: scene seed supplied as `:seed` to the builder. + graphics: graphics map to extend. + + Returns: + `graphics` with `:snapshot`, `:fingerprint` and `:ordered`. + + Throws: + The stale marker when `epoch` is no longer current. Other scene + preparation failures throw an exception with phase `scene-build`." + [epoch scene case-desc seed graphics] + (guard-current! epoch "scene-build") + (try + (let [snapshot ((:build scene) (assoc (:params case-desc) :seed seed))] + (assoc graphics + :snapshot snapshot + :fingerprint (fingerprint/fingerprint snapshot) + :ordered (snapshot/upload-order snapshot))) + (catch :default cause + (if (errors/stale? cause) + (throw cause) + (throw (phase-error "scene-build" "scene build failed" cause)))))) + +(defn- upload-scene! + "Sets the initial view and uploads prepared shapes inside the upload timer. + + The timer covers view setup, pool initialization, the loading lifecycle and + batch serialization. The loading lifecycle ends even when serialization + fails. + + Args: + epoch: owner epoch checked before uploading. + case-desc: collected case descriptor whose `:view` sets the camera. + graphics: prepared graphics map with `:snapshot` and `:ordered`. + + Returns: + `graphics` with `:upload-ms`, the upload duration in milliseconds, and + `:view`, the camera view that was set. + + Throws: + The stale marker when `epoch` is no longer current. Other upload failures + throw an exception with phase `upload`." + [epoch case-desc graphics] + (guard-current! epoch "upload") + (try + (let [u0 (now) + view (:view case-desc)] + (h/call wasm/internal-module "_set_view" + (:scale view) (- (:x view)) (- (:y view))) + (h/call wasm/internal-module "_init_shapes_pool" + (count (:objects (:snapshot graphics)))) + (h/call wasm/internal-module "_begin_loading") + (let [uploaded (attempt :catch-fatal + (serialize-shape/serialize-shapes-batch + (:ordered graphics) + {:include-layout? false + :include-fills-strokes? true}) + nil) + ended (attempt :catch-fatal (h/call wasm/internal-module "_end_loading") nil)] + (when (or uploaded ended) + (throw (errors/combine + [{:phase "upload" :operation :serialize-shapes :error uploaded} + {:phase "upload" :operation :end-loading :error ended}])))) + (h/call wasm/internal-module "_set_view_end") + (assoc graphics :upload-ms (- (now) u0) :view view)) + (catch :default cause + (if (errors/stale? cause) + (throw cause) + (throw (errors/with-context cause "upload failed" {:phase "upload"})))))) + +(defn- preparation-data + "Projects completed setup into preparation evidence. + + Args: + id: preparation identifier. + case-desc: collected case descriptor. + effective-graphics: graphics facts read from the live context. + graphics: browser-local graphics map with timings, snapshot, fingerprint + and optional `:setup-render`. + phases: raw preparation phase blocks. + + Returns: + Portable identity data, graphics facts, raw phase blocks and setup + diagnostics." + [id case-desc effective-graphics graphics phases] + (let [{:keys [module-ms graphics-ms upload-ms snapshot fingerprint setup-render]} graphics] + {:preparation-id id + :case-id (:id case-desc) + :phases phases + :fingerprint fingerprint + :effective-graphics effective-graphics + :diagnostics (cond-> {:module-init-ms module-ms :graphics-init-ms graphics-ms + :upload-ms upload-ms + :scene {:shapes (count (:objects snapshot)) :seed (:scene-seed case-desc)}} + setup-render (assoc :setup-render setup-render))})) + +(defn- command-context + "Returns the four-field command context for one preparation. + + Args: + case-desc: collected case descriptor. + hooks: browser hooks for the current owner and deadline. + graphics: prepared graphics map containing `:snapshot` and `:view`. + + Returns: + The command context with the uploaded snapshot in `:scene` and the + current camera view in `:view`." + [case-desc hooks graphics] + {:case case-desc + :scene (:snapshot graphics) + :view (:view graphics) + :hooks hooks}) + +(defn- ^:async prepare-initial-view + "Runs the preparation callback, then renders the initial Full frame. + + Args: + callback: function from a command context to the next context or a + Promise. + render-mode: `:none` skips the initial render; every other value renders. + setup-render: volatile that receives the drain trace when rendering runs. + rtx: command context passed to the callback. + + Returns: + A Promise of the callback's resulting context. The drain uses that + context's hooks, starts with flags 0 and records its traces and durations + in `setup-render`. A callback or drain failure rejects the Promise." + [callback render-mode setup-render rtx] + (let [rtx (await (callback rtx))] + (when-not (= :none render-mode) + (let [hooks (:hooks rtx)] + (vreset! setup-render + (await (protocol/drain hooks {:flags 0 + :origin ((:now hooks)) + :immediate false}))))) + rtx)) + +(defn- ^:async prepare-context + "Runs the preparation phase and attaches its context to `graphics`. + + The local case supplies the preparation callback; `identity` is used when + none is registered. + + Args: + preparation: map with `:owner`, `:case-desc`, `:local-case` and + `:preparation-id`. + phases: volatile that receives the raw phase block. + graphics: prepared graphics map with the uploaded snapshot and view. + + Returns: + A Promise of `graphics` with `:context`, the context returned by the + callback and initial render, and `:setup-render`, the automatic render + trace or nil. A failed preparation phase rejects the Promise." + [{:keys [owner case-desc local-case preparation-id]} phases graphics] + (let [setup-render (volatile! nil) + callback (get-in local-case [:preparation :callback] identity) + render-mode (get-in case-desc [:preparation :render]) + context (command-context case-desc (:hooks owner) graphics) + context (await (run-phase :preparation preparation-id context + (partial prepare-initial-view callback render-mode setup-render) + phases))] + (assoc graphics :context context :setup-render @setup-render))) + +(defn- prepared-state + "Collects the reusable context, graphics resources and preparation evidence. + + Args: + preparation: case definitions, owner and timeout settings. + graphics: prepared graphics and the completed command context. + evidence: portable preparation evidence. + + Returns: + State ready for its first execution, with the context stored separately + from graphics." + [preparation graphics evidence] + (assoc (select-keys preparation [:owner :case-desc :local-case :timeout-ms]) + :busy? false + :used? false + :context (:context graphics) + :graphics (assoc (dissoc graphics :context) :preparation evidence))) + +(defn- complete-preparation + "Reads effective graphics, records the completed preparation and returns it. + + Args: + preparation: map with `:owner`, `:case-desc` and `:preparation-id`. + phases: volatile of raw preparation blocks. + graphics: prepared graphics map with the live context and setup facts. + + Returns: + `{:status \"ok\" :preparation evidence}` after storing the preparation for + later attempts. + + Throws: + A top-level stale marker when the owner changed during preparation." + [{:keys [owner case-desc preparation-id] :as preparation} phases graphics] + (guard-current! (:epoch owner) "prepare") + (let [{:keys [width height dpr]} (:dims owner) + {:keys [ctx renderer]} graphics + effective-graphics (read-graphics (:context ctx) width height dpr renderer) + evidence (preparation-data preparation-id case-desc effective-graphics graphics @phases)] + (reset! prepared* (prepared-state preparation graphics evidence)) + {:status "ok" :preparation evidence})) + +(defn- record-setup + "Records setup facts for partial failure evidence and returns `graphics`. + + The stored keys are `:module-ms`, `:graphics-ms`, `:fingerprint` and + `:upload-ms` when present. + + Args: + available: volatile map that receives the available setup facts. + graphics: graphics map to read the facts from. + + Returns: + `graphics`, unchanged." + [available graphics] + (vreset! available (select-keys graphics [:module-ms :graphics-ms :fingerprint :upload-ms])) + graphics) + +(defn- preparation-failure + "Returns failure evidence for a preparation that did not complete. + + A top-level stale cause returns the stale result. + + Args: + epoch: owner epoch that the preparation belongs to. + available: volatile of setup facts recorded before the failure. + phases: volatile of raw preparation blocks. + cause: failure that ended preparation. + + Returns: + `{:status \"stale\"}` for a top-level stale cause; otherwise failed + evidence that keeps the recorded setup facts under `:partial :setup`, any + partial render data from the cause, and the raw preparation blocks at the + top level under `:phases`. It releases the owner when `epoch` is still + current." + [epoch available phases cause] + (if (errors/stale? cause) + (terminal-failure epoch cause "aborted") + (assoc (terminal-failure epoch + (errors/with-context cause (errors/message-of cause "setup failed") + {:partial {:setup @available}}) + "aborted") + :phases @phases))) + +(defn- ^:async prepare-owned-scene + "Prepares a claimed scene from a supplied module installation Promise. + + Args: + preparation: map with `:owner`, `:case-desc`, `:scene`, `:local-case` and + `:preparation-id`. + module-promise: Promise of the installed module map; the function awaits it + before graphics initialization and scene preparation. + + Returns: + A Promise of `{:status \"ok\" :preparation evidence}` on success. + Ordinary failure returns failed evidence with the failed phase, any + recorded setup facts under `:partial :setup`, and the raw preparation + blocks at the top level under `:phases`. A top-level stale marker produces + the stale result." + [{:keys [owner case-desc scene] :as preparation} module-promise] + (let [{:keys [epoch dims]} owner + available (volatile! {}) + phases (volatile! [])] + (try + (let [module (await module-promise) + graphics (->> module + (initialize-graphics! epoch dims) + (record-setup available)) + graphics (->> graphics + (prepare-scene epoch scene case-desc (:scene-seed case-desc)) + (record-setup available)) + graphics (->> graphics + (upload-scene! epoch case-desc) + (record-setup available)) + graphics (await (prepare-context preparation phases graphics))] + (complete-preparation preparation phases graphics)) + (catch :default cause + (preparation-failure epoch available phases cause))))) + +(defn- prepare-request + "Validates a Transit scene request, claims the page and prepares the scene. + + Validation runs before the owner changes. A valid request releases the prior + owner, starts a new owner epoch, builds browser hooks with a deadline and + prepares the scene. `:preparation-id` defaults to the new epoch and + `:timeout-ms` defaults to `case-ms-budget`. + + Args: + request: Transit string matching `schema:load-scene-args`. + + Returns: + A Promise of `{:status \"ok\" :preparation evidence}`, failed evidence with + a phase, message, cause and optional partial data, or the stale result. A + preparation failure also carries its available blocks under top-level + `:phases`. Invalid arguments resolve as failed `invalid-args` evidence and + leave the current owner untouched." + [request] + (try + (let [parsed (parse-load-request request)] + (if (failure? parsed) + (js/Promise.resolve (errors/fail-data (errors/evidence "invalid-args" parsed))) + (let [{:keys [params case-desc]} parsed + {:keys [epoch] :as owner} (begin-load! case-desc) + timeout-ms (or (:timeout-ms params) case-ms-budget) + deadline-ms (+ (now) timeout-ms) + owner (assoc owner :deadline-ms deadline-ms + :hooks (browser-hooks epoch deadline-ms))] + (prepare-owned-scene (assoc parsed + :owner owner + :timeout-ms timeout-ms + :preparation-id (or (:preparation-id params) epoch)) + (load-module epoch params))))) + (catch :default cause + (js/Promise.resolve (errors/fail-data (errors/evidence "setup" cause)))))) + +(defn- execute-prepared + "Runs main and optional reset against the stored preparation. + + Args: + preparation-id: identifier of the prepared scene to use. + timeout-ms: optional remaining budget in milliseconds, bounded by the + preparation's case deadline. + + Returns: + A Promise of status ok with `:preparation` evidence, top-level `:phases` + for main and any reset, `:scene` shape count and seed, and + `:effective-graphics`; or failed evidence. A top-level stale marker + produces the stale result. + + A missing, busy or mismatched preparation, or a fresh preparation already + used, resolves as failed `invalid-args` evidence, leaving the owner + unchanged. A non-stale phase failure releases that preparation, so later + attempts find it unavailable." + [{:keys [preparation-id timeout-ms]}] + (let [{:keys [owner case-desc local-case graphics context busy? used?] :as prepared} @prepared* + id (get-in graphics [:preparation :preparation-id])] + (if (or (nil? prepared) (not= id preparation-id) busy? + (and used? (= :fresh (:context case-desc)))) + (js/Promise.resolve (errors/fail-data {:phase "invalid-args" + :message "preparation unavailable, busy or already used"})) + (let [epoch (:epoch owner) + deadline-ms (if timeout-ms + (min (:deadline-ms owner) (+ (now) timeout-ms)) + (:deadline-ms owner)) + owner (assoc owner :hooks (browser-hooks epoch deadline-ms))] + (swap! prepared* assoc :busy? true :used? true) + (-> (run-case owner case-desc local-case graphics context) + (.then (fn [evidence] + (when (= epoch @owner-epoch*) + (swap! prepared* assoc :busy? false)) + evidence)) + (.catch #(terminal-failure epoch % "attempt"))))))) + +(defn- encodable-phase-prefix + [phases] + (reduce (fn [kept phase] + (try + (codec/encode-str phase) + (conj kept phase) + (catch :default _ (reduced kept)))) + [] phases)) + +(defn- encode-response + [epoch evidence] + (try + (codec/encode-str evidence) + (catch :default cause + (dispose-if-current! epoch) + (let [encoding (assoc (errors/evidence :bridge cause) :operation :encode-response) + original (when (= "failed" (:status evidence)) (errors/project-diagnostic evidence)) + recovered (if original + (assoc original :errors (conj (or (:errors original) + [(dissoc original :partial)]) encoding)) + encoding) + recovered (cond-> recovered + (seq (get-in evidence [:preparation :phases])) + (assoc :phases (encodable-phase-prefix (get-in evidence [:preparation :phases])))) + recovered (reduce (fn [out key] + (if (contains? evidence key) + (try + (codec/encode-str (get evidence key)) + (assoc out key (get evidence key)) + (catch :default _ + (if (= :phases key) + (assoc out :phases (encodable-phase-prefix (:phases evidence))) + out))) + out)) + recovered + [:preparation :phases :scene :effective-graphics])] + (t/encode-str (errors/fail-data recovered)))))) + +(defn- encode-evidence + "Encodes bridge evidence after timing and resolves failures as bridge + failures. + + Encoding runs after all measured work. If `promise` rejects or encoding + fails, the owner is disposed when the captured epoch is still current. + + Args: + promise: Promise of bridge evidence to encode. + + Returns: + A Promise of Transit text for the supplied evidence, or for failed bridge + evidence when a failure occurs." + [promise] + (let [epoch @owner-epoch*] + (-> promise + (.then (partial encode-response epoch)) + (.catch (fn [cause] + (dispose-if-current! epoch) + (t/encode-str (errors/fail-data (errors/evidence :bridge cause)))))))) + +(defn prepare-scene-request + "Prepares a scene from a Transit request and returns Transit evidence. + + Args: + request: Transit string accepted by `schema:load-scene-args`. + + Returns: + A Promise of Transit text. Success contains status ok and `:preparation` + evidence. Preparation failures contain status failed, phase, message and + available cause, partial evidence and blocks under top-level `:phases`. + Encoding failures report the bridge phase with status, phase and message. + A failure with a top-level stale marker returns status stale." + [request] + (encode-evidence (prepare-request request))) + +(defn run-prepared + "Runs one attempt against a prepared scene. + + Args: + request: Transit `{:preparation-id id :timeout-ms optional-positive-ms}`. + + Returns: + A Promise of Transit evidence. Success contains status ok, the preparation + evidence, top-level main and optional reset phase blocks, scene shape count + and seed, and effective graphics. Failures contain status failed, phase and + message. Phase failures also include their cause, preparation evidence, + recorded blocks and available partial data. Invalid requests report the + invalid-args phase; encoding failures report the bridge phase. A failure + with a top-level stale marker returns the stale result." + [request] + (encode-evidence + (try + (let [params (t/decode-str request)] + (if (sm/validate [:map {:closed true} + [:preparation-id some?] + [:timeout-ms {:optional true} [:int {:min 1}]]] params) + (execute-prepared params) + (js/Promise.resolve (errors/fail-data {:phase "invalid-args" + :message "runPrepared needs a preparation ID"})))) + (catch :default cause + (js/Promise.resolve (errors/fail-data {:phase "invalid-args" + :message (errors/message-of cause "invalid Transit request")})))))) + +(defn load-scene + "Prepares a collected case and runs one attempt. + + Args: + request: Transit string accepted by `prepare-scene-request`. + + Returns: + A Promise of Transit evidence. On success, `:preparation` holds the + preparation evidence and top-level `:phases` holds main and any reset. A + failed preparation places its available blocks at the top level. An + attempt failure retains the preparation evidence and recorded blocks + captured for that attempt. A failure with a top-level stale marker returns + the stale result." + [request] + (encode-evidence + (-> (prepare-request request) + (.then (fn [{:keys [status preparation] :as evidence}] + (if (= status "ok") + (execute-prepared {:preparation-id (:preparation-id preparation)}) + evidence)))))) + +(defn ping + "Reports that the bridge is callable. + + Returns: + A JavaScript object with status `\"ok\"`." + [] + #js {"status" "ok"}) + +(defn summarize-record + "Returns a Transit summary of a saved run record. + + Args: + record-text: Transit text for a supported run record. + analysis-text: optional Transit map of analysis settings. + + Returns: + Transit text for the summary. + + Throws: + When the run or settings cannot be decoded, validated, summarized or + encoded." + ([record-text] (summarize-record record-text nil)) + ([record-text analysis-text] + (codec/encode-str (summary/summarize (result/decode record-text) + (when analysis-text (t/decode-str analysis-text)))))) + +(defn compare-records + "Compares two saved runs and returns Transit comparison data. + + Args: + baseline-text: Transit text for the baseline run. + candidate-text: Transit text for the candidate run. + options-text: Transit map of comparison options. + + Returns: + Transit text for the comparison. + + Throws: + When a run or the options cannot be decoded, validated, compared or + encoded." + [baseline-text candidate-text options-text] + (result/encode (compare/compare-runs (result/decode baseline-text) + (result/decode candidate-text) + (t/decode-str options-text)))) + +(defn format-run + "Returns human-readable text for a saved run record. + + Args: + record-text: Transit text for a supported run record. + + Returns: + Text for the run summary. + + Throws: + When the run cannot be decoded, validated, summarized or formatted." + [record-text] + (format/format-run (summary/summarize (result/decode record-text)))) + +(defn format-comparison + "Returns human-readable text for a saved comparison record. + + Args: + record-text: Transit text for a supported comparison record. + + Returns: + Text for the comparison report. + + Throws: + When the comparison cannot be decoded, validated or formatted." + [record-text] + (format/format-comparison (result/decode record-text))) + +(def bridge + "Browser-side functions exported from the compiled module. + + Keys use the camelCase names used by the runner: `ping`, + `buildIdentity`, `loadScene`, `prepareScene`, `runPrepared`, `summarize`, + `compare`, `formatRun`, `formatComparison` and `dispose`. Each entry returns + Transit text, report text, or, for `ping` and `dispose`, a JavaScript status + object." + #js {"ping" ping + "buildIdentity" #(t/encode-str (identity/data)) + "loadScene" load-scene + "prepareScene" prepare-scene-request + "runPrepared" run-prepared + "summarize" summarize-record + "compare" compare-records + "formatRun" format-run + "formatComparison" format-comparison + "dispose" dispose!}) diff --git a/frontend/performance/benches/render_wasm/browser/index.html b/frontend/performance/benches/render_wasm/browser/index.html new file mode 100644 index 0000000000..dcf0d38dd9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/browser/index.html @@ -0,0 +1,33 @@ + + + + + Renderer benchmarks + + + + + + + + diff --git a/frontend/performance/benches/render_wasm/builder.cljc b/frontend/performance/benches/render_wasm/builder.cljc new file mode 100644 index 0000000000..3f7c35431b --- /dev/null +++ b/frontend/performance/benches/render_wasm/builder.cljc @@ -0,0 +1,512 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.builder + "Seeded construction of benchmark scenes. + + A `scene` consists of one seeded number generator, one object map, and one + label map. + + Shape and container calls attach objects to the current parent in evaluation + order. Coordinates are page-relative. Frames use authored bounds; groups and + booleans derive geometry from their children after the body completes. + + `scene` accepts a required :seed, optional :root attributes and :defaults. + It returns a validated snapshot with :objects and :refs. Labels name objects + within that snapshot. A container whose body or finalizer throws becomes + unfinished, and `finish!` rejects the snapshot. + + Default attributes follow the same theme: `*defaults*` maps a shape type to + an attribute generator map, bound by the `scene` macro from the scope + params. Per-shape attrs are merged over the generated values, so a recipe + can override values locally and rebind `*defaults*` around a section. + + Ordinary :require loads the macros and runtime helpers. Scene bodies support + let, doseq and calls to functions defined outside the scope." + #?(:cljs (:require-macros [benches.render-wasm.builder])) + #?(:cljs + (:require + [app.common.data :as d] + [app.common.files.helpers :as cfh] + [app.common.geom.shapes :as gsh] + [app.common.types.path :as path] + [app.common.types.shape :as cts] + [app.common.types.shape-tree :as ctst] + [app.common.uuid :as uuid] + [benches.render-wasm.random :as rnd :refer + [rng-float rng-hex-color rng-int rng-opacity rng-uuid]] + [benches.render-wasm.snapshot :as snapshot]))) + +#?(:clj + (do + (defn- emit-shape + [constructor label attrs] + `(~constructor + benches.render-wasm.builder/*state* + benches.render-wasm.builder/*defaults* + ~label + ~attrs)) + + (defn- emit-container + [type label-or-attrs body] + (let [labeled? (or (keyword? label-or-attrs) (vector? label-or-attrs)) + attrs (if labeled? (first body) label-or-attrs) + label (when labeled? label-or-attrs) + body (if labeled? (rest body) body)] + `(benches.render-wasm.builder/with-container + ~type ~attrs ~label (fn [] ~@body)))) + + (defmacro scene + "Runs `body` in a scene scope and returns the validated snapshot." + [params & body] + `(let [params# ~params + state# (benches.render-wasm.builder/start params#)] + (binding [benches.render-wasm.builder/*state* state# + benches.render-wasm.builder/*defaults* (:defaults params#)] + ~@body + (benches.render-wasm.builder/finish! state#)))) + + (defmacro rect + "Adds a rectangle and returns its uuid. + + Forms: (rect), (rect attrs), (rect label), (rect label attrs). + A single map argument supplies attributes; keywords and vectors label + the object in :refs. Attributes override the generated defaults." + ([] + (emit-shape 'benches.render-wasm.builder/rect! nil {})) + ([label-or-attrs] + (emit-shape 'benches.render-wasm.builder/rect! label-or-attrs {})) + ([label attrs] + (emit-shape 'benches.render-wasm.builder/rect! label attrs))) + + (defmacro path + "Adds a path and returns its uuid. + + Forms: (path), (path attrs), (path label), (path label attrs). + A single map argument supplies attributes; keywords and vectors label + the object in :refs. Attributes override the generated defaults." + ([] + (emit-shape 'benches.render-wasm.builder/path! nil {})) + ([label-or-attrs] + (emit-shape 'benches.render-wasm.builder/path! label-or-attrs {})) + ([label attrs] + (emit-shape 'benches.render-wasm.builder/path! label attrs))) + + (defmacro circle + "Adds an ellipse and returns its uuid. + + Forms: (circle), (circle attrs), (circle label), (circle label attrs). + A single map argument supplies attributes; keywords and vectors label + the object in :refs. Attributes override the generated defaults." + ([] + (emit-shape 'benches.render-wasm.builder/circle! nil {})) + ([label-or-attrs] + (emit-shape 'benches.render-wasm.builder/circle! label-or-attrs {})) + ([label attrs] + (emit-shape 'benches.render-wasm.builder/circle! label attrs))) + + (defmacro frame + "Adds a frame, runs `body` with the frame as parent, and returns its uuid. + + Forms: (frame attrs & body), (frame label attrs & body). + A keyword or vector form supplies a label in :refs. Attributes must + carry numeric :x :y :width :height. Children use page-relative + coordinates. Frames allow an empty body." + [label-or-attrs & body] + (emit-container :frame label-or-attrs body)) + + (defmacro group + "Adds a group, runs `body` with the group as parent, and returns its uuid. + + Forms: (group attrs & body), (group label attrs & body). + A keyword or vector form supplies a label in :refs. Groups derive their + geometry from children and require at least one child. With + :masked-group true, the first child supplies the mask geometry." + [label-or-attrs & body] + (emit-container :group label-or-attrs body)) + + (defmacro bool + "Adds a boolean shape, runs `body` with it as parent, and returns its uuid. + + Forms: (bool attrs & body), (bool label attrs & body). + A keyword or vector form supplies a label in :refs. Attributes require + :bool-type :union, :difference, :intersection or :exclude. + + Content and geometry derive from the children. Styles inherit from the + first child for :difference and the last child otherwise; supplied + non-nil styles take precedence. A bool requires at least one child and + accepts paths, rectangles, ellipses, groups and other booleans." + [label-or-attrs & body] + (emit-container :bool label-or-attrs body)))) + +#?(:cljs + (do + + + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Attribute generators + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + (defn gen-int + "Integer generator over `[min, max)`." + [min max] + (fn [rng] (rng-int rng min max))) + + (defn gen-float + "Float generator rounded to three decimals so snapshots stay readable." + [min max] + (fn [rng] (rnd/round3 (rng-float rng min max)))) + + (defn gen-hex-color + "Generator of a random `#rrggbb` string." + [] + (fn [rng] (rng-hex-color rng))) + + (defn gen-one-of + "Generator that picks one value per draw. Rejects an empty collection." + [values] + (when (empty? values) + (throw (ex-info "gen-one-of requires at least one value" + {:type ::no-values}))) + (fn [rng] (nth values (rng-int rng 0 (count values))))) + + (defn gen-fill + "Generator of one translucent solid fill." + [] + (fn [rng] + {:fill-color (rng-hex-color rng) + :fill-opacity (rng-opacity rng)})) + + (defn gen-stroke + "Centered stroke like the interim rectangle workload." + [width] + (fn [rng] + {:stroke-width width + :stroke-alignment :center + :stroke-color (rng-hex-color rng) + :stroke-opacity (rng-opacity rng)})) + + (defn gen-vector + "Composes generators into a generator of a vector, one value per generator." + [& generators] + (fn [rng] (mapv #(% rng) generators))) + + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Scope state + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + (def ^:dynamic *state* + "Current scene scope; bound by the `scene` macro." + nil) + + (def ^:dynamic *defaults* + "Per-shape-type attribute generator maps bound by the `scene` macro, + e.g. `{:rect {:x (gen-int 0 1920) ...}}`. Values are literals or one-arg + functions of the scope random source." + {}) + + (def ^:dynamic *parent* + "Current insertion point as `{:id parent-id :frame-id containing-frame}`; + bound by the container scopes, `uuid/zero` at the top." + {:id uuid/zero :frame-id uuid/zero}) + + (defn- check-seed! + "Requires a scene seed from 0 through 4294967295, retaining the builder + error type." + [seed] + (when-not (rnd/seed? seed) + (throw (ex-info "scene scope requires an integer :seed in [0, 2^32)" + {:type ::invalid-seed + :seed seed})))) + + (defn start + "Starts a scene scope and returns its state atom with objects, refs and :rng. + The seeded generator belongs to this scope; other scopes and report draws + cannot advance its state." + [params] + (when (some? *state*) + (throw (ex-info "scene scopes cannot nest" + {:type ::nested-scope}))) + (check-seed! (:seed params)) + (let [seed (:seed params) + root (cts/setup-shape (merge (:root params) + {:id uuid/zero + :type :frame + :name "Root Frame" + :parent-id uuid/zero + :frame-id uuid/zero + :shapes []}))] + (atom {:seed seed + :rng (rnd/create seed) + :objects {uuid/zero root} + :refs {} + :unfinished {}}))) + + (defn- ensure-state + [state] + (or state + (throw (ex-info "shape used outside a scene scope" + {:type ::outside-scope})))) + + (defn- resolve-attr + [rng spec] + (if (fn? spec) (spec rng) spec)) + + (defn- resolve-attrs + [rng defaults attrs] + (merge (reduce-kv (fn [result key spec] + (assoc result key (resolve-attr rng spec))) + {} + defaults) + attrs)) + + (defn- register-ref! + [state label id] + (when (some? label) + (when (contains? (:refs @state) label) + (throw (ex-info (str "duplicate scene label: " label) + {:type ::duplicate-label + :label label}))) + (swap! state assoc-in [:refs label] id))) + + (defn- add-object! + "Inserts `shape` under the current parent and registers `label`." + [state label shape] + (let [{:keys [id frame-id]} *parent*] + (register-ref! state label (:id shape)) + (swap! state + #(ctst/add-shape (:id shape) shape % frame-id id nil true)))) + + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; Shape constructors + ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + (defn- normalize-label-and-attrs + "Resolves map attributes and keyword/vector labels for shape construction." + [label attrs] + (cond + (and (map? label) (empty? attrs)) [nil label] + (nil? label) [nil attrs] + (or (keyword? label) (vector? label)) [label attrs] + :else (throw (ex-info "rect label must be a keyword or vector" + {:type ::invalid-label + :label label})))) + + (defn- construct-shape + [state defaults type label attrs] + (let [state (ensure-state state) + [label attrs] (normalize-label-and-attrs label attrs) + rng (:rng @state) + attrs (resolve-attrs rng (get defaults type) attrs) + id (rng-uuid rng) + shape (cts/setup-shape (merge attrs + {:id id + :type type}))] + (add-object! state label shape) + id)) + + (defn rect! + "Adds a rectangle with the selected defaults and attributes, returning its uuid." + [state defaults label attrs] + (construct-shape state defaults :rect label attrs)) + + (defn path! + "Adds a path with the selected defaults and attributes, returning its uuid." + [state defaults label attrs] + (construct-shape state defaults :path label attrs)) + + (defn circle! + "Adds an ellipse with the selected defaults and attributes, returning its uuid." + [state defaults label attrs] + (construct-shape state defaults :circle label attrs)) + + (def ^:private frame-bounds-keys + "Frames do not derive their bounds from children." + [:x :y :width :height]) + + (def ^:private group-geometry-keys + "Group geometry derives from its children." + [:x :y :width :height :selrect :points]) + + (def ^:private bool-geometry-keys + "Bool geometry, content and placement derive from its children." + [:x :y :width :height :selrect :points :content]) + + (def ^:private bool-transform-keys + "Construction does not accept transforms: `path/update-geometry` would + pivot them on the empty pre-finalization selrect." + [:transform :transform-inverse :rotation :flip-x :flip-y]) + + (defn- reject-attrs! + "Throws when `attrs` carries any of `attr-keys`. Shared by containers + whose geometry or content derives from their children." + [attrs id error-type message attr-keys] + (doseq [key attr-keys] + (when (some? (get attrs key)) + (throw (ex-info message + {:type error-type + :id id + :key key}))))) + + (defn- setup-frame + [id attrs] + (doseq [key frame-bounds-keys] + (let [value (get attrs key)] + (cond + (nil? value) + (throw (ex-info "frame requires :x :y :width :height" + {:type ::frame-bounds + :id id + :missing key})) + + (not (d/num? value)) + (throw (ex-info "frame bounds must be numbers" + {:type ::frame-bounds + :id id + :invalid key}))))) + (cts/setup-shape (assoc attrs :id id :type :frame))) + + (defn- setup-group + [id attrs] + (reject-attrs! attrs id ::group-geometry + "group derives its geometry from children" + group-geometry-keys) + (cts/setup-shape (assoc attrs :id id :type :group))) + + (defn- setup-bool + [id attrs] + (reject-attrs! attrs id ::bool-geometry + "bool derives its geometry and content from children" + bool-geometry-keys) + (reject-attrs! attrs id ::bool-transform + "bool construction does not accept transforms" + bool-transform-keys) + (when-not (contains? cts/bool-types (:bool-type attrs)) + (throw (ex-info "bool requires :bool-type :union :difference :intersection or :exclude" + {:type ::bool-type + :id id + :bool-type (:bool-type attrs) + :supported cts/bool-types}))) + (cts/setup-shape (assoc attrs :id id :type :bool))) + + (defn- bool-head + "Canonical style head: the first child is the base for `:difference`, the + last child otherwise." + [bool-shape children] + (if (= :difference (:bool-type bool-shape)) + (first children) + (last children))) + + (defn- finalize-bool! + [state id] + (let [scope @state + bool (get-in scope [:objects id]) + children (mapv (:objects scope) (:shapes bool))] + (when (empty? children) + (throw (ex-info "bool requires at least one child" + {:type ::empty-bool + :id id}))) + (when-let [frame (first (filter cfh/frame-shape? children))] + (throw (ex-info "bool children cannot be frames" + {:type ::frame-in-bool + :id id + :child (:id frame)}))) + (let [inherited (d/without-nils + (select-keys (bool-head bool children) + path/bool-style-properties)) + supplied (d/without-nils + (select-keys (get-in scope [:container-attrs id]) + path/bool-style-properties)) + bool (-> bool (merge inherited) (merge supplied)) + + ;; `path/update-bool-shape` dispatches to `path/wasm:calc-bool-content`, + ;; the renderer override, so scene content would depend on renderer + ;; initialization and A/B configuration. Compose the pure helpers: + ;; scene generation is untimed and must not vary with the renderer + ;; being compared. + content (path/calc-bool-content bool (:objects scope)) + bool (path/update-geometry bool content)] + (swap! state assoc-in [:objects id] bool)))) + + (defn- finalize-group! + [state id] + (let [scope @state + group (get-in scope [:objects id]) + children (mapv (:objects scope) (:shapes group))] + (when (empty? children) + (throw (ex-info "group requires at least one child" + {:type ::empty-group + :id id}))) + (let [group (if (:masked-group group) + (gsh/update-mask-selrect group children) + (gsh/update-group-selrect group children))] + (swap! state assoc-in [:objects id] group)))) + + (def ^:private container-specs + "Per-container behavior: how to build it, which parent scope it opens, + and how to finalize it after the body. `:keep-attrs?` records the raw + attrs in the scope so the finalizer can let supplied styles win over + inherited ones." + {:frame {:setup setup-frame + :parent (fn [id _parent] {:id id :frame-id id}) + :finalize (fn [_state _id] nil)} + :group {:setup setup-group + :parent (fn [id parent] {:id id :frame-id (:frame-id parent)}) + :finalize finalize-group!} + :bool {:setup setup-bool + :parent (fn [id parent] {:id id :frame-id (:frame-id parent)}) + :finalize finalize-bool! + :keep-attrs? true}}) + + (defn- check-container-attrs + [type id attrs] + (when-not (map? attrs) + (throw (ex-info (str (name type) " attrs must be a map; the label form takes the attrs map next") + {:type ::invalid-attrs :id id}))) + attrs) + + (defn with-container + "Creates a frame, group or bool, runs `thunk` with the new parent in + scope, finalizes the containers that derive state from children, and + returns the container uuid. + + Any exception from the body or the finalizer marks the container + unfinished and propagates; `finish!` then rejects the instance even if the + caller catches the exception." + [type attrs label thunk] + (let [{:keys [setup parent finalize keep-attrs?]} + (or (get container-specs type) + (throw (ex-info "unsupported container type" + {:type ::invalid-container + :container-type type}))) + state (ensure-state *state*) + id (rng-uuid (:rng @state)) + attrs (check-container-attrs type id attrs) + shape (setup id attrs)] + (when keep-attrs? + (swap! state assoc-in [:container-attrs id] attrs)) + (add-object! state label shape) + (binding [*parent* (parent id *parent*)] + (try + (thunk) + (finalize state id) + (catch :default cause + (swap! state assoc-in [:unfinished id] true) + (throw cause)))) + id)) + + (defn finish! + "Validates the scope snapshot and returns it." + [state] + (let [{:keys [objects refs unfinished]} @state] + (when (seq unfinished) + (throw (ex-info "scene contains an unfinished container" + {:type ::unfinished-container + :ids (vec (keys unfinished))}))) + (let [instance {:objects objects + :refs refs}] + (snapshot/validate! instance) + instance))))) diff --git a/frontend/performance/benches/render_wasm/cases.cljs b/frontend/performance/benches/render_wasm/cases.cljs new file mode 100644 index 0000000000..f67962538e --- /dev/null +++ b/frontend/performance/benches/render_wasm/cases.cljs @@ -0,0 +1,128 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +;; Discovery loads each scene and its transitive requires, including any +;; colocated operation functions. +;; Keep this entire import subgraph safe for Node. + +(ns benches.render-wasm.cases + "Case collection for the renderer benchmarks. + + Loading this namespace registers the scene namespaces listed here. The require + list is the discovery manifest for case collection. + + NOTE: Nothing here should import browser or renderer code, so the Node runner + and offline reports load scenes without requiring a browser. + + Usage: + + (cases/collect-cases {:master-seed 42}) + ;; => + [{:id :rects/load + :scene :rects + :scene-description \"Seeded rectangles with translucent fills and centered strokes\" + :scene-seed 2602546180 + :params {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {:render :none}} + {:id :rects/pan ...} + {:id :rects/zoom ...}]" + (:require + [app.common.schema :as sm] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :as random] + benches.render-wasm.scenes.masks + benches.render-wasm.scenes.paths + benches.render-wasm.scenes.plus + benches.render-wasm.scenes.rects + [clojure.string :as str])) + +(defn- check-master-seed! + "Collection requires an integer master seed in [0, 2^32)." + [seed] + (when-not (random/seed? seed) + (throw (ex-info "case collection requires an integer master seed in [0, 2^32)" + {:type ::invalid-master-seed + :seed seed})))) + +(defn- as-invalid-case! + "Runs `thunk`, translating core case-validation failures into this + namespace's ::invalid-case so `collect-cases` has a single error + contract. Registry mechanics (::duplicate-*, ::unknown-scene) and scene + declarations (::invalid-scene) keep their own types." + [thunk] + (try + (thunk) + (catch :default cause + (let [data (ex-data cause)] + (if (contains? #{::decl/invalid-case ::decl/non-serializable-case} (:type data)) + (throw (ex-info (ex-message cause) + (assoc data :type ::invalid-case) + cause)) + (throw cause)))))) + +(defn- check-params! + "Validates case params against the scene's closed schema." + [case-id scene-id schema params] + (when-not (sm/validate schema params) + (throw (ex-info (str "invalid params for case " case-id) + {:type ::invalid-case + :id case-id + :scene scene-id + ::sm/explain (sm/explain schema params)})))) + +(defn- validate-case! + "Validates a registered case entry against its scene: unknown scene, + case-namespace mismatch, declaration schema and closed params." + [{:keys [id scene params] :as registered} scene-by-id] + (let [entry (get scene-by-id scene)] + (when (nil? entry) + (throw (ex-info (str "case " id " names an unknown scene: " scene) + {:type ::unknown-scene + :id id + :scene scene}))) + (when-not (= (name scene) (namespace id)) + (throw (ex-info (str "case id " id " is not namespaced by its scene: " scene) + {:type ::invalid-case + :id id + :scene scene}))) + (as-invalid-case! #(decl/check-registered-case! registered)) + (check-params! id scene (:params-schema entry) params) + registered)) + +(defn- describe-case + "Projects a validated case to its collected descriptor: scene description + and seed attached, internal keys stripped, transit wire checked." + [master-seed registered scene-entry] + (let [projected (-> (decl/project-case registered) + (assoc :scene-description (:description scene-entry) + :scene-seed (decl/derive-seed master-seed + (:scene registered) + (:params registered))))] + (as-invalid-case! #(decl/check-collected-case projected)))) + +(defn collect-cases + "Selected benchmark cases as plain data. + + Validates scenes, cases and parameters before any browser work. `:filter` + is a substring match on the case id; an empty selection throws ::no-cases." + [{:keys [master-seed] :as options}] + (check-master-seed! master-seed) + (let [pattern (:filter options) + scene-by-id (into {} (map (juxt :id identity)) (decl/scenes))] + (doseq [scene (decl/scenes)] + (decl/check-registered-scene! scene)) + (let [selected (->> (decl/cases) + (map #(validate-case! % scene-by-id)) + (filter #(or (str/blank? pattern) + (str/includes? (str (:id %)) pattern))) + (mapv #(describe-case master-seed % (get scene-by-id (:scene %)))))] + (when (empty? selected) + (throw (ex-info "no benchmark cases match the selection" + {:type ::no-cases + :filter pattern}))) + selected))) diff --git a/frontend/performance/benches/render_wasm/codec.cljs b/frontend/performance/benches/render_wasm/codec.cljs new file mode 100644 index 0000000000..868514d21f --- /dev/null +++ b/frontend/performance/benches/render_wasm/codec.cljs @@ -0,0 +1,84 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.codec + "Verifies that measurements survive transit encoding. + + This ns exists mainly to preserve NaNs in the transit encoding of results. + We treat two NaN values as the same measurement. + + Any data we cannot represent raises an error." + (:require + [app.common.transit :as transit])) + +(declare equivalent?) + +(defn check-value + "Checks optional transport data within depth and size limits. + + Args: + value: Trace or diagnostic partial data to validate. + + Returns: + The supplied value when traversal stays within depth 32, 100000 visited + values and 1048576 UTF-16 text units. Sequential collections have a finite + count. Numbers retain NaN and infinities. + + Throws: + An error for unsupported values or data that exceeds either limit." + [value] + (loop [pending (list [value 0]) + budget 100000 + text-budget 1048576] + (when (neg? text-budget) + (throw (ex-info "Transport data exceeds text limits" {:type ::data-limit}))) + (if-let [[item depth] (first pending)] + (do + (when (or (zero? budget) (> depth 32)) + (throw (ex-info "Transport data exceeds traversal limits" {:type ::data-limit}))) + (let [children (cond + (map? item) (mapcat (fn [[key value]] [[key (inc depth)] [value (inc depth)]]) item) + (or (and (sequential? item) (counted? item)) (set? item)) (map #(vector % (inc depth)) item) + (or (nil? item) (boolean? item) (number? item) (string? item) + (keyword? item) (symbol? item) (uuid? item) (inst? item)) nil + :else (throw (ex-info "Unsupported transport value" {:type ::unsupported-value})))] + (recur (concat children (rest pending)) (dec budget) + (- text-budget (cond + (string? item) (count item) + (or (keyword? item) (symbol? item)) (+ (count (name item)) (count (namespace item))) + :else 0))))) + value))) + +(defn- equivalent-entry? + "Finds a map key and compares its value, including keys that contain NaN." + [other [k v]] + (if-let [entry (find other k)] + (equivalent? v (val entry)) + (boolean (some (fn [[other-k other-v]] + (and (equivalent? k other-k) (equivalent? v other-v))) other)))) + +(defn equivalent? + "Checks Transit value equality while retaining NaN as a valid raw value." + [a b] + (or (= a b) + (cond + (and (number? a) (number? b)) (and (js/Number.isNaN a) (js/Number.isNaN b)) + (and (map? a) (map? b)) (and (= (count a) (count b)) + (every? #(equivalent-entry? b %) a)) + (and (sequential? a) (sequential? b)) (and (= (count a) (count b)) + (every? true? (map equivalent? a b))) + (and (set? a) (set? b)) (and (= (count a) (count b)) + (every? (fn [item] (some #(equivalent? item %) b)) a)) + :else false))) + +(defn encode-str + "Encodes through app.common.transit and rejects any loss of supplied evidence." + ([value] (encode-str value nil)) + ([value options] + (let [text (transit/encode-str value options)] + (when-not (equivalent? value (transit/decode-str text)) + (throw (ex-info "Transit cannot preserve this value" {:type ::unrepresentable-value}))) + text))) diff --git a/frontend/performance/benches/render_wasm/declarations.cljc b/frontend/performance/benches/render_wasm/declarations.cljc new file mode 100644 index 0000000000..5a131e825e --- /dev/null +++ b/frontend/performance/benches/render_wasm/declarations.cljc @@ -0,0 +1,373 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.declarations + "Declarations, registry and contracts for renderer benchmark scenes and cases. + + A scene declares an id, description, parameters schema and build function. + A case declares an id namespaced by its scene, parameters, view, context + policy and a run function. Optional preparation and reset functions transform + the context before and after its main execution. + + Ordinary :require loads the declaration macros and runtime helpers. + Collection validates declarations and produces plain descriptors." + #?(:cljs (:require-macros + [app.common.exceptions :as ex] + [benches.render-wasm.declarations])) + #?(:clj (:require [app.common.schema :as-alias sm])) + #?(:cljs + (:require + [app.common.schema :as sm] + [app.common.transit :as t] + [clojure.string :as str]))) + +#?(:clj + (do + (defn- single-arg-vector? + "True when `form` is a vector holding one plain symbol." + [form] + (and (vector? form) + (= 1 (count form)) + (symbol? (first form)) + (not= '& (first form)))) + + (defn- check-arg-vector! + [macro id argv] + (when-not (single-arg-vector? argv) + (throw (ex-info (str macro " " id " requires one argument vector with one symbol") + {:type ::invalid-run-function + :id id + :form argv})))) + + (defmacro defscene + "Registers a scene and returns its id. + + `opts` supplies :description and :params-schema. `argv` names one scene + parameter symbol, and `body` builds the snapshot. + + Example: + + (defscene :rects opts [params] (scene params))" + [id opts argv & body] + (check-arg-vector! "defscene" id argv) + `(benches.render-wasm.declarations/register-scene! + (assoc ~opts :id ~id :ns ~(str *ns*) :build (fn ~argv ~@body)))) + + (defn- case-registration-form + [id scene opts run-fn] + (let [options (gensym "options")] + `(let [~options ~opts] + (when (contains? ~options :run!) + (throw (ex-info (str "defcase " ~id " requires its run function as the fourth argument") + {:type :benches.render-wasm.declarations/invalid-run-function + :id ~id}))) + (when (nil? ~scene) + (throw (ex-info (str "defcase " ~id " requires a :scene/case id") + {:type :benches.render-wasm.declarations/invalid-run-function + :id ~id + :form ~id}))) + (benches.render-wasm.declarations/register-case! + (assoc ~options :id ~id :scene ~scene :ns ~(str *ns*) + ~@(when run-fn [:run! run-fn])))))) + + (defn- scene-from-id + "Returns the scene keyword from a qualified case id, or nil." + [id] + (when-let [scene-ns (namespace id)] + (keyword scene-ns))) + + (defmacro defcase + "Registers a case and returns its id. + + The scene comes from the namespace part of the :scene/case id. An + optional argument vector and body supply the run function, which accepts + one command context and returns its next context or a Promise of it. + `[:preparation :callback]` runs once after upload. Preparation renders + Full by default; `:render :none` selects a cold first render. `:reset` + runs after each successful main execution. Collection produces plain + case descriptors. + + Example: + + (defcase :rects/pan options [context] (pan* context pan-options))" + ([id opts] + (case-registration-form id (scene-from-id id) opts nil)) + ([id opts argv & body] + (check-arg-vector! "defcase" id argv) + (case-registration-form id (scene-from-id id) opts `(fn ~argv ~@body)))))) + +#?(:cljs + (do + ;; Case execution context: + ;; :case - collected descriptor + ;; :scene - scene snapshot {:objects ... :refs ...} + ;; :view - current camera view + ;; :hooks - clock, scheduling, guard and renderer operations + ;; Case bodies return the next context or a Promise of it. + + ;; Declarations and their transitive imports load in Node and the browser. + ;; Browser capabilities enter through the execution context; resource + ;; acquisition belongs to untimed preparation. Scenes keep their imports + ;; safe for Node discovery. + + (def default-viewport + "Default viewport dimensions and device pixel ratio for a case." + {:width 1920 :height 1080 :dpr 2}) + + (def schema:viewport + [:map {:closed true} + [:width {:optional true} ::sm/positive-safe-number] + [:height {:optional true} ::sm/positive-safe-number] + [:dpr {:optional true} ::sm/positive-safe-number]]) + + (def schema:view + "Camera placement with finite coordinates, positive scale and optional + viewport overrides." + [:map {:closed true} + [:scale ::sm/positive-safe-number] + [:x ::sm/safe-number] + [:y ::sm/safe-number] + [:viewport {:optional true} schema:viewport]]) + + (defn resolve-viewport + "Resolves a view's viewport overrides against the default viewport." + [view] + (merge default-viewport (:viewport view))) + + (def schema:scene + [:map {:closed true} + [:id simple-keyword?] + [:ns string?] + [:description string?] + [:params-schema vector?] + [:build fn?]]) + + (def schema:registered-case + [:map {:closed true} + [:id qualified-keyword?] + [:scene simple-keyword?] + [:ns string?] + [:params [:map]] + [:view schema:view] + [:context [:enum :fresh :reuse]] + [:preparation {:optional true} + [:map + [:render {:optional true} [:enum :full :none]] + [:callback {:optional true} fn?]]] + [:reset {:optional true} fn?] + [:run! {:optional true} fn?]]) + + (def schema:collected-case + [:map {:closed true} + [:id qualified-keyword?] + [:scene simple-keyword?] + [:scene-description string?] + [:scene-seed [:int {:min 0 :max 4294967295}]] + [:params [:map]] + [:view schema:view] + [:context [:enum :fresh :reuse]] + [:preparation [:map [:render {:optional true} [:enum :full :none]]]]]) + + (defonce registry + (atom {:scenes {} + :scene-order [] + :cases {} + :case-order []})) + + (defn- register-entry + [entries-key order-key kind duplicate-type {:keys [id ns] :as entry}] + (let [existing (get-in @registry [entries-key id])] + (when (and (some? existing) (not= ns (:ns existing))) + (throw (ex-info (str "duplicate " kind " id: " id) + {:type duplicate-type + :id id + :ns ns + :existing-ns (:ns existing)}))) + (swap! registry + #(cond-> (assoc-in % [entries-key id] entry) + (not (contains? (get % entries-key) id)) + (update order-key conj id)))) + id) + + (defn register-scene! + "Registers a scene. The same id from the same namespace replaces the + entry in declaration order. Another namespace raises ::duplicate-scene." + [scene] + (register-entry :scenes :scene-order "scene" ::duplicate-scene scene)) + + (defn register-case! + "Registers a case. The same id from the same namespace replaces the + entry in declaration order. Another namespace raises ::duplicate-case." + [bench-case] + (register-entry :cases :case-order "case" ::duplicate-case bench-case)) + + (defn- unregister-entry + [entries-key order-key id] + (swap! registry + (fn [reg] + (-> reg + (update entries-key dissoc id) + (update order-key #(vec (remove #{id} %)))))) + id) + + (defn unregister-scene! + "Removes a scene registration and returns its id." + [id] + (unregister-entry :scenes :scene-order id)) + + (defn unregister-case! + "Removes a case registration and returns its id." + [id] + (unregister-entry :cases :case-order id)) + + (defn scenes + "Returns registered scenes in declaration order." + [] + (mapv (:scenes @registry) (:scene-order @registry))) + + (defn cases + "Returns registered cases in declaration order." + [] + (mapv (:cases @registry) (:case-order @registry))) + + (defn registered-scene + "Returns the full registration for a scene id." + [id] + (get-in @registry [:scenes id])) + + (defn registered-case + "Returns the full registration for a case id, including its run function." + [id] + (get-in @registry [:cases id])) + + (defn- check-params-schema! + [{:keys [id params-schema]}] + (try + (sm/schema params-schema) + (catch :default cause + (throw (ex-info (str "invalid params schema for scene " id) + {:type ::invalid-scene :id id} + cause))))) + + (defn- check-declaration + [schema declaration message error-type] + (when-not (sm/validate schema declaration) + (throw (ex-info (str message (:id declaration)) + {:type error-type + :id (:id declaration) + ::sm/explain (sm/explain schema declaration)}))) + declaration) + + (defn check-registered-scene! + "Validates a scene declaration and returns it unchanged." + [scene] + (check-params-schema! scene) + (check-declaration schema:scene scene "invalid scene declaration: " ::invalid-scene)) + + (defn check-registered-case! + "Validates a registered case declaration and returns it unchanged." + [bench-case] + (check-declaration schema:registered-case bench-case "invalid case declaration: " ::invalid-case)) + + (defn- check-param-keys! + "Rejects params keys that `canonical-params` cannot tell apart: only + simple keywords render distinctly by name." + [scene-id params] + (letfn [(check [value path] + (cond + (map? value) + (doseq [[k v] value] + (when-not (simple-keyword? k) + (throw (ex-info (str "seed identity requires simple keyword keys at " path) + {:type ::invalid-params + :scene scene-id + :path path + :key k}))) + (check v (str path "/" (name k)))) + + (vector? value) + (dorun (map-indexed (fn [index v] (check v (str path "[" index "]"))) value))))] + (check params ""))) + + (defn- canonical-params + "Stable textual identity of a params map: keys render sorted by name, + vectors keep their order, nested values render the same way. Map key + order is an accident of construction, so sorting keeps the identity on + the parameters themselves: equal parameters give the same seed whatever + their literal order. Keys render by name, so params maps use simple + keyword keys; derive-seed rejects anything else." + [params] + (letfn [(render [value] + (cond + (map? value) + (str "{" + (str/join "," + (map (fn [[k v]] (str (name k) ":" (render v))) + (sort-by (comp name key) value))) + "}") + + (vector? value) + (str "[" (str/join "," (map render value)) "]") + + :else + (pr-str value)))] + (render params))) + + (defn derive-seed + "FNV-1a 32-bit keyed hash of a case's parameter identity. + + Fowler-Noll-Vo, 32-bit, offset basis 2166136261, prime 16777619, hashed + over `\\0/`. A keyed hash, not + the scene PRNG: the scene's mulberry32 generator is seeded with the + result, and a sequential draw cannot substitute because it would couple + cases through collection order. Keyed by scene and params only, so the + seed does not depend on case name, collection order, operation body, + attempts or A/B feature selection. Params maps use simple keyword keys; + namespaced keywords and string keys cannot be told apart by name and make + derive-seed throw ::invalid-params." + [master-seed scene-id params] + (check-param-keys! scene-id params) + (let [text (str master-seed "\u0000" (name scene-id) "/" (canonical-params params))] + (loop [index 0 + seed 2166136261] + (if (< index (count text)) + (recur (inc index) + (unsigned-bit-shift-right + (js/Math.imul (bit-xor seed (.charCodeAt text index)) 16777619) + 0)) + seed)))) + + (defn transit-round-trips? + "True when `value` survives Transit encoding and decoding with the same + wire representation, including NaN and infinities." + [value] + (true? (ex/ignoring + (let [encoded (t/encode-str value)] + (= encoded (-> encoded t/decode-str t/encode-str)))))) + + (defn project-case + "Returns a data descriptor with the declared preparation settings. + + Args: + bench-case: registered case with local executable functions. + + Returns: + Case data with an explicit preparation map." + [bench-case] + (-> bench-case + (select-keys [:id :scene :params :view :context :preparation]) + (update :preparation #(dissoc (or % {}) :callback)))) + + (defn check-collected-case + "Validates a collected case descriptor and its Transit representation, + returning the descriptor unchanged." + [projected] + (check-declaration schema:collected-case projected "invalid collected case: " ::invalid-case) + (when-not (transit-round-trips? projected) + (throw (ex-info (str "collected case does not survive the transit wire: " (:id projected)) + {:type ::non-serializable-case + :id (:id projected)}))) + projected))) diff --git a/frontend/performance/benches/render_wasm/errors.cljc b/frontend/performance/benches/render_wasm/errors.cljc new file mode 100644 index 0000000000..c30d05be68 --- /dev/null +++ b/frontend/performance/benches/render_wasm/errors.cljc @@ -0,0 +1,500 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.errors + "Failure values and plain diagnostics for renderer benchmarks. + + A step that can fail returns its value or an Error. `attempt` catches thrown + values and `failure?` identifies the returned Error. Fatal failures mark + writes to the owner's record or log and propagate to a boundary that uses + `:catch-fatal`. + + `evidence` and `describe-cause` produce diagnostics for saved records and + Transit responses." + #?(:cljs (:require + [benches.render-wasm.codec :as codec] + [clojure.string :as str])) + #?(:cljs (:require-macros + [app.common.exceptions :as ex] + [benches.render-wasm.errors]))) + +#?(:clj + (defmacro attempt + "Evaluates `body` and returns its value, or the failure it throws. + + A thrown value becomes an Error through `as-failure`. Fatal failures + propagate unless a leading `:catch-fatal` requests them as values. + Inside an `^:async` function, `body` may use `await`. + + Examples: + + (attempt (parse text)) + (attempt (await (open-page owner))) + (attempt :catch-fatal (await (run-session state)))" + [& forms] + (let [catch-fatal? (= :catch-fatal (first forms)) + body (if catch-fatal? (rest forms) forms) + thrown (gensym "thrown") + failure (gensym "failure")] + `(try + ~@body + (catch :default ~thrown + (let [~failure (benches.render-wasm.errors/as-failure ~thrown)] + ~(if catch-fatal? + failure + `(if (benches.render-wasm.errors/fatal? ~failure) + (throw ~failure) + ~failure)))))))) + +#?(:cljs + (do + (def ^:private schema:cause-value + [:or nil? boolean? keyword? number? string?]) + + (def schema:cause-detail + "One level of a portable exception description." + [:map {:closed true} + [:message string?] + [:fn {:optional true} schema:cause-value] + [:code {:optional true} schema:cause-value] + [:type {:optional true} schema:cause-value] + [:hint {:optional true} schema:cause-value]]) + + (def ^:private diagnostic-fields + [[:phase string?] + [:message string?] + [:operation {:optional true} keyword?] + [:path {:optional true} [:string {:max 1024}]] + [:status {:optional true} [:enum "failed" "stale"]] + [:cause {:optional true} [:vector schema:cause-detail]] + [:partial {:optional true} map?] + [:reason {:optional true} :any] + [:deadline-exceeded {:optional true} boolean?] + [:expected-id {:optional true} [:string {:max 1024}]] + [:actual-id {:optional true} [:string {:max 1024}]] + [:command {:optional true} [:string {:max 1024}]] + [:exit-code {:optional true} :int] + [:pid {:optional true} [:int {:min 1}]] + [:pgid {:optional true} [:int {:min 1}]] + [:signals {:optional true} [:vector [:string {:max 1024}]]] + [:profile-path {:optional true} [:string {:max 1024}]] + [:fatal {:optional true} boolean?] + [:unresolved-process {:optional true} boolean?] + [:signal {:optional true} [:string {:max 1024}]] + [:endpoint {:optional true} [:string {:max 1024}]]]) + + (def schema:error-entry + "One failure occurrence with its operation and portable diagnostics." + (into [:map {:closed true}] diagnostic-fields)) + + (def schema:diagnostic + "Failure location, message and an optional ordered collection of errors." + (into [:map] + (conj diagnostic-fields + [:errors {:optional true} [:vector {:min 1} schema:error-entry]]))) + + (def ^:private evidence-cause-depth 3) + (def ^:private max-cause-chars 500) + (def ^:private unrenderable-value "unrenderable value") + (def ^:private unrenderable-cause-text "unrenderable cause") + + (defn failure? + "True when `x` is an Error." + [x] + (instance? js/Error x)) + + (defn message-of + "Returns the error message or string form of `cause`, with a fallback + when formatting fails. The default fallback is unknown failure." + ([cause] + (message-of cause "unknown failure")) + ([cause fallback] + (or (ex/ignoring (let [message (ex-message cause)] + (when (string? message) message))) + (ex/ignoring (str cause)) + fallback))) + + (defn fatal-error + "Returns an exception that `fatal?` recognizes, with `data` and `cause`." + [message data cause] + (ex-info message (assoc data :fatal true) cause)) + + (defn fatal? + "True when `cause` or one of its causes has the ex-data `:fatal` marker." + [cause] + (loop [cause cause + seen #{}] + (cond + (nil? cause) false + (:fatal (ex-data cause)) true + (contains? seen cause) false + :else (recur (ex-cause cause) (conj seen cause))))) + + (defn as-failure + "Returns an Error for `thrown`. Errors pass through; other values supply + the message and the `:thrown` ex-data." + [thrown] + (if (failure? thrown) + thrown + (ex-info (if (string? thrown) + thrown + (str "non-error value thrown: " (or (ex/ignoring (pr-str thrown)) "unprintable value"))) + {:thrown thrown}))) + + (defn with-data + "Adds default ex-data to an Error, keeping its message and using it as + the cause. Existing keys win. Other values pass through." + [x data] + (if (failure? x) + (ex-info (message-of x) (merge data (ex-data x)) x) + x)) + + (declare combine) + + (defn with-context + "Returns an exception with `message` and added diagnostic context. + New fields override existing fields; existing partial measurements win + over earlier measurements in `data`. An exception with ex-data keeps + its underlying cause. Other values become the cause." + [cause message data] + (let [existing (ex-data cause) + malformed (and (contains? existing :partial) (not (map? (:partial existing))) + (map? (:partial data))) + source (if malformed (combine [{:error cause :partial (:partial existing)}]) cause) + context (cond-> (merge (ex-data source) data) + (contains? data :partial) + (assoc :partial (if (map? (:partial data)) + (merge (:partial data) (when (map? (:partial existing)) (:partial existing))) + (:partial data))))] + (ex-info message context (if (and (some? existing) (not malformed)) (ex-cause cause) source)))) + + (defn- merge-error-context + [outer inner] + (cond-> (merge outer inner) + (and (map? (:partial outer)) (map? (:partial inner))) + (assoc :partial (merge (:partial outer) (:partial inner))))) + + (defn- get-error-collection + [cause] + (loop [current cause + seen #{} + context {}] + (let [data (ex-data current) + context (merge-error-context context (dissoc data ::errors))] + (if-let [entries (::errors data)] + (update entries 0 #(merge-error-context context %)) + (when (and (ex-cause current) (not (contains? seen current))) + (recur (ex-cause current) (conj seen current) context)))))) + + (defn get-errors + "Returns failure occurrences in order, retaining their Error objects. + + Args: + cause: An Error, including one wrapped with additional context. + + Returns: + A flat vector of maps with an Error under :error and available phase, + operation and path context." + [cause] + (or (get-error-collection cause) + [(assoc (select-keys (ex-data cause) [:phase :operation :path]) :error cause)])) + + (defn combine + "Combines failure occurrences into an Error, or returns nil on success. + + Args: + entries: Ordered maps with :error and optional :phase, :operation and + :path. Nil errors represent successful operations. An Error may + contain a collection from an earlier operation. + + Returns: + An Error retaining a flat collection and the first error's message + and data. Retained Error objects expose their causes. Any fatal + occurrence makes the combined error fatal." + [entries] + (let [sources (filterv #(failure? (:error %)) entries) + error (:error (first sources)) + entries (reduce (fn [collected {:keys [error] :as entry}] + (let [context (dissoc entry :error) + group (get-errors error) + group (if (seq context) + (mapv #(merge-error-context context %) group) + group)] + (if (empty? collected) group (into collected group)))) + [] sources)] + (when error + (ex-info (message-of error) + (cond-> (merge (select-keys (first entries) [:phase :operation :path]) + (ex-data error) + {::errors entries}) + (some #(fatal? (:error %)) sources) (assoc :fatal true)) + (if (::errors (ex-data error)) (ex-cause error) error))))) + + (defn stale? + "True when the exception's own data marks superseded work." + [cause] + (true? (::stale (ex-data cause)))) + + (defn fail-data + "Marks diagnostic data with the failed status." + [data] + (assoc data :status "failed")) + + (defn bounded-text + "Returns text within a size limit, preserving UTF-16 boundaries. + + Converts `value` to a string and limits it to max-chars UTF-16 code + units. If conversion fails, it returns 'unrenderable value'. When + truncating, we check whether the cut would leave a high surrogate + without its matching low surrogate. If so, we remove that last code unit + to keep the pair intact. + + Args: + value: Value to convert to text. + max-chars: Positive limit in UTF-16 units. + + Returns: + Bounded text, or a fallback string when conversion fails." + [value max-chars] + (or (ex/ignoring + (let [text (str value)] + (if (<= (count text) max-chars) + text + (let [cut (subs text 0 max-chars) + last (.charCodeAt cut (dec max-chars))] + (if (and (>= last 0xD800) + (<= last 0xDBFF) + (let [nxt (.charCodeAt text max-chars)] + (and (>= nxt 0xDC00) (<= nxt 0xDFFF)))) + (subs cut 0 (dec max-chars)) + cut))))) + (subs unrenderable-value 0 (min max-chars (count unrenderable-value))))) + + (defn terminal-text + "Returns bounded text with terminal controls escaped. + + Args: + value: Value to render as text. + max-chars: Positive limit in UTF-16 units after escaping. + + Returns: + Text with C0, DEL and C1 controls represented as Unicode escapes." + [value max-chars] + (bounded-text + (or (ex/ignoring + (str/replace (bounded-text value max-chars) #"[\x00-\x1f\x7f-\x9f]" + (fn [character] + (str "\\u" (.padStart (.toString (.charCodeAt character 0) 16) 4 "0"))))) + unrenderable-value) + max-chars)) + + (defn format-value + "Returns a bounded, readable value with terminal controls escaped. + + Args: + value: Value to print. Strings use JSON quoting. + max-chars: Positive limit in UTF-16 units after escaping. + + Returns: + Readable text, or fallback text when printing fails." + [value max-chars] + (terminal-text + (or (ex/ignoring + (if (string? value) + (js/JSON.stringify (bounded-text value max-chars)) + (binding [*print-level* 8 *print-length* 32] + (pr-str value)))) + unrenderable-value) + max-chars)) + + (defn project-context + "Returns bounded identity, process and ownership details. + + Args: + data: Diagnostic data with available identity or process fields. + + Returns: + Bounded text, integer process IDs and exit code, attempted signals, + and available fatal or unresolved-ownership markers." + [data] + (cond-> {} + (string? (:expected-id data)) (assoc :expected-id (bounded-text (:expected-id data) 1024)) + (string? (:actual-id data)) (assoc :actual-id (bounded-text (:actual-id data) 1024)) + (string? (:command data)) (assoc :command (bounded-text (:command data) 1024)) + (integer? (:exit-code data)) (assoc :exit-code (:exit-code data)) + (and (integer? (:pid data)) (pos? (:pid data))) (assoc :pid (:pid data)) + (and (integer? (:pgid data)) (pos? (:pgid data))) (assoc :pgid (:pgid data)) + (and (vector? (:signals data)) (every? string? (:signals data))) + (assoc :signals (mapv #(bounded-text % 1024) (:signals data))) + (string? (:profile-path data)) (assoc :profile-path (bounded-text (:profile-path data) 1024)) + (true? (:fatal data)) (assoc :fatal true) + (true? (:unresolved-process data)) (assoc :unresolved-process true) + (string? (:signal data)) (assoc :signal (bounded-text (:signal data) 1024)) + (string? (:endpoint data)) (assoc :endpoint (bounded-text (:endpoint data) 1024)))) + + (declare project-diagnostic) + + (defn- sanitize-cause-value + [value] + (cond + (or (nil? value) (boolean? value) (keyword? value) (number? value)) + value + + (string? value) + (bounded-text value max-cause-chars) + + :else + (bounded-text (or (ex/ignoring + (binding [*print-level* 8 *print-length* 32] + (pr-str value))) + unrenderable-value) + max-cause-chars))) + + (defn- describe-cause-level + [cause] + (or (ex/ignoring + (let [message (ex/ignoring (ex-message cause)) + data (ex/ignoring (ex-data cause)) + text (cond + (string? message) message + (string? cause) cause + (nil? cause) "unknown failure" + :else (binding [*print-level* 8 *print-length* 32] (pr-str cause))) + out {:message (sanitize-cause-value text)}] + (if (map? data) + (reduce (fn [m k] + (if (contains? data k) + (assoc m k (sanitize-cause-value (get data k))) + m)) + out + [:fn :code :type :hint]) + out))) + {:message unrenderable-cause-text})) + + (defn describe-cause + "Returns up to three cause levels as plain data. Each level keeps a + bounded message and available :fn, :code, :type and :hint details. + Unprintable values produce fallback strings." + [cause] + (or (ex/ignoring + (loop [current cause + depth 0 + acc [] + seen #{}] + (if (or (nil? current) (contains? seen current) (>= depth evidence-cause-depth)) + (if (seq acc) + acc + [{:message "unknown failure"}]) + (let [level (describe-cause-level current) + next (ex/ignoring (ex-cause current))] + (recur next (inc depth) (conj acc level) (conj seen current)))))) + [{:message unrenderable-cause-text}])) + + (defn- diagnostic-data + [phase cause] + (let [partial (ex/ignoring (:partial (ex-data cause)))] + (cond-> (merge (project-context (ex-data cause)) + (select-keys (ex-data cause) [:operation]) + {:phase (if (keyword? phase) (name phase) (str phase)) + :message (bounded-text (message-of cause) 1024) + :cause (describe-cause cause)}) + (contains? (ex-data cause) :partial) (assoc :partial partial) + (string? (:path (ex-data cause))) + (assoc :path (bounded-text (:path (ex-data cause)) 1024))))) + + (defn evidence + "Returns plain diagnostics, including ordered failure occurrences. + + Args: + phase: Failure location for the enclosing operation. + cause: Error with available operation context and collected failures. + + Returns: + Portable diagnostics with bounded cause descriptions. Collected + errors retain occurrence order and messages within 1024 UTF-16 units." + [phase cause] + (let [entries (get-error-collection cause) + projected (when entries + (mapv (fn [{:keys [error operation path partial] entry-phase :phase :as entry}] + (cond-> (merge (diagnostic-data (or entry-phase phase) error) + (project-context entry) + (select-keys entry [:reason :deadline-exceeded]) + {:message (bounded-text (message-of error) 1024)}) + operation (assoc :operation operation) + partial (assoc :partial partial) + (string? path) (assoc :path (bounded-text path 1024)))) + entries))] + (project-diagnostic + (cond-> (diagnostic-data phase cause) + projected (assoc :errors projected) + (map? (:partial (first projected))) (assoc :partial (:partial (first projected))) + (some :unresolved-process projected) (assoc :unresolved-process true))))) + + (defn- partial-encoding-error + [path cause] + {:phase "diagnostic-encoding" :operation :encode-diagnostic + :path (format-value path 1024) + :message "Diagnostic partial evidence cannot cross Transit" + :cause (describe-cause cause)}) + + (defn- project-partial + [partial] + (if (map? partial) + (reduce-kv (fn [[kept rejected] key value] + (try + (codec/check-value {key value}) + (codec/encode-str {key value}) + [(assoc kept key value) rejected] + (catch :default cause + [kept (conj rejected (partial-encoding-error [:partial key] cause))]))) + [{} []] + partial) + [nil [(partial-encoding-error [:partial] + (ex-info "Diagnostic partial evidence must be a map" {}))]])) + + (defn- project-entry + [entry] + (let [[partial rejected] (when (contains? entry :partial) (project-partial (:partial entry))) + projected (cond-> (project-context entry) + (contains? #{"failed" "stale"} (:status entry)) (assoc :status (:status entry)) + (boolean? (:deadline-exceeded entry)) (assoc :deadline-exceeded (:deadline-exceeded entry)) + (keyword? (:operation entry)) (assoc :operation (:operation entry)) + (contains? entry :phase) (assoc :phase (bounded-text (:phase entry) 1024)) + (contains? entry :message) (assoc :message (bounded-text (:message entry) 1024)) + (contains? entry :reason) (assoc :reason (sanitize-cause-value (:reason entry))) + (contains? entry :cause) + (assoc :cause (mapv (fn [level] + (reduce-kv (fn [out key value] + (assoc out key (sanitize-cause-value value))) + {} (select-keys level [:message :fn :code :type :hint]))) + (take evidence-cause-depth (:cause entry)))) + (some? partial) (assoc :partial partial) + (string? (:path entry)) (assoc :path (bounded-text (:path entry) 1024)))] + [projected rejected])) + + (defn project-diagnostic + "Returns bounded diagnostic fields and representable partial evidence. + + Args: + evidence: Execution evidence with optional failure details. + + Returns: + Diagnostic fields with bounded text. Partial entries that cannot + cross Transit produce ordered encoding diagnostics. Representable + partial values retain their raw numeric values." + [evidence] + (let [alias? (and (seq (:errors evidence)) (contains? evidence :partial) + (identical? (:partial evidence) (:partial (first (:errors evidence))))) + [projected rejected] (project-entry (if alias? (dissoc evidence :partial) evidence)) + entries (mapv project-entry (:errors evidence)) + rejected (into (vec rejected) (mapcat second) entries) + errors (mapv first entries) + projected (cond-> projected + (and alias? (contains? (first errors) :partial)) + (assoc :partial (:partial (first errors))))] + (cond-> projected + (seq errors) (assoc :errors errors) + (seq rejected) (assoc :errors (into (if (seq errors) errors [(dissoc projected :partial)]) rejected))))))) diff --git a/frontend/performance/benches/render_wasm/features/plan.cljs b/frontend/performance/benches/render_wasm/features/plan.cljs new file mode 100644 index 0000000000..e2b49d49d8 --- /dev/null +++ b/frontend/performance/benches/render_wasm/features/plan.cljs @@ -0,0 +1,76 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.features.plan + "Requested-feature configurations and comparison edges." + (:require + [benches.render-wasm.features.selection :as selection] + [clojure.string :as str])) + +(def ^:const default-max-features 3) + +(defn check-manifest + "Checks names and empty declared defaults against Cargo metadata's feature + definitions. Cargo owns activation; these definitions are not resolved here." + [features definitions] + (let [features (:features (selection/feature-selection {:features features}))] + (when-not (and (map? definitions) + (every? string? (keys definitions)) + (every? #(and (vector? %) (every? string? %)) (vals definitions))) + (throw (ex-info "invalid Cargo feature definitions" {:type ::invalid-manifest}))) + (when (seq (get definitions "default")) + (throw (ex-info "ab requires empty declared Cargo defaults" {:type ::nonempty-defaults}))) + (when-let [unknown (seq (remove #(contains? definitions %) features))] + (throw (ex-info "unknown requested Cargo features" + {:type ::unknown-features :features (vec unknown)}))) + features)) + +(defn plan + "Plans sorted, MSB-first reflected Gray configurations and all off-to-on + edges. Each selection disables Cargo defaults. Execution needs confirmation + above eight configurations or with an empty filter; dry-run needs none." + [features {:keys [max-features confirm dry-run filter] + :or {max-features default-max-features confirm false dry-run false}}] + (let [features (:features (selection/feature-selection {:features features})) + n (count features)] + (when-not (and (int? max-features) (<= 1 max-features 6)) + (throw (ex-info "max-features must be an integer from 1 to 6" {:type ::invalid-limit}))) + (when-not (and (boolean? confirm) (boolean? dry-run) + (or (nil? filter) (string? filter))) + (throw (ex-info "invalid feature planning options" {:type ::invalid-options}))) + (when (zero? n) + (throw (ex-info "ab requires at least one requested feature" {:type ::missing-features}))) + (when (> n max-features) + (throw (ex-info "requested features exceed max-features" + {:type ::too-many-features :count n :max-features max-features}))) + (when (and (not dry-run) (not confirm) + (or (> (bit-shift-left 1 n) 8) (str/blank? filter))) + (throw (ex-info "ab requires --confirm above eight configurations or with an empty filter" + {:type ::confirmation-required}))) + (let [slug (fn [bits] + (str/join (map #(if (bit-test bits %) "1" "0") + (reverse (range n))))) + gray-values (mapv #(bit-xor % (bit-shift-right % 1)) + (range (bit-shift-left 1 n))) + configurations (mapv (fn [bits] + {:slug (slug bits) + :selection {:default-features? false + :features (into [] + (keep-indexed + (fn [i feature] + (when (bit-test bits (- n i 1)) + feature))) + features)}}) + gray-values) + edges (vec + (for [bits gray-values + [i feature] (map-indexed vector features) + :let [bit (- n i 1)] + :when (not (bit-test bits bit))] + {:from (slug bits) + :to (slug (bit-set bits bit)) + :feature feature}))] + {:features features :configurations configurations :edges edges}))) diff --git a/frontend/performance/benches/render_wasm/features/selection.cljs b/frontend/performance/benches/render_wasm/features/selection.cljs new file mode 100644 index 0000000000..338c1e713e --- /dev/null +++ b/frontend/performance/benches/render_wasm/features/selection.cljs @@ -0,0 +1,29 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.features.selection + "Feature validation.") + +(def ^:private diagnostic-features #{"stats" "profile" "profile-macros" "profile-raf"}) + +(defn feature-name? + "True for a Cargo feature name made of letters, digits, underscores or hyphens." + [value] + (and (string? value) + (boolean (re-matches #"[A-Za-z0-9_-]+" value)))) + +(defn feature-selection + "Validates measurement feature selection before allocating an attempt. + Cargo validates functional feature names during the retained build." + [{:keys [default-features? features] + :or {default-features? true features []}}] + (when-not (and (boolean? default-features?) (vector? features) + (every? feature-name? features)) + (throw (ex-info "invalid Cargo feature selection" {:type ::invalid-features}))) + (when-let [selected (seq (filter diagnostic-features features))] + (throw (ex-info "measurement builds cannot enable diagnostic Cargo features" + {:type ::diagnostic-features :features (vec selected)}))) + {:default-features? default-features? :features (vec (sort (distinct features)))}) diff --git a/frontend/performance/benches/render_wasm/fingerprint.cljs b/frontend/performance/benches/render_wasm/fingerprint.cljs new file mode 100644 index 0000000000..1d8be614cc --- /dev/null +++ b/frontend/performance/benches/render_wasm/fingerprint.cljs @@ -0,0 +1,77 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.fingerprint + "Scene fingerprints to uniquely identify snapshots by their content." + (:require + [benches.render-wasm.codec :as codec] + [goog.crypt :as crypt] + [goog.crypt.Sha256])) + +(declare canonicalize) + +(defn- canonical-text + "Encodes a Transit representation as text." + [value] + (js/JSON.stringify value)) + +(defn- canonical-array + "Preserves array order while canonicalizing each element." + [value] + (into-array (map canonicalize (array-seq value)))) + +(defn- canonical-pairs + "Orders a compound map by canonical key text, preserving key/value pairs." + [value] + (->> (partition 2 (array-seq value)) + (map (fn [[k v]] [(canonicalize k) (canonicalize v)])) + (sort-by (comp canonical-text first)) + (mapcat identity) + (into-array))) + +(defn- canonicalize + "Orders verbose Transit objects, compound maps and unordered sets. + Tagged ordered-map and ordered-set representations retain their array order." + [value] + (cond + (array? value) + (canonical-array value) + + (and (some? value) (= "object" (goog/typeOf value))) + (let [keys (sort (array-seq (js/Object.keys value))) + out (js/Object.create nil)] + (doseq [k keys] + (let [v (unchecked-get value k)] + (aset out k + (cond + (= k "~#cmap") (canonical-pairs v) + (= k "~#set") (->> (array-seq v) + (map canonicalize) + (sort-by canonical-text) + (into-array)) + :else (canonicalize v))))) + out) + + :else value)) + +(defn snapshot-text + "Returns canonical Transit text for a validated snapshot. + Realizing :objects removes ObjectsMap's serialized shape strings while keeping + production shape, geometry and path tags in the values." + [snapshot] + (-> (assoc snapshot :objects (into {} (:objects snapshot))) + (codec/encode-str {:type :json-verbose}) + (js/JSON.parse) + (canonicalize) + (canonical-text))) + +(defn fingerprint + "Returns the SHA-256 and digest for a validated snapshot." + [snapshot] + (let [hash (goog.crypt.Sha256.)] + (.update hash (crypt/stringToUtf8ByteArray (snapshot-text snapshot))) + {:algorithm :sha-256 + :digest (crypt/byteArrayToHex (.digest hash))})) diff --git a/frontend/performance/benches/render_wasm/identity.cljc b/frontend/performance/benches/render_wasm/identity.cljc new file mode 100644 index 0000000000..79e4c3ef4a --- /dev/null +++ b/frontend/performance/benches/render_wasm/identity.cljc @@ -0,0 +1,36 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.identity + "Build identity shared by the isolated Node catalog and browser bundle." + #?(:cljs (:require-macros [benches.render-wasm.identity :refer [compiled-profile-marks]])) + #?(:clj (:require [app.render-wasm.performance :as performance]))) + +#?(:clj + (defmacro compiled-profile-marks + "Captures the profiling switch used by renderer macros in this compiler." + [] + (performance/enabled?))) + +#?(:cljs + (do + (goog-define build-id "unprepared") + + (defn data + [] + {:build-id build-id :cljs-profile-marks (compiled-profile-marks)}) + + (defn check + "Checks a loaded bundle's identity against the selected build." + [expected-id actual] + (when-not (and (map? actual) + (string? (:build-id actual)) + (= expected-id (:build-id actual))) + (throw (ex-info "renderer build identity mismatch" + {:type ::mismatch + :expected-id expected-id + :actual-id (:build-id actual)}))) + actual))) diff --git a/frontend/performance/benches/render_wasm/measurement.cljs b/frontend/performance/benches/render_wasm/measurement.cljs new file mode 100644 index 0000000000..a605ac2aa9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/measurement.cljs @@ -0,0 +1,85 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.measurement + "Projects browser phase evidence into saved attempts." + (:require + [app.common.schema :as sm] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.result :as result])) + +(def ^:private schema:response + [:multi {:dispatch :status} + ["ok" [:map [:status [:= "ok"]]]] + ["failed" (sm/merge errors/schema:diagnostic [:map [:status [:= "failed"]]])] + ["stale" [:map {:closed true} [:status [:= "stale"]]]]]) + +(defn partial-phases + "Returns the valid prefix of delivered phase history. + + Args: + phases: raw blocks from a browser response. + preparation-id: expected owner of the blocks. + order: allowed phase names in execution order. + + Returns: + Valid blocks up to the first malformed block or failed phase." + [phases preparation-id order] + (loop [phases (when (sequential? phases) (seq phases)) order order out []] + (if-let [phase (first phases)] + (if (and (sm/validate result/schema:phase phase) + (= preparation-id (:preparation-id phase)) + (= (first order) (:phase phase)) + (every? #(= 1 (count (get-in % [:metrics :command/elapsed]))) (:events phase))) + (let [out (conj out phase)] + (if (= :failed (:outcome phase)) out + (recur (next phases) (next order) out))) + out) + out))) + +(defn check-response + "Returns checked bridge evidence or an invalid-output failure. + + Stale responses retain their status and gain an ownership diagnostic. + Malformed responses retain their valid phase prefix and raw response. + + Args: + phase: The preparation or attempt receiving the response. + preparation-id: Expected owner of delivered phase blocks. + response: Decoded bridge envelope. + + Returns: + Execution evidence with a known status and diagnostic fields on failure." + [phase preparation-id response] + (if (sm/validate schema:response response) + (if (= "stale" (:status response)) + (assoc response :phase (name phase) :message "browser context was superseded") + response) + (assoc (errors/fail-data + (errors/evidence :invalid-output + (ex-info "invalid bridge response" + {:type ::invalid-response :partial {:response response}}))) + :phases (partial-phases (:phases response) preparation-id + (if (= :preparation phase) [:preparation] [:main :reset]))))) + +(defn attempt + "Returns an attempt with raw phase histories and available failure evidence. + + Args: + case-id: saved case identifier. + preparation-id: preparation identifier, or nil for an early failure. + warmup?: whether this execution occupies a warmup slot. + evidence: browser response with status, phases and optional failure details. + + Returns: + An attempt ready for chronological recording." + [case-id preparation-id warmup? evidence] + (cond-> {:case-id case-id :preparation-id preparation-id :warmup? warmup? + :outcome (if (= "ok" (:status evidence)) :completed :failed) + :phases (or (:phases evidence) [])} + (not= "ok" (:status evidence)) + (assoc :failure (dissoc (errors/project-diagnostic evidence) :partial)) + (:partial evidence) (assoc :partial (:partial evidence)))) diff --git a/frontend/performance/benches/render_wasm/node/catalog/entry.cljs b/frontend/performance/benches/render_wasm/node/catalog/entry.cljs new file mode 100644 index 0000000000..3d3e61194b --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/catalog/entry.cljs @@ -0,0 +1,59 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.catalog.entry + "Node entry points of a compiled renderer benchmark build. + + The `:bench-render-wasm-catalog` build in `frontend/shadow-cljs.edn` names + this namespace as its compilation entry through `:exports`, which exposes + `collect-cases` to Node as `collectCases` and `build-identity` as + `buildIdentity`. Both functions use Transit-encoded strings at the Node + boundary. + + Requiring this namespace loads the benchmark scene declarations. Each build's + `cases.cjs` artifact has its own scene registry and compiled identity; load + the artifact of the build being queried." + (:require + [app.common.transit :as transit] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.identity :as identity])) + +(defn collect-cases + "Collects the benchmark cases selected from this build. + + Args: + encoded-selection: A Transit-encoded map. The required :master-seed is an + integer from 0 through 4294967295 used to derive the seed of each case. + The optional :filter is a substring; a blank value selects all cases, + otherwise only case ids that contain it are collected. + + Returns: + A Transit-encoded map with :build-id, the identity compiled into this build, + and :cases, a non-empty vector of validated case descriptors. + Each descriptor has the case's :id and :scene, a :scene-description, its + :params and :scene-seed, and the execution settings used by the runner. + + Throws: + When the selection cannot be decoded, :master-seed is missing or not in + 0 through 4294967295, no cases are selected, or a registered scene or case + fails validation. Case collection failures carry a :type in their + exception data." + [encoded-selection] + (codec/encode-str + {:build-id identity/build-id + :cases (cases/collect-cases (transit/decode-str encoded-selection))})) + +(defn build-identity + "Returns the identity compiled into this build. + + Returns: + A Transit-encoded string. Decoding it yields a map with :build-id, the + identifier compiled into this artifact, defaulting to `unprepared`, and + :cljs-profile-marks, a boolean that records whether renderer profiling + marks were enabled at compile time." + [] + (codec/encode-str (identity/data))) diff --git a/frontend/performance/benches/render_wasm/node/catalog/loader.cljs b/frontend/performance/benches/render_wasm/node/catalog/loader.cljs new file mode 100644 index 0000000000..0d30924d96 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/catalog/loader.cljs @@ -0,0 +1,47 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.catalog.loader + "Loads case descriptors from a selected build, without the runner's registry." + (:require + ["module" :as node-module] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.declarations :as declarations] + [benches.render-wasm.identity :as identity])) + +(defn- load-catalog + [build] + (let [cases-path (:cases-path build)] + (if (exists? js/require) + (js/require cases-path) + ((node-module/createRequire cases-path) cases-path)))) + +(defn build-identity + "Reads and validates the selected catalog's compiled build facts." + [build] + (let [loaded (transit/decode-str (.buildIdentity ^js (load-catalog build)))] + (identity/check (:id build) loaded) + loaded)) + +(defn collect + "Requires the selected cases.cjs and checks both identity responses. + + build comes from artifacts/load-build or a newly completed build. The + createRequire branch also supports the Node ESM test entry points." + [build selection] + (let [catalog (load-catalog build) + loaded (transit/decode-str (.buildIdentity ^js catalog))] + (identity/check (:id build) loaded) + (let [response (transit/decode-str + (.collectCases ^js catalog (codec/encode-str selection))) + cases (:cases response)] + (identity/check (:id build) response) + (when-not (and (vector? cases) (seq cases)) + (throw (ex-info "renderer build returned an invalid case catalog" + {:type ::invalid-catalog + :build-id (:id build)}))) + (mapv declarations/check-collected-case cases)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/ab.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/ab.cljs new file mode 100644 index 0000000000..279bde3100 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/ab.cljs @@ -0,0 +1,55 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.ab + "Options for comparing configurations of Cargo features." + (:require + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.settings :as settings])) + +(def schema:max-features + [:int {:min 1 :max 6}]) + +(def schema:options + [:merge shared/schema:measurement shared/schema:comparison-permissions + [:map {:closed true} + [:features [:vector {:min 1} shared/schema:feature]] + [:max-features {:default feature-plan/default-max-features} schema:max-features] + [:confirm {:default false} :boolean] + [:dry-run {:default false} :boolean] + [:build-timeout-ms {:default settings/default-build-timeout-ms} shared/schema:timeout-ms]]]) + +(def ^:const usage "runner ab --features A,B [options]") + +(def ^:const description + "Compare configurations of Cargo features. + Managed output: frontend/target/performance/render-wasm/feature-comparisons//feature-comparison.transit") + +(def cli-options + (into [[nil "--features FEATURES" "Comma-separated functional Cargo features" + :parse-fn shared/parse-feature-list] + (shared/number-option "--max-features N" "Maximum number of requested features" + schema:max-features "must be an integer from 1 to 6") + [nil "--confirm" "Confirm above eight configurations or with a blank filter"] + [nil "--dry-run" "Validate and print the plan"] + (shared/build-timeout-option "Deadline for metadata and each configuration build")] + (concat shared/measurement-options shared/comparison-options))) + +(defn resolve-options + "Resolves AB arguments and requires a feature selection." + [options arguments] + (shared/check-arguments arguments 0) + (when-not (contains? options :features) + (throw (ex-info "ab requires at least one requested feature (--features)" + {:type ::shared/invalid-args}))) + options) + +(defn check-options + "Checks feature limits and confirmation for resolved AB options." + [options] + (feature-plan/plan (:features options) options) + options) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/build.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/build.cljs new file mode 100644 index 0000000000..ae5b376236 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/build.cljs @@ -0,0 +1,41 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.build + "Options for preparing a benchmark build." + (:require + [benches.render-wasm.features.selection :as selection] + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.settings :as settings])) + +(def schema:options + [:map {:closed true} + [:default-features {:default true} :boolean] + [:features {:default []} [:vector shared/schema:feature]] + [:build-timeout-ms {:default settings/default-build-timeout-ms} shared/schema:timeout-ms]]) + +(def ^:const usage "runner build [options]") + +(def ^:const description "Prepare a renderer build with the selected Cargo features.") + +(def cli-options + [[nil "--[no-]default-features" "Enable Cargo defaults"] + [nil "--features FEATURES" "Comma-separated functional Cargo features" + :parse-fn shared/parse-feature-list] + (shared/build-timeout-option "Complete preparation deadline")]) + +(defn resolve-options + "Resolves build arguments and supplied flags." + [options arguments] + (shared/check-arguments arguments 0) + options) + +(defn check-options + "Checks measurement feature policy and returns the resolved build options." + [resolved] + (selection/feature-selection {:default-features? (:default-features resolved) + :features (:features resolved)}) + resolved) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/compare.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/compare.cljs new file mode 100644 index 0000000000..eefc8fe0e2 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/compare.cljs @@ -0,0 +1,50 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.compare + "Options for comparing saved benchmark records." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.shared :as shared] + [clojure.string :as str])) + +(def schema:case-pair + [:tuple ::sm/text ::sm/text]) + +(def schema:options + [:merge shared/schema:comparison-permissions + [:map {:closed true} + [:baseline-path ::sm/text] + [:candidate-path ::sm/text] + [:case-pair {:default nil} [:maybe schema:case-pair]]]]) + +(def ^:const usage "runner compare BASELINE CANDIDATE [options]") + +(def ^:const description + "Compare two saved records. Paths resolve from the caller's working directory. +--cases selects one exact baseline,candidate pair by saved case ID or name.") + +(def ^:private valid-case-pair? + (sm/validator schema:case-pair)) + +(defn- parse-case-pair + [text] + (let [pair (str/split text #"," -1)] + (when-not (valid-case-pair? pair) + (throw (ex-info "--cases requires one exact nonblank A,B pair" + {:type ::shared/invalid-args}))) + pair)) + +(def cli-options + (into [[nil "--cases A,B" "Select one baseline,candidate pair" :id :case-pair + :parse-fn parse-case-pair]] + shared/comparison-options)) + +(defn resolve-options + "Resolves two literal paths and supplied comparison flags." + [options arguments] + (shared/check-arguments arguments 2) + (assoc options :baseline-path (first arguments) :candidate-path (second arguments))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/merge.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/merge.cljs new file mode 100644 index 0000000000..070e8c6c79 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/merge.cljs @@ -0,0 +1,31 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.merge + "Options for merging saved benchmark records." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.shared :as shared])) + +(def schema:options + [:map {:closed true} + [:base-path ::sm/text] + [:additional-path ::sm/text] + [:output-dir {:default nil} [:maybe ::sm/text]]]) + +(def ^:const usage "runner merge BASE ADDITIONAL [options]") + +(def ^:const description + "Merge two saved records. Paths resolve from the caller's working directory.") + +(def cli-options + [shared/output-option]) + +(defn resolve-options + "Resolves two literal paths and the supplied output directory." + [options arguments] + (shared/check-arguments arguments 2) + (assoc options :base-path (first arguments) :additional-path (second arguments))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/purge.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/purge.cljs new file mode 100644 index 0000000000..b00c7a6105 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/purge.cljs @@ -0,0 +1,32 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.purge + "Options for purging managed benchmark artifacts." + (:require + [benches.render-wasm.node.cli.shared :as shared])) + +(def targets [:builds :runs :comparisons :all]) + +(def schema:options + [:map {:closed true} + [:target (into [:enum] targets)]]) + +(def ^:const usage "runner purge builds|runs|comparisons|all [options]") + +(def ^:const description "Purge files in one explicit category of managed benchmark artifacts.") + +(def cli-options []) + +(defn resolve-options + "Resolves the required artifact category." + [options arguments] + (shared/check-arguments arguments 1) + (let [target (some #(when (= (name %) (first arguments)) %) targets)] + (when-not target + (throw (ex-info "purge requires builds, runs, comparisons or all" + {:type ::shared/invalid-args}))) + (assoc options :target target))) diff --git a/frontend/performance/benches/render_wasm/node/cli/commands/run.cljs b/frontend/performance/benches/render_wasm/node/cli/commands/run.cljs new file mode 100644 index 0000000000..a6280e1541 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/commands/run.cljs @@ -0,0 +1,69 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.commands.run + "Options for a benchmark run and its selected build source." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.commands.build :as build] + [benches.render-wasm.node.cli.shared :as shared])) + +(def schema:build + [:multi {:dispatch :kind} + [:prepare + [:merge build/schema:options + [:map {:closed true} + [:kind [:= :prepare]]]]] + [:existing + [:map {:closed true} + [:kind [:= :existing]] + [:id shared/schema:build-id]]]]) + +(def schema:options + [:merge shared/schema:measurement + [:map {:closed true} + [:build schema:build] + [:remove {:default false} :boolean]]]) + +(def ^:const usage "runner run [options]") + +(def ^:const description + "Run cases using existing or new builds. +Managed output: frontend/target/performance/render-wasm/runs//run.transit") + +(def cli-options + (into [[nil "--build ID" "Reuse a ready build" :id :build-id + :validate [(sm/validator shared/schema:build-id) "must be a benchmark artifact ID"]]] + (concat build/cli-options shared/measurement-options + [[nil "--remove" "Remove the used build after shutdown"]]))) + +(defn resolve-options + "Selects an existing build or a preparation branch from supplied flags." + [options arguments] + (shared/check-arguments arguments 0) + (let [preparation (select-keys options [:default-features :features :build-timeout-ms])] + (when (and (contains? options :build-id) (seq preparation)) + (throw (ex-info "--build cannot be combined with --features, --[no-]default-features or --build-timeout-ms" + {:type ::shared/invalid-args}))) + (assoc (dissoc options :build-id :default-features :features :build-timeout-ms) + :build (if (contains? options :build-id) + {:kind :existing :id (:build-id options)} + (assoc preparation :kind :prepare))))) + +(defn check-options + "Checks measurement feature policy for the selected build source." + [options] + (when (= :prepare (get-in options [:build :kind])) + (build/check-options (:build options))) + options) + +(defn get-help-defaults + "Returns CLI defaults for run options with a preparation source." + [options] + (assoc (dissoc options :build) + :default-features (get-in options [:build :default-features]) + :features (get-in options [:build :features]) + :build-timeout-ms (get-in options [:build :build-timeout-ms]))) diff --git a/frontend/performance/benches/render_wasm/node/cli/console.cljs b/frontend/performance/benches/render_wasm/node/cli/console.cljs new file mode 100644 index 0000000000..16c9720347 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/console.cljs @@ -0,0 +1,40 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.console + "Console settings carried by resolved benchmark requests." + (:require + [benches.render-wasm.node.cli.shared :as shared])) + +(def schema:console + [:map {:closed true} + [:level {:default :info} [:enum :info :warn :debug]] + [:color {:default :auto} [:enum :auto :always :never]]]) + +(def cli-options + [["-q" "--quiet" "Only warnings, errors and results"] + ["-v" "--verbose" "Show debug details and raw build output"] + [nil "--[no-]color" "Colorize terminal output"]]) + +(defn console-level + "Returns the console level selected by quiet or verbose options." + [opts] + (cond (:quiet opts) :warn (:verbose opts) :debug :else :info)) + +(defn color-mode + "Returns automatic, forced or disabled console coloring." + [opts] + (case (:color opts) true :always false :never :auto)) + +(defn resolve-options + "Resolves supplied console flags into console settings." + [options] + (when (and (:quiet options) (:verbose options)) + (throw (ex-info "--quiet and --verbose are mutually exclusive" + {:type ::shared/invalid-args}))) + (cond-> {} + (or (:quiet options) (:verbose options)) (assoc :level (console-level options)) + (contains? options :color) (assoc :color (color-mode options)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/deferred.cljs b/frontend/performance/benches/render_wasm/node/cli/deferred.cljs new file mode 100644 index 0000000000..3cda492436 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/deferred.cljs @@ -0,0 +1,74 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.deferred + "Deferred execution entry for build, run and feature comparison commands." + (:require + [app.common.transit :as transit] + [benches.render-wasm.errors :refer [attempt failure?]] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.feature-comparison :as feature-comparison] + [benches.render-wasm.node.runner.run :as run] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.report.format :as format])) + +(defn- ^:async build-command + [{:keys [get-root signal]} options] + (let [created (atom nil) + result (attempt :catch-fatal + (await (build/prepare + {:root (get-root) + :selection {:default-features? (:default-features options) + :features (:features options)} + :build-timeout-ms (:build-timeout-ms options) + :signal signal :on-created #(reset! created %)}))) + result (if (failure? result) + (outcome/failed result {:build @created}) + (outcome/succeeded {:build result :log-path (:log-path result)}))] + (cond-> result + @created (assoc :report (str "Build " (:id @created) "\nOutput " (:dir @created)))))) + +(defn- ^:async run-command + [{:keys [get-root signal services]} options] + (let [{:keys [home report] :as result} (await (run/run (get-root) options signal services))] + (assoc result :report (str "Run " (:run-id home) "\nOutput " (:dir home) + (when report (str "\nReport\n" report)))))) + +(defn- ^:async ab-command + [{:keys [get-root signal services]} options] + (let [root (get-root) + prepared (await (feature-comparison/prepare root options signal))] + (if (:dry-run options) + (outcome/succeeded {:report (feature-comparison/format-plan prepared)}) + (let [{:keys [home exit-code] :as result} + (await (feature-comparison/execute root prepared signal {:services services})) + interrupted (.-aborted signal) + failures (when (and home (= 1 exit-code) (not interrupted) + (= :failed (get-in result [:comparison :status])) + (empty? (:errors result))) + (format/format-feature-comparison-failures (:comparison result) home))] + (cond-> result + home (assoc :report (str "Comparison " (:comparison-id home) + "\nOutput " (:dir home) + "\nStatus " (cond + interrupted "interrupted" + (zero? exit-code) "completed" + :else "failed") + (when failures (str "\n" failures))))))))) + +(def ^:private handlers + {"build" build-command "run" run-command "ab" ab-command}) + +(defn ^:async execute + "Accepts a Transit string containing a complete request and returns its + numeric exit status after final publication." + [text] + (let [request (attempt :catch-fatal (transit/decode-str text))] + (if (failure? request) + (execution/fail request) + (await (execution/execute request handlers {:get-root artifacts/frontend-root}))))) diff --git a/frontend/performance/benches/render_wasm/node/cli/execution.cljs b/frontend/performance/benches/render_wasm/node/cli/execution.cljs new file mode 100644 index 0000000000..db3d45aa4b --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/execution.cljs @@ -0,0 +1,258 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.execution + "Validate benchmark command requests, run their handlers and print results. + + Both the CLI entry and the separately compiled renderer command entry use + this lifecycle. Callers supply a complete request and the handlers available + at that entry. Handlers own the command's files, processes and cleanup; this + module supplies cancellation, prints final reports and errors, and returns + a numeric status for the entry to use as its exit code. + + `fail` reports parsing, loading or decoding failures that occur before a + request reaches `execute`. Signal helpers also support work outside the + checked command lifecycle. Execution and failure reporting share console + settings; do not overlap calls that require different settings or streams." + (:require + [app.common.schema :as sm] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.node.cli.registry :as registry] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.report.format :as format])) + +(defn on-signals + "Listen for process interrupts so the caller can stop work and finish cleanup. + + Args: + callback: Function called with the string `SIGINT` or `SIGTERM` each time the + Node process receives that signal while the listeners are installed. + + Returns: + An idempotent function that removes only the listeners installed by this + call. Call it when the work settles. This helper does not abort work, + wait for cleanup or choose an exit status." + [callback] + (let [interrupt (fn [] (callback "SIGINT")) + terminate (fn [] (callback "SIGTERM"))] + (.on js/process "SIGINT" interrupt) + (.on js/process "SIGTERM" terminate) + (fn [] + (.removeListener js/process "SIGINT" interrupt) + (.removeListener js/process "SIGTERM" terminate)))) + +(defn ^:async with-interruption + "Run work with cooperative cancellation and preserve its result on interruption. + + The first SIGINT or SIGTERM aborts the supplied signal. Its reason is an + exception with `:signal` set to that signal's name. Later signals do not + replace the reason or force termination. The work must observe cancellation + and settle after its own cleanup; this wrapper imposes no timeout and cannot + preempt synchronous code. Listeners are removed after the work settles. + + Args: + f: Function called once with an AbortSignal. Return an outcome map, or a + promise of one, containing `:exit-code`, an `:errors` vector and + `:log-path` (nil when there is no log). Additional fields may retain partial + results, file paths or a final `:report` string. + + Returns: + A promise of the outcome. Thrown or rejected failures become outcomes with + exit code 1. After interruption, set the exit code to 130 for SIGINT or 143 + for SIGTERM and retain only errors recognized by `errors/fatal?`. Other + outcome fields are preserved. This function does not print the outcome." + [f] + (let [controller (js/AbortController.) + signal (.-signal controller) + interrupted (atom nil) + remove-signals (on-signals + (fn [signal-name] + (when-not (.-aborted signal) + (reset! interrupted signal-name) + (.abort controller + (ex-info "interrupted" + {:type ::interrupted :signal signal-name})))))] + (try + (let [result (attempt :catch-fatal (await (f signal))) + finished (if (failure? result) (outcome/failed result {}) result)] + (cond-> finished + @interrupted (-> (update :errors #(filterv errors/fatal? %)) + (assoc :exit-code (outcome/interruption-code @interrupted))))) + (finally + (remove-signals))))) + +(defn- configure-console + [console local] + (log/configure (merge (select-keys local [:stdout :stderr]) console))) + +(defn- check-gpu + [command options] + (when (and (:require-gpu options) + (or (= command "run") + (and (= command "ab") (not (:dry-run options)))) + (not= "1" (.-PENPOT_GPU (.-env js/process)))) + (throw (ex-info (str "--require-gpu needs PENPOT_GPU=1. " settings/gpu-guidance) + {:type ::gpu-required})))) + +(defn- log-failures + [finished] + (doseq [[index cause] (map-indexed vector (:errors finished))] + (let [data (ex-data cause) + context (mapv (fn [[key value]] [key (errors/format-value value 1024)]) + (errors/project-context data)) + path (when-let [path (:path data)] + (errors/format-value path 1024)) + validation (some-> (::sm/explain data) sm/simplify) + entries (:errors (errors/evidence (or (:phase data) :execution) cause)) + alias? (and (seq entries) + (= (errors/bounded-text (errors/message-of cause) 1024) (:message (first entries)))) + entry-context (when (and alias? + (not= (select-keys data [:phase :operation :path]) + (select-keys (first entries) [:phase :operation :path]))) + (format/format-error-entry + (select-keys (first entries) [:phase :operation :path]))) + entries (if alias? + (rest entries) + entries)] + (log/error {:message (errors/message-of cause) + :fields (into (mapv (fn [[key value]] + [key (if (string? value) + (errors/terminal-text value 1024) + (errors/format-value value 1024))]) + (filterv (comp some? second) + [[:type (:type data)] [:cause-type (:cause-type data)] + [:path path] + [:role (:role data)] [:operation (:operation data)] + [:phase (:phase data)] [:build (:build-id data)] + [:reason (:reason data)] [:manifest-error (:manifest-error data)] + [:baseline (:baseline data)] [:candidate (:candidate data)] + [:features (:features data)] + [:count (:count data)] [:max-features (:max-features data)] + [:validation validation]])) + (concat context + (map #(vector :difference (errors/format-value (:field %) 1024)) (:mismatches data)) + (when entry-context [[:error-context entry-context]]) + (map #(vector :error (format/format-error-entry %)) (take 8 entries)) + (when (> (count entries) 8) + [[:errors-omitted (- (count entries) 8)]]))) + :tail-log (:log-path data) + :tail? (zero? index)})))) + +(defn- report-fallback + [cause local] + (attempt :catch-fatal + (let [message (errors/message-of cause) + stream (or (:stderr local) (.-stderr js/process))] + (.write ^js stream + (str "Could not report benchmark failure: " + (errors/terminal-text message 1024) "\n"))))) + +(defn- finish + "Print the optional report and errors without retrying failed report writes." + [result local] + (binding [log/*log-path* (:log-path result)] + (let [finished (if-some [report (:report result)] + (outcome/publish result #(log/result report)) + result) + reported (attempt :catch-fatal (log-failures finished))] + (if (failure? reported) + (do + (report-fallback reported local) + (:exit-code (outcome/add-error finished reported))) + (:exit-code finished))))) + +(defn fail + "Report a failure that occurred before checked command execution could begin. + + Use this for argument parsing, module loading or request decoding failures. + Do not report a failure again after `execute` has returned its numeric status. + + Args: + cause: Error or thrown value to report. Error data may supply a log path, + usage text, validation details or comparison differences for diagnosis. + console: Trusted console settings with `:level` (`:info`, `:warn`, `:debug`) + and `:color` (`:auto`, `:always`, `:never`). An empty map uses defaults. + Do not take these settings from a request that failed validation. + local: Optional `:stdout` and `:stderr` writable streams with a Node-style + `write` method. Omitted streams use the process streams. + + Returns: + The number 1 after attempting to write the error to stderr and any log + named by its error data. Reporting failures get one bounded stderr fallback + attempt. The one-argument form uses default console settings and streams." + ([cause] + (fail cause {} {})) + ([cause console local] + (configure-console console local) + (finish (outcome/failed (errors/as-failure cause) {}) local))) + +(defn ^:async execute + "Validate a command request, await its handler and print its final outcome. + + Check the request without adding defaults or coercing values. Invalid + requests use default console settings; valid requests choose their console + settings before handler lookup and command policy checks. Unsupported + commands and policy failures are reported without invoking a handler or + looking up the frontend root. + + For `run` and non-dry-run `ab` commands, `:require-gpu` requires the environment + variable `PENPOT_GPU` to equal the string `1` before invoking the handler. + This checks an environment prerequisite, not actual GPU availability. + + Args: + received: Complete request map with a string `:command`, resolved + `:console` settings and command-specific `:options`. Obtain it from + `cli/parse-resolve` or decode it from the CLI's Transit request. Missing + required fields and unknown fields fail validation. + handlers: Map of supported command-name strings to functions called as + `(handler context options)`. Return an outcome or a promise of one, with + `:exit-code`, an `:errors` vector and `:log-path` (nil when there is no log). + An optional `:report` string supplies all final stdout text. Other + result fields do not generate output. Handlers own progress logging + and must await their work and cleanup before returning. + local: Optional map containing `:stdout` and `:stderr` writable streams, + a zero-argument `:get-root` function returning the frontend directory, + and a `:services` map of handler-specific overrides. The context contains + an AbortSignal at `:signal`, plus `:get-root` and `:services` when supplied. + Root lookup and service calls remain the handler's responsibility. These + functions, streams and services are local, not part of the request. + + Returns: + A promise of a numeric exit status after the handler settles and final + output writes have been attempted. Print `:report` once, when present, + followed by errors on stderr, using the outcome's log path for file output. + No report means no final stdout text. Output writes do not wait for streams + to drain. The two-argument form uses process streams and supplies no root + function or services. + + Validation, policy and handler failures produce status 1. Interruption + follows `with-interruption`: await cleanup, preserve partial result fields + and fatal errors, and return 130 for SIGINT or 143 for SIGTERM. A report or + error-reporting failure changes other statuses to 1, with one bounded + stderr fallback if error reporting fails. Interrupted statuses take + precedence. Failed report writes are not retried." + ([received handlers] + (execute received handlers {})) + ([received handlers local] + (configure-console {} local) + (let [result (attempt :catch-fatal + (let [{:keys [command console options]} (request/check received)] + (configure-console console local) + (let [handler (or (get handlers command) + (throw (ex-info "command is unavailable at this execution entry" + {:type ::unsupported-command :command command}))) + options (registry/check-options command options)] + (check-gpu command options) + (await (with-interruption + (fn [signal] + (handler (assoc (select-keys local [:get-root :services]) + :signal signal) + options))))))) + result (if (failure? result) (outcome/failed result {}) result)] + (finish result local)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/help.cljs b/frontend/performance/benches/render_wasm/node/cli/help.cljs new file mode 100644 index 0000000000..7ab3f06c11 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/help.cljs @@ -0,0 +1,25 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.help + "Options for benchmark CLI help." + (:require + [benches.render-wasm.node.cli.registry :as registry])) + +(def schema:options + [:map {:closed true} + [:topic {:default nil} + [:maybe (into [:enum] registry/command-names)]]]) + +(def ^:const usage "runner COMMAND [options]") + +(def ^:const description + "Renderer benchmarks. Use COMMAND --help for command options. +Numeric options accept JavaScript number syntax, including exponents, radix +prefixes and surrounding whitespace, within each option's integer bounds.") + +(def cli-options + [["-h" "--help" "Print help"]]) diff --git a/frontend/performance/benches/render_wasm/node/cli/main.cljs b/frontend/performance/benches/render_wasm/node/cli/main.cljs new file mode 100644 index 0000000000..8f9a5f0414 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/main.cljs @@ -0,0 +1,60 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.main + "Command-line entry for renderer benchmarks and saved results." + (:require + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :refer [attempt failure?]] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.cli.resolve :as cli] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.report.compare :as compare] + [benches.render-wasm.node.report.merge :as merge] + [benches.render-wasm.node.storage.artifacts :as artifacts])) + +(defn- help-command + [_ {:keys [topic]}] + (outcome/succeeded {:report (cli/get-help topic)})) + +(defn- compare-command + [_ {:keys [baseline-path candidate-path] :as options}] + (let [result (compare/execute baseline-path candidate-path options)] + (outcome/succeeded {:comparison (:comparison result) :report (:text result)}))) + +(defn- merge-command + [{:keys [get-root]} {:keys [base-path additional-path] :as options}] + (let [{:keys [run home]} (merge/execute (get-root) base-path additional-path options)] + (outcome/succeeded {:run run :home home :log-path (:log-path home) + :report (str "Run " (:run-id run) "\nOutput " (:dir home))}))) + +(defn- purge-command + [{:keys [get-root]} {:keys [target]}] + (artifacts/purge (get-root) target) + (outcome/succeeded {})) + +(def ^:private handlers + {"help" help-command "compare" compare-command "merge" merge-command "purge" purge-command}) + +(defn- ^:async run-cli + [argv] + (let [request (attempt :catch-fatal (cli/parse-resolve argv))] + (if (failure? request) + (execution/fail request) + (if (contains? handlers (:command request)) + (await (execution/execute request handlers {:get-root artifacts/frontend-root})) + (let [status (attempt :catch-fatal + (let [commands (js/require (path/join js/__dirname "commands.cjs"))] + (await (.execute commands (transit/encode-str request)))))] + (if (failure? status) + (execution/fail status (:console request) {}) + status)))))) + +(defn ^:async -main + "Runs argv and sets the process exit status after publication and cleanup." + [& argv] + (set! (.-exitCode js/process) (await (run-cli argv)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/registry.cljs b/frontend/performance/benches/render_wasm/node/cli/registry.cljs new file mode 100644 index 0000000000..a9541448ec --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/registry.cljs @@ -0,0 +1,68 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.registry + "Ordered command definitions for parsing, request schemas and help." + (:require + [benches.render-wasm.node.cli.commands.ab :as ab] + [benches.render-wasm.node.cli.commands.build :as build] + [benches.render-wasm.node.cli.commands.compare :as compare] + [benches.render-wasm.node.cli.commands.merge :as merge] + [benches.render-wasm.node.cli.commands.purge :as purge] + [benches.render-wasm.node.cli.commands.run :as run])) + +(def commands + [["build" {:usage build/usage + :description build/description + :options build/cli-options + :resolve build/resolve-options + :schema build/schema:options + :check build/check-options}] + ["run" {:usage run/usage + :description run/description + :options run/cli-options + :resolve run/resolve-options + :schema run/schema:options + :help-options {:build {:kind :prepare}} + :help-defaults run/get-help-defaults + :check run/check-options}] + ["ab" {:usage ab/usage + :description ab/description + :options ab/cli-options + :resolve ab/resolve-options + :schema ab/schema:options + :check ab/check-options}] + ["compare" {:usage compare/usage + :description compare/description + :options compare/cli-options + :resolve compare/resolve-options + :schema compare/schema:options}] + ["merge" {:usage merge/usage + :description merge/description + :options merge/cli-options + :resolve merge/resolve-options + :schema merge/schema:options}] + ["purge" {:usage purge/usage + :description purge/description + :options purge/cli-options + :resolve purge/resolve-options + :schema purge/schema:options}]]) + +(def command-names (mapv first commands)) + +(def ^:private definitions (into {} commands)) + +(defn get-definition + "Returns a command definition, or nil for an unknown command." + [command] + (get definitions command)) + +(defn check-options + "Checks command policy and returns the resolved options." + [command options] + (if-let [check (:check (get-definition command))] + (check options) + options)) diff --git a/frontend/performance/benches/render_wasm/node/cli/request.cljs b/frontend/performance/benches/render_wasm/node/cli/request.cljs new file mode 100644 index 0000000000..0aedff2c2c --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/request.cljs @@ -0,0 +1,41 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.request + "Resolved benchmark requests, sender defaults and receiver validation." + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.console :as console] + [benches.render-wasm.node.cli.help :as help] + [benches.render-wasm.node.cli.registry :as registry] + [malli.transform :as mt])) + +(defn- command-schema + [command options] + [:map {:closed true} + [:command [:= command]] + [:console {:default {}} console/schema:console] + [:options {:default {}} options]]) + +(def schema:request + (into [:multi {:dispatch :command}] + (map (fn [[command definition]] + [command (command-schema command (:schema definition))])) + (conj registry/commands ["help" {:schema help/schema:options}]))) + +(def ^:private default-decoder + (delay (sm/decoder schema:request (mt/default-value-transformer)))) + +(defn apply-defaults + "Fills contextual defaults for the selected command and build source." + [partial-request] + (@default-decoder partial-request)) + +(def check + "Returns a complete valid request, or throws with schema explain data." + (sm/check-fn schema:request + :hint "invalid renderer benchmark request" + :type ::invalid-request)) diff --git a/frontend/performance/benches/render_wasm/node/cli/resolve.cljs b/frontend/performance/benches/render_wasm/node/cli/resolve.cljs new file mode 100644 index 0000000000..3f7508f84c --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/resolve.cljs @@ -0,0 +1,106 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.resolve + "Command-specific argument resolution and help for renderer benchmarks." + (:require + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.cli.console :as console] + [benches.render-wasm.node.cli.help :as help] + [benches.render-wasm.node.cli.registry :as registry] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.cli.shared :as shared] + [clojure.string :as str] + [clojure.tools.cli :as tools])) + +(defn- get-definition + [topic] + (if (nil? topic) + {:usage help/usage :description help/description :options []} + (registry/get-definition topic))) + +(defn- get-declarations + [definition] + (into help/cli-options (concat console/cli-options (:options definition)))) + +(defn- argument-error + [topic message cause] + (let [usage (or (:usage (get-definition topic)) help/usage)] + (ex-info (str message "\nUsage: " usage) + (assoc (ex-data cause) + :type ::invalid-args :cause-type (:type (ex-data cause)) + :command topic :usage usage) + cause))) + +(defn parse-resolve + "Returns a complete request for command-first argv, or throws an argument error. + A leading pnpm forwarding delimiter is accepted." + [argv] + (let [argv (if (= "--" (first argv)) (rest argv) argv) + first-arg (first argv) + topic (when (and first-arg (not (str/starts-with? first-arg "-"))) first-arg) + args (if topic (rest argv) argv)] + (when (and topic (nil? (registry/get-definition topic))) + (throw (argument-error topic (str "unknown command: " topic) nil))) + (let [definition (get-definition topic)] + (try + (let [{:keys [options arguments errors]} + (tools/parse-opts args (get-declarations definition) :no-defaults true) + _ (when (seq errors) + (throw (ex-info (str/join "\n" errors) {:type ::shared/invalid-args}))) + console (console/resolve-options options) + partial (if (:help options) + (do + (when (and (nil? topic) (seq arguments)) + (throw (ex-info "a command must precede its options" + {:type ::shared/invalid-args}))) + {:command "help" :console console :options {:topic topic}}) + (do + (when-not topic + (throw (ex-info "a command is required" {:type ::shared/invalid-args}))) + {:command topic :console console + :options ((:resolve definition) + (dissoc options :help :quiet :verbose :color) arguments)})) + resolved (request/check (request/apply-defaults partial))] + (update resolved :options #(registry/check-options (:command resolved) %))) + (catch :default cause + (if (contains? #{::shared/invalid-args ::request/invalid-request + ::feature-plan/too-many-features ::feature-plan/confirmation-required} + (:type (ex-data cause))) + (throw (argument-error topic (ex-message cause) cause)) + (throw cause))))))) + +(defn- get-defaults + [topic] + (let [definition (get-definition topic) + partial {:command (or topic "help") :options (or (:help-options definition) {})} + resolved (request/apply-defaults partial) + options ((or (:help-defaults definition) identity) (:options resolved))] + (merge (:console resolved) options))) + +(defn get-help + "Returns root help for nil or help for a known command name." + [topic] + (let [definition (get-definition topic) + defaults (get-defaults topic) + summary (:summary + (tools/parse-opts + [] (get-declarations definition) + :no-defaults true + :summary-fn + (fn [specs] + (tools/summarize + (mapv (fn [{:keys [id] :as spec}] + (if (contains? defaults id) + (assoc spec :default (get defaults id) + :default-desc (pr-str (get defaults id))) + spec)) + specs)))))] + (str "Usage: " (:usage definition) "\n\n" (:description definition) "\n\n" + (when (nil? topic) + (str (str/join "\n" (map (fn [[_ definition]] (str " " (:usage definition))) registry/commands)) + "\n\n")) + "Options:\n" summary))) diff --git a/frontend/performance/benches/render_wasm/node/cli/shared.cljs b/frontend/performance/benches/render_wasm/node/cli/shared.cljs new file mode 100644 index 0000000000..671d9036e9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/shared.cljs @@ -0,0 +1,109 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.cli.shared + "Value constraints and option groups shared by benchmark commands." + (:require + [app.common.schema :as sm] + [benches.render-wasm.features.selection :as selection] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.random :as random] + [clojure.string :as str])) + +(def schema:nonnegative-count + [:int {:min 0 :max js/Number.MAX_SAFE_INTEGER}]) + +(def schema:positive-count + [:int {:min 1 :max js/Number.MAX_SAFE_INTEGER}]) + +(def schema:timeout-ms + "Positive integer milliseconds within Node's timer range." + [:int {:min 1 :max 2147483647}]) + +(def schema:seed + [:fn {:error/message "expected a uint32 seed"} random/seed?]) + +(def schema:build-id + [:fn {:error/message "expected a benchmark artifact ID"} artifacts/valid-id?]) + +(def schema:feature + [:fn {:error/message "expected a Cargo feature name"} selection/feature-name?]) + +(def schema:measurement + [:map {:closed true} + [:seed {:default 42} schema:seed] + [:filter {:default nil} [:maybe :string]] + [:warmups {:default 3} schema:nonnegative-count] + [:repetitions {:default 10} schema:positive-count] + [:timeout-ms {:default 30000} schema:timeout-ms] + [:startup-timeout-ms {:default settings/default-startup-timeout-ms} schema:timeout-ms] + [:output-dir {:default nil} [:maybe ::sm/text]] + [:screenshots {:default false} :boolean] + [:require-gpu {:default false} :boolean]]) + +(def schema:comparison-permissions + [:map {:closed true} + [:allow-build-differences {:default false} :boolean] + [:allow-scene-differences {:default false} :boolean] + [:allow-all-differences {:default false} :boolean]]) + +(defn parse-feature-list + [text] + (:features (selection/feature-selection {:features (mapv str/trim (str/split text #"," -1))}))) + +(defn- parse-number + [text] + (if (str/blank? text) js/NaN (js/Number text))) + +(defn number-option + "Declares a numeric option with the given value constraint. + Values accept JavaScript number syntax, including exponents, radix prefixes + and surrounding whitespace." + [flag description schema message] + [nil flag description + :parse-fn parse-number + :validate [(sm/validator schema) message]]) + +(def output-option + [nil "--output-dir DIR" "Explicit output home; must not exist" + :validate [(sm/validator ::sm/text) "must be a nonblank path"]]) + +(def measurement-options + [(number-option "--seed N" "Master scene seed" schema:seed + "must be a uint32 integer") + [nil "--filter SUBSTR" "Select cases whose id contains SUBSTR"] + (number-option "--warmups N" "Warmup attempts per case" schema:nonnegative-count + "must be a nonnegative JavaScript-safe integer") + (number-option "--repetitions N" "Measured attempts per case" schema:positive-count + "must be a positive JavaScript-safe integer") + (number-option "--timeout-ms MS" "Case deadline including setup, attempts and diagnostics" schema:timeout-ms + "must be a positive integer within Node's timer range") + (number-option "--startup-timeout-ms MS" "Host/browser readiness deadline" schema:timeout-ms + "must be a positive integer within Node's timer range") + output-option + [nil "--screenshots" "Capture one screenshot per case"] + [nil "--require-gpu" "Fail when GPU rendering is unavailable"]]) + +(def comparison-options + [[nil "--allow-build-differences" "Allow recorded build differences"] + [nil "--allow-scene-differences" "Allow scene inputs and fingerprint content differences"] + [nil "--allow-all-differences" "Allow all comparison compatibility differences"]]) + +(defn build-timeout-option + "Declares a preparation deadline with the given description." + [description] + (number-option "--build-timeout-ms MS" description schema:timeout-ms + "must be a positive integer within Node's timer range")) + +(defn check-arguments + "Checks the number of positional arguments." + [arguments expected] + (when-not (= expected (count arguments)) + (throw (ex-info (str "expected " expected " positional " + (if (= expected 1) "argument" "arguments") + ", received " (count arguments)) + {:type ::invalid-args :arguments arguments})))) diff --git a/frontend/performance/benches/render_wasm/node/files.cljs b/frontend/performance/benches/render_wasm/node/files.cljs new file mode 100644 index 0000000000..e902a5ca87 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/files.cljs @@ -0,0 +1,51 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.files + "Filesystem paths and storage for benchmark artifacts." + (:require + ["fs" :as fs] + ["path" :as path] + [benches.render-wasm.errors :as errors :refer-macros [attempt]] + [clojure.string :as str])) + +(defn contained? + "True when `file` is `root` or a path below it." + [root file] + (let [relative (path/relative root file)] + (and (not (path/isAbsolute relative)) + (not= relative "..") + (not (str/starts-with? relative (str ".." path/sep)))))) + +(defn get-real-path + "Resolves symlink ancestors even when the final path does not exist yet." + [file] + (let [file (path/resolve file)] + (if (fs/existsSync file) + (fs/realpathSync file) + (path/join (get-real-path (path/dirname file)) (path/basename file))))) + +(defn atomic-write + "Writes (UTF-8) content to a temporary file and renames it over `file`, + so readers see either the previous complete file or the new one. The parent + directory must exist. + + A failed write or rename leaves the previous file untouched and attempts to + remove the temporary file. Failures retain operation context in order." + [file content] + (let [temporary (str file "." (random-uuid) ".tmp") + written (attempt :catch-fatal + (fs/writeFileSync temporary content #js {:encoding "utf8" :flag "wx"}) + nil) + renamed (when-not written + (attempt :catch-fatal (fs/renameSync temporary file) nil)) + removed (when (or written renamed) + (attempt :catch-fatal (fs/rmSync temporary #js {:force true}) nil))] + (when-let [failure (errors/combine + [{:phase "storage" :operation :write-temporary :path file :error written} + {:phase "storage" :operation :rename-temporary :path file :error renamed} + {:phase "storage" :operation :remove-temporary :path temporary :error removed}])] + (throw failure)))) diff --git a/frontend/performance/benches/render_wasm/node/log.cljs b/frontend/performance/benches/render_wasm/node/log.cljs new file mode 100644 index 0000000000..1db99a5f80 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/log.cljs @@ -0,0 +1,181 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.log + "Console output and scoped file logs for the benchmark CLI. + + Public API: + + (log/configure {:level :info + :color :auto + :stdout stream + :stderr stream}) + (log/result text) + (log/info text) + (log/debug text) + (log/warn text) + (log/error {:message ... + :fields [...] + :tail-log ...}) + + Every output function uses the same private one: + + ... + └── log/write <-- this guy + ├── append text to the bound log file + └── filter and render console output + ├── result → stdout + └── diagnostics → stderr + + The CLI selects the logging threshold for the console: + - Default: :info + - --quiet: :warn + - --verbose: :debug + + But console filtering and file logging are independent: + + | Function | Console destination | Console visibility | File log | + |----------|---------------------|---------------------|----------| + | result | stdout | Always | Always | + | info | stderr | Default and verbose | Always | + | debug | stderr | Verbose | Always | + | warn | stderr | Always | Always | + | error | stderr | Always | Always | + + So: quiet mode still records detailed diagnostics in files. + Also, the console renderer adds headings, prefixes, and colors." + (:require-macros + [app.common.exceptions :as ex]) + (:require + ["fs" :as fs] + [benches.render-wasm.errors :as errors] + [clojure.string :as str])) + +;; Each high-level task binds its own path and awaits its work inside that scope: +;; (binding [log/*log-path* (:log-path home)] +;; (await (execute-task ...))) +;; Returning or throwing from a child restores the parent's destination +(def ^:dynamic *log-path* nil) + +(defonce ^:private config (atom nil)) + +(defn configure + "Configures console level, color mode and output streams." + [options] + (reset! config (merge {:level :info :color :auto + :stdout (.-stdout js/process) + :stderr (.-stderr js/process)} + options))) + +(defn- color? + [stream mode] + (case mode + :always true + :never false + (and (.-isTTY ^js stream) + (empty? (.-NO_COLOR (.-env js/process))) + (not= "dumb" (.-TERM (.-env js/process)))))) + +(defn- paint + [enabled code text] + (if enabled (str "\u001b[" code "m" text "\u001b[0m") text)) + +(defn- render + [level text color] + (str/join "\n" + (map (fn [line] + (case level + :info (if (re-find #"^\S" line) (paint color "1;36" line) line) + :debug (paint color "2" (str "· " line)) + :warn (paint color "33" (str "! " line)) + :error (paint color "31" line) + line)) + (str/split-lines text)))) + +(defn- write + [level text] + (when-not @config (configure {})) + (let [{:keys [stdout stderr color] console-level :level} @config + text (str/replace (str text) #"\n$" "") + failure (when *log-path* + (try + (fs/appendFileSync *log-path* (str text "\n") "utf8") + nil + (catch :default cause + (errors/fatal-error "cannot append benchmark log" + {:type ::storage-error :log-path *log-path*} + cause)))) + enabled (case level + :debug (= :debug console-level) + :info (contains? #{:info :debug} console-level) + true) + stream (if (= :result level) stdout stderr) + display (if (contains? #{:debug :info :warn} level) + (->> (str/split-lines (errors/bounded-text text 1024)) + (map #(errors/terminal-text % 1024)) + (str/join "\n") + (#(errors/bounded-text % 1024))) + text)] + (when enabled + (.write ^js stream (str (if (= :result level) + display + (render level display (color? stream color))) "\n"))) + (when failure + (if (= :error level) + (.write ^js stderr (str "Could not append log: " (errors/terminal-text *log-path* 1024) "\n")) + (throw failure))))) + +(defn result [text] (write :result text)) +(defn info [text] (write :info text)) +(defn debug [text] (write :debug text)) +(defn warn [text] (write :warn text)) + +(defn- get-tail + [path] + (when path + (ex/ignoring + (let [fd (fs/openSync path "r")] + (try + (let [size (.-size (fs/fstatSync fd)) + length (min size 8192) + buffer (js/Buffer.alloc length) + read (fs/readSync fd buffer 0 length (- size length)) + text (->> (str/split-lines (.toString buffer "utf8" 0 read)) + (remove str/blank?) + (take-last 20) + (map #(errors/terminal-text % (* 6 8192))) + (str/join "\n")) + start (max 0 (- (count text) 8192)) + first (.charCodeAt text start) + start (if (and (pos? start) (<= 0xDC00 first 0xDFFF) + (<= 0xD800 (.charCodeAt text (dec start)) 0xDBFF)) + (inc start) + start)] + (str/split-lines (subs text start))) + (finally (fs/closeSync fd))))))) + +(defn error + "Prints a terminal failure with ordered fields and an optional bounded log tail." + [{:keys [message fields tail-log tail?] :or {tail? true}}] + (let [tail-path (or tail-log *log-path*) + fields (cond-> (vec fields) + (and *log-path* (not (some #(= :log (first %)) fields))) + (conj [:log (errors/terminal-text *log-path* 1024)]) + (and tail-log (not= tail-log *log-path*)) + (conj [:tail-log (errors/terminal-text tail-log 1024)])) + width (reduce max 0 (map #(count (name (first %))) fields)) + tail (when tail? (get-tail tail-path))] + (write :error + (str "✖ " (errors/terminal-text message 1024) + (apply str + (map (fn [[key value]] + (let [label (name key)] + (str "\n " label + (apply str (repeat (+ 2 (- width (count label))) " ")) + value))) + fields)) + (when (seq tail) + (str "\n tail\n" (str/join "\n" (map #(str " " %) tail)))))))) diff --git a/frontend/performance/benches/render_wasm/node/outcome.cljs b/frontend/performance/benches/render_wasm/node/outcome.cljs new file mode 100644 index 0000000000..0f630f8668 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/outcome.cljs @@ -0,0 +1,57 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.outcome + "Result of a benchmark command. + + An outcome is a map with `:exit-code`. A command that failed adds `:errors` + (a vector, primary failure first) and `:log-path`. Exit code 130 and 143 + mark an interruption and no failure replaces them." + (:require + [benches.render-wasm.errors :as errors :refer [attempt failure?]])) + +(defn interruption-code + "Returns the exit code for the interrupting signal name: 143 for SIGTERM, + 130 otherwise." + [signal-name] + (case signal-name "SIGTERM" 143 130)) + +(defn succeeded + "Returns a successful outcome with command result data." + [data] + (merge {:exit-code 0 :errors [] :log-path nil} data)) + +(defn failed + "Returns an outcome for `cause` with defaults for errors, exit code and log. + + Args: + cause: Failure supplying the default error and log path. + data: Outcome fields that override defaults, including :errors, + :exit-code and :log-path. + + Returns: + An outcome with exit code 1 and the failure's log path as defaults." + [cause data] + (merge {:errors [cause] + :exit-code 1 + :log-path (:log-path (ex-data cause))} + data)) + +(defn add-error + "Appends `cause` to the errors of `outcome`. An exit code that is not an + interruption becomes 1." + [outcome cause] + (-> outcome + (update :errors (fnil conj []) cause) + (update :exit-code #(if (contains? #{130 143} %) % 1)))) + +(defn publish + "Calls `f` to print a result of `outcome`. A failure of `f` is added to the + errors of the returned outcome." + [outcome f] + (let [published (attempt :catch-fatal (f))] + (cond-> outcome + (failure? published) (add-error published)))) diff --git a/frontend/performance/benches/render_wasm/node/process.cljs b/frontend/performance/benches/render_wasm/node/process.cljs new file mode 100644 index 0000000000..a01af9077e --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/process.cljs @@ -0,0 +1,604 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.process + "Deadline-bound Node commands with owned POSIX process groups." + (:require + ["child_process" :as child-process] + ["fs" :as fs] + ["node:readline" :as readline] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [clojure.string :as str])) + +(def default-termination-grace-ms 5000) +(def ^:private final-termination-wait-ms 5000) + +(def ^:private max-timer-delay-ms + "Largest delay that Node timers accept; longer delays overflow a signed 32-bit integer." + 2147483647) + +(def ^:private default-options + {:termination-grace-ms default-termination-grace-ms + :capture-output? true}) + +(defn signal-name + "Returns the signal name for an aborted AbortSignal, defaulting to SIGINT." + [signal] + (when (and signal (.-aborted ^js signal)) + (let [reason (.-reason ^js signal)] + (cond + (string? reason) reason + (keyword? reason) (name reason) + :else (or (:signal (ex-data reason)) "SIGINT"))))) + +(defn deadline + "Returns an absolute Date.now() timestamp, in milliseconds. All commands + in a preparation can share it, including commands waiting for a lock. + This is a wall clock deadline, so clock changes affect the time left." + [timeout-ms] + (+ (js/Date.now) timeout-ms)) + +(defn remaining + "Returns the nonnegative milliseconds left on a Date.now() deadline." + [deadline] + (max 0 (- deadline (js/Date.now)))) + + +(defn- unresolved-cause? + [cause seen] + (cond + (nil? cause) false + (:unresolved-process (ex-data cause)) true + (contains? seen cause) false + :else (recur (ex-cause cause) (conj seen cause)))) + +(defn unresolved? + "True when a command's process group may still be running. + + Args: + cause: An Error, including collected errors and wrapped causes, or nil. + + Returns: + True if any error reports :unresolved-process true after termination." + [cause] + (boolean + (some (fn [{:keys [error] :as entry}] + (or (:unresolved-process entry) (unresolved-cause? error #{}))) + (errors/get-errors cause)))) + +(defn- signal-group + [pid signal] + (try + (.kill js/process (- pid) signal) + true + (catch :default cause + (if (= "ESRCH" (.-code cause)) + false + (throw cause))))) + +(defn- linux-group-member? + [pid entry] + (when (re-matches #"\d+" entry) + (try + ;; comm may contain spaces or ')'; fields after its final ')' start + ;; with state, ppid and pgrp. Zombies retain a group but no resources. + (let [stat (fs/readFileSync (str "/proc/" entry "/stat") "utf8") + fields (str/split (str/trim (subs stat (inc (.lastIndexOf stat ")")))) #"\s+")] + (and (= pid (js/Number (nth fields 2))) + (not (contains? #{"Z" "X" "x"} (first fields))))) + (catch :default cause + (if (contains? #{"ENOENT" "ESRCH"} (.-code cause)) + false + (throw cause)))))) + +(defn- group-running? + [pid] + (and (signal-group pid 0) + (if (= "linux" (.-platform js/process)) + ;; Container init need not reap orphan zombies promptly. Waiting + ;; for kill(-pgid, 0) to return ESRCH can therefore hang forever. + (some #(linux-group-member? pid %) (array-seq (fs/readdirSync "/proc"))) + true))) + +(defn- pause + [ms] + (js/Promise. (fn [resolve _] (js/setTimeout resolve ms)))) + + +(defn- get-group-observation + "Checks whether a process group has a live member. + + Args: + pid: The group's leader PID. + + Returns: + A map with :status set to :running, :stopped or :unknown, and :error + when the operating system rejects the probe." + [pid] + (let [result (attempt :catch-fatal (group-running? pid))] + (if (failure? result) + {:status :unknown :error result} + {:status (if result :running :stopped)}))) + +(defn- wait-again? + "Decides whether to take another process-group observation. + + Args: + observation: A map with :status :running, :stopped or :unknown. + context: The absolute :deadline and :phase, either :grace before SIGKILL + or :confirmation after SIGKILL. + now: Current time in milliseconds since the Unix epoch. + + Returns: + True while time remains and the group may be running. Grace ends on a + failed probe; confirmation continues through probe failures." + [observation {:keys [phase deadline]} now] + (and (not= :stopped (:status observation)) + (< now deadline) + (or (= :confirmation phase) (= :running (:status observation))))) + +(defn- group-failed + "Returns the failure of an operation on a command group. + + Args: + kind: The failure type. + action: Text that describes the failed operation. The message appends the + bounded text of the cause. + cause: The Error that the operation raised. + data: Diagnostic data of the failure." + [kind action cause data] + {:type kind + :message (str action ": " (errors/bounded-text (errors/message-of cause) 500)) + :cause cause + :data data}) + +(defn- ^:async wait-for-group + [{:keys [pid deadline signals] :as context} record] + (let [observation (get-group-observation pid) + now (js/Date.now)] + (when-let [cause (:error observation)] + (record :probe-group (group-failed :process-probe "Could not inspect command group" cause {:signals signals}))) + (if (wait-again? observation context now) + (do + (await (pause (min 10 (- deadline now)))) + (await (wait-for-group context record))) + observation))) + +(defn- send-group-signal + [pid signal record] + (let [result (attempt :catch-fatal (signal-group pid signal))] + (when (failure? result) + (record :signal-group + (group-failed :process-signal (str "Could not send " signal " to command group") result {:signal signal}))) + result)) + +(defn- ^:async terminate-group + [pid grace-deadline record] + (let [signals ["SIGTERM"] + term (send-group-signal pid "SIGTERM" record) + grace {:pid pid :phase :grace :signals signals + :deadline (if (failure? term) (js/Date.now) grace-deadline)} + observation (await (wait-for-group grace record))] + (if (= :stopped (:status observation)) + {:signals signals :unresolved-process false} + (let [signals (conj signals "SIGKILL")] + (send-group-signal pid "SIGKILL" record) + (let [confirmation (assoc grace :phase :confirmation :signals signals + :deadline (deadline final-termination-wait-ms)) + observation (await (wait-for-group confirmation record)) + unresolved (not= :stopped (:status observation))] + (when unresolved + (record :terminate-group + {:type :process-unresolved + :message "Command group ownership remains unresolved after SIGKILL" + :data {:fatal true :unresolved-process true :signals signals}})) + {:signals signals :unresolved-process unresolved}))))) + +(defn- process-error + [{:keys [command args cwd]} result {:keys [type message cause data]}] + (ex-info message + (merge {:type type :command command :args args :cwd cwd} result data) + cause)) + +(defn- cancelled + [^js signal] + (let [reason (.-reason signal)] + {:type :process-cancelled + :message "Command cancelled" + :cause (when (failure? reason) reason) + :data {:reason (signal-name signal)}})) + +(defn- timed-out + [deadline] + {:type :process-timeout + :message "Command deadline expired" + :data {:deadline deadline}}) + +(defn- spawn-failed + [^js cause] + {:type :process-spawn + :message "Could not spawn command" + :cause cause + :data {:code (.-code cause)}}) + +(defn- stream-done + [^js stream] + (js/Promise. + (fn [resolve _] + (.once stream "end" resolve) + (.once stream "close" resolve) + (.once stream "error" resolve)))) + +(defn- drain-output + "Allow queued pipe data to drain, but never wait indefinitely for an EOF." + [done grace-deadline] + (js/Promise. + (fn [resolve _] + (let [timer (js/setTimeout resolve (max 10 (remaining grace-deadline)))] + (.then done + (fn [_] + (js/clearTimeout timer) + (resolve))))))) + +(def ^:private empty-result {:exit-code nil :signal nil :stdout "" :stderr ""}) +(def ^:private read-failed-message "Could not read command output") +(def ^:private callback-failed-message "Command output callback failed") + +(defn- group-ids + "Returns {:pid pid :pgid pid} for the leader of a detached process group." + [pid] + {:pid pid :pgid pid}) + +(defn- launch-error + [options failure] + (process-error options empty-result failure)) + +(defn- launch-failure + "Returns the failure that forbids starting a command, or nil. + + Args: + options: Command settings with the absolute :deadline and the optional + AbortSignal :signal." + [{:keys [signal deadline]}] + (cond + (and signal (.-aborted ^js signal)) (cancelled signal) + (zero? (remaining deadline)) (timed-out deadline))) + +(defn- output-failed + [stream cause message] + {:type :process-output :message message :cause cause :data {:stream stream}}) + +(defn- completion-failure + "Returns the failure to record for the reason a command completes. + + Args: + reason: The exit status {:exit-code :signal} or a failure map with :type. + + Returns: + The failure, or nil for a zero exit code. Output failures are recorded + when they occur." + [reason] + (let [kind (:type reason)] + (cond + (= :process-output kind) nil + kind reason + (not= 0 (:exit-code reason)) {:type :process-exit :message "Command exited unsuccessfully"}))) + +(defn- spawn-child + "Spawns the command in a new process group. + + Args: + options: Command settings with :command and literal :args, and optional + :cwd and :env. A map or JS object under :env merges over process.env. + + Returns: + The ChildProcess. + + Throws: + A launch error when the operating system rejects the spawn." + [{:keys [command args cwd env] :as options}] + (let [environment (js/Object.assign #js {} (.-env js/process) (if (map? env) (clj->js env) env))] + (try + (child-process/spawn command (clj->js args) + #js {:cwd cwd :env environment :detached true + :stdio #js ["pipe" "pipe" "pipe"]}) + (catch :default cause + (throw (launch-error options (spawn-failed cause))))))) + +(defn- watch-exit + "Observes the exit and the process errors of a child. + + Returns: + A map with :promise, which resolves with the first outcome, and :state, + a volatile holding the latest outcome. An outcome is {:exit-code :signal} + or the failure of a process error." + [^js child] + (let [state (volatile! nil) + observe (fn [resolve outcome] + (vreset! state outcome) + (resolve outcome))] + {:state state + :promise (js/Promise. + (fn [resolve _] + (.once child "exit" (fn [code signal] (observe resolve {:exit-code code :signal signal}))) + (.once child "error" #(observe resolve (spawn-failed %)))))})) + +(defn- destroy-stream + [^js stream] + (when stream (.destroy stream))) + +(defn- failure-recorder + "Returns (record operation failure), which appends a failure occurrence. + + Args: + options: Command settings. Occurrences carry its :command, :args and :cwd. + child: The ChildProcess. + exit-state: Volatile with the latest exit outcome. Occurrences carry the + exit status known when they occur. + failures: Volatile vector that receives the occurrences in order." + [options ^js child exit-state failures] + (fn [operation failure] + (let [context (merge empty-result + (group-ids (.-pid child)) + (select-keys @exit-state [:exit-code :signal]))] + (vswap! failures conj {:phase "process" :operation operation + :error (process-error options context failure)})))) + +(defn- new-completion + "Returns the settlement of the completion of a command. + + Args: + record: Function from `failure-recorder`. + + Returns: + A map with :promise and :settle. `settle` takes a completion reason. The + first reason records its failure, if any, and resolves :promise. Later + reasons do nothing." + [record] + (let [^js completion (js/Promise.withResolvers) + settled (volatile! false)] + {:promise (.-promise completion) + :settle (fn [reason] + (when-not @settled + (vreset! settled true) + (when-let [failure (completion-failure reason)] + (record :run-command failure)) + ((.-resolve completion) reason)))})) + +(defn- watch-deadline + "Calls `settle` with a timeout failure once `deadline` passes. + + Args: + deadline: Absolute Date.now() timestamp in milliseconds. + settle: Function of a completion reason. + + Returns: + A function that cancels the pending check." + [deadline settle] + (let [timer (volatile! nil) + check (fn check [] + (let [ms (remaining deadline)] + (if (zero? ms) + (settle (timed-out deadline)) + (vreset! timer (js/setTimeout check (min ms max-timer-delay-ms))))))] + (check) + #(js/clearTimeout @timer))) + +(defn- watch-abort + "Calls `settle` with a cancellation failure when `signal` aborts, or at once + when it already aborted. + + Args: + signal: An AbortSignal, or nil. + settle: Function of a completion reason. + + Returns: + A function that removes the listener." + [^js signal settle] + (if signal + (let [abort (fn [_] (settle (cancelled signal)))] + (.addEventListener signal "abort" abort #js {:once true}) + (when (.-aborted signal) (abort nil)) + #(.removeEventListener signal "abort" abort)) + (constantly nil))) + +(defn- watch-completion + "Calls `settle` with the first completion reason among the exit outcome, the + deadline and the abort. + + Args: + options: Command settings with :deadline and the optional :signal. + exited: Promise of the exit outcome. + settle: Function of a completion reason. + + Returns: + A function that stops watching the deadline and the signal." + [{:keys [deadline signal]} ^js exited settle] + (.then exited settle) + (let [stop-deadline (watch-deadline deadline settle) + stop-abort (watch-abort signal settle)] + (fn [] + (stop-deadline) + (stop-abort)))) + +(defn- open-line-reader + "Returns a readline interface that passes the complete lines of `input` to + `on-line` with `stream`. Callback failures and stream errors go to `fail`. + Stream errors keep going to `fail` after the reader closes." + [^js input stream on-line fail] + (let [reader (readline/createInterface #js {:input input :terminal false :crlfDelay js/Infinity}) + unreadable #(fail stream % read-failed-message)] + (.on reader "line" + (fn [line] + (try + (on-line line stream) + (catch :default cause + (fail stream cause callback-failed-message))))) + (.on reader "error" unreadable) + (.once reader "close" #(.on input "error" unreadable)) + reader)) + +(defn- observe-output + "Decodes stdout and stderr of a child as UTF-8 and routes their text. + + Appends each chunk to `captured` when :capture-output? is set, and calls + :on-output with each chunk and :on-line with each complete line. + + Args: + options: Command settings with :capture-output?, :on-output and :on-line. + child: The ChildProcess. + captured: Volatile map from :stdout and :stderr to vectors of text. + fail: Function of a stream, a cause and a message that receives the + failures of streams and callbacks. + + Returns: + A vector with the line readers it opened." + [{:keys [on-output on-line capture-output?]} ^js child captured fail] + (let [route (fn [stream text] + (when capture-output? (vswap! captured update stream conj text)) + (when on-output + (try + (on-output text stream) + (catch :default cause + (fail stream cause callback-failed-message))))) + listen (fn [^js input stream] + (.setEncoding input "utf8") + (.on input "data" #(route stream %)) + (when-not on-line + (.on input "error" #(fail stream % read-failed-message))))] + (listen (.-stdout child) :stdout) + (listen (.-stderr child) :stderr) + (if on-line + [(open-line-reader (.-stdout child) :stdout on-line fail) + (open-line-reader (.-stderr child) :stderr on-line fail)] + []))) + +(defn- close-stdin + "Closes the stdin of a child. Write errors go to `fail`, except EPIPE, which + a command that exits without reading stdin causes." + [^js child fail] + (.on (.-stdin child) "error" + (fn [cause] + (when-not (= "EPIPE" (.-code cause)) + (fail :stdin cause "Could not close command stdin")))) + (.end (.-stdin child))) + +(defn- command-result + "Returns the exit status and the captured text of a finished command." + [status captured] + {:exit-code (:exit-code status) + :signal (:signal status) + :stdout (str/join "" (:stdout captured)) + :stderr (str/join "" (:stderr captured))}) + +(defn- ^:async terminate-and-drain + "Terminates the process group of a child and drains its queued output. + + Args: + options: Command settings with :termination-grace-ms. + child: The ChildProcess. + exit: Map from `watch-exit`. + output-done: Promise that resolves when stdout and stderr end. + record: Function from `failure-recorder`. + + Returns: + A promise of {:status exit-status :termination termination}. The status + comes from the exit of the child. When the group stays unresolved, the + status is the one already known, and the child no longer keeps this + process alive." + [{:keys [termination-grace-ms]} ^js child {exit-state :state exited :promise} output-done record] + (let [grace-deadline (deadline termination-grace-ms) + termination (await (terminate-group (.-pid child) grace-deadline record)) + unresolved? (:unresolved-process termination)] + (when unresolved? (.unref child)) + (let [status (if unresolved? @exit-state (await exited))] + (await (drain-output output-done grace-deadline)) + {:status status :termination termination}))) + +(defn- ^:async supervise + "Watches a spawned command until it completes, then terminates its group. + + Args: + options: The options of `run` with defaults applied. + child: The spawned ChildProcess. + exit: Map from `watch-exit`. + + Returns: + A promise of {:exit-code 0 :stdout string :stderr string}. + + Throws: + The Error that combines the recorded failures with the result, the + termination and the process identity." + [options ^js child exit] + (let [pid (.-pid child) + failures (volatile! []) + captured (volatile! {:stdout [] :stderr []}) + record (failure-recorder options child (:state exit) failures) + {completed :promise settle :settle} (new-completion record) + fail (fn [stream cause message] + (let [failure (output-failed stream cause message)] + (record :read-output failure) + (settle failure))) + output-done (js/Promise.all #js [(stream-done (.-stdout child)) + (stream-done (.-stderr child))]) + readers (observe-output options child captured fail) + stop-waiting (watch-completion options (:promise exit) settle)] + (try + (close-stdin child fail) + (await completed) + (stop-waiting) + (let [{:keys [status termination]} (await (terminate-and-drain options child exit output-done record)) + result (command-result status @captured) + failure (errors/combine @failures)] + (if failure + (throw (errors/with-context failure (errors/message-of failure) + (merge result termination (group-ids pid)))) + (dissoc result :signal))) + (finally + (run! #(.close ^js %) readers) + (stop-waiting))))) + +(defn ^:async run + "Runs a command and owns its detached POSIX process group. + + An expired deadline or aborted signal starts no process. Output callbacks + receive UTF-8 chunks or complete lines, including a final unterminated + line. Completion follows the exit of the root process, not the close of its + output pipes, because descendants can keep the pipes open. Cleanup sends + SIGTERM and waits the termination grace. If the group still runs, it sends + SIGKILL once and allows five more seconds for the group to stop. Unresolved + ownership produces a fatal failure. + + Args: + options: Command settings with :command, literal :args, optional :cwd and + :env, absolute :deadline and optional AbortSignal under :signal. + :termination-grace-ms defaults to 5000. :on-output receives text and + :stdout/:stderr; :on-line receives a line and its stream. Output capture + defaults to true and accepts :capture-output? false for streamed output. + + Returns: + A promise resolving to {:exit-code 0 :stdout string :stderr string}. + + Throws: + Failures reject with command, output and process identity. Collected + occurrences retain execution, output, signalling and probe failures in + order. :process-unresolved marks unresolved group ownership as fatal. + Cancellation records the signal name as :reason and retains an Error + supplied as the abort reason as a cause." + [options] + (let [options (merge default-options options)] + (when-let [failure (launch-failure options)] + (throw (launch-error options failure))) + (let [^js child (spawn-child options) + exit (watch-exit child)] + (try + (if (.-pid child) + (await (supervise options child exit)) + (throw (launch-error options (await (:promise exit))))) + (finally + (destroy-stream (.-stdin child)) + (destroy-stream (.-stdout child)) + (destroy-stream (.-stderr child))))))) diff --git a/frontend/performance/benches/render_wasm/node/report/compare.cljs b/frontend/performance/benches/render_wasm/node/report/compare.cljs new file mode 100644 index 0000000000..98261f82e4 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/report/compare.cljs @@ -0,0 +1,46 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.report.compare + "Offline comparison of two saved schema:run records. + + Inputs are user file paths resolved from the caller's cwd." + (:require + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.result :as result])) + +(defn execute + "Reads and compares two saved run files. + + Runs are only compared if the build options and scene configuration match. + These can be overriden via flags. + + It is also possible to restrict comparison to specific scene ids. + + Options: + - `:case-pair` contains the baseline and candidate selectors, or nil. + - The three `:allow-*-differences` options select build, scene or all + compatibility permissions. + + Returns the validated `comparison` with its selected permissions, retained + differences and formatted text." + [baseline-path candidate-path {:keys [case-pair allow-build-differences + allow-scene-differences allow-all-differences]}] + (let [baseline (runs/get-record baseline-path {:role :baseline}) + candidate (runs/get-record candidate-path {:role :candidate}) + comparison (result/check-record + (compare/compare-runs + baseline candidate + (cond-> {:allow-build-differences (boolean allow-build-differences) + :allow-scene-differences (boolean allow-scene-differences) + :allow-all-differences (boolean allow-all-differences)} + case-pair + (assoc :pairs (compare/selector-pair baseline candidate + case-pair)))))] + {:comparison comparison + :text (format/format-comparison comparison)})) diff --git a/frontend/performance/benches/render_wasm/node/report/merge.cljs b/frontend/performance/benches/render_wasm/node/report/merge.cljs new file mode 100644 index 0000000000..d719265544 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/report/merge.cljs @@ -0,0 +1,28 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.report.merge + "Offline merge of two saved schema:run records." + (:require + ["fs" :as fs] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.result :as result])) + +(defn execute + "Reads and merges two saved run files from the caller's cwd, then writes a + new run home. Inputs remain unchanged. Returns the merged record and home." + [root base-path additional-path {:keys [output-dir]}] + (let [base (runs/get-record base-path {:role :base}) + additional (runs/get-record additional-path {:role :additional}) + run-id (str (random-uuid)) + merged (result/merge-runs base additional {:run-id run-id}) + home (runs/create-home {:root root + :run-id run-id + :output-dir output-dir})] + (runs/write-record! home merged) + (fs/appendFileSync (:log-path home) + (str "merge " (:run-id base) " + " (:run-id additional) "\n")) + {:run merged :home home})) diff --git a/frontend/performance/benches/render_wasm/node/runner/browser.cljs b/frontend/performance/benches/render_wasm/node/runner/browser.cljs new file mode 100644 index 0000000000..6d1a237745 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/browser.cljs @@ -0,0 +1,385 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.browser + "Owns Chromium through process/run from spawn, then attaches Playwright by CDP. + Shutdown stops the owned process group before disconnecting the client." + (:require-macros + [app.common.exceptions :as ex]) + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + ["playwright" :as playwright] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors :refer [attempt]] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.settings :as settings])) + +(defn gpu-active? + "Returns whether the development container enables GPU access." + [] + (= "1" (.-PENPOT_GPU (.-env js/process)))) + +(defn launch-settings + "Returns the resolved Chromium settings for this environment." + [] + {:headless true + :transport :cdp + :args (cond-> ["--headless" + "--no-sandbox" + "--disable-dev-shm-usage" + "--enable-unsafe-swiftshader" + "--no-first-run" + "--no-default-browser-check" + "--disable-background-networking" + "--disable-background-timer-throttling" + "--disable-renderer-backgrounding" + "--hide-scrollbars" + "--mute-audio"] + (gpu-active?) (into ["--enable-gpu" + "--use-gl=angle" + "--use-angle=vulkan"]))}) + +(defn within + "Calls work while its absolute deadline and optional AbortSignal allow it. + Clears the timer/listener on every outcome and handles late rejections. + The caller owns shutdown of the browser running the work." + [work {:keys [deadline signal phase]}] + (js/Promise. + (fn [resolve reject] + (let [settled (atom false) + timer (atom nil)] + (letfn [(cleanup [] + (js/clearTimeout @timer) + (when signal + (.removeEventListener ^js signal "abort" aborted))) + (succeeded [value] + (when (compare-and-set! settled false true) + (cleanup) + (resolve value))) + (failed [cause] + (when (compare-and-set! settled false true) + (cleanup) + (reject cause))) + (aborted [] + (failed (or (.-reason ^js signal) + (ex-info "browser work aborted" {:type ::aborted :phase phase})))) + (expired [] + (failed (ex-info "browser work timed out" {:type ::timeout :phase phase})))] + (when signal + (.addEventListener ^js signal "abort" aborted #js {:once true})) + (cond + (and signal (.-aborted ^js signal)) + (aborted) + + (zero? (process/remaining deadline)) + (expired) + + :else + (do + (reset! timer (js/setTimeout expired (process/remaining deadline))) + (try + (.then (js/Promise.resolve (work)) succeeded failed) + (catch :default cause + (failed cause)))))))))) + +(defn- deadline-failure + [cause deadline] + (if (contains? (ex-data cause) ::deadline-exceeded) + cause + (errors/with-context cause (errors/message-of cause) + {::deadline-exceeded (or (= ::timeout (:type (ex-data cause))) + (zero? (process/remaining deadline)))}))) + +(defn evaluate + "Calls a fixed bridge method, optionally with a Transit string. + JSON quoting embeds the argument; descriptors and results remain Transit." + ([^js page method] + (.evaluate page (str "window.__benchBridge." method "()"))) + ([^js page method text] + (.evaluate page (str "window.__benchBridge." method "(" (.stringify js/JSON text) ")")))) + +(defn- endpoint-waiter + "Keeps the last 64 KiB of startup stderr until a complete endpoint line arrives." + [] + (let [callbacks (atom nil) + settled (atom false) + output (atom "") + address (atom nil) + promise (js/Promise. (fn [resolve reject] + (reset! callbacks {:resolve resolve :reject reject})))] + ;; Cancellation may precede the first within call. Keep the endpoint's + ;; rejection handled even when that call never subscribes to it. + (.catch promise (fn [_] nil)) + {:promise promise + :address address + :on-output (fn [text stream] + (when (and (= :stderr stream) (not @settled)) + (let [buffer (str @output text)] + (if-let [[_ endpoint] (re-find #"DevTools listening on (ws://[^\s]+)\r?\n" buffer)] + (do + (reset! settled true) + (reset! address endpoint) + (reset! output "") + ((:resolve @callbacks) endpoint)) + (reset! output (subs buffer (max 0 (- (count buffer) 65536)))))))) + :failed (fn [cause] + (when (compare-and-set! settled false true) + ((:reject @callbacks) + (ex-info "Chromium exited before its CDP endpoint became ready" + (merge (errors/project-context (ex-data cause)) + {:type ::endpoint-exit :stderr @output}) + cause))))})) + +(defn- ^:async stop-process + [{:keys [controller process]}] + (.abort controller) + (try + (await process) + nil + (catch :default cause + (let [entries (remove (fn [{:keys [error]}] + (and (= :process-cancelled (:type (ex-data error))) + (not (errors/fatal? error)))) + (errors/get-errors cause))] + (when-let [failure (errors/combine entries)] + (throw (errors/with-data failure (errors/project-context (ex-data cause))))))))) + +(defn- disconnect + "Disconnects only after the group has stopped. A late CDP attachment is also + closed by this continuation; the caller waits at most one termination grace." + [{:keys [connection]}] + (within (fn [] + (if-let [pending @connection] + (.then pending + (fn [^js browser] + (when (.isConnected browser) + (.close browser))) + (fn [_] nil)) + nil)) + {:deadline (process/deadline process/default-termination-grace-ms) + :phase :browser-disconnect})) + +(defn ^:async close-page + "Closes an owned page and its context within one termination grace. + Repeated or already-closed pages resolve without work." + [{:keys [page context]}] + (when (or page context) + (let [deadline (process/deadline process/default-termination-grace-ms) + page-failure + (attempt :catch-fatal + (when (and page (not (.isClosed ^js page))) + (await (within #(.close ^js page) + {:deadline deadline :phase :browser-page-close}))) + nil) + context-failure + (attempt :catch-fatal + (when context + (await (within #(.close ^js context) + {:deadline deadline :phase :browser-context-close}))) + nil)] + (when-let [cause (errors/combine + [{:phase "cleanup" :operation :close-page :error page-failure} + {:phase "cleanup" :operation :close-context :error context-failure}])] + (throw cause))))) + +(defn- ^:async shutdown-owner + "Disposes the owned process group, CDP connection and profile. + Successful process shutdown releases page/context resources. Other shutdown + failures retain the available page/context failure evidence." + [{:keys [user-data-dir] :as owner}] + (let [page-failure (attempt :catch-fatal (await (close-page owner)) nil) + process-failure (attempt :catch-fatal (await (stop-process owner)) nil) + disconnect-failure (attempt :catch-fatal (await (disconnect owner)) nil) + profile-failure (when-not (process/unresolved? process-failure) + (attempt :catch-fatal + (fs/rmSync user-data-dir #js {:recursive true :force true}) + nil))] + (when (or process-failure disconnect-failure profile-failure) + (throw (errors/with-data + (errors/combine + [{:phase "cleanup" :operation :close-page :error page-failure} + {:phase "cleanup" :operation :stop-process :profile-path user-data-dir :error process-failure} + {:phase "cleanup" :operation :disconnect-browser :error disconnect-failure} + {:phase "cleanup" :operation :remove-profile :path user-data-dir :error profile-failure}]) + {:profile-path user-data-dir}))))) + +(defn close + "Returns one shutdown Promise for this owner. Aborting the internal process + lifecycle invokes process/run's SIGTERM/5s/SIGKILL group cleanup, even during + startup or blocked page JS. A settled process exit or cancellation releases + the owned process resources." + [{:keys [shutdown signal abort-listener] :as owner}] + (or @shutdown + (do + (when signal + (.removeEventListener ^js signal "abort" @abort-listener)) + (let [closing (shutdown-owner owner)] + (reset! shutdown closing) + closing)))) + +(defn ^:async launch + "Spawns a Chromium instance and attaches via Chrome DevTools Protocol (CDP). + + Options include an optional signal, startup-timeout-ms (30000 by default) + and an optional absolute deadline shared with server startup. The result + owns the process group, user-data directory and CDP connection. It holds no + page or context; use new-page for those. + + Startup failures include available process details, endpoint and profile." + [{:keys [signal startup-timeout-ms deadline] + :or {startup-timeout-ms settings/default-startup-timeout-ms}}] + (let [deadline (or deadline (process/deadline startup-timeout-ms)) + budget (process/remaining deadline)] + (when (and signal (.-aborted ^js signal)) + (throw (or (.-reason ^js signal) (ex-info "browser startup aborted" {:type ::aborted})))) + (when (zero? budget) + (throw (ex-info "browser startup timed out" {:type ::timeout :phase :browser-startup}))) + (let [^js chromium (.-chromium playwright) + command (.executablePath chromium) + user-data-dir (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-browser-")) + controller (js/AbortController.) + endpoint (endpoint-waiter) + running (process/run {:command command + :args (into (:args (launch-settings)) + ["--remote-debugging-port=0" + "--remote-debugging-address=127.0.0.1" + (str "--user-data-dir=" user-data-dir) + "about:blank"]) + :deadline js/Number.MAX_SAFE_INTEGER + :signal (.-signal controller) + :capture-output? false + :on-output (:on-output endpoint)}) + owner {:controller controller :process running :shutdown (atom nil) + :connection (atom nil) :user-data-dir user-data-dir + :signal signal :abort-listener (atom nil)} + bounds {:deadline deadline :signal signal :phase :browser-startup} + abort (fn [] + ;; Startup/caller cleanup also awaits this same promise. + (.catch (close owner) (fn [_] nil)))] + (.then running + (fn [result] + ((:failed endpoint) (ex-info "Chromium exited during startup" + {:type ::process-exit :command command + :exit-code (:exit-code result)}))) + (:failed endpoint)) + (reset! (:abort-listener owner) abort) + (when signal + (.addEventListener ^js signal "abort" abort #js {:once true})) + (try + (let [address (await (within (fn [] (:promise endpoint)) bounds)) + ^js browser (await (within (fn [] + (let [connecting (.connectOverCDP chromium address + #js {:timeout (max 1 (process/remaining deadline))})] + (reset! (:connection owner) connecting) + connecting)) + bounds))] + (assoc owner :browser browser :version (.version browser) :endpoint address)) + (catch :default cause + (let [observed cause + cause (deadline-failure cause deadline) + cleanup-cause (attempt :catch-fatal (await (close owner)) nil) + cleanup (when (errors/failure? cleanup-cause) + (let [reported (loop [current observed seen #{}] + (if (or (nil? current) (contains? seen current)) + (into seen (map :error (errors/get-errors observed))) + (recur (ex-cause current) (conj seen current))))] + (errors/combine + (remove #(contains? reported (:error %)) + (errors/get-errors cleanup-cause)))))] + (if cleanup + (throw (errors/with-context + (errors/combine [{:phase "browser-startup" :operation :start-browser :error cause} + {:phase "cleanup" :operation :close-browser :error cleanup}]) + "browser shutdown failed after startup error" + (merge (errors/project-context (ex-data cause)) + (errors/project-context (ex-data cleanup-cause)) + {:type ::startup-cleanup + :endpoint @(:address endpoint) :user-data-dir user-data-dir + :startup-failure (errors/describe-cause cause)}))) + (throw (ex-info (errors/message-of cause) + (assoc (ex-data cause) :endpoint @(:address endpoint) :user-data-dir user-data-dir) + cause))))))))) + +(defn ^:async new-page + "Creates a context and page in a browser process we own, navigates to host, + waits for the bridge and checks its compiled build identity. + + Returns {:page :context}. + + Failures close the started page/context before rejecting." + [owner host {:keys [viewport signal startup-timeout-ms deadline] + :or {startup-timeout-ms settings/default-startup-timeout-ms}}] + (let [deadline (or deadline (process/deadline startup-timeout-ms)) + bounds {:deadline deadline :signal signal :phase :browser-page-startup} + ^js browser (:browser owner) + ^js context (await (within #(.newContext browser + #js {:viewport #js {:width (:width viewport) + :height (:height viewport)} + :deviceScaleFactor (:dpr viewport)}) + bounds))] + (try + (let [page (await (within #(.newPage context) bounds)) + page-own {:page page :context context}] + (try + (await (within #(.goto ^js page (str (:base-url host) "/") + #js {:timeout (max 1 (process/remaining deadline))}) + bounds)) + (await (within #(.waitForFunction ^js page "window.__benchReady === true" nil + #js {:timeout (max 1 (process/remaining deadline))}) + bounds)) + (identity/check (get-in host [:build :id]) + (transit/decode-str (await (within #(evaluate page "buildIdentity") bounds)))) + page-own + (catch :default cause + (let [cause (deadline-failure cause deadline)] + (ex/ignoring (await (close-page page-own))) + (throw cause))))) + (catch :default cause + (let [cause (deadline-failure cause deadline)] + (ex/ignoring (await (close-page {:context context}))) + (throw cause)))))) + +(defn connected? + "True while the owned CDP connection is live." + [{:keys [browser]}] + (boolean (and browser (.isConnected ^js browser)))) + +(defn prepare-scene + "Calls prepareScene and decodes its Transit response under bounds." + [page request bounds] + (.then (within #(evaluate page "prepareScene" request) bounds) + transit/decode-str)) + +(defn run-prepared + "Calls runPrepared and decodes its Transit response under bounds." + [page request bounds] + (.then (within #(evaluate page "runPrepared" request) bounds) + transit/decode-str)) + +(defn screenshot + "Writes one PNG under bounds. The timeout belongs to Playwright's call." + [page file {:keys [deadline] :as bounds}] + (.then (within #(.screenshot ^js page + #js {:path file + :timeout (max 1 (process/remaining deadline))}) + bounds) + (fn [_] nil))) + +(def service + "Interface to the browser used by the serial runner. + + Tests can replace this map with mocks." + {:launch launch + :new-page new-page + :close-page close-page + :close close + :connected? connected? + :prepare prepare-scene + :run run-prepared + :screenshot screenshot}) diff --git a/frontend/performance/benches/render_wasm/node/runner/build.cljs b/frontend/performance/benches/render_wasm/node/runner/build.cljs new file mode 100644 index 0000000000..ca7697354e --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/build.cljs @@ -0,0 +1,216 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.build + "These functions build the renderer and package it with the code that runs + benchmarks in Node and the browser. + + Each build gets its own directory under + frontend/target/performance/render-wasm/builds/. It contains the renderer + compiled to WebAssembly, a list of benchmarks that Node can load, and the + page and JavaScript that run them in a browser. + + Run `prepare` from Node in a checkout with the project's build tools and + dependencies installed. Stop other builds and any processes that rebuild the + renderer on file changes until it finishes." + (:require + ["fs" :as fs] + ["path" :as path] + ["url" :as url] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.features.selection :as selection] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [clojure.string :as str])) + +(defn ^:async git-provenance + "Read the Git commit ID and whether the checkout containing `root` has local + changes, including staged changes and untracked files. + + `bounds` supplies :deadline, an absolute time in milliseconds since the Unix + epoch, and an optional :signal AbortSignal to cancel the work. + + Return a promise of a map containing :sha, the commit ID, and :dirty, which + is true when the checkout has local changes. If Git information is + unavailable, log a warning and return nil for both values. Fatal process + failures, timeouts and cancellation reject the promise." + [root bounds] + (try + (let [options (merge bounds {:command "git" :cwd root}) + sha (await (process/run (assoc options :args ["rev-parse" "HEAD"]))) + status (await (process/run (assoc options :args ["status" "--porcelain"])))] + {:sha (str/trim (:stdout sha)) :dirty (not (str/blank? (:stdout status)))}) + (catch :default cause + (if (or (errors/fatal? cause) + (contains? #{:process-timeout :process-cancelled} (:type (ex-data cause)))) + (throw cause) + (do + (log/warn (str "Git provenance unavailable: " (errors/message-of cause))) + {:sha nil :dirty nil}))))) + +(defn- check-profile-marks + "Require :cljs-profile-marks to be false in the map describing the compiled + code. This flag indicates whether extra calls record timings for profiling; + those calls can affect benchmark measurements. Return the map or throw." + [compiled] + (when-not (false? (:cljs-profile-marks compiled)) + (throw (ex-info "compiled benchmark enables profiling marks or omits its setting" + {:type ::profile-marks :compiled compiled}))) + compiled) + +(defn- ^:async validate-browser + "Checks that the JavaScript compiled for the browser belongs to the requested + build and has profiling disabled. + + build: supplies the expected :id and :browser-dir, the directory containing + browser.js. + root: the frontend directory. + bounds: supplies :deadline, an absolute time in milliseconds since the Unix + epoch, and an optional :signal AbortSignal for cancellation. + + Returns a promise of the code's build information, including :build-id and + :cljs-profile-marks. Rejects the promise if loading or checking the code fails." + [build root bounds] + (let [module-url (.-href (url/pathToFileURL (path/join (:browser-dir build) "browser.js"))) + script "const {default: bridge} = await import(process.env.RENDER_BENCH_MODULE_URL); process.stdout.write(bridge.buildIdentity());" + result (await (process/run (merge bounds + {:command (.-execPath js/process) :cwd root + :args ["--input-type=module" "-e" script] + :env {"RENDER_BENCH_MODULE_URL" module-url}})))] + (check-profile-marks (identity/check (:id build) (transit/decode-str (:stdout result)))))) + +(defn ^:async prepare + "Build the renderer and the programs that run benchmarks, and save the files + for later benchmark runs. + + Pass an options map with :root set to the frontend directory. If omitted, + the function locates that directory from the Node program's location. + + Cargo features choose which optional parts of the renderer to compile. Use + :selection {:default-features? true :features []} to enable Cargo's defaults. + Put additional feature names from render-wasm/Cargo.toml in the vector, and + set :default-features? to false to disable the declared defaults. The + :features option accepts the same map when :selection is absent. Omit both + options to use Cargo's defaults. + + :build-timeout-ms sets the time allowed for building, in milliseconds + (300000 by default). An optional `:signal AbortSignal` cancels the work. + An optional :on-created function receives a map with the build's ID and + paths after its directory, log and initial metadata file exist. This callback + runs before compilation. Await the promise from `prepare` before using the + compiled files. + + The promise resolves to a map with :id, the build's unique identifier; + :dir, its output directory; :log-path, its compiler log; and :manifest, + metadata describing the build and its settings. :cases-path points to the + JavaScript module that lists the benchmarks for Node. :browser-dir and + :renderer-dir contain the compiled programs, and :index-path points to the + HTML page. + + A failure rejects the promise and keeps any files already written for + inspection. Error data includes :build-id and :log-path when available." + [{:keys [root features selection build-timeout-ms signal on-created] + :or {build-timeout-ms settings/default-build-timeout-ms}}] + (when-not (and (js/Number.isSafeInteger build-timeout-ms) (pos? build-timeout-ms)) + (throw (ex-info "build timeout must be a positive integer" {:type ::invalid-timeout}))) + (let [root (or root (artifacts/frontend-root)) + selection (selection/feature-selection (or selection features)) + deadline (process/deadline build-timeout-ms) + bounds {:deadline deadline :signal signal} + initial (artifacts/create-build root selection {:git {:sha nil :dirty nil} + :feature-selection selection}) + build-ref (atom initial) + phase (atom :storage)] + (binding [log/*log-path* (:log-path initial)] + (try + (when on-created (on-created initial)) + (reset! phase :provenance) + (let [git (await (git-provenance root bounds))] + (swap! build-ref #(artifacts/write-manifest % (assoc-in (:manifest %) [:provenance :git] git)))) + (reset! phase :renderer) + (log/info "Renderer compiling") + (let [build @build-ref + facts-path (path/join (:renderer-dir build) "cargo-build.json") + args (cond-> ["frontend"] + (not (:default-features? selection)) (conj "--no-default-features") + (seq (:features selection)) (into ["--features" (str/join "," (:features selection))]))] + (await (process/run (merge bounds + {:command (path/resolve root ".." "render-wasm" "build") + :args args :cwd (path/resolve root ".." "render-wasm") + :capture-output? false :on-line (fn [line _] (log/debug line)) + :env {"PENPOT_WASM_PREPARED" "1" "NODE_ENV" "production" + "BUILD_MODE" "release" + "VERSION_TAG" (:id build) + "FRONTEND_DEST" (:renderer-dir build) + "FRONTEND_SHARED_DEST" (path/join root "src" "app" "render_wasm" "api" "shared.js") + "RENDER_WASM_SHARED_ARTIFACT" (path/join (:renderer-dir build) "shared.js") + "RENDER_WASM_BUILD_INFO" facts-path}}))) + (let [facts (js/JSON.parse (fs/readFileSync facts-path "utf8")) + profile (js->clj (.-profile facts) :keywordize-keys true) + resolved (:features (selection/feature-selection {:features (js->clj (.-features facts)) + :default-features? (:default-features? selection)}))] + (when-not (and (= "3" (:opt_level profile)) + (false? (:debug_assertions profile))) + (throw (ex-info "Cargo did not produce an optimized renderer" + {:type ::renderer-profile :profile profile}))) + (swap! build-ref #(artifacts/write-manifest + % (cond-> (-> (:manifest %) + (assoc-in [:functional :features] resolved) + (assoc-in [:functional :default-features?] (boolean (some #{"default"} resolved))) + (assoc-in [:provenance :cargo-profile] profile)) + (js/Object.hasOwn facts "environment") + (assoc :environment (js->clj (.-environment facts)))))))) + (reset! phase :cljs) + (log/info "Modules compiling") + (let [build @build-ref] + (await (process/run (merge bounds + {:command "clojure" :cwd root + :args ["-M:dev:renderer-bench" "release" + "bench-render-wasm-catalog" "bench-render-wasm-browser"] + :capture-output? false :on-line (fn [line _] (log/debug line)) + :env {"RENDER_BENCH_BUILD_ID" (:id build) + "RENDER_BENCH_CASES_PATH" (:cases-path build) + "RENDER_BENCH_BROWSER_DIR" (:browser-dir build)}}))) + (fs/copyFileSync (path/join root "performance" "benches" "render_wasm" "browser" "index.html") + (:index-path build)) + (reset! phase :identity) + (log/info "Validating") + (check-profile-marks (catalog/build-identity build)) + (catalog/collect build {:master-seed 42}) + (await (validate-browser build root bounds)) + (artifacts/check-files build) + (when (zero? (process/remaining deadline)) + (throw (ex-info "preparation deadline expired" {:type :process-timeout}))) + (log/info " ready") + (let [ready (artifacts/write-manifest build (assoc (:manifest build) :status :ready))] + (reset! build-ref ready) + (artifacts/load-build root (:id ready)))) + (catch :default cause + (let [build @build-ref + context (errors/project-context (ex-data cause)) + failure (errors/evidence @phase cause) + saved (when (= :building (get-in build [:manifest :status])) + (attempt :catch-fatal + (artifacts/write-manifest + build (assoc (:manifest build) :status :failed :failure failure)))) + manifest-error (when (failure? saved) (errors/message-of saved)) + data (cond-> (merge context + {:type ::preparation-failed :build-id (:id build) + :phase @phase :log-path (or (:log-path (ex-data cause)) (:log-path build)) + :reason (str/join " -> " (keep :message (:cause failure)))}) + manifest-error (assoc :manifest-error manifest-error))] + (throw (if manifest-error + (errors/fatal-error + "renderer build preparation failed" data + (errors/combine [{:error cause} + {:phase "storage" :operation :write-manifest + :path (:manifest-path build) :error saved}])) + (ex-info "renderer build preparation failed" data cause))))))))) diff --git a/frontend/performance/benches/render_wasm/node/runner/feature_comparison.cljs b/frontend/performance/benches/render_wasm/node/runner/feature_comparison.cljs new file mode 100644 index 0000000000..4e04f071b4 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/feature_comparison.cljs @@ -0,0 +1,880 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.feature-comparison + "Plan and run benchmarks for combinations of optional Cargo renderer features. + + `prepare` reads Cargo metadata and collects the selected benchmark cases. + Call `format-plan` to inspect that data, or await `execute` to build and run + each configuration in sequence. Each configuration disables Cargo defaults. + An edge is a comparison between two configurations that differ by one + requested feature, from disabled to enabled. + + Execution saves progress, run records, logs and edge reports for inspection + after completion, failure or cancellation. Builds use the checkout's renderer + and benchmark sources. Keep other renderer builds and watchers stopped until + execution finishes." + (:require + ["fs" :as fs] + ["node:timers/promises" :as timers] + ["path" :as path] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.run :as run] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.node.time :as time] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.result :as result] + [clojure.set :as set] + [clojure.string :as str])) + +(def ^:private warnings + ["Timing values do not determine success." + "Intervals are exploratory; the comparison records no p-value verdicts."]) + +(def default-services + {:prepare-build build/prepare + :collect-catalog catalog/collect + :create-run-home runs/create-home + :execute-run run/execute}) + +(defn create-home + "Creates a directory and an empty log for a new feature comparison. + + Args: + root: The frontend directory. + comparison-id: Contains 1 to 128 ASCII letters, digits, underscores or + hyphens, starting with a letter or digit. + output-dir: With nil `output-dir`, uses + `target/performance/render-wasm/feature-comparisons/` under + root. An explicit path resolves from the Node process's working directory. + The chosen directory must not exist and must not be inside the managed + builds tree, including through symlink ancestors. + + Returns: + A map containing `:comparison-id`, `:dir`, `:log-path` and `:record-path`. + The log is `feature-comparison.log`; `:record-path` names the future + `feature-comparison.transit` file, which this function does not write. + + Throws: + Invalid identifiers, rejected paths and filesystem failures throw an error; + path and filesystem errors include `:path` when available." + [root comparison-id output-dir] + (artifacts/check-home-id comparison-id "invalid feature comparison id") + (let [default-dir (path/join (artifacts/managed-root root) + "feature-comparisons" comparison-id) + dir (artifacts/check-output-dir (or output-dir default-dir))] + (when (artifacts/inside-builds? root dir) + (artifacts/invalid-home "feature comparison cannot use a directory inside builds" {:path dir})) + (assoc (artifacts/create-home dir "feature-comparison" + "cannot create feature comparison home") + :comparison-id comparison-id))) + +(defn write-record + "Validates and saves a feature-comparison record. + + Encodes `record` as Transit after schema validation, then atomically replaces + that file so readers see either the previous complete record or the new one. + + Args: + home: Supplies `:record-path` in an existing directory. + record: The feature-comparison record. + + Returns: + The record. + + Throws: + Validation, write and rename failures throw without replacing the previous + record." + [home record] + (files/atomic-write (:record-path home) (result/encode record)) + record) + +(defn- permissions + [options] + {:allow-build-differences (boolean (:allow-build-differences options)) + :allow-scene-differences (boolean (:allow-scene-differences options)) + :allow-all-differences (boolean (:allow-all-differences options))}) + +(defn- storage-error + [cause log-path] + (errors/fatal-error "cannot persist feature comparison" + {:type ::storage-error :cause (errors/message-of cause) :log-path log-path} + cause)) + +(defn- change-config + [record slug f] + (update record :configurations + #(mapv (fn [config] + (if (= slug (:slug config)) + (f config) + config)) + %))) + +(defn- change-edge + [record from to f] + (update record :edges + #(mapv (fn [edge] + (if (and (= from (:from edge)) (= to (:to edge))) + (f edge) + edge)) + %))) + +(defn- config-by-slug + [record slug] + (some #(when (= slug (:slug %)) %) (:configurations record))) + +(defn- persist + "Save updated progress before exposing it as the current and last saved record." + [state update] + (let [next (assoc (update @(:record-ref state)) :updated-at (time/now)) + saved (try + (write-record (:home state) next) + (catch :default cause + (throw (storage-error cause (get-in state [:home :log-path])))))] + (reset! (:record-ref state) saved) + (reset! (:saved-ref state) saved) + saved)) + +(defn- initial-record + [prepared comparison-id started-at] + (let [plan (:plan prepared) + options (:options prepared)] + {:type :feature-comparison + :comparison-id comparison-id + :started-at started-at + :updated-at started-at + :status :running + :options options + :permissions (permissions options) + :cases (:cases prepared) + :features (:features plan) + :configurations (mapv #(assoc % :status :planned) (:configurations plan)) + :edges (mapv #(assoc % :status :planned) (:edges plan)) + :warnings warnings})) + +(defn- fail-config + [state slug phase cause] + (persist state + (fn [record] + (change-config record slug + #(-> % (assoc :status :failed + :failure (errors/evidence phase cause)) + (dissoc :reason)))))) + +(defn- interrupt-config + [state slug signal failure] + (persist state + (fn [record] + (change-config record slug + #(cond-> (assoc % :status :failed :reason signal) + (some? failure) (assoc :failure failure)))))) + +(defn- mark-configs-unattempted + [state configurations reason] + (when-let [[configuration & pending] (seq configurations)] + (let [slug (:slug configuration)] + (when (= :planned (:status (config-by-slug @(:record-ref state) slug))) + (persist state #(change-config % slug (fn [entry] + (assoc entry :status :unattempted :reason reason))))) + (mark-configs-unattempted state pending reason)))) + +(defn- mark-edge + [state edge status data] + (persist state + (fn [record] + (change-edge record (:from edge) (:to edge) + #(merge % (assoc data :status status)))))) + +(defn- mark-edges-unattempted + ([state reason] + (mark-edges-unattempted state (:edges @(:record-ref state)) reason)) + ([state edges reason] + (when-let [[edge & pending] (seq edges)] + (when (= :planned (:status edge)) + (mark-edge state edge :unattempted {:reason reason})) + (mark-edges-unattempted state pending reason)))) + +(defn- storage-failure + "Combines earlier failures with a failed comparison-storage operation. + + Args: + operation: Operation that failed while saving comparison evidence. + causes: Earlier failures in occurrence order. + cause: Failure of the storage operation. + + Returns: + An Error retaining earlier failures followed by the storage failure." + [operation causes cause] + (errors/combine (conj (mapv #(hash-map :error %) causes) + {:phase "comparison-storage" :operation operation :error cause}))) + +(defn- finish + [state status exit-code causes] + (let [saved (attempt :catch-fatal (persist state #(assoc % :status status :ended-at (time/now)))) + failure (when (failure? saved) + (storage-failure :finish-comparison causes saved))] + {:comparison (if failure @(:saved-ref state) saved) + :home (:home state) + :log-path (get-in state [:home :log-path]) + :errors (if failure [failure] (vec (remove nil? causes))) + :exit-code (if (and failure (not (contains? #{130 143} exit-code))) 1 exit-code)})) + +(defn- finish-root + [state error] + (let [record @(:record-ref state) + completed (and (every? #(= :completed (:status %)) (:configurations record)) + (every? #(= :completed (:status %)) (:edges record)))] + (if completed + (finish state :completed 0 [error]) + (finish state :failed 1 [error])))) + +(defn- failure-outcome + [state signal cause] + (assoc (outcome/failed cause {:comparison @(:saved-ref state) + :home (:home state) + :log-path (get-in state [:home :log-path])}) + :exit-code (if signal (outcome/interruption-code signal) 1))) + +(defn- interrupt-comparison + [state signal error] + (let [reason (str "interrupted by " signal) + recorded (attempt :catch-fatal + (mark-configs-unattempted state (:configurations @(:record-ref state)) reason) + (mark-edges-unattempted state reason) + nil)] + (if (failure? recorded) + (failure-outcome state signal + (storage-failure :record-interruption [error] recorded)) + (finish state :interrupted (outcome/interruption-code signal) [error])))) + +(defn- record-created-build + [state slug build] + (persist state + #(change-config % slug + (fn [configuration] + (-> configuration + (assoc :status :building + :build (select-keys build [:id :manifest-path :log-path])) + (dissoc :failure :reason)))))) + +(defn- stop-outcome + [slug reason cause] + {:stop? true + :reason (str "not attempted after " reason " in configuration " slug) + :error cause}) + +(defn- fail-configuration + [state slug phase reason cause] + (fail-config state slug phase cause) + (stop-outcome slug reason cause)) + +(defn- interrupt-configuration + [state slug signal phase cause] + (interrupt-config state slug signal (when cause (errors/evidence phase cause))) + (cond-> {:interrupted? true :signal signal} + cause (assoc :error cause))) + +(defn- fail-or-interrupt + [state slug phase cause] + (try + (if-let [signal (process/signal-name (:signal state))] + (interrupt-configuration state slug signal phase cause) + (fail-configuration state slug phase (str (name phase) " failure") cause)) + (catch :default recorded + (throw (storage-failure :record-configuration-failure [cause] recorded))))) + +(defn- begin-configuration + [state configuration] + (let [slug (:slug configuration)] + (if-let [signal (process/signal-name (:signal state))] + {:configuration configuration + :outcome (interrupt-configuration state slug signal nil nil)} + (do + (persist state #(change-config % slug (fn [entry] + (-> entry (assoc :status :building) + (dissoc :failure :reason))))) + {:configuration configuration})))) + +(defn- accept-build + [state flow selected] + (let [{:keys [slug selection]} (:configuration flow) + effective (get-in selected [:manifest :functional :features]) + saved-selection (get-in selected [:manifest :provenance :feature-selection]) + valid-features? (and (vector? effective) (every? string? effective))] + (if (and (= selection saved-selection) valid-features?) + (do + (persist state #(change-config % slug (fn [entry] + (assoc entry :status :ready :effective-features effective)))) + (log/info (str "Configuration " slug " built")) + (assoc flow :build selected)) + (let [cause (ex-info "renderer build manifest disagrees with the planned selection" + {:type ::invalid-build :log-path (:log-path selected) + :selection selection :saved-selection saved-selection})] + (assoc flow :outcome (fail-configuration state slug :build "build failure" cause)))))) + +(defn- ^:async build-configuration + [state flow] + (if (:outcome flow) + flow + (let [{:keys [slug selection]} (:configuration flow) + built (attempt :catch-fatal + (await ((:prepare-build (:services state)) + {:root (:root state) + :selection selection + :build-timeout-ms (get-in state [:options :build-timeout-ms]) + :signal (:signal state) + :on-created (partial record-created-build state slug)}))) + signal (process/signal-name (:signal state))] + (cond + (and (failure? built) (errors/fatal? built) (not (process/unresolved? built))) + (throw built) + + (failure? built) + (assoc flow :outcome (fail-or-interrupt state slug :build built)) + + signal + (assoc flow :outcome (interrupt-configuration state slug signal nil nil)) + + :else (accept-build state flow built))))) + +(defn- check-catalog + [state flow] + (if (:outcome flow) + flow + (let [slug (get-in flow [:configuration :slug]) + selected (:build flow) + mismatch? (-> (attempt + (not= ((:collect-catalog (:services state)) + selected + {:master-seed (get-in state [:options :seed]) + :filter (get-in state [:options :filter])}) + (:cases state))) + (errors/with-data {:log-path (:log-path selected)})) + signal (process/signal-name (:signal state)) + step (cond + (failure? mismatch?) (fail-or-interrupt state slug :catalog mismatch?) + signal (interrupt-configuration state slug signal nil nil) + mismatch? + (fail-configuration state slug :catalog "catalog mismatch" + (ex-info "build catalog disagrees with the prepared cases" + {:type ::catalog-mismatch + :log-path (:log-path selected) + :configuration slug})))] + (cond-> flow step (assoc :outcome step))))) + +(defn- allocate-run + [state flow] + (if (:outcome flow) + flow + (let [slug (get-in flow [:configuration :slug]) + home (attempt + ((:create-run-home (:services state)) + {:root (:root state) :run-id (str (random-uuid)) + :output-dir (path/join (get-in state [:home :dir]) "configs" slug)}))] + (if (failure? home) + (assoc flow :outcome (fail-configuration state slug :run-home "run-home failure" home)) + (assoc flow :run-home home))))) + +(defn- ^:async execute-run + [state flow] + (attempt + (await ((:execute-run (:services state)) + (:root state) (:run-home flow) (:build flow) (:cases state) + (run/get-execution-options (:options state)) + (:signal state))))) + +(defn- get-evidence-error + [slug home response] + (or (some #(when (errors/fatal? %) %) (:errors response)) + (attempt + (when-not (= (get-in response [:run :run-id]) (:run-id (runs/get-record (:record-path home)))) + (throw (ex-info "saved run identity disagrees with execution result" + {:type ::invalid-run :configuration slug})))))) + +(defn- get-run-evidence + [slug home response] + (let [run (:run response) + home (or (:home response) home) + evidence-error (get-evidence-error slug home response) + error (errors/with-data (or evidence-error (first (:errors response))) + {:log-path (:log-path home)})] + {:run run + :home home + :evidence-error evidence-error + :error error + :gpu-failure (some #(when (= :gpu (:stage %)) %) (get-in run [:metadata :diagnostics :failures])) + :run-ref {:run-id (:run-id run) :record-path (:record-path home) + :log-path (:log-path home) :exit-code (:exit-code response)}})) + +(defn- get-run-failure + [{:keys [run evidence-error gpu-failure error]}] + (cond + evidence-error (errors/evidence :run-evidence evidence-error) + gpu-failure gpu-failure + (map? (get-in run [:termination :failure])) (get-in run [:termination :failure]) + error (errors/evidence :run error))) + +(defn- classify-run + [signal evidence] + (let [termination (get-in evidence [:run :termination :reason]) + completed (and (nil? (:evidence-error evidence)) (= :completed termination) + (zero? (get-in evidence [:run-ref :exit-code]))) + interrupted (or signal (when (= :interrupted termination) + (or (get-in evidence [:run :termination :signal]) "SIGINT"))) + failure (get-run-failure evidence) + reason (or interrupted (when (and (not completed) (nil? failure)) "run did not complete"))] + (assoc evidence :completed completed :signal interrupted :failure failure :reason reason))) + +(defn- record-run-result + [state slug {:keys [completed run-ref reason failure]}] + (persist state + #(change-config % slug + (fn [entry] + (if completed + (-> entry (assoc :status :completed :run run-ref) (dissoc :failure :reason)) + (cond-> (assoc entry :status :failed :run run-ref) + (some? reason) (assoc :reason reason) + (some? failure) (assoc :failure failure))))))) + +(defn- zero-attempt-outcome + [slug home] + (stop-outcome slug "a run with no attempts" + (ex-info "configuration run made no attempts" + {:type ::zero-attempts :log-path (:log-path home) :configuration slug}))) + +(defn- config-run-outcome + [slug {:keys [run home completed signal reason error evidence-error gpu-failure]}] + (cond + completed + (if (empty? (:attempts run)) + (zero-attempt-outcome slug home) + (do (log/info (str "Configuration " slug " completed")) {:stop? false})) + + signal {:interrupted? true :signal reason :error error} + evidence-error (stop-outcome slug "run evidence failure" error) + gpu-failure (stop-outcome slug "GPU failure" + (ex-info (:message gpu-failure) + {:type ::gpu-required :log-path (:log-path home) :configuration slug})) + (empty? (:attempts run)) (zero-attempt-outcome slug home) + :else (do (log/warn (str "Configuration " slug " failed")) {:stop? false :error error}))) + +(defn- finish-config-run + [state flow response] + (let [slug (get-in flow [:configuration :slug]) + facts (get-run-evidence slug (:run-home flow) response) + evidence (classify-run (process/signal-name (:signal state)) facts)] + (try + (record-run-result state slug evidence) + (catch :default cause + (throw (storage-failure :record-run-result [(:error evidence)] cause)))) + (config-run-outcome slug evidence))) + +(defn- get-missing-run-cause + "Returns why execution produced no run record, or nil when it produced one." + [response home] + (-> (cond + (failure? response) response + (nil? (:run response)) (or (first (:errors response)) + (ex-info "run executor returned no record" {:type ::missing-run}))) + (errors/with-data {:log-path (:log-path home)}))) + +(defn- ^:async run-configuration + [state flow] + (if (:outcome flow) + flow + (let [slug (get-in flow [:configuration :slug]) + home (:run-home flow)] + (persist state #(change-config % slug (fn [entry] + (assoc entry :status :running + :run {:run-id (:run-id home) + :record-path (:record-path home) + :log-path (:log-path home)})))) + (log/info (str "Configuration " slug " running")) + (let [response (await (execute-run state flow)) + cause (get-missing-run-cause response home)] + (assoc flow :outcome (if cause + (fail-configuration state slug :run "run failure" cause) + (finish-config-run state flow response))))))) + +(defn- ^:async execute-config + [state configuration] + (let [started (begin-configuration state configuration) + built (await (build-configuration state started)) + catalogued (check-catalog state built) + allocated (allocate-run state catalogued) + executed (await (run-configuration state allocated))] + (:outcome executed))) + +(defn- sorted-features + [features] + (when-not (and (vector? features) (every? string? features)) + (throw (ex-info "run metadata lacks an effective feature vector" + {:type ::missing-features}))) + (vec (sort (distinct features)))) + +(defn- edge-facts + [baseline candidate edge] + (let [baseline-features (sorted-features + (get-in baseline [:metadata :build :functional :features])) + candidate-features (sorted-features + (get-in candidate [:metadata :build :functional :features])) + baseline-set (set baseline-features) + candidate-set (set candidate-features)] + {:from (:from edge) + :to (:to edge) + :feature (:feature edge) + :baseline-features baseline-features + :candidate-features candidate-features + :added (vec (sort (set/difference candidate-set baseline-set))) + :removed (vec (sort (set/difference baseline-set candidate-set)))})) + +(defn- save-comparison + [path comparison log-path] + (let [text (result/encode comparison)] + (try + (files/atomic-write path text) + (catch :default cause + (throw (storage-error cause log-path)))))) + +(defn- unacknowledged-count + [comparison] + (reduce + 0 (map (fn [pair] (count (remove :waived? (:mismatches pair)))) + (:pairs comparison)))) + +(defn- get-comparison-options + [record facts] + (assoc (:permissions record) + :expected-feature-change (select-keys facts [:feature :baseline-features + :candidate-features :added :removed]) + :edge facts + :on-unallowed-difference :record)) + +(defn- analyze-edge + "Returns the path of the saved comparison and its count of unacknowledged + differences, or the failure, fatal failures included." + [state edge] + (attempt + :catch-fatal + (let [record @(:record-ref state) + baseline-config (config-by-slug record (:from edge)) + candidate-config (config-by-slug record (:to edge)) + baseline (runs/get-record (get-in baseline-config [:run :record-path])) + candidate (runs/get-record (get-in candidate-config [:run :record-path])) + facts (edge-facts baseline candidate edge) + comparison (compare/compare-runs baseline candidate (get-comparison-options record facts)) + path (path/join (get-in state [:home :dir]) + "comparisons" + (str (:from edge) "__" (:to edge) ".transit"))] + (save-comparison path comparison (get-in state [:home :log-path])) + {:path path + :unacknowledged (unacknowledged-count comparison)}))) + +(defn- record-edge-outcome + "Records the result of analyzing an edge. + + Returns: + An Error retaining accumulated analysis failures, or nil on success. + + Throws: + Fatal analysis or persistence failures with available earlier failures." + [state edge result error] + (let [fatal? (and (failure? result) (errors/fatal? result)) + analyzed (if (failure? result) + (errors/combine [{:error error} + {:phase "comparison" :operation :analyze-edge :error result}]) + error) + recorded (attempt + :catch-fatal + (cond + fatal? + (do + (mark-edge state edge :failed {:failure (errors/evidence :comparison-storage result)}) + (mark-edges-unattempted state "not attempted after comparison storage failure")) + + (failure? result) + (mark-edge state edge :failed {:failure (errors/evidence :comparison result)}) + + (pos? (:unacknowledged result)) + (do + (log/warn (str "Comparison " (:from edge) " -> " (:to edge) + " has " (:unacknowledged result) " unacknowledged differences")) + (mark-edge state edge :failed {:path (:path result) + :failure {:unacknowledged-differences (:unacknowledged result)}})) + + :else + (mark-edge state edge :completed {:path (:path result)})) + nil)] + (cond + (failure? recorded) + (throw (storage-failure :record-edge [analyzed] recorded)) + + fatal? (throw analyzed) + :else analyzed))) + +(defn- ^:async analyze-edges + [state edges error] + (await (timers/setImmediate)) + (if-let [signal (process/signal-name (:signal state))] + (do + (try + (mark-edges-unattempted state (str "interrupted by " signal)) + (catch :default cause + (throw (storage-failure :record-interruption [error] cause)))) + {:interrupted signal :error error}) + (if-let [[edge & pending] (seq edges)] + (analyze-edges state pending (record-edge-outcome state edge (analyze-edge state edge) error)) + {:done true :error error}))) + +(defn- create-comparison-directory + [state] + (try + (fs/mkdirSync (path/join (get-in state [:home :dir]) "comparisons") #js {:recursive true}) + (catch :default cause + (throw (storage-error cause (get-in state [:home :log-path])))))) + +(defn- stop-unresolved-comparison + [state error] + (let [recorded (attempt :catch-fatal + (mark-edges-unattempted state "process group may still be running") + nil)] + (if (failure? recorded) + (failure-outcome state (process/signal-name (:signal state)) + (storage-failure :skip-comparisons [error] recorded)) + (finish state :failed 1 [error])))) + +(defn- ^:async run-edges + [state error] + (let [signal (process/signal-name (:signal state))] + (cond + signal (interrupt-comparison state signal error) + (process/unresolved? error) (stop-unresolved-comparison state error) + :else + (do + (try + (create-comparison-directory state) + (catch :default cause + (throw (storage-failure :create-comparison-directory [error] cause)))) + (let [analyzed (attempt :catch-fatal + (await (analyze-edges state (:edges @(:record-ref state)) error)))] + (cond + (failure? analyzed) + (if-let [signal (process/signal-name (:signal state))] + (failure-outcome state signal analyzed) + (finish state :failed 1 [analyzed])) + (:interrupted analyzed) (interrupt-comparison state (:interrupted analyzed) (:error analyzed)) + :else (finish-root state (:error analyzed)))))))) + +(defn- ^:async run-comparisons + [state configurations] + (if-let [signal (process/signal-name (:signal state))] + (interrupt-comparison state signal nil) + (if-let [[configuration & pending] (seq configurations)] + (let [step (await (execute-config state configuration))] + (cond + (:interrupted? step) + (interrupt-comparison state (:signal step) (:error step)) + + (:stop? step) + (do + (try + (mark-configs-unattempted state pending (:reason step)) + (catch :default cause + (throw (storage-failure :stop-configurations [(:error step)] cause)))) + (run-edges state (:error step))) + + :else (run-comparisons state pending))) + (run-edges state nil)))) + +(defn- create-comparison-state + [root prepared signal services home comparison-id] + {:root root + :home home + :options (:options prepared) + :cases (:cases prepared) + :signal signal + :services services + :record-ref (atom (initial-record prepared comparison-id (time/now))) + :saved-ref (atom nil)}) + +(defn- initialize-comparison + [state] + (let [initial @(:record-ref state) + home (:home state) + saved (write-record home initial)] + (reset! (:record-ref state) saved) + (reset! (:saved-ref state) saved))) + +(defn- ^:async execute-comparison + [state] + (let [home (:home state)] + (binding [log/*log-path* (:log-path home)] + (let [result (attempt :catch-fatal + (initialize-comparison state) + (await (run-comparisons state (:configurations @(:record-ref state)))))] + (if (failure? result) + (failure-outcome state (process/signal-name (:signal state)) result) + result))))) + +(defn ^:async execute + "Build and benchmark every prepared feature configuration, then compare runs. + + The caller must handle `:dry-run` by calling `format-plan` instead; this function + always executes. + + Create a fresh comparison directory and save its initial record before the + first build. Build and run configurations serially, checking each compiled + catalog against the prepared cases. Runs live in `configs/` under the + comparison directory; edge reports live in `comparisons/__.transit`. + Build artifacts remain in the managed builds directory. + + Timing differences alone do not cause failure. Unacknowledged differences + in build, scene or other comparison metadata do cause an edge to fail. + The prepared options `:allow-build-differences` and `:allow-scene-differences` + acknowledge their respective categories; `:allow-all-differences` acknowledges + every compatibility mismatch. Reports retain acknowledged differences. + + Build or catalog failures stop later configurations. A failed run may allow + later configurations when it retains attempts and usable evidence; missing + evidence, GPU failure or zero attempts stop them. Cancellation skips remaining + work. Files already written remain for diagnosis. + + Args: + root: The frontend directory. + prepared: The result of `prepare`. + signal: An optional AbortSignal that cancels execution. + settings: May contain `:services` overrides with the same contracts as the + defaults: `:prepare-build` uses `node.runner.build/prepare`, `:collect-catalog` + uses `node.catalog.loader/collect`, `:create-run-home` uses `node.storage.runs/create-home`, + and `:execute-run` uses `node.runner.run/execute`. Omitted services use + `default-services`. + + Returns: + Await the returned promise for a map with `:comparison`, the last + successfully saved progress record; `:home`, the comparison ID and paths; + `:log-path`; `:errors`, retained error objects; and `:exit-code`. The exit + code is 0 only when all configurations and edges complete, 1 on failure, + or 130/143 on interruption. + + If storage fails, the returned `:comparison` reflects the last successful + atomic write, or nil if no record was saved; `:home` is nil if directory + creation failed. Inspect the saved statuses and `:errors` together, since + `:errors` is not an exhaustive list of failed configurations or edges." + [root prepared signal settings] + (let [services (merge default-services (:services settings)) + comparison-id (str (random-uuid)) + home (attempt (create-home root comparison-id (:output-dir prepared)))] + (if (failure? home) + (outcome/failed home {:comparison nil :home nil}) + (await + (execute-comparison + (create-comparison-state root prepared signal services home comparison-id)))))) + + + +(defn- get-root-features + "Read feature definitions for cwd/Cargo.toml from Cargo metadata JSON." + [cwd text] + (let [metadata (js->clj (js/JSON.parse text)) + manifest (fs/realpathSync (path/join cwd "Cargo.toml")) + package (first (filter #(= manifest (fs/realpathSync + (path/resolve cwd (get % "manifest_path")))) + (get metadata "packages")))] + (when-not package + (throw (ex-info "Cargo metadata does not contain the renderer manifest" + {:type ::missing-package :manifest-path manifest}))) + (get package "features"))) + +(defn ^:async prepare + "Collect and validate a feature comparison plan without building or running it. + + Run `cargo metadata --offline --locked --no-deps` in `render-wasm` with the + inherited environment. Cargo and its offline metadata must be available. + `:build-timeout-ms` bounds that subprocess in milliseconds, not the entire + planning operation. + Require empty declared Cargo defaults and known requested feature names; + Cargo remains responsible for resolving feature activation during builds. + + Planning creates no comparison output and starts no build, server or browser, + including for a dry-run. + + Args: + root: The frontend directory, with render-wasm as its sibling. + opts: Resolved CLI options, including defaults. `:features` supplies the + requested Cargo feature names, `:seed` supplies the master scene seed, + and `:filter` selects case IDs by substring. `:max-features` limits the + requested feature count. Execution requires `:confirm` for more than + eight configurations or a blank filter; `:dry-run` bypasses that + confirmation requirement only. + + An optional `:output-dir` resolves from the Node process's working + directory and must not exist. Check it without creating or reserving it. + `execute` checks it again and also rejects paths inside the managed + builds tree. + signal: An optional AbortSignal that cancels the process. + + Returns: + Await the promise for a map with `:plan`, `:cases`, `:output-dir` and + `:options`. The `:plan` map contains `:features` in sorted order, + `:configurations` with a `:slug` and Cargo `:selection`, and `:edges` with + `:from` and `:to` slugs and a `:feature`. Slugs are bit strings in sorted + feature order: `0` means disabled, `1` enabled. The `:cases` vector holds + case descriptions collected once. `:output-dir` is the absolute explicit + path or nil. `:options` contains the supplied options with normalized + features and output path. Pass this map to `format-plan` or `execute`. + + Throws: + Invalid options or paths, Cargo failures or cancellation, unsupported + manifest features, and case collection failures reject the promise. + An empty case selection is an error." + [root opts signal] + (let [plan (feature-plan/plan (:features opts) opts) + output-dir (artifacts/check-output-dir (:output-dir opts)) + cwd (path/resolve root ".." "render-wasm") + metadata (await (process/run + {:command "cargo" + :args ["metadata" "--offline" "--locked" "--no-deps" "--format-version" "1"] + :cwd cwd + :deadline (process/deadline (:build-timeout-ms opts settings/default-build-timeout-ms)) + :signal signal}))] + (feature-plan/check-manifest (:features plan) (get-root-features cwd (:stdout metadata))) + {:plan plan + :cases (cases/collect-cases {:master-seed (:seed opts) :filter (:filter opts)}) + :output-dir output-dir + :options (assoc opts :features (:features plan) :output-dir output-dir)})) + +(defn format-plan + "Return a text summary of the map produced by `prepare`. + + This function neither collects cases again nor reads the filesystem, + starts processes or executes benchmarks. + + Returns: + Feature names, configurations, comparison edges, case count, sampling + counts and configured timeouts." + [{:keys [plan cases options]}] + (str "Feature comparison plan\n" + "Features: " (str/join ", " (:features plan)) "\n" + "Configurations (Cargo defaults disabled):\n" + (str/join "\n" (map #(str " " (:slug %) ": " + (str/join ", " (get-in % [:selection :features]))) + (:configurations plan))) "\n" + "Edges:\n" + (str/join "\n" (map #(str " " (:from %) " -> " (:to %) ": " (:feature %)) + (:edges plan))) "\n" + "Cases: " (count cases) "\n" + "Attempts per case: " (:warmups options) " warmups, " (:repetitions options) " measured\n" + "Timeouts (ms): case " (:timeout-ms options) + ", build " (:build-timeout-ms options) ", startup " (:startup-timeout-ms options) "\n")) diff --git a/frontend/performance/benches/render_wasm/node/runner/run.cljs b/frontend/performance/benches/render_wasm/node/runner/run.cljs new file mode 100644 index 0000000000..47bcc0fc0b --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/run.cljs @@ -0,0 +1,916 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.run + "Renderer benchmark runs over fixed builds. + + Runs are stored in a `home`, and execute via a loopback host and at most one + Chromium process. + + `:fresh` cases prepare once per slot on a new page and context. + `:reuse` cases prepare once and run every slot on the same page and context. + + A failed attempt consumes its slot. + + Browser failures leave the current case's remaining slots unattempted and + continue with the next case in a new browser." + (:require-macros + [app.common.exceptions :as ex]) + (:require + ["module" :as node-module] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.declarations :as declarations] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.measurement :as measurement] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.browser :as browser] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.server :as server] + [benches.render-wasm.node.settings :as settings] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.node.time :as time] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [clojure.string :as str])) + +(defn get-execution-options + "Returns resolved settings for executing a prepared run." + [options] + (select-keys options [:seed :filter :warmups :repetitions :timeout-ms + :startup-timeout-ms :screenshots :require-gpu :remove])) + +(defn- cause-evidence + "Plain failure evidence for a rejected browser, server or persistence action." + [cause fallback-phase] + (cond-> (errors/fail-data + (errors/evidence (or (:phase (ex-data cause)) fallback-phase) cause)) + (contains? (ex-data cause) ::browser/deadline-exceeded) + (assoc :deadline-exceeded (::browser/deadline-exceeded (ex-data cause))))) + +(defn- interruption + [signal] + (when-let [name (process/signal-name signal)] + {:signal name :cause (.-reason ^js signal)})) + +(defn- interrupted? + [state] + (some? (interruption (:signal state)))) + +(defn- check-deadline + [deadline] + (when (zero? (process/remaining deadline)) + (throw (ex-info "case deadline exceeded" {:type ::browser/timeout :phase :deadline})))) + +(defn- failure-outcome + [state deadline evidence] + (cond + (interrupted? state) :interrupted + (if (contains? evidence :deadline-exceeded) + (:deadline-exceeded evidence) + (or (zero? (process/remaining deadline)) (= "deadline" (:phase evidence)))) :timed-out + :else :failed)) + +(defn- keep-sink-failure + "Remembers the first failed write to the run record or log. The failure stays + in the outcome even when a later write succeeds." + [state cause] + (swap! (:sink-failure-ref state) #(or % cause)) + cause) + +(defn- write-log + [state output text] + (try + (output text) + (catch :default cause + (throw (keep-sink-failure + state + (if (errors/fatal? cause) + cause + (errors/fatal-error "cannot write the run log" {} cause))))))) + +(defn- warn-safely + "Appends runner diagnostics without replacing an error already under handling." + [state text] + (ex/ignoring (write-log state log/warn text)) + nil) + +(defn- persist + [state] + (try + (runs/write-record! (:home state) @(:run-ref state)) + (catch :default cause + (throw (keep-sink-failure + state + (errors/fatal-error "cannot save the run record" {:log-path (get-in state [:home :log-path])} cause)))))) + +(defn- record-preparation + [state preparation] + (swap! (:run-ref state) result/record-preparation preparation) + (persist state) + (write-log state log/debug (str "Prepared " (result/case-name (:case-id preparation))))) + +(defn- record-attempt + [state measured] + (swap! (:run-ref state) result/record-attempt measured) + (persist state) + (let [remaining (get-in @(:run-ref state) [:remaining (:case-id measured)]) + total (+ (:warmups (:opts state)) (:repetitions (:opts state))) + index (- total (:warmups remaining) (:repetitions remaining))] + (write-log state (if (= :failed (:outcome measured)) log/warn log/info) + (str " attempt " index "/" total " " (name (:outcome measured)) + (when (:warmup? measured) " (warmup)"))))) + +(defn- record-failure + [state case-id stage evidence] + (let [failure (cond-> (assoc (errors/project-diagnostic evidence) + :stage stage) + (some? case-id) (assoc :case-id case-id))] + (reset! (:harness-failed? state) true) + (swap! (:run-ref state) update-in [:metadata :diagnostics :failures] + (fnil conj []) failure) + (persist state) + (write-log state log/warn (str (name stage) " failed" + (when (some? case-id) (str " for " (result/case-name case-id))) + ": " (:message failure) "\n")) + failure)) + +(defn- record-screenshot-status + [state case-id status] + (swap! (:run-ref state) assoc-in + [:metadata :diagnostics :screenshots (result/case-name case-id)] status) + (persist state)) + +(defn- screenshot-status + [state case-id] + (get-in @(:run-ref state) + [:metadata :diagnostics :screenshots (result/case-name case-id)])) + +(defn- record-screenshot-skip + [state case-id reason] + (when (:screenshots (:opts state)) + (when-not (screenshot-status state case-id) + (record-screenshot-status state case-id {:status :skipped :reason reason})))) + +(defn- ensure-screenshot-status + [state case reason] + (when (:screenshots (:opts state)) + (when-not (screenshot-status state (:id case)) + (record-screenshot-skip state (:id case) reason)))) + +(defn- ^:async screenshot-case + [state case page deadline] + (if (:screenshots (:opts state)) + (let [{:keys [file path]} (runs/screenshot-path (:home state) (:id case)) + bounds {:deadline deadline + :signal (:signal state) :phase :screenshot} + shot (attempt (await ((get-in state [:services :screenshot]) page path bounds)))] + (if (failure? shot) + (do + (record-screenshot-status state (:id case) + {:file file :status :failed + :message (errors/message-of shot) + :cause (errors/describe-cause shot)}) + (write-log state log/warn (str "Screenshot failed for " + (result/case-name (:id case)) ": " + (errors/message-of shot) "\n")) + (let [outcome (failure-outcome state deadline (cause-evidence shot :screenshot))] + (when (= :timed-out outcome) + (record-failure state (:id case) :case-deadline (cause-evidence shot :screenshot))) + (if (= :failed outcome) :ok outcome))) + (do + (record-screenshot-status state (:id case) {:file file :status :ok}) + (write-log state log/debug (str "Screenshot " file)) + :ok))) + :ok)) + +(defn- case-slots + "Returns each scheduled slot in order: all warmups, then measured slots. + True marks a warmup." + [opts] + (vec (concat (repeat (:warmups opts) true) + (repeat (:repetitions opts) false)))) + +(defn- renderer-url + [build-id file] + (str "/renderer/" file "?build=" (js/encodeURIComponent build-id))) + +(defn- ^:async prepare-case + [state case page deadline warmup?] + (check-deadline deadline) + (let [build-id (:id (:selected-build state)) + preparation-id (str (random-uuid)) + response (attempt + (measurement/check-response + :preparation preparation-id + (await ((get-in state [:services :prepare]) + page + (transit/encode-str + {:case case + :preparation-id preparation-id + :timeout-ms (max 1 (process/remaining deadline)) + :module-url (renderer-url build-id "render-wasm.js") + :wasm-url (renderer-url build-id "render-wasm.wasm")}) + {:deadline deadline :signal (:signal state) :phase :preparation})))) + accepted (when (and (not (failure? response)) (= "ok" (:status response))) + (attempt + (let [preparation (assoc (:preparation response) :warmup? warmup?)] + (when-not (and (= preparation-id (:preparation-id preparation)) + (= (:id case) (:case-id preparation)) + (not (:failure preparation))) + (throw (ex-info "preparation success response is inconsistent" {}))) + (record-preparation state preparation) + preparation)))] + (if (and accepted (not (failure? accepted))) + {:preparation accepted} + (let [failure (cond + (failure? accepted) (assoc (cause-evidence accepted :invalid-output) + :partial {:response response}) + (failure? response) (cause-evidence response :preparation) + :else response) + phases (if (failure? accepted) (get-in response [:preparation :phases]) (:phases failure))] + (record-preparation state {:preparation-id preparation-id :case-id (:id case) + :warmup? warmup? + :phases (measurement/partial-phases phases preparation-id [:preparation]) + :diagnostics (or (get-in failure [:partial :setup]) {}) + :failure (errors/project-diagnostic failure)}) + {:failure failure :preparation-id preparation-id})))) + +(defn- ^:async run-attempt + "Returns the bridge evidence of one attempt, or the failure." + [state page preparation-id deadline] + (attempt + (measurement/check-response + :attempt preparation-id + (await ((get-in state [:services :run]) + page + (transit/encode-str + {:preparation-id preparation-id + :timeout-ms (max 1 (process/remaining deadline))}) + {:deadline deadline :signal (:signal state) :phase :attempt}))))) + +(defn- ^:async recycle-browser + [state] + (when-let [owner @(:owner-ref state)] + (let [closed (attempt (await ((get-in state [:services :close]) owner)))] + (if (failure? closed) + (do + (record-failure state nil :browser-close (cause-evidence closed :browser-close)) + (reset! (:abort-run? state) true)) + (do + (reset! (:owner-ref state) nil) + (reset! (:page-ref state) nil)))))) + +(defn- ^:async close-current-page + [state case] + (let [owner @(:owner-ref state) + connected? (get-in state [:services :connected?])] + (if (and owner connected? (not (connected? owner))) + (do + (record-failure state (:id case) :browser-disconnected + {:phase "browser-disconnected" :message "browser disconnected"}) + (reset! (:page-ref state) nil) + (await (recycle-browser state)) + false) + (if-let [page @(:page-ref state)] + (do + (reset! (:page-ref state) nil) + (let [closed (attempt (await ((get-in state [:services :close-page]) page)))] + (if (failure? closed) + (do + (record-failure state (:id case) :browser-close (cause-evidence closed :browser-close)) + (await (recycle-browser state)) + false) + true))) + true)))) + +(defn- ^:async ensure-browser + [state deadline] + (or @(:owner-ref state) + (let [opts (:opts state) + options {:startup-timeout-ms (:startup-timeout-ms opts) + :signal (:signal state) :deadline deadline} + owner (await ((get-in state [:services :launch]) options))] + (reset! (:owner-ref state) owner) + (swap! (:run-ref state) assoc-in [:metadata :environment :browser] (:version owner)) + (persist state) + (write-log state log/debug (str "Browser " (:version owner))) + owner))) + +(defn- usable-graphics-identity? + [value] + (and (string? value) + (not (contains? #{"" "unknown" "unavailable" "null" "undefined"} + (str/lower-case (str/trim value)))))) + +(defn- check-gpu + [state preparation] + (let [graphics (:effective-graphics preparation) + accepted (and (true? (:webgl2 graphics)) + (false? (:software graphics)) + (usable-graphics-identity? (:vendor graphics)) + (usable-graphics-identity? (:renderer graphics)))] + (if (and (:require-gpu (:opts state)) (not accepted)) + (do + (record-failure state (:case-id preparation) :gpu + {:status "failed" :phase "gpu" + :message (str "GPU check failed; renderer: " (pr-str (:renderer graphics)) + ". Hardware WebGL2 with a usable vendor and renderer is required." + " " settings/gpu-guidance) + :partial {:graphics graphics}}) + (reset! (:abort-run? state) true) + :abort) + :ok))) + +(defn- ^:async open-case-page + [state case owner deadline] + (let [opened (attempt + (await ((get-in state [:services :new-page]) + owner @(:host-ref state) + {:viewport (declarations/resolve-viewport (:view case)) + :deadline deadline + :startup-timeout-ms (:startup-timeout-ms (:opts state)) + :signal (:signal state)})))] + (cond + (not (failure? opened)) + (do + (reset! (:page-ref state) opened) + {:page (:page opened)}) + + (interrupted? state) + {:outcome :interrupted} + + :else + (let [evidence (cause-evidence opened :browser-page) + outcome (failure-outcome state deadline evidence)] + (record-failure state (:id case) :browser-page evidence) + (await (recycle-browser state)) + {:outcome outcome})))) + +(defn- ^:async open-page + [state case deadline] + (check-deadline deadline) + (if (interrupted? state) + {:outcome :interrupted} + (let [owner (attempt (await (ensure-browser state deadline)))] + (cond + (not (failure? owner)) + (await (open-case-page state case owner deadline)) + + (interrupted? state) + {:outcome :interrupted} + + :else + (let [evidence (cause-evidence owner :browser-launch) + outcome (failure-outcome state deadline evidence)] + (record-failure state (:id case) :browser-launch evidence) + (if (and (= :timed-out outcome) + (not= ::browser/startup-cleanup (:type (ex-data owner)))) + {:outcome outcome} + (do + (reset! (:abort-run? state) true) + {:outcome :abort}))))))) + +(defn- ^:async execute-slot + [state case page preparation-id warmup? deadline] + (let [response (await (run-attempt state page preparation-id deadline)) + evidence (if (failure? response) (cause-evidence response :attempt) response) + accepted (attempt + (let [measured (measurement/attempt (:id case) preparation-id warmup? evidence)] + (record-attempt state measured) + measured)) + measured (if (failure? accepted) + (let [invalid (assoc (cause-evidence accepted :invalid-output) + :phases (measurement/partial-phases (:phases evidence) preparation-id [:main :reset]) + :partial {:response evidence}) + failed (measurement/attempt (:id case) preparation-id warmup? invalid)] + (record-attempt state failed) + failed) + accepted)] + (cond + (and (failure? response) (interrupted? state)) :interrupted + (= :completed (:outcome measured)) :ok + :else (failure-outcome state deadline (if (failure? response) + (cause-evidence response :attempt) + response))))) + +(defn- ^:async finish-slot + [state case page outcome final? deadline] + (if (and final? (= :ok outcome)) + (await (screenshot-case state case page deadline)) + (do + (when (not= :ok outcome) + (record-screenshot-skip state (:id case) :no-completed-measured-attempt)) + outcome))) + +(defn- ^:async run-fresh-slot + [state case page warmup? final? deadline] + (let [prepared (await (prepare-case state case page deadline warmup?))] + (if-let [evidence (:failure prepared)] + (do + (record-attempt state (measurement/attempt (:id case) (:preparation-id prepared) warmup? + (dissoc evidence :phases))) + (await (finish-slot state case page + (failure-outcome state deadline evidence) final? deadline))) + (let [preparation (:preparation prepared)] + (if (= :abort (check-gpu state preparation)) + :abort + (let [outcome (await (execute-slot state case page (:preparation-id preparation) + warmup? deadline))] + (await (finish-slot state case page outcome final? deadline)))))))) + +(defn- ^:async run-fresh-page + [state case warmup? final? deadline] + (let [{:keys [page outcome]} (await (open-page state case deadline))] + (if page + (let [outcome (await (run-fresh-slot state case page warmup? final? deadline))] + (if (= :ok outcome) + (if (await (close-current-page state case)) outcome :failed) + outcome)) + (or outcome :abandon)))) + +(defn- ^:async run-fresh-slots + [state case slots deadline] + (if-let [[warmup? & pending] (seq slots)] + (let [outcome (await (run-fresh-page state case warmup? (empty? pending) deadline))] + (if (= :ok outcome) + (run-fresh-slots state case pending deadline) + outcome)) + :done)) + +(defn- ^:async run-fresh-case + [state case deadline] + (let [outcome (await (run-fresh-slots state case (case-slots (:opts state)) deadline))] + (ensure-screenshot-status state case :case-abandoned) + outcome)) + +(defn- ^:async run-reuse-slots + [state case page preparation-id slots deadline] + (if-let [[warmup? & pending] (seq slots)] + (let [_ (check-deadline deadline) + executed (await (execute-slot state case page preparation-id warmup? deadline)) + outcome (await (finish-slot state case page executed (empty? pending) deadline))] + (if (= :ok outcome) + (run-reuse-slots state case page preparation-id pending deadline) + outcome)) + :done)) + +(defn- ^:async abandon-preparation + [state case evidence deadline] + (record-failure state (:id case) :preparation evidence) + (ensure-screenshot-status state case :no-completed-measured-attempt) + (failure-outcome state deadline evidence)) + +(defn- ^:async run-reuse-prepared + [state case page preparation deadline] + (let [accepted (check-gpu state preparation) + outcome (if (= :abort accepted) + :abort + (await (run-reuse-slots state case page (:preparation-id preparation) + (case-slots (:opts state)) deadline)))] + (ensure-screenshot-status state case :case-abandoned) + (if (= :done outcome) + (if (await (close-current-page state case)) outcome :failed) + outcome))) + +(defn- ^:async run-reuse-page + [state case page deadline] + (let [prepared (await (prepare-case state case page deadline false))] + (if-let [evidence (:failure prepared)] + (await (abandon-preparation state case evidence deadline)) + (await (run-reuse-prepared state case page (:preparation prepared) deadline))))) + +(defn- ^:async run-reuse-case + [state case deadline] + (let [{:keys [page outcome]} (await (open-page state case deadline))] + (cond + page (await (run-reuse-page state case page deadline)) + (= :abort outcome) :abort + + :else + (do + (ensure-screenshot-status state case :case-abandoned) + (or outcome :abandon))))) + + +(defn- fail-startup + "Records the startup failure and aborts the run. An interrupted run records + nothing. Returns false." + [state stage cause] + (when-not (interrupted? state) + (record-failure state nil stage (cause-evidence cause stage)) + (reset! (:abort-run? state) true)) + false) + +(defn- ^:async launch-initial + [state deadline] + (let [owner (attempt (await (ensure-browser state deadline)))] + (if (failure? owner) + (fail-startup state :browser-launch owner) + true))) + +(defn- ^:async record-provenance + [state deadline] + (let [git (attempt (await (build/git-provenance (:root state) + {:deadline deadline :signal (:signal state)})))] + (if (failure? git) + (fail-startup state :runner-git git) + (do + (swap! (:run-ref state) assoc-in [:metadata :diagnostics :runner-git] git) + (persist state) + true)))) + +(defn- ^:async start-server + [state deadline] + (let [remaining (process/remaining deadline) + host (attempt + (when (zero? remaining) + (throw (ex-info "server startup timed out" + {:type ::startup-timeout :phase :server-startup}))) + (await (server/open (:selected-build state) + {:startup-timeout-ms remaining :signal (:signal state)})))] + (if (failure? host) + (fail-startup state :server-startup host) + (do + (reset! (:host-ref state) host) + true)))) + +(defn- session-active? + [state] + (and (not @(:abort-run? state)) (not (interrupted? state)))) + +(defn- get-case-failure + [run case-id] + (or (some :failure (reverse (filter #(= case-id (:case-id %)) (:attempts run)))) + (some :failure (reverse (filter #(= case-id (:case-id %)) (:preparations run)))) + (some #(when (= case-id (:case-id %)) %) + (reverse (get-in run [:metadata :diagnostics :failures]))))) + +(defn- ^:async run-case + [state case deadline] + (let [executed (attempt + (if (= :fresh (:context case)) + (await (run-fresh-case state case deadline)) + (await (run-reuse-case state case deadline)))) + outcome (if (failure? executed) + (let [evidence (cause-evidence executed :case)] + (record-failure state (:id case) :case evidence) + (failure-outcome state deadline evidence)) + executed) + unmeasured? (and (= :done outcome) + (not (result/has-measurements? @(:run-ref state) (:id case)))) + outcome (if unmeasured? + (do + (record-failure state (:id case) :case + {:phase "case" :reason :no-measurements + :message "case has no measurements"}) + :failed) + outcome) + saved-outcome (cond + (= :done outcome) :completed + (= :interrupted outcome) :interrupted + (= :timed-out outcome) :timed-out + :else :failed) + failure (when (not= :completed saved-outcome) + (get-case-failure @(:run-ref state) (:id case)))] + (swap! (:run-ref state) result/record-case-outcome (:id case) + (cond-> {:outcome saved-outcome} + failure (assoc :failure failure))) + (ensure-screenshot-status state case :case-abandoned) + (persist state) + (when (not= :done outcome) + (await (recycle-browser state))) + outcome)) + +(defn- ^:async run-cases + [state cases] + (when (and (seq cases) (session-active? state)) + (let [case (first cases) + deadline (process/deadline (:timeout-ms (:opts state)))] + (write-log state log/info (str "▸ Case " (result/case-name (:id case)) + " (" (name (:context case)) ")")) + (let [outcome (await (run-case state case deadline))] + (if (#{:abort :interrupted} outcome) + (do + (reset! (:abort-run? state) true) + outcome) + (run-cases state (rest cases))))))) + +(defn- ^:async run-session + [state] + (let [deadline (process/deadline (:startup-timeout-ms (:opts state))) + provenance-ready? (await (record-provenance state deadline)) + server-ready? (and provenance-ready? (await (start-server state deadline))) + browser-ready? (and server-ready? (session-active? state) (await (launch-initial state deadline)))] + (when (and browser-ready? (session-active? state)) + (await (run-cases state (:cases state)))))) + +(defn- get-termination + [run interrupted harness-failed?] + (let [all-attempted? (every? (fn [[_ slots]] + (and (zero? (:warmups slots)) + (zero? (:repetitions slots)))) + (:remaining run)) + failed? (or (some #(= :failed (:outcome %)) (:attempts run)) + (some #(not= :completed (:outcome %)) (vals (:case-outcomes run)))) + reason (cond + interrupted :interrupted + all-attempted? (if failed? :failed :completed) + :else :incomplete) + failures (get-in run [:metadata :diagnostics :failures]) + attempt-failure (some #(when (:failure %) (:failure %)) + (reverse (:attempts run))) + termination (cond-> {:reason reason} + interrupted + (assoc :signal (:signal interrupted)) + + (and (seq failures) + (or (not= :completed reason) harness-failed?)) + (assoc :failure (last failures)) + + (and (not= :completed reason) + (empty? failures) + (some? attempt-failure)) + (assoc :failure attempt-failure))] + termination)) + +(defn- complete-screenshots + [screenshots cases] + (let [defaults (into {} (map (fn [case] + [(result/case-name (:id case)) {:status :skipped :reason :not-run}])) + cases)] + (merge defaults screenshots))) + +(defn- finish-run + [state] + (let [termination (get-termination @(:run-ref state) (interruption (:signal state)) + @(:harness-failed? state))] + (when (:screenshots (:opts state)) + (swap! (:run-ref state) update-in [:metadata :diagnostics :screenshots] + complete-screenshots (:cases state))) + (swap! (:run-ref state) result/finish-run (time/now) termination) + (persist state) + @(:run-ref state))) + +(defn- ^:async release-resource + "Closes the resource in `reference`. Returns nil or the failure." + [reference close] + (attempt :catch-fatal + (when-let [resource @reference] + (reset! reference nil) + (await (close resource))) + nil)) + +(defn- ^:async cleanup + [state] + (let [page-failure (await (release-resource (:page-ref state) (get-in state [:services :close-page]))) + browser-failure (await (release-resource (:owner-ref state) (get-in state [:services :close]))) + server-failure (await (release-resource (:host-ref state) server/close)) + failure (errors/combine + [{:phase "cleanup" :operation :close-page :error page-failure} + {:phase "cleanup" :operation :close-browser :error browser-failure} + {:phase "cleanup" :operation :close-server :error server-failure}])] + (when failure + (warn-safely state (str "run: cleanup failed: " (errors/message-of failure) "\n"))) + failure)) + +(defn- remove-build + [root selected-build] + (artifacts/remove-builds root [(:id selected-build)])) + +(defn- git-facts + [selected-build] + (merge {:sha nil :dirty nil} + (get-in selected-build [:manifest :provenance :git]))) + +(defn- playwright-version + [root] + (ex/ignoring + (let [require-package (if (exists? js/require) + js/require + (node-module/createRequire (path/join root "package.json")))] + (.-version (require-package "playwright/package.json"))))) + +(defn- environment + [root _selected-build] + (let [cpus (array-seq (os/cpus))] + {:node (.-version js/process) + :playwright (playwright-version root) + :browser nil + :platform (os/platform) + :release (os/release) + :arch (os/arch) + :cpu {:models (vec (distinct (map #(.-model %) cpus))) + :logical-count (count cpus)} + :launch (browser/launch-settings)})) + +(defn- metadata + [root selected-build opts] + {:git (git-facts selected-build) + :build (cond-> {:functional (or (get-in selected-build [:manifest :functional]) {}) + :provenance (assoc (get-in selected-build [:manifest :provenance]) + :build-id (:id selected-build))} + (contains? (:manifest selected-build) :environment) + (assoc :environment (get-in selected-build [:manifest :environment]))) + :environment (environment root selected-build) + :scored? true + :diagnostics (cond-> {:seed (:seed opts) + :filter (:filter opts) + :warmups (:warmups opts) + :repetitions (:repetitions opts) + :timeout-ms (:timeout-ms opts) + :startup-timeout-ms (:startup-timeout-ms opts) + :screenshots? (boolean (:screenshots opts)) + :require-gpu? (boolean (:require-gpu opts)) + :penpot-gpu (or (.-PENPOT_GPU (.-env js/process)) nil) + :runner-git {:sha nil :dirty nil} + :runtime-helper-identity :unverified + :screenshots {} + :failures []} + (:build-timeout-ms opts) (assoc :build-timeout-ms (:build-timeout-ms opts)))}) + +(defn- create-run-record + [root run-id selected-build cases opts] + (result/create-run + {:run-id run-id + :started-at (time/now) + :plan {:cases cases + :warmups (:warmups opts) + :repetitions (:repetitions opts) + :timeout-policy {:case-ms (:timeout-ms opts)}} + :metadata (metadata root selected-build opts)})) + +(defn- create-run-state + [root home selected-build cases opts signal] + {:root root + :selected-build selected-build + :cases cases + :opts opts + :signal signal + :home home + :run-ref (atom (create-run-record root (:run-id home) selected-build cases opts)) + :sink-failure-ref (atom nil) + :owner-ref (atom nil) + :page-ref (atom nil) + :host-ref (atom nil) + :abort-run? (atom false) + :harness-failed? (atom false) + :services (merge browser/service (:browser opts))}) + +(defn- record-session-failure + [state stage cause] + (let [recorded (attempt :catch-fatal + (record-failure state nil stage (cause-evidence cause stage)))] + (when (failure? recorded) + (warn-safely state (str "Could not record " (when (= :cleanup stage) "cleanup ") + "failure: " (errors/message-of recorded) "\n"))))) + +(defn- ^:async execute-session + "Runs the session and retains failures that require final reporting. + + Args: + state: The session's execution state. + + Returns: + The failure that ended execution, or nil. Interrupted sessions retain + fatal failures." + [state] + (let [ended (attempt :catch-fatal (await (run-session state)))] + (when (and (failure? ended) (or (not (interrupted? state)) (errors/fatal? ended))) + (record-session-failure state :runner ended) + ended))) + +(defn- ^:async cleanup-session + [state] + (when-let [cause (await (cleanup state))] + (record-session-failure state :cleanup cause) + cause)) + +(defn- remove-run-build + [state saved? failures] + (when (and saved? (:remove (:opts state)) (not-any? process/unresolved? failures)) + (attempt :catch-fatal + (remove-build (:root state) (:selected-build state)) + nil))) + +(defn- format-run-report + "Returns the report text or the failure." + [state run] + (let [report (attempt (format/format-run (summary/summarize run)))] + (when (failure? report) + (warn-safely state (str "Report failed: " (errors/message-of report) "\n"))) + report)) + +(defn- log-run-completion + [state run] + (attempt :catch-fatal + (write-log state log/debug (str "Run " (name (get-in run [:termination :reason])))) + nil)) + +(defn- get-exit-code + [run causes harness-failed?] + (case (get-in run [:termination :reason]) + :interrupted (outcome/interruption-code (get-in run [:termination :signal])) + :completed (if (or (seq causes) harness-failed?) 1 0) + 1)) + +(defn- finish-execution + [state session-error cleanup-error] + (let [saved (attempt :catch-fatal (finish-run state)) + run (if (failure? saved) @(:run-ref state) saved) + removal (remove-run-build state (not (failure? saved)) + (filter failure? [session-error cleanup-error saved @(:sink-failure-ref state)])) + report (format-run-report state run) + logged (log-run-completion state run) + causes (->> [cleanup-error session-error saved removal report logged + @(:sink-failure-ref state)] + (filter failure?) + distinct + vec) + home (:home state)] + {:run run + :home home + :log-path (:log-path home) + :report (when-not (failure? report) report) + :errors causes + :exit-code (get-exit-code run causes @(:harness-failed? state))})) + +(defn ^:async execute + "Executes a prepared selection. Owns host and browser. + + - `root`: the frontend checkout root. + - `home`: the reserved output home. + - `selected-build`: the prepared build. + - `cases`: the selected cases. + - `options`: resolved execution settings, including optional local :browser services. + - `signal`: an optional abort signal. + + A browser startup failure leaves a valid partial record. + + Returns the run, home, report, log path, exit code and the failures in + :errors, a cleanup failure first. :errors includes any failed record or log + write, including a failure followed by a successful write. An interruption + is not a failure." + [root home selected-build cases options signal] + (binding [log/*log-path* (:log-path home)] + (let [state (create-run-state root home selected-build cases options signal)] + (persist state) + (let [session-error (await (execute-session state)) + cleanup-error (await (cleanup-session state))] + (finish-execution state session-error cleanup-error))))) + +(defn- ^:async get-selected-build + [root opts signal] + (let [{:keys [kind id default-features features build-timeout-ms]} (:build opts)] + (case kind + :existing (artifacts/load-build root id) + :prepare (await (build/prepare + {:root root + :selection {:default-features? default-features :features features} + :build-timeout-ms build-timeout-ms :signal signal}))))) + +(defn- get-cases + [selected-build opts] + (try + (catalog/collect selected-build {:master-seed (:seed opts) :filter (:filter opts)}) + (catch :default cause + (throw (ex-info (errors/message-of cause) + (assoc (ex-data cause) :build-id (:id selected-build) + :log-path (:log-path selected-build)) + cause))))) + +(defn ^:async run + "Prepares or loads a build, collects matching cases and executes one run. + + - `root`: the frontend checkout root. + - `options`: resolved run options with a tagged :build source. + - `signal`: an optional abort signal. + - `services`: local service overrides, with browser functions under :browser. + + Returns an outcome with the reserved home, available evidence and failures." + [root options signal services] + (let [home (runs/create-home {:root root :run-id (str (random-uuid)) + :output-dir (:output-dir options) :remove (:remove options)}) + opts (cond-> (assoc (get-execution-options options) :browser (:browser services)) + (= :prepare (get-in options [:build :kind])) + (assoc :build-timeout-ms (get-in options [:build :build-timeout-ms])))] + (binding [log/*log-path* (:log-path home)] + (try + (let [selected-build (await (get-selected-build root options signal)) + cases (get-cases selected-build opts)] + (log/info (str " build " (:id selected-build))) + (log/info (str " selected " (count cases) " cases")) + (await (execute root home selected-build cases opts signal))) + (catch :default cause + (outcome/failed (-> cause errors/as-failure + (errors/with-data {:log-path (:log-path home)})) + {:home home :run nil :report nil})))))) diff --git a/frontend/performance/benches/render_wasm/node/runner/server.cljs b/frontend/performance/benches/render_wasm/node/runner/server.cljs new file mode 100644 index 0000000000..24457ebb36 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/runner/server.cljs @@ -0,0 +1,309 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.runner.server + "An owned loopback HTTP server for one immutable renderer build." + (:require + ["fs" :as fs] + ["http" :as http] + ["path" :as path] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.settings :as settings] + [clojure.string :as str])) + +(def ^:private mime + {".html" "text/html; charset=utf-8" + ".js" "application/javascript" + ".cjs" "application/javascript" + ".mjs" "application/javascript" + ".wasm" "application/wasm" + ".css" "text/css; charset=utf-8" + ".json" "application/json" + ".svg" "image/svg+xml" + ".png" "image/png"}) + +(def ^:private status-text + {400 "bad request" + 403 "forbidden" + 404 "not found" + 405 "method not allowed"}) + +(def ^:private fs-error-status + {"ENOENT" 404 + "ENOTDIR" 404 + "EACCES" 403}) + +(def ^:private owners + "Maps each owned server to {:sockets Set :shutdown atom}." + (js/WeakMap.)) + +;; Request URL to file + +(defn- rejection + "Returns an Error that `error-status` maps to `status`." + [status message] + (ex-info message {:status status})) + +(defn- unsafe-pathname? + [pathname] + (or (str/includes? pathname "\u0000") + (str/includes? pathname "\\") + (some #{".."} (str/split pathname #"/")))) + +(defn- parse-pathname + "Returns the decoded path of a request URL without its query and fragment. + + Throws: + 400 for a path without a leading slash and for a malformed escape. + 403 for a path with a NUL, a backslash or a `..` segment." + [url] + (let [pathname (js/decodeURIComponent (first (str/split url #"[?#]" 2)))] + (cond + (not (str/starts-with? pathname "/")) (throw (rejection 400 "invalid request path")) + (unsafe-pathname? pathname) (throw (rejection 403 "forbidden request path")) + :else pathname))) + +(defn- pathname->relative + [pathname] + (if (= pathname "/") "index.html" (subs pathname 1))) + +(defn- contained + "Returns `file` when it lies within `root`, otherwise throws 403 with `message`." + [root message file] + (if (files/contained? root file) + file + (throw (rejection 403 message)))) + +(defn- request-file + "Returns the real path of the file that `url` selects below `root`. + + Throws: + 400 or 403 for a bad path. + 403 for a path or a symlink that leaves `root`. + The filesystem error when the file cannot be resolved." + [root url] + (->> (parse-pathname url) + (pathname->relative) + (path/resolve root) + (contained root "forbidden request path") + (fs/realpathSync) + (contained root "forbidden artifact symlink"))) + +;; File to response + +(defn- error-status + "Returns the HTTP status for a failure while serving a request." + [error] + (or (:status (ex-data error)) + (when (instance? js/URIError error) 400) + (get fs-error-status (.-code ^js error) 500))) + +(defn- error-response + [method status] + {:status status + :headers {"Content-Type" "text/plain; charset=utf-8"} + :body (when-not (= "HEAD" method) + (get status-text status "server error"))}) + +(defn- method-not-allowed + [method] + (assoc-in (error-response method 405) [:headers "Allow"] "GET, HEAD")) + +(defn- file-response + [method file ^js stat] + {:status 200 + :headers {"Content-Type" (get mime (path/extname file) "application/octet-stream") + "Content-Length" (.-size stat)} + :body (when (= "GET" method) + (fs/readFileSync file))}) + +(defn- get-file-response + [root method url] + (try + (let [file (request-file root url) + ^js stat (fs/statSync file)] + (if (.isFile stat) + (file-response method file stat) + (error-response method 404))) + (catch :default error + (error-response method (error-status error))))) + +(defn- respond + "Returns {:status :headers :body} for a request with `method` and `url`." + [root method url] + (if (contains? #{"GET" "HEAD"} method) + (get-file-response root method url) + (method-not-allowed method))) + +(defn- write-response + [^js response {:keys [status headers body]}] + (.writeHead response status (clj->js headers)) + (.end response body)) + +(defn- serve + [root ^js request ^js response] + (write-response response (respond root (.-method request) (.-url request)))) + +;; Shutdown + +(defn- close-failure + "Returns `error` unless it is nil or reports a server that was not running." + [error] + (when-not (or (nil? error) + (= "ERR_SERVER_NOT_RUNNING" (.-code ^js error))) + error)) + +(defn- close-server + "Closes `server` and destroys its `sockets`. + + Returns a promise that resolves once the server closed. A server that is + not running counts as closed." + [^js server ^js sockets] + (js/Promise. + (fn [resolve reject] + (let [settle (fn [error] + (if-let [failure (close-failure error)] + (reject failure) + (resolve nil)))] + (try + (.close server settle) + (run! #(.destroy ^js %) (array-seq (js/Array.from sockets))) + (catch :default error + (settle error))))))) + +(defn close + "Closes an owned host once and destroys only that server's connections. + + Args: + host: Map returned by `open`. + + Returns: + Promise that resolves once the server closed. Repeated calls return the + same promise, including during startup." + [{:keys [server]}] + (let [{:keys [sockets shutdown]} (.get owners server)] + (or @shutdown + (reset! shutdown (close-server server sockets))))) + +;; Startup + +(defn- track-connection + [^js sockets shutdown ^js socket] + (.add sockets socket) + (.once socket "close" #(.delete sockets socket)) + (when @shutdown + (.destroy socket))) + +(defn- create-host + "Returns {:server :build} for a server of `build`'s files that does not + listen yet." + [build] + (let [root (fs/realpathSync (:dir build)) + ^js server (http/createServer (partial serve root)) + sockets (js/Set.) + shutdown (atom nil)] + (.set owners server {:sockets sockets :shutdown shutdown}) + (.on server "connection" (partial track-connection sockets shutdown)) + {:server server :build build})) + +(defn- aborted? + [^js signal] + (and signal (.-aborted signal))) + +(defn- aborted-error + [] + (ex-info "renderer server startup aborted" {:type ::aborted})) + +(defn- timeout-error + [timeout-ms] + (ex-info "renderer server startup timed out" + {:type ::startup-timeout :timeout-ms timeout-ms})) + +(defn- subscribe-startup + "Routes the end of `server` startup to `succeed` or `fail`. + + `succeed` receives the listening event. `fail` receives the first error + event of `server`, an `::aborted` error when `signal` (optional) aborts and + a `::startup-timeout` error after `timeout-ms`. + + Returns a function that removes the subscriptions." + [^js server signal timeout-ms succeed fail] + (let [on-abort #(fail (aborted-error)) + timer (js/setTimeout #(fail (timeout-error timeout-ms)) timeout-ms)] + (.once server "listening" succeed) + (.once server "error" fail) + (when signal + (.addEventListener ^js signal "abort" on-abort #js {:once true})) + (fn [] + (js/clearTimeout timer) + (.removeListener server "listening" succeed) + (.removeListener server "error" fail) + (when signal + (.removeEventListener ^js signal "abort" on-abort))))) + +(defn- ^:async listen + "Waits until `server` listens on 127.0.0.1 and an ephemeral port. + + Aborting `close-signal` closes the server. Rejects as `subscribe-startup` + describes, and when `signal` is already aborted, without listening." + [^js server close-signal signal timeout-ms] + (when (aborted? signal) + (throw (aborted-error))) + ;; Nothing awaits before the subscription, so an abort that follows the + ;; call reaches it. + (let [^js startup (js/Promise.withResolvers) + release (subscribe-startup server signal timeout-ms + (.-resolve startup) (.-reject startup))] + (try + (.listen server #js {:host "127.0.0.1" :port 0 :signal close-signal}) + (await (.-promise startup)) + (finally + (release))))) + +(defn- ^:async startup-failure + "Closes `host` after a failed startup. + + Returns `error`, or an Error that also holds the failure of the close." + [host ^js controller error] + (.abort controller) + (try + (await (close host)) + error + (catch :default close-error + (errors/combine + [{:phase "server-startup" :operation :listen :error error} + {:phase "cleanup" :operation :close-server :error close-error}])))) + +(defn- base-url + [^js server] + (str "http://127.0.0.1:" (.-port (.address server)))) + +(defn ^:async open + "Starts serving build's directory on 127.0.0.1 and an ephemeral port. + + Args: + build: Map with the directory to serve at `:dir`. + options: Optional map with `:startup-timeout-ms` (30000 by default) and + `:signal`, an AbortSignal. + + Returns: + Promise of {:server NodeHttpServer :base-url string :build build}. + + Throws: + Rejects on abort, timeout and listen errors, after closing the owned + server. A failed close joins the startup error." + ([build] (open build {})) + ([build {:keys [startup-timeout-ms signal] + :or {startup-timeout-ms settings/default-startup-timeout-ms}}] + (let [host (create-host build) + controller (js/AbortController.)] + (try + (await (listen (:server host) (.-signal controller) signal startup-timeout-ms)) + (assoc host :base-url (base-url (:server host))) + (catch :default error + (let [failure (await (startup-failure host controller error))] + (throw failure))))))) diff --git a/frontend/performance/benches/render_wasm/node/settings.cljs b/frontend/performance/benches/render_wasm/node/settings.cljs new file mode 100644 index 0000000000..100d8ac791 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/settings.cljs @@ -0,0 +1,14 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.settings + "Shared operational settings and guidance.") + +(def default-build-timeout-ms 300000) +(def default-startup-timeout-ms 30000) + +(def gpu-guidance + "Start the container with ./manage.sh run-devenv --gpu and use pnpm run render-bench.") diff --git a/frontend/performance/benches/render_wasm/node/storage/artifacts.cljs b/frontend/performance/benches/render_wasm/node/storage/artifacts.cljs new file mode 100644 index 0000000000..61e53643d2 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/storage/artifacts.cljs @@ -0,0 +1,403 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.storage.artifacts + "Managed renderer builds and output homes for benchmark runs and comparisons. + + Here, 'managed' means that the runner owns the build directory and its + contents, which can be deleted by means of the `purge` command." + (:require + ["fs" :as fs] + ["path" :as path] + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.time :as time] + [clojure.string :as str])) + +(def schema:build + "Stored renderer build identity, lifecycle and provenance." + [:map + [:type [:= :renderer-build]] + [:id [:string {:min 1}]] + [:status [:enum :building :ready :failed]] + [:created-at :string] + [:updated-at :string] + [:environment {:optional true} [:map-of :string :string]] + [:failure {:optional true} errors/schema:diagnostic] + [:functional :map] + [:provenance :map]]) + +(def ^:private executable-dir + ;; CJS boots use __dirname. Node's ESM test boots have an absolute argv[1] + ;; instead; capture it before a caller changes cwd. Neither uses a cwd root. + (if (exists? js/__dirname) + js/__dirname + (let [script (aget js/process.argv 1)] + (when (and (string? script) (path/isAbsolute script)) + (path/dirname script))))) + +(defn frontend-root + "Finds the frontend checkout by walking up from the Node executable. + + Supports target/performance/render-wasm/runner.cjs and the ESM + target/{performance-tests,tests}/test.js boots, independent of caller cwd." + [] + (when-not executable-dir + (throw (ex-info "cannot locate the Node executable directory" + {:type ::missing-executable-dir}))) + (loop [dir (path/resolve executable-dir)] + (let [deps (path/join dir "deps.edn") + scenes (path/join dir "performance" "benches" "render_wasm") + parent (path/dirname dir)] + (cond + (and (fs/existsSync deps) (.isFile ^js (fs/statSync deps)) + (fs/existsSync scenes) (.isDirectory ^js (fs/statSync scenes))) + dir + + (= parent dir) + (throw (ex-info "cannot locate the frontend checkout" + {:type ::missing-frontend-root + :executable-dir executable-dir})) + + :else + (recur parent))))) + +(defn valid-id? + "True for IDs of up to 128 letters, digits, underscores or hyphens, + starting with a letter or digit." + [id] + (and (string? id) + (boolean (re-matches #"[A-Za-z0-9][A-Za-z0-9_-]{0,127}" id)))) + +(defn- check-id + [id] + (when-not (valid-id? id) + (throw (ex-info "invalid renderer build id" {:type ::invalid-id :id id}))) + id) + +(defn invalid-home + "Throws an ::invalid-home exception with `data` and an optional cause." + ([message data] + (throw (ex-info message (assoc data :type ::invalid-home)))) + ([message data cause] + (throw (ex-info message (assoc data :type ::invalid-home) cause)))) + +(defn check-home-id + "Returns `id` when it is valid, or throws ::invalid-home with `message`." + [id message] + (when-not (valid-id? id) + (invalid-home message {:id id})) + id) + +(defn check-output-dir + "Resolves an explicit user directory from the caller's cwd and checks that it + does not exist. Returns nil for nil. Does not create anything." + [dir] + (when (some? dir) + (when (str/blank? dir) + (invalid-home "output directory requires a nonempty path" {})) + (let [resolved (path/resolve dir) + exists? (try + (fs/lstatSync resolved #js {:throwIfNoEntry false}) + (catch :default cause + (let [detail (errors/message-of cause)] + (invalid-home (str "cannot inspect output directory: " detail) + {:path resolved :cause detail} + cause))))] + (when exists? + (invalid-home "output directory already exists" {:path resolved})) + resolved))) + +(defn- create-home* + [dir stem] + (let [log-path (path/join dir (str stem ".log"))] + (fs/mkdirSync dir #js {:recursive true}) + (fs/writeFileSync log-path "" #js {:encoding "utf8" :flag "wx"}) + {:dir dir + :log-path log-path + :record-path (path/join dir (str stem ".transit"))})) + +(defn create-home + "Creates `dir` and its parents, reserves an empty log. + + A filesystem failure throws ::invalid-home with `message` and `:path`, and + keeps the original cause. + + Args: + dir: path to create. + stem: names the .log and .transit files. + message: used for errors + + Returns: + Directory created, log path and record path." + [dir stem message] + (try + (create-home* dir stem) + (catch :default cause + (invalid-home message + {:path dir :cause (errors/message-of cause)} + cause)))) + +(defn- checked-path + [root file] + (let [resolved (fs/realpathSync file)] + (when-not (files/contained? root resolved) + (throw (ex-info "renderer artifact path escapes its managed directory" + {:type ::unsafe-path :path file}))) + resolved)) + +(defn managed-root + "Returns the managed path under an existing frontend checkout. + + Checks its nearest existing ancestor before creating or removing paths, + so a managed-root or ancestor symlink cannot escape the frontend root." + [root] + (let [frontend (fs/realpathSync root) + managed (path/resolve root "target" "performance" "render-wasm")] + (loop [existing managed] + (if (fs/existsSync existing) + (checked-path frontend existing) + (recur (path/dirname existing)))) + managed)) + +(defn inside-builds? + "Checks the whole builds tree, resolving symlink ancestors of both paths." + [frontend-root output-path] + (files/contained? (files/get-real-path (path/join (managed-root frontend-root) "builds")) + (files/get-real-path output-path))) + +(defn- build-reference + [root id manifest] + (let [dir (path/join (managed-root root) "builds" id)] + {:id id + :dir dir + :manifest-path (path/join dir "build.transit") + :log-path (path/join dir "build.log") + :cases-path (path/join dir "cases.cjs") + :browser-dir (path/join dir "browser") + :renderer-dir (path/join dir "renderer") + :index-path (path/join dir "index.html") + :manifest manifest})) + +(defn create-build + "Allocates a UUID build and saves its :building build.transit and empty log. + + - root: the frontend checkout. + - selection: {:default-features? boolean :features vector} + - provenance: {:git {:sha string-or-nil :dirty boolean-or-nil}}. + + Returns a build reference. + " + [root selection provenance] + (let [id (str (random-uuid)) + now (time/now) + manifest {:type :renderer-build + :id id + :status :building + :created-at now + :updated-at now + :functional {:mode :release + :default-features? (:default-features? selection) + :features (:features selection) + :cljs-profile-marks false} + :provenance provenance} + build (build-reference root id manifest)] + (fs/mkdirSync (path/dirname (:dir build)) #js {:recursive true}) + (checked-path (fs/realpathSync (managed-root root)) (path/dirname (:dir build))) + (fs/mkdirSync (:dir build)) + (fs/mkdirSync (:browser-dir build)) + (fs/mkdirSync (:renderer-dir build)) + (fs/writeFileSync (:log-path build) "" "utf8") + (files/atomic-write (:manifest-path build) (codec/encode-str manifest)) + build)) + +(defn write-manifest + "Atomically replaces a manifest and returns the updated build reference. + + Only :building builds can change. An unchanged terminal manifest is a + no-op; :ready and :failed builds cannot otherwise change. + Read the saved status so an old build reference cannot reopen a build." + [build manifest] + (let [current (transit/decode-str (fs/readFileSync (:manifest-path build) "utf8"))] + (when-not (and (= (:id build) (:id current) (:id manifest)) + (sm/validate schema:build current) + (sm/validate schema:build manifest)) + (throw (ex-info "invalid schema:build" + {:type ::invalid-manifest :id (:id build)}))) + (if (contains? #{:ready :failed} (:status current)) + (if (= current manifest) + (assoc build :manifest current) + (throw (ex-info "renderer build is already terminal" + {:type ::terminal-build + :id (:id build) + :status (:status current)}))) + (let [manifest (assoc manifest :updated-at (time/now))] + (files/atomic-write (:manifest-path build) (codec/encode-str manifest)) + (assoc build :manifest manifest))))) + +(defn- require-file + [root file] + (let [resolved (checked-path root file)] + (when-not (.isFile ^js (fs/statSync resolved)) + (throw (ex-info "renderer artifact is not a file" + {:type ::missing-artifact :path file}))) + resolved)) + +(defn check-files + "Checks required files and selected directory containment; returns build. + + build is a reference from create-build or load-build. This check neither + reads nor changes manifest status. The build helper must also check catalog + and browser identities before writing its terminal :ready manifest." + [build] + ;; Build directories use /target/performance/render-wasm/builds/. + (let [root (path/resolve (:dir build) ".." ".." ".." ".." "..") + managed (fs/realpathSync (managed-root root)) + dir (checked-path managed (:dir build))] + (require-file dir (:cases-path build)) + (require-file dir (path/join (:browser-dir build) "browser.js")) + (require-file dir (path/join (:renderer-dir build) "render-wasm.js")) + (require-file dir (path/join (:renderer-dir build) "render-wasm.wasm")) + (require-file dir (path/join (:renderer-dir build) "shared.js")) + (require-file dir (:index-path build)) + build)) + +(defn load-build + "Loads a ready schema:build and checks every required artifact. + + IDs cannot contain path segments. The build, manifest and artifact paths + must remain inside the managed root and the selected build after resolving + symlinks. Building, failed, incomplete and unsupported builds are rejected." + [root id] + (check-id id) + (let [managed (fs/realpathSync (managed-root root)) + build (build-reference root id nil) + dir (checked-path managed (:dir build)) + file (require-file dir (:manifest-path build)) + manifest (transit/decode-str (fs/readFileSync file "utf8"))] + (when-not (and (sm/validate schema:build manifest) + (= id (:id manifest))) + (throw (ex-info "invalid schema:build" + {:type ::unsupported-manifest :id id}))) + (when-not (= :ready (:status manifest)) + (throw (ex-info "renderer build is not ready" + {:type ::not-ready :id id :status (:status manifest)}))) + (check-files (assoc build :manifest manifest)))) + +(def ^:private build-outputs + ["build.transit" "build.log" "cases.cjs" "index.html" + "browser/browser.js" "renderer/render-wasm.js" "renderer/render-wasm.wasm" + "renderer/render-wasm.wasm.map" "renderer/shared.js" "renderer/cargo-build.json" + "renderer/worker/render.js"]) + +(defn- directory? + [file] + (and (fs/existsSync file) (.isDirectory (fs/statSync file)))) + +(defn- unlink-if-present + [file] + (when-not (directory? file) + (try + (fs/unlinkSync file) + (catch :default error + (when-not (contains? #{"ENOENT" "ENOTDIR"} (.-code error)) + (throw error)))))) + +(defn- clean-build + [dir] + (when (directory? dir) + (doseq [file build-outputs] + (unlink-if-present (path/join dir file))))) + +(defn- purge-builds + [root] + (when (directory? root) + (doseq [id (array-seq (fs/readdirSync root))] + (clean-build (path/join root id))))) + +(defn- purge-runs + [root] + (loop [pending [root] + visited #{}] + (when-let [dir (peek pending)] + (let [pending (pop pending) + real (when (directory? dir) (fs/realpathSync dir))] + (if (or (nil? real) (contains? visited real)) + (recur pending visited) + (do + (unlink-if-present (path/join dir "run.log")) + (let [names (array-seq (fs/readdirSync dir))] + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".png")] + (unlink-if-present (path/join dir file))) + (unlink-if-present (path/join dir "run.transit")) + (recur (into pending (map #(path/join dir %) names)) + (conj visited real))))))))) + +(defn- purge-feature-comparisons + "Unlinks feature-comparison records and logs, configuration run evidence and + edge reports from the output directory. + + Keeps directories, builds and unrelated files." + [root] + (loop [pending [root] + visited #{}] + (when-let [dir (peek pending)] + (let [pending (pop pending) + real (when (directory? dir) (fs/realpathSync dir))] + (if (or (nil? real) (contains? visited real)) + (recur pending visited) + (let [parent (path/basename (path/dirname dir)) + base (path/basename dir) + names (array-seq (fs/readdirSync dir))] + (when (= "feature-comparisons" parent) + (unlink-if-present (path/join dir "feature-comparison.log")) + (unlink-if-present (path/join dir "feature-comparison.transit"))) + (when (= "configs" parent) + (unlink-if-present (path/join dir "run.log")) + (unlink-if-present (path/join dir "run.transit"))) + (when (= "comparisons" base) + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".transit")] + (unlink-if-present (path/join dir file)))) + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".png")] + (unlink-if-present (path/join dir file))) + (recur (into pending (map #(path/join dir %) names)) + (conj visited real)))))))) + +(defn purge + "Unlinks known outputs from :builds, :runs, :comparisons or all of them (:all). + + The caller must be idle: no build, server, run or comparison may still own + these paths. Follows directory symlinks; keeps directories and unrelated + files." + [root category] + (let [managed (managed-root root)] + (case category + :builds (purge-builds (path/join managed "builds")) + :runs (purge-runs (path/join managed "runs")) + :comparisons (purge-feature-comparisons + (path/join managed "feature-comparisons")) + :all (do + (purge-builds (path/join managed "builds")) + (purge-runs (path/join managed "runs")) + (purge-feature-comparisons + (path/join managed "feature-comparisons"))) + (throw (ex-info "invalid renderer artifact category" + {:type ::invalid-category :category category}))))) + +(defn remove-builds + "Unlinks known outputs of selected build IDs. The caller must be idle." + [root ids] + (let [ids (mapv check-id ids) + managed (managed-root root) + builds (path/join managed "builds")] + (doseq [id ids] + (clean-build (path/join builds id))))) diff --git a/frontend/performance/benches/render_wasm/node/storage/runs.cljs b/frontend/performance/benches/render_wasm/node/storage/runs.cljs new file mode 100644 index 0000000000..1dbe608070 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/storage/runs.cljs @@ -0,0 +1,78 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.storage.runs + "Manages saved run destinations, schema:run persistence and diagnostic logs. + + Runs are saved under /target/performance/render-wasm/runs. + + --output-dir selects another destination. Purge cleans files reachable from + the default runs tree, including explicit destinations and directory links." + (:require + ["fs" :as fs] + ["path" :as path] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.result :as result] + [clojure.string :as str])) + +(defn create-home + "Creates one run home and its empty log. The record is written separately + after the plan exists. An explicit directory must not exist; a managed home + uses the frontend root regardless of cwd." + [{:keys [root run-id output-dir remove]}] + (artifacts/check-home-id run-id "invalid run id") + (let [dir (artifacts/check-output-dir + (or output-dir (path/join (artifacts/managed-root root) "runs" run-id)))] + (when (and remove (artifacts/inside-builds? root dir)) + (artifacts/invalid-home "--remove cannot use an output directory inside builds" {:path dir})) + (assoc (artifacts/create-home dir "run" "cannot create run home") :run-id run-id))) + +(defn get-record + "Reads and validates a saved run record. + + Args: + file: Input path, resolved relative to cwd. + options: Optional :role identifying the input within an operation. + + Returns: + The validated run record. + + Throws: + An error retaining its message, type and cause, with :path, :operation + and the supplied :role in ex-data." + ([file] + (get-record file {})) + ([file {:keys [role]}] + (try + (when-not (and (string? file) (fs/existsSync file)) + (throw (ex-info "run record is missing" {:type ::missing-run}))) + (let [record (result/decode (fs/readFileSync file "utf8"))] + (when-not (= :run-record (:type record)) + (throw (ex-info "expected a saved run record" {:type ::invalid-run}))) + record) + (catch :default cause + (throw (errors/with-data + (errors/as-failure cause) + (cond-> {:path file :operation :read-record} + role (assoc :role role)))))))) + +(defn write-record! + "Validates schema:run and atomically replaces the saved file." + [home run] + (files/atomic-write (:record-path home) (result/encode run))) + +(defn screenshot-file + "Returns the relative PNG filename for a case, replacing path separators." + [case-id] + (str (str/replace (result/case-name case-id) #"[/\\]" "-") ".png")) + +(defn screenshot-path + "Returns the relative filename and absolute path under the run home." + [home case-id] + (let [file (screenshot-file case-id)] + {:file file :path (path/join (:dir home) file)})) diff --git a/frontend/performance/benches/render_wasm/node/time.cljs b/frontend/performance/benches/render_wasm/node/time.cljs new file mode 100644 index 0000000000..db0c9aff34 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/time.cljs @@ -0,0 +1,12 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.time) + +(defn now + "Returns the current UTC time as an ISO 8601 string." + [] + (.toISOString (js/Date.))) diff --git a/frontend/performance/benches/render_wasm/quantities.cljs b/frontend/performance/benches/render_wasm/quantities.cljs new file mode 100644 index 0000000000..ea114106ed --- /dev/null +++ b/frontend/performance/benches/render_wasm/quantities.cljs @@ -0,0 +1,25 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.quantities + "Portable descriptions of raw measured quantities.") + +(def schema:description + [:map {:closed true} + [:unit keyword?] + [:description {:optional true} string?]]) + +(def descriptions + {:command/elapsed + {:unit :ms :description "Command core start through its completion endpoint."} + :renderer/call-elapsed + {:unit :ms :description "Elapsed time of each synchronous renderer call."} + :renderer/cache-call-elapsed + {:unit :ms :description "Elapsed time of each synchronous cached renderer call."} + :renderer/prepared-scene-to-full + {:unit :ms :description "Prepared scene render request through the renderer return at Full."} + :camera/finalization-to-full + {:unit :ms :description "Camera finalization through the renderer return at Full."}}) diff --git a/frontend/performance/benches/render_wasm/random.cljs b/frontend/performance/benches/render_wasm/random.cljs new file mode 100644 index 0000000000..2ad63e7a80 --- /dev/null +++ b/frontend/performance/benches/render_wasm/random.cljs @@ -0,0 +1,73 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.random + "Seeded numbers for reproducible scene construction and report resampling. + + A seed is an integer from 0 through 4294967295. Each call to `create` owns a + separate Mulberry32 state. Calling the returned function advances that state + and returns a number in [0,1). Equal seeds and equal numbers of calls produce + equal sequences, including seed zero." + (:require + [app.common.types.color :as clr] + [app.common.uuid :as uuid])) + + +(defn seed? + "Returns true for integers from 0 through 4294967295, inclusive." + [value] + (and (integer? value) (<= 0 value 4294967295))) + +(defn create + "Returns a function that draws seeded numbers from [0, 1) using Mulberry32. + Each function owns its state. A seed outside the uint32 range throws + ::invalid-seed; uint32 means an integer from 0 through 4294967295." + [seed] + (when-not (seed? seed) + (throw (ex-info "random seed must be an integer in [0, 2^32)" + {:type ::invalid-seed :seed seed}))) + (let [state (volatile! (bit-or seed 0))] + (fn [] + (let [s (bit-or (+ @state 0x6d2b79f5) 0) + _ (vreset! state s) + v (js/Math.imul (bit-xor s (unsigned-bit-shift-right s 15)) + (bit-or s 1)) + v (bit-xor v (+ v (js/Math.imul (bit-xor v (unsigned-bit-shift-right v 7)) + (bit-or v 61)))) + v (bit-xor v (unsigned-bit-shift-right v 14))] + (/ (unsigned-bit-shift-right v 0) 4294967296))))) + +(defn round3 + "Rounds to 3 decimal places" + [value] + (/ (js/Math.round (* 1000 value)) 1000)) + + +(defn rng-float + "Draws a float in `[min, max)` from the random source." + [rng min max] + (+ min (* (rng) (- max min)))) + +(defn rng-int + "Draws an integer in `[min, max)` from the random source." + [rng min max] + (js/Math.floor (rng-float rng min max))) + +(defn rng-uuid + "Draws a deterministic uuid from two 32-bit values of the random source." + [rng] + (uuid/custom (rng-int rng 0 4294967296) + (rng-int rng 0 4294967296))) + +(defn rng-hex-color + "Draws a hex color from three RGB channels in [0,256)." + [rng] + (clr/rgb->hex [(rng-int rng 0 256) (rng-int rng 0 256) (rng-int rng 0 256)])) + +(defn rng-opacity + "Draws opacity in [0.1,0.999], rounded to three decimal places." + [rng] + (min 0.999 (round3 (rng-float rng 0.1 1)))) diff --git a/frontend/performance/benches/render_wasm/report/analysis.cljs b/frontend/performance/benches/render_wasm/report/analysis.cljs new file mode 100644 index 0000000000..ef2e11c958 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/analysis.cljs @@ -0,0 +1,241 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.analysis + "Calculates statistics and median changes for benchmark measurements. + + summarize/summarize-metric passes one vector to `statistics`. An attempt is + one execution of a benchmark case. Each value represents one attempt, including + a batch average: the attempt's total divided by its number of inner calls. + compare/compare-runs passes baseline and candidate vectors to changes. + Those callers exclude warmups, failed attempts and invalid measurements, + and supply already validated finite values and resolved settings. + + Report fields use maps: {:value number} contains a calculated number; + {:lower number :upper number} contains interval bounds; {:reason keyword} + explains why a valid input cannot supply that result. For example, + :insufficient-observations can include :minimum, the required sample size. + This contract preserves unavailable fields through Transit and lets + report.format display the reason. quantile and resolve-settings reject invalid + arguments at their boundaries. + + Resampling draws a sample of the original size, allowing repeated values. + Repeating this process gives medians whose central :confidence fraction + defines a percentile confidence interval. These intervals describe the + supplied measurements; evolving caches limit their interpretation." + (:require + [app.common.schema :as sm] + [benches.render-wasm.random :as random])) + +(def defaults + "Default settings for report calculations. + :draws is the number of replacement samples; :seed controls their generator. + :confidence is the fraction between interval bounds. :min-ci, :min-p95 and + :min-p99 set the required observation counts for those results." + {:draws 2000 + :seed 0x70656e70 + :confidence 0.95 + :min-ci 10 + :min-p95 100 + :min-p99 500}) + +(def schema:settings + "Valid settings for sample draws, seed and report limits. + The seed is an integer in [0, 2^32); confidence lies strictly between 0 and 1. + Draw and minimum observation counts are positive integers." + [:map {:closed true} + [:draws [:int {:min 1 :max 2147483647}]] + [:seed [:fn random/seed?]] + [:confidence [:fn #(and (number? %) (js/Number.isFinite %) (< 0 % 1))]] + [:min-ci [:int {:min 1}]] + [:min-p95 [:int {:min 1}]] + [:min-p99 [:int {:min 1}]]]) + +(defn resolve-settings + "Returns defaults merged with supplied overrides, or defaults for nil. + Accepts a complete settings map too. Invalid values or unknown keys throw + ::invalid-settings. Callers store this resolved map with their results." + [overrides] + (let [settings (merge defaults overrides)] + (when-not (sm/validate schema:settings settings) + (throw (ex-info "invalid analysis settings" {:type ::invalid-settings + :settings settings}))) + settings)) + +(defn- finite-number? + "Returns true for numbers other than NaN and positive or negative infinity." + [value] + (and (number? value) (js/Number.isFinite value))) + +(defn- observation-vector + "Returns a vector of supplied finite measurements, or an empty vector for nil. + A non-sequence or any non-finite/non-numeric entry throws ::invalid-observations. + Report callers exclude unusable raw entries before requesting calculations." + [values] + (when-not (and (or (nil? values) (sequential? values)) + (every? finite-number? values)) + (throw (ex-info "observations must be a sequence of finite numbers" + {:type ::invalid-observations :values values}))) + (vec values)) + +(defn- sorted-quantile + "Returns an interpolated value from a nonempty vector sorted in ascending order. + Probability 0 selects the minimum, 1 the maximum and 0.5 the median. + At position (count - 1) * probability, a fractional index gives the weighted + average of its neighboring values. Callers validate the probability." + [sorted probability] + (let [position (* (dec (count sorted)) probability) + lower (js/Math.floor position) + upper (js/Math.ceil position) + weight (- position lower)] + (+ (* (- 1 weight) (nth sorted lower)) + (* weight (nth sorted upper))))) + +(defn quantile + "Returns an interpolated quantile as a number, or nil for an empty sequence. + Probability 0 selects the minimum, 1 the maximum and 0.5 the median. + For example, [1 2 3 4] at probability 0.25 gives 1.75. + Values must form a sequence of finite numbers. Probability must be a finite + number in [0, 1]. Invalid arguments throw ::invalid-observations or + ::invalid-probability; report callers handle unavailable results separately." + [values probability] + (when-not (and (finite-number? probability) (<= 0 probability 1)) + (throw (ex-info "quantile probability must be a finite number in [0, 1]" + {:type ::invalid-probability :probability probability}))) + (let [values (observation-vector values)] + (when (seq values) + (sorted-quantile (vec (sort values)) probability)))) + +(defn- number-result + "Returns {:value value} for a finite calculated number. + Non-finite arithmetic yields {:reason :undefined-estimate}, so a report can + retain the field and explain why it has no number." + [value] + (if (finite-number? value) + {:value value} + {:reason :undefined-estimate})) + +(defn- resampled-median + "Returns the median of one replacement sample drawn from a nonempty vector. + The sample has the same size as values; repeated indices are allowed. + rng is a function that draws a number from [0, 1) on each call." + [values rng] + (let [n (count values) + sample (vec (sort (repeatedly n #(nth values (js/Math.floor (* n (rng)))))))] + (sorted-quantile sample 0.5))) + +(defn- interval + "Returns bounds containing the central confidence fraction of calculated draws. + For confidence 0.95, the bounds are the 0.025 and 0.975 quantiles. + Empty or non-finite draws yield {:reason :undefined-bootstrap-estimate}; + callers retain that reason rather than discarding an undefined draw." + [values confidence] + (if (and (seq values) (every? finite-number? values)) + (let [sorted (vec (sort values)) + tail (/ (- 1 confidence) 2) + lower (sorted-quantile sorted tail) + upper (sorted-quantile sorted (- 1 tail))] + (if (and (finite-number? lower) (finite-number? upper)) + {:lower lower :upper upper} + {:reason :undefined-bootstrap-estimate})) + {:reason :undefined-bootstrap-estimate})) + +(defn- quantile-result + "Returns a quantile in {:value number} when the sorted sample reaches minimum. + Otherwise returns {:reason :insufficient-observations :minimum minimum}. + Probability selects the quantile, such as 0.95 for p95." + [sorted probability minimum] + (if (< (count sorted) minimum) + {:reason :insufficient-observations :minimum minimum} + (number-result (sorted-quantile sorted probability)))) + +(defn statistics + "Returns statistics for one vector of finite measurement values. + + `values` must be an already validated vector of finite measurements, and + `settings` must be the complete map returned by `resolve-settings`. + Callers enforce these preconditions. + `:count` is the number of original measurements, including each batch average + once. Numeric fields contain {:value number}; unavailable fields contain + {:reason keyword}, with :minimum for an observation requirement. + + :sample-sd is sqrt(sum((x - mean)^2) / (count - 1)). + :mad is the median absolute distance from the median, without scaling. + :p95 and :p99 are observed 0.95 and 0.99 quantiles. :median-ci contains + bounds from repeatedly drawing replacement samples and taking their medians. + Empty input retains every field with :no-valid-observations. + Insufficient counts and undefined arithmetic retain reasons." + [values settings] + (let [{:keys [draws seed confidence min-ci min-p95 min-p99]} settings + n (count values)] + (if (zero? n) + (let [absent {:reason :no-valid-observations}] + {:count 0 :mean absent :median absent :min absent :max absent + :sample-sd absent :mad absent :median-ci absent :p95 absent :p99 absent}) + (let [sorted (vec (sort values)) + mean (reduce (fn [total x] (+ total (/ x n))) 0 values) + median (sorted-quantile sorted 0.5) + rng (random/create seed)] + {:count n + :mean (number-result mean) + :median (number-result median) + :min (number-result (first sorted)) + :max (number-result (peek sorted)) + :sample-sd (if (= n 1) + {:reason :insufficient-observations :minimum 2} + (number-result (js/Math.sqrt + (reduce (fn [total x] + (+ total (/ (* (- x mean) (- x mean)) (dec n)))) + 0 values)))) + :mad (number-result (sorted-quantile (vec (sort (map #(js/Math.abs (- % median)) values))) 0.5)) + :median-ci (if (< n min-ci) + {:reason :insufficient-observations :minimum min-ci} + (interval (vec (repeatedly draws #(resampled-median values rng))) + confidence)) + :p95 (quantile-result sorted 0.95 min-p95) + :p99 (quantile-result sorted 0.99 min-p99)})))) + +(defn changes + "Returns candidate-minus-baseline changes between the two observed medians. + Inputs must be already validated vectors of finite measurements; settings + must be the complete map returned by resolve-settings. Callers enforce these + preconditions. :absolute and :percentage contain + {:value number}; :absolute-ci and :percentage-ci contain {:lower number :upper number}. + Any unavailable result carries {:reason keyword} instead. + + Each interval draw samples both inputs independently, allowing repeated + measurements, then calculates the change between those two sample medians. + Both original counts must reach :min-ci. A zero observed baseline suppresses + percentages; any zero resampled baseline suppresses the whole percentage + interval. Empty inputs or undefined arithmetic retain reasons. + compare/compare-runs calls this once per shared metric." + [baseline candidate settings] + (let [{:keys [draws seed confidence min-ci]} settings] + (if (or (empty? baseline) (empty? candidate)) + {:absolute {:reason :no-valid-observations} + :percentage {:reason :no-valid-observations} + :absolute-ci {:reason :no-valid-observations} + :percentage-ci {:reason :no-valid-observations}} + (let [b (sorted-quantile (vec (sort baseline)) 0.5) + c (sorted-quantile (vec (sort candidate)) 0.5) + supported (and (>= (count baseline) min-ci) (>= (count candidate) min-ci)) + rng (random/create seed) + samples (when supported + (vec (repeatedly draws + (fn [] [(resampled-median baseline rng) + (resampled-median candidate rng)])))) + low-count {:reason :insufficient-observations :minimum min-ci}] + {:absolute (number-result (- c b)) + :percentage (if (zero? b) {:reason :zero-baseline} (number-result (* 100 (/ (- c b) b)))) + :absolute-ci (if supported + (interval (mapv (fn [[b c]] (- c b)) samples) confidence) + low-count) + :percentage-ci (cond + (zero? b) {:reason :zero-baseline} + (not supported) low-count + (some (fn [[b _]] (zero? b)) samples) {:reason :zero-bootstrap-baseline} + :else (interval (mapv (fn [[b c]] (* 100 (/ (- c b) b))) samples) + confidence))})))) diff --git a/frontend/performance/benches/render_wasm/report/compare.cljs b/frontend/performance/benches/render_wasm/report/compare.cljs new file mode 100644 index 0000000000..904ebb48b3 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/compare.cljs @@ -0,0 +1,281 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.compare + "Offline comparisons of paired cases from two saved schema:run records. + + Matching case identifiers select default pairs; explicit pairs only select + cases. Permissions acknowledge recorded differences without hiding them." + (:require + [benches.render-wasm.report.analysis :as analysis] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [clojure.set :as set] + [clojure.string :as str])) + +(defn resolve-case + "Resolves an identifier or exact name only from the saved plan. + Unknown or ambiguous selectors report the available names for that file." + [run selector] + (let [ids (mapv :id (get-in run [:plan :cases])) + matches (if (some #(= selector %) ids) + [selector] + (filterv #(= selector (result/case-name %)) ids))] + (if (= 1 (count matches)) + (first matches) + (throw (ex-info (str "cannot resolve saved case " (pr-str selector) + "; available cases: " (str/join ", " (map result/case-name ids))) + {:type ::unknown-case :selector selector :available (mapv result/case-name ids)}))))) + +(defn selector-pair + "Resolves baseline and candidate selectors into an extensible pair list." + [baseline candidate [baseline-selector candidate-selector]] + [[(resolve-case baseline baseline-selector) (resolve-case candidate candidate-selector)]]) + +(defn- difference + "Returns one compatibility difference when recorded values disagree." + [field baseline candidate] + (when (not= baseline candidate) + {:field field :baseline baseline :candidate candidate :waived? false})) + +(defn- preparations + "Returns chronological preparation evidence for an exact saved case." + [run id] + (filterv #(= id (:case-id %)) (:preparations run))) + +(defn- preparation-facts + "Collects distinct recorded fingerprints or graphics, independent of prep IDs." + [run id field] + (set (map field (preparations run id)))) + +(defn- scene-settings + "Selects scene identity and generation inputs, separate from execution." + [definition] + (select-keys definition [:scene :scene-seed :params])) + +(defn- execution-settings + "Selects view, context and preparation settings for execution compatibility." + [definition] + (select-keys definition [:view :context :preparation])) + +(defn- evidence-differences + "Checks cross-run equality plus missing or varying preparation evidence." + [field baseline candidate] + (concat + (when-let [d (difference field baseline candidate)] [d]) + (when (or (not= 1 (count baseline)) (not= 1 (count candidate)) + (contains? baseline nil) (contains? candidate nil)) + [{:field [field :incomplete-or-varying-evidence] + :baseline baseline :candidate candidate :waived? false}]))) + +(defn- feature-names? + "Checks the vector-of-strings shape of recorded or expected feature names." + [features] + (and (vector? features) (every? string? features))) + +(defn- get-effective-feature-evidence + "Returns comparison values and eligibility for automatic feature acknowledgment. + Missing, nil and explicitly empty features remain distinct. Collections compare + as sets, but only saved vectors of strings qualify for automatic acknowledgment." + [run] + (let [features (get-in run [:metadata :build :functional :features] ::missing-features)] + {:value (if (coll? features) (set features) features) + :valid (feature-names? features)})) + +(defn- pair-differences + "Checks saved inputs, graphics, environment, build, sampling and metric definitions." + [baseline candidate b c shared] + (let [bdef (some #(when (= b (:id %)) %) (get-in baseline [:plan :cases])) + cdef (some #(when (= c (:id %)) %) (get-in candidate [:plan :cases]))] + (vec + (concat + (keep identity + [(difference :scene (scene-settings bdef) (scene-settings cdef)) + (difference :execution (execution-settings bdef) (execution-settings cdef)) + (difference :environment (get-in baseline [:metadata :environment]) + (get-in candidate [:metadata :environment])) + (difference :effective-features (:value (get-effective-feature-evidence baseline)) + (:value (get-effective-feature-evidence candidate))) + (difference :functional-build (dissoc (get-in baseline [:metadata :build :functional]) :features) + (dissoc (get-in candidate [:metadata :build :functional]) :features)) + (difference :build-environment (get-in baseline [:metadata :build :environment]) + (get-in candidate [:metadata :build :environment])) + (difference :warmups (get-in baseline [:plan :warmups]) + (get-in candidate [:plan :warmups])) + (difference :timeout-policy (get-in baseline [:plan :timeout-policy]) + (get-in candidate [:plan :timeout-policy])) + (difference :scored? (get-in baseline [:metadata :scored?]) + (get-in candidate [:metadata :scored?]))]) + (evidence-differences :fingerprint (preparation-facts baseline b :fingerprint) + (preparation-facts candidate c :fingerprint)) + (evidence-differences :effective-graphics (preparation-facts baseline b :effective-graphics) + (preparation-facts candidate c :effective-graphics)) + (mapcat (fn [metric-id] + (let [bdefs (map #(dissoc % :description) (summary/definitions baseline b metric-id)) + cdefs (map #(dissoc % :description) (summary/definitions candidate c metric-id))] + (evidence-differences [:metric metric-id] (set bdefs) (set cdefs)))) + shared))))) + +(defn- feature-change-evidence? + "Checks feature facts against saved effective features and requested selections. + Both selections must record one off-to-on requested toggle with equal default + policy. Effective transitive additions/removals remain unrestricted, but must + match the supplied facts." + [baseline candidate {:keys [feature baseline-features candidate-features added removed]}] + (let [b (get-effective-feature-evidence baseline) + c (get-effective-feature-evidence candidate) + b-request (get-in baseline [:metadata :build :provenance :feature-selection]) + c-request (get-in candidate [:metadata :build :provenance :feature-selection])] + (and (:valid b) (:valid c) + (string? feature) + (every? feature-names? + [baseline-features candidate-features added removed]) + (map? b-request) (map? c-request) + (boolean? (:default-features? b-request)) + (boolean? (:default-features? c-request)) + (= (:default-features? b-request) (:default-features? c-request)) + (feature-names? (:features b-request)) + (feature-names? (:features c-request)) + (not (contains? (set (:features b-request)) feature)) + (= (conj (set (:features b-request)) feature) (set (:features c-request))) + (= (:value b) (set baseline-features)) + (= (:value c) (set candidate-features)) + (= (set/difference (:value c) (:value b)) (set added)) + (= (set/difference (:value b) (:value c)) (set removed))))) + +(defn- expected-feature-change? + [baseline candidate expected] + (and (feature-change-evidence? baseline candidate expected) + (contains? (set (:added expected)) (:feature expected)))) + +(defn- acknowledge-difference + "Retains a mismatch and records its applicable rule, if any. + Automatic feature evidence and scoped permissions take precedence over all. + Fingerprint content is scene data; incomplete/varying evidence is not." + [{:keys [field] :as mismatch} permissions expected-feature-change?] + (let [rule (cond + (and (= :effective-features field) expected-feature-change?) :feature-toggle + (and (:allow-build-differences permissions) + (contains? #{:effective-features :functional-build :build-environment} field)) :build + (and (:allow-scene-differences permissions) + (contains? #{:scene :fingerprint} field)) :scene + (:allow-all-differences permissions) :all)] + (cond-> mismatch + rule (assoc :waived? true :rule rule)))) + +(defn- side-report + "Reports one side with saved definitions, setup evidence and sample accounting." + [run id shared analysis] + (summary/case-report run id shared analysis)) + +(defn- metric-changes + "Keeps diagnostic changes unavailable for differing or varying definitions. + Only all-differences permits comparing their valid raw values without conversion." + [baseline candidate b c id mismatches allow-all-differences analysis] + (if (and (not allow-all-differences) + (some #(contains? #{[:metric id] [[:metric id] :incomplete-or-varying-evidence]} + (:field %)) mismatches)) + {:absolute {:reason :inconsistent-metric-definitions} + :percentage {:reason :inconsistent-metric-definitions} + :absolute-ci {:reason :inconsistent-metric-definitions} + :percentage-ci {:reason :inconsistent-metric-definitions}} + (analysis/changes (:values (summary/observations baseline b id)) + (:values (summary/observations candidate c id)) analysis))) + +(defn- compare-pair + "Compares shared numeric observations after resolving the compatibility policy." + [baseline candidate [b c] explicit? permissions expected-feature-change? on-unallowed-difference analysis] + (let [b (resolve-case baseline b) + c (resolve-case candidate c) + candidate-ids (set (summary/metric-ids candidate c)) + shared (filterv #(contains? candidate-ids %) (summary/metric-ids baseline b)) + mismatches (mapv #(acknowledge-difference % permissions expected-feature-change?) + (pair-differences baseline candidate b c shared)) + allow-all-differences (:allow-all-differences permissions)] + (when (empty? shared) + (throw (ex-info "selected cases have no shared metric identifiers" + {:type ::no-shared-metrics :baseline b :candidate c}))) + (when (and (not= :record on-unallowed-difference) (some #(not (:waived? %)) mismatches)) + (throw (ex-info "incompatible saved runs" + {:type ::incompatible :baseline b :candidate c :mismatches mismatches}))) + {:explicit? explicit? + :mismatches mismatches + :baseline (side-report baseline b shared analysis) + :candidate (side-report candidate c shared analysis) + :metrics (into {} + (map (fn [id] + [id (metric-changes baseline candidate b c id mismatches + allow-all-differences analysis)])) + shared)})) + +(defn- provenance + "Retains run identity and facts without using Git/run/time as compatibility gates." + [run] + (select-keys run [:run-id :started-at :ended-at :metadata :termination])) + +(defn compare-runs + "Creates a schema:comparison from constructed or decoded schema:run records. + + Inputs must already satisfy schema:run. Encoding validates the returned + comparison at the boundary. + + Default pairs use identical identifiers. Explicit pairs select cases without + changing compatibility checks. + + - `:pairs` accepts an extensible vector of explicit [baseline candidate] pairs. + - `:allow-build-differences` acknowledges build settings and effective features. + - `:allow-scene-differences` acknowledges scene inputs and fingerprint content. + It never acknowledges incomplete or varying evidence or execution settings. + - `:allow-all-differences` acknowledges any compatibility mismatch and permits + mixed-definition statistics on valid raw observations, without conversions. + Scoped permissions combine by union. Every mismatch retains its values, + legacy `:waived?` field and, only when acknowledged, its `:rule`. + - `:expected-feature-change` is planner evidence with string `:feature` and + vectors `:baseline-features`, `:candidate-features`, `:added`, `:removed`. + Automatic acknowledgment requires saved effective vectors of strings, exact + sets/deltas with that feature added, and saved requested selections differing + only by that toggle, with the same recorded default-feature policy. + - `:edge` attaches checked feature-comparison context: `:from`, `:to`, and the five expected + feature fields. When both options are supplied, their feature facts must agree. + The ordinary encoding boundary validates the saved edge's shape. + - Internal `:on-unallowed-difference :record` returns diagnostic reports instead + of throwing for unacknowledged differences. It keeps strict statistics. + - `:analysis` resolves from method defaults, independent of each run's defaults." + ([baseline candidate] (compare-runs baseline candidate {})) + ([baseline candidate {:keys [pairs allow-build-differences allow-scene-differences + allow-all-differences expected-feature-change edge + on-unallowed-difference analysis]}] + (let [explicit? (some? pairs) + candidate-ids (set (map :id (get-in candidate [:plan :cases]))) + pairs (or pairs (->> (get-in baseline [:plan :cases]) + (map :id) + (filter #(contains? candidate-ids %)) + (mapv #(vector % %)))) + analysis (analysis/resolve-settings analysis) + permissions {:allow-build-differences (boolean allow-build-differences) + :allow-scene-differences (boolean allow-scene-differences) + :allow-all-differences (boolean allow-all-differences)} + expected? (expected-feature-change? baseline candidate expected-feature-change)] + (when (and (some? edge) (not (feature-change-evidence? baseline candidate edge))) + (throw (ex-info "edge context disagrees with saved run feature evidence" + {:type ::invalid-edge :edge edge}))) + (when (and (some? edge) (some? expected-feature-change) + (not= expected-feature-change + (select-keys edge [:feature :baseline-features :candidate-features :added :removed]))) + (throw (ex-info "edge context disagrees with expected feature facts" + {:type ::invalid-edge :edge edge :expected-feature-change expected-feature-change}))) + (when (or (empty? pairs) (not (every? #(and (vector? %) (= 2 (count %))) pairs))) + (throw (ex-info "comparison needs saved case pairs" + {:type ::invalid-pairs + :baseline (mapv (comp result/case-name :id) (get-in baseline [:plan :cases])) + :candidate (mapv (comp result/case-name :id) (get-in candidate [:plan :cases]))}))) + (cond-> {:type :comparison :analysis analysis + :baseline (provenance baseline) :candidate (provenance candidate) + :permissions permissions + :source-coverage summary/source-coverage :interpretation summary/interpretation + :pairs (mapv #(compare-pair baseline candidate % explicit? permissions expected? + on-unallowed-difference analysis) pairs)} + (some? edge) (assoc :edge edge))))) diff --git a/frontend/performance/benches/render_wasm/report/format.cljs b/frontend/performance/benches/render_wasm/report/format.cljs new file mode 100644 index 0000000000..d586167e30 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/format.cljs @@ -0,0 +1,357 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.format + "Text formatting for run summaries and saved comparisons. + + Text leads with observed median, bootstrap interval and distinct counts. + Full statistics and acknowledged differences include units and phase identity." + (:require + [benches.render-wasm.errors :as errors] + [benches.render-wasm.result :as result] + [clojure.string :as str])) + +(defn- number->text + "Formats safe integers in decimal form and other estimates with six significant digits." + [value] + (if (js/Number.isSafeInteger value) + (str value) + (.toPrecision value 6))) + +(defn- estimate->text + "Formats an estimate, interval or suppression reason." + [estimate] + (cond + (contains? estimate :value) (number->text (:value estimate)) + (contains? estimate :lower) (str "[" (number->text (:lower estimate)) ", " + (number->text (:upper estimate)) "]") + :else (str "unavailable (" (some-> (:reason estimate) name) + (when-some [n (:minimum estimate)] (str "; need " n)) ")"))) + +(defn- counts->text + "Prints valid, invalid, absent, failed, unattempted and excluded warmup counts." + [{:keys [valid invalid absent failed unattempted warmup invocations raw-values]}] + (str "valid=" valid " invalid=" invalid " absent=" absent " failed=" failed + " unattempted=" unattempted " warmup=" warmup + " invocations=" invocations " raw-values=" raw-values " n=" valid)) + +(defn metric->text + "Formats statistics across phase execution sums with their units and counts. + + Args: + id: vector of phase, command or global scope, and metric type. + report: quantity descriptions, accounting counts, statistics and diagnostics. + + Returns: + Text with observed estimates, support limits and raw-value counts." + [id {:keys [definitions counts statistics diagnostics]}] + (str (pr-str id) " sum per execution; median " (estimate->text (or (:median statistics) statistics)) + " CI " (estimate->text (or (:median-ci statistics) statistics)) " | " (counts->text counts) + "\n definitions " (pr-str definitions) + (when (seq diagnostics) (str "\n diagnostics " (pr-str diagnostics))) + (if-let [reason (:reason statistics)] + (str "\n unavailable (" (name reason) ")") + (str "\n mean " (estimate->text (:mean statistics)) + " min " (estimate->text (:min statistics)) " max " (estimate->text (:max statistics)) + " sample SD " (estimate->text (:sample-sd statistics)) + " MAD " (estimate->text (:mad statistics)) + " p95 " (estimate->text (:p95 statistics)) " p99 " (estimate->text (:p99 statistics)))))) + +(defn- quoted-detail + [value] + (errors/format-value value 1024)) + +(defn- identifier->text + [value] + (errors/format-value value 1024)) + +(def ^:private process-detail-keys + [:command :pid :pgid :signals :profile-path :fatal :unresolved-process]) + +(defn format-error-entry + "Formats an error with its location and available process details. + + Args: + entry: A projected error occurrence. + + Returns: + Bounded error text and process details when a group may still be running." + [{:keys [phase operation message path] :as entry}] + (str (when operation (str (errors/format-value operation 1024) " ")) + (quoted-detail phase) + (when (some? message) (str ": " (quoted-detail message))) + (when path (str "\n Path " (quoted-detail path))) + (when (:unresolved-process entry) + (apply str + (map (fn [[key value]] (str "\n " (name key) " " (identifier->text value))) + (select-keys entry process-detail-keys)))))) + +(defn- failure->text + [{:keys [failure reason]}] + (if failure + (if-let [n (:unacknowledged-differences failure)] + (str n " unacknowledged " (if (= n 1) "difference" "differences")) + (if-let [entries (seq (:errors failure))] + (let [header (apply dissoc failure process-detail-keys) + omitted (drop 8 entries) + unresolved (filter :unresolved-process omitted)] + (str (if (not= (errors/bounded-text (:message failure) 1024) (:message (first entries))) + (str (format-error-entry header) "\n ") + (when (not= (select-keys failure [:phase :operation :path]) + (select-keys (first entries) [:phase :operation :path])) + (str "Context " (format-error-entry (dissoc header :message)) "\n "))) + (str/join "\n " (map format-error-entry (take 8 entries))) + (when (seq omitted) + (str "\n " (count omitted) " more errors recorded")) + (when-let [entry (first unresolved)] + (str "\n " (count unresolved) " unresolved process errors in the saved record; first: " + (format-error-entry entry))))) + (format-error-entry failure))) + (quoted-detail reason))) + +(defn- same-diagnostic? + [left right] + (= (dissoc (errors/project-diagnostic left) :status :partial) + (dissoc (errors/project-diagnostic right) :status :partial))) + +(defn- case-failures + [{:keys [case-id preparations failed-attempts]}] + (let [preparations (filter :failure preparations) + by-id (into {} (map (juxt :preparation-id identity)) preparations)] + (concat + (map (fn [preparation] + (assoc (select-keys preparation [:preparation-id :warmup? :failure]) + :kind :preparation :case-id case-id)) + preparations) + (keep (fn [{:keys [preparation-id failure phases] :as attempt}] + (let [preparation (get by-id preparation-id) + copied? (and preparation + (every? #(= :preparation (:phase %)) phases) + (same-diagnostic? failure (:failure preparation)))] + (when (and failure (not copied?)) + (assoc (select-keys attempt [:attempt-id :preparation-id :warmup? :failure :partial]) + :kind :attempt :case-id case-id)))) + failed-attempts)))) + +(defn- run-failures + [cases metadata termination] + (let [case-entries (vec (mapcat case-failures cases)) + by-case (group-by :case-id case-entries) + harness (keep-indexed + (fn [index failure] + (let [copied? (and (= :preparation (:stage failure)) + (some #(and (= :preparation (:kind %)) + (same-diagnostic? (:failure %) failure)) + (get by-case (:case-id failure))))] + (when-not copied? + {:kind :harness :index index :stage (:stage failure) + :case-id (:case-id failure) :failure failure}))) + (get-in metadata [:diagnostics :failures])) + recorded (into case-entries harness) + by-owner (group-by :case-id recorded) + outcomes (keep (fn [{:keys [case-id outcome]}] + (when-let [failure (:failure outcome)] + (when-not (some #(same-diagnostic? failure (:failure %)) (get by-owner case-id)) + {:kind :case :case-id case-id :failure failure}))) + cases) + entries (into recorded outcomes)] + (cond-> entries + (and (:failure termination) + (not-any? #(same-diagnostic? (:failure termination) (:failure %)) entries)) + (conj {:kind :termination :failure (:failure termination)})))) + +(defn- diagnostic->text + [failure] + (let [context (errors/project-context failure) + context (if (:unresolved-process failure) (apply dissoc context process-detail-keys) context) + details (merge (select-keys (first (:cause failure)) [:type :code :fn]) + context + (select-keys failure [:reason :deadline-exceeded]))] + (str (failure->text {:failure failure}) + (apply str (map (fn [[key value]] + (str "\n " (name key) " " (identifier->text value))) + (sort-by (comp name key) (filter (comp some? val) details))))))) + +(defn- failure-entry->text + [{:keys [kind case-id preparation-id attempt-id index stage warmup? failure partial]}] + (str " " (case kind + :preparation (str "Preparation " (identifier->text preparation-id)) + :attempt (str "Attempt " attempt-id " (preparation " (identifier->text preparation-id) ")") + :harness (str "Harness " index " " (identifier->text stage)) + :case "Case outcome" + :termination "Termination") + (when (some? case-id) (str " (case " (identifier->text case-id) ")")) + (when warmup? " warmup") + ": " (diagnostic->text failure) + (when (or partial (:partial failure)) "\n Partial evidence recorded"))) + +(defn- failures->text + [heading entries] + (when (seq entries) + (str "\n\n" heading "\n" (str/join "\n" (map failure-entry->text entries))))) + +(defn- screenshots->text + [heading metadata failures cases] + (let [screenshots (get-in metadata [:diagnostics :screenshots]) + names (frequencies (map result/case-name + (distinct (concat (map :case-id cases) (keep :case-id failures))))) + deadlines (group-by #(result/case-name (:case-id %)) + (filter #(and (= :harness (:kind %)) + (= :case-deadline (:stage %)) + (some? (:case-id %))) failures))] + (when (seq screenshots) + (str "\n\n" heading "\n" + (str/join + "\n" + (map (fn [[case-name {:keys [status message reason file cause]}]] + (let [matches (when (and (= 1 (get names case-name)) (string? message) (seq cause)) + (filter #(and (= message (get-in % [:failure :message])) + (= cause (get-in % [:failure :cause]))) + (get deadlines case-name))) + related (when (= 1 (count matches)) (first matches))] + (str " " (identifier->text case-name) ": " + (if related + (str "failed; see Harness " (:index related)) + (case status + :failed "warning" + :ok "ok" + :skipped "skipped" + (identifier->text status))) + (when (and message (not related)) (str "; " (quoted-detail message))) + (when reason (str "; reason " (identifier->text reason))) + (when file (str "\n File " (quoted-detail file)))))) + (sort-by (comp str key) screenshots))))))) + +(defn- case-status->text + "Displays pending slots, phase counts and failed execution identities." + [{:keys [remaining failed-attempts phases outcome]}] + (str " remaining " (pr-str remaining) + (when outcome (str "\n outcome " (pr-str (:outcome outcome)))) + "\n phases " (pr-str phases) + " failed-warmups=" (count (filter :warmup? failed-attempts)) + (when (seq failed-attempts) + (str "\n failed attempts " + (pr-str (mapv #(select-keys % [:attempt-id :preparation-id :warmup? :outcome]) failed-attempts)))))) + +(defn format-run + "Formats run statistics, recorded failures and screenshot diagnostics." + [{:keys [run-id metadata analysis termination source-coverage interpretation cases]}] + (let [failures (run-failures cases metadata termination)] + (str "Run " (pr-str run-id) " " (pr-str (dissoc termination :failure)) + "\nGit " (pr-str (:git metadata)) + "\nAnalysis " (pr-str analysis) + "\n" source-coverage "\n" interpretation + (when (some #(get-in % [:effective-graphics :software]) + (mapcat :preparations cases)) "\nSOFTWARE RENDERER recorded") + (apply str (map (fn [{:keys [case-id metrics] :as case-report}] + (str "\n\nCase " (result/case-name case-id) + (case-status->text case-report) + (apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics)))) + cases)) + (failures->text "Failures" failures) + (screenshots->text "Screenshots" metadata failures cases)))) + +(defn- side->text + "Displays one comparison side's definitions, preparation facts and observations." + [label {:keys [case-id definition preparations metrics] :as side}] + (str "\n" label " " (result/case-name case-id) + (case-status->text side) + "\n case " (pr-str definition) + "\n preparations " (pr-str (mapv #(select-keys % [:preparation-id :warmup? :fingerprint :effective-graphics]) + preparations)) + (apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics)))) + +(defn- configuration-failure->text + [{:keys [slug run build] :as configuration}] + (str " Configuration " (quoted-detail slug) ": " (failure->text configuration) + (cond + run (str "\n Run record " (quoted-detail (:record-path run)) + "\n Log " (quoted-detail (:log-path run))) + build (str "\n Build manifest " (quoted-detail (:manifest-path build)) + "\n Log " (quoted-detail (:log-path build)))))) + +(defn- edge-failure->text + [{:keys [from to feature path] :as edge}] + (str " Comparison " (quoted-detail from) " -> " (quoted-detail to) + " (feature " (quoted-detail feature) "): " (failure->text edge) + (when path (str "\n Report " (quoted-detail path))))) + +(defn format-feature-comparison-failures + "Formats failed configurations and comparisons with artifact references. + + Args: + record: Validated feature-comparison record. + home: Comparison :record-path and :log-path. + + Returns: + Text for up to eight failures in configuration and comparison order, + with an omitted count and references to the complete evidence. String + fields escape C0 controls (U+0000 through U+001F) and use at most + 1024 UTF-16 units. + Returns nil when the record contains no failed entries." + [record home] + (let [configurations (filter #(= :failed (:status %)) (:configurations record)) + edges (filter #(= :failed (:status %)) (:edges record)) + total (+ (count configurations) (count edges)) + entries (concat (map configuration-failure->text configurations) + (map edge-failure->text edges))] + (when (pos? total) + (str "Failures\n" (str/join "\n" (take 8 entries)) + (when (> total 8) (str "\n " (- total 8) " more failures in the saved record")) + "\n Record " (quoted-detail (:record-path home)) + "\n Log " (quoted-detail (:log-path home)))))) + +(defn format-comparison + "Formats selected permissions, acknowledged/unacknowledged differences and changes. + Both saved definitions remain visible. Edge labels describe actual feature + evidence, including equal effective sets, without predicting a timing effect." + [{:keys [baseline candidate analysis permissions edge source-coverage interpretation pairs]}] + (let [side-cases (fn [side] + (let [cases (map side pairs) + by-id (group-by :case-id cases)] + (map #(first (get by-id %)) (distinct (map :case-id cases))))) + baseline-cases (side-cases :baseline) + candidate-cases (side-cases :candidate) + baseline-failures (run-failures baseline-cases (:metadata baseline) (:termination baseline)) + candidate-failures (run-failures candidate-cases (:metadata candidate) (:termination candidate))] + (str "Permissions " (pr-str permissions) "\n" + (when edge + (str "Edge " (:from edge) " -> " (:to edge) " feature " (:feature edge) + (when (= (set (:baseline-features edge)) (set (:candidate-features edge))) + " (effective feature sets identical)") + "\n effective features " (pr-str (select-keys edge [:baseline-features :candidate-features :added :removed])) + "\n")) + (when (some #(get-in % [:effective-graphics :software]) + (mapcat (fn [pair] (concat (get-in pair [:baseline :preparations]) + (get-in pair [:candidate :preparations]))) pairs)) + "SOFTWARE RENDERER recorded\n") + "Baseline " (pr-str (:run-id baseline)) " Git " (pr-str (get-in baseline [:metadata :git])) + "\nBaseline termination " (pr-str (dissoc (:termination baseline) :failure)) + "\nCandidate " (pr-str (:run-id candidate)) " Git " (pr-str (get-in candidate [:metadata :git])) + "\nCandidate termination " (pr-str (dissoc (:termination candidate) :failure)) + "\nAnalysis " (pr-str analysis) "\n" source-coverage "\n" interpretation + (failures->text "Baseline failures" baseline-failures) + (screenshots->text "Baseline screenshots" (:metadata baseline) baseline-failures baseline-cases) + (failures->text "Candidate failures" candidate-failures) + (screenshots->text "Candidate screenshots" (:metadata candidate) candidate-failures candidate-cases) + (apply str + (map (fn [{:keys [baseline candidate mismatches metrics]}] + (str "\n\nPair " (result/case-name (:case-id baseline)) "," + (result/case-name (:case-id candidate)) + (apply str + (map (fn [mismatch] + (str "\n" (if (:waived? mismatch) "ACKNOWLEDGED" "UNACKNOWLEDGED") + " DIFFERENCE " (pr-str mismatch))) mismatches)) + (side->text "Baseline" baseline) + (side->text "Candidate" candidate) + (str/join "\n" + (map (fn [[id change]] + (str "\n" (pr-str id) + " median change " (estimate->text (:absolute change)) + " CI " (estimate->text (:absolute-ci change)) + "; percent " (estimate->text (:percentage change)) + " CI " (estimate->text (:percentage-ci change)))) metrics)))) + pairs))))) diff --git a/frontend/performance/benches/render_wasm/report/summarize.cljs b/frontend/performance/benches/render_wasm/report/summarize.cljs new file mode 100644 index 0000000000..e6c1e8e0c9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/report/summarize.cljs @@ -0,0 +1,238 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.report.summarize + "Offline analysis of raw command histories, grouped within phase executions." + (:require + [app.common.schema :as sm] + [benches.render-wasm.quantities :as quantities] + [benches.render-wasm.report.analysis :as analysis])) + +(def source-coverage + "Recorded inputs and settings checked; runtime-helper identity is unverified. Git SHA and dirty state record provenance.") + +(def interpretation + "Intervals are exploratory and conditional on available executions. Incomplete and timed-out cases can select which observations survive. Caches can correlate attempts; repeat whole runs to investigate findings. Full denotes renderer submission completion. Sums combine only one metric type within one phase execution. Finite clock resolution can produce valid zero durations.") + +(defn case-attempts + "Returns chronological attempts for an exact saved case identifier." + [run case-id] + (filterv #(= case-id (:case-id %)) (:attempts run))) + +(defn get-failed-attempts + "Returns failed warmup and measured attempts with their saved evidence." + [run case-id] + (filterv #(= :failed (:outcome %)) (case-attempts run case-id))) + +(defn executions + "Returns actual phase executions with their eligibility and saved identities. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + phase: preparation, main or reset. + + Returns: + Chronological phase histories with phase and preparation/attempt outcomes." + [run case-id phase] + (let [attempts (case-attempts run case-id)] + (if (= :preparation phase) + (let [definition (some #(when (= case-id (:id %)) %) (get-in run [:plan :cases]))] + (mapv (fn [prep] + (let [block (first (:phases prep))] + {:preparation-id (:preparation-id prep) + :events (:events block) + :warmup? (and (not= :reuse (:context definition)) (:warmup? prep)) + :outcome (:outcome block) + :preparation-outcome (if (:failure prep) :failed :completed)})) + (filter #(and (= case-id (:case-id %)) (seq (:phases %))) (:preparations run)))) + (into [] + (keep (fn [attempt] + (when-let [block (some #(when (= phase (:phase %)) %) (:phases attempt))] + (merge (select-keys attempt [:attempt-id :preparation-id :warmup?]) + {:outcome (:outcome block) :attempt-outcome (:outcome attempt) + :events (:events block)})))) + attempts)))) + +(defn phase-counts + "Counts actual phase executions, including empty histories and excluded work. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + + Returns: + Execution, completion, failure and warmup counts for each phase." + [run case-id] + (into {} + (map (fn [phase] + (let [blocks (executions run case-id phase)] + [phase {:executions (count blocks) + :completed (count (filter #(= :completed (:outcome %)) blocks)) + :failed (count (filter #(= :failed (:outcome %)) blocks)) + :warmup (count (filter :warmup? blocks))}]))) + [:preparation :main :reset])) + +(defn metric-ids + "Returns [phase command-or-global metric-type] keys in first appearance order. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + + Returns: + Distinct report keys, including quantities recorded during failed work." + [run case-id] + (vec (distinct + (for [phase [:preparation :main :reset] + block (executions run case-id phase) + event (:events block) + metric (keys (:metrics event)) + scope [(:command event) :global]] + [phase scope metric])))) + +(defn definitions + "Returns valid saved quantity descriptions for a report key. + + Args: + run: validated saved run. + _case-id: exact saved case identifier. + id: vector of phase, command or global scope, and metric type. + + Returns: + A vector containing the description, or an empty vector for invalid metadata." + [run _case-id [_phase _scope metric]] + (let [description (get-in run [:quantities metric])] + (if (sm/validate quantities/schema:description description) [description] []))) + +(defn- invalid-reason + [value] + (cond + (not (js/Number.isFinite value)) :non-finite + (neg? value) :negative + :else nil)) + +(defn- execution-value + [block scope metric] + (let [events (->> (:events block) + (map-indexed vector) + (filter (fn [[_ event]] + (and (or (= :global scope) (= scope (:command event))) + (contains? (:metrics event) metric))))) + values (vec (mapcat #(get-in (second %) [:metrics metric]) events)) + invalid (vec (for [[event-index event] events + [value-index value] (map-indexed vector (get-in event [:metrics metric])) + :let [reason (invalid-reason value)] + :when reason] + (merge (select-keys block [:attempt-id :preparation-id]) + {:event-index event-index :value-index value-index + :value value :reason reason}))) + total (when (and (seq values) (empty? invalid)) (reduce + 0 values))] + {:invocations (count events) :raw-values (count values) + :diagnostics (cond-> invalid + (and (some? total) (not (js/Number.isFinite total))) + (conj {:reason :non-finite-sum})) + :value total})) + +(defn observations + "Sums one metric type per eligible phase execution. + Invocation and raw-value counts cover eligible executions, including invalid + values. Statistical n counts valid execution sums. Failed attempts exclude + both main and reset. Successful preparation survives later attempt failures. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + id: vector of phase, command or global scope, and metric type. + + Returns: + Valid execution sums, accounting counts and invalidity diagnostics." + [run case-id [phase scope metric :as id]] + (let [description (get-in run [:quantities metric]) + diagnostic (when (empty? (definitions run case-id id)) + {:reason (if (contains? (:quantities run) metric) + :malformed-quantity :unknown-quantity) + :metric metric :description description})] + (reduce + (fn [out {:keys [warmup? outcome attempt-outcome preparation-outcome] :as block}] + (let [{:keys [value invocations raw-values diagnostics]} (execution-value block scope metric) + kind (cond + warmup? :warmup + (or (= :failed outcome) (= :failed attempt-outcome) + (= :failed preparation-outcome)) :failed + (zero? invocations) :absent + (or diagnostic (seq diagnostics)) :invalid + :else :valid) + eligible? (not (#{:warmup :failed} kind))] + (cond-> (update-in out [:counts kind] inc) + eligible? (update-in [:counts :invocations] + invocations) + eligible? (update-in [:counts :raw-values] + raw-values) + eligible? (update :diagnostics into diagnostics) + (= kind :valid) (update :values conj value)))) + {:values [] :diagnostics (if diagnostic [diagnostic] []) + :counts {:valid 0 :invalid 0 :absent 0 :failed 0 :warmup 0 + :invocations 0 :raw-values 0 + :unattempted (if (= :preparation phase) 0 + (get-in run [:remaining case-id :repetitions]))}} + (executions run case-id phase)))) + +(defn summarize-metric + "Computes statistics across execution sums for one phase, scope and metric type. + + Args: + run: validated saved run. + case-id: exact saved case identifier. + id: vector of phase, command or global scope, and metric type. + analysis: resolved statistical settings. + + Returns: + Quantity descriptions, counts, diagnostics and statistics across executions." + [run case-id id analysis] + (let [{:keys [values counts diagnostics]} (observations run case-id id)] + {:definitions (definitions run case-id id) + :statistic :sum :counts counts :diagnostics diagnostics + :statistics (analysis/statistics values analysis)})) + +(defn case-report + "Reports saved case facts and phase quantities using one analysis method. + + Args: + run: validated saved run. + id: exact saved case identifier. + ids: selected report keys. + analysis: resolved statistical settings. + + Returns: + Case definition, raw preparation/failure evidence, counts and metric reports." + [run id ids analysis] + {:case-id id + :definition (some #(when (= id (:id %)) %) (get-in run [:plan :cases])) + :outcome (get-in run [:case-outcomes id]) + :preparations (filterv #(= id (:case-id %)) (:preparations run)) + :phases (phase-counts run id) + :remaining (get-in run [:remaining id]) + :failed-attempts (get-failed-attempts run id) + :metrics (into {} (map (fn [metric-id] + [metric-id (summarize-metric run id metric-id analysis)])) ids)}) + +(defn summarize + "Reports a validated saved run using saved cases and quantity descriptions. + + Args: + run: validated saved run. + overrides: optional statistical settings overriding the saved defaults. + + Returns: + A report with provenance, termination and separate analyses for each case." + ([run] (summarize run nil)) + ([run overrides] + (let [analysis (analysis/resolve-settings (merge (:analysis run) overrides))] + {:run-id (:run-id run) :metadata (:metadata run) :analysis analysis + :termination (:termination run) + :source-coverage source-coverage :interpretation interpretation + :cases (mapv (fn [{:keys [id]}] + (case-report run id (metric-ids run id) analysis)) + (get-in run [:plan :cases]))}))) diff --git a/frontend/performance/benches/render_wasm/result.cljs b/frontend/performance/benches/render_wasm/result.cljs new file mode 100644 index 0000000000..5fd87b64d4 --- /dev/null +++ b/frontend/performance/benches/render_wasm/result.cljs @@ -0,0 +1,696 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.result + "Schemas for experiments and reports." + (:require + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.quantities :as quantities] + [benches.render-wasm.report.analysis :as analysis])) + +(def schema:event + "One measured command invocation with its ordered raw values." + [:map {:closed true} + [:command qualified-keyword?] + [:metrics [:map-of qualified-keyword? [:vector {:min 1} number?]]] + [:trace {:optional true} :any]]) + +(def schema:phase + "Raw command history for one phase execution, including partial failures." + [:map {:closed true} + [:phase [:enum :preparation :main :reset]] + [:preparation-id some?] + [:outcome [:enum :completed :failed]] + [:events [:vector schema:event]] + [:diagnostics {:optional true} [:vector :any]]]) + +(def schema:fingerprint + "The content digest of the actual validated starting snapshot." + [:map {:closed true} + [:algorithm [:= :sha-256]] + [:digest [:re #"^[0-9a-f]{64}$"]]]) + +(def schema:preparation + "Setup diagnostics and callback measurements for one owned scene." + [:map {:closed true} + [:preparation-id some?] + [:case-id some?] + [:warmup? boolean?] + [:fingerprint {:optional true} schema:fingerprint] + [:effective-graphics {:optional true} map?] + [:diagnostics map?] + [:phases [:vector schema:phase]] + [:failure {:optional true} errors/schema:diagnostic]]) + +(def schema:attempt + "One chronological execution; failure consumes a slot and may retain evidence." + [:map {:closed true} + [:attempt-id [:int {:min 0}]] + [:case-id some?] + [:preparation-id [:maybe some?]] + [:warmup? boolean?] + [:outcome [:enum :completed :failed]] + [:phases [:vector schema:phase]] + [:failure {:optional true} errors/schema:diagnostic] + [:partial {:optional true} map?]]) + +(def schema:metadata + "Git, functional build settings, environment and diagnostic provenance." + [:map {:closed true} + [:git [:map [:sha [:maybe string?]] [:dirty [:maybe boolean?]]]] + [:build [:map + [:functional map?] + [:provenance {:optional true} map?] + [:environment {:optional true} [:map-of :string :string]]]] + [:environment map?] + [:scored? boolean?] + [:diagnostics {:optional true} + [:map [:failures {:optional true} [:vector errors/schema:diagnostic]]]]]) + +(def schema:termination + "A run's current state or final reason, with optional diagnostic details." + [:map + [:reason keyword?] + [:failure {:optional true} errors/schema:diagnostic]]) + +(def schema:case-outcome + [:map {:closed true} + [:outcome [:enum :completed :failed :timed-out :interrupted]] + [:failure {:optional true} errors/schema:diagnostic]]) + +(def schema:run + "schema:run stores saved case definitions, independent of any registry. + Build :functional holds compatibility settings; :provenance holds identities." + [:map {:closed true} + [:type [:= :run-record]] + [:run-id some?] + [:started-at string?] + [:ended-at [:maybe string?]] + [:plan [:map {:closed true} + [:cases [:vector [:map [:id some?]]]] + [:warmups [:int {:min 0}]] + [:repetitions [:int {:min 0}]] + [:timeout-policy [:map {:closed true} [:case-ms [:int {:min 1}]]]]]] + [:metadata schema:metadata] + [:analysis analysis/schema:settings] + [:quantities [:map-of qualified-keyword? :any]] + [:preparations [:vector schema:preparation]] + [:attempts [:vector schema:attempt]] + [:case-outcomes [:map-of :any schema:case-outcome]] + [:remaining [:map-of :any [:map {:closed true} + [:warmups [:int {:min 0}]] + [:repetitions [:int {:min 0}]]]]] + [:termination schema:termination]]) + +(def schema:estimate + "A finite computed estimate or an explicit suppression reason." + [:or + [:map {:closed true} [:value [:fn #(and (number? %) (js/Number.isFinite %))]]] + [:map {:closed true} [:reason keyword?] [:minimum {:optional true} [:int {:min 1}]]]]) + +(def schema:interval + "Finite ordered percentile bounds or an explicit suppression reason." + [:or + [:and + [:map {:closed true} + [:lower [:fn #(and (number? %) (js/Number.isFinite %))]] + [:upper [:fn #(and (number? %) (js/Number.isFinite %))]]] + [:fn #(<= (:lower %) (:upper %))]] + [:map {:closed true} [:reason keyword?] [:minimum {:optional true} [:int {:min 1}]]]]) + +(def schema:statistics + "Observed statistics and supported intervals for one metric." + [:or + [:map {:closed true} + [:count [:int {:min 0}]] + [:mean schema:estimate] [:median schema:estimate] [:min schema:estimate] [:max schema:estimate] + [:sample-sd schema:estimate] [:mad schema:estimate] [:p95 schema:estimate] [:p99 schema:estimate] + [:median-ci schema:interval]] + [:map {:closed true} [:reason keyword?] [:count [:int {:min 0}]]]]) + +(def schema:metric-report + "A sum within each phase execution and statistics across eligible executions." + [:map {:closed true} + [:definitions [:vector quantities/schema:description]] + [:statistic [:= :sum]] + [:counts [:map {:closed true} + [:valid [:int {:min 0}]] [:invalid [:int {:min 0}]] [:absent [:int {:min 0}]] + [:failed [:int {:min 0}]] [:unattempted [:int {:min 0}]] [:warmup [:int {:min 0}]] + [:invocations [:int {:min 0}]] [:raw-values [:int {:min 0}]]]] + [:diagnostics [:vector map?]] + [:statistics schema:statistics]]) + +(def schema:comparison-side + "One saved case's definition, preparations and shared metric reports." + [:map {:closed true} + [:case-id some?] [:definition [:map [:id some?]]] + [:outcome [:maybe schema:case-outcome]] + [:preparations [:vector schema:preparation]] + [:phases [:map-of :keyword [:map {:closed true} + [:executions nat-int?] [:completed nat-int?] + [:failed nat-int?] [:warmup nat-int?]]]] + [:remaining [:map {:closed true} [:warmups [:int {:min 0}]] [:repetitions [:int {:min 0}]]]] + [:failed-attempts [:vector schema:attempt]] + [:metrics [:map-of :any schema:metric-report]]]) + +(def schema:provenance + "Saved run identity, timestamps, metadata and termination for comparison." + [:map {:closed true} + [:run-id some?] [:started-at string?] [:ended-at [:maybe string?]] + [:metadata schema:metadata] [:termination schema:termination]]) + +(def schema:permissions + [:map {:closed true} + [:allow-build-differences boolean?] + [:allow-scene-differences boolean?] + [:allow-all-differences boolean?]]) + +(def schema:feature-comparison + "Saved feature comparison progress. Case data and options remain ordinary + extensible maps; configuration/edge states and retained references are typed." + [:map {:closed true} + [:type [:= :feature-comparison]] + [:comparison-id string?] + [:started-at string?] + [:updated-at string?] + [:ended-at {:optional true} string?] + [:status [:enum :planned :running :completed :failed :interrupted]] + [:options :map] + [:permissions schema:permissions] + [:cases [:vector [:map [:id some?]]]] + [:features [:vector string?]] + [:configurations + [:vector + [:map + [:slug string?] + [:selection [:map {:closed true} + [:default-features? boolean?] + [:features [:vector string?]]]] + [:status [:enum :planned :building :ready :running :completed :failed :unattempted]] + [:build {:optional true} [:map [:id string?] [:manifest-path string?] [:log-path string?]]] + [:run {:optional true} [:map [:run-id string?] [:record-path string?] [:log-path string?] + [:exit-code {:optional true} :int]]] + [:effective-features {:optional true} [:vector string?]] + [:failure {:optional true} errors/schema:diagnostic] + [:reason {:optional true} string?]]]] + [:edges + [:vector + [:map + [:from string?] [:to string?] [:feature string?] + [:status [:enum :planned :completed :failed :unattempted]] + [:path {:optional true} string?] + [:failure {:optional true} + [:or errors/schema:diagnostic + [:map {:closed true} [:unacknowledged-differences [:int {:min 1}]]]]] + [:reason {:optional true} string?]]]] + [:warnings [:vector string?]]]) + +(def schema:comparison + "Resolved analysis, selected permissions, every mismatch and per-side reports. + Every acknowledged difference has a rule; unacknowledged differences have none. + Optional edge context records requested toggles and Cargo's actual effective + sets/deltas, including aliases." + [:map {:closed true} + [:type [:= :comparison]] + [:analysis analysis/schema:settings] + [:baseline schema:provenance] + [:candidate schema:provenance] + [:permissions schema:permissions] + [:edge {:optional true} + [:map {:closed true} + [:from string?] [:to string?] [:feature string?] + [:baseline-features [:vector string?]] [:candidate-features [:vector string?]] + [:added [:vector string?]] [:removed [:vector string?]]]] + [:source-coverage string?] + [:interpretation string?] + [:pairs [:vector + [:map {:closed true} + [:explicit? boolean?] + [:mismatches [:vector + [:and + [:map {:closed true} + [:field :any] [:baseline :any] [:candidate :any] [:waived? boolean?] + [:rule {:optional true} [:enum :feature-toggle :build :scene :all]]] + [:fn #(= (:waived? %) (contains? % :rule))]]]] + [:baseline schema:comparison-side] [:candidate schema:comparison-side] + [:metrics [:map-of :any [:map {:closed true} + [:absolute schema:estimate] [:percentage schema:estimate] + [:absolute-ci schema:interval] [:percentage-ci schema:interval]]]]]]]]) + +(defn case-name + "Returns the saved case identifier as shown in commands and reports." + [id] + (if (keyword? id) (subs (str id) 1) (str id))) + +(defn- require-valid! + "Rejects malformed external data with enough context for the caller." + [valid? message data] + (when-not valid? + (throw (ex-info message (assoc data :type ::invalid-record))))) + +(defn- incompatible! + "Rejects runs whose recorded evidence cannot be merged." + [message data] + (throw (ex-info message (assoc data :type ::incompatible-runs)))) + +(defn- require-compatible! + "Requires equal values at `path`, naming the disagreeing field." + [base additional path label] + (let [base-value (get-in base path) + additional-value (get-in additional path)] + (when-not (= base-value additional-value) + (incompatible! (str "cannot merge runs with different " label) + {:field label + :path path + :base base-value + :additional additional-value})))) + +(defn- initial-remaining + "Creates each saved case's fixed budget of unattempted slots." + [{:keys [cases warmups repetitions]}] + (into {} (map (fn [{:keys [id]}] [id {:warmups warmups :repetitions repetitions}])) cases)) + +(defn- consume-slot + "Consumes a warmup or measured slot; completed and failed attempts count alike." + [remaining {:keys [case-id warmup?]}] + (let [kind (if warmup? :warmups :repetitions) + path [case-id kind] + left (get-in remaining path)] + (require-valid! (and (some? left) (pos? left)) "no planned slot for attempt" + {:case-id case-id :slot kind}) + (require-valid! (or warmup? (zero? (get-in remaining [case-id :warmups]))) + "measured attempt precedes planned warmups" {:case-id case-id}) + (update-in remaining path dec))) + +(defn has-measurements? + "Reports whether a case has measured commands in its saved phase histories. + + Args: + run: constructed or decoded run record. + case-id: exact saved case identifier. + + Returns: + True when any preparation, main or reset block contains a command event." + [run case-id] + (boolean + (some (comp seq :events) + (mapcat :phases (filter #(= case-id (:case-id %)) + (concat (:preparations run) (:attempts run))))))) + +(defn get-unmeasured-cases + "Returns saved case ids whose phase histories contain no measured commands. + + Args: + run: constructed or decoded run record. + + Returns: + A vector of case identifiers in plan order." + [run] + (into [] + (keep (fn [{:keys [id]}] + (when-not (has-measurements? run id) + id))) + (get-in run [:plan :cases]))) + +(defn- check-completion + "Requires all planned attempts to succeed before a run can be completed." + [run termination] + (when (= :completed (:reason termination)) + (require-valid! (every? #(every? zero? (vals %)) (vals (:remaining run))) + "completed run still has unattempted slots" {}) + (require-valid! (not-any? #(= :failed (:outcome %)) (:attempts run)) + "completed run has failed attempts" {}) + (require-valid! (not-any? :failure (:preparations run)) + "completed run has failed preparations" {}) + (require-valid! (every? #(= :completed (:outcome %)) (vals (:case-outcomes run))) + "completed run has failed cases" {}) + (require-valid! (= (set (map :id (get-in run [:plan :cases]))) (set (keys (:case-outcomes run)))) + "completed run needs every case outcome" {}) + (require-valid! (empty? (get-unmeasured-cases run)) + "completed case has no measurements" {:case-ids (get-unmeasured-cases run)}))) + +(defn- check-phases + [record allowed] + (let [phases (:phases record)] + (require-valid! (= (mapv :phase phases) + (filterv (set (map :phase phases)) allowed)) + "phase order or identity is invalid" {}) + (doseq [phase phases] + (require-valid! (= (:preparation-id record) (:preparation-id phase)) + "phase references a different preparation" {}) + (doseq [event (:events phase)] + (require-valid! (= 1 (count (get-in event [:metrics :command/elapsed]))) + "command needs one elapsed value" {:command (:command event)}))) + (when-not (or (:failure record) (= :failed (:outcome record))) + (require-valid! (every? #(= :completed (:outcome %)) phases) + "successful execution has a failed phase" {})))) + +(defn- check-preparation + [preparation] + (require-valid! (sm/validate schema:preparation preparation) "invalid preparation" {}) + (check-phases preparation [:preparation]) + (when-not (:failure preparation) + (require-valid! (and (:fingerprint preparation) (:effective-graphics preparation) + (= [:preparation] (mapv :phase (:phases preparation)))) + "completed preparation needs scene, graphics and phase evidence" {}))) + +(defn- check-attempt + [attempt preparation] + (require-valid! (sm/validate schema:attempt attempt) "invalid attempt" {}) + (check-phases attempt [:main :reset]) + (when (:preparation-id attempt) + (require-valid! (= (:case-id attempt) (:case-id preparation)) + "attempt references an unknown or different preparation" {})) + (when (= :completed (:outcome attempt)) + (require-valid! (and preparation (not (:failure preparation)) + (not (:failure attempt)) + (= :main (get-in attempt [:phases 0 :phase]))) + "completed attempt needs successful preparation and main" {})) + (when (seq (:phases attempt)) + (require-valid! (= :main (get-in attempt [:phases 0 :phase])) + "attempt phases must start with main" {}) + (require-valid! (not (:failure preparation)) "attempt phases need successful preparation" {})) + (when (= :failed (get-in attempt [:phases 0 :outcome])) + (require-valid! (= 1 (count (:phases attempt))) "reset follows failed main" {}))) + +(defn- check-case-outcome + [run case-id outcome] + (require-valid! (contains? (:remaining run) case-id) "unknown case outcome" {}) + (require-valid! (sm/validate schema:case-outcome outcome) "invalid case outcome" {}) + (when (= :completed (:outcome outcome)) + (require-valid! (and (not (:failure outcome)) + (every? zero? (vals (get-in run [:remaining case-id]))) + (not-any? #(and (= case-id (:case-id %)) (= :failed (:outcome %))) (:attempts run)) + (not-any? #(and (= case-id (:case-id %)) (:failure %)) (:preparations run)) + (has-measurements? run case-id)) + "completed case has pending, failed or unmeasured work" {:case-id case-id}))) + +(defn check-run + "Validates schema:run and chronological accounting at a saved-data boundary." + [run] + (require-valid! (sm/validate schema:run run) "invalid schema:run" {}) + (let [case-ids (mapv :id (get-in run [:plan :cases])) + preps (:preparations run) + prep-ids (mapv :preparation-id preps) + prep-map (into {} (map (juxt :preparation-id identity)) preps) + left (reduce consume-slot (initial-remaining (:plan run)) (:attempts run))] + (require-valid! (= (count case-ids) (count (set case-ids))) "duplicate saved case" {}) + (require-valid! (= (count prep-ids) (count (set prep-ids))) "duplicate preparation" {}) + (require-valid! (every? #(contains? (set case-ids) %) (map :case-id preps)) "unknown preparation case" {}) + (require-valid! (= left (:remaining run)) "remaining slots disagree with attempts" {}) + (doseq [[case-id outcome] (:case-outcomes run)] + (check-case-outcome run case-id outcome)) + (doseq [prep preps] + (check-preparation prep)) + (doseq [[attempt-id attempt] (map-indexed vector (:attempts run))] + (require-valid! (= attempt-id (:attempt-id attempt)) "attempts are not chronological" {}) + (check-attempt attempt (get prep-map (:preparation-id attempt)))) + (check-completion run (:termination run))) + run) + +(defn create-run + "Creates an empty raw run from externally supplied identities and facts. + The plan defaults to 3 warmups, 10 repetitions and a 30000 ms case timeout. + For each case, all warmup slots precede measured slots. Failed attempts consume + their slot too; attempts retain chronological order across cases." + [{:keys [run-id started-at plan metadata analysis quantities]}] + (let [plan (merge {:warmups 3 :repetitions 10 :timeout-policy {:case-ms 30000}} plan)] + (check-run {:type :run-record + :run-id run-id :started-at started-at :ended-at nil + :plan plan :metadata metadata :analysis (analysis/resolve-settings analysis) + :quantities (merge quantities/descriptions quantities) + :preparations [] :attempts [] :case-outcomes {} :remaining (initial-remaining plan) + :termination {:reason :running}}))) + +(defn- check-open! + "Checks that a constructed or decoded run still accepts evidence." + [run] + (require-valid! (= :running (get-in run [:termination :reason])) "run already terminated" {})) + +(defn record-preparation + "Checks and appends new setup evidence without consuming an attempt slot. + The run must come from create-run or decode; earlier evidence is trusted." + [run preparation] + (check-open! run) + (require-valid! (not (contains? (:case-outcomes run) (:case-id preparation))) "case already terminated" {}) + (check-preparation preparation) + (require-valid! (contains? (:remaining run) (:case-id preparation)) "unknown preparation case" {}) + (require-valid! (not-any? #(= (:preparation-id preparation) (:preparation-id %)) + (:preparations run)) + "duplicate preparation" {}) + (update run :preparations conj preparation)) + +(defn record-attempt + "Checks and appends one new execution, consuming its scheduled slot. + The run must come from create-run or decode. + Missing preparation is allowed for failures before scene setup completes. + Earlier attempts are trusted; encode and decode check the complete record." + [run attempt] + (check-open! run) + (require-valid! (not (contains? (:case-outcomes run) (:case-id attempt))) "case already terminated" {}) + (let [attempt (-> (merge {:preparation-id nil :phases []} attempt) + (assoc :attempt-id (count (:attempts run))))] + (check-attempt attempt (some #(when (= (:preparation-id attempt) (:preparation-id %)) %) + (:preparations run))) + (-> run + (update :attempts conj attempt) + (update :remaining consume-slot attempt)))) + +(defn finish-run + "Records externally supplied end time and termination details, retaining gaps." + [run ended-at termination] + (check-open! run) + (require-valid! (sm/validate schema:termination termination) "invalid termination" {}) + (require-valid! (not= :running (:reason termination)) "termination needs an end reason" {}) + (check-completion run termination) + (assoc run :ended-at ended-at :termination termination)) + +(defn record-case-outcome + "Records the terminal outcome of one case. + + Args: + run: constructed or decoded run that accepts new evidence. + case-id: exact saved case identifier. + outcome: completion, failure, timeout or interruption with available cause. + + Returns: + The run with its case outcome. + + Throws: + ExceptionInfo for unknown cases, duplicate outcomes or malformed evidence." + [run case-id outcome] + (check-open! run) + (require-valid! (not (contains? (:case-outcomes run) case-id)) "duplicate case outcome" {}) + (check-case-outcome run case-id outcome) + (assoc-in run [:case-outcomes case-id] outcome)) + +(defn merge-runs + "Returns a new validated run that appends `additional`'s measured attempts to + `base`. + + Both runs must be completed and compatible. The merged plan keeps `base`'s + warmups and adds `additional`'s measured repetitions. Warmup attempts + and preparations used only by them are excluded from `additional`. + + The merged record uses the base run's quantity descriptions. + The caller supplies the merged run id and may supply its end time." + [base additional {:keys [run-id ended-at]}] + (let [base (check-run base) + additional (check-run additional)] + (require-valid! (and (string? run-id) (seq run-id)) + "merged run needs a run id" {}) + (doseq [[side run] [[:base base] [:additional additional]]] + (when-not (= :completed (get-in run [:termination :reason])) + (incompatible! "only completed runs can be merged" + {:side side + :run-id (:run-id run) + :reason (get-in run [:termination :reason])}))) + (when-not (and (string? (:ended-at base)) + (string? (:ended-at additional))) + (incompatible! "completed runs need end times" + {:base (:ended-at base) + :additional (:ended-at additional)})) + (doseq [[path label] [[[:plan :cases] "case definitions"] + [[:plan :timeout-policy] "timeout policy"] + [[:plan :warmups] "warmup count"] + [[:analysis] "analysis settings"] + [[:metadata :git] "git provenance"] + [[:metadata :build] "build facts"] + [[:metadata :environment] "runtime environment"] + [[:metadata :scored?] "scored flag"]]] + (require-compatible! base additional path label)) + (doseq [run [base additional]] + (require-valid! (every? #(sm/validate quantities/schema:description %) + (vals (:quantities run))) + "cannot merge malformed quantity descriptions" {:run-id (:run-id run)})) + (when-not (= (update-vals (:quantities base) #(dissoc % :description)) + (update-vals (:quantities additional) #(dissoc % :description))) + (incompatible! "cannot merge runs with different quantity units" {})) + (let [base-attempts (:attempts base) + additional-measured (filterv #(not (:warmup? %)) (:attempts additional)) + base-preparations (:preparations base) + base-preparation-ids (into #{} (map :preparation-id) base-preparations) + used-preparation-ids (into #{} (keep :preparation-id) additional-measured) + additional-preparations (filterv #(contains? used-preparation-ids + (:preparation-id %)) + (:preparations additional))] + (when-let [duplicate (some #(contains? base-preparation-ids (:preparation-id %)) + additional-preparations)] + (incompatible! "cannot merge runs with duplicate preparation ids" + {:preparation-id (:preparation-id duplicate)})) + (let [offset (count base-attempts) + attempts (into (vec base-attempts) + (map-indexed (fn [index attempt] + (assoc attempt :attempt-id (+ offset index))) + additional-measured)) + plan (assoc (:plan base) + :repetitions (+ (get-in base [:plan :repetitions]) + (get-in additional [:plan :repetitions]))) + remaining (reduce consume-slot (initial-remaining plan) attempts) + ended-at (or ended-at + (let [base-end (:ended-at base) + additional-end (:ended-at additional)] + (if (neg? (compare base-end additional-end)) + additional-end + base-end))) + metadata (update (:metadata base) :diagnostics + #(assoc (or % {}) + :merge + {:base-run-id (:run-id base) + :additional-run-id (:run-id additional) + :additional-warmups-excluded + (count (filter :warmup? (:attempts additional))) + :merged-at ended-at}))] + (check-run {:type :run-record + :run-id run-id + :started-at (:started-at base) + :ended-at ended-at + :plan plan + :metadata metadata + :analysis (:analysis base) + :quantities (:quantities base) + :preparations (into (vec base-preparations) additional-preparations) + :attempts attempts + :case-outcomes (merge (:case-outcomes base) (:case-outcomes additional)) + :remaining remaining + :termination {:reason :completed}}))))) + +(defn- check-feature-configuration + "Requires evidence that matches each saved configuration's progress." + [{:keys [status build run effective-features failure reason]}] + (when (#{:ready :running :completed} status) + (require-valid! (some? build) "feature comparison configuration needs a build" {:status status})) + (when (#{:ready :completed} status) + (require-valid! (some? effective-features) + "feature comparison configuration needs effective features" {:status status})) + (when (#{:running :completed} status) + (require-valid! (some? run) "feature comparison configuration needs a run" {:status status})) + (when (= :completed status) + (require-valid! (some? (:exit-code run)) + "completed feature comparison configuration needs a run exit code" {}) + (require-valid! (zero? (:exit-code run)) + "completed feature comparison configuration needs a successful run" {})) + (when (= :failed status) + (require-valid! (or (some? failure) (some? reason)) + "failed feature comparison configuration needs a failure or reason" {})) + (when (= :unattempted status) + (require-valid! (some? reason) + "unattempted feature comparison configuration needs a reason" {}))) + +(defn- check-feature-edge + "Requires a path for completed analysis and a reason for unattempted work." + [{:keys [status path failure reason]}] + (case status + :completed + (require-valid! (some? path) "completed feature comparison edge needs a report path" {}) + + :failed + (require-valid! (or (some? failure) (some? reason)) + "failed feature comparison edge needs a failure or reason" {}) + + :unattempted + (require-valid! (some? reason) + "unattempted feature comparison edge needs a reason" {}) + + nil)) + +(defn check-feature-comparison + "Validates saved feature-comparison progress against its planned experiment. + Configurations and edges must match the pure planner after status evidence is + attached. Terminal records cannot retain active work, and completed work must + retain its build, run, effective-feature and report evidence." + [record] + (require-valid! (sm/validate schema:feature-comparison record) + "invalid schema:feature-comparison" {}) + (let [features (:features record) + configurations (:configurations record) + edges (:edges record) + expected (feature-plan/plan features + {:max-features (count features) + :dry-run true + :confirm true}) + terminal? (contains? #{:completed :failed :interrupted} (:status record)) + active? (comp #{:planned :building :ready :running} :status)] + (require-valid! (= features (:features expected)) + "feature comparison features disagree with normalized selection" {}) + (require-valid! (= (mapv #(select-keys % [:slug :selection]) configurations) + (:configurations expected)) + "feature comparison configurations disagree with planned features" {}) + (require-valid! (= (mapv #(select-keys % [:from :to :feature]) edges) + (:edges expected)) + "feature comparison edges disagree with planned configurations" {}) + (require-valid! (= (count (:cases record)) (count (set (map :id (:cases record))))) + "duplicate feature comparison case" {}) + (when terminal? + (require-valid! (some? (:ended-at record)) + "terminal feature comparison needs an end time" {}) + (require-valid! (not-any? active? configurations) + "terminal feature comparison still has active configurations" {}) + (require-valid! (not-any? active? edges) + "terminal feature comparison still has active edges" {})) + (when-not terminal? + (require-valid! (nil? (:ended-at record)) + "unfinished feature comparison has an end time" {})) + (when (= :planned (:status record)) + (require-valid! (every? (comp #{:planned} :status) configurations) + "planned feature comparison has started configurations" {}) + (require-valid! (every? (comp #{:planned} :status) edges) + "planned feature comparison has started edges" {})) + (when (= :completed (:status record)) + (require-valid! (every? (comp #{:completed} :status) configurations) + "completed feature comparison has unfinished configurations" {}) + (require-valid! (every? (comp #{:completed} :status) edges) + "completed feature comparison has unfinished edges" {})) + (doseq [configuration configurations] + (check-feature-configuration configuration)) + (doseq [edge edges] + (check-feature-edge edge))) + record) + +(defn check-record + "Validates a run, comparison or feature-comparison at a Transit boundary." + [record] + (case (:type record) + :run-record (check-run record) + :comparison (do + (require-valid! (sm/validate schema:comparison record) "invalid schema:comparison" {}) + record) + :feature-comparison (check-feature-comparison record) + (throw (ex-info "unsupported benchmark format" {:type ::unsupported-format})))) + +(defn encode + "Encodes a supported record with transit." + [record] + (codec/encode-str (check-record record))) + +(defn decode + "Decodes from transit and validates a supported record." + [text] + (check-record (transit/decode-str text))) diff --git a/frontend/performance/benches/render_wasm/runtime/camera.cljs b/frontend/performance/benches/render_wasm/runtime/camera.cljs new file mode 100644 index 0000000000..cc334fe604 --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/camera.cljs @@ -0,0 +1,151 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.camera + "Rendering and camera commands over a prepared snapshot. + + Commands accept a context or its Promise and return a Promise of the next + context. Pan, zoom and restore update its view. Starred variants record raw + elapsed time and renderer evidence in the current execution phase. + + Load waits for an animation frame and renders through Full. Pan and zoom + submit paced cached previews, settle, then render through Full. Restore + applies a view and renders immediately with SyncTiles. Render-sync uses the + blocking renderer path. Full denotes renderer submission completion. + + Measured async commands finish at the final renderer return captured in + `:end-ms`. Input resolution and argument validation precede timing; output + validation and event recording follow it. Traces retain renderer calls, + cached previews, waits and protocol endpoints." + (:require + [app.common.schema :as sm] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.runtime.commands :refer [defcommand]] + [benches.render-wasm.runtime.protocol :as protocol])) + +(defcommand render-sync + "Renders the prepared scene synchronously. + + Args: + rtx: context or Promise of it. + + Returns: + A Promise of the context after the blocking renderer call." + [rtx] + ((get-in rtx [:hooks :render-sync])) + rtx) + +(def ^:private valid-pan? + (sm/validator + [:map {:closed true} + [:dx ::sm/safe-number] + [:dy ::sm/safe-number] + [:steps [:int {:min 1}]] + [:settle-ms [:and ::sm/safe-number [:>= 0]]]])) + +(def ^:private valid-zoom? + (sm/validator + [:map {:closed true} + [:factor [:and ::sm/safe-number [:> 0]]] + [:steps [:int {:min 1}]] + [:settle-ms [:and ::sm/safe-number [:>= 0]]]])) + +(def ^:private valid-view? (sm/validator decl/schema:view)) + +(defn- pan-view + [view dx dy] + (-> view (update :x + dx) (update :y + dy))) + +(defn- zoom-view + [view factor] + (update view :scale * factor)) + +(defn- render-evidence + [evidence] + {:end-ms (:end-ms evidence) + :metrics {:renderer/call-elapsed (mapv :duration-ms (:slices evidence))} + :trace evidence}) + +(defn- camera-evidence + [evidence] + (-> (render-evidence evidence) + (assoc-in [:metrics :renderer/cache-call-elapsed] (mapv :duration-ms (:cached-slices evidence))) + (assoc-in [:metrics :camera/finalization-to-full] [(:time-to-full-ms evidence)]))) + +(defn- run-camera + [rtx target {:keys [steps settle-ms]}] + (-> (protocol/start-camera! rtx {:settle-ms settle-ms}) + (protocol/animate-view! {:to target :steps steps}) + (protocol/finish-camera!))) + +(defcommand pan + "Moves the view by `:dx` and `:dy` through `:steps` cached previews. + Waits `:settle-ms` milliseconds, ends the interaction and renders until Full. + + Args: + rtx: context or Promise of it. + options: finite offsets, positive step count and nonnegative settling wait. + + Returns: + A Promise of the context with its updated view." + {:async [complete fail]} + [rtx {:keys [dx dy] :as options}] + {:pre [(valid-pan? options) (valid-view? (pan-view (:view rtx) dx dy))]} + (let [target (pan-view (:view rtx) dx dy)] + (-> (run-camera rtx target options) + (.then #(complete (assoc rtx :view target) (camera-evidence %))) + (.catch fail)))) + +(defcommand load + "Renders the uploaded snapshot until the renderer returns Full. + Waits for an animation frame before the first renderer call. + + Args: + rtx: context or Promise of it. + + Returns: + A Promise of the context after renderer completion." + {:async [complete fail]} + [rtx] + (let [hooks (:hooks rtx)] + (-> (protocol/drain hooks {:flags 0 :origin ((:now hooks)) :immediate false}) + (.then #(complete rtx (assoc-in (render-evidence %) + [:metrics :renderer/prepared-scene-to-full] [(:full-ms %)]))) + (.catch fail)))) + +(defcommand zoom + "Multiplies view scale through paced cached previews, then renders through Full. + + Args: + rtx: context or Promise of it. + options: positive finite factor, positive steps and nonnegative settle-ms. + + Returns: + A Promise of the context with its updated scale." + {:async [complete fail]} + [rtx {:keys [factor] :as options}] + {:pre [(valid-zoom? options) (valid-view? (zoom-view (:view rtx) factor))]} + (let [target (zoom-view (:view rtx) factor)] + (-> (run-camera rtx target options) + (.then #(complete (assoc rtx :view target) (camera-evidence %))) + (.catch fail)))) + +(defcommand restore + "Restores a view and renders immediately through Full with SyncTiles. + + Args: + rtx: context or Promise of it. + view: scale and offsets to restore. Its resolved viewport must match rtx. + + Returns: + A Promise of the context with the restored view." + {:async [complete fail]} + [rtx view] + {:pre [(valid-view? view) + (= (decl/resolve-viewport view) (decl/resolve-viewport (:view rtx)))]} + (-> (protocol/restore (:hooks rtx) view) + (.then #(complete (assoc rtx :view view) (render-evidence %))) + (.catch fail))) diff --git a/frontend/performance/benches/render_wasm/runtime/commands.clj b/frontend/performance/benches/render_wasm/runtime/commands.clj new file mode 100644 index 0000000000..438e71eb85 --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/commands.clj @@ -0,0 +1,90 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.commands + "Declarations of renderer commands. + A command transforms a command context, a map of case data, a scene snapshot, + the current camera view and hooks. A scene is a recipe for shapes; a case + pairs a scene with its work. A snapshot contains concrete shapes and their + references. The view stores zoom scale and camera offsets. Hooks are functions + for clocks, scheduling and renderer operations. A measured command records + an event containing raw numeric values for named quantities, called metrics.") + +(defmacro defcommand + "Defines a base command and its measured function with a `*` suffix. + + Both accept a command context or its Promise and return a Promise of the next + context. A synchronous body returns that context. The measured function + requires an active command execution and records one event with + `:command/elapsed`, the body duration in milliseconds. + + Options: + - `:measured-variant false` defines a composite's base function. A composite + combines commands and records events from the measured commands it calls. + - `:async [complete fail]` lets the body complete asynchronously. Successful + completion requires `complete`; `fail` reports a cause. The first call to + either callback determines the outcome. A returned Promise forwards its + rejection to `fail`. + + `complete` accepts the next context and an optional map. `:end-ms` supplies + the clock reading at the end of the work. When the map omits `:end-ms`, + `complete` captures a reading on entry. `:metrics` maps namespaced metric + keywords to nonempty vectors of raw numbers. + `:trace` contains renderer details such as individual render calls and waits. + + Argument destructuring and `:pre` conditions precede work and timing in every + build. A failed condition rejects the returned Promise. The condition map + accepts `:pre`. Output validation and event recording follow the endpoint." + [command & forms] + (let [[doc forms] (if (string? (first forms)) + [(first forms) (next forms)] + [nil forms]) + [options forms] (if (map? (first forms)) + [(first forms) (next forms)] + [{} forms]) + [argv & body] forms + [conditions body] (if (and (map? (first body)) (next body)) + [(first body) (next body)] + [nil body]) + measured (get options :measured-variant true) + handlers (:async options) + command-id (keyword (str (get-in &env [:ns :name])) (name command)) + starred (symbol (str command "*")) + prepare (gensym "prepare")] + (when-not (and (simple-symbol? command) + (vector? argv) + (seq argv) + (boolean? measured) + (every? #{:measured-variant :async} (keys options)) + (or (not (contains? options :async)) + (and (vector? handlers) + (= 2 (count handlers)) + (every? simple-symbol? handlers) + (apply distinct? handlers))) + (every? #{:pre} (keys conditions)) + (or (nil? (:pre conditions)) (vector? (:pre conditions)))) + (throw (ex-info "invalid command declaration" {:command command}))) + `(let [~prepare (fn ~argv + ~@(for [condition (:pre conditions)] + `(when-not ~condition + (throw (ex-info "command precondition failed" + {:type ::invalid-arguments + :command ~command-id + :condition '~condition})))) + (fn ~(or handlers []) ~@body))] + (defn ~command + ~@(when doc [doc]) + [rtx# & args#] + (invoke {:command ~command-id :measured false :composite ~(not measured) + :async ~(some? handlers)} + rtx# ~prepare args#)) + ~@(when measured + [`(defn ~starred + ~(str "Measured variant of `" command "`." (when doc (str " " doc))) + [rtx# & args#] + (invoke {:command ~command-id :measured true :composite false + :async ~(some? handlers)} + rtx# ~prepare args#))])))) diff --git a/frontend/performance/benches/render_wasm/runtime/commands.cljs b/frontend/performance/benches/render_wasm/runtime/commands.cljs new file mode 100644 index 0000000000..6fa8f86ecc --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/commands.cljs @@ -0,0 +1,241 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.commands + "Execution of renderer commands and collection of their measurements. + + A command transforms a command context: a map containing case data (`:case`), + the prepared scene snapshot (`:scene`), the current camera view (`:view`) and + functions for clocks, scheduling and renderer operations (`:hooks`). A scene + is a recipe for shapes; a case pairs a scene with its work. A snapshot contains + the concrete shapes and their references. A view contains zoom scale, camera + offsets and optional viewport settings for canvas dimensions and pixel ratio. + The `:check` hook is a guard: + returning authorizes work, and throwing an error rejects it. + + A command execution evaluates a function from one context to the next and + records measured command calls in completion order. Each call records an + event with the command name and its metrics. A metric names a quantity; + each event keeps its raw numeric + values in a vector. An optional trace contains details such as render calls + and waits. Each execution owns its event history and authorizes work while + active. Its outcome contains the next context or an error and earlier events." + (:require-macros [benches.render-wasm.runtime.commands]) + (:require + [app.common.schema :as sm] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors :refer [attempt failure?]])) + +(def ^:private schema:context + [:map {:closed true} + [:case [:map [:id qualified-keyword?]]] + [:scene [:map [:objects map?] [:refs map?]]] + [:view decl/schema:view] + [:hooks [:and + [:map-of :keyword fn?] + [:map + [:now fn?] + [:check fn?] + [:frame fn?] + [:sleep fn?] + [:render fn?] + [:render-sync fn?] + [:render-from-cache fn?] + [:set-view fn?] + [:set-view-start fn?] + [:set-view-end fn?]]]]]) + +(def ^:private valid-context? (sm/validator schema:context)) +(def ^:private schema:completion + [:map {:closed true} + [:end-ms {:optional true} number?] + [:metrics {:optional true} + [:map-of qualified-keyword? [:vector {:min 1} number?]]] + [:trace {:optional true} :any]]) +(def ^:private valid-completion? (sm/validator schema:completion)) +(def ^:private owners (js/WeakMap.)) + +(defn- check-context + [rtx] + (when-not (valid-context? rtx) + (throw (ex-info "invalid command context" + {:type ::invalid-context + ::sm/explain (sm/explain schema:context rtx)})))) + +(defn- check-output + [rtx output] + (check-context output) + (when-not (identical? (get-in rtx [:hooks :check]) + (get-in output [:hooks :check])) + (throw (ex-info "command changed execution ownership" + {:type ::ownership-lost}))) + ((get-in rtx [:hooks :check])) + output) + +(defn- scope-context + [rtx events diagnostics active] + (check-context rtx) + (let [previous-check (get-in rtx [:hooks :check]) + previous-owner (.get owners previous-check) + base-check (or (:base-check previous-owner) previous-check) + check (fn [] + (when-not @active + (throw (ex-info "command execution has ended" + {:type ::inactive-execution}))) + (base-check)) + scoped (assoc-in rtx [:hooks :check] check)] + (.set owners check {:base-check base-check + :record #(do (check) (vswap! events conj %)) + :diagnose #(when @active (vswap! diagnostics conj %))}) + (check) + scoped)) + +(defn ^:async ^:no-doc execute + "Runs `run` with a command context and resolves to its outcome. + + `run` returns the next context or its Promise. A successful outcome contains + `:outcome :completed`, `:context` and ordered `:events`. A failed outcome + contains `:outcome :failed`, `:error` and events recorded before failure. + Existing Errors retain their identity; other thrown values become Errors + that retain the value in `:thrown`. + + The context's guard authorizes work during this execution. Passing a returned + context to another execution establishes a new authorization. Repeated calls + to asynchronous completion callbacks can supply `:diagnostics` while the + execution is active. Returned contexts must retain this execution's `:check` + guard. A changed guard rejects completion." + [rtx run] + (let [events (volatile! []) + diagnostics (volatile! []) + active (volatile! true) + result (try + (attempt :catch-fatal + (let [scoped (scope-context (await rtx) events diagnostics active)] + (check-output scoped (await (run scoped))))) + (finally + (vreset! active false)))] + (cond-> (if (failure? result) + {:outcome :failed :error result :events @events} + {:outcome :completed :context result :events @events}) + (seq @diagnostics) (assoc :diagnostics @diagnostics)))) + +(defn- prepare-invocation + [{:keys [command measured] :as invocation} rtx prepare args] + (check-context rtx) + (let [{:keys [check now]} (:hooks rtx) + owner (.get owners check)] + (check) + (when (and measured (nil? owner)) + (throw (ex-info "measured command requires an execution" + {:type ::missing-execution :command command}))) + (assoc invocation + :input rtx + :core (apply prepare rtx args) + :now now + :record (:record owner) + :diagnose (:diagnose owner)))) + +(defn- thenable? + [output] + (and (some? output) (fn? (.-then output)))) + +(defn- run-core + [{:keys [command measured composite core now] :as invocation}] + (if composite + (assoc invocation :output (core)) + (let [start (when measured (now)) + output (core) + end (when measured (now))] + (when (thenable? output) + (.catch (js/Promise.resolve output) errors/as-failure) + (throw (ex-info "synchronous command returned asynchronous work" + {:type ::async-result :command command}))) + (cond-> (assoc invocation :output output) + measured (assoc :metrics {:command/elapsed [(- end start)]}))))) + +(defn- finish-invocation + [{:keys [measured input output record] :as invocation}] + (check-output input output) + (when (contains? invocation :trace) + (try + (codec/check-value (:trace invocation)) + (codec/encode-str (:trace invocation)) + (catch :default cause + (throw (ex-info "command trace cannot cross Transit" + {:type ::invalid-trace} cause))))) + (when measured + (record (select-keys invocation [:command :metrics :trace]))) + output) + +(defn- complete-invocation + [{:keys [command measured now] :as invocation} output evidence start] + ;; A completion callback can run after the renderer returns. A supplied + ;; endpoint preserves the renderer return time across that scheduling delay. + (let [end (when measured + (if (and (map? evidence) (contains? evidence :end-ms)) + (:end-ms evidence) + (now)))] + (when-not (and (valid-completion? evidence) + (not (contains? (:metrics evidence) :command/elapsed))) + (throw (ex-info "invalid command completion evidence" + {:type ::invalid-completion :command command + ::sm/explain (sm/explain schema:completion evidence)}))) + (cond-> (merge invocation evidence {:output output}) + measured (assoc-in [:metrics :command/elapsed] [(- end start)])))) + +(defn- claim-terminal + [terminal diagnose command signal] + (if-some [first-signal @terminal] + (do + (when diagnose + (diagnose {:type ::duplicate-terminal :command command + :first-signal first-signal :signal signal})) + false) + (do + (vreset! terminal signal) + true))) + +(defn- run-async-core + [{:keys [command measured core now diagnose] :as invocation}] + (js/Promise. + (fn [resolve reject] + (let [terminal (volatile! nil) + start (volatile! nil) + complete (fn complete + ([output] (complete output {})) + ([output evidence] + (when (claim-terminal terminal diagnose command :complete) + (let [result (attempt :catch-fatal + (-> (complete-invocation invocation output evidence @start) + (finish-invocation)))] + (if (failure? result) (reject result) (resolve result)))) + nil)) + fail (fn [cause] + (when (claim-terminal terminal diagnose command :fail) + (reject (errors/as-failure cause))) + nil) + result (attempt :catch-fatal + (vreset! start (when measured (now))) + (let [returned (core complete fail)] + (when (thenable? returned) + (.catch (js/Promise.resolve returned) fail))))] + (when (failure? result) + (fail result)))))) + +(defn ^:async invoke + "Runs a command with a context or its Promise and returns the next context. + Resolves to that context as a Promise; errors reject it. A measured command + records one event with elapsed time, additional metric values and any trace." + [command rtx prepare args] + (let [invocation (prepare-invocation command (await rtx) prepare args)] + (if (:async command) + (await (run-async-core invocation)) + (let [result (run-core invocation)] + (finish-invocation + (if (:composite command) + (assoc result :output (await (:output result))) + result)))))) diff --git a/frontend/performance/benches/render_wasm/runtime/protocol.clj b/frontend/performance/benches/render_wasm/runtime/protocol.clj new file mode 100644 index 0000000000..0b8768b259 --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/protocol.clj @@ -0,0 +1,30 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.protocol + "Forms for evaluating renderer operations as Promises. + A Promise represents a result that can complete with a value or reject with + a cause. These forms include synchronous errors in that result.") + +(defmacro resolve-or-reject + "Evaluates `body` and returns a Promise of its result. + A synchronous throw rejects the Promise." + [& body] + `(try + (js/Promise.resolve (do ~@body)) + (catch :default cause# + (js/Promise.reject cause#)))) + +(defmacro resolve-or-reject-with + "Evaluates `body` and returns a Promise of its result. + A synchronous throw rejects with `on-error`, evaluated with `cause` bound + to the thrown value." + [cause on-error & body] + `(try + (js/Promise.resolve (do ~@body)) + (catch :default cause# + (let [~cause cause#] + (js/Promise.reject ~on-error))))) diff --git a/frontend/performance/benches/render_wasm/runtime/protocol.cljs b/frontend/performance/benches/render_wasm/runtime/protocol.cljs new file mode 100644 index 0000000000..84ed08f9ba --- /dev/null +++ b/frontend/performance/benches/render_wasm/runtime/protocol.cljs @@ -0,0 +1,424 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.runtime.protocol + "Rendering until completion and timing camera movement. + + A frame type describes the progress of one renderer call. A render drain is + a sequence of calls that ends when the renderer returns Full. Each call + produces a slice: its timestamp, flags, frame type and duration. Partial and + ViewportReady schedule another call on the next animation frame. Full marks + renderer submission of the completed request. GPU execution and display + proceed asynchronously. + + A view contains zoom `:scale` and offsets `:x` and `:y`. A cached preview + draws from existing tiles, the fixed regions of the scene that the renderer + caches. A camera session starts an interaction, applies views with cached + previews, waits for the requested settling time, ends the interaction and + renders until Full. Each preview waits for an animation frame. Explicit + pauses can extend the active interval between camera start and settling. + Camera finalization starts immediately before ending the interaction and + ends at the final renderer return. + + ## Hooks and errors + + Hooks are functions that supply clock, scheduling and renderer operations: + + - `:now` returns a clock reading in milliseconds. + - `:frame` resolves a Promise with the next animation frame's timestamp. + - `:sleep` accepts a requested wait in milliseconds and returns its Promise. + - `:check` is a guard. Returning authorizes work; throwing rejects it. It can + report cancellation, an expired deadline, loss of the WebGL2 graphics + context or replacement of the active renderer owner. An owner is the + execution authorized to use the renderer. + - `:render` accepts a timestamp and flags and returns a frame type. + - `:render-from-cache` draws one cached preview. + - `:set-view` accepts a view containing scale and both offsets. + - `:set-view-start` and `:set-view-end` start and end the interaction. + + Guards run before work, after awaited frames and waits, and around render + calls. Guard and renderer errors reject the returned Promises. Errors keep + available traces of render calls and pauses under `:partial`. + + Frame types come from `app.common.render-wasm.wasm`. Partial and ViewportReady + continue rendering, Full completes it, and other values reject with + `::unexpected-frame-type`. Each subsequent call receives the preceding frame + type as its flags. SyncTiles asks the renderer to update which tiles the + current view requires. + View restoration and camera finalization use that flag for their first call + and start immediately. A drain can also wait for an animation frame before + its first call. Later calls wait for animation frames. + + ## Render calls and measured durations + + A trace records individual calls and waits. Durations use milliseconds and + preserve raw numeric values, including zero. A drain origin is the clock + reading from which its completion durations start. + + - `:slices` contains one entry per renderer call: + - `:timestamp`: the animation frame's timestamp or the current clock + reading for an immediate call. + - `:flags`: the flags passed to the renderer. + - `:frame-type`: the returned frame type. + - `:duration-ms`: elapsed time during the call. + - `:cached-slices` contains one entry per cached preview: + - `:timestamp`: the animation frame's timestamp for the view update. + - `:duration-ms`: elapsed time during the cached render call. + - `:view-and-preview-ms`: elapsed time for the view update and preview. + - `:viewport-ready-ms` measures from the drain origin to the first call that + returns ViewportReady or Full. It equals `:full-ms` when the first such + call returns Full. + - `:full-ms` measures from the drain origin to the call that returns Full. + - `:end-ms` is the clock reading immediately after that final call returns. + - `:interact-ms` measures from immediately before camera start through the + final renderer return. + - `:active-ms` measures from camera start through the last cached preview and + its guard check, or through an explicit pause that follows the preview. + - `:authored-sleeps` contains requested and observed durations of explicit + pauses during the active interval. + - `:settling-requested-ms` is the requested settling wait. + - `:settling-actual-ms` measures from the end of the active interval to the + start of camera finalization. + - `:set-view-end-ms` measures the call that ends the interaction. + - `:time-to-viewport-ready-ms` measures from camera finalization to the first + ViewportReady or Full result. `:time-to-full-ms` ends at Full. Both include + the call that ends the interaction. + - `:last-input-to-full-ms` measures from immediately before the last view + update to the final renderer return. The initial input time equals camera + start; each view update replaces it." + (:require-macros + [benches.render-wasm.runtime.protocol :refer [resolve-or-reject resolve-or-reject-with]]) + (:require + [app.common.render-wasm.wasm :as wasm] + [benches.render-wasm.errors :as errors])) + + +(defn- partial-error + "Returns an error with completed traces under `:partial`. + Trace fields already present in the error take precedence over `evidence`." + [cause evidence] + (errors/with-context cause (errors/message-of cause "renderer protocol failed") + {:partial evidence})) + +(defn- next-render-timestamp + "Resolves to the current clock reading for an immediate render, or to the + next animation frame's timestamp." + [{:keys [now frame]} immediate?] + (if immediate? + (js/Promise.resolve (now)) + (frame))) + + +(defn- render-slice + "Returns the updated drain state, frame type and endpoint for one render call. + The state includes the completed call in its `:slices` trace." + [{:keys [now check render]} + {:keys [flags] :as state} + timestamp] + (check) + (let [started-ms (now) + frame-type (render timestamp flags) + ended-ms (now) + slice {:timestamp timestamp + :flags flags + :frame-type frame-type + :duration-ms (- ended-ms started-ms)}] + (try + (check) + (catch :default cause + (throw (partial-error cause {:slices (conj (:slices state) slice)})))) + {:state (update state :slices conj slice) + :frame-type frame-type + :ended-ms ended-ms})) + + +(defn- valid-frame-type? + "Recognizes the progressive and final renderer frame types." + [frame-type] + (or (= frame-type wasm/FRAME_TYPE_PARTIAL) + (= frame-type wasm/FRAME_TYPE_VIEWPORT_READY) + (= frame-type wasm/FRAME_TYPE_FULL))) + + +(defn- viewport-ready-time + "Returns the elapsed time to the first ViewportReady or Full result." + [current-ready-ms frame-type ended-ms origin-ms] + (or current-ready-ms + (when (or (= frame-type wasm/FRAME_TYPE_VIEWPORT_READY) + (= frame-type wasm/FRAME_TYPE_FULL)) + (- ended-ms origin-ms)))) + + +(defn- advance-drain-state + "Updates the continuation flags and completion time for a returned frame type. + An unexpected type throws an error containing the completed render calls." + [{:keys [viewport-ready-ms] :as state} + frame-type + ended-ms + origin-ms] + (when-not (valid-frame-type? frame-type) + (throw (ex-info (str "unexpected frame type: " (pr-str frame-type)) + {:type ::unexpected-frame-type + :frame-type frame-type + :partial (select-keys state [:slices :viewport-ready-ms])}))) + (assoc state + :flags frame-type + :viewport-ready-ms + (viewport-ready-time viewport-ready-ms + frame-type + ended-ms + origin-ms))) + +(defn- drain-complete? + "Returns whether the renderer reported Full completion." + [frame-type] + (= frame-type wasm/FRAME_TYPE_FULL)) + +(defn- drain-result + "Returns traces of render calls, the final clock reading and completion durations + measured from the drain origin." + [{:keys [slices viewport-ready-ms]} + ended-ms + origin-ms] + {:slices slices + :end-ms ended-ms + :viewport-ready-ms viewport-ready-ms + :full-ms (- ended-ms origin-ms)}) + + +(defn drain + "Renders until a call returns Full and resolves to its traces and durations. + + Options: + - `:flags` supplies the first render call's flags. + - `:origin` supplies the starting clock reading for completion durations. + - `:immediate` starts the first call immediately when true. Otherwise, the + first call waits for an animation frame. Subsequent calls wait for frames. + + The result contains `:slices`, the traces of render calls; `:end-ms`, the final + renderer clock reading; and `:viewport-ready-ms` and `:full-ms`, durations + measured from `:origin`. Partial and ViewportReady continue with the returned + frame type as the next flags. A guard failure or unexpected frame type + rejects the Promise with completed traces in the error's `:partial` field." + [{:keys [check] :as hooks} + {:keys [flags origin immediate]}] + (resolve-or-reject + (check) + (letfn [(step [state immediate?] + (-> (next-render-timestamp hooks immediate?) + (.then ;; recurse or drain + (fn [timestamp] + (let [{:keys [frame-type ended-ms] :as rendered} + (render-slice hooks state timestamp) + next-state + (advance-drain-state (:state rendered) + frame-type + ended-ms + origin)] + + (if (drain-complete? frame-type) + (drain-result next-state ended-ms origin) + (step next-state false))))) + (.catch (fn [cause] + (throw (partial-error cause (select-keys state [:slices :viewport-ready-ms])))))))] + (step {:flags flags + :slices [] + :viewport-ready-ms nil} + immediate)))) + +(defn restore + "Restores `view` and renders its tiles until Full. + The first call uses SyncTiles and starts immediately. Completion durations + begin after restoring the view and ending the interaction. Returns a Promise + with recorded render calls, completion durations and the final clock reading + `:end-ms`." + [{:keys [now check set-view set-view-start set-view-end] :as hooks} view] + (resolve-or-reject + (check) + (set-view-start) + (set-view view) + (set-view-end) + (drain hooks {:flags wasm/RENDER_FLAG_SYNC_TILES + :origin (now) + :immediate true}))) + +(defn start-camera! + "Starts a camera session from the context's current `:view` and `:hooks`. + Resolves to the session with its start time and requested settling wait. + The initial active interval covers the operation that starts the camera. + `:settle-ms` must give a finite, nonnegative duration in milliseconds." + [{:keys [hooks view]} {:keys [settle-ms]}] + (resolve-or-reject + (when-not (and (number? settle-ms) + (js/Number.isFinite settle-ms) + (<= 0 settle-ms)) + (throw (ex-info "settle-ms must be a nonnegative finite number" + {:type ::invalid-settle-ms :settle-ms settle-ms}))) + (let [{:keys [now check set-view-start]} hooks] + (check) + (let [started-ms (now)] + (set-view-start) + {:hooks hooks + :view view + :settle-ms settle-ms + :started-ms started-ms + :last-input-ms started-ms + :active-end-ms (now) + :cached-slices [] + :authored-sleeps []})))) + +(defn preview-view! + "Updates the view and draws one cached preview on the next animation frame. + Resolves to the session with the new view and measured preview durations. + Errors retain completed previews and pauses under `:partial`." + [session-p view] + (-> (js/Promise.resolve session-p) + (.then (fn [{:keys [hooks] :as session}] + (let [{:keys [now frame check render-from-cache set-view]} hooks + evidence (select-keys session [:cached-slices :authored-sleeps])] + (resolve-or-reject-with cause + (partial-error cause evidence) + (check) + (-> (frame) + (.then (fn [timestamp] + (check) + (let [input-ms (now) + call-start (now)] + (set-view view) + (let [cache-start (now)] + (render-from-cache) + (let [cache-end (now) + slices (conj (:cached-slices session) + {:timestamp timestamp + :duration-ms (- cache-end cache-start) + :view-and-preview-ms (- cache-end call-start)})] + (try + (check) + (catch :default cause + (throw (partial-error cause (assoc evidence :cached-slices slices))))) + (assoc session :view view :last-input-ms input-ms + :active-end-ms (now) :cached-slices slices)))))) + (.catch (fn [cause] (throw (partial-error cause evidence))))))))))) + +(defn- interpolate + "Interpolates $v(t) = (1-t)a + tb$ between finite endpoints. + + Args: + a: starting value. + b: target value. + t: interpolation fraction in [0, 1]. + + Returns: + The interpolated value, preserving positive endpoints near zero." + [a b t] + (if (< t 0.5) + (+ a (* t (- b a))) + (+ b (* (- 1 t) (- a b))))) + +(defn animate-view! + "Moves to `to` through `steps` linearly interpolated views in sequence. + Each update waits for an animation frame and draws one cached preview. + Resolves to the session with its final view. Steps must be a positive integer; + errors retain completed previews and pauses under `:partial`." + [session-p {:keys [to steps]}] + (-> (js/Promise.resolve session-p) + (.then (fn [{from :view :as session}] + (when-not (and (integer? steps) (pos? steps)) + (throw (ex-info "camera steps must be a positive integer" + {:type ::invalid-steps :steps steps + :partial (select-keys session [:cached-slices :authored-sleeps])}))) + (reduce (fn [session-p i] + (let [t (/ i steps) + view (if (= i steps) + to + {:scale (interpolate (:scale from) (:scale to) t) + :x (interpolate (:x from) (:x to) t) + :y (interpolate (:y from) (:y to) t)})] + (preview-view! session-p + view))) + (js/Promise.resolve session) + (range 1 (inc steps))))))) + +(defn sleep! + "Pauses the camera session and records the requested and observed wait. + The wait must be finite and nonnegative. The pause extends the active + interval. Resolves to the updated session; + errors retain completed previews and pauses under `:partial`." + [session-p requested-ms] + (-> (js/Promise.resolve session-p) + (.then (fn [{:keys [hooks] :as session}] + (let [{:keys [now sleep check]} hooks + evidence (select-keys session [:cached-slices :authored-sleeps])] + (resolve-or-reject-with cause + (partial-error cause evidence) + (when-not (and (number? requested-ms) + (js/Number.isFinite requested-ms) + (<= 0 requested-ms)) + (throw (ex-info "sleep duration must be a nonnegative finite number" + {:type ::invalid-sleep :requested-ms requested-ms}))) + (check) + (let [started-ms (now)] + (-> (sleep requested-ms) + (.then (fn [_] + (check) + (let [ended-ms (now)] + (-> session + (assoc :active-end-ms ended-ms) + (update :authored-sleeps conj + {:requested-ms requested-ms + :actual-ms (- ended-ms started-ms)}))))) + (.catch (fn [cause] (throw (partial-error cause evidence)))))))))))) + +(defn- finish-interaction + "Ends the interaction and resolves to camera durations and recorded render calls. + Errors retain completed previews and pauses under `:partial`." + [{:keys [now check set-view-end] :as hooks} + {:keys [settle-ms started-ms cached-slices last-input-ms active-end-ms authored-sleeps]}] + (check) + + (let [finalization-start (now) + t0 (now)] + (set-view-end) + + (let [set-view-end-ms (- (now) t0)] + (-> (drain hooks + {:flags wasm/RENDER_FLAG_SYNC_TILES + :origin finalization-start + :immediate true}) + (.then + (fn [{:keys [slices end-ms viewport-ready-ms full-ms]}] + {:cached-slices cached-slices + :slices slices + :end-ms end-ms + :interact-ms (- end-ms started-ms) + :active-ms (- active-end-ms started-ms) + :authored-sleeps authored-sleeps + :settling-requested-ms settle-ms + :settling-actual-ms (- finalization-start active-end-ms) + :set-view-end-ms set-view-end-ms + :time-to-viewport-ready-ms viewport-ready-ms + :time-to-full-ms full-ms + :last-input-to-full-ms (- end-ms last-input-ms)})) + (.catch (fn [cause] + (throw (partial-error cause {:cached-slices cached-slices + :authored-sleeps authored-sleeps})))))))) + + +(defn finish-camera! + "Waits for settling, ends the interaction and renders until Full. + Resolves to cached previews, traces of final render calls, pauses and measured + camera durations. `:end-ms` is the clock reading at the final renderer return. + Errors retain completed previews and pauses under `:partial`." + [session-p] + (-> (js/Promise.resolve session-p) + (.then (fn [{:keys [hooks settle-ms] :as session}] + (let [evidence (select-keys session [:cached-slices :authored-sleeps])] + (resolve-or-reject-with cause + (partial-error cause evidence) + ((:check hooks)) + (-> ((:sleep hooks) settle-ms) + (.then (fn [_] (finish-interaction hooks session))) + (.catch (fn [cause] (throw (partial-error cause evidence))))))))))) diff --git a/frontend/performance/benches/render_wasm/scenes/masks.cljs b/frontend/performance/benches/render_wasm/scenes/masks.cljs new file mode 100644 index 0000000000..8deb1c4cbb --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/masks.cljs @@ -0,0 +1,59 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.masks + "Fixed grid of masked groups with circle masks and ordered descendants." + (:require + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.runtime.camera :as camera])) + +(def ^:const ^:private default-params + {:width 1920 :height 1080}) + +(def ^:const ^:private schema:params + [:map {:closed true} + [:width [:int {:min 1920}]] + [:height [:int {:min 1080}]]]) + +(decl/defscene :masks + {:description "Twenty-four circle masks with ordered striped descendants" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)}} + (doseq [i (range 24)] + (let [x (+ 25 (* 310 (mod i 6))) + y (+ 25 (* 255 (quot i 6)))] + (sb/group [:mask i] {:masked-group true} + (sb/circle {:x x :y y :width 180 :height 180 + :fills [{:fill-color "#ffffff"}]}) + (doseq [j (range 8)] + (sb/rect {:x (+ x (* 28 j) -22) + :y (- y 12) + :width 25 + :height 205 + :fills [{:fill-color (if (even? j) "#1254b8" "#ff8b38")}]})))))))) + +(def ^:const ^:private base-view {:scale 1 :x 0 :y 0}) + +(decl/defcase :masks/load + {:params default-params :view base-view :context :fresh :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :masks/pan + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 120 :dy 40 :settle-ms 100})) + +(decl/defcase :masks/zoom + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.6 :settle-ms 100})) diff --git a/frontend/performance/benches/render_wasm/scenes/paths.cljs b/frontend/performance/benches/render_wasm/scenes/paths.cljs new file mode 100644 index 0000000000..d96382ad30 --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/paths.cljs @@ -0,0 +1,78 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.paths + "Curved paths" + (:require + [app.common.schema :as sm] + [app.common.types.path :as path] + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :refer [rng-int]] + [benches.render-wasm.runtime.camera :as camera])) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Config +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:const ^:private default-params + {:count 1000 :width 1920 :height 1080}) + +(def ^:const ^:private schema:params + [:map {:closed true} + [:count [:int {:min 1 :max 100000}]] + [:width ::sm/positive-safe-number] + [:height ::sm/positive-safe-number]]) + +(def ^:const ^:private base-view {:scale 1 :x 0 :y 0}) + +(defn- curved-content + [{:keys [width height]}] + (fn [rng] + (let [x (rng-int rng 0 width) + y (rng-int rng 0 height) + w (rng-int rng 20 100) + h (rng-int rng 20 100)] + (path/from-plain + [{:command :move-to :params {:x x :y y}} + {:command :curve-to + :params {:c1x (+ x w) :c1y y + :c2x (+ x w) :c2y (+ y h) + :x (+ x w) :y (+ y h)}} + {:command :line-to :params {:x x :y (+ y h)}} + {:command :close-path}])))) + + +(decl/defscene :paths + {:description "Curved paths" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)} + :defaults {:path {:content (curved-content params) + :fills (sb/gen-vector (sb/gen-fill)) + :strokes (sb/gen-vector (sb/gen-stroke 3))}}} + (doseq [_ (range (:count params))] + (sb/path))))) + + +(decl/defcase :paths/load + {:params default-params :view base-view :context :fresh :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :paths/pan + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 160 :dy 40 :settle-ms 100})) + +(decl/defcase :paths/zoom + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.5 :settle-ms 100})) diff --git a/frontend/performance/benches/render_wasm/scenes/plus.cljs b/frontend/performance/benches/render_wasm/scenes/plus.cljs new file mode 100644 index 0000000000..d4021bcd85 --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/plus.cljs @@ -0,0 +1,68 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.plus + "Plus signs made of two open line subpaths" + (:require + [app.common.types.path :as path] + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :refer [rng-int]] + [benches.render-wasm.runtime.camera :as camera])) + +(def ^:private default-params + {:count 1000 :width 1920 :height 1080}) + +(def ^:private schema:params + [:map {:closed true} + [:count [:int {:min 1 :max 100000}]] + [:width [:int {:min 100}]] + [:height [:int {:min 100}]]]) + +(defn- plus-content + [{:keys [width height]}] + (fn [rng] + (let [x (rng-int rng 20 width) + y (rng-int rng 20 height) + r (rng-int rng 10 40)] + (path/from-plain + [{:command :move-to :params {:x (- x r) :y y}} + {:command :line-to :params {:x (+ x r) :y y}} + {:command :move-to :params {:x x :y (- y r)}} + {:command :line-to :params {:x x :y (+ y r)}}])))) + + +(decl/defscene :plus + {:description "Plus signs made of two open line subpaths" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 :y 0 :width (:width params) :height (:height params)} + :defaults {:path {:content (plus-content params) + :fills [] + :strokes (sb/gen-vector (sb/gen-stroke 4))}}} + (doseq [_ (range (:count params))] + (sb/path))))) + +(def ^:private base-view {:scale 1 :x 0 :y 0}) + +(decl/defcase :plus/load + {:params default-params :view base-view :context :fresh :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :plus/pan + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 140 :dy 60 :settle-ms 100})) + +(decl/defcase :plus/zoom + {:params default-params :view base-view :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.4 :settle-ms 100})) diff --git a/frontend/performance/benches/render_wasm/scenes/rects.cljs b/frontend/performance/benches/render_wasm/scenes/rects.cljs new file mode 100644 index 0000000000..6e8ec5b83c --- /dev/null +++ b/frontend/performance/benches/render_wasm/scenes/rects.cljs @@ -0,0 +1,118 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.scenes.rects + "Rectangle scene, the smallest benchmark workload. + + Distributes rectangles over the canvas with varied sizes, translucent + fills and centered strokes, all drawn from the scope's seeded generator. + The `defscene` body produces the scene snapshot; `defcase` declares + the scene and its standard cases. The three cases share parameters, so + they render the same scene." + (:require + [app.common.schema :as sm] + [benches.render-wasm.builder :as sb] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.runtime.camera :as camera])) + +;; This namespace is also loaded by Node discovery. Keep operation helpers +;; portable: browser API/helper imports belong in browser adapters supplied +;; through rtx, not in this require list or its transitive dependencies. +;; See declarations' dependency contract before adding a capability. + +(def ^:const ^:private default-params + {:count 1000 + :width 1920 + :height 1080 + :min-size 20 + :max-size 100}) + +(def ^:private schema:params + "Workload parameters. Sizes are finite. `:min-size` > `:max-size` is rejected" + [:and + [:map {:closed true} + [:count [:int {:min 1 :max 100000}]] + [:width ::sm/positive-safe-number] + [:height ::sm/positive-safe-number] + [:min-size ::sm/positive-safe-number] + [:max-size ::sm/positive-safe-number]] + [:fn {:error/message "min-size exceeds max-size"} + (fn [{:keys [min-size max-size]}] + (<= min-size max-size))]]) + +(defn- rect-defaults + "Attribute generators for one rectangle of the workload." + [{:keys [width height min-size max-size]}] + {:x (sb/gen-int 0 width) + :y (sb/gen-int 0 height) + :width (sb/gen-int min-size max-size) + :height (sb/gen-int min-size max-size) + :fills (sb/gen-vector (sb/gen-fill)) + :strokes (sb/gen-vector (sb/gen-stroke 10)) + :r1 (sb/gen-int 0 24) + :r2 (sb/gen-int 0 24) + :r3 (sb/gen-int 0 24) + :r4 (sb/gen-int 0 24)}) + +(decl/defscene :rects + {:description "Seeded rectangles with translucent fills and centered strokes" + :params-schema schema:params} + [params] + (let [params (merge default-params params)] + (sb/scene {:seed (:seed params) + :root {:x 0 + :y 0 + :width (:width params) + :height (:height params)} + :defaults {:rect (rect-defaults params)}} + (doseq [_ (range (:count params))] + (sb/rect))))) + +(def ^:private base-view + {:scale 1 :x 0 :y 0}) + +(decl/defcase :rects/load + {:params default-params + :view base-view + :context :fresh + :preparation {:render :none}} + [rtx] + (camera/load* rtx)) + +(decl/defcase :rects/pan + {:params default-params + :view base-view + :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/pan* rtx {:steps 20 :dx 200 :dy 0 :settle-ms 100})) + +(decl/defcase :rects/zoom + {:params default-params + :view base-view + :context :reuse + :reset #(camera/restore % (get-in % [:case :view]))} + [rtx] + (camera/zoom* rtx {:steps 20 :factor 1.5 :settle-ms 100})) + +(decl/defcase :rects/blocking-full + {:params default-params + :view base-view + :context :fresh + :preparation {:render :none}} + [rtx] + (camera/render-sync* rtx)) + +(decl/defcase :rects/mixed-pans + {:params default-params + :view base-view + :context :reuse + :reset #(camera/restore* % (get-in % [:case :view]))} + [rtx] + (-> rtx + (camera/pan* {:dx 120 :dy 0 :steps 12 :settle-ms 50}) + (camera/pan {:dx 0 :dy 80 :steps 8 :settle-ms 50}) + (camera/pan* {:dx -120 :dy 0 :steps 12 :settle-ms 50}))) diff --git a/frontend/performance/benches/render_wasm/snapshot.cljs b/frontend/performance/benches/render_wasm/snapshot.cljs new file mode 100644 index 0000000000..50c4f337e2 --- /dev/null +++ b/frontend/performance/benches/render_wasm/snapshot.cljs @@ -0,0 +1,203 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.snapshot + "Utilities shared by generated and saved-page scenes. + + A scene snapshot is + `{:objects {uuid shape ...} :refs {label uuid ...}}`. + + - `:objects` is a Penpot page object map. + - `:refs` is a snapshot-local lookup from custom labels to object uuids. + + A scene snapshot is just static data. It is independent of camera, operations, + WASM buffers and renderer calls, and it never mutates after upload. + + In order for benchmark scenes to follow editor semantics faithfully, + validation uses penpot's facilities. We do perform a manual check for + reachability from the root here because neither the editor validator nor file + loading guarantees it. + + Authoring and scoped construction are ticket17; containers ticket18/19; + saved-page loading ticket21. + + Which shape types may own children is container semantics owned by + ticket18/19; this contract only checks hierarchy consistency." + (:require + [app.common.files.helpers :as cfh] + [app.common.files.validate :as cfv] + [app.common.geom.shapes.tree-seq :as gts] + [app.common.schema :as sm] + [app.common.types.page :as ctp] + [app.common.uuid :as uuid])) + +(def schema:snapshot + "Envelope schema for a scene snapshot." + [:map {:title "RendererBenchmarkSceneSnapshot"} + [:objects ctp/schema:objects] + [:refs [:map-of :any ::sm/uuid]]]) + +(defn- fail! + [hint data] + (throw (ex-info (str "Invalid scene snapshot: " hint) + (merge {:type ::invalid-snapshot + :hint hint} + data)))) + +(defn- check-schema! + [instance] + (when-not (sm/validate schema:snapshot instance) + (fail! "instance does not match the scene snapshot schema" + {::sm/explain (sm/explain schema:snapshot instance)}))) + +(defn- check-root! + [instance] + (let [root (get-in instance [:objects uuid/zero])] + (when-not (and (some? root) (cfh/root? root)) + (fail! "root object must be a frame at uuid/zero" + {:id uuid/zero})) + (when-not (= uuid/zero (:parent-id root)) + (fail! "root object :parent-id must be uuid/zero" + {:id uuid/zero + :parent-id (:parent-id root)})) + (when-not (= uuid/zero (:frame-id root)) + (fail! "root object :frame-id must be uuid/zero" + {:id uuid/zero + :frame-id (:frame-id root)})))) + +(defn- check-object-ids! + [objects] + (doseq [[id shape] objects] + (when-not (= id (:id shape)) + (fail! "object key does not match the shape :id" + {:key id + :shape-id (:id shape)})))) + +(defn- check-child-links! + "Guard the reachability walk against malformed child links. + + `objects` is an object map keyed by id. A `shape` can have children + in `:shapes`. When present, `:shapes` must be a vector, and each listed + child that exists must point back to the listing shape." + [objects] + (doseq [[id shape] objects + :let [children (:shapes shape)]] + (when (some? children) + (when-not (vector? children) + (fail! "shape :shapes must be a vector" + {:id id})) + (doseq [child-id children + :let [child (get objects child-id)] + :when (some? child)] + (when-not (= id (:parent-id child)) + (fail! "shape :shapes lists a child with a different :parent-id" + {:id id + :child child-id})))))) + +(defn- check-reachability! + "Check the root graph through the cycle-safe descendant walk. + + Runs `check-child-links!` first, which keeps the walk linear: one parent + per child and vector `:shapes` mean the walk cannot fan out over shared + children or loop on a cycle. + + Detects missing child ids, objects reachable more than once (shared + children or cycles), detached subtrees and a root listed as its own + descendant. Runs before the production validator, whose walk follows + `:shapes` links and would loop on a reachable cycle." + [objects] + (check-child-links! objects) + (let [ids (cfh/get-children-ids objects uuid/zero) + seen (set ids)] + + (when (contains? seen uuid/zero) + (fail! "root must not be a descendant of itself" + {:id uuid/zero})) + + (doseq [id ids] + (when-not (contains? objects id) + (fail! "shape :shapes references a missing object" + {:child id}))) + + (when-let [dup (first (for [[id n] (frequencies ids) :when (> n 1)] id))] + (fail! "an object is reachable more than once" + {:id dup})) + + (doseq [id (keys objects) + :when (not= id uuid/zero)] + (when-not (contains? seen id) + (fail! "object is not reachable from the root" + {:id id}))))) + +;; Captured-page policy (tickets 02/26; implementation pending): extraction +;; maps canonical component/detach-shape over every selected-page shape, +;; without resolving component libraries, then rejects remaining unsupported +;; references/resources before validation. Warn and record when detachment +;; changes shapes; do not carry document context or use editor detach flows. +;; Thus the empty file/libraries below describe a standalone snapshot, not +;; support for arbitrary component-bearing pages. Move reusable checks to +;; shared .cljc code for JVM extraction and CLJS loading; do not copy them. + +(defn- check-referential-integrity! + "Run the production referential validation over the reachable graph. + + `cfv/validate-shape` checks parent/child links, frame ids and geometry + with editor semantics. The file and page only carry the uuids the + validation error schema requires; scenes contain no component shapes. + Unreachable objects never reach this walk; `check-reachability!` owns + them." + [instance] + (let [file {:id uuid/zero :data {}} + page {:id uuid/zero :objects (:objects instance)} + errors (cfv/validate-shape uuid/zero file page {})] + (when-let [{:keys [hint] :as error} (first errors)] + (fail! hint (select-keys error [:code :shape-id :args]))))) + +(defn- check-refs! + [instance] + (let [objects (:objects instance)] + (doseq [[label target] (:refs instance)] + (when-not (contains? objects target) + (fail! "ref target is not present in :objects" + {:label label + :target target}))))) + +(defn validate! + "Validates a scene snapshot and returns it unchanged. + + Throws `ex-info` with `{:type ::invalid-snapshot :hint ...}` when the + snapshot is invalid. Schema failures also carry `::sm/explain`. + + Explicit `throw`s are used instead of assertions so the checks survive + elided assertions." + [instance] + (check-schema! instance) + (let [objects (:objects instance)] + (check-root! instance) + (check-object-ids! objects) + ;; Reachability must run before the production validator, because that walk + ;; follows `:shapes` links and would not terminate on a reachable cycle + (check-reachability! objects) + (check-referential-integrity! instance) + (check-refs! instance)) + instance) + +(defn upload-order + "Returns the instance objects as a vector in parent-before-child order. + + First root, then `:shapes` in order. + + Note that upload order carries no renderer semantics because a shape's child + list is uploaded from its own `:shapes` vector. + + Assumes a validated instance (every child id exists)." + [instance] + (vec (gts/get-children-seq uuid/zero (:objects instance)))) + +(defn ref-id + "Resolves an author label to the referenced object uuid, or nil." + [instance label] + (get-in instance [:refs label])) diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index ef226a62b0..998ad8d7ea 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -175,4 +175,92 @@ {:compiler-options {:fn-invoke-direct true :elide-asserts true - :anon-fn-naming-policy :off}}}}} + :anon-fn-naming-policy :off}}} + + :performance-tests + {:target :esm + :output-dir "target/performance-tests" + :runtime :custom + :js-options {:js-provider :import} + + + :modules + {:test {:init-fn performance-tests.render-wasm.runner/init + :prepend-js ";if (typeof globalThis.navigator?.userAgent === 'undefined') { globalThis.navigator = {userAgent: ''}; };"}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-runner + {:target :node-script + :output-to "target/performance/render-wasm/runner.cjs" + :output-dir "target/performance/render-wasm/runner/" + :main benches.render-wasm.node.cli.main/-main + :devtools {:autoload false} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false + :warnings {:fn-deprecated false}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-commands + {:target :node-library + :output-to "target/performance/render-wasm/commands.cjs" + :output-dir "target/performance/render-wasm/commands/" + :exports {:execute benches.render-wasm.node.cli.deferred/execute} + :devtools {:autoload false} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false + :warnings {:fn-deprecated false}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-browser + {:target :esm + :output-dir #shadow/env ["RENDER_BENCH_BROWSER_DIR" :default "target/performance/render-wasm/browser/"] + :closure-defines {benches.render-wasm.identity/build-id #shadow/env ["RENDER_BENCH_BUILD_ID" :default "unprepared"]} + ;; Bundle npm dependencies for the dedicated static host. + :js-options {:js-provider :shadow} + + :modules + {:browser + {:exports {default benches.render-wasm.browser.bridge/bridge}}} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false + :warnings {:fn-deprecated false}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + + :bench-render-wasm-catalog + {:target :node-library + :output-to #shadow/env ["RENDER_BENCH_CASES_PATH" :default "target/performance/render-wasm/cases.cjs"] + :output-dir "target/performance/render-wasm/catalog/" + :exports {:collectCases benches.render-wasm.node.catalog.entry/collect-cases + :buildIdentity benches.render-wasm.node.catalog.entry/build-identity} + :closure-defines {benches.render-wasm.identity/build-id #shadow/env ["RENDER_BENCH_BUILD_ID" :default "unprepared"]} + :compiler-options {:output-feature-set :es2020 + :warnings {:fn-deprecated false}} + :release {:compiler-options {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}}}} diff --git a/frontend/test/frontend_tests/render_wasm/serialization_test.cljs b/frontend/test/frontend_tests/render_wasm/serialization_test.cljs index b204bc13b6..88fba33b67 100644 --- a/frontend/test/frontend_tests/render_wasm/serialization_test.cljs +++ b/frontend/test/frontend_tests/render_wasm/serialization_test.cljs @@ -92,15 +92,6 @@ path? (conj [:path id (:content shape)]))))) cases)) -(t/deftest tail-cases-flags-are-consistent - (doseq [{:keys [shape tail? svg? path?]} tail-cases] - (t/is (boolean? tail?) "tail? is a boolean") - (t/is (= tail? (boolean (or svg? path?))) "tail? matches the write flags") - (t/is (= svg? (some? (:svg-attrs shape))) "svg? matches the shape") - (t/is (= path? (boolean (and (contains? #{:path :bool} (:type shape)) - (some? (:content shape))))) - "path? matches the shape"))) - (t/deftest needs-shape-tail-matches-spec-table (doseq [{:keys [shape tail?]} tail-cases] (t/is (= tail? (serialize-shape/needs-shape-tail? shape)) diff --git a/frontend/test/performance_tests/render_wasm/artifacts_test.cljs b/frontend/test/performance_tests/render_wasm/artifacts_test.cljs new file mode 100644 index 0000000000..4e4efbf696 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/artifacts_test.cljs @@ -0,0 +1,566 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.artifacts-test + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + ["worker_threads" :as workers] + [app.common.transit :as transit] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.catalog.loader :as catalog] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [cljs.test :as t :include-macros true])) + +(def ^:private temporary-root (atom nil)) +(def ^:private selection {:default-features? true :features []}) +(def ^:private provenance {:git {:sha "test-revision" :dirty false}}) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-artifacts-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- create-build + [] + (artifacts/create-build @temporary-root selection provenance)) + +(defn- write-file + [file content] + (fs/mkdirSync (path/dirname file) #js {:recursive true}) + (fs/writeFileSync file content "utf8")) + +(defn- complete-files + [build] + (write-file (:cases-path build) "module.exports = {};") + (write-file (path/join (:browser-dir build) "browser.js") "export default {};") + (write-file (path/join (:renderer-dir build) "render-wasm.js") "export default {};") + (fs/writeFileSync (path/join (:renderer-dir build) "render-wasm.wasm") + (js/Buffer.from #js [0 97 115 109 1 0 0 0])) + (write-file (path/join (:renderer-dir build) "shared.js") "export default {};") + (write-file (:index-path build) "selected build") + build) + +(defn- ready-build + [] + (let [build (complete-files (create-build))] + (artifacts/write-manifest build (assoc (:manifest build) :status :ready)))) + +(defn- failure + [f] + (try + (f) + nil + (catch :default error error))) + +(t/deftest frontend-root-is-independent-of-caller-cwd + (let [expected (artifacts/frontend-root) + previous (.cwd js/process)] + (try + (.chdir js/process @temporary-root) + (t/is (= expected (artifacts/frontend-root))) + (t/is (.isFile (fs/statSync (path/join expected "deps.edn")))) + (t/is (.isDirectory (fs/statSync + (path/join expected "performance" "benches" "render_wasm")))) + (finally + (.chdir js/process previous))))) + +(t/deftest create-build-retains-its-initial-release-evidence + (let [selected {:default-features? false :features ["feature-one" "feature-two"]} + build (artifacts/create-build @temporary-root selected provenance) + manifest (transit/decode-str (fs/readFileSync (:manifest-path build) "utf8"))] + (t/is (re-matches #"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + (:id build))) + (t/is (= (path/join (artifacts/managed-root @temporary-root) "builds" (:id build)) + (:dir build))) + (t/is (= (path/join (:dir build) "build.transit") (:manifest-path build))) + (t/is (= (path/join (:dir build) "index.html") (:index-path build))) + (t/is (= manifest (:manifest build))) + (t/is (= #{:type :id :status :created-at :updated-at :functional :provenance} + (set (keys manifest)))) + (t/is (= {:type :renderer-build :status :building} + (select-keys manifest [:type :status]))) + (t/is (= {:mode :release :default-features? false + :features ["feature-one" "feature-two"] :cljs-profile-marks false} + (:functional manifest))) + (t/is (= provenance (:provenance manifest))) + (t/is (= (:created-at manifest) (.toISOString (js/Date. (:created-at manifest))))) + (t/is (= (:created-at manifest) (:updated-at manifest))) + (t/is (= "" (fs/readFileSync (:log-path build) "utf8"))) + (t/is (= (path/join (:dir build) "cases.cjs") (:cases-path build))) + (t/is (.isDirectory (fs/statSync (:browser-dir build)))) + (t/is (.isDirectory (fs/statSync (:renderer-dir build)))))) + +(t/deftest ready-build-loads-all-selected-paths + (let [build (ready-build) + loaded (artifacts/load-build @temporary-root (:id build))] + (t/is (= build loaded)) + (t/is (= (path/join (:dir build) "index.html") (:index-path loaded))))) + +(t/deftest complete-building-files-can-be-checked-before-publishing-ready + (let [build (complete-files (create-build))] + (t/is (= build (artifacts/check-files build))) + (t/is (= :building (:status (transit/decode-str + (fs/readFileSync (:manifest-path build) "utf8"))))) + (t/is (= ::artifacts/not-ready + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + +(t/deftest incomplete-building-files-are-rejected-before-publishing-ready + (doseq [relative ["cases.cjs" "browser/browser.js" "renderer/render-wasm.js" + "renderer/render-wasm.wasm" "renderer/shared.js" "index.html"]] + (let [build (complete-files (create-build))] + (fs/unlinkSync (path/join (:dir build) relative)) + (t/is (some? (failure #(artifacts/check-files build))) relative) + (t/is (= :building (:status (transit/decode-str + (fs/readFileSync (:manifest-path build) "utf8")))))))) + +(t/deftest building-and-failed-builds-are-not-loadable-and-retain-evidence + (let [building (create-build) + failed (create-build) + log "renderer compilation failed\n"] + (fs/appendFileSync (:log-path failed) log "utf8") + (artifacts/write-manifest failed (assoc (:manifest failed) :status :failed + :failure {:phase "renderer" :message "compile failed"})) + (doseq [build [building failed]] + (t/is (= ::artifacts/not-ready + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build)))))))) + (t/is (= :failed (:status (transit/decode-str + (fs/readFileSync (:manifest-path failed) "utf8"))))) + (t/is (= log (fs/readFileSync (:log-path failed) "utf8"))) + (t/is (fs/existsSync (:dir building))))) + +(t/deftest stale-references-cannot-reopen-terminal-builds + (doseq [status [:ready :failed]] + (let [build (create-build) + terminal (artifacts/write-manifest build (assoc (:manifest build) :status status)) + saved (fs/readFileSync (:manifest-path build) "utf8")] + (t/is (= terminal (artifacts/write-manifest terminal (:manifest terminal)))) + (doseq [next-status [:building :ready :failed]] + (let [attempt (assoc (:manifest terminal) :status next-status :extra "changed")] + (t/is (= ::artifacts/terminal-build + (:type (ex-data (failure #(artifacts/write-manifest build attempt)))))))) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8")))))) + +(t/deftest unencodable-update-keeps-the-last-manifest-and-log + (let [build (create-build) + saved (fs/readFileSync (:manifest-path build) "utf8")] + (fs/appendFileSync (:log-path build) "saved diagnostic\n" "utf8") + (t/is (some? (failure #(artifacts/write-manifest + build (assoc (:manifest build) :callback (fn [] nil)))))) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8"))) + (t/is (= "saved diagnostic\n" (fs/readFileSync (:log-path build) "utf8"))) + (t/is (= #{"browser" "renderer" "build.log" "build.transit"} + (set (array-seq (fs/readdirSync (:dir build)))))))) + +(t/deftest malformed-builds-cannot-replace-saved-manifests + (doseq [side [:current :replacement]] + (let [build (create-build) + valid (:manifest build) + malformed (dissoc valid :functional) + proposed (if (= :replacement side) malformed valid)] + (when (= :current side) + (fs/writeFileSync (:manifest-path build) (codec/encode-str malformed) "utf8")) + (let [saved (fs/readFileSync (:manifest-path build) "utf8") + error (failure #(artifacts/write-manifest build proposed))] + (t/is (= ::artifacts/invalid-manifest (:type (ex-data error))) (name side)) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8")) (name side)))))) + +(def ^:private manifest-reader + (str "const {parentPort, workerData} = require('worker_threads');" + "const fs = require('fs'); const stop = new Int32Array(workerData.stop);" + "let reads = 0; const errors = [];" + "function read() { try { JSON.parse(fs.readFileSync(workerData.path, 'utf8'));" + "reads++; } catch (error) { if (errors.length < 5) errors.push(error.message); } }" + "read(); parentPort.postMessage('ready');" + "while (Atomics.load(stop, 0) === 0) read();" + "parentPort.postMessage({reads, errors});")) + +(defn- worker-message + [^js worker] + (js/Promise. + (fn [resolve reject] + (letfn [(received [value] + (.removeListener worker "error" failed) + (resolve value)) + (failed [error] + (.removeListener worker "message" received) + (reject error))] + (.once worker "message" received) + (.once worker "error" failed))))) + +(t/deftest ^:async manifest-replacements-never-expose-a-partial-file + (let [build (create-build) + stop (js/SharedArrayBuffer. 4) + ^js worker (workers/Worker. manifest-reader + #js {:eval true + :workerData #js {:path (:manifest-path build) :stop stop}})] + (try + (await (worker-message worker)) + (let [result (worker-message worker)] + (dotimes [n 24] + (artifacts/write-manifest + build (assoc (:manifest build) :progress n + :diagnostic (apply str (repeat (+ 8192 (* n 1024)) "evidence "))))) + (js/Atomics.store (js/Int32Array. stop) 0 1) + (let [^js observed (await result)] + (t/is (> (.-reads observed) 1) "a separate thread read during replacement") + (t/is (empty? (array-seq (.-errors observed))) + "every observed manifest remained a complete JSON document"))) + (t/is (= 23 (:progress (transit/decode-str + (fs/readFileSync (:manifest-path build) "utf8"))))) + (finally + (await (.terminate worker)))))) + +(t/deftest malformed-manifests-and-incomplete-builds-are-rejected + (doseq [change [{:functional []} {:type :run-record} {:id "another-build"} + {:status :unknown}]] + (let [build (ready-build)] + (fs/writeFileSync (:manifest-path build) (codec/encode-str (merge (:manifest build) change)) + "utf8") + (t/is (= ::artifacts/unsupported-manifest + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + (doseq [relative ["cases.cjs" "browser/browser.js" "renderer/render-wasm.js" + "renderer/render-wasm.wasm" "renderer/shared.js" "index.html"]] + (let [build (ready-build)] + (fs/unlinkSync (path/join (:dir build) relative)) + (t/is (some? (failure #(artifacts/load-build @temporary-root (:id build)))) relative)))) + +(t/deftest saved-builds-validate-current-shapes + (let [build (ready-build) + manifest (select-keys (:manifest build) + [:type :id :status :created-at :updated-at :functional :provenance])] + (fs/writeFileSync (:manifest-path build) (codec/encode-str manifest) "utf8") + (t/is (= manifest (:manifest (artifacts/load-build @temporary-root (:id build))))) + (fs/writeFileSync (:manifest-path build) + (codec/encode-str (dissoc manifest :functional)) "utf8") + (t/is (= ::artifacts/unsupported-manifest + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + +(t/deftest build-environment-roundtrips-without-inventing-missing-evidence + (t/is (not (contains? (:manifest (create-build)) :environment))) + (doseq [environment [{} {"CFLAGS" "" "RUSTFLAGS" "-C\u001fopt-level=3"}]] + (let [build (complete-files (create-build)) + ready (artifacts/write-manifest build (assoc (:manifest build) + :status :ready :environment environment)) + saved (artifacts/load-build @temporary-root (:id ready))] + (t/is (= environment (get-in saved [:manifest :environment])))))) + +(t/deftest build-environment-rejects-non-string-keys-and-values + (doseq [environment [nil [] {:CFLAGS ""} {"CFLAGS" nil} {"CFLAGS" 0}]] + (let [build (create-build) + saved (fs/readFileSync (:manifest-path build) "utf8") + cause (failure #(artifacts/write-manifest build (assoc (:manifest build) :environment environment)))] + (t/is (= ::artifacts/invalid-manifest (:type (ex-data cause)))) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8")))))) + +(t/deftest ids-cannot-select-or-delete-outside-paths + (let [build (ready-build) + outside (path/join @temporary-root "outside.txt")] + (write-file outside "keep this") + (doseq [id [nil "" "." ".." "../outside.txt" "safe/../../outside.txt" + "..\\outside.txt" "/tmp/outside" "id%2fother"]] + (t/is (= ::artifacts/invalid-id + (:type (ex-data (failure #(artifacts/load-build @temporary-root id)))))) + (t/is (= ::artifacts/invalid-id + (:type (ex-data (failure #(artifacts/remove-builds + @temporary-root [(:id build) id]))))))) + (t/is (= build (artifacts/load-build @temporary-root (:id build)))) + (t/is (= "keep this" (fs/readFileSync outside "utf8"))))) + +(t/deftest safe-non-uuid-ids-can-load-existing-builds + (let [build (ready-build) + id "retained-build_1" + dir (path/join (path/dirname (:dir build)) id)] + (fs/renameSync (:dir build) dir) + (fs/writeFileSync (path/join dir "build.transit") + (codec/encode-str (assoc (:manifest build) :id id)) "utf8") + (t/is (= id (:id (artifacts/load-build @temporary-root id)))) + (t/is (= dir (:dir (artifacts/load-build @temporary-root id)))))) + +(t/deftest build-and-artifact-symlinks-cannot-escape-the-managed-root + (let [build (ready-build) + outside (path/join @temporary-root "outside.txt")] + (write-file outside "private") + (fs/unlinkSync (:cases-path build)) + (fs/symlinkSync outside (:cases-path build)) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/check-files build)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build)))))))) + (let [build (ready-build) + outside (path/join @temporary-root "outside-build")] + (fs/renameSync (:dir build) outside) + (fs/symlinkSync outside (:dir build) "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/check-files build)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) + +(t/deftest category-purge-unlinks-only-known-outputs-and-keeps-directories + (doseq [category [:builds :runs :all]] + (let [root (fs/mkdtempSync (path/join @temporary-root (str (name category) "-"))) + building (complete-files (artifacts/create-build root selection provenance)) + failed (artifacts/create-build root selection provenance) + managed (artifacts/managed-root root) + run-dir (path/join managed "runs" "group" "saved-run") + outputs ["build.transit" "build.log" "cases.cjs" "index.html" + "browser/browser.js" "renderer/render-wasm.js" + "renderer/render-wasm.wasm" "renderer/render-wasm.wasm.map" + "renderer/shared.js" "renderer/cargo-build.json" + "renderer/worker/render.js"] + boot (path/join managed "runner.cjs") + outside (path/join root "explicit-output" "run.transit")] + (artifacts/write-manifest failed (assoc (:manifest failed) :status :failed)) + (doseq [file outputs] + (write-file (path/join (:dir building) file) "build output")) + (doseq [file ["run.log" "run.transit" "shot.png" "SHOT.PNG"]] + (write-file (path/join run-dir file) "run output")) + (doseq [dir ["run.log" "run.transit" "directory.PNG"]] + (write-file (path/join managed "runs" dir "keep.txt") "keep directory")) + (write-file (path/join managed "runs" "no-marker" "only.PNG") "screenshot") + (write-file (path/join run-dir "record.transit") "unrelated run file") + (write-file (path/join (:dir building) "custom" "build.log") "unrelated build file") + (write-file boot "boot executable") + (write-file outside "unmanaged") + (artifacts/purge root category) + (doseq [dir [(:dir building) (:dir failed) (:browser-dir building) + (:renderer-dir building) run-dir]] + (t/is (.isDirectory (fs/statSync dir)))) + (doseq [file outputs] + (t/is (= (= category :runs) (fs/existsSync (path/join (:dir building) file))))) + (t/is (= (= category :runs) (fs/existsSync (:manifest-path failed)))) + (doseq [file ["run.log" "run.transit" "shot.png" "SHOT.PNG"]] + (t/is (= (= category :builds) (fs/existsSync (path/join run-dir file))))) + (t/is (= (= category :builds) + (fs/existsSync (path/join managed "runs" "no-marker" "only.PNG")))) + (doseq [dir ["run.log" "run.transit" "directory.PNG"]] + (t/is (fs/existsSync (path/join managed "runs" dir "keep.txt")))) + (t/is (fs/existsSync (path/join run-dir "record.transit"))) + (t/is (fs/existsSync (path/join (:dir building) "custom" "build.log"))) + (t/is (= "boot executable" (fs/readFileSync boot "utf8"))) + (t/is (= "unmanaged" (fs/readFileSync outside "utf8"))) + (artifacts/purge root category)))) + +(t/deftest comparison-purge-unlinks-only-feature-comparison-evidence + (doseq [category [:comparisons :all]] + (let [root (fs/mkdtempSync (path/join @temporary-root (str (name category) "-"))) + managed (artifacts/managed-root root) + build-file (path/join managed "builds" "saved" "build.transit") + ab-dir (path/join managed "feature-comparisons" "ab-1") + config-dir (path/join ab-dir "configs" "00") + edge-dir (path/join ab-dir "comparisons") + run-dir (path/join managed "runs" "saved") + outside (path/join root "unmanaged" "feature-comparison.transit") + outputs [["feature-comparison.log" ab-dir] + ["feature-comparison.transit" ab-dir] + ["run.log" config-dir] + ["run.transit" config-dir] + ["shot.PNG" config-dir] + ["00__10.transit" edge-dir]] + kept [["notes.txt" ab-dir] + ["keep.txt" config-dir] + ["keep.txt" edge-dir]]] + (doseq [[file dir] outputs + :let [path (path/join dir file)]] + (write-file path "comparison output")) + (doseq [[file dir] kept] + (write-file (path/join dir file) "unrelated")) + (write-file (path/join managed "feature-comparisons" "keep.txt") + "unrelated root") + (write-file (path/join managed "feature-comparisons" "ab-1" "directory.PNG" + "keep.txt") + "keep directory") + (write-file build-file "retained build") + (write-file (path/join run-dir "run.transit") "standalone run") + (write-file outside "unmanaged") + (artifacts/purge root category) + (doseq [[file dir] outputs] + (t/is (not (fs/existsSync (path/join dir file))) (str category " " file))) + (doseq [[file dir] kept] + (t/is (fs/existsSync (path/join dir file)) (str category " " file))) + (t/is (= "unrelated root" + (fs/readFileSync (path/join managed "feature-comparisons" "keep.txt") + "utf8"))) + (t/is (fs/existsSync (path/join ab-dir "directory.PNG" "keep.txt"))) + (t/is (.isDirectory (fs/statSync ab-dir))) + (t/is (.isDirectory (fs/statSync config-dir))) + (t/is (.isDirectory (fs/statSync edge-dir))) + (t/is (= (= category :comparisons) + (fs/existsSync (path/join run-dir "run.transit"))) + (str category " standalone run")) + (t/is (= (= category :comparisons) (fs/existsSync build-file)) + (str category " retained build")) + (when (= category :comparisons) + (t/is (= "standalone run" + (fs/readFileSync (path/join run-dir "run.transit") "utf8"))) + (t/is (= "retained build" (fs/readFileSync build-file "utf8")))) + (t/is (= "unmanaged" (fs/readFileSync outside "utf8")))))) + +(t/deftest purge-follows-category-and-nested-directory-symlinks-and-stops-cycles + (let [managed (artifacts/managed-root @temporary-root)] + (fs/mkdirSync managed #js {:recursive true}) + (doseq [category [:builds :runs]] + (let [outside (path/join @temporary-root (str "outside-" (name category))) + detached (path/join @temporary-root (str "detached-" (name category))) + output (if (= category :builds) "build.transit" "run.transit") + saved (path/join outside "keep.txt") + link (path/join managed (name category))] + (write-file saved "keep symlink target") + (write-file (path/join outside "saved" output) "output, not a parsed record") + (write-file (path/join detached output) "linked output") + (write-file (path/join detached "keep.txt") "unrelated") + (fs/symlinkSync outside (path/join outside "saved" "cycle") "dir") + (fs/symlinkSync detached (path/join outside "linked") "dir") + (fs/symlinkSync outside link "dir") + (artifacts/purge @temporary-root category) + (t/is (.isSymbolicLink (fs/lstatSync link))) + (t/is (.isSymbolicLink (fs/lstatSync (path/join outside "saved" "cycle")))) + (t/is (.isSymbolicLink (fs/lstatSync (path/join outside "linked")))) + (t/is (not (fs/existsSync (path/join outside "saved" output)))) + (t/is (not (fs/existsSync (path/join detached output)))) + (t/is (fs/existsSync (path/join detached "keep.txt"))) + (t/is (= "keep symlink target" (fs/readFileSync saved "utf8"))))))) + +(t/deftest managed-root-symlink-cannot-create-or-delete-outside-the-frontend + (let [root (path/join @temporary-root "frontend") + managed (path/join root "target" "performance" "render-wasm") + outside (path/join @temporary-root "outside-managed") + build-log (path/join outside "builds" "retained" "build.log") + run-file (path/join outside "runs" "retained" "record.transit")] + (write-file build-log "outside build") + (write-file run-file "outside run") + (fs/mkdirSync (path/dirname managed) #js {:recursive true}) + (fs/symlinkSync outside managed "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/managed-root root)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/create-build root selection provenance)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/purge root :all)))))) + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/remove-builds root ["retained"])))))) + (t/is (= "outside build" (fs/readFileSync build-log "utf8"))) + (t/is (= "outside run" (fs/readFileSync run-file "utf8"))) + (t/is (= #{"retained"} (set (array-seq (fs/readdirSync (path/join outside "builds")))))))) + +(t/deftest escaping-target-ancestor-is-rejected-before-the-managed-path-exists + (let [root (path/join @temporary-root "frontend-ancestor") + outside (path/join @temporary-root "outside-target")] + (fs/mkdirSync root) + (fs/mkdirSync outside) + (fs/symlinkSync outside (path/join root "target") "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/create-build root selection provenance)))))) + (t/is (empty? (array-seq (fs/readdirSync outside)))))) + +(t/deftest create-build-rejects-an-escaping-builds-parent-before-writing-files + (let [managed (artifacts/managed-root @temporary-root) + outside (path/join @temporary-root "outside-builds") + saved (path/join outside "retained" "build.log")] + (write-file saved "keep external build") + (fs/mkdirSync managed #js {:recursive true}) + (fs/symlinkSync outside (path/join managed "builds") "dir") + (t/is (= ::artifacts/unsafe-path + (:type (ex-data (failure #(artifacts/create-build + @temporary-root selection provenance)))))) + (t/is (= "keep external build" (fs/readFileSync saved "utf8"))) + (t/is (= #{"retained"} (set (array-seq (fs/readdirSync outside))))))) + +(t/deftest remove-builds-cleans-selected-ids-and-keeps-directories-and-unrelated-files + (let [removed (ready-build) + kept (ready-build) + managed (artifacts/managed-root @temporary-root) + run (path/join managed "runs" "saved" "run.transit") + outside (path/join @temporary-root "outside" "keep.txt") + browser (path/join @temporary-root "linked-browser") + link (path/join managed "builds" "outside-link")] + (write-file run "keep run") + (write-file outside "keep target") + (write-file (path/join (:dir removed) "screenshot.PNG") "keep build screenshot") + (write-file (path/join (path/dirname outside) "build.transit") "linked build output") + (write-file (path/join browser "browser.js") "linked browser output") + (write-file (path/join browser "keep.js") "unrelated") + (fs/symlinkSync browser (path/join (path/dirname outside) "browser") "dir") + (fs/symlinkSync (path/dirname outside) link "dir") + (artifacts/remove-builds @temporary-root [(:id removed) "outside-link" "absent-build"]) + (t/is (.isDirectory (fs/statSync (:dir removed)))) + (t/is (not (fs/existsSync (:manifest-path removed)))) + (t/is (not (fs/existsSync (:cases-path removed)))) + (t/is (fs/existsSync (path/join (:dir removed) "screenshot.PNG"))) + (t/is (.isSymbolicLink (fs/lstatSync link))) + (t/is (not (fs/existsSync (path/join link "build.transit")))) + (t/is (not (fs/existsSync (path/join browser "browser.js")))) + (t/is (fs/existsSync (path/join browser "keep.js"))) + (t/is (= kept (artifacts/load-build @temporary-root (:id kept)))) + (t/is (= "keep run" (fs/readFileSync run "utf8"))) + (t/is (= "keep target" (fs/readFileSync outside "utf8"))))) + +(t/deftest inside-builds-resolves-symlink-ancestors-with-missing-output-components + (let [managed (artifacts/managed-root @temporary-root) + builds (path/join managed "builds") + outside (path/join @temporary-root "external-builds") + alias (path/join @temporary-root "build-alias")] + (t/is (artifacts/inside-builds? @temporary-root (path/join builds "future" "run"))) + (fs/mkdirSync managed #js {:recursive true}) + (fs/mkdirSync outside) + (fs/symlinkSync outside builds "dir") + (fs/symlinkSync builds alias "dir") + (doseq [output [builds outside (path/join builds "new" "run") + (path/join outside "new" "run") (path/join alias "new" "run")]] + (t/is (artifacts/inside-builds? @temporary-root output))) + (t/is (not (artifacts/inside-builds? @temporary-root (str outside "-neighbor/run")))) + (t/is (not (artifacts/inside-builds? @temporary-root (path/join managed "runs" "new")))))) + +(defn- descriptor + [id] + {:id id + :scene :artifact-only + :scene-description "A descriptor supplied only by this build" + :scene-seed 42 + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {}}) + +(defn- write-catalog + [build loaded-id response] + (write-file + (:cases-path build) + (str "const identity = " (js/JSON.stringify (codec/encode-str {:build-id loaded-id})) ";" + "const response = " (js/JSON.stringify (codec/encode-str response)) ";" + "module.exports = {buildIdentity() {return identity;}," + "collectCases(encodedSelection) {return response;}};"))) + +(t/deftest catalogs-use-the-selected-commonjs-module-with-isolated-scope + (let [first-build (ready-build) + second-build (ready-build) + first-case (descriptor :artifact-only/first) + second-case (descriptor :artifact-only/second) + selection {:master-seed 42}] + (write-catalog first-build (:id first-build) + {:build-id (:id first-build) :cases [first-case]}) + (write-catalog second-build (:id second-build) + {:build-id (:id second-build) :cases [second-case]}) + (t/is (= [first-case] (catalog/collect first-build selection))) + (t/is (= [second-case] (catalog/collect second-build selection))) + (t/is (= [first-case] (catalog/collect first-build selection))))) + +(t/deftest catalog-rejects-both-loaded-and-response-identity-mismatches + (doseq [boundary [:loaded :response]] + (let [build (ready-build)] + (write-catalog build (if (= boundary :loaded) "wrong-build" (:id build)) + {:build-id (if (= boundary :response) "wrong-build" (:id build)) + :cases [(descriptor :artifact-only/load)]}) + (t/is (= ::identity/mismatch + (:type (ex-data (failure #(catalog/collect build {:master-seed 42}))))))))) + +(t/deftest catalog-rejects-malformed-descriptors-and-empty-responses + (doseq [cases [[(assoc (descriptor :artifact-only/load) :scene-seed -1)] [] nil]] + (let [build (ready-build)] + (write-catalog build (:id build) {:build-id (:id build) :cases cases}) + (t/is (some? (failure #(catalog/collect build {:master-seed 42}))))))) diff --git a/frontend/test/performance_tests/render_wasm/boolean_test.cljs b/frontend/test/performance_tests/render_wasm/boolean_test.cljs new file mode 100644 index 0000000000..ccea94ce2e --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/boolean_test.cljs @@ -0,0 +1,214 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.boolean-test + (:require + [app.common.geom.point :as gpt] + [app.common.geom.shapes.intersect :as gint] + [app.common.types.path :as path] + [app.common.types.path.segment :as segm] + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as b] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(t/deftest difference-of-overlapping-rects-keeps-hierarchy-and-bounds + (let [red {:fill-color "#ff0000" :fill-opacity 1} + blue {:fill-color "#0000ff" :fill-opacity 1} + instance (b/scene {:seed 1} + (b/bool :l-shape {:bool-type :difference :name "L"} + (b/rect {:x 0 :y 0 :width 100 :height 100 :fills [red]}) + (b/rect {:x 50 :y 50 :width 100 :height 100 :fills [blue]}))) + bool (first (helpers/children-of instance uuid/zero)) + children (helpers/children-of instance (:id bool))] + (t/is (= :bool (:type bool))) + (t/is (= :difference (:bool-type bool))) + (t/is (= (:id bool) (snapshot/ref-id instance :l-shape))) + (t/is (= 2 (count children))) + (t/is (= [(:id (first children)) (:id (second children))] (:shapes bool))) + (t/is (= [(:id bool) (:id bool)] + [(:parent-id (first children)) (:parent-id (second children))])) + (t/is (= [uuid/zero uuid/zero] + [(:frame-id (first children)) (:frame-id (second children))])) + (t/is (= [0 0 100 100] (helpers/selrect bool))))) + +(t/deftest difference-excludes-points-in-removed-overlap + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100}))) + bool (first (helpers/children-of instance uuid/zero)) + lines (segm/path->lines bool)] + (t/is (true? (gint/is-point-inside-nonzero? (gpt/point 25 25) lines)) + "a point in the kept region stays inside the content") + (t/is (false? (gint/is-point-inside-nonzero? (gpt/point 75 75) lines)) + "a point in the subtracted overlap is excluded from the content"))) + +(t/deftest union-derives-canonical-bounds + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :union} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= :union (:bool-type bool))) + (t/is (= [0 0 150 150] (helpers/selrect bool))))) + +(t/deftest difference-picks-the-first-child-as-style-head + (let [red {:fill-color "#ff0000" :fill-opacity 1} + blue {:fill-color "#0000ff" :fill-opacity 1} + blur {:id (uuid/custom 3 3) :type :layer-blur :value 4 :hidden false} + difference (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 10 :height 10 + :fills [red] :blur blur}) + (b/rect {:x 5 :y 0 :width 10 :height 10 + :fills [blue]}))) + diff-bool (first (helpers/children-of difference uuid/zero))] + (t/is (= [red] (:fills diff-bool))) + (t/is (= blur (:blur diff-bool))))) + +(t/deftest supplied-styles-override-inherited + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :union :fills []} + (b/rect {:x 0 :y 0 :width 10 :height 10 + :fills [{:fill-color "#ff0000" :fill-opacity 1}]}) + (b/rect {:x 5 :y 0 :width 10 :height 10 + :fills [{:fill-color "#0000ff" :fill-opacity 1}]}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= [] (:fills bool))))) + +(t/deftest nil-styles-do-not-count-as-supplied + (let [red {:fill-color "#ff0000" :fill-opacity 1} + blue {:fill-color "#0000ff" :fill-opacity 1} + instance (b/scene {:seed 1} + (b/bool {:bool-type :union :fills nil} + (b/rect {:x 0 :y 0 :width 10 :height 10 :fills [red]}) + (b/rect {:x 5 :y 0 :width 10 :height 10 :fills [blue]}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= [blue] (:fills bool))))) + +(t/deftest single-child-bool-uses-the-child-path + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 10 :y 20 :width 30 :height 40}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= 1 (count (:shapes bool)))) + (t/is (= [10 20 30 40] (helpers/selrect bool))) + (t/is (= (:selrect bool) (path/calc-selrect (:content bool)))))) + +(t/deftest nested-bools-finalize-inside-out + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :union} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100})) + (b/rect {:x 200 :y 0 :width 50 :height 50}))) + outer (first (helpers/children-of instance uuid/zero)) + inner (first (helpers/children-of instance (:id outer)))] + (t/is (= :bool (:type inner))) + (t/is (= [0 0 100 100] (helpers/selrect inner))) + (t/is (= [0 0 250 100] (helpers/selrect outer))) + (t/is (= (:selrect outer) (path/calc-selrect (:content outer)))))) + +(t/deftest bool-inside-frame-keeps-frame-ids + (let [instance (b/scene {:seed 1} + (b/frame {:x 0 :y 0 :width 200 :height 200} + (b/bool {:bool-type :union} + (b/rect {:x 10 :y 10 :width 10 :height 10}) + (b/rect {:x 30 :y 10 :width 10 :height 10})))) + frame (first (helpers/children-of instance uuid/zero)) + bool (first (helpers/children-of instance (:id frame))) + rects (helpers/children-of instance (:id bool))] + (t/is (= (:id frame) (:parent-id bool))) + (t/is (= (:id frame) (:frame-id bool))) + (t/is (= [(:id frame) (:id frame)] (mapv :frame-id rects))) + (t/is (= [(:id bool) (:id bool)] (mapv :parent-id rects))))) + +(t/deftest bool-inside-group-derives-group-bounds + (let [instance (b/scene {:seed 1} + (b/group {} + (b/bool {:bool-type :union} + (b/rect {:x 10 :y 20 :width 30 :height 40}) + (b/rect {:x 50 :y 80 :width 10 :height 10})))) + group (first (helpers/children-of instance uuid/zero)) + bool (first (helpers/children-of instance (:id group)))] + (t/is (= :bool (:type bool))) + (t/is (= [10 20 50 70] (helpers/selrect group))) + (t/is (= uuid/zero (:frame-id bool))))) + +(t/deftest empty-bool-is-rejected + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool {:bool-type :union})))] + (t/is (= ::b/empty-bool (:type data))))) + +(t/deftest caught-empty-bool-still-fails-finish + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (b/bool {:bool-type :union}) + (catch :default _ nil))))] + (t/is (= ::b/unfinished-container + (:type (helpers/failure-data #(b/finish! state))))))) + +(t/deftest bool-geometry-attrs-are-rejected + (doseq [attrs [{:bool-type :union :x 1} + {:bool-type :union :selrect {:x 0 :y 0 :width 1 :height 1}} + {:bool-type :union :content []}]] + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool attrs (b/rect))))] + (t/is (= ::b/bool-geometry (:type data)) (str attrs)) + (t/is (contains? #{:x :selrect :content} (:key data)) (str attrs))))) + +(t/deftest bool-transform-attrs-are-rejected + (doseq [attrs [{:bool-type :union :rotation 45} + {:bool-type :union :transform "m"} + {:bool-type :union :transform-inverse "m"} + {:bool-type :union :flip-x true} + {:bool-type :union :flip-y true}]] + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool attrs (b/rect))))] + (t/is (= ::b/bool-transform (:type data)) (str attrs))))) + +(t/deftest bool-type-is-required-and-validated + (t/is (= ::b/bool-type + (:type (helpers/failure-data #(b/scene {:seed 1} (b/bool {:name "B"} (b/rect))))))) + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/bool {:bool-type :merge} (b/rect))))] + (t/is (= ::b/bool-type (:type data))) + (t/is (= :merge (:bool-type data))) + (t/is (= #{:union :difference :intersection :exclude} (set (:supported data)))))) + +(t/deftest frame-child-is-rejected + (let [data (helpers/failure-data + #(b/scene {:seed 1} + (b/bool {:bool-type :union} + (b/frame {:x 0 :y 0 :width 10 :height 10} + (b/rect)))))] + (t/is (= ::b/frame-in-bool (:type data))) + (t/is (uuid? (:child data))))) + +(t/deftest bool-generation-ignores-the-renderer-override + (let [original path/wasm:calc-bool-content] + (try + (set! path/wasm:calc-bool-content + (fn [& _] + (throw (ex-info "renderer override ran during generation" + {:type ::unexpected-renderer-call})))) + (let [instance (b/scene {:seed 1} + (b/bool {:bool-type :difference} + (b/rect {:x 0 :y 0 :width 100 :height 100}) + (b/rect {:x 50 :y 50 :width 100 :height 100}))) + bool (first (helpers/children-of instance uuid/zero))] + (t/is (= [0 0 100 100] (helpers/selrect bool)))) + (finally + (set! path/wasm:calc-bool-content original))))) + +(t/deftest non-map-bool-attrs-are-rejected + (t/is (= ::b/invalid-attrs + (:type (helpers/failure-data #(b/scene {:seed 1} (b/bool :hero (b/rect))))))) + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (b/bool :hero (b/rect)) + (catch :default _ nil))))] + (t/is (= 1 (count (get-in @state [:objects uuid/zero :shapes])))))) diff --git a/frontend/test/performance_tests/render_wasm/browser_test.cljs b/frontend/test/performance_tests/render_wasm/browser_test.cljs new file mode 100644 index 0000000000..e0b88af31d --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/browser_test.cljs @@ -0,0 +1,328 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.browser-test + (:require + ["fs" :as fs] + ["http" :as http] + ["net" :as net] + ["os" :as os] + ["path" :as path] + ["playwright" :as playwright] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.browser :as browser] + [benches.render-wasm.node.runner.server :as server] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) +(def ^:private build-id "owned-browser-fixture") +(def ^:private viewport {:width 320 :height 240 :dpr 1}) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-browser-test-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(t/deftest gpu-container-selects-the-vulkan-launch-backend + (let [env (.-env js/process) + previous (.-PENPOT_GPU env)] + (try + (doseq [value [nil "0" "true" "1"]] + (if (nil? value) + (js-delete env "PENPOT_GPU") + (set! (.-PENPOT_GPU env) value)) + (let [args (set (:args (browser/launch-settings)))] + (t/is (= (= value "1") (contains? args "--use-angle=vulkan"))) + (t/is (= (= value "1") (contains? args "--use-gl=angle"))) + (t/is (contains? args "--enable-unsafe-swiftshader")))) + (finally + (if (undefined? previous) + (js-delete env "PENPOT_GPU") + (set! (.-PENPOT_GPU env) previous)))))) + +(defn- ^:async open-host + [actual-id ready?] + (let [identity-text (.stringify js/JSON (transit/encode-str {:build-id actual-id}))] + (fs/writeFileSync (path/join @temporary-root "index.html") + (str "browser ownership"))) + (await (server/open {:id build-id :dir @temporary-root} {:startup-timeout-ms 3000}))) + +(defn- request-status + [base-url request-path] + (js/Promise. + (fn [resolve reject] + (let [address (js/URL. base-url) + req (http/get #js {:hostname (.-hostname address) :port (.-port address) + :path request-path :agent false} + (fn [^js response] + (.once response "error" reject) + (.once response "end" #(resolve (.-statusCode response))) + (.resume response)))] + (.once ^js req "error" reject) + (.setTimeout ^js req 3000 #(.destroy ^js req (js/Error. "HTTP test timed out"))))))) + +(defn- socket-status + [url] + (js/Promise. + (fn [resolve _] + (let [address (js/URL. url) + ^js socket (net/createConnection #js {:host (.-hostname address) + :port (js/Number (.-port address))})] + (.once socket "connect" #(do (.destroy socket) (resolve :open))) + (.once socket "error" #(do (.destroy socket) (resolve (.-code ^js %)))) + (.setTimeout socket 3000 #(do (.destroy socket) (resolve :timeout))))))) + +(defn- ^:async rejected + [promise] + (try + (await promise) + (t/is false "expected browser work to reject") + nil + (catch :default cause cause))) + +(t/deftest ^:async page-and-context-close-failures-both-survive + (let [page-error (js/Error. "page disposal failed") + context-error (js/Error. "context disposal failed") + owner {:page #js {:isClosed (constantly false) + :close #(js/Promise.reject page-error)} + :context #js {:close #(js/Promise.reject context-error)}} + cause (await (rejected (browser/close-page owner)))] + (t/is (= [page-error context-error] (mapv :error (errors/get-errors cause)))))) + +(t/deftest ^:async browser-disposal-collects-failures-and-repeated-close-shares-the-result + (doseq [stop-fails? [false true]] + (let [page-error (js/Error. "page disposal failed") + context-error (js/Error. "context disposal failed") + process-error (js/Error. "process disposal failed") + disconnect-error (js/Error. "CDP disconnect failed") + controller (js/AbortController.) + running (js/Promise. + (fn [_ reject] + (.addEventListener (.-signal controller) "abort" + #(reject (if stop-fails? + process-error + (ex-info "cancelled" {:type :process-cancelled})))))) + owner {:controller controller :process running :shutdown (atom nil) + :user-data-dir @temporary-root + :page #js {:isClosed (constantly false) :close #(js/Promise.reject page-error)} + :context #js {:close #(js/Promise.reject context-error)} + :connection (atom (js/Promise.resolve + #js {:isConnected (constantly true) + :close #(if stop-fails? + (js/Promise.reject disconnect-error) + (js/Promise.resolve nil))}))} + closing (browser/close owner)] + (t/is (identical? closing (browser/close owner))) + (if stop-fails? + (let [cause (await (rejected closing))] + (t/is (= [page-error context-error process-error disconnect-error] + (mapv :error (errors/get-errors cause))))) + (t/is (nil? (await closing))))))) + + +(t/deftest ^:async unresolved-cancellation-retains-the-browser-profile + (let [profile (path/join @temporary-root "profile.info") + controller (js/AbortController.) + failure (errors/combine + [{:error (ex-info "Command cancelled" {:type :process-cancelled})} + {:error (errors/fatal-error "Command group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil)}]) + running (js/Promise. (fn [_ reject] + (.addEventListener (.-signal controller) "abort" #(reject failure)))) + owner {:controller controller :process running :shutdown (atom nil) + :connection (atom nil) :user-data-dir @temporary-root}] + (fs/writeFileSync profile "retained") + (let [cause (try (await (browser/close owner)) nil (catch :default cause cause))] + (t/is (errors/fatal? cause)) + (t/is (process/unresolved? cause)) + (t/is (= @temporary-root (:profile-path (errors/evidence :cleanup cause)))) + (t/is (= "retained" (when (fs/existsSync profile) (fs/readFileSync profile "utf8"))))))) + + +(t/deftest ^:async process-stop-retains-other-errors-after-cancellation + (let [controller (js/AbortController.) + probe (ex-info "Probe failed" {:type :process-probe}) + failure (errors/combine [{:error (ex-info "Command cancelled" {:type :process-cancelled})} + {:error probe}]) + running (js/Promise. (fn [_ reject] + (.addEventListener (.-signal controller) "abort" #(reject failure)))) + owner {:controller controller :process running :shutdown (atom nil) + :connection (atom nil) :user-data-dir @temporary-root} + cause (try (await (browser/close owner)) nil (catch :default cause cause))] + (t/is (= [probe] (mapv :error (errors/get-errors cause)))))) + +(t/deftest ^:async browser-startup-retains-a-failure-of-its-disposal + (let [run-process process/run + close browser/close + cleanup-error (js/Error. "startup disposal failed")] + (await + (mock/with-mocks* + {process/run (fn [options] + (run-process (assoc options :command "/bin/sh" :args ["-c" "exit 17"]))) + browser/close (fn [owner] + (.then (close owner) + (fn [_] (throw cleanup-error)) + (fn [_] (throw cleanup-error))))} + (let [cause (await (rejected (browser/launch {:startup-timeout-ms 10000}))) + entries (errors/get-errors cause)] + (t/is (= ::browser/startup-cleanup (:type (ex-data cause)))) + (t/is (= ["Command exited unsuccessfully" "startup disposal failed"] + (mapv #(errors/message-of (:error %)) entries)))))))) + +(defn- ^:async check-stopped + [{:keys [endpoint user-data-dir]}] + (t/is (not (fs/existsSync user-data-dir)) "owned Chromium profile was removed") + (t/is (= "ECONNREFUSED" (await (socket-status endpoint))) "the owned CDP listener is gone")) + +(defn- ^:async close-fixture + [owner page host] + (try + (when page + (try + (await (browser/close-page page)) + (catch :default _ nil))) + (when owner + (try + (await (browser/close owner)) + (catch :default _ nil))) + (finally + (await (server/close host))))) + +(t/deftest ^:async startup-process-details-survive-browser-failure-wrappers + (let [run-process process/run] + (doseq [exit-code [0 17]] + (await + (mock/with-mocks* + {process/run (fn [options] + (run-process (assoc options :command "/bin/sh" + :args ["-c" (str "exit " exit-code)])))} + (let [cause (await (rejected (browser/launch {:startup-timeout-ms 10000}))) + saved (-> (errors/evidence :browser-startup cause) + transit/encode-str + transit/decode-str) + command (if (zero? exit-code) + (.executablePath (.-chromium playwright)) + "/bin/sh")] + (t/is (= {:command command :exit-code exit-code} + (select-keys saved [:command :exit-code]))) + (t/is (= ::browser/endpoint-exit (:type (ex-data cause)))) + (t/is (= "Chromium exited before its CDP endpoint became ready" (errors/message-of cause))) + (t/is (= 1 (count (errors/get-errors cause)))))))))) + +(t/deftest ^:async successful-close-removes-the-profile-and-cdp-listener + (let [host (await (open-host build-id true)) + owner-ref (atom nil) + page-ref (atom nil)] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000})) + _ (reset! owner-ref owner) + page (await (browser/new-page owner host {:viewport viewport + :startup-timeout-ms 30000}))] + (reset! page-ref page) + (t/is (= 200 (await (request-status (:endpoint owner) "/json/version")))) + (t/is (= viewport + (js->clj (await (.evaluate ^js (:page page) + "({width:innerWidth,height:innerHeight,dpr:devicePixelRatio})")) + :keywordize-keys true))) + (await (browser/close-page page)) + (reset! page-ref nil) + (await (browser/close owner)) + (await (check-stopped owner)) + (t/is (not (.isConnected ^js (:browser owner)))) + (t/is (= 200 (await (request-status (:base-url host) "/"))) "browser shutdown leaves its host live") + (await (server/close host)) + (t/is (= "ECONNREFUSED" (await (socket-status (:base-url host)))))) + (finally + (await (close-fixture @owner-ref @page-ref host)))))) + +(t/deftest ^:async identity-mismatch-rejects-and-the-caller-closes-the-owned-browser + (let [host (await (open-host "wrong-compiled-build" true)) + owner-ref (atom nil)] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000})) + _ (reset! owner-ref owner) + cause (await (rejected (browser/new-page + owner host + {:viewport viewport :startup-timeout-ms 30000}))) + facts (ex-data cause)] + (t/is (= ::identity/mismatch (:type facts))) + (t/is (= "wrong-compiled-build" (:actual-id facts))) + (await (browser/close owner)) + (reset! owner-ref nil) + (await (check-stopped owner))) + (finally + (await (close-fixture @owner-ref nil host)))))) + +(t/deftest ^:async abort-stops-a-synchronously-blocked-page + (let [host (await (open-host build-id true)) + controller (js/AbortController.) + owner-ref (atom nil) + page-ref (atom nil) + started-ref (atom nil) + started (js/Promise. (fn [resolve _] (reset! started-ref resolve)))] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000 + :signal (.-signal controller)})) + _ (reset! owner-ref owner) + page-owner (await (browser/new-page + owner host + {:viewport viewport + :startup-timeout-ms 30000 + :signal (.-signal controller)})) + ^js page (:page page-owner)] + (reset! page-ref page-owner) + (await (.exposeFunction page "__blockingStarted" (fn [] (@started-ref)))) + (let [blocking (.evaluate page "window.__blockingStarted(); while (true) {}")] + (.catch blocking (fn [_] nil)) + (await (browser/within (fn [] started) + {:deadline (process/deadline 5000) :phase :test-blocking-start})) + (let [cause (await (rejected (browser/within #(.evaluate page "1") + {:deadline (process/deadline 1000) + :phase :test-blocked-page})))] + (t/is (= ::browser/timeout (:type (ex-data cause))))) + (.abort controller) + (reset! page-ref nil) + (await (browser/close owner)) + (reset! owner-ref nil) + (await (check-stopped owner)) + (t/is (not (.isConnected ^js (:browser owner)))) + (t/is (instance? js/Error (await (rejected blocking)))))) + (finally + (.abort controller) + (await (close-fixture @owner-ref @page-ref host)))))) + +(t/deftest ^:async abort-during-readiness-cleans-the-starting-browser + (let [host (await (open-host build-id false)) + controller (js/AbortController.) + owner-ref (atom nil) + requested (js/Promise. + (fn [resolve _] + (.once ^js (:server host) "request" (fn [_] (resolve nil)))))] + (try + (let [owner (await (browser/launch {:startup-timeout-ms 30000 + :signal (.-signal controller)})) + _ (reset! owner-ref owner) + page (browser/new-page owner host {:viewport viewport + :startup-timeout-ms 30000 + :signal (.-signal controller)})] + (.catch page (fn [_] nil)) + (await (browser/within (fn [] requested) + {:deadline (process/deadline 30000) :phase :test-host-request})) + (.abort controller (ex-info "caller cancelled browser readiness" {:type ::caller-abort})) + (let [cause (await (rejected page))] + (t/is (= ::caller-abort (:type (ex-data cause))))) + (await (browser/close owner)) + (reset! owner-ref nil) + (await (check-stopped owner))) + (finally + (.abort controller) + (await (close-fixture @owner-ref nil host)))))) diff --git a/frontend/test/performance_tests/render_wasm/build_test.cljs b/frontend/test/performance_tests/render_wasm/build_test.cljs new file mode 100644 index 0000000000..97e0d4f732 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/build_test.cljs @@ -0,0 +1,306 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.build-test + "Real preparation failures in temporary frontend/render-wasm checkouts. + + A scripted compiler keeps these tests independent of Cargo and Skia while + exercising the process, filesystem and failure-retention boundary. Release + integration checks cover compiler success and Cargo feature forwarding." + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.build :as build] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) + +(defn- frontend-root + [] + (path/join @temporary-root "frontend")) + +(defn- renderer-root + [] + (path/join @temporary-root "render-wasm")) + +(t/use-fixtures :each + {:before (fn [] + (reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-build-test-"))) + (fs/mkdirSync (frontend-root)) + (fs/mkdirSync (renderer-root))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(def ^:private renderer-wrapper + (str "#!/usr/bin/env node\n" + "const fs = require('node:fs');\n" + "const path = require('node:path');\n" + "const {spawnSync} = require('node:child_process');\n" + "const dir = path.dirname(process.env.FRONTEND_DEST);\n" + "const observation = {\n" + " cwd: process.cwd(),\n" + " manifest: fs.readFileSync(path.join(dir, 'build.transit'), 'utf8'),\n" + " log: fs.readFileSync(path.join(dir, 'build.log'), 'utf8'),\n" + " created: JSON.parse(fs.readFileSync(path.join(dir, 'created.json'), 'utf8'))\n" + "};\n" + "fs.writeFileSync(path.join(dir, 'renderer-observation.json'), JSON.stringify(observation));\n" + "const result = spawnSync('flock', [\n" + " '--exclusive', path.join(__dirname, 'build.lock'),\n" + " process.execPath, path.join(__dirname, 'compiler.cjs')\n" + "], {stdio: 'inherit'});\n" + "if (result.error) throw result.error;\n" + "process.exitCode = result.status === null ? 1 : result.status;\n")) + +(def ^:private failing-compiler + (str "const fs = require('node:fs');\n" + "const path = require('node:path');\n" + "fs.writeFileSync(path.join(process.env.FRONTEND_DEST, 'partial.js'), 'partial renderer output\\n');\n" + "process.stdout.write('compiler stdout before failure\\n');\n" + "process.stderr.write('compiler stderr before failure\\n');\n" + "process.exitCode = 23;\n")) + +(def ^:private sleeping-compiler + (str "const fs = require('node:fs');\n" + "const path = require('node:path');\n" + "fs.writeFileSync(path.join(process.env.FRONTEND_DEST, 'lock-acquired.txt'), 'held');\n" + "process.stdout.write('compiler lock acquired\\n');\n" + "process.stderr.write('waiting in compiler\\n');\n" + "setInterval(() => {}, 1000);\n")) + +(defn- install-renderer + [compiler] + (fs/writeFileSync (path/join (renderer-root) "build") renderer-wrapper #js {:mode 493}) + (fs/writeFileSync (path/join (renderer-root) "compiler.cjs") compiler "utf8")) + +(defn- read-manifest + [build-ref] + (transit/decode-str (fs/readFileSync (:manifest-path build-ref) "utf8"))) + +(defn- record-created + [build-ref] + ;; These reads must succeed before the executable can read created.json. + (let [observation {:id (:id build-ref) + :manifest (fs/readFileSync (:manifest-path build-ref) "utf8") + :log (fs/readFileSync (:log-path build-ref) "utf8") + :renderer-started? (fs/existsSync (path/join (:dir build-ref) "renderer-observation.json"))}] + (fs/writeFileSync (path/join (:dir build-ref) "created.json") + (js/JSON.stringify (clj->js observation)) "utf8"))) + +(defn- retained-build + [cause] + (let [id (:build-id (ex-data cause)) + dir (path/join (frontend-root) "target" "performance" "render-wasm" "builds" id)] + {:id id :dir dir + :manifest-path (path/join dir "build.transit") + :log-path (path/join dir "build.log") + :renderer-dir (path/join dir "renderer")})) + +(defn- ^:async rejected + [promise] + (try + (let [result (await promise)] + (t/is false (str "Expected preparation to fail, got " (pr-str result))) + nil) + (catch :default cause cause))) + +(defn- complete + [promise done] + (.then promise + (fn [_] (done)) + (fn [cause] + (t/is false (str "Unexpected test error: " cause)) + (done)))) + +(t/deftest ^:async renderer-facts-retain-string-environment-keys-and-missing-evidence + (let [bin (path/join @temporary-root "bin") + original-path (.. js/process -env -PATH)] + (fs/mkdirSync bin) + (fs/writeFileSync (path/join bin "clojure") "#!/bin/sh\nexit 24\n" #js {:mode 493}) + (try + (set! (.. js/process -env -PATH) (str bin path/delimiter original-path)) + (doseq [environment [::missing {} {"RUSTFLAGS" "-C\u001fopt-level=3" "CFLAGS" ""}]] + (let [facts (cond-> {:features [] :profile {:opt_level "3" :debug_assertions false}} + (not= ::missing environment) (assoc :environment environment))] + (install-renderer + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, " + (js/JSON.stringify (js/JSON.stringify (clj->js facts))) ");\n")) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + manifest (read-manifest (retained-build cause))] + (t/is (= :cljs (:phase (ex-data cause)))) + (t/is (= 24 (:exit-code (ex-data (ex-cause cause))))) + (if (= ::missing environment) + (t/is (not (contains? manifest :environment))) + (t/is (= environment (:environment manifest))))))) + (finally + (set! (.. js/process -env -PATH) original-path))))) + +(t/deftest ^:async explicit-selection-preserves-disabled-cargo-defaults + (let [bin (path/join @temporary-root "bin") + original-path (.. js/process -env -PATH)] + (fs/mkdirSync bin) + (fs/writeFileSync (path/join bin "clojure") "#!/bin/sh\nexit 24\n" #js {:mode 493}) + (try + (set! (.. js/process -env -PATH) (str bin path/delimiter original-path)) + (install-renderer + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, " + (js/JSON.stringify + (js/JSON.stringify + (clj->js {:features ["a"] + :profile {:opt_level "3" :debug_assertions false}}))) + ");\n")) + (let [cause (await (rejected (build/prepare + {:root (frontend-root) + :selection {:default-features? false :features ["a"]} + :build-timeout-ms 10000 + :on-created record-created}))) + manifest (read-manifest (retained-build cause))] + (t/is (= :cljs (:phase (ex-data cause)))) + (t/is (= {:default-features? false :features ["a"]} + (get-in manifest [:provenance :feature-selection]))) + (t/is (= ["a"] (get-in manifest [:functional :features]))) + (t/is (false? (get-in manifest [:functional :default-features?])))) + (finally + (set! (.. js/process -env -PATH) original-path))))) + +(t/deftest real-git-outside-a-repository-has-unknown-provenance + (t/async done + (complete + ((^:async fn [] + (let [git (await (build/git-provenance (frontend-root) + {:deadline (process/deadline 10000)}))] + (t/is (= {:sha nil :dirty nil} git))))) + done))) + + +(t/deftest ^:async git-provenance-propagates-unresolved-process-failures + (let [failure (errors/fatal-error "Git process may still be running" + {:type :process-unresolved :unresolved-process true + :command "git" :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil)] + (await + (mock/with-mocks* + {process/run (fn [_] (js/Promise.reject failure))} + (let [cause (try + (await (build/git-provenance (frontend-root) {:deadline (process/deadline 1000)})) + nil + (catch :default cause cause))] + (t/is (errors/fatal? cause)) + (t/is (process/unresolved? cause))))))) + +(t/deftest nonzero-renderer-retains-created-evidence-output-and-failed-manifest + (t/async done + (complete + ((^:async fn [] + (install-renderer failing-compiler) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + data (ex-data cause) + child-data (ex-data (ex-cause cause)) + retained (retained-build cause) + manifest (read-manifest retained) + log (fs/readFileSync (:log-path retained) "utf8") + observation (js->clj (js/JSON.parse + (fs/readFileSync (path/join (:dir retained) "renderer-observation.json") "utf8")) + :keywordize-keys true) + created (:created observation) + initial (transit/decode-str (:manifest created)) + at-renderer (transit/decode-str (:manifest observation))] + (t/is (= ::build/preparation-failed (:type data))) + (t/is (= :renderer (:phase data))) + (t/is (= (:log-path retained) (:log-path data))) + (t/is (= :process-exit (:type child-data))) + (t/is (= 23 (:exit-code child-data))) + (t/is (= (path/join (renderer-root) "build") (:command child-data))) + (t/is (= (renderer-root) (:cwd observation))) + (t/is (= (:id retained) (:id created) (:id manifest) + (:id initial) (:id at-renderer))) + (t/is (= :building (:status initial) (:status at-renderer))) + (t/is (false? (:renderer-started? created))) + (t/is (= "" (:log created))) + (t/is (str/includes? (:log observation) "Renderer compiling\n")) + (t/is (= :failed (:status manifest))) + (t/is (= "renderer" (get-in manifest [:failure :phase]))) + (t/is (= "Command exited unsuccessfully" (get-in manifest [:failure :message]))) + (t/is (= [:run-command] (mapv :operation (get-in manifest [:failure :errors])))) + (t/is (= :process-exit (get-in manifest [:failure :cause 0 :type]))) + (t/is (= {:sha nil :dirty nil} (get-in manifest [:provenance :git]))) + (t/is (str/includes? log "compiler stdout before failure\n")) + (t/is (str/includes? log "compiler stderr before failure\n")) + (t/is (= "partial renderer output\n" + (fs/readFileSync (path/join (:renderer-dir retained) "partial.js") "utf8")))))) + done))) + +(t/deftest failed-manifest-save-makes-the-preparation-error-fatal + (t/async done + (complete + ((^:async fn [] + (install-renderer + (str failing-compiler + "fs.unlinkSync(path.join(path.dirname(process.env.FRONTEND_DEST), 'build.transit'));\n")) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + data (ex-data cause) + entries (errors/get-errors cause)] + (t/is (errors/fatal? cause)) + (t/is (string? (:manifest-error data))) + (t/is (= :process-exit (:type (ex-data (:error (first entries)))))) + (t/is (= 2 (count entries))) + (t/is (= :write-manifest (:operation (second entries)))) + (t/is (= (:manifest-path (retained-build cause)) (:path (second entries)))) + (t/is (str/includes? (errors/message-of (:error (second entries))) "ENOENT"))))) + done))) + +(t/deftest timed-out-lock-holder-retains-its-attempt-and-retry-gets-a-new-id + (t/async done + (complete + ((^:async fn [] + (install-renderer sleeping-compiler) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 3000 + :on-created record-created}))) + retained (retained-build cause) + manifest (read-manifest retained) + before (fs/readFileSync (:manifest-path retained) "utf8") + log (fs/readFileSync (:log-path retained) "utf8")] + (t/is (= ::build/preparation-failed (:type (ex-data cause)))) + (t/is (= :renderer (:phase (ex-data cause)))) + (t/is (= (:log-path retained) (:log-path (ex-data cause)))) + (t/is (= :process-timeout (:type (ex-data (ex-cause cause))))) + (t/is (= (:id retained) (:id manifest))) + (t/is (= :failed (:status manifest))) + (t/is (= "renderer" (get-in manifest [:failure :phase]))) + (t/is (= :process-timeout (get-in manifest [:failure :cause 0 :type]))) + (t/is (= "held" (fs/readFileSync (path/join (:renderer-dir retained) "lock-acquired.txt") "utf8"))) + (t/is (str/includes? log "compiler lock acquired\n")) + (t/is (str/includes? log "waiting in compiler\n")) + ;; The retry must acquire the same flock before its compiler can exit. + (install-renderer failing-compiler) + (let [retry-cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + retry (retained-build retry-cause) + retry-log (fs/readFileSync (:log-path retry) "utf8")] + (t/is (not= (:id retained) (:id retry))) + (t/is (= :renderer (:phase (ex-data retry-cause)))) + (t/is (= 23 (:exit-code (ex-data (ex-cause retry-cause))))) + (t/is (= :failed (:status (read-manifest retry)))) + (t/is (str/includes? retry-log "compiler stderr before failure\n")) + (t/is (= before (fs/readFileSync (:manifest-path retained) "utf8"))) + (t/is (= log (fs/readFileSync (:log-path retained) "utf8"))) + (t/is (= #{(:id retained) (:id retry)} + (set (array-seq (fs/readdirSync (path/dirname (:dir retained))))))))))) + done))) diff --git a/frontend/test/performance_tests/render_wasm/builder_test.cljs b/frontend/test/performance_tests/render_wasm/builder_test.cljs new file mode 100644 index 0000000000..4e2194b4f8 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/builder_test.cljs @@ -0,0 +1,288 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.builder-test + "Tests canonical scene construction and the shared seeded number generator. + Fixed random sequences preserve scene draws during generator extraction." + (:require + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as b] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.random :as random] + [benches.render-wasm.scenes.rects] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(defn- build + ([] (build {})) + ([params] ((:build (decl/registered-scene :rects)) (merge {:seed 42 :count 5} params)))) + +(defn- add-tile + "Helper defined outside a scene scope, called from inside one." + [index] + (b/rect [:tile index] {:x (* 10 index)})) + +(t/deftest repeatable-builds-are-identical + (t/is (= (build) (build))) + (t/is (not= (build) (build {:seed 43})))) + +(t/deftest shared-generator-retains-scene-draws-and-independent-state + (doseq [[seed expected] [[0 [1144304738 1416247 958946056 627933444 2007157716]] + [42 [2581720956 1925393290 3661312704 2876485805 750819978]] + [4294967295 [3850105811 813802916 3073704848 4054706436 3630262831]]]] + (let [first-source (random/create seed) + second-source (random/create seed)] + (t/is (= expected (mapv #(* 4294967296 %) (repeatedly 5 first-source)))) + (t/is (= expected (mapv #(* 4294967296 %) (repeatedly 5 second-source)))) + (t/is (every? #(and (<= 0 %) (< % 1)) (repeatedly 1000 first-source))))) + (let [rng (:rng @(b/start {:seed 42}))] + (t/is (= [2581720956 1925393290 3661312704 2876485805 750819978] + (mapv #(* 4294967296 %) (repeatedly 5 rng)))))) + +(t/deftest shared-generator-rejects-invalid-seeds + (doseq [seed [-1 4294967296 1.5 nil false "42" js/NaN js/Infinity]] + (t/is (= ::random/invalid-seed + (:type (helpers/failure-data #(random/create seed))))))) + +(t/deftest rectangle-workload-keeps-default-count + (let [instance (build {:seed 42 :count 1000}) + rects (helpers/root-shapes instance)] + (t/is (= 1000 (count rects))) + (t/is (< 1 (count (distinct (map :x rects))))))) + +(t/deftest default-workload-is-canonical + (let [instance (build {:count 25}) + rects (helpers/root-shapes instance)] + (t/is (= 25 (count rects))) + (t/is (every? #(= :rect (:type %)) rects)) + (t/is (every? (fn [rect] + (and (= 1 (count (:fills rect))) + (= 1 (count (:strokes rect))) + (= 10 (:stroke-width (first (:strokes rect)))) + (= :center (:stroke-alignment (first (:strokes rect)))))) + rects)) + (t/is (every? (fn [rect] + (and (re-matches #"^#[0-9a-f]{6}$" (:fill-color (first (:fills rect)))) + (<= 0.1 (:fill-opacity (first (:fills rect))) 0.999) + (every? #(<= 0 % 24) [(:r1 rect) (:r2 rect) (:r3 rect) (:r4 rect)]))) + rects)))) + +(t/deftest geometry-is-page-relative-and-in-range + (let [params {:seed 7 :count 40 :width 800 :height 600 :min-size 5 :max-size 15} + instance (build params) + rects (helpers/root-shapes instance)] + (t/is (every? #(<= 0 (:x %) 800) rects)) + (t/is (every? #(<= 0 (:y %) 600) rects)) + (t/is (every? #(<= 5 (:width %) 15) rects)) + (t/is (every? #(<= 5 (:height %) 15) rects)) + (t/is (= 800 (get-in instance [:objects uuid/zero :width]))) + (t/is (= 600 (get-in instance [:objects uuid/zero :height]))))) + +(t/deftest labels-are-snapshot-local + (let [instance (b/scene {:seed 1} + (b/rect) + (b/rect :hero {}) + (b/rect [:tile 1] {})) + refs (:refs instance) + children (helpers/child-ids instance uuid/zero)] + (t/is (= #{:hero [:tile 1]} (set (keys refs)))) + (t/is (= (get refs :hero) (second children))) + (t/is (= (get refs [:tile 1]) (nth children 2))))) + +(t/deftest attrs-override-generated-values + (let [instance (b/scene {:seed 1 + :defaults {:rect {:x (b/gen-int 0 100)}}} + (b/rect {:x 7 :fills []})) + rect (first (helpers/root-shapes instance))] + (t/is (= 7 (:x rect))) + (t/is (= [] (:fills rect))))) + +(t/deftest shapes-use-their-defaults-before-drawing-ids + (doseq [[type low high construct] [[:rect 0.1 0.2 #(b/rect %)] + [:path 0.3 0.4 #(b/path %)] + [:circle 0.5 0.6 #(b/circle %)]] + attrs [{} {:opacity 0.75}]] + (let [rng (random/create 7) + opacity (random/round3 (random/rng-float rng low high)) + id (random/rng-uuid rng) + instance (b/scene {:seed 7 + :defaults {:rect {:opacity (b/gen-float 0.1 0.2)} + :path {:opacity (b/gen-float 0.3 0.4)} + :circle {:opacity (b/gen-float 0.5 0.6)}}} + (construct attrs)) + shape (first (helpers/root-shapes instance))] + (t/is (= {:type type :id id :opacity (get attrs :opacity opacity)} + (select-keys shape [:type :id :opacity])))))) + +(t/deftest scene-and-shape-expressions-evaluate-once-in-order + (let [events (atom []) + instance (b/scene (do (swap! events conj :params) {:seed 1}) + (b/rect (do (swap! events conj :label) :hero) + (do (swap! events conj :attrs) {:x 11}))) + hero (get-in instance [:objects (get (:refs instance) :hero)])] + (t/is (= [:params :label :attrs] @events)) + (t/is (= 11 (:x hero))))) + +(t/deftest ordinary-forms-work-inside-the-scope + (let [instance (b/scene {:seed 1} + (let [first-id (b/rect)] + (t/is (uuid? first-id))) + (doseq [index (range 3)] + (add-tile index)) + (b/rect :last {:x 99})) + refs (:refs instance) + rects (helpers/root-shapes instance)] + (t/is (= 5 (count rects))) + (t/is (= [0 10 20] + (mapv #(:x (get-in instance [:objects (get refs [:tile %])])) + (range 3)))) + (t/is (= 99 (:x (get-in instance [:objects (get refs :last)])))))) + +(t/deftest duplicate-labels-are-rejected + (let [data (helpers/failure-data + #(b/scene {:seed 1} + (b/rect :dup {}) + (b/rect :dup {})))] + (t/is (= ::b/duplicate-label (:type data))) + (t/is (= :dup (:label data))))) + +(t/deftest failed-builds-leak-no-state + (let [clean (build)] + (helpers/failure-data #(b/scene {:seed 99} + (b/rect :dup {}) + (b/rect :dup {}))) + (t/is (= clean (build))))) + +(t/deftest rect-outside-scope-is-rejected + (let [data (helpers/failure-data #(b/rect))] + (t/is (= ::b/outside-scope (:type data))))) + +(t/deftest nested-scopes-are-rejected + (let [data (helpers/failure-data + #(b/scene {:seed 1} + (b/scene {:seed 2} + (b/rect))))] + (t/is (= ::b/nested-scope (:type data))))) + +(t/deftest invalid-seeds-are-rejected + (t/is (= ::b/invalid-seed + (:type (helpers/failure-data #(b/scene {} (b/rect)))))) + (t/is (= ::b/invalid-seed + (:type (helpers/failure-data #(b/scene {:seed -1} (b/rect)))))) + (t/is (= ::b/invalid-seed + (:type (helpers/failure-data #(b/scene {:seed 4294967296} (b/rect))))))) + +(t/deftest runtime-attrs-maps-are-resolved + (let [attrs {:x 11} + instance (b/scene {:seed 1} + (b/rect attrs)) + rect (first (helpers/root-shapes instance))] + (t/is (= 11 (:x rect))))) + +(t/deftest single-label-form-is-supported + (let [instance (b/scene {:seed 1} + (b/rect :solo)) + id (get (:refs instance) :solo)] + (t/is (uuid? id)) + (t/is (= [id] (helpers/child-ids instance uuid/zero))))) + +(t/deftest built-snapshot-feeds-the-scene-api + (let [instance (b/scene {:seed 1} + (b/rect :hero {})) + order (snapshot/upload-order instance) + hero-id (snapshot/ref-id instance :hero)] + (t/is (= [uuid/zero hero-id] (mapv :id order))))) + +(t/deftest invalid-labels-are-rejected + (let [data (helpers/failure-data #(b/scene {:seed 1} (b/rect "hero" {})))] + (t/is (= ::b/invalid-label (:type data))))) + +(t/deftest protected-keys-cannot-be-overridden + (let [other (uuid/custom 9 9) + instance (b/scene {:seed 1} + (b/rect {:type :circle :id other :parent-id other})) + rect (first (helpers/root-shapes instance))] + (t/is (= :rect (:type rect))) + (t/is (not= other (:id rect))) + (t/is (= uuid/zero (:parent-id rect))))) + +(t/deftest root-protected-keys-cannot-be-overridden + (let [other (uuid/custom 9 8) + instance (b/scene {:seed 1 + :root {:id other :type :rect :width 10}} + (b/rect)) + root (get-in instance [:objects uuid/zero])] + (t/is (= uuid/zero (:id root))) + (t/is (= :frame (:type root))) + (t/is (= 10 (:width root))))) + +(t/deftest attrs-override-generated-fills + (let [fill {:fill-color "#ffffff" :fill-opacity 1} + instance (b/scene {:seed 1} + (b/rect {:fills [fill]})) + rect (first (helpers/root-shapes instance))] + (t/is (= [fill] (:fills rect))))) + +(t/deftest generator-helpers-produce-expected-values + (let [instance (b/scene {:seed 3 + :defaults {:rect {:opacity (b/gen-float 0.5 0.6) + :x (b/gen-one-of [4 5]) + :r1 (b/gen-int 3 5) + :fills (b/gen-vector (b/gen-fill))}}} + (b/rect)) + rect (first (helpers/root-shapes instance))] + (t/is (<= 0.5 (:opacity rect) 0.6)) + (t/is (contains? #{4 5} (:x rect))) + (t/is (<= 3 (:r1 rect) 4)) + (t/is (re-matches #"^#[0-9a-f]{6}$" (:fill-color (first (:fills rect))))))) + +(t/deftest empty-one-of-is-rejected + (let [data (helpers/failure-data #(b/gen-one-of []))] + (t/is (= ::b/no-values (:type data))))) + +(t/deftest circle-creates-ellipse-type + (let [instance (b/scene {:seed 1} + (b/circle {:x 10 :y 20 :width 30 :height 40})) + circle (first (helpers/root-shapes instance))] + (t/is (= :circle (:type circle))) + (t/is (= [10 20 30 40] [(:x circle) (:y circle) (:width circle) (:height circle)])) + (t/is (= 1 (count (helpers/child-ids instance uuid/zero)))))) + +(t/deftest circle-uses-circle-defaults + (let [instance (b/scene {:seed 3 + :defaults {:circle {:x (b/gen-int 10 12)}}} + (b/circle)) + circle (first (helpers/root-shapes instance))] + (t/is (contains? #{10 11} (:x circle))))) + +(t/deftest circle-ignores-rect-defaults + (let [instance (b/scene {:seed 1 + :defaults {:rect {:x 999}}} + (b/circle)) + circle (first (helpers/root-shapes instance))] + (t/is (not= 999 (:x circle))))) + +(t/deftest circle-geometry-is-page-relative-and-in-range + (let [instance (b/scene {:seed 7 + :defaults {:circle {:x (b/gen-int 0 800) + :y (b/gen-int 0 600) + :width (b/gen-int 5 15) + :height (b/gen-int 5 15)}}} + (doseq [_ (range 40)] + (b/circle))) + circles (helpers/root-shapes instance)] + (t/is (= 40 (count circles))) + (t/is (every? #(<= 0 (:x %) 800) circles)) + (t/is (every? #(<= 0 (:y %) 600) circles)) + (t/is (every? #(<= 5 (:width %) 15) circles)) + (t/is (every? #(<= 5 (:height %) 15) circles)))) + +(t/deftest circle-preserves-radius-attrs + (let [instance (b/scene {:seed 1} + (b/circle {:x 0 :y 0 :width 20 :height 20 :rx 5 :ry 7})) + circle (first (helpers/root-shapes instance))] + (t/is (= [5 7] [(:rx circle) (:ry circle)])))) diff --git a/frontend/test/performance_tests/render_wasm/cli_integration.cjs b/frontend/test/performance_tests/render_wasm/cli_integration.cjs new file mode 100644 index 0000000000..648ac8be43 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/cli_integration.cjs @@ -0,0 +1,2197 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// Copyright (c) KALEIDOS SUBSIDIARY SL + +// Our CLJS test suite already covers the internals: +// - runner_test.cljs tests parse-args, +// - build_test.cljs tests preparation with fake compilers, +// - feature_comparison_test.cljs tests orchestration. +// Those compile to an :esm bundle with :runtime :custom and run in-process from +// target/performance-tests/test.js. +// +// It cannot test the assembled executable, so here we test: +// - exit codes of the real process, including 130/143 for SIGINT/SIGTERM, +// - stdout and stderr separation as the shell sees it, +// - that runner.cjs finds commands.cjs next to itself, +// - that help, compare, and merge never load execution dependencies, +// - release-compiled file and manifest behavior on the real filesystem, + +// Opt-in: build the release runner and the test bundle first: +// +// pnpm run build:renderer-benchmarks:runner +// pnpm run build:renderer-benchmarks:tests +// RENDER_WASM_RUNNER_BOOT=target/performance/render-wasm/runner.cjs \ +// node --test test/performance_tests/render_wasm/cli_integration.cjs +// +// The normal unit test build does not build this CLI. + +const assert = require("node:assert/strict"); +const { execFile, spawn, spawnSync } = require("node:child_process"); +const { once } = require("node:events"); +const { setTimeout: pause } = require("node:timers/promises"); +const { randomUUID } = require("node:crypto"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { before, test } = require("node:test"); +const { pathToFileURL } = require("node:url"); +const { promisify } = require("node:util"); + +const execute = promisify(execFile); +const testOptions = { timeout: 60000 }; +let bootstrap; +let frontendRoot; +let transit; + +before(async () => { + assert.ok( + process.env.RENDER_WASM_RUNNER_BOOT, + "Set RENDER_WASM_RUNNER_BOOT to the release-compiled runner.cjs", + ); + bootstrap = path.resolve(process.env.RENDER_WASM_RUNNER_BOOT); + assert.ok( + fs.existsSync(bootstrap) && fs.statSync(bootstrap).isFile(), + "Run pnpm run build:renderer-benchmarks:runner before the CLI tests", + ); + bootstrap = fs.realpathSync(bootstrap); + frontendRoot = path.resolve(path.dirname(bootstrap), "../../.."); + assert.ok(fs.statSync(path.join(frontendRoot, "deps.edn")).isFile()); + assert.ok( + fs + .statSync( + path.join(frontendRoot, "performance", "benches", "render_wasm"), + ) + .isDirectory(), + ); + assert.ok(fs.statSync(path.join(frontendRoot, "node_modules")).isDirectory()); + + // Load the actual Transit JS library emitted by Shadow, without starting + // either the CLJS test runner or the production CLI in this process. + const transitModule = path.join( + frontendRoot, + "target/performance-tests/cljs-runtime/com.cognitect.transit.js", + ); + assert.ok( + fs.existsSync(transitModule), + "Run pnpm run build:renderer-benchmarks:tests before the CLI tests", + ); + await import(pathToFileURL(transitModule).href); + transit = globalThis.com.cognitect.transit; +}); + +function createFrontend(t) { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "render-wasm-cli-")); + t.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const root = path.join(temporary, "frontend"); + const managed = path.join(root, "target", "performance", "render-wasm"); + const renderer = path.join(temporary, "render-wasm"); + const runner = path.join(managed, "runner.cjs"); + fs.mkdirSync(path.join(root, "performance", "benches", "render_wasm"), { + recursive: true, + }); + fs.writeFileSync(path.join(root, "deps.edn"), "{}\n"); + fs.mkdirSync(managed, { recursive: true }); + fs.copyFileSync(bootstrap, runner); + fs.copyFileSync( + path.join(path.dirname(bootstrap), "commands.cjs"), + path.join(managed, "commands.cjs"), + ); + fs.symlinkSync( + path.join(frontendRoot, "node_modules"), + path.join(root, "node_modules"), + "dir", + ); + fs.mkdirSync(renderer); + fs.mkdirSync(path.join(managed, "compiler-cache")); + fs.writeFileSync( + path.join(managed, "compiler-cache", "retained.cache"), + "cache\n", + ); + return { + managed, + runner, + compiler: path.join(renderer, "build"), + rendererRoot: renderer, + builds: path.join(managed, "builds"), + runs: path.join(managed, "runs"), + comparisons: path.join(managed, "feature-comparisons"), + env: {}, + }; +} + +async function runCli(fixture, args) { + try { + const output = await execute( + process.execPath, + [...(fixture.nodeArgs ?? []), fixture.runner, ...args], + { + cwd: os.tmpdir(), + encoding: "utf8", + timeout: 15000, + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + env: { ...process.env, ...fixture.env }, + }, + ); + return { exitCode: 0, ...output }; + } catch (error) { + // A timeout, signal or spawn failure must fail the test, rather than + // masquerade as the expected nonzero CLI exit. + if (!Number.isInteger(error.code) || error.killed || error.signal) { + throw error; + } + return { + exitCode: error.code, + stdout: error.stdout, + stderr: error.stderr, + }; + } +} + +function readManifest(file) { + return transit.reader("json").read(fs.readFileSync(file, "utf8")); +} + +function requestText(command, options, color = "never") { + const k = transit.keyword; + return transit.writer("json").write(transit.map([ + k("command"), command, + k("console"), transit.map([k("level"), k("info"), k("color"), k(color)]), + k("options"), transit.map(Object.entries(options).flatMap(([key, value]) => [k(key), value])), + ])); +} + +test("deferred transport carries a complete request and numeric status", testOptions, async (t) => { + const fixture = createFrontend(t); + const capture = path.join(fixture.managed, "request.transit"); + fs.writeFileSync(path.join(fixture.managed, "commands.cjs"), ` + module.exports.execute = (text) => { + if (typeof text !== "string") throw new Error("expected Transit text"); + require("node:fs").writeFileSync(${JSON.stringify(capture)}, text); + return Promise.resolve(37); + }; + `); + const result = await runCli(fixture, ["run", "--build", "ready", "--warmups", "0", "--quiet", "--no-color"]); + assert.equal(result.exitCode, 37, result.stderr); + const k = transit.keyword; + const request = readManifest(capture); + assert.equal(request.get(k("command")), "run"); + assert.equal(request.get(k("console")).get(k("level")).toString(), ":warn"); + assert.equal(request.get(k("console")).get(k("color")).toString(), ":never"); + const options = request.get(k("options")); + assert.equal(options.get(k("build")).get(k("kind")).toString(), ":existing"); + assert.equal(options.get(k("build")).get(k("id")), "ready"); + assert.equal(options.get(k("warmups")), 0); + assert.equal(options.get(k("repetitions")), 10); + assert.equal(options.get(k("startup-timeout-ms")), 30000); +}); + +test("decoded requests reject incomplete, unsupported and policy-invalid commands before acquisition", testOptions, async (t) => { + const fixture = createFrontend(t); + fs.unlinkSync(path.resolve(fixture.managed, "../../../deps.edn")); + const entry = path.join(fixture.managed, "receive.cjs"); + fs.writeFileSync(entry, ` + require("./commands.cjs").execute(process.env.COMMAND_REQUEST) + .then(code => { process.exitCode = code; }); + `); + const before = tree(fixture.managed); + const measurement = { seed: 42, filter: null, warmups: 3, repetitions: 10, + "timeout-ms": 30000, "startup-timeout-ms": 30000, "output-dir": null, + screenshots: false, "require-gpu": false }; + const ab = { ...measurement, features: ["a"], "max-features": 3, confirm: false, + "dry-run": false, "build-timeout-ms": 300000, "allow-build-differences": false, + "allow-scene-differences": false, "allow-all-differences": false }; + const k = transit.keyword; + const build = { "default-features": true, features: [], "build-timeout-ms": 300000 }; + for (const [text, diagnostic] of [ + ["invalid Transit", /JSON/], + [requestText("build", {}), /default-features|build-timeout-ms/], + [requestText("purge", { target: k("all") }, "always"), /unsupported-command/], + [requestText("build", { ...build, features: ["profile"] }), /diagnostic-features/], + [requestText("run", { ...measurement, remove: false, + build: transit.map([k("kind"), k("prepare"), ...Object.entries({ ...build, features: ["stats"] }).flatMap(([key, value]) => [k(key), value])]) }), /diagnostic-features/], + [requestText("ab", ab), /confirmation-required/], + [requestText("ab", { ...ab, features: ["a", "b"], "max-features": 1, "dry-run": true }), /too-many-features/], + ]) { + fixture.env.COMMAND_REQUEST = text; + const result = await runCli({ ...fixture, runner: entry }, []); + assert.equal(result.exitCode, 1, result.stderr); + assert.equal(result.stdout, ""); + assert.match(result.stderr, diagnostic); + assert.equal((result.stderr.match(/✖/g) ?? []).length, 1); + assert.deepEqual(tree(fixture.managed), before); + } +}); + +test("final publication failures select a failing status in local and deferred commands", testOptions, async (t) => { + for (const command of [["--help"], ["build"]]) { + const fixture = createFrontend(t); + if (command[0] === "build") installPreparedCompiler(fixture, false); + const guard = path.join(fixture.managed, "writer.cjs"); + fs.writeFileSync(guard, `process.stdout.write = () => { throw new Error("publication unavailable"); };`); + fixture.nodeArgs = ["--require", guard]; + const result = await runCli(fixture, command); + assert.equal(result.exitCode, 1, result.stderr); + assert.match(result.stderr, /publication unavailable/); + assert.equal((result.stderr.match(/✖/g) ?? []).length, 1); + if (command[0] === "build") { + const [id] = fs.readdirSync(fixture.builds); + assert.equal(readManifest(path.join(fixture.builds, id, "build.transit")).get(transit.keyword("status")).toString(), ":ready"); + } + } +}); + +test("merge and purge execute with unavailable process, HTTP and browser dependencies", testOptions, async (t) => { + const fixture = createFrontend(t); + const guard = path.join(fixture.managed, "offline.cjs"); + fs.writeFileSync(guard, ` + const Module = require("node:module"); + const load = Module._load; + Module._load = function (id, ...args) { + if (["playwright", "http", "node:http", "child_process", "node:child_process"].includes(id)) { + throw new Error("Unavailable dependency: " + id); + } + return load.call(this, id, ...args); + }; + `); + fixture.nodeArgs = ["--require", guard]; + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + for (const [file, runId, value] of [[base, "base", 10], [additional, "additional", 12]]) { + fs.writeFileSync(file, runFixtureText({ runId, caseId: "case/load", value, gitSha: "same" })); + } + const merged = await runCli(fixture, ["merge", base, additional]); + assert.equal(merged.exitCode, 0, merged.stderr); + assert.match(merged.stdout, /^Run .+\nOutput .+\n$/); + seedBuilds(fixture); + const purged = await runCli(fixture, ["purge", "builds"]); + assert.equal(purged.exitCode, 0, purged.stderr); + for (const id of fs.readdirSync(fixture.builds)) { + assert.deepEqual(fs.readdirSync(path.join(fixture.builds, id)), ["keep.txt"]); + } +}); + +function manifestText(id, status, type) { + const k = transit.keyword; + return transit + .writer("json") + .write( + transit.map([k("type"), k(type), k("id"), id, k("status"), k(status)]), + ); +} + +function seedBuilds(fixture) { + for (const status of ["building", "failed", "ready"]) { + const id = randomUUID(); + const directory = path.join(fixture.builds, id); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, "build.transit"), + manifestText(id, status, "renderer-build"), + ); + fs.writeFileSync(path.join(directory, "build.log"), `${status} build\n`); + fs.writeFileSync(path.join(directory, "keep.txt"), "keep\n"); + } +} + +function seedRuns(fixture) { + for (const status of ["complete", "failed"]) { + const id = randomUUID(); + const directory = path.join(fixture.runs, id); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, "run.transit"), + `${status} run output\n`, + ); + fs.writeFileSync(path.join(directory, "run.log"), "run\n"); + fs.writeFileSync(path.join(directory, "case-load.PNG"), "screenshot\n"); + fs.writeFileSync(path.join(directory, "keep.txt"), "keep\n"); + } +} + +function seedComparisons(fixture) { + const id = randomUUID(); + const directory = path.join(fixture.comparisons, id); + const config = path.join(directory, "configs", "00"); + const edges = path.join(directory, "comparisons"); + fs.mkdirSync(config, { recursive: true }); + fs.mkdirSync(edges, { recursive: true }); + fs.mkdirSync(path.join(config, "shot.PNG"), { recursive: true }); + fs.writeFileSync( + path.join(directory, "feature-comparison.transit"), + "comparison\n", + ); + fs.writeFileSync( + path.join(directory, "feature-comparison.log"), + "progress\n", + ); + fs.writeFileSync(path.join(directory, "keep.txt"), "keep\n"); + for (const file of ["run.transit", "run.log", "case-load.PNG"]) { + fs.writeFileSync(path.join(config, file), "configuration output\n"); + } + fs.writeFileSync(path.join(config, "keep.txt"), "keep\n"); + fs.writeFileSync(path.join(config, "shot.PNG", "keep.txt"), "keep\n"); + for (const file of ["00__10.transit", "keep.txt"]) { + fs.writeFileSync(path.join(edges, file), "edge output\n"); + } + fs.writeFileSync(path.join(fixture.comparisons, "keep.txt"), "keep\n"); +} + +function installCargoMetadata(fixture, definitions) { + const manifest = path.join(fixture.rendererRoot, "Cargo.toml"); + const bin = path.join(path.dirname(fixture.rendererRoot), "fixture-bin"); + const metadata = path.join(fixture.rendererRoot, "metadata.json"); + const cargo = path.join(bin, "cargo"); + fs.writeFileSync( + manifest, + "[package]\nname = 'fixture'\nversion = '0.1.0'\n", + ); + fs.writeFileSync( + path.join(fixture.rendererRoot, "Cargo.lock"), + "version = 4\n", + ); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync( + metadata, + JSON.stringify({ + packages: [{ manifest_path: manifest, features: definitions }], + }), + ); + fs.writeFileSync( + cargo, + `#!/usr/bin/env node\nconst fs = require("node:fs");\nprocess.stdout.write(fs.readFileSync(process.env.CARGO_METADATA_FIXTURE, "utf8"));\n`, + { mode: 0o755 }, + ); + fixture.env.PATH = `${bin}${path.delimiter}${process.env.PATH}`; + fixture.env.CARGO_METADATA_FIXTURE = metadata; + return cargo; +} + +function installPreparedCompiler(fixture, breakLog, identityId = null) { + const k = transit.keyword; + const m = (pairs) => transit.map(pairs); + const marker = "RENDER_FIXTURE_BUILD_ID"; + const identity = transit + .writer("json") + .write( + m([k("build-id"), identityId ?? marker, k("cljs-profile-marks"), false]), + ); + const descriptor = m([ + k("id"), + k("fixture/load"), + k("scene"), + k("fixture"), + k("scene-description"), + "fixture", + k("scene-seed"), + 42, + k("params"), + m([]), + k("view"), + m([k("scale"), 1, k("x"), 0, k("y"), 0]), + k("context"), + k("fresh"), + k("preparation"), + m([]), + ]); + const catalog = transit + .writer("json") + .write(m([k("build-id"), identityId ?? marker, k("cases"), [descriptor]])); + const frontend = path.resolve(fixture.managed, "../../.."); + const host = path.join( + frontend, + "performance", + "benches", + "render_wasm", + "browser", + ); + const bin = path.join(fixture.rendererRoot, "fixture-bin"); + fs.mkdirSync(host, { recursive: true }); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(host, "index.html"), ""); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require('node:fs'); const path = require('node:path'); +fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, JSON.stringify({features: ['default'], profile: {opt_level: '3', debug_assertions: false}})); +for (const file of ['render-wasm.js', 'render-wasm.wasm', 'shared.js']) fs.writeFileSync(path.join(process.env.FRONTEND_DEST, file), 'fixture'); +`, + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(bin, "clojure"), + `#!/usr/bin/env node +const fs = require('node:fs'); const path = require('node:path'); +const id = process.env.RENDER_BENCH_BUILD_ID; +const identity = ${JSON.stringify(identity)}.replaceAll(${JSON.stringify(marker)}, id); +const catalog = ${JSON.stringify(catalog)}.replaceAll(${JSON.stringify(marker)}, id); +fs.writeFileSync(process.env.RENDER_BENCH_CASES_PATH, 'module.exports = {buildIdentity() {return ' + JSON.stringify(identity) + ';}, collectCases() {return ' + JSON.stringify(catalog) + ';}};'); +const log = path.join(path.dirname(process.env.RENDER_BENCH_CASES_PATH), 'build.log'); +const sabotage = ${breakLog ? "'fs.unlinkSync(' + JSON.stringify(log) + '); fs.mkdirSync(' + JSON.stringify(log) + ');'" : "''"}; +fs.writeFileSync(path.join(process.env.RENDER_BENCH_BROWSER_DIR, 'browser.js'), 'const fs = require("node:fs"); module.exports = {buildIdentity() {' + sabotage + 'return ' + JSON.stringify(identity) + ';}};'); +`, + { mode: 0o755 }, + ); + fixture.env.PATH = `${bin}${path.delimiter}${process.env.PATH}`; +} + + +test("unresolved compiler ownership exits while retaining its build and lock", testOptions, async (t) => { + for (const [command, interrupt] of [["build", null], ["run", "SIGINT"], ["ab", "SIGTERM"]]) { + const fixture = createFrontend(t); + if (command === "ab") installCargoMetadata(fixture, { default: [], a: [] }); + const groupFile = path.join(fixture.managed, "compiler-group.pid"); + const signalsFile = path.join(fixture.managed, "termination-signals.log"); + const lockFile = path.join(fixture.rendererRoot, "compiler.lock"); + const guard = path.join(fixture.managed, "unresolved-process.cjs"); + fs.writeFileSync(fixture.compiler, `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const { spawn } = require("node:child_process"); +fs.writeFileSync(path.join(process.env.FRONTEND_DEST, "partial.js"), "retained compiler output"); +const worker = spawn("flock", ["--exclusive", ${JSON.stringify(lockFile)}, process.execPath, "-e", + "process.stdout.write('compiler ready'); setInterval(() => {}, 1000);"], { stdio: "inherit" }); +worker.on("exit", (code) => { process.exitCode = code; }); +`, { mode: 0o755 }); + fs.writeFileSync(guard, ` +const fs = require("node:fs"); +const cp = require("node:child_process"); +const spawn = cp.spawn; +const kill = process.kill; +const interrupt = ${JSON.stringify(interrupt)}; +let group; +cp.spawn = function (command, ...args) { + const child = spawn.call(this, command, ...args); + if (command === ${JSON.stringify(fixture.compiler)} && child.pid) { + group = child.pid; + fs.writeFileSync(${JSON.stringify(groupFile)}, String(group)); + if (interrupt) child.stdout.once("data", () => process.kill(process.pid, interrupt)); + } + return child; +}; +process.kill = function (pid, signal) { + if (pid === -group && ["SIGTERM", "SIGKILL"].includes(signal)) { + fs.appendFileSync(${JSON.stringify(signalsFile)}, signal + "\\n"); + return true; + } + return kill.apply(this, arguments); +}; +require("node:module").syncBuiltinESMExports(); +`); + fixture.nodeArgs = ["--require", guard]; + const args = command === "ab" ? ["ab", "--features", "a", "--filter", "rects/load"] : [command]; + if (!interrupt) args.push("--build-timeout-ms", "1000"); + if (command === "run") args.push("--remove"); + args.push("--no-color"); + try { + const result = await runCli(fixture, args); + assert.equal(result.exitCode, interrupt === "SIGINT" ? 130 : interrupt === "SIGTERM" ? 143 : 1, result.stderr); + assert.match(result.stderr, /unresolved/); + assert.match(result.stderr, /pgid/); + assert.ok(result.stderr.includes(fixture.compiler), result.stderr); + assert.deepEqual(fs.readFileSync(signalsFile, "utf8").trim().split("\n"), ["SIGTERM", "SIGKILL"]); + const group = Number(fs.readFileSync(groupFile, "utf8")); + assert.ok(process.kill(group, 0)); + assert.equal(spawnSync("flock", ["--nonblock", lockFile, "true"], { timeout: 1000 }).status, 1); + const ids = fs.readdirSync(fixture.builds); + assert.equal(ids.length, 1); + const directory = path.join(fixture.builds, ids[0]); + const manifest = readManifest(path.join(directory, "build.transit")); + assert.equal(manifest.get(transit.keyword("status")).toString(), ":failed"); + assert.equal(manifest.get(transit.keyword("failure")).get(transit.keyword("unresolved-process")), true); + assert.equal(fs.readFileSync(path.join(directory, "renderer", "partial.js"), "utf8"), "retained compiler output"); + if (command === "ab") { + const k = transit.keyword; + const [id] = fs.readdirSync(fixture.comparisons); + const comparison = readManifest(path.join(fixture.comparisons, id, "feature-comparison.transit")); + assert.equal(comparison.get(k("status")).toString(), ":interrupted"); + assert.deepEqual(comparison.get(k("configurations")).map((entry) => entry.get(k("status")).toString()), + [":failed", ":unattempted"]); + assert.deepEqual(comparison.get(k("edges")).map((entry) => entry.get(k("status")).toString()), [":unattempted"]); + } + } finally { + if (fs.existsSync(groupFile)) { + const group = Number(fs.readFileSync(groupFile, "utf8")); + assert.ok(Number.isSafeInteger(group) && group > 0); + try { + process.kill(-group, "SIGKILL"); + } catch (error) { + if (error.code !== "ESRCH") throw error; + } + const deadline = Date.now() + 3000; + while (spawnSync("flock", ["--nonblock", lockFile, "true"], { timeout: 1000 }).status === 1) { + assert.ok(Date.now() < deadline, "Owned fixture lock did not release"); + await pause(10); + } + } + } + } +}); + +function runFixtureText({ + runId, + caseId, + value, + environment = { node: "v24" }, + mode = "release", + features = [], + scene = "case", + sceneSeed = 7, + count = 10, + fingerprint = "a".repeat(64), + view = { scale: 1, x: 0, y: 0 }, + context = "fresh", + warmups = 0, + repetitions = 1, + gitSha = runId, +}) { + const k = transit.keyword; + const map = (pairs) => transit.map(pairs); + const event = map([ + k("command"), k("fixture/work"), + k("metrics"), map([k("command/elapsed"), [value]]), + ]); + const preparation = map([ + k("preparation-id"), + `${runId}-preparation`, + k("case-id"), + k(caseId), + k("warmup?"), + false, + k("fingerprint"), + map([k("algorithm"), k("sha-256"), k("digest"), fingerprint]), + k("effective-graphics"), + map([k("webgl2"), true]), + k("diagnostics"), + map([]), + k("phases"), + [map([ + k("phase"), k("preparation"), + k("preparation-id"), `${runId}-preparation`, + k("outcome"), k("completed"), + k("events"), [], + ])], + ]); + const makeAttempt = (attemptId, warmup) => + map([ + k("attempt-id"), + attemptId, + k("case-id"), + k(caseId), + k("preparation-id"), + `${runId}-preparation`, + k("warmup?"), + warmup, + k("outcome"), + k("completed"), + k("phases"), + [map([ + k("phase"), k("main"), + k("preparation-id"), `${runId}-preparation`, + k("outcome"), k("completed"), + k("events"), [event], + ])], + ]); + const attempts = [ + ...Array.from({ length: warmups }, (_, index) => makeAttempt(index, true)), + ...Array.from({ length: repetitions }, (_, index) => + makeAttempt(warmups + index, false), + ), + ]; + const record = map([ + k("type"), + k("run-record"), + k("run-id"), + runId, + k("started-at"), + "2026-10-26T00:00:00.000Z", + k("ended-at"), + "2026-10-26T00:01:00.000Z", + k("plan"), + map([ + k("cases"), + [ + map([ + k("id"), + k(caseId), + k("scene"), + k(scene), + k("scene-seed"), + sceneSeed, + k("params"), + map([k("count"), count]), + k("view"), + map(Object.entries(view).flatMap(([key, entry]) => [k(key), entry])), + k("context"), + k(context), + k("preparation"), + map([]), + ]), + ], + k("warmups"), + warmups, + k("repetitions"), + repetitions, + k("timeout-policy"), + map([k("case-ms"), 30000]), + ]), + k("metadata"), + map([ + k("git"), + map([k("sha"), gitSha, k("dirty"), false]), + k("build"), + map([ + k("functional"), + map([k("mode"), k(mode), k("features"), features]), + ]), + k("environment"), + map( + Object.entries(environment).flatMap(([key, entry]) => [k(key), entry]), + ), + k("scored?"), + true, + k("diagnostics"), + map([]), + ]), + k("analysis"), + map([ + k("draws"), + 2000, + k("seed"), + 1885695600, + k("confidence"), + 0.95, + k("min-ci"), + 10, + k("min-p95"), + 100, + k("min-p99"), + 500, + ]), + k("quantities"), + map([k("command/elapsed"), map([k("unit"), k("ms")])]), + k("preparations"), + [preparation], + k("attempts"), + attempts, + k("case-outcomes"), + map([k(caseId), map([k("outcome"), k("completed")])]), + k("remaining"), + map([k(caseId), map([k("warmups"), 0, k("repetitions"), 0])]), + k("termination"), + map([k("reason"), k("completed")]), + ]); + return transit.writer("json").write(record); +} + +function tree(directory) { + return fs + .readdirSync(directory, { withFileTypes: true }) + .sort((a, b) => a.name.localeCompare(b.name)) + .map((entry) => { + const file = path.join(directory, entry.name); + return [ + entry.name, + entry.isDirectory() ? tree(file) : fs.readFileSync(file), + ]; + }); +} + +const failingCompiler = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const directory = path.dirname(process.env.FRONTEND_DEST); +fs.copyFileSync( + path.join(directory, "build.transit"), + path.join(directory, "before-compiler.transit") +); +fs.writeFileSync( + path.join(process.env.FRONTEND_DEST, "partial.js"), + "partial renderer output\\n" +); +process.stdout.write("compiler stdout before failure\\n"); +process.stderr.write("compiler stderr before failure\\n"); +process.exitCode = 23; +`; + +test( + "build prints its allocation and one failure block with retained compiler evidence", + testOptions, + async (t) => { + const fixture = createFrontend(t); + fs.writeFileSync(fixture.compiler, failingCompiler, { mode: 0o755 }); + + const args = ["build", "--build-timeout-ms", "10000"]; + const result = await runCli(fixture, args); + assert.equal(result.exitCode, 1); + const [buildLine] = result.stdout.trim().split("\n"); + const id = buildLine.slice("Build ".length); + assert.match( + id, + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i, + ); + assert.equal( + result.stdout, + `Build ${id}\nOutput ${path.join(fixture.builds, id)}\n`, + ); + assert.deepEqual(fs.readdirSync(fixture.builds), [id]); + + const directory = path.join(fixture.builds, id); + const logPath = path.join(directory, "build.log"); + const initial = readManifest( + path.join(directory, "before-compiler.transit"), + ); + const manifest = readManifest(path.join(directory, "build.transit")); + const k = transit.keyword; + assert.equal(initial.get(k("id")), id); + assert.equal(initial.get(k("status")).toString(), ":building"); + assert.equal(manifest.get(k("id")), id); + assert.equal(manifest.get(k("status")).toString(), ":failed"); + const failure = manifest.get(k("failure")); + assert.equal(failure.get(k("phase")), "renderer"); + assert.equal(failure.get(k("exit-code")), 23); + assert.equal(failure.get(k("command")), fixture.compiler); + assert.equal( + failure.get(k("cause"))[0].get(k("type")).toString(), + ":process-exit", + ); + assert.equal( + fs.readFileSync(path.join(directory, "renderer", "partial.js"), "utf8"), + "partial renderer output\n", + ); + + const log = fs.readFileSync(logPath, "utf8"); + for (const output of [log, result.stderr]) { + assert.ok(output.includes("compiler stdout before failure\n")); + assert.ok(output.includes("compiler stderr before failure\n")); + } + assert.match(result.stderr, /renderer build preparation failed/); + assert.match( + result.stderr, + /type\s+:benches.render-wasm.node.runner.build\/preparation-failed/, + ); + assert.match(result.stderr, new RegExp(`build\\s+${id}`)); + assert.ok(result.stderr.includes(logPath)); + assert.match(result.stderr, /phase\s+:renderer/); + assert.match(result.stderr, /exit-code\s+23/); + assert.ok(result.stderr.includes(fixture.compiler), result.stderr); + assert.equal(result.stderr.split("✖ ").length - 1, 1); + assert.doesNotMatch(result.stderr, /^compiler (stdout|stderr)/m); + assert.match(log, /✖ renderer build preparation failed/); + }, +); + +test( + "build retains selected and loaded identities in saved and terminal failures", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const actualId = `other\n\u001b${"x".repeat(2048)}`; + installPreparedCompiler(fixture, false, actualId); + + const result = await runCli(fixture, [ + "build", + "--no-color", + "--build-timeout-ms", + "10000", + ]); + + assert.equal(result.exitCode, 1, result.stderr); + const id = result.stdout.match(/^Build ([0-9a-f-]+)$/m)?.[1]; + assert.ok(id, result.stdout); + const k = transit.keyword; + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + const failure = manifest.get(k("failure")); + assert.equal(manifest.get(k("status")).toString(), ":failed"); + assert.equal(failure.get(k("phase")), "identity"); + assert.equal( + failure.get(k("cause"))[0].get(k("type")).toString(), + ":benches.render-wasm.identity/mismatch", + ); + assert.equal(failure.get(k("expected-id")), id); + assert.equal(failure.get(k("actual-id")), actualId.slice(0, 1024)); + assert.match(result.stderr, new RegExp(`expected-id\\s+"${id}"`)); + const actualField = result.stderr.match(/^ actual-id\s+(.+)$/m)?.[1]; + assert.ok(actualField, result.stderr); + assert.ok(actualField.includes("other"), actualField); + assert.ok(actualField.length <= 1024, actualField.length); + assert.ok( + [...actualField].every((character) => character.charCodeAt(0) >= 32), + actualField, + ); + }, +); + +test( + "build failure diagnostics retain semantic and storage causes", + testOptions, + async (t) => { + for (const [source, reason, broken] of [ + [ + `#!/usr/bin/env node +const fs = require('node:fs'); +fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, JSON.stringify({features: [], profile: {opt_level: '0', debug_assertions: false}})); +`, + /Cargo did not produce an optimized renderer/, + ], + [ + failingCompiler + + "\nfs.unlinkSync(require('node:path').join(require('node:path').dirname(process.env.FRONTEND_DEST), 'build.transit'));\n", + /Command exited unsuccessfully/, + "manifest", + ], + [ + `#!/usr/bin/env node +const fs = require('node:fs'); +const path = require('node:path'); +const log = path.join(path.dirname(process.env.FRONTEND_DEST), 'build.log'); +fs.unlinkSync(log); fs.mkdirSync(log); +process.stdout.write('compiler output\\n'); +`, + /cannot append benchmark log/, + "log", + ], + ]) { + const fixture = createFrontend(t); + fs.writeFileSync(fixture.compiler, source, { mode: 0o755 }); + const result = await runCli(fixture, ["build"]); + const [id] = fs.readdirSync(fixture.builds); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /✖ renderer build preparation failed/); + assert.match(result.stderr, reason); + assert.ok(result.stderr.includes(id)); + assert.ok( + result.stderr.includes(path.join(fixture.builds, id, "build.log")), + ); + if (broken !== "log") { + const log = fs.readFileSync( + path.join(fixture.builds, id, "build.log"), + "utf8", + ); + assert.match(log, /✖ renderer build preparation failed/); + assert.match(log, reason); + } + if (broken === "manifest") { + assert.match(result.stderr, /manifest-error\s+.*ENOENT/); + } else { + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + assert.equal( + manifest.get(transit.keyword("status")).toString(), + ":failed", + ); + } + } + }, +); + +test( + "build completion retains a usable ready build or the original log failure", + testOptions, + async (t) => { + for (const breakLog of [false, true]) { + const fixture = createFrontend(t); + installPreparedCompiler(fixture, breakLog); + const result = await runCli(fixture, ["build"]); + const [id] = fs.readdirSync(fixture.builds); + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + assert.equal(result.exitCode, breakLog ? 1 : 0, result.stderr); + assert.equal( + manifest.get(transit.keyword("status")).toString(), + breakLog ? ":failed" : ":ready", + ); + if (breakLog) { + assert.match(result.stderr, /cannot append benchmark log/); + assert.ok( + result.stderr.includes(path.join(fixture.builds, id, "build.log")), + ); + } else { + assert.match(result.stderr, /ready/); + } + } + }, +); + +test( + "output flags control progress, raw output and console colors", + testOptions, + async (t) => { + for (const [flags, phases, raw, colors] of [ + [[], true, false, false], + [["--verbose"], true, true, false], + [["--quiet"], false, false, false], + [["--color"], true, false, true], + [["--no-color"], true, false, false], + ]) { + const fixture = createFrontend(t); + fs.writeFileSync(fixture.compiler, failingCompiler, { mode: 0o755 }); + const result = await runCli(fixture, ["build", ...flags]); + assert.equal(result.exitCode, 1, result.stderr); + assert.match(result.stdout, /^Build [^\n]+\nOutput \/[^\n]+\n$/); + assert.equal( + /^(?:\u001b\[[0-9;]*m)?Renderer compiling/m.test(result.stderr), + phases, + ); + assert.equal( + result.stderr.includes("· compiler stdout before failure"), + raw, + ); + assert.equal(result.stderr.includes("\u001b["), colors); + assert.match(result.stderr, /! Git provenance unavailable:/); + const [id] = fs.readdirSync(fixture.builds); + const log = fs.readFileSync( + path.join(fixture.builds, id, "build.log"), + "utf8", + ); + assert.match(log, /^compiler stdout before failure$/m); + assert.match(log, /^Renderer compiling$/m); + assert.doesNotMatch(log, /\u001b\[/); + } + }, +); + +test( + "ab retains its log and names the nested build tail on failure", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + fs.writeFileSync(fixture.compiler, failingCompiler, { mode: 0o755 }); + const result = await runCli(fixture, [ + "ab", + "--features", + "a", + "--filter", + "rects/load", + ]); + assert.equal(result.exitCode, 1, result.stderr); + const [idLine, outputLine, statusLine] = result.stdout.trim().split("\n"); + assert.match(idLine, /^Comparison [0-9a-f-]+$/); + const directory = outputLine.slice("Output ".length); + assert.equal(statusLine, "Status failed"); + const logPath = path.join(directory, "feature-comparison.log"); + assert.equal(result.stderr.split("✖ ").length - 1, 1); + assert.ok(result.stderr.includes(logPath)); + assert.match(result.stderr, /tail-log\s+.*build\.log/); + assert.match( + fs.readFileSync(logPath, "utf8"), + /compiler stderr before failure/, + ); + await runCli(fixture, ["purge", "comparisons"]); + assert.equal(fs.existsSync(logPath), false); + }, +); + +test( + "ab prints status once and preserves the build failure when its log becomes unwritable", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + const output = path.join(fixture.managed, "comparison-output"); + fixture.env.COMPARISON_LOG = path.join(output, "feature-comparison.log"); + fs.writeFileSync( + fixture.compiler, + failingCompiler + + "\nfs.unlinkSync(process.env.COMPARISON_LOG); fs.mkdirSync(process.env.COMPARISON_LOG);\n", + { mode: 0o755 }, + ); + const result = await runCli(fixture, [ + "ab", + "--features", + "a", + "--filter", + "rects/load", + "--output-dir", + output, + ]); + assert.equal(result.exitCode, 1); + assert.equal(result.stdout.split("Status failed\n").length - 1, 1); + assert.equal(result.stdout.trim().split("\n").length, 3); + assert.match(result.stderr, /✖ renderer build preparation failed/); + }, +); + +test( + "run and ab retain signal exit codes during preparation", + testOptions, + async (t) => { + for (const command of ["run", "ab"]) { + for (const [signal, code] of [ + ["SIGINT", 130], + ["SIGTERM", 143], + ]) { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + const marker = path.join(fixture.managed, "compiler-started"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node\nconst fs = require('node:fs'); fs.writeFileSync(${JSON.stringify(marker)}, 'ready'); setInterval(() => {}, 1000);\n`, + { mode: 0o755 }, + ); + const args = + command === "ab" + ? ["ab", "--features", "a", "--filter", "rects/load"] + : ["run"]; + const failPublication = command === "run" && signal === "SIGINT"; + if (failPublication) { + const writer = path.join(fixture.managed, "writer.cjs"); + fs.writeFileSync(writer, `process.stdout.write = () => { throw new Error("publication unavailable"); };`); + fixture.nodeArgs = ["--require", writer]; + } + const child = spawn(process.execPath, [...(fixture.nodeArgs ?? []), fixture.runner, ...args], { + cwd: os.tmpdir(), + env: { ...process.env, ...fixture.env }, + stdio: ["ignore", "pipe", "pipe"], + }); + t.after(() => { + if (child.exitCode === null) child.kill("SIGKILL"); + }); + const exited = once(child, "exit"); + const closed = once(child, "close"); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const deadline = Date.now() + 10000; + while ( + !fs.existsSync(marker) && + child.exitCode === null && + Date.now() < deadline + ) + await pause(10); + assert.ok(fs.existsSync(marker), stderr); + child.kill(signal); + assert.deepEqual(await exited, [code, null], stderr); + await closed; + if (command === "ab") { + assert.match(stdout, /^Status interrupted$/m); + assert.equal(stdout.trim().split("\n").length, 3); + } + if (failPublication) assert.match(stderr, /publication unavailable/); + } + } + }, +); + +test( + "run records an early retained-build failure in its allocated log", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const output = path.join(fixture.managed, "early-run"); + const result = await runCli(fixture, [ + "run", + "--build", + "missing", + "--output-dir", + output, + ]); + assert.equal(result.exitCode, 1); + const logPath = path.join(output, "run.log"); + assert.ok(result.stderr.includes(logPath)); + assert.match(fs.readFileSync(logPath, "utf8"), /✖ /); + }, +); + +test( + "ab dry-run prints the Gray plan without creating comparison output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + fixture.env.PENPOT_GPU = "0"; + installCargoMetadata(fixture, { default: [], a: [], b: ["a"] }); + const retained = tree(fixture.managed); + + const result = await runCli(fixture, [ + "ab", + "--features", + "b,a", + "--filter", + "rects/load", + "--dry-run", + "--require-gpu", + ]); + + assert.equal(result.exitCode, 0); + assert.equal(result.stderr, ""); + assert.match(result.stdout, /^Feature comparison plan/); + for (const part of [ + "Features: a, b", + "00: ", + "01: b", + "11: a, b", + "10: a", + "00 -> 10: a", + "00 -> 01: b", + "01 -> 11: a", + "10 -> 11: b", + "Cases: 1", + ]) { + assert.ok(result.stdout.includes(part), part); + } + assert.equal(fs.existsSync(fixture.comparisons), false); + assert.deepEqual(tree(fixture.managed), retained); + }, +); + +test( + "ab validates requested features before creating comparison output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + const retained = tree(fixture.managed); + + const result = await runCli(fixture, [ + "ab", + "--features", + "missing", + "--filter", + "rects/load", + "--dry-run", + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /unknown requested Cargo features/); + assert.equal(fs.existsSync(fixture.comparisons), false); + assert.deepEqual(tree(fixture.managed), retained); + }, +); + +test("purge deletes only the selected artifacts", testOptions, async (t) => { + const fixture = createFrontend(t); + const retained = tree(fixture.managed); + seedBuilds(fixture); + seedRuns(fixture); + seedComparisons(fixture); + const runs = tree(fixture.runs); + const comparisons = tree(fixture.comparisons); + + assert.deepEqual(await runCli(fixture, ["purge", "builds"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + for (const id of fs.readdirSync(fixture.builds)) { + assert.deepEqual(fs.readdirSync(path.join(fixture.builds, id)), [ + "keep.txt", + ]); + } + assert.deepEqual(tree(fixture.runs), runs); + assert.deepEqual(tree(fixture.comparisons), comparisons); + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); + + seedBuilds(fixture); + const builds = tree(fixture.builds); + assert.deepEqual(await runCli(fixture, ["purge", "runs"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + for (const id of fs.readdirSync(fixture.runs)) { + assert.deepEqual(fs.readdirSync(path.join(fixture.runs, id)), ["keep.txt"]); + } + assert.deepEqual(tree(fixture.builds), builds); + assert.deepEqual(tree(fixture.comparisons), comparisons); + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); + + const buildsBeforeComparisonPurge = tree(fixture.builds); + const runsBeforeComparisonPurge = tree(fixture.runs); + assert.deepEqual(await runCli(fixture, ["purge", "comparisons"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + const [comparisonId] = fs.readdirSync(fixture.comparisons); + const comparison = path.join(fixture.comparisons, comparisonId); + for (const file of [ + "feature-comparison.log", + "feature-comparison.transit", + "configs/00/run.transit", + "configs/00/run.log", + "configs/00/case-load.PNG", + "comparisons/00__10.transit", + ]) { + assert.equal(fs.existsSync(path.join(comparison, file)), false, file); + } + for (const file of [ + "keep.txt", + "configs/00/keep.txt", + "configs/00/shot.PNG/keep.txt", + "comparisons/keep.txt", + ]) { + assert.equal(fs.existsSync(path.join(comparison, file)), true, file); + } + assert.deepEqual(tree(fixture.builds), buildsBeforeComparisonPurge); + assert.deepEqual(tree(fixture.runs), runsBeforeComparisonPurge); + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); + + seedRuns(fixture); + seedComparisons(fixture); + assert.deepEqual(await runCli(fixture, ["purge", "all"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + for (const root of [fixture.builds, fixture.runs]) { + for (const id of fs.readdirSync(root)) { + assert.deepEqual(fs.readdirSync(path.join(root, id)), ["keep.txt"]); + } + } + for (const id of fs.readdirSync(fixture.comparisons)) { + const directory = path.join(fixture.comparisons, id); + if (id === "keep.txt") continue; + for (const file of [ + "feature-comparison.transit", + "configs/00/run.transit", + "configs/00/run.log", + "configs/00/case-load.PNG", + "comparisons/00__10.transit", + ]) { + assert.equal(fs.existsSync(path.join(directory, file)), false, file); + } + assert.equal(fs.existsSync(path.join(directory, "keep.txt")), true); + } + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); +}); + +test( + "run reserves nested output before the compiler starts", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const output = path.join(fixture.managed, "missing", "parents", "output"); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require("node:fs"); +if (!fs.statSync(${JSON.stringify(path.join(output, "run.log"))}).isFile()) process.exit(99); +fs.writeFileSync(${JSON.stringify(marker)}, "reserved"); +process.exit(23); +`, + { mode: 0o755 }, + ); + + const result = await runCli(fixture, ["run", "--output-dir", output]); + + assert.equal(result.exitCode, 1); + assert.match( + result.stdout, + new RegExp(`^Run [0-9a-f-]+\\nOutput ${output}\\n$`), + ); + assert.equal(fs.readFileSync(marker, "utf8"), "reserved"); + assert.equal(fs.existsSync(path.join(output, "run.transit")), false); + }, +); + +test( + "run rejects invalid output parents before invoking the compiler", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const parent = path.join(fixture.managed, "not-a-directory"); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync(parent, "keep"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +require("node:fs").writeFileSync(${JSON.stringify(marker)}, "ran"); +process.exit(23); +`, + { mode: 0o755 }, + ); + + const result = await runCli(fixture, [ + "run", + "--output-dir", + path.join(parent, "output"), + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /ENOTDIR|cannot create run home/); + assert.equal(fs.existsSync(marker), false); + assert.equal(fs.readFileSync(parent, "utf8"), "keep"); + }, +); + +test( + "run rejects --remove outputs inside builds including directory aliases", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const marker = path.join(fixture.managed, "compiler-ran"); + const alias = path.join(path.dirname(fixture.managed), "build-alias"); + fs.mkdirSync(fixture.builds); + fs.symlinkSync(fixture.builds, alias, "dir"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +require("node:fs").writeFileSync(${JSON.stringify(marker)}, "ran"); +process.exit(23); +`, + { mode: 0o755 }, + ); + + for (const root of [fixture.builds, alias]) { + const output = path.join(root, "missing", "output"); + const result = await runCli(fixture, [ + "run", + "--remove", + "--output-dir", + output, + ]); + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /--remove.*inside builds/); + assert.equal(fs.existsSync(output), false); + } + assert.equal(fs.existsSync(marker), false); + }, +); + +test( + "malformed purge categories leave all files intact", + testOptions, + async (t) => { + const fixture = createFrontend(t); + seedBuilds(fixture); + seedRuns(fixture); + const original = tree(fixture.managed); + + for (const args of [ + ["purge"], + ["purge", "unknown"], + ["purge", "builds", "runs"], + ]) { + const result = await runCli(fixture, args); + assert.equal(result.exitCode, 1, args.join(" ")); + assert.equal(result.stdout, ""); + assert.match( + result.stderr, + /Usage: runner purge/, + ); + assert.deepEqual(tree(fixture.managed), original, args.join(" ")); + } + }, +); + +test( + "required GPU access rejects run and ab before allocating output or checking builds", + testOptions, + async (t) => { + for (const command of [ + ["run"], + ["ab", "--features", "a", "--filter", "rects/load"], + ]) { + const fixture = createFrontend(t); + fixture.env.PENPOT_GPU = ""; + const original = tree(fixture.managed); + const result = await runCli(fixture, [ + ...command, + "--require-gpu", + "--output-dir", + path.join(fixture.managed, "gpu-run"), + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /PENPOT_GPU=1/); + assert.match(result.stderr, /\.\/manage\.sh run-devenv --gpu/); + assert.match(result.stderr, /pnpm run render-bench/); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test( + "run validates an explicit output dir before preparing a build", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require("node:fs"); +fs.writeFileSync(${JSON.stringify(marker)}, "ran"); +`, + { mode: 0o755 }, + ); + const output = path.join(fixture.managed, "existing-output"); + fs.mkdirSync(output); + const original = tree(fixture.managed); + + const result = await runCli(fixture, [ + "run", + "--output-dir", + output, + "--warmups", + "0", + "--repetitions", + "1", + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /output directory already exists/); + assert.equal(fs.existsSync(marker), false); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test( + "run rejects retained-build feature selection before any work", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const marker = path.join(fixture.managed, "compiler-ran"); + fs.writeFileSync( + fixture.compiler, + `#!/usr/bin/env node +const fs = require("node:fs"); +fs.writeFileSync(${JSON.stringify(marker)}, "ran"); +`, + { mode: 0o755 }, + ); + const original = tree(fixture.managed); + + for (const flags of [["--features", "functional"], ["--build-timeout-ms", "1000"]]) { + const result = await runCli(fixture, ["run", "--build", "retained-id", ...flags]); + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /cannot be combined/); + assert.equal(fs.existsSync(marker), false); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test( + "help and compare work without a frontend checkout or execution dependencies", + testOptions, + async (t) => { + const temporary = fs.mkdtempSync( + path.join(os.tmpdir(), "render-wasm-offline-"), + ); + t.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const runner = path.join(temporary, "runner.cjs"); + const guard = path.join(temporary, "guard.cjs"); + fs.copyFileSync(bootstrap, runner); + fs.symlinkSync( + path.join(frontendRoot, "node_modules"), + path.join(temporary, "node_modules"), + "dir", + ); + fs.writeFileSync( + guard, + ` + const Module = require("node:module"); + const load = Module._load; + Module._load = function (id, ...args) { + if (["playwright", "http", "node:http", "child_process", "node:child_process"].includes(id)) { + throw new Error("Offline command imported execution dependency: " + id); + } + return load.call(this, id, ...args); + }; + `, + ); + const baseline = path.join(temporary, "baseline.transit"); + const candidate = path.join(temporary, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }), + ); + const fixture = { runner, nodeArgs: ["--require", guard] }; + + const help = await runCli(fixture, ["--help"]); + assert.equal(help.exitCode, 0, help.stderr); + assert.match(help.stdout, /runner compare/); + assert.match(help.stdout, /runner merge/); + const abHelp = await runCli(fixture, ["ab", "--help"]); + assert.equal(abHelp.exitCode, 0, abHelp.stderr); + assert.match(abHelp.stdout, /runner ab --features/); + assert.match(abHelp.stdout, /--max-features/); + const mergeHelp = await runCli(fixture, ["merge", "--help"]); + assert.equal(mergeHelp.exitCode, 0, mergeHelp.stderr); + assert.match(mergeHelp.stdout, /runner merge BASE ADDITIONAL/); + for (const args of [["ab"], ["ab", "--dry-run"]]) { + const invalid = await runCli(fixture, args); + assert.equal(invalid.exitCode, 1, invalid.stderr); + assert.match(invalid.stderr, /at least one requested feature/); + assert.doesNotMatch( + invalid.stderr, + /execution dependency|Cannot find module/, + ); + } + const comparison = await runCli(fixture, ["compare", baseline, candidate]); + assert.equal(comparison.exitCode, 0, comparison.stderr); + assert.match(comparison.stdout, /median change/); + for (const flag of [ + "--allow-build-differences", + "--allow-scene-differences", + "--allow-all-differences", + ]) { + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + flag, + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /median change/); + } + assert.equal(fs.existsSync(path.join(temporary, "commands.cjs")), false); + }, +); + +test( + "help lists run and compare without touching managed storage", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const original = tree(fixture.managed); + + const result = await runCli(fixture, ["--help"]); + + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /runner run/); + assert.match(result.stdout, /runner compare/); + assert.match(result.stdout, /runner merge/); + const compareHelp = await runCli(fixture, ["compare", "--help"]); + assert.equal(compareHelp.exitCode, 0, compareHelp.stderr); + assert.match(compareHelp.stdout, /--cases/); + assert.match(compareHelp.stdout, /--allow-build-differences/); + assert.match(compareHelp.stdout, /--allow-scene-differences/); + assert.match(compareHelp.stdout, /--allow-all-differences/); + assert.match(result.stdout, /-q, --quiet/); + assert.match(result.stdout, /-v, --verbose/); + assert.match(result.stdout, /--\[no-\]color/); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test( + "compare and merge identify unreadable saved inputs", + testOptions, + async (t) => { + const cases = [ + { + command: "compare", + role: "baseline", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "corrupt", + message: /JSON/, + type: null, + }, + { + command: "compare", + role: "baseline", + kind: "unsupported", + message: /unsupported benchmark format/, + type: /result\/unsupported-format/, + }, + { + command: "compare", + role: "candidate", + kind: "invalid", + message: /invalid schema:run/, + type: /result\/invalid-record/, + }, + { + command: "compare", + role: "candidate", + kind: "directory", + message: /EISDIR/, + type: null, + }, + { + command: "merge", + role: "base", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "merge", + role: "additional", + kind: "corrupt", + message: /JSON/, + type: null, + }, + ]; + for (const { command, role, kind, message, type } of cases) { + await t.test(`${command}: ${role} ${kind}`, async (t) => { + const fixture = createFrontend(t); + const valid = path.join(fixture.managed, "valid.transit"); + const input = path.join(fixture.managed, `${kind}.transit`); + fs.writeFileSync( + valid, + runFixtureText({ runId: "valid", caseId: "case/load", value: 10 }), + ); + if (kind === "directory") { + fs.mkdirSync(input); + } else if (kind !== "missing") { + const text = + kind === "corrupt" + ? "not JSON" + : kind === "unsupported" + ? "{}" + : transit + .writer("json") + .write( + transit.map([ + transit.keyword("type"), + transit.keyword("run-record"), + ]), + ); + fs.writeFileSync(input, text); + } + const inputs = + role === "baseline" || role === "base" + ? [input, valid] + : [valid, input]; + const result = await runCli(fixture, [ + command, + ...inputs, + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, message); + assert.ok(result.stderr.includes(input), result.stderr); + assert.match(result.stderr, new RegExp(`role\\s+:${role}\\b`)); + assert.match(result.stderr, /operation\s+:read-record\b/); + if (type) assert.match(result.stderr, type); + }); + } + }, +); + +test( + "compare bounds and escapes the reported input path", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const prefix = 'missing\n"\\\u001b'; + const prefixLength = JSON.stringify(prefix).length - 1; + const input = `${prefix}${"x".repeat(1023 - prefixLength)}😀.transit`; + const result = await runCli(fixture, [ + "compare", + input, + "unused.transit", + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /role\s+:baseline\b/); + const reportedPath = result.stderr.match(/^ path\s+(.+)$/m)?.[1]; + assert.ok(reportedPath, result.stderr); + assert.match(reportedPath, /^"missing\\n\\"\\\\\\u001b/); + assert.ok(reportedPath.length <= 1024, reportedPath.length); + assert.ok(reportedPath.endsWith("x"), reportedPath); + }, +); + +test( + "compare reads saved run files without starting resources", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }), + ); + const original = tree(fixture.managed); + + const result = await runCli(fixture, ["compare", baseline, candidate]); + + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /Baseline/); + assert.match(result.stdout, /case\/load/); + assert.match(result.stdout, /median change/); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test( + "compare supports explicit pairs and all-differences from the CLI", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "a/pan", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "b/pan", + value: 12, + environment: { node: "v25" }, + }), + ); + const original = tree(fixture.managed); + + const incompatible = await runCli(fixture, [ + "compare", + baseline, + candidate, + ]); + assert.equal(incompatible.exitCode, 1); + assert.deepEqual(tree(fixture.managed), original); + + const selected = ["compare", baseline, candidate, "--cases", "a/pan,b/pan"]; + const strict = await runCli(fixture, selected); + assert.equal(strict.exitCode, 1); + assert.match(strict.stderr, /incompatible saved runs/); + + const allowed = await runCli(fixture, [ + ...selected, + "--allow-all-differences", + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /:allow-all-differences true/); + assert.match(allowed.stdout, /ACKNOWLEDGED DIFFERENCE/); + assert.match(allowed.stdout, /:rule :all/); + assert.match(allowed.stdout, /Pair a\/pan,b\/pan/); + assert.deepEqual(tree(fixture.managed), original); + + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "a/pan", + value: 12, + environment: { node: "v25" }, + }), + ); + const automatic = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-all-differences", + ]); + assert.equal(automatic.exitCode, 0, automatic.stderr); + assert.match(automatic.stdout, /:allow-all-differences true/); + assert.match(automatic.stdout, /ACKNOWLEDGED DIFFERENCE/); + assert.match(automatic.stdout, /Pair a\/pan,a\/pan/); + }, +); + +test( + "compare applies build and scene permissions separately and together", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + + const build = "--allow-build-differences"; + const scene = "--allow-scene-differences"; + for (const scenario of [ + { + changes: { mode: "debug", features: ["fixture"] }, + allowed: [build], + denied: [[], [scene]], + fields: ["functional-build", "effective-features"], + rules: ["build"], + }, + { + changes: { + scene: "other", + sceneSeed: 8, + count: 20, + fingerprint: "b".repeat(64), + }, + allowed: [scene], + denied: [[], [build]], + fields: ["scene", "fingerprint"], + rules: ["scene"], + }, + { + changes: { mode: "debug", count: 20 }, + allowed: [build, scene], + denied: [[], [build], [scene]], + fields: ["functional-build", "scene"], + rules: ["build", "scene"], + }, + ]) { + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "case/load", + value: 12, + ...scenario.changes, + }), + ); + const original = tree(fixture.managed); + for (const flags of scenario.denied) { + const rejected = await runCli(fixture, [ + "compare", + baseline, + candidate, + ...flags, + ]); + assert.equal(rejected.exitCode, 1, rejected.stderr); + assert.match(rejected.stderr, /incompatible saved runs/); + } + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + ...scenario.allowed, + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /:allow-all-differences false/); + assert.match(allowed.stdout, /ACKNOWLEDGED DIFFERENCE/); + for (const flag of scenario.allowed) { + assert.ok(allowed.stdout.includes(`:${flag.slice(2)} true`)); + } + for (const field of scenario.fields) { + assert.ok(allowed.stdout.includes(`:field :${field}`)); + } + for (const rule of scenario.rules) { + assert.ok(allowed.stdout.includes(`:rule :${rule}`)); + } + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test( + "scoped compare permissions keep execution settings and runtime environment strict", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + for (const [field, changes] of [ + ["execution", { context: "reuse" }], + ["execution", { view: { scale: 2, x: 0, y: 0 } }], + ["environment", { environment: { node: "v25" } }], + ]) { + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "case/load", + value: 12, + ...changes, + }), + ); + const original = tree(fixture.managed); + const rejected = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-build-differences", + "--allow-scene-differences", + ]); + assert.equal(rejected.exitCode, 1); + assert.match(rejected.stderr, /incompatible saved runs/); + assert.match(rejected.stderr, new RegExp(`difference\\s+:${field}`)); + + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-all-differences", + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.ok(allowed.stdout.includes(`:field :${field}`)); + assert.match(allowed.stdout, /:rule :all/); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test("compare rejects --force without an alias", testOptions, async (t) => { + const fixture = createFrontend(t); + const original = tree(fixture.managed); + for (const flags of [["--force"], ["--force", "--allow-all-differences"]]) { + const rejected = await runCli(fixture, [ + "compare", + "missing-baseline.transit", + "missing-candidate.transit", + ...flags, + ]); + assert.equal(rejected.exitCode, 1); + assert.match(rejected.stderr, /Unknown option.*--force/); + assert.doesNotMatch(rejected.stderr, /ENOENT/); + assert.deepEqual(tree(fixture.managed), original); + } +}); + +test( + "merge writes a validated run home with appended measured attempts", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + const baseText = runFixtureText({ + runId: "base", + caseId: "case/load", + value: 10, + warmups: 1, + repetitions: 2, + gitSha: "same-sha", + }); + const additionalText = runFixtureText({ + runId: "additional", + caseId: "case/load", + value: 12, + warmups: 1, + repetitions: 3, + gitSha: "same-sha", + }); + fs.writeFileSync(base, baseText); + fs.writeFileSync(additional, additionalText); + const output = path.join(fixture.managed, "merged-run"); + + const result = await runCli(fixture, [ + "merge", + base, + additional, + "--output-dir", + output, + ]); + + assert.equal(result.exitCode, 0, result.stderr); + const runId = result.stdout.match(/Run ([0-9a-f-]{36})/)[1]; + assert.ok(fs.existsSync(path.join(output, "run.transit"))); + assert.equal(fs.readFileSync(base, "utf8"), baseText); + assert.equal(fs.readFileSync(additional, "utf8"), additionalText); + + const k = transit.keyword; + const merged = readManifest(path.join(output, "run.transit")); + const plan = merged.get(k("plan")); + const attempts = merged.get(k("attempts")); + const remaining = merged.get(k("remaining")).get(k("case/load")); + const mergeInfo = merged + .get(k("metadata")) + .get(k("diagnostics")) + .get(k("merge")); + + assert.equal(merged.get(k("run-id")), runId); + assert.equal(plan.get(k("warmups")), 1); + assert.equal(plan.get(k("repetitions")), 5); + assert.deepEqual( + attempts.map((attempt) => attempt.get(k("attempt-id"))), + [0, 1, 2, 3, 4, 5], + ); + assert.deepEqual( + attempts.map((attempt) => attempt.get(k("warmup?"))), + [true, false, false, false, false, false], + ); + assert.equal(remaining.get(k("warmups")), 0); + assert.equal(remaining.get(k("repetitions")), 0); + assert.equal(mergeInfo.get(k("base-run-id")), "base"); + assert.equal(mergeInfo.get(k("additional-run-id")), "additional"); + assert.equal(mergeInfo.get(k("additional-warmups-excluded")), 1); + assert.match( + fs.readFileSync(path.join(output, "run.log"), "utf8"), + /merge base \+ additional/, + ); + }, +); + +test( + "merge rejects incompatible runs before creating output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + fs.writeFileSync( + base, + runFixtureText({ + runId: "base", + caseId: "case/load", + value: 10, + gitSha: "sha-a", + }), + ); + fs.writeFileSync( + additional, + runFixtureText({ + runId: "additional", + caseId: "case/load", + value: 12, + gitSha: "sha-b", + }), + ); + const output = path.join(fixture.managed, "merged-run"); + const original = tree(fixture.managed); + + const result = await runCli(fixture, [ + "merge", + base, + additional, + "--output-dir", + output, + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /different git provenance/); + assert.equal(fs.existsSync(output), false); + assert.deepEqual(tree(fixture.managed), original); + }, +); + +test("merge retains rename and temporary-file cleanup failures", testOptions, async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + for (const [file, runId] of [[base, "base"], [additional, "additional"]]) { + fs.writeFileSync(file, runFixtureText({ runId, caseId: "case/load", value: 10, gitSha: "same" })); + } + const output = path.join(fixture.managed, "merged-run"); + const record = path.join(output, "run.transit"); + const guard = path.join(fixture.managed, "storage-failures.cjs"); + fs.writeFileSync(guard, ` + const fs = require("node:fs"); + const rename = fs.renameSync; + const remove = fs.rmSync; + const write = fs.writeFileSync; + const record = ${JSON.stringify(record)}; + fs.writeFileSync = function (file) { + if (String(file).startsWith(record + ".") && String(file).endsWith(".tmp")) { + write.call(fs, record, "previous complete record"); + } + return write.apply(this, arguments); + }; + fs.renameSync = function (source, destination) { + if (destination === record) throw new Error("record rename failed"); + return rename.apply(this, arguments); + }; + fs.rmSync = function (file) { + if (String(file).startsWith(record + ".") && String(file).endsWith(".tmp")) { + throw new Error("temporary record removal failed"); + } + return remove.apply(this, arguments); + }; + `); + fixture.nodeArgs = ["--require", guard]; + + const result = await runCli(fixture, ["merge", base, additional, "--output-dir", output, "--no-color"]); + + assert.equal(result.exitCode, 1, result.stderr); + assert.match(result.stderr, /record rename failed/); + assert.match(result.stderr, /temporary record removal failed/); + assert.ok(result.stderr.indexOf("record rename failed") < result.stderr.indexOf("temporary record removal failed"), result.stderr); + assert.equal(fs.readFileSync(record, "utf8"), "previous complete record"); + assert.match(result.stderr, /rename-temporary/); + assert.match(result.stderr, /remove-temporary/); + assert.ok(result.stderr.includes(record), result.stderr); +}); + +test("merge refuses an existing output directory", testOptions, async (t) => { + const fixture = createFrontend(t); + const base = path.join(fixture.managed, "base.transit"); + const additional = path.join(fixture.managed, "additional.transit"); + fs.writeFileSync( + base, + runFixtureText({ + runId: "base", + caseId: "case/load", + value: 10, + gitSha: "same-sha", + }), + ); + fs.writeFileSync( + additional, + runFixtureText({ + runId: "additional", + caseId: "case/load", + value: 12, + gitSha: "same-sha", + }), + ); + const output = path.join(fixture.managed, "existing-run"); + fs.mkdirSync(output); + fs.writeFileSync(path.join(output, "keep.txt"), "keep\n"); + + const result = await runCli(fixture, [ + "merge", + base, + additional, + "--output-dir", + output, + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /already exists/); + assert.equal( + fs.readFileSync(path.join(output, "keep.txt"), "utf8"), + "keep\n", + ); +}); diff --git a/frontend/test/performance_tests/render_wasm/cli_test.cljs b/frontend/test/performance_tests/render_wasm/cli_test.cljs new file mode 100644 index 0000000000..c001463051 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/cli_test.cljs @@ -0,0 +1,278 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.cli-test + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.cli.resolve :as cli] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private measurement-defaults + {:seed 42 :filter nil :warmups 3 :repetitions 10 + :timeout-ms 30000 :startup-timeout-ms 30000 :output-dir nil + :screenshots false :require-gpu false}) + +(def ^:private comparison-defaults + {:allow-build-differences false :allow-scene-differences false + :allow-all-differences false}) + +(defn- resolved + [command options] + {:command command :console {:level :info :color :auto} :options options}) + +(defn- failure + [argv] + (try + (cli/parse-resolve argv) + nil + (catch :default cause cause))) + +(t/deftest commands-resolve-complete-requests + (doseq [[argv expected] + [[["build"] + (resolved "build" {:default-features true :features [] :build-timeout-ms 300000})] + [["run"] + (resolved "run" (assoc measurement-defaults :remove false + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000}))] + [["run" "--build" "ready-build"] + (resolved "run" (assoc measurement-defaults :remove false + :build {:kind :existing :id "ready-build"}))] + [["ab" "--features" "cache" "--dry-run"] + (resolved "ab" (merge measurement-defaults comparison-defaults + {:features ["cache"] :max-features 3 :confirm false + :dry-run true :build-timeout-ms 300000}))] + [["compare" "base" "candidate"] + (resolved "compare" (assoc comparison-defaults :baseline-path "base" + :candidate-path "candidate" :case-pair nil))] + [["compare" "base" "candidate" "--allow-build-differences" + "--allow-scene-differences" "--allow-all-differences"] + (resolved "compare" {:baseline-path "base" :candidate-path "candidate" :case-pair nil + :allow-build-differences true :allow-scene-differences true + :allow-all-differences true})] + [["ab" "--features" "cache" "--filter" "rects" "--require-gpu" "--screenshots" + "--allow-build-differences" "--allow-scene-differences" "--allow-all-differences"] + (resolved "ab" (assoc measurement-defaults :filter "rects" :require-gpu true + :screenshots true :features ["cache"] :max-features 3 + :confirm false :dry-run false :build-timeout-ms 300000 + :allow-build-differences true :allow-scene-differences true + :allow-all-differences true))] + [["merge" "base" "extra"] + (resolved "merge" {:base-path "base" :additional-path "extra" :output-dir nil})] + [["merge" " ./base " " ./extra " "--output-dir" " ./merged "] + (resolved "merge" {:base-path " ./base " :additional-path " ./extra " + :output-dir " ./merged "})] + [["purge" "comparisons"] (resolved "purge" {:target :comparisons})] + [["--help"] (resolved "help" {:topic nil})]]] + (t/testing (pr-str argv) + (t/is (= expected (cli/parse-resolve argv)))))) + +(t/deftest console-flags-compose-with-every-command + (doseq [argv [["build"] ["run"] ["ab" "--features" "cache" "--dry-run"] + ["compare" "base" "candidate"] ["merge" "base" "extra"] + ["purge" "runs"] ["--help"]] + [flags console] [[["-q" "--no-color"] {:level :warn :color :never} + [["-v" "--color"] {:level :debug :color :always}]]]] + (t/is (= console (:console (cli/parse-resolve (into argv flags)))))) + (doseq [argv [["run" "-qv"] ["build" "--quiet" "--verbose"] + ["compare" "--help" "-qv"] ["--help" "-qv"]]] + (let [error (failure argv)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "mutually exclusive"))))) + +(t/deftest explicit-build-and-measurement-values-survive-resolution + (t/is (= (resolved "build" {:default-features false :features ["a" "b"] + :build-timeout-ms 1000}) + (cli/parse-resolve ["--" "build" "--no-default-features" + "--features=b,a,b" "--build-timeout-ms" "1e3"]))) + (t/is (= {:command "run" :console {:level :info :color :never} + :options {:build {:kind :prepare :default-features false + :features ["a" "b"] :build-timeout-ms 5000} + :seed 0 :filter " rects " :warmups 0 :repetitions 2 + :timeout-ms 1000 :startup-timeout-ms 2000 :output-dir " ./runs " + :screenshots true :remove true :require-gpu true}} + (cli/parse-resolve + ["run" "--no-default-features" "--features" " b,a,b " + "--build-timeout-ms" "5000" "--seed" "0" "--filter" " rects " + "--warmups" "0" "--repetitions" "2" "--timeout-ms" "1000" + "--startup-timeout-ms" "2000" "--output-dir" " ./runs " + "--screenshots" "--remove" "--require-gpu" "--no-color"])))) + +(t/deftest retained-build-conflicts-use-supplied-flags + (doseq [flags [["--features" "cache"] ["--default-features"] + ["--no-default-features"] ["--build-timeout-ms=300000"]] + argv [(into ["run" "--build" "ready"] flags) + (into (into ["run"] flags) ["--build=ready"])]] + (t/is (str/includes? (ex-message (failure argv)) "--build cannot be combined"))) + (t/is (= (resolved "run" (assoc measurement-defaults :remove false :warmups 0 + :build {:kind :existing :id "ready"})) + (cli/parse-resolve ["run" "--build" "ready" "--warmups" "0"])))) + +(t/deftest parsers-reject-options-owned-by-other-commands + (doseq [[argv flags] + [[["build"] ["--warmups" "--cases" "--confirm"]] + [["run"] ["--cases" "--dry-run" "--allow-build-differences"]] + [["ab" "--features" "cache" "--dry-run"] + ["--build" "--remove" "--default-features" "--no-default-features" "--cases"]] + [["compare" "base" "candidate"] ["--output-dir" "--seed" "--features"]] + [["merge" "base" "extra"] ["--warmups" "--cases" "--allow-all-differences"]] + [["purge" "runs"] ["--features" "--seed" "--only-runs"]]] + flag flags] + (doseq [args [(conj argv flag) (conj argv "--help" flag)]] + (let [error (failure args)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "Unknown option")))))) + +(t/deftest command-first-syntax-and-positional-arguments-are-checked + (doseq [argv [[] ["unknown" "--help"] ["--help" "run"] ["-q" "build"] + ["build" "extra"] ["run" "extra"] + ["compare"] ["compare" "one"] ["compare" "one" "two" "three"] + ["compare" "" "candidate"] ["merge" "base"] ["merge" "base" " "] + ["run" "--build" "../escape"] ["run" "--output-dir" " "]]] + (t/testing (pr-str argv) + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv))))))) + (doseq [target ["builds" "runs" "comparisons" "all"]] + (t/is (= (resolved "purge" {:target (keyword target)}) + (cli/parse-resolve ["purge" target])))) + (doseq [args [[] ["unknown"] ["runs" "builds"]]] + (let [error (failure (into ["purge"] args))] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "runner purge builds|runs|comparisons|all"))))) + +(t/deftest argument-errors-include-command-and-validation-details + (let [data (ex-data (failure ["unknown" "--help"]))] + (t/is (= "unknown" (:command data))) + (t/is (= "runner COMMAND [options]" (:usage data)))) + (doseq [[argv path] [[["compare" "" "candidate"] [:options :baseline-path]] + [["merge" "base" " "] [:options :additional-path]]]] + (let [data (ex-data (failure argv))] + (t/is (= ::cli/invalid-args (:type data))) + (t/is (= ::request/invalid-request (:cause-type data))) + (t/is (some #(= path (:in %)) (:errors (::sm/explain data))))))) + +(t/deftest literal-paths-selectors-and-option-looking-values-are-preserved + (t/is (= (resolved "compare" (assoc comparison-defaults :baseline-path " ./base " + :candidate-path " ./candidate " + :case-pair [" rects/load " "other/load"])) + (cli/parse-resolve ["compare" " ./base " " ./candidate " + "--cases" " rects/load ,other/load"]))) + (t/is (= (resolved "compare" (assoc comparison-defaults :baseline-path "--seed" + :candidate-path "-candidate" :case-pair nil)) + (cli/parse-resolve ["compare" "--" "--seed" "-candidate"]))) + (doseq [text ["" " " "--allow-all-differences" "--" "-h"]] + (t/is (= (resolved "run" (assoc measurement-defaults :filter text :remove false + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000})) + (cli/parse-resolve ["run" "--filter" text]))))) + +(t/deftest malformed-selector-pairs-fail-before-record-loading + (doseq [text ["" "one" "one,two,three" ",two" "one," " ,two" "one, "]] + (let [error (failure ["compare" "base" "candidate" (str "--cases=" text)])] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "--cases")))) + (t/is (= ::cli/invalid-args + (:type (ex-data (failure ["compare" "base" "candidate" "--cases"])))))) + +(t/deftest numeric-values-use-integer-and-timer-bounds + (doseq [[flag values] + [["--seed" ["0" "4294967295"]] + ["--warmups" ["0" "9007199254740991"]] + ["--repetitions" ["1" "9007199254740991"]] + ["--timeout-ms" ["1" "2147483647"]] + ["--startup-timeout-ms" ["1" "2147483647"]] + ["--build-timeout-ms" ["1" " 1000 " "1e3" "0x3e8" "0o1750" "0b1111101000" "2147483647"]]] + value values] + (t/is (= (js/Number value) + (let [options (:options (cli/parse-resolve ["run" flag value]))] + (if (= flag "--build-timeout-ms") + (get-in options [:build :build-timeout-ms]) + (get options (keyword (subs flag 2)))))))) + (doseq [[flag values] + [["--seed" ["-1" "4294967296" "1.5"]] + ["--warmups" ["-1" "9007199254740992" "0.5"]] + ["--repetitions" ["0" "9007199254740992" "1.5"]] + ["--timeout-ms" ["0" "2147483648" "1.5"]] + ["--startup-timeout-ms" ["0" "2147483648"]] + ["--build-timeout-ms" ["0" "2147483648" "1.5" "1000ms"]]] + value (concat values ["" " " "NaN" "Infinity"])] + (t/testing (str flag " " value) + (t/is (= ::cli/invalid-args (:type (ex-data (failure ["run" flag value])))))))) + +(t/deftest feature-selection-and-ab-policy-use-domain-rules + (doseq [command ["build" "run" "ab"] + [feature message] [["stats" "diagnostic Cargo features"] + ["profile" "diagnostic Cargo features"] + ["profile-macros" "diagnostic Cargo features"] + ["profile-raf" "diagnostic Cargo features"] + ["" "invalid Cargo feature selection"] + ["a,,b" "invalid Cargo feature selection"] + ["a/b" "invalid Cargo feature selection"]]] + (let [argv (cond-> [command "--features" feature] (= command "ab") (conj "--dry-run")) + error (failure argv)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) message)))) + (doseq [argv [["ab"] ["ab" "--features" "cache"] + ["ab" "--features" "cache" "--filter" " "] + ["ab" "--features" "a,b,c,d" "--dry-run"] + ["ab" "--features" "a,b,c,d" "--max-features" "4" "--filter" "rects"]]] + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv)))))) + (doseq [limit ["" " " "0" "7" "1.5" "NaN" "Infinity" "3x"]] + (t/is (= ::cli/invalid-args + (:type (ex-data (failure ["ab" "--features" "cache" "--dry-run" + "--max-features" limit])))))) + (doseq [flags [["--confirm"] ["--dry-run"] ["--filter" "rects"]]] + (t/is (= ["a" "b" "c"] + (get-in (cli/parse-resolve (into ["ab" "--features" "c,b,a,c"] flags)) + [:options :features])))) + (let [error (failure ["ab" "--features" "cache"])] + (t/is (str/includes? (ex-message (ex-cause error)) "--confirm")))) + +(t/deftest help-skips-required-command-arguments-and-keeps-syntax-checks + (doseq [command ["build" "run" "ab" "compare" "merge" "purge"]] + (t/is (= (resolved "help" {:topic command}) + (cli/parse-resolve [command "--help"])))) + (t/is (= (assoc (resolved "help" {:topic "compare"}) + :console {:level :debug :color :never}) + (cli/parse-resolve ["compare" "--help" "--verbose" "--no-color"]))) + (doseq [argv [["--help" "--unknown"] ["run" "--help" "--warmups" "bad"] + ["compare" "--help" "--cases="] ["ab" "--help" "--features"]]] + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv))))))) + +(t/deftest help-shows-command-options-and-schema-defaults + (doseq [[topic flags] + [["build" #{"--[no-]default-features" "--features" "--build-timeout-ms"}] + ["run" #{"--build" "--[no-]default-features" "--features" "--build-timeout-ms" + "--seed" "--filter" "--warmups" "--repetitions" "--timeout-ms" + "--startup-timeout-ms" "--output-dir" "--screenshots" "--require-gpu" "--remove"}] + ["ab" #{"--features" "--max-features" "--confirm" "--dry-run" "--build-timeout-ms" + "--seed" "--filter" "--warmups" "--repetitions" "--timeout-ms" + "--startup-timeout-ms" "--output-dir" "--screenshots" "--require-gpu" + "--allow-build-differences" "--allow-scene-differences" "--allow-all-differences"}] + ["compare" #{"--cases" "--allow-build-differences" + "--allow-scene-differences" "--allow-all-differences"}] + ["merge" #{"--output-dir"}] + ["purge" #{}]]] + (let [text (cli/get-help topic)] + (t/is (str/includes? text (str "Usage: runner " topic))) + (t/is (= (into #{"--help" "--quiet" "--verbose" "--[no-]color"} flags) + (set (re-seq #"--(?:\[no-\])?[a-z][a-z-]*" text)))))) + (t/is (= ["build" "run" "ab" "compare" "merge" "purge"] + (mapv second (re-seq #"(?m)^ runner ([a-z]+)" (cli/get-help nil))))) + (doseq [[topic pattern] [["build" #"--build-timeout-ms MS\s+300000"] + ["run" #"--warmups N\s+3"] + ["run" #"--seed N\s+42"] + ["run" #"--repetitions N\s+10"] + ["run" #"--timeout-ms MS\s+30000\s+Case deadline"] + ["ab" #"--timeout-ms MS\s+30000\s+Case deadline"] + ["run" #"--features FEATURES\s+\[\]"] + ["run" #"--\[no-\]default-features\s+true"] + ["run" #"--build-timeout-ms MS\s+300000"] + ["ab" #"--max-features N\s+3"] + ["merge" #"--output-dir DIR\s+nil"] + ["compare" #"--cases A,B\s+nil"]]] + (t/is (re-find pattern (cli/get-help topic))))) diff --git a/frontend/test/performance_tests/render_wasm/commands_test.cljs b/frontend/test/performance_tests/render_wasm/commands_test.cljs new file mode 100644 index 0000000000..a2b60587bd --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/commands_test.cljs @@ -0,0 +1,520 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.commands-test + (:require + [app.common.render-wasm.wasm :as wasm] + [app.common.schema :as sm] + [benches.render-wasm.builder :as builder] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.runtime.camera :as camera] + [benches.render-wasm.runtime.commands :as cmd :refer [defcommand]] + [cljs.test :as t])) + +(defcommand move + "Adds the supplied offsets to the command context's view." + [rtx {:keys [dx dy]}] + (-> rtx + (update-in [:view :x] + dx) + (update-in [:view :y] + dy))) + +(defcommand selected-renders + {:measured-variant false} + [rtx] + (-> rtx + (camera/render-sync*) + (move {:dx 10 :dy 5}) + (camera/render-sync) + (camera/render-sync*))) + +(defcommand unexpected-promise + [rtx cause] + (if cause + (js/Promise.reject cause) + (js/Promise.resolve rtx))) + +(defcommand reject-offset + [rtx offset] + {:pre [(pos? offset)]} + ((get-in rtx [:hooks :render-sync])) + (update-in rtx [:view :x] + offset)) + +(defcommand replace-owner + [rtx] + (assoc-in rtx [:hooks :check] (fn []))) + +(defcommand async-work + {:async [complete fail]} + [rtx work] + (work rtx complete fail)) + +(defcommand selected-async + {:async [complete fail] :measured-variant false} + [rtx] + (-> (camera/render-sync* rtx) + (move {:dx 10 :dy 5}) + (camera/render-sync*) + (.then complete) + (.catch fail))) + +(defn- context + [clock] + {:case {:id :commands/example} + :scene (builder/scene {:seed 1}) + :view {:scale 1 :x 0 :y 0} + :hooks {:now #(deref clock) + :check (fn []) + :frame #(js/Promise.resolve (vswap! clock + 16)) + :sleep #(do (vswap! clock + %) (js/Promise.resolve nil)) + :render (fn [_ _] (vswap! clock + 7) wasm/FRAME_TYPE_FULL) + :render-sync #(vswap! clock + 7) + :render-from-cache #(vswap! clock + 2) + :set-view (fn [_]) + :set-view-start (fn []) + :set-view-end #(vswap! clock + 3)}}) + +(t/deftest ^:async measurement-uses-core-time + (let [clock (volatile! 0) + rtx (assoc-in (context clock) [:hooks :render-sync] + #(do + (vswap! clock + 7) + (js/queueMicrotask (fn [] (vswap! clock + 500))))) + {:keys [outcome events context]} + (await + (cmd/execute + rtx + (fn [rtx] + (let [input (-> (js/Promise.resolve rtx) + (.then (fn [rtx] + (vswap! clock + 1000) + rtx)))] + (-> (camera/render-sync* input) + (.then (fn [rtx] + (vswap! clock + 2000) + rtx)))))))] + (t/is (= :completed outcome)) + (t/is (= (:view rtx) (:view context))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + events)) + (t/is (= 3507 @clock)))) + +(t/deftest ^:async composite-preserves-view-and-selected-observations + (doseq [promised [false true]] + (let [clock (volatile! 0) + cost (volatile! 0) + rtx (assoc-in (context clock) [:hooks :render-sync] + #(vswap! clock + (vswap! cost + 7))) + {:keys [outcome context events]} + (await (cmd/execute rtx #(selected-renders + (if promised (js/Promise.resolve %) %))))] + (t/is (= :completed outcome)) + (t/is (= {:scale 1 :x 10 :y 5} (:view context))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}} + {:command ::camera/render-sync + :metrics {:command/elapsed [21]}}] + events)) + (t/is (= 42 @clock))))) + +(t/deftest measured-command-documentation + (t/is (.startsWith (:doc (meta #'camera/render-sync*)) + "Measured variant of `render-sync`.")) + (t/is (= "Measured variant of `unexpected-promise`." + (:doc (meta #'unexpected-promise*))))) + +(t/deftest ^:async zero-duration-is-a-raw-observation + (let [rtx (context (volatile! 0)) + result (await (cmd/execute rtx #(move* % {:dx 2 :dy 3})))] + (t/is (= :completed (:outcome result))) + (t/is (= {:scale 1 :x 2 :y 3} (get-in result [:context :view]))) + (t/is (= [{:command ::move :metrics {:command/elapsed [0]}}] + (:events result))))) + +(t/deftest ^:async async-completion-keeps-endpoints-and-raw-evidence + (doseq [command [async-work async-work*] + supplied [false true]] + (let [clock (volatile! 0) + rtx (context clock) + metrics {:renderer/call-elapsed [1 2 4]} + trace {:steps [1 2 3]} + result (await + (cmd/execute + rtx + (fn [rtx] + (command + (-> (js/Promise.resolve rtx) + (.then #(do (vswap! clock + 1000) %))) + (fn [rtx complete _fail] + (vswap! clock + 7) + (let [end @clock + evidence {:metrics metrics :trace trace} + output (assoc-in rtx [:view :x] 10)] + (if supplied + (-> (js/Promise.resolve nil) + (.then (fn [_] + (vswap! clock + 500) + (complete output (assoc evidence :end-ms end))))) + (do + (complete output evidence) + (js/queueMicrotask #(vswap! clock + 500))))))))))] + (t/is (= :completed (:outcome result))) + (t/is (= 10 (get-in result [:context :view :x]))) + (t/is (= 1507 @clock)) + (t/is (= (if (= command async-work*) + [{:command ::async-work + :metrics (assoc metrics :command/elapsed [7]) + :trace trace}] + []) + (:events result))))) + (let [result (await (cmd/execute (context (volatile! 0)) selected-async))] + (t/is (= :completed (:outcome result))) + (t/is (= {:scale 1 :x 10 :y 5} (get-in result [:context :view]))) + (t/is (= [::camera/render-sync ::camera/render-sync] + (mapv :command (:events result)))))) + +(t/deftest ^:async invalid-trace-stops-the-command-and-retains-earlier-measurements + (doseq [trace [{:handle (fn [] nil)} (repeat 0) (nth (iterate vector 0) 40)]] + (let [run (fn [rtx] + (async-work* rtx (fn [rtx complete _] (complete rtx {:trace trace})))) + result (await (cmd/execute (context (volatile! 0)) + (fn [rtx] (.then (camera/render-sync* rtx) run))))] + (t/is (= :failed (:outcome result))) + (t/is (= "command trace cannot cross Transit" (errors/message-of (:error result)))) + (t/is (= [::camera/render-sync] (mapv :command (:events result))))))) + +(t/deftest ^:async async-completion-retains-invalid-numeric-observations + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (complete rtx {:end-ms js/Infinity + :metrics {:sample/raw [-1 0 js/NaN js/Infinity]}}))))) + values (get-in result [:events 0 :metrics :sample/raw])] + (t/is (= :completed (:outcome result))) + (t/is (= [js/Infinity] (get-in result [:events 0 :metrics :command/elapsed]))) + (t/is (= [-1 0] (subvec values 0 2))) + (t/is (js/Number.isNaN (nth values 2))) + (t/is (= js/Infinity (nth values 3))))) + +(t/deftest ^:async async-commands-use-the-first-terminal-signal + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete fail] + (complete (assoc-in rtx [:view :x] 10)) + (complete nil) + (fail (js/Error. "late failure")) + (throw (js/Error. "late throw"))))))] + (t/is (= :completed (:outcome result))) + (t/is (= 10 (get-in result [:context :view :x]))) + (t/is (= [{:command ::async-work :metrics {:command/elapsed [0]}}] (:events result))) + (t/is (= [:complete :fail :fail] (mapv :signal (:diagnostics result))))) + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (js/queueMicrotask (fn [] (complete (assoc-in rtx [:view :x] 20)))) + rtx))))] + (t/is (= :completed (:outcome result))) + (t/is (= 20 (get-in result [:context :view :x])))) + (let [unhandled (atom []) + observe #(swap! unhandled conj %)] + (.on js/process "unhandledRejection" observe) + (try + (let [result (await + (cmd/execute + (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (complete rtx) + (js/Promise.reject (js/Error. "rejected after completion"))))))] + (await (js/Promise. #(js/setImmediate %))) + (t/is (= :completed (:outcome result))) + (t/is (= [{:command ::async-work :metrics {:command/elapsed [0]}}] (:events result))) + (t/is (= [] @unhandled))) + (finally + (.removeListener js/process "unhandledRejection" observe))))) + +(t/deftest ^:async async-failures-retain-causes-and-earlier-events + (doseq [mode [:throw :reject :fail]] + (let [cause (ex-info "renderer failed" {:partial {:slices [{:duration-ms 3}]}}) + result (await + (cmd/execute + (context (volatile! 0)) + #(-> % + (camera/render-sync*) + (async-work* (fn [rtx complete fail] + (case mode + :throw (throw cause) + :reject (js/Promise.reject cause) + :fail (do (fail cause) (complete rtx))))))))] + (t/is (= :failed (:outcome result))) + (t/is (identical? cause (:error result))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [7]}}] + (:events result)))))) + +(t/deftest ^:async malformed-async-completion-fails-before-recording + (doseq [[output evidence expected] + [[identity {:end-ms nil} ::cmd/invalid-completion] + [identity {:metrics {:sample/raw []}} ::cmd/invalid-completion] + [identity {:metrics {:sample/raw ["bad"]}} ::cmd/invalid-completion] + [identity {:metrics {:command/elapsed [123]}} ::cmd/invalid-completion] + [#(assoc-in % [:view :scale] 0) {} ::cmd/invalid-context] + [#(assoc-in % [:hooks :check] (fn [])) {} ::cmd/ownership-lost]]] + (let [result (await (cmd/execute (context (volatile! 0)) + #(async-work* % (fn [rtx complete _fail] + (complete (output rtx) evidence)))))] + (t/is (= :failed (:outcome result))) + (t/is (= expected (:type (ex-data (:error result))))) + (t/is (= [] (:events result)))))) + +(t/deftest ^:async async-completion-checks-the-owner-guard + (doseq [cause [(ex-info "deadline expired" {:phase :deadline}) + (ex-info "owner superseded" {::errors/stale true})]] + (let [blocked (volatile! nil) + rtx (assoc-in (context (volatile! 0)) [:hooks :check] + #(when @blocked (throw @blocked))) + result (await (cmd/execute rtx #(async-work* % (fn [rtx complete _fail] + (vreset! blocked cause) + (complete rtx)))))] + (t/is (= :failed (:outcome result))) + (t/is (identical? cause (:error result))) + (t/is (= [] (:events result)))))) + +(t/deftest ^:async late-completion-cannot-record-in-another-execution + (let [callbacks (volatile! nil) + pending (volatile! nil) + first-run (await + (cmd/execute + (context (volatile! 0)) + (fn [rtx] + (vreset! pending + (.catch (async-work* rtx (fn [rtx complete fail] + (vreset! callbacks [rtx complete fail]))) + errors/as-failure)) + rtx))) + [rtx complete fail] @callbacks + next-run (await (cmd/execute (:context first-run) + (fn [next-rtx] + (complete rtx) + (fail (js/Error. "late failure")) + (camera/render-sync* next-rtx)))) + cause (await @pending)] + (t/is (= :completed (:outcome first-run))) + (t/is (= [] (:events first-run))) + (t/is (= ::cmd/inactive-execution (:type (ex-data cause)))) + (t/is (= :completed (:outcome next-run))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [7]}}] + (:events next-run))))) + +(t/deftest ^:async camera-gestures-return-the-view-and-renderer-evidence + (doseq [[command options target command-id measured] + [[camera/pan {:dx 10 :dy 5 :steps 2 :settle-ms 5} {:scale 1 :x 10 :y 5} ::camera/pan false] + [camera/pan* {:dx 10 :dy 5 :steps 2 :settle-ms 5} {:scale 1 :x 10 :y 5} ::camera/pan true] + [camera/zoom {:factor 2 :steps 2 :settle-ms 5} {:scale 2 :x 0 :y 0} ::camera/zoom false] + [camera/zoom* {:factor 2 :steps 2 :settle-ms 5} {:scale 2 :x 0 :y 0} ::camera/zoom true]]] + (let [clock (volatile! 0) + view (volatile! nil) + remaining (volatile! [wasm/FRAME_TYPE_PARTIAL wasm/FRAME_TYPE_FULL]) + rtx (update (context clock) :hooks merge + {:set-view #(vreset! view %) + :render (fn [_ _] + (vswap! clock + 7) + (let [frame-type (first @remaining)] + (vswap! remaining rest) + (when (= wasm/FRAME_TYPE_FULL frame-type) + (js/queueMicrotask #(vswap! clock + 500))) + frame-type))}) + result (await (cmd/execute rtx #(command % options)))] + (t/is (= :completed (:outcome result))) + (t/is (= target @view (get-in result [:context :view]))) + (t/is (= 574 @clock)) + (if measured + (do + (t/is (= command-id (get-in result [:events 0 :command]))) + (t/is (= {:command/elapsed [74] + :renderer/call-elapsed [7 7] + :renderer/cache-call-elapsed [2 2] + :camera/finalization-to-full [33]} + (get-in result [:events 0 :metrics]))) + (t/is (= 74 (get-in result [:events 0 :trace :end-ms])))) + (t/is (= [] (:events result)))) + (vreset! clock 0) + (let [invalid (await (cmd/execute rtx #(command % (assoc options :steps 0))))] + (t/is (= :failed (:outcome invalid))) + (t/is (= ::cmd/invalid-arguments (:type (ex-data (:error invalid))))) + (t/is (= [] (:events invalid))) + (t/is (zero? @clock)))))) + +(t/deftest ^:async load-and-restore-finish-at-the-renderer-endpoint + (doseq [[command restore measured] [[camera/load false false] [camera/load* false true] + [camera/restore true false] [camera/restore* true true]]] + (let [clock (volatile! 0) + view (volatile! nil) + target {:scale 2 :x 30 :y -10} + rtx (update (context clock) :hooks merge + {:set-view #(vreset! view %) + :render (fn [_ _] + (vswap! clock + 7) + (js/queueMicrotask #(vswap! clock + 500)) + wasm/FRAME_TYPE_FULL)}) + result (await (cmd/execute rtx #(if restore + (command (js/Promise.resolve %) target) + (command (js/Promise.resolve %)))))] + (t/is (= :completed (:outcome result))) + (t/is (= (if restore target (:view rtx)) (get-in result [:context :view]))) + (when restore (t/is (= target @view))) + (t/is (= (if restore 510 523) @clock)) + (when measured + (let [event (first (:events result))] + (t/is (= (if restore ::camera/restore ::camera/load) (:command event))) + (t/is (= [(if restore 10 23)] (get-in event [:metrics :command/elapsed]))) + (t/is (= [7] (get-in event [:metrics :renderer/call-elapsed]))) + (t/is (= (if restore 10 23) (get-in event [:trace :end-ms]))) + (when-not restore + (t/is (= [23] (get-in event [:metrics :renderer/prepared-scene-to-full]))))))))) + +(t/deftest ^:async invalid-camera-targets-fail-before-changing-renderer-state + (doseq [[command options view] + [[camera/pan* {:dx sm/max-safe-int :dy 0 :steps 1 :settle-ms 0} {:scale 1 :x 1 :y 0}] + [camera/zoom* {:factor sm/max-safe-int :steps 1 :settle-ms 0} {:scale 2 :x 0 :y 0}] + [camera/restore* {:scale 1 :x 10 :y 0 :viewport {:width 800}} {:scale 1 :x 0 :y 0}]]] + (let [clock (volatile! 0) + applied (volatile! view) + rtx (-> (context clock) + (assoc :view view) + (assoc-in [:hooks :set-view] #(vreset! applied %))) + result (await (cmd/execute rtx #(command % options)))] + (t/is (= :failed (:outcome result))) + (t/is (= ::cmd/invalid-arguments (:type (ex-data (:error result))))) + (t/is (= view @applied)) + (t/is (zero? @clock))))) + +(t/deftest ^:async declared-reference-cases-execute-selected-work-and-reset-the-view + (let [clock (volatile! 0) + descriptor (first (cases/collect-cases {:master-seed 42 :filter "rects/blocking-full"})) + result (await (cmd/execute (assoc (context clock) :case descriptor) + (:run! (decl/registered-case (:id descriptor)))))] + (t/is (= :completed (:outcome result))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [7]}}] (:events result)))) + (let [clock (volatile! 0) + descriptor (first (cases/collect-cases {:master-seed 42 :filter "rects/mixed-pans"})) + registered (decl/registered-case (:id descriptor)) + applied (volatile! (:view descriptor)) + initial (-> (context clock) (assoc :case descriptor) + (assoc-in [:hooks :set-view] #(vreset! applied %)))] + (loop [rtx initial remaining 2] + (when (pos? remaining) + (let [main (await (cmd/execute rtx (:run! registered))) + _ (t/is (= {:scale 1 :x 0 :y 80} @applied (get-in main [:context :view]))) + reset (await (cmd/execute (:context main) (:reset registered)))] + (t/is (= :completed (:outcome main) (:outcome reset))) + (t/is (= [::camera/pan ::camera/pan] (mapv :command (:events main)))) + (t/is (= [::camera/restore] (mapv :command (:events reset)))) + (t/is (= (:view descriptor) @applied (get-in reset [:context :view]))) + (recur (:context reset) (dec remaining))))))) + +(t/deftest ^:async small-positive-zoom-scales-stay-positive-and-reach-the-target + (doseq [[scale factor] [[1 1e-20] [js/Number.MIN_VALUE 1]]] + (let [applied (volatile! {:scale scale :x 0 :y 0}) + minimum (volatile! scale) + rtx (-> (context (volatile! 0)) + (assoc :view @applied) + (assoc-in [:hooks :set-view] + #(do (vreset! applied %) + (vswap! minimum min (:scale %))))) + result (await (cmd/execute rtx #(camera/zoom* % {:factor factor :steps 2 :settle-ms 0})))] + (t/is (= :completed (:outcome result))) + (t/is (pos? @minimum)) + (t/is (= (* scale factor) (:scale @applied) (get-in result [:context :view :scale])))))) + +(t/deftest ^:async failed-work-retains-earlier-observations + (let [unhandled (atom []) + observe #(swap! unhandled conj %)] + (.on js/process "unhandledRejection" observe) + (try + (doseq [cause [nil (js/Error. "rejected core")]] + (let [rtx (context (volatile! 0)) + result (await (cmd/execute rtx #(-> % + (camera/render-sync*) + (unexpected-promise* cause))))] + (t/is (= :failed (:outcome result))) + (t/is (= ::cmd/async-result (:type (ex-data (:error result))))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + (:events result))))) + (await (js/Promise. #(js/setImmediate %))) + (t/is (= [] @unhandled)) + (finally + (.removeListener js/process "unhandledRejection" observe))))) + +(t/deftest ^:async execution-failures-preserve-causes-and-earlier-observations + (doseq [cause [(js/Error. "renderer failed") + "renderer disconnected" + {:reason :renderer-disconnected} + (errors/fatal-error "recording failed" {} (js/Error. "storage"))]] + (let [clock (volatile! 0) + rtx (assoc-in (context clock) [:hooks :render-sync] + #(if (zero? @clock) + (vswap! clock + 7) + (throw cause))) + result (await (cmd/execute rtx #(-> % + (camera/render-sync*) + (camera/render-sync*)))) + error (:error result)] + (t/is (= :failed (:outcome result))) + (t/is (errors/failure? error)) + (if (errors/failure? cause) + (t/is (identical? cause error)) + (t/is (= cause (:thrown (ex-data error))))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + (:events result)))))) + +(t/deftest ^:async invalid-boundaries-fail-the-execution + (doseq [[input command expected] + [[#(assoc-in % [:view :scale] 0) camera/render-sync* ::cmd/invalid-context] + [#(assoc % :module #js {}) camera/render-sync* ::cmd/invalid-context] + [#(update % :hooks dissoc :render-sync) camera/render-sync* ::cmd/invalid-context] + [#(update % :hooks dissoc :frame) camera/render-sync* ::cmd/invalid-context] + [#(assoc-in % [:hooks :sleep] 0) camera/render-sync* ::cmd/invalid-context] + [identity #(reject-offset* % -1) ::cmd/invalid-arguments]]] + (let [clock (volatile! 0) + now-reads (volatile! 0) + rtx (assoc-in (context clock) [:hooks :now] + #(do (vswap! now-reads inc) @clock)) + result (await (cmd/execute rtx #(command (input %))))] + (t/is (= :failed (:outcome result))) + (t/is (= expected (:type (ex-data (:error result))))) + (t/is (= [] (:events result))) + (t/is (zero? @now-reads)) + (t/is (zero? @clock))))) + +(t/deftest ^:async commands-enforce-execution-ownership + (let [rtx (context (volatile! 0)) + result (await (cmd/execute rtx replace-owner*)) + cause (await (.then (camera/render-sync* rtx) (fn [_] nil) identity))] + (t/is (= :failed (:outcome result))) + (t/is (= ::cmd/ownership-lost (:type (ex-data (:error result))))) + (t/is (= [] (:events result))) + (t/is (= ::cmd/missing-execution (:type (ex-data cause)))))) + +(t/deftest ^:async completed-context-starts-a-new-execution + (let [first-result (await (cmd/execute (context (volatile! 0)) + #(move* % {:dx 10 :dy 5}))) + rtx (:context first-result) + late-error (await (.then (camera/render-sync* rtx) + (fn [_] nil) + identity)) + next-result (await (cmd/execute rtx camera/render-sync*))] + (t/is (= ::cmd/inactive-execution (:type (ex-data late-error)))) + (t/is (= :completed (:outcome next-result))) + (t/is (= {:scale 1 :x 10 :y 5} (get-in next-result [:context :view]))) + (t/is (= [{:command ::camera/render-sync + :metrics {:command/elapsed [7]}}] + (:events next-result))))) diff --git a/frontend/test/performance_tests/render_wasm/compare_test.cljs b/frontend/test/performance_tests/render_wasm/compare_test.cljs new file mode 100644 index 0000000000..7809f03713 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/compare_test.cljs @@ -0,0 +1,278 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.compare-test + (:require + ["fs" :as fs] + ["node:stream" :as stream] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.report.compare :as compare] + [benches.render-wasm.report.compare :as report-compare] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-compare-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- metric + [value] + {:command :fixture/work :metrics {:command/elapsed [value]}}) + +(defn- case-definition + [case-id scene] + {:id case-id + :scene scene + :scene-description "comparison scene" + :scene-seed 7 + :params {:count 10} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {}}) + +(defn- make-run + [run-id case-id value & {:keys [environment functional fingerprint scene case-settings] + :or {environment {:node "v24" :browser "Fake"} + functional {:mode :release} + fingerprint (apply str (repeat 64 "a")) + scene :case}}] + (let [preparation-id (str run-id "-preparation") + run (result/create-run + {:run-id run-id + :started-at "2026-10-26T00:00:00.000Z" + :plan {:cases [(merge (case-definition case-id scene) case-settings)] + :warmups 0 + :repetitions 1 + :timeout-policy {:case-ms 30000}} + :metadata {:git {:sha run-id :dirty false} + :build {:functional functional} + :environment environment + :scored? true}}) + run (result/record-preparation + run + {:preparation-id preparation-id + :case-id case-id :warmup? false + :fingerprint {:algorithm :sha-256 + :digest fingerprint} + :effective-graphics {:webgl2 true} + :diagnostics {} + :phases [{:phase :preparation :preparation-id preparation-id + :outcome :completed :events []}]}) + run (result/record-attempt + run + {:case-id case-id + :preparation-id preparation-id + :warmup? false + :outcome :completed + :phases [{:phase :main :preparation-id preparation-id + :outcome :completed :events [(metric value)]}]})] + (-> run + (result/record-case-outcome case-id {:outcome :completed}) + (result/finish-run "2026-10-26T00:01:00.000Z" {:reason :completed})))) + +(defn- write-run + [file run] + (fs/writeFileSync file (result/encode run) "utf8") + file) + +(defn- failure + [f] + (try + (f) + nil + (catch :default cause cause))) + +(t/deftest compares-saved-records-from-the-caller-cwd + (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12)) + (let [previous (.cwd js/process)] + (try + (.chdir js/process @temporary-root) + (let [{:keys [comparison text]} (compare/execute "baseline.transit" + "candidate.transit" {})] + (t/is (= :comparison (:type comparison))) + (t/is (= {:allow-build-differences false + :allow-scene-differences false + :allow-all-differences false} + (:permissions comparison))) + (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (= 2 (get-in comparison [:pairs 0 :metrics [:main :fixture/work :command/elapsed] :absolute :value]))) + (t/is (str/includes? text "Baseline")) + (t/is (str/includes? text "Candidate")) + (t/is (str/includes? text "case/load")) + (t/is (str/includes? text "median change"))) + (finally + (.chdir js/process previous))))) + +(t/deftest cases-selects-one-explicit-baseline-candidate-pair + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :a/pan 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :b/pan 12)) + {:keys [text]} (compare/execute baseline candidate + {:case-pair ["a/pan" "b/pan"]})] + (t/is (str/includes? text "Pair a/pan,b/pan")))) + +(t/deftest cases-requires-scene-permission-for-different-scenes + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :a/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :b/load 12 :scene :other + :fingerprint (apply str (repeat 64 "b")))) + selector {:case-pair ["a/load" "b/load"]}] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate selector)))))) + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate + (assoc selector :allow-build-differences true))))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate (assoc selector :allow-scene-differences true))] + (t/is (= {:allow-build-differences false + :allow-scene-differences true + :allow-all-differences false} + (:permissions comparison))) + (t/is (= #{:scene :fingerprint} + (set (map :field (get-in comparison [:pairs 0 :mismatches]))))) + (t/is (every? #(and (:waived? %) (= :scene (:rule %))) + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) + (t/is (str/includes? text "Pair a/load,b/load"))))) + +(t/deftest build-permission-allows-build-differences-only + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :functional {:mode :debug :features ["fixture"]}))] + (doseq [options [{} {:allow-scene-differences true}]] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate options))))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate {:allow-build-differences true})] + (t/is (= {:allow-build-differences true + :allow-scene-differences false + :allow-all-differences false} + (:permissions comparison))) + (t/is (= #{:functional-build :effective-features} + (set (map :field (get-in comparison [:pairs 0 :mismatches]))))) + (t/is (every? #(and (:waived? %) (= :build (:rule %))) + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? text ":rule :build"))))) + +(t/deftest build-and-scene-permissions-combine + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 :scene :other + :functional {:mode :debug}))] + (doseq [options [{:allow-build-differences true} {:allow-scene-differences true}]] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate options))))))) + (let [{:keys [comparison]} + (compare/execute baseline candidate {:allow-build-differences true + :allow-scene-differences true})] + (t/is (= {:allow-build-differences true + :allow-scene-differences true + :allow-all-differences false} + (:permissions comparison))) + (t/is (= {:scene :scene :functional-build :build} + (into {} (map (juxt :field :rule)) + (get-in comparison [:pairs 0 :mismatches]))))))) + +(t/deftest comparison-failure-identifies-incompatible-fields + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :environment {:node "v25" :browser "Fake"})) + cause (failure #(compare/execute baseline candidate {})) + text (atom "") + stderr (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))})] + (try + (t/is (some? cause)) + (t/is (= 1 (execution/fail cause {:color :never} {:stderr stderr}))) + (t/is (str/includes? @text "incompatible saved runs")) + (t/is (str/includes? @text ":environment")) + (finally + (log/configure {}))))) + +(t/deftest scoped-permissions-keep-view-and-runtime-checks-strict + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10))] + (doseq [[field settings] + [[:execution {:case-settings {:view {:scale 2 :x 0 :y 0}}}] + [:environment {:environment {:node "v25" :browser "Fake"}}]]] + (let [candidate (write-run (path/join @temporary-root "candidate.transit") + (apply make-run "candidate" :case/load 12 (mapcat identity settings))) + cause (failure #(compare/execute baseline candidate + {:allow-build-differences true + :allow-scene-differences true}))] + (t/is (= ::report-compare/incompatible (:type (ex-data cause)))) + (t/is (= [field] (mapv :field (:mismatches (ex-data cause))))) + (t/is (every? (comp not :waived?) (:mismatches (ex-data cause)))))))) + +(t/deftest all-permission-allows-runtime-differences-and-displays-them + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :environment {:node "v25" :browser "Fake"}))] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate {})))))) + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate {:force true})))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate {:allow-all-differences true})] + (t/is (= {:allow-build-differences false + :allow-scene-differences false + :allow-all-differences true} + (:permissions comparison))) + (t/is (= [{:field :environment + :baseline {:node "v24" :browser "Fake"} + :candidate {:node "v25" :browser "Fake"} + :waived? true :rule :all}] + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/starts-with? text "Permissions ")) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) + (t/is (str/includes? text ":rule :all"))))) + +(t/deftest unreadable-records-fail-at-the-boundary + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12))] + (let [file (path/join @temporary-root "comparison.transit") + record (:comparison (compare/execute baseline candidate {}))] + (fs/writeFileSync file (result/encode record)) + (t/is (thrown-with-msg? js/Error #"expected a saved run record" + (compare/execute baseline file {})))) + (fs/writeFileSync (path/join @temporary-root "broken.transit") "not transit") + (t/is (thrown? js/Error (compare/execute baseline + (path/join @temporary-root "broken.transit") + {}))) + (fs/writeFileSync (path/join @temporary-root "malformed.transit") + (transit/encode-str (dissoc (make-run "malformed" :case/load 12) :plan))) + (doseq [options [{} {:allow-all-differences true}]] + (t/is (= ::result/invalid-record + (:type (ex-data (failure #(compare/execute baseline + (path/join @temporary-root "malformed.transit") + options))))))) + (t/is (thrown? js/Error (compare/execute baseline + (path/join @temporary-root "missing.transit") + {}))))) diff --git a/frontend/test/performance_tests/render_wasm/containers_test.cljs b/frontend/test/performance_tests/render_wasm/containers_test.cljs new file mode 100644 index 0000000000..19d68dc833 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/containers_test.cljs @@ -0,0 +1,251 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.containers-test + (:require + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as builder] + [benches.render-wasm.random :as random] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(t/deftest frame-attaches-children-in-order + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 100 :height 100 :name "F"} + (builder/rect {:x 10 :y 10 :width 5 :height 5}) + (builder/rect {:x 20 :y 20 :width 5 :height 5}))) + frame (first (helpers/children-of instance uuid/zero)) + rects (helpers/children-of instance (:id frame))] + (t/is (= :frame (:type frame))) + (t/is (= [(:id frame)] (helpers/child-ids instance uuid/zero))) + (t/is (= 2 (count rects))) + (t/is (every? #(= (:id frame) (:parent-id %)) rects)) + (t/is (every? #(= (:id frame) (:frame-id %)) rects)) + (t/is (= [10 20] (mapv :x rects))) + (t/is (= [10 20] (mapv :y rects))) + (t/is (= [100 100] [(:width frame) (:height frame)])))) + +(t/deftest nested-frames-keep-frame-ids + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 100 :height 100 :name "Outer"} + (builder/frame {:x 10 :y 10 :width 50 :height 50 :name "Inner"} + (builder/rect {:x 20 :y 20 :width 5 :height 5})))) + outer (first (helpers/children-of instance uuid/zero)) + inner (first (helpers/children-of instance (:id outer))) + rect (first (helpers/children-of instance (:id inner)))] + (t/is (= uuid/zero (:frame-id outer))) + (t/is (= (:id outer) (:parent-id inner))) + (t/is (= (:id outer) (:frame-id inner))) + (t/is (= (:id inner) (:parent-id rect))) + (t/is (= (:id inner) (:frame-id rect))) + (t/is (= [20 20] [(:x rect) (:y rect)])))) + +(t/deftest group-derives-bounds-from-children + (let [instance (builder/scene {:seed 1} + (builder/group {} + (builder/rect {:x 10 :y 20 :width 30 :height 40}) + (builder/rect {:x 50 :y 80 :width 10 :height 10}))) + group (first (helpers/children-of instance uuid/zero)) + rects (helpers/children-of instance (:id group))] + (t/is (= :group (:type group))) + (t/is (= [10 20 50 70] + [(:x group) (:y group) (:width group) (:height group)])) + (t/is (every? #(= uuid/zero (:frame-id %)) rects)) + (t/is (every? #(= (:id group) (:parent-id %)) rects)))) + +(t/deftest nested-groups-derive-inside-out + (let [instance (builder/scene {:seed 1} + (builder/group {:name "Outer"} + (builder/rect {:x 0 :y 0 :width 10 :height 10}) + (builder/group {:name "Inner"} + (builder/rect {:x 100 :y 100 :width 10 :height 10})))) + outer (first (helpers/children-of instance uuid/zero)) + inner (second (helpers/children-of instance (:id outer)))] + (t/is (= :group (:type inner))) + (t/is (= [100 100 10 10] + [(:x inner) (:y inner) (:width inner) (:height inner)])) + (t/is (= [0 0 110 110] + [(:x outer) (:y outer) (:width outer) (:height outer)])))) + +(t/deftest group-inside-frame-points-at-the-frame + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 200 :height 200} + (builder/group {} + (builder/rect {:x 10 :y 10 :width 10 :height 10})))) + frame (first (helpers/children-of instance uuid/zero)) + group (first (helpers/children-of instance (:id frame))) + rect (first (helpers/children-of instance (:id group)))] + (t/is (= (:id frame) (:frame-id group))) + (t/is (= (:id frame) (:frame-id rect))) + (t/is (= (:id group) (:parent-id rect))))) + +(t/deftest masked-group-takes-the-first-child-geometry + (let [instance (builder/scene {:seed 1} + (builder/group {:masked-group true} + (builder/rect {:x 10 :y 20 :width 30 :height 40}) + (builder/rect {:x 0 :y 0 :width 500 :height 500}))) + group (first (helpers/children-of instance uuid/zero))] + (t/is (true? (:masked-group group))) + (t/is (= [10 20 30 40] + [(:x group) (:y group) (:width group) (:height group)])))) + +(t/deftest container-labels-are-snapshot-local + (let [instance (builder/scene {:seed 1} + (builder/frame :hero {:x 0 :y 0 :width 10 :height 10} + (builder/group [:g 1] {} + (builder/rect)))) + refs (:refs instance)] + (t/is (= #{:hero [:g 1]} (set (keys refs)))) + (t/is (= (get refs :hero) (first (helpers/child-ids instance uuid/zero)))) + (t/is (= (get refs [:g 1]) (first (helpers/child-ids instance (get refs :hero))))) + (t/is (= 1 (count (helpers/child-ids instance uuid/zero)))))) + +(t/deftest container-expressions-evaluate-once-in-order + (let [events (atom []) + instance (builder/scene {:seed 1} + (builder/frame [:frame (do (swap! events conj :label) 3)] + (do (swap! events conj :attrs) + {:x 0 :y 0 :width 20 :height 20}) + (swap! events conj :body) + (builder/rect {:x 10}))) + frame-id (get (:refs instance) [:frame 3])] + (t/is (= [:attrs :label :body] @events)) + (t/is (= [10] (mapv :x (helpers/children-of instance frame-id)))))) + +(t/deftest invalid-container-attrs-retain-seeded-construction-order + (doseq [construct [#(builder/frame :hero :invalid) + #(builder/group :hero :invalid) + #(builder/bool :hero :invalid)]] + (let [rng (random/create 7) + failed-id (random/rng-uuid rng) + next-id (random/rng-uuid rng) + failure (volatile! nil) + state (helpers/run-scope + {:seed 7} + (fn [] + (vreset! failure (helpers/failure-data construct)) + (builder/rect))) + instance (builder/finish! state)] + (t/is (= {:type ::builder/invalid-attrs :id failed-id} @failure)) + (t/is (= [next-id] (helpers/child-ids instance uuid/zero)))))) + +(t/deftest empty-group-is-rejected + (let [data (helpers/failure-data #(builder/scene {:seed 1} (builder/group {})))] + (t/is (= ::builder/empty-group (:type data))))) + +(t/deftest group-geometry-attrs-are-rejected + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/group {:x 1} + (builder/rect))))] + (t/is (= ::builder/group-geometry (:type data))) + (t/is (= :x (:key data))))) + +(t/deftest frame-requires-bounds + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame {:name "F"} + (builder/rect))))] + (t/is (= ::builder/frame-bounds (:type data))))) + +(t/deftest runtime-attrs-expressions-are-accepted + (let [frame-attrs {:x 0 :y 0 :width 10 :height 10} + instance (builder/scene {:seed 1} + (builder/frame frame-attrs + (builder/rect))) + frame (first (helpers/children-of instance uuid/zero))] + (t/is (= :frame (:type frame))) + (t/is (= 10 (:width frame))))) + +(t/deftest non-map-attrs-are-rejected + (t/is (= ::builder/invalid-attrs + (:type (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame :hero (builder/rect))))))) + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (builder/frame :hero (builder/rect)) + (catch :default _ nil))))] + (t/is (= 1 (count (helpers/child-ids @state uuid/zero)))))) + +(t/deftest non-numeric-frame-bounds-are-rejected + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width "100" :height 10} + (builder/rect))))] + (t/is (= ::builder/frame-bounds (:type data))) + (t/is (= :width (:invalid data))))) + +(t/deftest duplicate-container-labels-are-rejected + (let [data (helpers/failure-data + #(builder/scene {:seed 1} + (builder/frame :dup {:x 0 :y 0 :width 10 :height 10} + (builder/rect)) + (builder/group :dup {} + (builder/rect))))] + (t/is (= ::builder/duplicate-label (:type data))) + (t/is (= :dup (:label data))))) + +(t/deftest frame-inside-group-keeps-frame-ids + (let [instance (builder/scene {:seed 1} + (builder/group {} + (builder/frame {:x 0 :y 0 :width 50 :height 50} + (builder/rect)))) + group (first (helpers/children-of instance uuid/zero)) + frame (first (helpers/children-of instance (:id group))) + rect (first (helpers/children-of instance (:id frame)))] + (t/is (= (:id group) (:parent-id frame))) + (t/is (= uuid/zero (:frame-id frame))) + (t/is (= (:id frame) (:parent-id rect))) + (t/is (= (:id frame) (:frame-id rect))))) + +(t/deftest group-of-container-derives-bounds + (let [instance (builder/scene {:seed 1} + (builder/group {} + (builder/group {} + (builder/rect {:x 5 :y 5 :width 10 :height 10})))) + outer (first (helpers/children-of instance uuid/zero)) + inner (first (helpers/children-of instance (:id outer)))] + (t/is (= [5 5 10 10] + [(:x outer) (:y outer) (:width outer) (:height outer)])) + (t/is (= [5 5 10 10] + [(:x inner) (:y inner) (:width inner) (:height inner)])) + (t/is (= 1 (count (helpers/child-ids instance (:id outer))))))) + +(t/deftest empty-frame-is-allowed + (let [instance (builder/scene {:seed 1} + (builder/frame {:x 0 :y 0 :width 10 :height 10})) + frame (first (helpers/children-of instance uuid/zero))] + (t/is (= [] (helpers/child-ids instance (:id frame)))))) + +(t/deftest caught-empty-group-still-fails-finish + (let [state (helpers/run-scope {:seed 1} + (fn [] + (try + (builder/group {}) + (catch :default _ nil))))] + (t/is (= ::builder/unfinished-container + (:type (helpers/failure-data #(builder/finish! state))))))) + +(t/deftest container-failure-restores-parent-and-marks-unfinished + (let [state (helpers/run-scope {:seed 1} + (fn [] + (builder/rect {:x 0}) + (try + (builder/frame {:x 0 :y 0 :width 10 :height 10} + (builder/rect :dup {:x 1}) + (builder/rect :dup {:x 2})) + (catch :default _ nil)) + (builder/rect {:x 3}))) + root-children (helpers/child-ids @state uuid/zero) + last-child (get-in @state [:objects (last root-children)])] + (t/is (= 3 (count root-children))) + (t/is (= :rect (:type last-child))) + (t/is (= 3 (:x last-child))) + (t/is (= 1 (count (:unfinished @state)))) + (t/is (= ::builder/unfinished-container + (:type (helpers/failure-data #(builder/finish! state))))))) diff --git a/frontend/test/performance_tests/render_wasm/contracts_test.cljs b/frontend/test/performance_tests/render_wasm/contracts_test.cljs new file mode 100644 index 0000000000..0804f0e388 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/contracts_test.cljs @@ -0,0 +1,445 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.contracts-test + (:require + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [cljs.test :as t :include-macros true])) + +(defrecord Point [x y]) + +(def ^:private body-calls + (atom [])) + +(defn- touch! + [rtx] + (swap! body-calls conj :touched) + rtx) + +(decl/defscene :contracts-scene + {:description "Synthetic scene for contract tests" + :params-schema [:map {:closed true}]} + [params] + params) + +(decl/defcase :contracts-scene/case + {:params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh} + [ctx] + (touch! ctx)) + +(defn- failure-data + [f] + (try + (f) + nil + (catch :default cause + (ex-data cause)))) + +(defn- rect-cases + ([] (rect-cases 42 "rects/")) + ([master-seed filter] + (cases/collect-cases {:master-seed master-seed :filter filter}))) + +(defn- with-synthetic-scene + "Registers a synthetic scene and case under `scene-id`, runs `f`, then + unregisters both so the live registry survives." + [scene-id case-params f] + (decl/register-scene! {:id scene-id + :ns "contracts-test" + :description "synthetic" + :params-schema [:map {:closed true} [:count [:int {:min 1 :max 100000}]]] + :build (fn [params] params)}) + (decl/register-case! {:id (keyword (name scene-id) "case") + :scene scene-id + :ns "contracts-test" + :params case-params + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (try + (f) + (finally + (decl/unregister-case! (keyword (name scene-id) "case")) + (decl/unregister-scene! scene-id)))) + +(t/deftest defscene-registers-metadata-and-build + (let [scene (decl/registered-scene :rects)] + (t/is (= :rects (:id scene))) + (t/is (= "benches.render-wasm.scenes.rects" (:ns scene))) + (t/is (= "Seeded rectangles with translucent fills and centered strokes" + (:description scene))) + (t/is (vector? (:params-schema scene))) + (t/is (fn? (:build scene))))) + +(t/deftest rects-cases-are-collected-in-declaration-order + (let [collected (rect-cases)] + (t/is (= [:rects/load :rects/pan :rects/zoom :rects/blocking-full :rects/mixed-pans] (mapv :id collected))))) + +(t/deftest collected-cases-survive-the-wire + (doseq [scene-id [:rects :paths :plus :masks]] + (let [collected (cases/collect-cases {:master-seed 42 + :filter (str (name scene-id) "/")}) + decoded (transit/decode-str (transit/encode-str collected))] + (t/is (= collected decoded)) + (doseq [case-desc decoded] + (t/is (= case-desc (decl/check-collected-case case-desc))) + (t/is (not-any? #(contains? case-desc %) + [:run! :ns]) + (str (:id case-desc))))))) + +(t/deftest declared-preparation-survives-collection-and-wire + (with-synthetic-scene :contracts-prepared {:count 3} + (fn [] + (let [preparation {:render :none :settle-ms 25} + registered (decl/registered-case :contracts-prepared/case)] + (decl/register-case! (assoc registered + :preparation (assoc preparation :callback identity) + :reset identity)) + (let [collected (cases/collect-cases {:master-seed 42 :filter "contracts-prepared/"}) + decoded (transit/decode-str (transit/encode-str collected))] + (t/is (= [:contracts-prepared/case] (mapv :id decoded))) + (t/is (= collected decoded)) + (t/is (= preparation (:preparation (first decoded)))) + (t/is (= (first decoded) (decl/check-collected-case (first decoded))))) + (decl/register-case! (assoc registered :preparation {:asset {:loader identity}})) + (t/is (= ::cases/invalid-case + (:type (failure-data #(cases/collect-cases {:master-seed 42 :filter "contracts-prepared/"}))))))))) + +(t/deftest rects-cases-carry-declared-view-context-and-preparation + (let [collected (rect-cases)] + (t/is (= {:rects/load :fresh :rects/pan :reuse :rects/zoom :reuse + :rects/blocking-full :fresh :rects/mixed-pans :reuse} + (into {} (map (juxt :id :context)) collected))) + (doseq [case-desc collected] + (t/is (= {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100} + (:params case-desc)) + (str (:id case-desc))) + (t/is (= {:scale 1 :x 0 :y 0} (:view case-desc)) (str (:id case-desc))) + (t/is (= (if (#{:rects/load :rects/blocking-full} (:id case-desc)) {:render :none} {}) + (:preparation case-desc)) (str (:id case-desc)))))) + +(t/deftest rects-cases-carry-scene-metadata-and-seed + (let [collected (rect-cases)] + (doseq [case-desc collected] + (t/is (= :rects (:scene case-desc))) + (t/is (= "Seeded rectangles with translucent fills and centered strokes" + (:scene-description case-desc))) + (t/is (= 2602546180 (:scene-seed case-desc)) (str (:id case-desc))))) + (let [collected (rect-cases 43 "rects/")] + (t/is (every? #(= 2993218129 (:scene-seed %)) collected)))) + +(t/deftest derive-seed-depends-on-scene-and-params-only + (let [params {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100} + base (decl/derive-seed 42 :rects params)] + (t/is (= 2602546180 base)) + (t/is (= base (decl/derive-seed 42 :rects (into {} (reverse (seq params)))))) + (t/is (= 2993218129 (decl/derive-seed 43 :rects params))) + (t/is (not= base (decl/derive-seed 42 :rects (assoc params :count 10)))) + (t/is (not= base (decl/derive-seed 42 :paths params))))) + +(t/deftest derive-seed-rejects-keys-it-cannot-tell-apart + (t/is (= ::decl/invalid-params + (:type (failure-data #(decl/derive-seed 42 :rects {"count" 1}))))) + (t/is (= ::decl/invalid-params + (:type (failure-data #(decl/derive-seed 42 :rects {:rects/count 1}))))) + (t/is (= :rects (:scene (failure-data #(decl/derive-seed 42 :rects {"count" 1})))))) + +(t/deftest unknown-params-are-rejected + (with-synthetic-scene :contracts-unknown-params {:count 1 :bogus 2} + (fn [] + (let [data (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-unknown-params/"}))] + (t/is (= ::cases/invalid-case (:type data))) + (t/is (some? (::sm/explain data))))))) + +(t/deftest out-of-range-count-is-rejected + (with-synthetic-scene :contracts-bad-count {:count 0} + (fn [] + (let [data (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-bad-count/"}))] + (t/is (= ::cases/invalid-case (:type data))) + (t/is (some? (::sm/explain data))))))) + +(t/deftest rects-workload-ranges-are-rejected + (let [schema (:params-schema (decl/registered-scene :rects)) + good {:count 1000 :width 1920 :height 1080 :min-size 20 :max-size 100}] + (t/is (true? (sm/validate schema good))) + (t/testing "reversed size range" + (t/is (false? (sm/validate schema (assoc good :min-size 100 :max-size 20))))) + (t/testing "non-finite sizes" + (doseq [bad [(assoc good :width js/Infinity) + (assoc good :height js/NaN) + (assoc good :min-size js/Infinity) + (assoc good :max-size js/NaN)]] + (t/is (false? (sm/validate schema bad)) (pr-str bad)))))) + +(t/deftest view-coordinates-are-finite + (let [good {:scale 1 :x 0 :y 0}] + (t/is (true? (sm/validate decl/schema:view good))) + (doseq [bad [(assoc good :scale js/Infinity) + (assoc good :x js/NaN) + (assoc good :y js/Infinity) + (assoc good :viewport {:width js/NaN})]] + (t/is (false? (sm/validate decl/schema:view bad)) (pr-str bad))))) + +(t/deftest transit-rejects-unencodable-values + (t/is (false? (decl/transit-round-trips? {:v (fn [] 1)}))) + (t/is (false? (decl/transit-round-trips? {:point (->Point 1 2)})))) + +(t/deftest transit-wire-preserves-json-lossy-shapes + (doseq [value [{:foo 1 "foo" 2} + {:a/b 1 :c/b 2} + {:id :rects/load :scene :rects} + {:tags [:fresh :rects/load]} + {:v js/Infinity} + {:v js/-Infinity}]] + (t/is (= value (-> value transit/encode-str transit/decode-str)) + (pr-str value)))) + +(t/deftest transit-wire-preserves-non-finite-numbers + (let [value {:nan js/NaN :positive js/Infinity :negative js/-Infinity} + decoded (-> value transit/encode-str transit/decode-str) + case-desc (assoc (first (rect-cases)) :params value)] + (t/is (number? (:nan decoded))) + (t/is (js/Number.isNaN (:nan decoded))) + (t/is (= js/Infinity (:positive decoded))) + (t/is (= js/-Infinity (:negative decoded))) + (t/is (decl/transit-round-trips? value)) + (t/is (nil? (failure-data #(decl/check-collected-case case-desc)))))) + +(t/deftest unencodable-collected-values-are-rejected + (let [good (first (rect-cases))] + (doseq [bad [(assoc good :params {:run (fn [] 1)}) + (assoc good :params {:point (->Point 1 2)})]] + (t/is (= ::decl/non-serializable-case + (:type (failure-data #(decl/check-collected-case bad)))) + (pr-str bad))))) + +(t/deftest duplicate-scene-id-from-another-namespace-is-rejected + (t/is (= ::decl/duplicate-scene + (:type (failure-data + #(decl/register-scene! {:id :rects + :ns "another-namespace" + :description "x" + :params-schema [:map] + :build (fn [params] params)})))))) + +(t/deftest scene-replacement-preserves-order-and-unregister-removes + (let [ids [:contracts-first :contracts-middle :contracts-last] + scene {:ns "contracts-test" :description "first" + :params-schema [:map] :build identity} + get-scenes #(filterv (comp (set ids) :id) (decl/scenes))] + (try + (doseq [id ids] + (decl/register-scene! (assoc scene :id id))) + (decl/register-scene! (assoc scene :id :contracts-middle :description "second")) + (t/is (= ids (mapv :id (get-scenes)))) + (t/is (= ["first" "second" "first"] (mapv :description (get-scenes)))) + (decl/unregister-scene! :contracts-middle) + (t/is (= [:contracts-first :contracts-last] (mapv :id (get-scenes)))) + (t/is (nil? (decl/registered-scene :contracts-middle))) + (finally + (doseq [id ids] + (decl/unregister-scene! id)))))) + +(t/deftest duplicate-case-id-from-another-namespace-is-rejected + (try + (decl/register-scene! {:id :contracts-dup + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (decl/register-case! {:id :contracts-dup/case + :scene :contracts-dup + :ns "contracts-test" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (t/is (= ::decl/duplicate-case + (:type (failure-data + #(decl/register-case! {:id :contracts-dup/case + :scene :contracts-dup + :ns "another-namespace" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}))))) + (finally + (decl/unregister-case! :contracts-dup/case) + (decl/unregister-scene! :contracts-dup)))) + +(t/deftest unknown-scene-reference-is-rejected + (try + (decl/register-case! {:id :contracts-missing/case + :scene :contracts-missing + :ns "contracts-test" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (t/is (= ::cases/unknown-scene + (:type (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-missing/"}))))) + (finally + (decl/unregister-case! :contracts-missing/case)))) + +(t/deftest case-id-must-be-namespaced-by-its-scene + (try + (decl/register-scene! {:id :contracts-other + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (decl/register-case! {:id :contracts-other/case + :scene :rects + :ns "contracts-test" + :params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh}) + (t/is (= ::cases/invalid-case + (:type (failure-data + #(cases/collect-cases {:master-seed 42 + :filter "contracts-other/"}))))) + (finally + (decl/unregister-case! :contracts-other/case) + (decl/unregister-scene! :contracts-other)))) + +(t/deftest scene-without-params-schema-is-rejected + (try + (decl/register-scene! {:id :contracts-noschema + :ns "contracts-test" + :description "x" + :build (fn [params] params)}) + (t/is (= ::decl/invalid-scene + (:type (failure-data #(cases/collect-cases {:master-seed 42}))))) + (finally + (decl/unregister-scene! :contracts-noschema)))) + +(t/deftest malformed-params-schema-is-rejected + (try + (decl/register-scene! {:id :contracts-badschema + :ns "contracts-test" + :description "x" + :params-schema [:map [:count]] + :build (fn [params] params)}) + (let [data (failure-data #(cases/collect-cases {:master-seed 42}))] + (t/is (= ::decl/invalid-scene (:type data))) + (t/is (= :contracts-badschema (:id data)))) + (finally + (decl/unregister-scene! :contracts-badschema)))) + +(t/deftest qualified-scene-id-is-rejected + (try + (decl/register-scene! {:id :contracts-ns/scene + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (t/is (= ::decl/invalid-scene + (:type (failure-data #(cases/collect-cases {:master-seed 42}))))) + (finally + (decl/unregister-scene! :contracts-ns/scene)))) + +(t/deftest invalid-declarations-fail-outside-the-filter + (with-synthetic-scene :contracts-global {:count 1 :bogus 2} + (fn [] + (t/is (= ::cases/invalid-case + (:type (failure-data #(cases/collect-cases {:master-seed 42 + :filter "rects/"})))))))) + +(t/deftest incomplete-case-declaration-is-rejected + (try + (decl/register-scene! {:id :contracts-incomplete + :ns "contracts-test" + :description "x" + :params-schema [:map] + :build (fn [params] params)}) + (decl/register-case! {:id :contracts-incomplete/case + :scene :contracts-incomplete + :ns "contracts-test" + :params {} + :context :fresh}) + (t/is (= ::cases/invalid-case + (:type (failure-data #(cases/collect-cases {:master-seed 42 + :filter "contracts-incomplete/"}))))) + (finally + (decl/unregister-case! :contracts-incomplete/case) + (decl/unregister-scene! :contracts-incomplete)))) + +(t/deftest records-do-not-survive-the-wire + (let [point (->Point 1 2)] + (t/is (false? (decl/transit-round-trips? point))) + (t/is (false? (decl/transit-round-trips? {:point point}))))) + +(t/deftest case-replacement-preserves-order-and-unregister-removes + (let [ids [:contracts-replacecase/first :contracts-replacecase/middle :contracts-replacecase/last] + bench-case {:scene :contracts-replacecase :ns "contracts-test" + :params {} :view {:scale 1 :x 0 :y 0} :context :fresh} + get-cases #(filterv (comp (set ids) :id) (decl/cases))] + (try + (decl/register-scene! {:id :contracts-replacecase :ns "contracts-test" + :description "x" :params-schema [:map] :build identity}) + (doseq [id ids] + (decl/register-case! (assoc bench-case :id id))) + (decl/register-case! (assoc bench-case :id :contracts-replacecase/middle :view {:scale 2 :x 0 :y 0})) + (t/is (= ids (mapv :id (get-cases)))) + (t/is (= [1 2 1] (mapv #(get-in % [:view :scale]) (get-cases)))) + (decl/unregister-case! :contracts-replacecase/middle) + (t/is (= [:contracts-replacecase/first :contracts-replacecase/last] (mapv :id (get-cases)))) + (t/is (nil? (decl/registered-case :contracts-replacecase/middle))) + (finally + (doseq [id ids] + (decl/unregister-case! id)) + (decl/unregister-scene! :contracts-replacecase))))) + +(t/deftest invalid-master-seed-is-rejected + (doseq [seed [-1 4294967296 "42" nil 0.5 js/NaN js/Infinity js/-Infinity]] + (t/is (= ::cases/invalid-master-seed + (:type (failure-data #(cases/collect-cases {:master-seed seed})))) + (str seed)))) + +(t/deftest filter-matches-case-ids + (t/is (= [:rects/pan] + (mapv :id (cases/collect-cases {:master-seed 42 :filter "rects/pan"})))) + (t/is (= 5 (count (cases/collect-cases {:master-seed 42 :filter "rects/"}))))) + +(t/deftest empty-selection-is-rejected + (t/is (= ::cases/no-cases + (:type (failure-data #(cases/collect-cases {:master-seed 42 + :filter "missing"})))))) + +(t/deftest defcase-registers-function-without-sending-it + (reset! body-calls []) + (let [collected (cases/collect-cases {:master-seed 42 :filter "contracts-scene/"})] + (t/is (= [:contracts-scene/case] (mapv :id collected))) + (t/is (not (contains? (first collected) :run!))) + (t/is (decl/transit-round-trips? (first collected)))) + (t/is (empty? @body-calls)) + (let [entry (decl/registered-case :contracts-scene/case)] + (t/is (= "performance-tests.render-wasm.contracts-test" (:ns entry))) + (t/is (fn? (:run! entry))) + (t/is (= :runtime ((:run! entry) :runtime))) + (t/is (= [:touched] @body-calls)))) + +(t/deftest defcase-rejects-run-function-in-options + (t/is (= :benches.render-wasm.declarations/invalid-run-function + (:type (failure-data + #(decl/defcase :contracts-scene/unchecked + {:run! (fn [x y] [x y])}))))) + (t/is (nil? (decl/registered-case :contracts-scene/unchecked)))) + +(t/deftest defcase-rejects-unqualified-id + (t/is (= :benches.render-wasm.declarations/invalid-run-function + (:type (failure-data + #(decl/defcase :bare + {:params {} + :view {:scale 1 :x 0 :y 0} + :context :fresh})))))) diff --git a/frontend/test/performance_tests/render_wasm/errors_test.cljs b/frontend/test/performance_tests/render_wasm/errors_test.cljs new file mode 100644 index 0000000000..ef300c168e --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/errors_test.cljs @@ -0,0 +1,248 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.errors-test + (:require + [app.common.schema :as sm] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(t/deftest attempt-returns-the-value-of-its-body + (t/is (= 3 (attempt 1 2 3))) + (t/is (nil? (attempt nil))) + (t/is (= {:error :not-a-failure} (attempt {:error :not-a-failure})))) + +(t/deftest composed-failures-retain-occurrence-order-and-fatal-propagation + (let [upload (js/Error. "upload failed") + end (js/Error. "ending load failed") + close (js/Error. "browser disposal failed") + save (js/Error. "saving evidence failed") + initial (errors/combine [{:phase "upload" :operation :serialize-shapes :error upload} + {:phase "upload" :operation :end-loading :error end}]) + wrapped (errors/fatal-error "preparation failed" + {::errors/stale true :partial {:setup {:upload-ms 7}}} + (errors/with-context initial "preparation failed" + {:phase "preparation" :path "scene.transit"})) + later (errors/combine [{:phase "cleanup" :operation :close-browser :error close} + {:phase "storage" :operation :save-evidence :error save}]) + later (errors/fatal-error "cleanup failed" + {:partial {:setup {:upload-ms 9}} + :command "/bin/chromium" :exit-code 17} + (errors/with-data later {:partial {:render {:frames 2}}})) + combined (errors/combine [{:error wrapped} {:error later}]) + entries (errors/get-errors combined) + deep (nth (iterate #(ex-info "wrapped" {} %) combined) 12) + wire (-> (errors/evidence :runner deep) + transit/encode-str transit/decode-str errors/project-diagnostic)] + (t/is (= [upload end close save] (mapv :error entries))) + (t/is (= [:serialize-shapes :end-loading :close-browser :save-evidence] + (mapv :operation entries))) + (t/is (= "scene.transit" (:path (first entries)))) + (t/is (= {:setup {:upload-ms 7}} (:partial (first entries)))) + (t/is (errors/stale? combined)) + (t/is (= (mapv errors/message-of [upload end close save]) (mapv :message (:errors wire)))) + (t/is (= {:command "/bin/chromium" :exit-code 17 + :partial {:setup {:upload-ms 9} :render {:frames 2}}} + (select-keys (nth (:errors wire) 2) [:command :exit-code :partial]))) + (t/is (sm/validate errors/schema:diagnostic wire)) + (t/is (errors/fatal? deep)) + (t/is (errors/fatal? (errors/combine [{:error upload} {:error later}]))) + (t/is (try + (attempt (throw combined)) + false + (catch :default cause + (= [upload end close save] (mapv :error (errors/get-errors cause)))))))) + +(t/deftest ordinary-failures-project-operation-and-path-context + (let [cause (ex-info "record unavailable" {:operation :read-record :path "saved/run.transit"}) + wire (-> (errors/evidence :storage cause) transit/encode-str transit/decode-str)] + (t/is (= {:operation :read-record :path "saved/run.transit"} + (select-keys wire [:operation :path]))) + (t/is (sm/validate errors/schema:diagnostic wire)))) + +(t/deftest attempt-returns-the-error-its-body-throws + (let [cause (ex-info "boom" {:a 1}) + result (attempt (throw cause))] + (t/is (failure? result)) + (t/is (identical? cause result)))) + +(t/deftest attempt-turns-a-thrown-value-into-an-error-that-keeps-its-text + (let [text (attempt (throw "device disconnected")) + object (attempt (throw #js {:code 7}))] + (t/is (failure? text)) + (t/is (= "device disconnected" (ex-message text))) + (t/is (failure? object)) + (t/is (str/includes? (ex-message object) "code")))) + +(t/deftest attempt-throws-fatal-failures-again + (let [cause (errors/fatal-error "write failed" {:path "a"} (js/Error. "disk"))] + (t/is (thrown-with-msg? js/Error #"write failed" + (attempt (throw cause)))) + (t/is (identical? cause (attempt :catch-fatal (throw cause)))))) + +(t/deftest attempt-treats-a-fatal-cause-deep-in-the-chain-as-fatal + (let [fatal (errors/fatal-error "write failed" {} nil) + outer (ex-info "outer" {} (ex-info "middle" {} fatal))] + (t/is (true? (errors/fatal? outer))) + (t/is (thrown-with-msg? js/Error #"outer" (attempt (throw outer)))))) + +(t/deftest fatal-marker-is-not-assumed-from-other-data + (t/is (false? (errors/fatal? (ex-info "x" {:type :anything})))) + (t/is (false? (errors/fatal? nil))) + (t/is (false? (errors/fatal? "text")))) + +(t/deftest ^:async attempt-awaits-promises-in-an-async-function + (let [value (attempt (await (js/Promise.resolve 7))) + failure (attempt (await (js/Promise.reject (ex-info "rejected" {})))) + bound (let [x (attempt :catch-fatal (await (js/Promise.resolve 1)) 2)] x)] + (t/is (= 7 value)) + (t/is (= "rejected" (ex-message failure))) + (t/is (= 2 bound)))) + +(t/deftest ^:async attempt-keeps-fatal-rejections-fatal + (let [cause (errors/fatal-error "write failed" {} nil) + outer (try + (attempt (await (js/Promise.reject cause))) + nil + (catch :default thrown thrown))] + (t/is (identical? cause outer)))) + +(t/deftest with-data-passes-non-failures-through + (t/is (= 1 (errors/with-data 1 {:a 1}))) + (t/is (nil? (errors/with-data nil {:a 1}))) + (t/is (= {:a 1} (errors/with-data {:a 1} {:a 2})))) + +(t/deftest with-data-adds-keys-under-the-failure-data + (let [cause (ex-info "boom" {:log-path "inner"}) + result (errors/with-data cause {:log-path "outer" :build-id "b"})] + (t/is (= "boom" (ex-message result))) + (t/is (= {:log-path "inner" :build-id "b"} (ex-data result))) + (t/is (identical? cause (ex-cause result))))) + +(t/deftest message-of-falls-back-to-the-string-form + (t/is (= "boom" (errors/message-of (ex-info "boom" {})))) + (t/is (= "text" (errors/message-of "text")))) + +(t/deftest context-keeps-deeper-measurements-and-native-cause-details + (let [native (ex-info "render failed" {:fn "_render" :code 17}) + cause (ex-info "interact failed" {:phase "interact" :partial {:slices [1 2]}} native) + error (errors/with-context cause "setup failed" + {:phase "setup" :partial {:slices [1] :setup {:upload-ms 3}}}) + result (errors/evidence :setup error)] + (t/is (= {:phase "setup" :partial {:slices [1 2] :setup {:upload-ms 3}}} + (ex-data error))) + (t/is (= "setup failed" (:message result))) + (t/is (some #(= {:message "render failed" :fn "_render" :code 17} %) + (:cause result))))) + +(t/deftest context-keeps-native-errors-as-diagnostic-causes + (let [error (errors/with-context (js/Error. "device lost") "restore failed" {:phase "restore"})] + (t/is (some #(= "device lost" (:message %)) (errors/describe-cause error))))) + +(t/deftest hostile-cause-renders-plain-data + (let [hostile (ex-info "boom" + {:fn (fn [] 1) :code #js {:evil 1} :type :wasm-error + :hint (str (apply str (repeat 499 "x")) "😀")} + (js/Error. "inner boom")) + result (errors/evidence :aborted hostile) + decoded (-> result transit/encode-str transit/decode-str) + outer (first (:cause decoded))] + (t/is (= "aborted" (:phase decoded))) + (t/is (= "boom" (:message decoded))) + (t/is (string? (:fn outer))) + (t/is (string? (:code outer))) + (t/is (= (apply str (repeat 499 "x")) (:hint outer))) + (t/is (some #(= "inner boom" (:message %)) (:cause decoded))))) + +(t/deftest diagnostics-retain-bounded-identity-and-process-details + (let [long-text (str (apply str (repeat 1023 "x")) "😀") + context {:expected-id long-text :actual-id "other-build" + :command "compiler" :exit-code 23 :signal "SIGTERM" + :endpoint "ws://127.0.0.1/devtools/browser/owner"} + cause (ex-info "failed" context) + decoded (-> (errors/evidence :identity cause) + errors/project-diagnostic + transit/encode-str + transit/decode-str)] + (t/is (= (assoc context :expected-id (apply str (repeat 1023 "x"))) + (select-keys decoded (keys context)))))) + +(t/deftest unprintable-values-produce-portable-diagnostics + (let [evil (js-obj)] + (js/Object.defineProperty evil "evil" + #js {:enumerable true + :get (fn [] (throw (js/Error. "evil")))}) + (let [data (errors/evidence :upload (ex-info "evil boom" {:fn evil})) + cause (errors/evidence :upload evil) + decoded (-> [data cause] transit/encode-str transit/decode-str)] + (t/is (= "unrenderable value" (get-in decoded [0 :cause 0 :fn]))) + (t/is (= "unrenderable cause" (get-in decoded [1 :cause 0 :message])))))) + +(t/deftest diagnostic-text-is-bounded-without-splitting-surrogate-pairs + (let [message (str (apply str (repeat 1023 "x")) "😀" "tail") + cause (ex-info message {:command message}) + data (-> (errors/evidence :main cause) transit/encode-str transit/decode-str) + hostile (js/Error.)] + (set! (.-message hostile) #js {:toString #(throw (js/Error. "cannot print"))}) + (t/is (= (apply str (repeat 1023 "x")) (:message data))) + (t/is (<= (count (:command data)) 1024)) + (t/is (sm/validate errors/schema:diagnostic (errors/evidence :main hostile))))) + +(t/deftest malformed-partial-data-retains-the-failure-and-representable-evidence + (let [cycle #js {}] + (aset cycle "self" cycle) + (doseq [invalid [(fn [] nil) cycle (js/AbortController.)] + composed? [false true]] + (let [failure (ex-info "phase failed" {:partial {:safe {:elapsed [0 js/Infinity js/NaN]} + :handle invalid}}) + failure (if composed? + (errors/combine [{:error failure} {:error (js/Error. "cleanup failed")}]) + failure) + data (errors/evidence :main failure) + wire (try + (-> data transit/encode-str transit/decode-str) + (catch :default _ nil))] + (t/is (= "phase failed" (:message wire))) + (t/is (= "phase failed" (get-in wire [:cause 0 :message]))) + (t/is (= 0 (get-in wire [:partial :safe :elapsed 0]))) + (t/is (= js/Infinity (get-in wire [:partial :safe :elapsed 1]))) + (t/is (js/Number.isNaN (get-in wire [:partial :safe :elapsed 2]))) + (t/is (= 1 (count (filter #(= :encode-diagnostic (:operation %)) (:errors wire))))))))) + +(t/deftest unbounded-partial-data-produces-an-encoding-diagnostic + (let [data (errors/evidence :main (ex-info "phase failed" {:partial {:trace (repeat 0)}})) + wire (-> data transit/encode-str transit/decode-str)] + (t/is (= "phase failed" (:message wire))) + (t/is (some #(= :encode-diagnostic (:operation %)) (:errors wire))))) + +(t/deftest malformed-partial-shapes-and-cyclic-causes-produce-portable-diagnostics + (doseq [partial [nil false 42 "partial"]] + (let [data (errors/evidence :main (ex-info "phase failed" {:partial partial}))] + (t/is (sm/validate errors/schema:diagnostic data)) + (t/is (some #(= :encode-diagnostic (:operation %)) (:errors data))))) + (let [cycle (js/Error. "cyclic failure")] + (set! (.-cause cycle) cycle) + (let [data (-> (errors/evidence :main cycle) transit/encode-str transit/decode-str)] + (t/is (= "cyclic failure" (:message data))) + (t/is (<= (count (:cause data)) 3))))) + +(t/deftest malformed-optional-metadata-retains-a-valid-failure-diagnostic + (doseq [flag [42 (fn [] nil)]] + (let [cause (errors/combine [{:error (ex-info "main failed" {:deadline-exceeded flag})}]) + data (errors/evidence :main cause) + wire (try (-> data transit/encode-str transit/decode-str) (catch :default _ nil))] + (t/is (= "main failed" (:message wire))) + (t/is (sm/validate errors/schema:diagnostic wire))))) + +(t/deftest non-finite-failure-details-cross-transit-as-numbers + (let [failure (ex-info "bad measurement" {:code js/NaN :hint js/Infinity :type js/-Infinity}) + decoded (-> (errors/evidence :render failure) transit/encode-str transit/decode-str) + detail (first (:cause decoded))] + (t/is (js/Number.isNaN (:code detail))) + (t/is (= js/Infinity (:hint detail))) + (t/is (= js/-Infinity (:type detail))))) diff --git a/frontend/test/performance_tests/render_wasm/execution_test.cljs b/frontend/test/performance_tests/render_wasm/execution_test.cljs new file mode 100644 index 0000000000..53e4c26264 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/execution_test.cljs @@ -0,0 +1,346 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.execution-test + (:require + ["fs" :as fs] + ["node:module" :refer [createRequire]] + ["node:stream" :as stream] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.cli.resolve :as cli] + [benches.render-wasm.node.log :as log] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def temporary-root (atom nil)) + +(defn- within + [promise] + (let [timer (volatile! nil)] + (-> (js/Promise.race + #js [promise + (js/Promise. + (fn [_ reject] + (vreset! timer (js/setTimeout #(reject (js/Error. "Test stalled")) 5000))))]) + (.finally #(js/clearTimeout @timer))))) + +(t/deftest ^:async real-signals-reach-the-callback-and-remover-restores-listeners + (let [before-int (.listenerCount js/process "SIGINT") + before-term (.listenerCount js/process "SIGTERM") + received (atom []) + next-signal (volatile! nil) + remove (execution/on-signals + (fn [signal] + (swap! received conj signal) + (@next-signal signal))) + receive (fn [signal] + (within + (js/Promise. + (fn [resolve _] + (vreset! next-signal resolve) + (.kill js/process (.-pid js/process) signal)))))] + (try + (t/is (= "SIGINT" (await (receive "SIGINT")))) + (t/is (= "SIGTERM" (await (receive "SIGTERM")))) + (t/is (= ["SIGINT" "SIGTERM"] @received)) + (finally + (remove))) + (remove) + (t/is (= before-int (.listenerCount js/process "SIGINT"))) + (t/is (= before-term (.listenerCount js/process "SIGTERM"))))) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "benchmark-execution-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- capture + [] + (let [output (atom "") + errors (atom "") + writer (fn [text] + (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))}))] + {:stdout (writer output) :stderr (writer errors) :output output :errors errors})) + +(defn- resolved + [command options] + (request/apply-defaults {:command command :console {:color :never} :options options})) + +(defn- listeners + [] + [(.listenerCount js/process "SIGINT") (.listenerCount js/process "SIGTERM")]) + + +(t/deftest ^:async sync-and-async-handlers-publish-once-and-release-listeners + (doseq [async [false true]] + (let [local (capture) + before (listeners) + request (cli/parse-resolve ["compare" "--help" "--no-color"]) + handler (fn [_ {:keys [topic]}] + (log/info "Preparing help") + {:exit-code 0 :errors [] :log-path nil :report (cli/get-help topic)}) + handler (if async + (^:async fn [context options] + (await (timers/setTimeout 1)) + (handler context options)) + handler)] + (t/is (= 0 (await (execution/execute request {"help" handler} + (assoc local :get-root #(throw (js/Error. "root unavailable"))))))) + (t/is (= (str (cli/get-help "compare") "\n") @(:output local))) + (t/is (= "Preparing help\n" @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest ^:async handlers-use-local-root-and-services-with-complete-options + (let [local (capture) + file (path/join @temporary-root "input.txt")] + (fs/writeFileSync file "saved input" "utf8") + (t/is (= 0 + (await + (execution/execute + (cli/parse-resolve ["run" "--build" "ready" "--warmups" "0"]) + {"run" (fn [{:keys [get-root services]} {:keys [warmups build]}] + {:exit-code 0 :errors [] :log-path nil + :report (str (:id build) ":" warmups ":" + ((:read-text services) (path/join (get-root) "input.txt")))})} + (assoc local :get-root (constantly @temporary-root) + :services {:read-text #(fs/readFileSync % "utf8")}))))) + (t/is (= "ready:0:saved input\n" @(:output local))))) + +(t/deftest ^:async malformed-and-unsupported-requests-fail-before-output-allocation + (let [valid (resolved "run" {:build {:kind :existing :id "ready"}})] + (doseq [[received field handlers] + [[(dissoc valid :console) "console" nil] + [(update valid :options dissoc :warmups) "warmups" nil] + [(assoc (update valid :options dissoc :warmups) + :console {:level :warn :color :always}) "warmups" nil] + [(assoc-in valid [:options :extra] true) "extra" nil] + [(assoc-in valid [:console :color] :invalid) "color" nil] + [(assoc-in valid [:console :stdout] #js {:write #(throw (js/Error. "untrusted stream"))}) + "stdout" nil] + [valid "unsupported-command" {}]]] + (let [local (capture) + output (path/join @temporary-root (str (random-uuid))) + before (listeners) + handler (fn [_ _] + (fs/mkdirSync output) + {:exit-code 0 :errors [] :log-path nil})] + (t/is (= 1 (await (execution/execute received (or handlers {"run" handler}) + (assoc local :get-root #(fs/mkdirSync output)))))) + (t/is (str/includes? @(:errors local) field)) + (t/is (not (str/includes? @(:errors local) "\u001b["))) + (t/is (not (fs/existsSync output))) + (t/is (= before (listeners))))))) + +(t/deftest ^:async received-command-policies-reject-before-output-allocation + (doseq [[command options message] + [["build" {:features ["stats"]} "diagnostic-features"] + ["run" {:build {:kind :prepare :features ["profile"]}} "diagnostic-features"] + ["ab" {:features ["profile-raf"] :dry-run true} "diagnostic-features"] + ["ab" {:features ["cache"]} "confirmation-required"] + ["ab" {:features ["cache"] :filter " "} "confirmation-required"] + ["ab" {:features ["a" "b"] :max-features 1 :dry-run true} "too-many-features"]]] + (let [local (capture) + output (path/join @temporary-root (str (random-uuid))) + request (request/check (resolved command options))] + (t/is (= 1 (await (execution/execute + request + {command (fn [_ _] + (fs/mkdirSync output) + {:exit-code 0 :errors [] :log-path nil})} + (assoc local :get-root #(fs/mkdirSync output)))))) + (t/is (str/includes? @(:errors local) message)) + (t/is (not (fs/existsSync output)))))) + +(t/deftest ^:async gpu-policy-uses-checked-console-and-dry-run-ab-needs-no-gpu + (let [env (.-env js/process) + old (.-PENPOT_GPU env)] + (try + (js-delete env "PENPOT_GPU") + (doseq [[command options expected] + [["run" {:build {:kind :existing :id "ready"} :require-gpu true} 1] + ["ab" {:features ["cache"] :confirm true :require-gpu true} 1] + ["ab" {:features ["cache"] :dry-run true :require-gpu true} 0]]] + (let [local (capture) + request (assoc (resolved command options) :console {:level :warn :color :always})] + (t/is (= expected + (await (execution/execute + request {command (fn [_ _] {:exit-code 0 :errors [] :log-path nil + :report "dry plan"})} local)))) + (if (zero? expected) + (t/is (= "dry plan\n" @(:output local))) + (do + (t/is (str/includes? @(:errors local) "PENPOT_GPU=1")) + (t/is (str/includes? @(:errors local) "\u001b[")))))) + (finally + (if (nil? old) (js-delete env "PENPOT_GPU") (set! (.-PENPOT_GPU env) old)))))) + +(t/deftest ^:async thrown-and-rejected-failures-use-checked-console-and-release-listeners + (doseq [[handler expected] [[(fn [_ _] (throw (js/Error. "failed work"))) + "✖ failed work\n"] + [(^:async fn [_ _] + (await (timers/setTimeout 1)) + (throw (js/Error. "failed work"))) + "✖ failed work\n"] + [(^:async fn [_ _] + (await (timers/setTimeout 1)) + (throw (errors/fatal-error "failed work" {} (js/Error. "storage")))) + "✖ failed work\n fatal true\n"]]] + (let [local (capture) + before (listeners) + request (assoc (resolved "help" {:topic nil}) :console {:level :warn :color :never})] + (t/is (= 1 (await (execution/execute + request {"help" (fn [context options] + (log/info "hidden progress") + (handler context options))} local)))) + (t/is (= expected @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest ^:async final-publication-uses-the-outcomes-log-and-numeric-status + (let [local (capture) + file (path/join @temporary-root "command.log")] + (t/is (= 7 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 7 :errors [] :log-path file :report "saved report"})} + local)))) + (t/is (= "saved report\n" (fs/readFileSync file "utf8"))) + (t/is (= "saved report\n" @(:output local))) + (let [silent (capture)] + (t/is (= 0 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path nil})} silent)))) + (t/is (= "" @(:output silent)))))) + +(t/deftest ^:async final-errors-share-the-command-log-and-print-one-tail + (let [local (capture) + file (path/join @temporary-root "failures.log")] + (fs/writeFileSync file "saved evidence\n" "utf8") + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] + {:exit-code 1 :log-path file + :errors [(js/Error. "first failure") (js/Error. "second failure")]})} + local)))) + (t/is (= 2 (count (re-seq #"✖" @(:errors local))))) + (t/is (= 1 (count (re-seq #" tail\n" @(:errors local))))) + (t/is (str/includes? @(:errors local) "saved evidence")) + (t/is (str/includes? (fs/readFileSync file "utf8") "second failure")))) + +(t/deftest ^:async publication-failure-fails-the-command-and-preserves-interruption-status + (doseq [[code expected] [[0 1] [130 130] [143 143]]] + (let [local (capture) + written (atom []) + broken #js {:write (fn [text] + (swap! written conj text) + (throw (js/Error. "stdout closed")))}] + (t/is (= expected (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code code :errors [] :log-path nil :report "result"})} + (assoc local :stdout broken))))) + (t/is (= ["result\n"] @written)) + (t/is (= "✖ stdout closed\n" @(:errors local))))) + (let [local (capture)] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path @temporary-root :report "result"})} + local)))) + (t/is (str/includes? @(:errors local) "cannot append benchmark log")) + (t/is (str/includes? @(:errors local) "Could not append log")))) + +(t/deftest ^:async real-signals-retain-partial-evidence-and-fatal-errors + (doseq [[first-signal second-signal code] [["SIGINT" "SIGTERM" 130] ["SIGTERM" "SIGINT" 143]]] + (let [local (capture) + before (listeners) + file (path/join @temporary-root first-signal) + handler (fn [{:keys [signal]} _] + (js/Promise. + (fn [resolve reject] + (.addEventListener + signal "abort" + (fn [] + (-> (timers/setTimeout 1) + (.then (fn [] + (t/is (.emit js/process second-signal)) + (fs/writeFileSync file "partial evidence" "utf8") + (resolve {:exit-code 1 :log-path nil :report "partial report" + :errors [(js/Error. "aborted work") + (errors/fatal-error "fatal storage" {} (js/Error. "disk"))]}))) + (.catch reject))) + #js {:once true}) + (.kill js/process (.-pid js/process) first-signal))))] + (t/is (= code (await (within (execution/execute (resolved "help" {:topic nil}) + {"help" handler} local))))) + (t/is (= "partial evidence" (fs/readFileSync file "utf8"))) + (t/is (= "partial report\n" @(:output local))) + (t/is (= "✖ fatal storage\n fatal true\n" @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest failure-adapters-preserve-usage-validation-and-decoder-diagnostics + (doseq [[get-cause message] + [[#(cli/parse-resolve ["purge"]) "Usage: runner purge"] + [#(request/check (update (resolved "run" {:build {:kind :existing :id "ready"}}) + :options dissoc :warmups)) "warmups"] + [#(transit/decode-str "invalid transit") nil] + [#((createRequire (path/join @temporary-root "entry.cjs")) + (path/join @temporary-root "missing.cjs")) "missing.cjs"]]] + (let [local (capture) + cause (try + (get-cause) + (t/is false "expected a boundary failure") + (catch :default cause cause))] + (t/is (= 1 (execution/fail cause {} local))) + (t/is (str/includes? @(:errors local) (or message (errors/message-of cause)))) + (t/is (= 1 (count (re-seq #"✖" @(:errors local)))))))) + +(t/deftest ^:async failure-text-escapes-controls-and-survives-a-throwing-printer + (let [local (capture) + hostile (reify IPrintWithWriter + (-pr-writer [_ _ _] (throw (js/Error. "cannot print path")))) + failure (ex-info (str "work failed\n\u001b" (apply str (repeat 2000 "x"))) + {:path hostile :reason "reason\u0007\u007f\u0085" + :mismatches [{:field [:metric (keyword "metric" "unsafe\u001b")]}]})] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] (throw failure))} local)))) + (t/is (str/includes? @(:errors local) "work failed")) + (t/is (str/includes? @(:errors local) "unrenderable value")) + (t/is (str/includes? @(:errors local) "\\u001b")) + (t/is (str/includes? @(:errors local) "\\u0007")) + (t/is (every? #(<= (count %) 1100) (str/split-lines @(:errors local)))) + (t/is (every? #(let [code (.charCodeAt % 0)] + (or (= code 10) (and (>= code 32) (or (< code 127) (> code 159))))) + (seq @(:errors local)))))) + +(t/deftest ^:async error-reporting-failure-uses-a-bounded-fallback + (doseq [[code expected] [[1 1] [130 130]]] + (let [local (capture) + fallback (atom []) + broken #js {:write (fn [text] + (if (str/starts-with? text "Could not report") + (swap! fallback conj text) + (throw (js/Error. (str "\u001b" (apply str (repeat 1100 "😀")))))))}] + (t/is (= expected (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code code :log-path nil :errors [(js/Error. "work failed")]})} + (assoc local :stderr broken))))) + (t/is (= 1 (count @fallback))) + (t/is (< (count (first @fallback)) 1100)) + (t/is (str/includes? (first @fallback) "\\u001b")) + (t/is (not (re-find #"[\uD800-\uDBFF](?![\uDC00-\uDFFF])" (first @fallback)))))) + (let [broken #js {:write #(throw (js/Error. "closed"))}] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path nil :report "result"})} + {:stdout broken :stderr broken})))))) diff --git a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs new file mode 100644 index 0000000000..67bb5aff2d --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs @@ -0,0 +1,1363 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.feature-comparison-test + (:require + ["fs" :as fs] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.cli.execution :as execution] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.build :as build] + [benches.render-wasm.node.runner.feature-comparison :as feature-comparison] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-feature-plan-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- frontend-root [] (path/join @temporary-root "frontend")) +(defn- renderer-root [] (path/join @temporary-root "render-wasm")) + +(defn- write-cargo-fixture + [] + (fs/mkdirSync (frontend-root)) + (fs/mkdirSync (renderer-root)) + (fs/writeFileSync (path/join (renderer-root) "Cargo.toml") + (str "[package]\nname = 'feature-plan-fixture'\nversion = '0.1.0'\nedition = '2021'\n" + "[lib]\npath = 'lib.rs'\n[features]\ndefault = []\na = []\nb = ['a']\n")) + (fs/writeFileSync (path/join (renderer-root) "lib.rs") "") + (fs/writeFileSync (path/join (renderer-root) "Cargo.lock") + "version = 4\n[[package]]\nname = 'feature-plan-fixture'\nversion = '0.1.0'\n")) + +(defn- planning-options + [] + (:options (request/apply-defaults + {:command "ab" + :options {:features ["b" "a"] :filter "rects/load" :seed 42 :dry-run true + :warmups 0 :repetitions 1 :timeout-ms 2000 + :build-timeout-ms 10000 :startup-timeout-ms 3000}}))) + +(defn- metadata-output + [definitions] + {:stdout (js/JSON.stringify + (clj->js {"packages" [{"manifest_path" (path/join (frontend-root) "Cargo.toml") + "features" {"default" ["wrong-package"]}} + {"manifest_path" (path/join (renderer-root) "Cargo.toml") + "features" definitions}]}))}) + +(defn- error-message + [promise] + (.then promise (fn [_] nil) (fn [cause] (ex-message cause)))) + +(t/deftest one-feature-has-an-explicit-off-and-on-selection + (t/is (= {:features ["a"] + :configurations [{:slug "0" :selection {:default-features? false :features []}} + {:slug "1" :selection {:default-features? false :features ["a"]}}] + :edges [{:from "0" :to "1" :feature "a"}]} + (feature-plan/plan ["a"] {:filter "rects/load"})))) + +(t/deftest two-feature-order-and-all-edges-are-msb-first + (t/is (= {:features ["a" "b"] + :configurations [{:slug "00" :selection {:default-features? false :features []}} + {:slug "01" :selection {:default-features? false :features ["b"]}} + {:slug "11" :selection {:default-features? false :features ["a" "b"]}} + {:slug "10" :selection {:default-features? false :features ["a"]}}] + :edges [{:from "00" :to "10" :feature "a"} + {:from "00" :to "01" :feature "b"} + {:from "01" :to "11" :feature "a"} + {:from "10" :to "11" :feature "b"}]} + (feature-plan/plan ["b" "a" "a"] {:filter "rects/load"})))) + +(t/deftest three-feature-order-and-complete-graph + (let [{:keys [configurations edges]} (feature-plan/plan ["c" "a" "b"] {:filter "rects/load"}) + by-slug (into {} (map (juxt :slug #(set (get-in % [:selection :features])))) configurations)] + (t/is (= ["000" "001" "011" "010" "110" "111" "101" "100"] (mapv :slug configurations))) + (t/is (= 12 (count (set edges)))) + (doseq [{:keys [from to feature]} edges] + (t/is (not (contains? (by-slug from) feature))) + (t/is (= (conj (by-slug from) feature) (by-slug to)))))) + +(t/deftest upper-limit-still-emits-every-edge-once + (let [{:keys [configurations edges]} + (feature-plan/plan ["a" "b" "c" "d" "e" "f"] {:max-features 6 :confirm true})] + (t/is (= 64 (count (set configurations)))) + (t/is (= 192 (count (set edges)))) + (t/is (every? false? (map #(get-in % [:selection :default-features?]) configurations))))) + +(t/deftest invalid-inputs-fail-before-planning + (doseq [features [nil '("a") "a" [nil] [1] [""] ["a b"] ["a/b"] ["a.b"] [" a"]]] + (t/is (thrown? js/Error (feature-plan/plan features {:dry-run true})) (pr-str features))) + (t/is (thrown-with-msg? js/Error #"at least one" (feature-plan/plan [] {:dry-run true}))) + (doseq [limit [nil 0 7 -1 1.5 "3" js/NaN js/Infinity]] + (t/is (thrown-with-msg? js/Error #"integer from 1 to 6" + (feature-plan/plan ["a"] {:max-features limit :dry-run true})))) + (doseq [opts [{:confirm nil} {:confirm 1} {:dry-run nil} {:dry-run "true"} {:filter 1}]] + (t/is (thrown-with-msg? js/Error #"invalid feature planning options" + (feature-plan/plan ["a"] opts)))) + (t/is (thrown-with-msg? js/Error #"exceed max-features" + (feature-plan/plan ["a" "b" "c" "d"] {:dry-run true})))) + +(t/deftest confirmation-applies-to-execution-only + (doseq [filter [nil "" " "]] + (t/is (thrown-with-msg? js/Error #"requires --confirm" (feature-plan/plan ["a"] {:filter filter}))) + (t/is (= 2 (count (:configurations (feature-plan/plan ["a"] {:filter filter :dry-run true})))))) + (t/is (= 8 (count (:configurations (feature-plan/plan ["a" "b" "c"] {:filter "rects"}))))) + (t/is (thrown-with-msg? js/Error #"requires --confirm" + (feature-plan/plan ["a" "b" "c" "d"] {:max-features 4 :filter "rects"}))) + (doseq [opts [{:confirm true} {:dry-run true}]] + (t/is (= 16 (count (:configurations + (feature-plan/plan ["a" "b" "c" "d"] (assoc opts :max-features 4)))))))) + +(t/deftest manifest-validation-leaves-activation-to-cargo + (t/is (= ["a" "b"] (feature-plan/check-manifest ["b" "a"] + {"default" [] "a" [] "b" ["a" "dep:library"]}))) + (t/is (= ["a"] (feature-plan/check-manifest ["a"] {"a" []}))) + (t/is (thrown-with-msg? js/Error #"unknown requested" + (feature-plan/check-manifest ["missing"] {"default" [] "a" []}))) + (t/is (thrown-with-msg? js/Error #"empty declared Cargo defaults" + (feature-plan/check-manifest ["a"] {"default" ["a"] "a" []}))) + (doseq [definitions [nil [] {"a" "b"} {:a []} {"a" [1]}]] + (t/is (thrown-with-msg? js/Error #"invalid Cargo feature definitions" + (feature-plan/check-manifest ["a"] definitions)))) + (doseq [feature ["stats" "profile" "profile-macros" "profile-raf"]] + (t/is (thrown-with-msg? js/Error #"diagnostic Cargo features" + (feature-plan/plan [feature] {:dry-run true}))) + (t/is (thrown-with-msg? js/Error #"diagnostic Cargo features" + (feature-plan/check-manifest [feature] {"default" [] feature []})))) + ;; A declared alias is allowed here. Actual Cargo-reported diagnostics are + ;; rejected later by build validation, not by a second activation resolver. + (t/is (= ["alias"] (feature-plan/check-manifest ["alias"] {"default" [] "alias" ["stats"]})))) + +(t/deftest ^:async real-offline-metadata-and-collection-create-no-output + (write-cargo-fixture) + (let [output (path/join @temporary-root "new" "comparison") + opts (assoc (planning-options) :output-dir output) + before (vec (fs/readdirSync (renderer-root))) + ready (await (feature-comparison/prepare (frontend-root) opts nil))] + (t/is (= ["a" "b"] (get-in ready [:plan :features]))) + (t/is (= (cases/collect-cases {:master-seed 42 :filter "rects/load"}) (:cases ready))) + (t/is (= (assoc opts :features ["a" "b"]) + (:options ready))) + (t/is (= output (:output-dir ready))) + (t/is (not (fs/existsSync (path/join @temporary-root "new")))) + (t/is (= before (vec (fs/readdirSync (renderer-root))))) + (t/is (empty? (fs/readdirSync (frontend-root)))) + (let [text (feature-comparison/format-plan ready)] + (doseq [part ["a, b" "00" "01" "11" "10" "00 -> 10: a" "Cases: 1" + "0 warmups, 1 measured" "case 2000, build 10000, startup 3000"]] + (t/is (str/includes? text part)))))) + +(t/deftest ^:async metadata-selects-the-package-by-manifest-not-array-position + (write-cargo-fixture) + (fs/writeFileSync (path/join (frontend-root) "Cargo.toml") "decoy") + (await + (mock/with-mocks* + {process/run (fn [request] + (t/is (= "cargo" (:command request))) + (t/is (= ["metadata" "--offline" "--locked" "--no-deps" "--format-version" "1"] + (:args request))) + (t/is (= (renderer-root) (:cwd request))) + (t/is (not (contains? request :env))) + (js/Promise.resolve (metadata-output {"default" [] "a" [] "b" ["a"]})))} + (let [ready (await (feature-comparison/prepare (frontend-root) (planning-options) nil))] + (t/is (= ["a" "b"] (get-in ready [:plan :features]))))))) + +(t/deftest ^:async collected-custom-case-data-and-order-survive-planning + (write-cargo-fixture) + (decl/register-scene! {:id :feature-plan-fixture :ns "feature-comparison-test" + :description "Planning data fixture" + :params-schema [:map [:count :int]] + :build (fn [_] (throw (js/Error. "planning must not construct a scene")))}) + (doseq [id [:feature-plan-fixture/second :feature-plan-fixture/first]] + (decl/register-case! {:id id :scene :feature-plan-fixture :ns "feature-comparison-test" + :params {:count 17} :view {:scale 2 :x 10 :y -5} :context :reuse + :preparation {:reset :outside-timing :data [1 2]} + :run! (fn [_] (throw (js/Error. "planning must not run an operation")))})) + (try + (let [ready (await (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :filter "feature-plan-fixture/") nil)) + data (:cases ready)] + (t/is (= [:feature-plan-fixture/second :feature-plan-fixture/first] (mapv :id data))) + (t/is (= (cases/collect-cases {:master-seed 42 :filter "feature-plan-fixture/"}) data)) + (t/is (= {:params {:count 17} :view {:scale 2 :x 10 :y -5} :context :reuse + :preparation {:reset :outside-timing :data [1 2]}} + (select-keys (first data) [:params :view :context :preparation])))) + (finally + (decl/unregister-case! :feature-plan-fixture/second) + (decl/unregister-case! :feature-plan-fixture/first) + (decl/unregister-scene! :feature-plan-fixture)))) + +(t/deftest ^:async planning-propagates-manifest-and-collector-errors + (write-cargo-fixture) + (fs/writeFileSync (path/join (frontend-root) "Cargo.toml") "decoy") + (doseq [[definitions message] [[{"default" [] "a" []} "unknown requested Cargo features"] + [{"default" ["a"] "a" [] "b" []} "ab requires empty declared Cargo defaults"]]] + (await (mock/with-mocks* + {process/run (fn [_] (js/Promise.resolve (metadata-output definitions)))} + (t/is (= message (await (error-message + (feature-comparison/prepare (frontend-root) (planning-options) nil)))))))) + (t/is (= "no benchmark cases match the selection" + (await (error-message (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :filter "missing-case") nil)))))) + +(t/deftest ^:async invalid-input-and-existing-output-fail-before-cargo + (write-cargo-fixture) + (await + (mock/with-mocks* + {process/run (fn [_] (throw (js/Error. "unexpected Cargo invocation")))} + (t/is (= "output directory already exists" + (await (error-message (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :output-dir (frontend-root)) nil))))) + (t/is (= "ab requires at least one requested feature" + (await (error-message (feature-comparison/prepare (frontend-root) + (assoc (planning-options) :features []) nil)))))))) + +(t/deftest ^:async cancellation-reaches-the-process-boundary + (write-cargo-fixture) + (let [controller (js/AbortController.)] + (.abort controller "SIGINT") + (t/is (str/includes? (await (error-message + (feature-comparison/prepare (frontend-root) (planning-options) (.-signal controller)))) + "cancelled")))) + +(defn- planned-record + [] + (let [plan (feature-plan/plan ["a" "b"] {:dry-run true})] + {:type :feature-comparison :comparison-id "ab-id" + :started-at "start" :updated-at "start" :status :planned + :options (planning-options) + :permissions {:allow-build-differences false :allow-scene-differences true :allow-all-differences false} + :cases (cases/collect-cases {:master-seed 42 :filter "rects/load"}) + :features (:features plan) + :configurations (mapv #(assoc % :status :planned) (:configurations plan)) + :edges (mapv #(assoc % :status :planned) (:edges plan)) + :warnings ["Timing values do not determine success."]})) + +(defn- completed-configuration + [configuration] + (assoc configuration + :status :completed + :effective-features ["transitive"] + :build {:id "build-id" :manifest-path "/build.transit" :log-path "/build.log"} + :run {:run-id "run-id" :record-path "/run.transit" :log-path "/run.log" + :exit-code 0})) + +(defn- completed-record + [] + (let [record (planned-record)] + (assoc record + :status :completed :updated-at "end" :ended-at "end" + :configurations (mapv completed-configuration (:configurations record)) + :edges (mapv #(assoc % :status :completed :path "/comparison.transit") + (:edges record))))) + +(defn- terminal-record + [status] + (let [record (planned-record)] + (assoc record + :status status :updated-at "end" :ended-at "end" + :configurations + (mapv (fn [[index configuration]] + (if (zero? index) + (assoc (completed-configuration configuration) + :status :failed :failure {:phase "run" :message "run interrupted"} + :reason "SIGINT") + (assoc configuration :status :unattempted + :reason "stopped after failure"))) + (map-indexed vector (:configurations record))) + :edges + (mapv (fn [[index edge]] + (if (zero? index) + (assoc edge :status :failed :path "/comparison.transit" + :failure {:phase "comparison" :message "partial"}) + (assoc edge :status :unattempted + :reason "stopped after failure"))) + (map-indexed vector (:edges record)))))) + +(defn- running-record + [] + (-> (planned-record) + (assoc :status :running) + (assoc-in [:configurations 0 :status] :building) + (assoc-in [:configurations 0 :build] + {:id "build-id" :manifest-path "/build.transit" :log-path "/build.log"}))) + +(t/deftest feature-comparison-record-round-trips-plan-and-partial-evidence + (let [record (terminal-record :interrupted)] + (t/is (= (planned-record) (result/decode (result/encode (planned-record))))) + (t/is (= record (result/decode (result/encode record)))))) + +(t/deftest feature-comparison-progress-must-match-saved-evidence + (doseq [record [(planned-record) (running-record) (completed-record) + (terminal-record :failed) (terminal-record :interrupted)]] + (t/is (= record (result/check-record record)))) + (let [completed (completed-record) + failed (terminal-record :failed) + running (running-record)] + (doseq [[record message] + [[(assoc (planned-record) :status :completed) + #"terminal feature comparison needs an end time"] + [(assoc-in completed [:configurations 0 :status] :planned) + #"still has active configurations"] + [(assoc-in completed [:edges 0 :status] :planned) #"still has active edges"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :build) configurations))) + #"configuration needs a build"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :effective-features) configurations))) + #"configuration needs effective features"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :run) configurations))) + #"configuration needs a run"] + [(update completed :configurations + (fn [configurations] + (mapv #(update % :run dissoc :exit-code) configurations))) + #"run exit code"] + [(assoc-in completed [:configurations 0 :run :exit-code] 1) + #"successful run"] + [(update-in completed [:edges 0] dissoc :path) + #"edge needs a report path"] + [(update-in failed [:configurations 0] dissoc :failure :reason) + #"configuration needs a failure or reason"] + [(update-in failed [:configurations 1] dissoc :reason) + #"configuration needs a reason"] + [(update-in failed [:edges 0] dissoc :failure :reason) + #"edge needs a failure"] + [(update-in failed [:edges 1] dissoc :reason) + #"edge needs a reason"] + [(assoc running :ended-at "end") #"unfinished feature comparison has an end time"] + [(assoc-in (planned-record) [:configurations 0 :status] :building) + #"started configurations"] + [(assoc-in (planned-record) [:edges 0 :status] :completed) + #"started edges"] + [(assoc (planned-record) :features ["b" "a"]) + #"features disagree with normalized selection"] + [(update (planned-record) :configurations + #(assoc % 1 (first %))) + #"configurations disagree with planned features"] + [(update (planned-record) :configurations pop) + #"configurations disagree with planned features"] + [(assoc-in (planned-record) [:edges 0 :to] "11") + #"edges disagree with planned configurations"] + [(update (planned-record) :cases conj (first (:cases (planned-record)))) + #"duplicate feature comparison case"]]] + (t/is (thrown-with-msg? js/Error message (result/check-record record)) + (pr-str message))))) + +(t/deftest feature-comparison-current-state-shapes-are-checked + (doseq [[location value] [[[:comparison-id] 1] [[:updated-at] nil] [[:status] :ready] + [[:ended-at] nil] [[:features] [:a]] [[:cases] [{}]] + [[:permissions :allow-scene-differences] nil] + [[:configurations 0 :status] :interrupted] + [[:configurations 0 :selection :features] "a"] + [[:configurations 0 :selection :default-features?] nil] + [[:configurations 0 :build] {:id "id"}] + [[:configurations 0 :run] {:run-id "id" :record-path "p" :log-path "l" :exit-code "0"}] + [[:configurations 0 :failure] "error"] + [[:configurations 0 :effective-features] [:a]] + [[:edges 0 :status] :running] [[:edges 0 :path] 1] [[:warnings] [nil]]]] + (t/is (thrown-with-msg? js/Error #"invalid schema:feature-comparison" + (result/check-record (assoc-in (planned-record) location value))) + (pr-str location)))) + +(defn- create-frontend-root + [] + (fs/mkdirSync (frontend-root) #js {:recursive true})) + +(defn- comparison-output-dir + [] + (path/join @temporary-root "output" "comparison")) + +(defn- fixture-case + [] + {:id :fixture/comparison-case}) + +(defn- comparison-prepared + ([features] (comparison-prepared features {})) + ([features overrides] + (let [plan (feature-plan/plan features {:dry-run true}) + output (or (:output-dir overrides) (comparison-output-dir)) + opts (merge {:default-features false + :filter "fixture/" + :seed 42 + :warmups 0 + :repetitions 1 + :timeout-ms 1000 + :build-timeout-ms 5000 + :startup-timeout-ms 1000 + :allow-build-differences false + :allow-scene-differences false + :allow-all-differences false} + overrides + {:features (:features plan) + :output-dir output})] + {:plan plan + :cases [(fixture-case)] + :output-dir output + :options opts}))) + +(defn- fake-build + [selection extra-features] + {:id (str (random-uuid)) + :manifest-path (path/join @temporary-root "builds" "build.transit") + :log-path (path/join @temporary-root "builds" "build.log") + :manifest {:functional {:mode :release + :features (vec (concat (:features selection) extra-features))} + :provenance {:feature-selection selection}}}) + +(defn- build-service + [{:keys [extra-features fail-on inspect build-fn] + :or {extra-features (constantly []) + fail-on (constantly false) + build-fn fake-build}}] + (fn [request] + (when inspect (inspect request)) + (let [selection (get request :selection) + build (build-fn selection (extra-features selection))] + (when-let [on-created (:on-created request)] + (on-created build)) + (if (fail-on selection) + (js/Promise.reject (js/Error. "fixture build failure")) + (js/Promise.resolve build))))) + +(defn- metric-entry + [value] + {:command :fixture/work :metrics {:command/elapsed [value]}}) + +(defn- fingerprint + [character] + {:algorithm :sha-256 + :digest (apply str (repeat 64 character))}) + +(defn- preparation + [case-id digest] + (let [id (str (random-uuid))] + {:preparation-id id :case-id case-id :warmup? false + :fingerprint (fingerprint digest) + :effective-graphics {:backend :fixture} + :diagnostics {} + :phases [{:phase :preparation :preparation-id id :outcome :completed :events []}]})) + +(defn- fake-run-record + [selected-build run-home cases options {:keys [kind digest metric-value] + :or {kind :completed digest "a" metric-value 10}}] + (let [run (result/create-run + {:run-id (:run-id run-home) + :started-at (.toISOString (js/Date.)) + :plan {:cases cases + :warmups 0 + :repetitions 1 + :timeout-policy {:case-ms (:timeout-ms options)}} + :metadata {:git {:sha nil :dirty nil} + :build {:functional (get-in selected-build [:manifest :functional]) + :provenance (get-in selected-build [:manifest :provenance])} + :environment {} + :scored? true}}) + failed? (= kind :failed) + run (if (= kind :zero-attempts) + run + (reduce + (fn [run case] + (let [case-id (:id case)] + (if failed? + (result/record-attempt + run + {:case-id case-id + :preparation-id nil + :warmup? false + :outcome :failed + :phases [] + :failure {:phase "main" :message "fixture failure"}}) + (let [prep (preparation case-id digest)] + (-> run + (result/record-preparation prep) + (result/record-attempt + {:case-id case-id + :preparation-id (:preparation-id prep) + :warmup? false + :outcome :completed + :phases [{:phase :main :preparation-id (:preparation-id prep) + :outcome :completed :events [(metric-entry metric-value)]}]})))))) + run + cases)) + termination (case kind + :failed {:reason :failed :failure {:phase "main" :message "fixture failure"}} + :zero-attempts {:reason :incomplete} + :interrupted {:reason :interrupted :signal "SIGINT"} + {:reason :completed}) + run (if (= :completed (:reason termination)) + (reduce #(result/record-case-outcome %1 (:id %2) {:outcome :completed}) run cases) + run)] + (result/finish-run run (.toISOString (js/Date.)) termination))) + +(defn- execute-service + [spec-or-fn] + (fn [_ run-home selected-build cases options _signal] + (let [spec (if (fn? spec-or-fn) (spec-or-fn selected-build) spec-or-fn) + run (fake-run-record selected-build run-home cases options spec) + exit-code (case (:kind spec) + :failed 1 + :zero-attempts 1 + :interrupted 130 + 0)] + (files/atomic-write (:record-path run-home) + (result/encode (result/check-record run))) + (js/Promise.resolve {:run run :home run-home :exit-code exit-code :errors []})))) + +(defn- comparison-services + [{:keys [prepare-build collect-catalog execute-run]}] + {:prepare-build prepare-build + :collect-catalog (or collect-catalog (fn [_ _] [(fixture-case)])) + :execute-run execute-run}) + +(defn- saved-comparison + [outcome] + (result/decode (fs/readFileSync (get-in outcome [:home :record-path]) "utf8"))) + +(t/deftest ^:async invalid-build-selection-or-features-stops-later-configurations + (create-frontend-root) + (doseq [[field value] [[[:provenance :feature-selection] {:default-features? false :features ["other"]}] + [[:functional :features] #{"a"}] + [[:functional :features] [:invalid]]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:build-fn (fn [selection extra] + (assoc-in (fake-build selection extra) + (into [:manifest] field) value))}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status record))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= "build" (get-in record [:configurations 0 :failure :phase]))) + (t/is (str/includes? (get-in record [:configurations 0 :failure :message]) + "disagrees with the planned selection"))))) + +(t/deftest ^:async operational-failures-save-the-stage-and-stop-later-configurations + (create-frontend-root) + (doseq [[boundary phase message] [[:catalog "catalog" "catalog rejected"] + [:run-home "run-home" "already exists"] + [:run "run" "run rejected"] + [:missing-run "run" "run executor returned no record"]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + base (comparison-services {:prepare-build (build-service {}) :execute-run (execute-service {})}) + services (case boundary + :catalog (assoc base :collect-catalog (fn [_ _] (throw (js/Error. "catalog rejected")))) + :run-home (assoc base :prepare-build + (build-service {:inspect (fn [_] + (fs/mkdirSync (path/join (:output-dir prepared) "configs" "0") + #js {:recursive true}))})) + :run (assoc base :execute-run (fn [& _] (js/Promise.reject (js/Error. "run rejected")))) + :missing-run (assoc base :execute-run (fn [& _] (js/Promise.resolve {:exit-code 1})))) + outcome (await (feature-comparison/execute (frontend-root) prepared nil {:services services})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status record))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= phase (get-in record [:configurations 0 :failure :phase]))) + (t/is (str/includes? (get-in record [:configurations 0 :failure :message]) message))))) + +(t/deftest ^:async run-failure-saves-partial-measurements-and-nested-causes + (create-frontend-root) + (let [partial {:slices [{:duration-ms 3}]} + cause (ex-info "run rejected" {:partial partial} + (ex-info "render failed" {:fn "_render" :code 17})) + prepared (comparison-prepared ["a"] {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run (fn [& _] (js/Promise.reject cause))})})) + failure (get-in (saved-comparison outcome) [:configurations 0 :failure])] + (t/is (= {:phase "run" + :message "run rejected" + :partial partial} + (select-keys failure [:phase :message :partial]))) + (t/is (some #(= {:message "render failed" :fn "_render" :code 17} %) + (:cause failure))))) + +(t/deftest ^:async unavailable-or-mismatched-run-evidence-stops-later-configurations + (create-frontend-root) + (doseq [[kind message] [[:missing "run record is missing"] + [:mismatch "saved run identity disagrees with execution result"]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + execute (execute-service {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (^:async fn [root home build cases options signal] + (let [response (await (execute root home build cases options signal))] + (if (= kind :missing) + (fs/unlinkSync (:record-path home)) + (files/atomic-write (:record-path home) + (result/encode (assoc (:run response) :run-id (str (random-uuid)))))) + response))})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= "run-evidence" (get-in record [:configurations 0 :failure :phase]))) + (t/is (str/includes? (get-in record [:configurations 0 :failure :message]) message))))) + +(t/deftest ^:async returned-storage-failure-wins-over-readable-evidence-and-executor-errors + (create-frontend-root) + (doseq [[signal code] [[nil 1] ["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + execute (execute-service {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (^:async fn [root home build cases options signal-object] + (let [response (await (execute root home build cases options signal-object))] + (when signal (.abort controller signal)) + (assoc response :exit-code 1 + :errors [(js/Error. "executor failure") + (errors/fatal-error "recovered write failure" {} nil)])))})})) + record (saved-comparison outcome) + run (result/decode (fs/readFileSync (get-in record [:configurations 0 :run :record-path]) "utf8"))] + (t/is (= code (:exit-code outcome))) + (t/is (= (if signal :interrupted :failed) (:status record))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= "run-evidence" (get-in record [:configurations 0 :failure :phase]))) + (t/is (= "recovered write failure" (get-in record [:configurations 0 :failure :message]))) + (t/is (= "recovered write failure" (ex-message (first (:errors outcome)))))))) + +(t/deftest ^:async pre-aborted-comparison-preserves-all-work-as-unattempted + (create-frontend-root) + (doseq [[reason signal code] [["SIGINT" "SIGINT" 130] + ["SIGTERM" "SIGTERM" 143] + [:SIGTERM "SIGTERM" 143] + [(ex-info "stopped" {:signal "SIGTERM"}) "SIGTERM" 143] + [(js/Error. "stopped") "SIGINT" 130]]] + (let [controller (js/AbortController.) + _ (.abort controller reason) + prepared (comparison-prepared ["a" "b"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + outcome (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services {:prepare-build (build-service {}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= code (:exit-code outcome))) + (t/is (= :interrupted (:status record))) + (t/is (every? #(= :unattempted (:status %)) (concat (:configurations record) (:edges record)))) + (t/is (every? #(= (str "interrupted by " signal) (:reason %)) + (concat (:configurations record) (:edges record))))))) + +(t/deftest ^:async interrupted-build-and-catalog-preserve-stage-failures + (create-frontend-root) + (doseq [boundary [:build :catalog] + rejected [false true] + [signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str (random-uuid)))}) + build (build-service {}) + stop (fn [] + (.abort controller signal) + (when rejected (throw (js/Error. "interrupted work")))) + services (comparison-services + {:prepare-build (if (= boundary :build) + (^:async fn [request] + (let [selected (await (build request))] + (stop) + selected)) + build) + :collect-catalog (fn [_ _] + (when (= boundary :catalog) (stop)) + (:cases prepared)) + :execute-run (execute-service {})}) + outcome (await (feature-comparison/execute (frontend-root) prepared (.-signal controller) + {:services services})) + record (saved-comparison outcome)] + (t/is (= code (:exit-code outcome))) + (t/is (= :interrupted (:status record))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations record)))) + (t/is (= signal (get-in record [:configurations 0 :reason]))) + (t/is (= (if rejected [(name boundary)] []) + (mapv :phase (keep :failure (:configurations record))))) + (t/is (every? #(= :unattempted (:status %)) (:edges record)))))) + +(t/deftest ^:async failed-edge-evidence-save-retains-the-analysis-and-persistence-errors + (create-frontend-root) + (doseq [[kind point] [[nil :record] [:zero-attempts :record] [:zero-attempts :directory]]] + (let [prepared (comparison-prepared ["a" "b"] + {:output-dir (path/join @temporary-root (str "edge-save-" (or kind "completed") "-" (name point)))}) + write feature-comparison/write-record] + (await + (mock/with-mocks* + {feature-comparison/write-record + (fn [home record] + (if (= :failed (get-in record [:edges 0 :status])) + (throw (js/Error. "edge evidence unavailable")) + (write home record)))} + (let [response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:inspect (fn [_] + (if (= :directory point) + (fs/writeFileSync (path/join (:output-dir prepared) "comparisons") "blocked") + (fs/mkdirSync + (path/join (:output-dir prepared) "comparisons" "00__10.transit") + #js {:recursive true})))}) + :execute-run (execute-service {:kind kind})})})) + evidence (errors/evidence :comparison (first (:errors response))) + entries (:errors evidence)] + (t/is (= 1 (:exit-code response))) + (if kind + (do + (t/is (= "configuration run made no attempts" (:message (first entries)))) + (t/is (= 1 (count (filter #(= "configuration run made no attempts" (:message %)) entries))))) + (t/is (some #(str/includes? (:message %) "EISDIR") entries))) + (t/is (some #(some (fn [cause] + (if (= :directory point) + (str/includes? (:message cause) "EEXIST") + (= "edge evidence unavailable" (:message cause)))) (:cause %)) entries)) + (t/is (= (:comparison response) (saved-comparison response))))))))) + +(t/deftest ^:async comparison-save-failures-retain-progress-errors-and-signal-codes + (create-frontend-root) + (doseq [[signal point] [[nil :terminal] ["SIGINT" :terminal] ["SIGTERM" :terminal] + ["SIGINT" :configuration] ["SIGTERM" :edge] + ["SIGINT" :run] ["SIGTERM" :run]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str "finish-" signal "-" (name point)))}) + controller (js/AbortController.) + write feature-comparison/write-record + pending (js/Error. "interrupted operation failed") + execute (execute-service (if (= :terminal point) {} {:kind :interrupted}))] + (when (and signal (= :terminal point)) (.abort controller signal)) + (await + (mock/with-mocks* + {feature-comparison/write-record + (fn [home record] + (if (case point + :terminal (contains? #{:completed :interrupted} (:status record)) + :run (some #(= :failed (:status %)) (:configurations record)) + :configuration (some #(= :unattempted (:status %)) (:configurations record)) + :edge (some #(= :unattempted (:status %)) (:edges record))) + (throw (js/Error. "comparison save failed")) + (write home record)))} + (let [response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run (fn [& args] + (.then (apply execute args) + (fn [response] + (if (= :terminal point) + response + (do + (.abort controller signal) + (assoc response :errors [pending]))))))})})) + evidence (errors/evidence :comparison (first (:errors response)))] + (t/is (= (case signal "SIGINT" 130 "SIGTERM" 143 1) (:exit-code response))) + (t/is (= :running (get-in response [:comparison :status]))) + (t/is (= (:comparison response) (saved-comparison response))) + (t/is (some #(some (fn [cause] (= "comparison save failed" (:message cause))) (:cause %)) + (:errors evidence))) + (when (not= :terminal point) + (t/is (= 1 (count (filter #(= "interrupted operation failed" (:message %)) (:errors evidence)))))))))))) + + +(t/deftest ^:async unresolved-run-stops-the-feature-matrix + (create-frontend-root) + (let [prepared (comparison-prepared ["a"]) + execute (execute-service {}) + failure (errors/fatal-error "Command group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run (fn [& args] + (.then (apply execute args) + #(assoc % :exit-code 1 :errors [failure])))})})) + saved (saved-comparison response)] + (t/is (= 1 (:exit-code response))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations saved)))) + (t/is (= [:unattempted] (mapv :status (:edges saved)))))) + + +(t/deftest ^:async unresolved-build-finishes-the-feature-matrix + (create-frontend-root) + (doseq [signal [nil "SIGTERM"]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str "unresolved-build-" signal))}) + controller (js/AbortController.) + prepare (build-service {}) + failure (errors/fatal-error "Compiler group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (fn [request] + (.then (prepare request) + (fn [_] + (when signal (.abort controller signal)) + (throw failure)))) + :execute-run (execute-service {})})})) + saved (saved-comparison response)] + (t/is (= (if signal 143 1) (:exit-code response))) + (t/is (= (if signal :interrupted :failed) (:status saved))) + (t/is (= [:failed :unattempted] (mapv :status (:configurations saved)))) + (t/is (= [:unattempted] (mapv :status (:edges saved)))) + (t/is (true? (get-in saved [:configurations 0 :failure :unresolved-process]))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= (:comparison response) saved))))) + +(t/deftest ^:async unresolved-build-and-failed-comparison-save-retain-both-errors + (create-frontend-root) + (doseq [signal [nil "SIGTERM"]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root (str "unresolved-save-" signal))}) + controller (js/AbortController.) + write feature-comparison/write-record + prepare (build-service {}) + failure (errors/fatal-error "Compiler group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + save-error (js/Error. "comparison save failed")] + (await + (mock/with-mocks* + {feature-comparison/write-record + (fn [home record] + (if (= :failed (get-in record [:configurations 0 :status])) + (throw save-error) + (write home record)))} + (let [response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (fn [request] + (.then (prepare request) + (fn [_] + (when signal (.abort controller signal)) + (throw failure)))) + :execute-run (execute-service {})})})) + evidence (errors/evidence :comparison (first (:errors response)))] + (t/is (= (if signal 143 1) (:exit-code response))) + (t/is (= :running (get-in response [:comparison :status]))) + (t/is (= (:comparison response) (saved-comparison response))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= ["Compiler group may still be running" "cannot persist feature comparison"] + (mapv :message (:errors evidence)))) + (t/is (= 4242 (get-in evidence [:errors 0 :pgid]))) + (t/is (some #(= "comparison save failed" (:message %)) + (get-in evidence [:errors 1 :cause]))))))))) + +(t/deftest ^:async edge-report-storage-failure-preserves-failed-and-unattempted-edges + (create-frontend-root) + (let [prepared (comparison-prepared ["a" "b"]) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:inspect (fn [_] + (fs/mkdirSync + (path/join (:output-dir prepared) "comparisons" "00__10.transit") + #js {:recursive true}))}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status record))) + (t/is (every? #(= :completed (:status %)) (:configurations record))) + (t/is (= [:failed :unattempted :unattempted :unattempted] (mapv :status (:edges record)))) + (t/is (= "comparison-storage" (get-in record [:edges 0 :failure :phase]))) + (t/is (= (get-in outcome [:home :log-path]) + (:log-path (ex-data (first (:errors outcome)))))))) + +(t/deftest ^:async edge-analysis-failures-allow-independent-comparisons-to-complete + (create-frontend-root) + (let [prepared (comparison-prepared ["a" "b"]) + removed (path/join (:output-dir prepared) "configs" "00" "run.transit") + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:inspect (fn [{:keys [selection]}] + (when (= ["b"] (:features selection)) + (fs/unlinkSync removed)))}) + :execute-run (execute-service {})})})) + record (saved-comparison outcome)] + (t/is (= 1 (:exit-code outcome))) + (t/is (every? #(= :completed (:status %)) (:configurations record))) + (t/is (= [:failed :failed :completed :completed] (mapv :status (:edges record)))) + (t/is (= ["comparison" "comparison"] (mapv #(get-in % [:failure :phase]) (take 2 (:edges record))))) + (doseq [edge (drop 2 (:edges record))] + (t/is (= :comparison (:type (result/decode (fs/readFileSync (:path edge) "utf8")))))))) + +(t/deftest ^:async post-run-persistence-failure-retains-the-running-configuration + (create-frontend-root) + (let [prepared (comparison-prepared ["a"]) + path (path/join (:output-dir prepared) "feature-comparison.transit") + prior (str path ".saved") + execute (execute-service {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (^:async fn [root home build cases options signal] + (let [response (await (execute root home build cases options signal))] + (fs/renameSync path prior) + (fs/mkdirSync path) + response))})})) + record (result/decode (fs/readFileSync prior "utf8")) + run (result/decode (fs/readFileSync (get-in record [:configurations 0 :run :record-path]) "utf8"))] + (t/is (= 1 (:exit-code outcome))) + (t/is (= record (:comparison outcome))) + (t/is (= :running (:status record))) + (t/is (= [:running :planned] (mapv :status (:configurations record)))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= (get-in outcome [:home :log-path]) + (:log-path (ex-data (first (:errors outcome)))))))) + +(t/deftest create-home-creates-only-a-fresh-home + (create-frontend-root) + (let [home (feature-comparison/create-home (frontend-root) "comparison-a" nil)] + (t/is (= (path/join (frontend-root) "target" "performance" "render-wasm" + "feature-comparisons" "comparison-a") + (:dir home))) + (t/is (= "" (fs/readFileSync (:log-path home) "utf8")))) + (doseq [id [nil "" "../escape" "a/b" "a\\b"]] + (t/is (thrown-with-msg? js/Error #"invalid feature comparison id" + (feature-comparison/create-home (frontend-root) id nil)))) + (let [existing (path/join @temporary-root "existing")] + (fs/mkdirSync existing) + (t/is (thrown-with-msg? js/Error #"already exists" + (feature-comparison/create-home (frontend-root) + "comparison-b" existing)))) + (let [builds (path/join (frontend-root) "target" "performance" "render-wasm" "builds") + dir (path/join builds "build-a" "nested")] + (fs/mkdirSync (path/join builds "build-a") #js {:recursive true}) + (t/is (thrown-with-msg? js/Error #"inside builds" + (feature-comparison/create-home (frontend-root) + "comparison-c" dir))))) + +(t/deftest ^:async initial-record-is-saved-before-the-first-build + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record (path/join (:output-dir prepared) "feature-comparison.transit") + observed (atom nil) + services (comparison-services + {:prepare-build (build-service + {:inspect (fn [_] + (reset! observed + (result/decode (fs/readFileSync record "utf8")))) + :fail-on (constantly true)}) + :execute-run (execute-service {})}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services services}))] + (t/is (fs/existsSync (get-in result [:home :log-path]))) + (t/is (= :running (:status @observed))) + (t/is (= :building (get-in @observed [:configurations 0 :status]))) + (t/is (nil? (get-in @observed [:configurations 0 :build]))) + (t/is (= :planned (get-in @observed [:configurations 1 :status]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async completed-configurations-and-edge-return-zero + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build + (^:async fn [request] + (let [build (await ((build-service {}) request))] + (fs/mkdirSync (path/dirname (:log-path build)) #js {:recursive true}) + (binding [log/*log-path* (:log-path build)] + (await (timers/setImmediate)) + (log/debug "Build fixture")) + build)) + :execute-run + (^:async fn [root home build cases options signal] + (binding [log/*log-path* (:log-path home)] + (let [response (await ((execute-service {}) root home build cases options signal))] + (log/debug "Run fixture") + response)))})})) + record (:comparison result) + edge (first (:edges record)) + path (:path edge) + comparison (result/decode (fs/readFileSync path "utf8"))] + (t/is (= 0 (:exit-code result))) + (t/is (= :completed (:status record))) + (let [text (fs/readFileSync (get-in result [:home :log-path]) "utf8")] + (doseq [slug ["0" "1"] + phase ["built" "running" "completed"]] + (t/is (str/includes? text (str "Configuration " slug " " phase "\n"))))) + (t/is (= ["a"] (get-in record [:configurations 1 :effective-features]))) + (t/is (= :comparison (:type comparison))) + (t/is (= {:from "0" + :to "1" + :feature "a" + :baseline-features [] + :candidate-features ["a"] + :added ["a"] + :removed []} + (:edge comparison))) + (t/is (= record (result/decode (fs/readFileSync (get-in result [:home :record-path]) + "utf8")))))) + +(t/deftest ^:async attempted-failed-run-continues-and-failed-edge-returns-one + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (execute-service + (fn [build] + (if (seq (get-in build [:manifest :functional :features])) + {} + {:kind :failed})))})})) + record (:comparison result) + edge (first (:edges record))] + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (:status record))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= :completed (get-in record [:configurations 1 :status]))) + (t/is (fs/existsSync (get-in record [:configurations 0 :run :record-path]))) + (t/is (= :failed (:status edge))) + (t/is (not (contains? edge :path))) + (t/is (str/includes? (get-in edge [:failure :message]) "no shared metric")))) + +(t/deftest ^:async zero-attempts-stop-later-configurations + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + base (execute-service {:kind :zero-attempts}) + calls (atom 0) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @calls)) + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (str/includes? (get-in record [:configurations 1 :reason]) "no attempts")) + (t/is (= :failed (get-in record [:edges 0 :status]))))) + +(t/deftest ^:async build-failure-stops-and-missing-endpoint-fails + (create-frontend-root) + (let [prepared (comparison-prepared ["a" "b"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:fail-on #(= ["b"] (:features %))}) + :execute-run (execute-service {})})})) + record (:comparison result) + edge (first (:edges record))] + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (:status record))) + (t/is (= :completed (get-in record [:configurations 0 :status]))) + (t/is (= :failed (get-in record [:configurations 1 :status]))) + (t/is (= :unattempted (get-in record [:configurations 2 :status]))) + (t/is (= :unattempted (get-in record [:configurations 3 :status]))) + (t/is (= :failed (:status edge))) + (t/is (not (contains? edge :path))) + (t/is (str/includes? (get-in edge [:failure :message]) "run record is missing")))) + +(t/deftest ^:async catalog-mismatch-stops-before-a-run-home + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + calls (atom 0) + base (execute-service {}) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :collect-catalog (fn [_ _] [{:id :other/case}]) + :execute-run execute})})) + record (:comparison result)] + (t/is (zero? @calls)) + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= "catalog" (get-in record [:configurations 0 :failure :phase]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))))) + +(t/deftest ^:async catalog-identity-failure-retains-selected-and-loaded-build-ids + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :collect-catalog (fn [selected _] + (identity/check (:id selected) + {:build-id "other-build"})) + :execute-run (execute-service {})})})) + record (saved-comparison outcome) + config (first (:configurations record))] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status config))) + (t/is (= (get-in config [:build :id]) (get-in config [:failure :expected-id]))) + (t/is (= "other-build" (get-in config [:failure :actual-id]))) + (t/is (= ::identity/mismatch (get-in config [:failure :cause 0 :type]))))) + +(t/deftest ^:async unacknowledged-differences-save-a-failed-edge-report + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (execute-service + (fn [build] + (if (seq (get-in build [:manifest :functional :features])) + {:digest "b"} + {:digest "a"})))})})) + record (:comparison result) + edge (first (:edges record)) + saved (result/decode (fs/readFileSync (:path edge) "utf8")) + text (format/format-feature-comparison-failures record (:home result))] + (t/is (= 1 (:exit-code result))) + (t/is (empty? (:errors result))) + (t/is (= :failed (:status record))) + (t/is (= :failed (:status edge))) + (t/is (fs/existsSync (:path edge))) + (t/is (= {:unacknowledged-differences 1} (:failure edge))) + (t/is (= :comparison (:type saved))) + (t/is (= ["a"] (get-in saved [:edge :added]))) + (t/is (str/includes? text "Comparison \"0\" -> \"1\" (feature \"a\"): 1 unacknowledged difference")) + (t/is (str/includes? text (:path edge))) + (t/is (str/includes? text (get-in result [:home :record-path]))) + (t/is (some #(and (= :fingerprint (:field %)) (not (:waived? %))) + (get-in saved [:pairs 0 :mismatches]))))) + +(t/deftest failure-summary-reports-build-reasons-and-comparison-evidence + (let [record (-> (terminal-record :failed) + (update-in [:configurations 0] dissoc :run :reason) + (assoc-in [:configurations 0 :failure] {:phase "catalog" :message "catalog rejected"}) + (assoc-in [:configurations 1 :status] :failed) + (assoc-in [:configurations 1 :reason] "run did not complete") + result/check-record) + home {:record-path "/feature-comparison.transit" :log-path "/feature-comparison.log"} + text (format/format-feature-comparison-failures record home) + edge (first (:edges record))] + (t/is (str/includes? text "Configuration \"00\": \"catalog\": \"catalog rejected\"")) + (t/is (str/includes? text "Configuration \"01\": \"run did not complete\"")) + (t/is (str/includes? text (str "Comparison " (pr-str (:from edge)) " -> " (pr-str (:to edge))))) + (t/is (str/includes? text "\"comparison\": \"partial\"")) + (t/is (str/includes? text "/build.transit")) + (t/is (str/includes? text "/build.log")) + (t/is (str/includes? text "/comparison.transit")) + (t/is (str/includes? text (:record-path home))) + (t/is (str/includes? text (:log-path home))))) + +(t/deftest failure-summary-bounds-and-escapes-details-and-names-omitted-evidence + (let [plan (feature-plan/plan ["a" "b" "c"] {:dry-run true}) + message (str "detail\n\u001b" (apply str (repeat 10000 "x")) "😀") + record (-> (planned-record) + (assoc :status :failed :ended-at "end" + :features (:features plan) + :configurations (mapv (fn [[index configuration]] + (if (< index 4) + (assoc configuration :status :failed + :failure {:phase "run" :message message}) + (completed-configuration configuration))) + (map-indexed vector (:configurations plan))) + :edges (mapv #(assoc % :status :failed :reason message) (:edges plan))) + (assoc-in [:options :features] (:features plan)) + result/check-record) + home {:record-path "/saved\n\u001b.transit" :log-path "/saved.log"} + text (format/format-feature-comparison-failures record home) + lines (str/split-lines text) + detail (second (re-find #"(?m)^ Configuration \"000\": \"run\": (.+)$" text))] + (t/is (= 8 (count (re-seq #"(?m)^ (?:Configuration|Comparison) " text)))) + (t/is (= 4 (count (re-seq #"(?m)^ Comparison " text)))) + (t/is (str/includes? text "8 more failures in the saved record")) + (t/is (str/includes? text "Record \"/saved")) + (t/is (string? detail)) + (t/is (<= (count detail) 1024)) + (t/is (every? #(<= (count %) 1100) lines)) + (t/is (every? #(>= (.charCodeAt % 0) 32) (mapcat seq lines))) + (t/is (< (count text) 12000)))) + +(t/deftest ^:async interruption-skips-comparisons-and-returns-the-signal-exit + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + controller (js/AbortController.) + base (execute-service + (fn [build] + (when (empty? (get-in build [:manifest :functional :features])) + (.abort controller "SIGINT")) + {:kind :interrupted})) + calls (atom 0) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @calls)) + (t/is (= 130 (:exit-code result))) + (t/is (= :interrupted (:status record))) + (t/is (some? (:ended-at record))) + (t/is (= "SIGINT" (get-in record [:configurations 0 :reason]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (= :unattempted (get-in record [:edges 0 :status]))) + (t/is (not (fs/existsSync (path/join (:dir (:home result)) "comparisons")))))) + +(t/deftest ^:async interruption-between-configurations-stops-before-the-next-build + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + controller (js/AbortController.) + builds (atom 0) + base (execute-service {}) + execute (fn [& args] + (let [result (apply base args)] + (.abort controller "SIGTERM") + result)) + result (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build + (build-service + {:inspect (fn [_] (swap! builds inc))}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @builds)) + (t/is (= 143 (:exit-code result))) + (t/is (= :interrupted (:status record))) + (t/is (= :completed (get-in record [:configurations 0 :status]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (str/includes? (get-in record [:configurations 1 :reason]) "SIGTERM")) + (t/is (= :unattempted (get-in record [:edges 0 :status]))) + (t/is (not (fs/existsSync (path/join (:dir (:home result)) "comparisons")))))) + +(t/deftest ^:async persistence-failure-retains-the-last-saved-state + (create-frontend-root) + (let [stdout (atom "") + prepared (comparison-prepared ["a"] {}) + record-path (path/join (:output-dir prepared) "feature-comparison.transit") + prepare (fn [request] + (fs/rmSync record-path) + (fs/mkdirSync record-path) + (let [build (fake-build (:selection request) [])] + ((:on-created request) build) + (js/Promise.resolve build))) + result (do + (log/configure {:stdout #js {:write #(swap! stdout str %)}}) + (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build prepare + :execute-run (execute-service {})})}))) + record (:comparison result)] + (t/is (= 1 (:exit-code result))) + (t/is (seq (:errors result))) + (t/is (= :running (:status record))) + (t/is (= :building (get-in record [:configurations 0 :status]))) + (t/is (nil? (get-in record [:configurations 0 :build]))) + (t/is (= :planned (get-in record [:configurations 1 :status]))))) + +(t/deftest ^:async comparison-storage-error-uses-comparison-evidence-after-a-failed-config + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record (path/join (:output-dir prepared) "feature-comparison.transit") + response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services + (comparison-services + {:prepare-build + (build-service + {:inspect (fn [request] + (when (seq (get-in request [:selection :features])) + (fs/renameSync record (str record ".saved")) + (fs/mkdirSync record)))}) + :execute-run (execute-service {:kind :failed})})}))] + (t/is (= 1 (:exit-code response))) + (t/is (= :failed (get-in response [:comparison :configurations 0 :status]))) + (t/is (= (get-in response [:home :log-path]) + (:log-path (ex-data (first (:errors response)))))))) + +(t/deftest ^:async comparison-storage-error-keeps-its-origin-through-build-preparation + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record (path/join (:output-dir prepared) "feature-comparison.transit") + response (await (feature-comparison/execute + (frontend-root) prepared nil + {:services + {:prepare-build + (^:async fn [request] + (let [created (:on-created request)] + (await (build/prepare + (assoc request :on-created + (fn [selected] + (fs/renameSync record (str record ".saved")) + (fs/mkdirSync record) + (created selected)))))))}}))] + (t/is (= 1 (:exit-code response))) + (t/is (= (get-in response [:home :log-path]) + (:log-path (ex-data (first (:errors response)))))))) + +(t/deftest ^:async pending-real-signals-stop-edge-analysis + (create-frontend-root) + (doseq [[signal exit-code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root signal)}) + controller (js/AbortController.) + remove (execution/on-signals #(.abort controller %)) + execute (execute-service + (fn [build] + (when (seq (get-in build [:manifest :functional :features])) + (.kill js/process (.-pid js/process) signal)) + {}))] + (try + (let [response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})}))] + (await (timers/setImmediate)) + (let [saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))] + (t/is (= exit-code (:exit-code response))) + (t/is (= :interrupted (:status saved))) + (t/is (every? #(= :unattempted (:status %)) (:edges saved))))) + (finally + (remove)))))) diff --git a/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs b/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs new file mode 100644 index 0000000000..d351bf1b86 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/fingerprint_test.cljs @@ -0,0 +1,100 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.fingerprint-test + "Tests content identity with production records, paths and collection types. + Validated scene snapshots retain child order even when maps reorder." + (:require + [app.common.geom.matrix :as matrix] + [app.common.geom.point :as point] + [app.common.transit :as transit] + [app.common.types.objects-map :as objects-map] + [app.common.types.path.impl :as path] + [app.common.types.shape :as shape] + [app.common.uuid :as uuid] + [benches.render-wasm.codec :as codec] + [benches.render-wasm.fingerprint :as fingerprint] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [linked.map :as linked-map] + [linked.set :as linked-set])) + +(def a #uuid "00000000-0000-0000-0000-000000000001") +(def b #uuid "00000000-0000-0000-0000-000000000002") + +(defn- starting-snapshot + "Builds and validates two canonical rectangles under the root frame." + [] + (snapshot/validate! + {:objects {uuid/zero (shape/setup-shape {:id uuid/zero :type :frame :name "Root" + :x 0 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero :shapes [a b]}) + a (shape/setup-shape {:id a :type :rect :name "A" + :x 0 :y 0 :width 10 :height 10 :parent-id uuid/zero :frame-id uuid/zero}) + b (shape/setup-shape {:id b :type :rect :name "B" + :x 20 :y 20 :width 10 :height 10 :parent-id uuid/zero :frame-id uuid/zero})} + :refs {:a a :b b}})) + +(t/deftest unordered-representations-have-one-fingerprint + (let [scene (starting-snapshot) + first (assoc scene :extra {:map (array-map :a 1 :b 2) + :compound (array-map [:a 1] {:x 1 :y 2} [:b 2] #{:one :two}) + :set (set (range 20))}) + second (-> scene + (update :objects #(into {} (reverse (seq %)))) + (assoc :refs (array-map :b b :a a) + :extra {:set (set (reverse (range 20))) + :compound (array-map [:b 2] #{:two :one} [:a 1] (array-map :y 2 :x 1)) + :map (array-map :b 2 :a 1)}))] + (t/is (= (fingerprint/snapshot-text first) (fingerprint/snapshot-text second))) + (t/is (= (fingerprint/fingerprint first) (fingerprint/fingerprint second))))) + +(t/deftest ordered-collections-and-child-order-change-the-fingerprint + (let [scene (starting-snapshot)] + (t/is (not= (fingerprint/fingerprint scene) + (fingerprint/fingerprint (assoc-in scene [:objects uuid/zero :shapes] [b a])))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra [1 2])) + (fingerprint/fingerprint (assoc scene :extra [2 1])))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra (into linked-set/empty-linked-set [1 2]))) + (fingerprint/fingerprint (assoc scene :extra (into linked-set/empty-linked-set [2 1]))))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra (into linked-map/empty-linked-map [[:a 1] [:b 2]]))) + (fingerprint/fingerprint (assoc scene :extra (into linked-map/empty-linked-map [[:b 2] [:a 1]]))))) + (t/is (not= (fingerprint/fingerprint (assoc scene :extra (point/point 1 2))) + (fingerprint/fingerprint (assoc scene :extra {:x 1 :y 2})))))) + +(t/deftest unrepresentable-transit-content-never-gets-a-misleading-fingerprint + (let [scene (starting-snapshot)] + (t/is (= ::codec/unrepresentable-value + (try + (fingerprint/fingerprint (assoc scene :extra {"__proto__" "value"})) + nil + (catch :default cause (:type (ex-data cause)))))))) + +(t/deftest objects-map-materializes-before-hashing-and-production-tags-survive + (let [scene (starting-snapshot) + wrapped (update scene :objects objects-map/wrap) + data (assoc scene :extra {:point (point/point 1 2) + :matrix (matrix/matrix) + :path (path/path-data [{:command :move-to :params {:x 1 :y 2}} + {:command :line-to :params {:x 3 :y 4}}])}) + text (fingerprint/snapshot-text data)] + (t/is (= (fingerprint/fingerprint scene) (fingerprint/fingerprint wrapped))) + (t/is (not (str/includes? (fingerprint/snapshot-text wrapped) "penpot/objects-map"))) + (t/is (str/includes? text "~#shape")) + (t/is (str/includes? text "~#point")) + (t/is (str/includes? text "~#matrix")) + (t/is (str/includes? text "~#penpot/path-data")) + (t/is (= (fingerprint/fingerprint data) + (fingerprint/fingerprint (transit/decode-str (transit/encode-str data))))))) + +(t/deftest digest-matches-known-sha256-vector + ;; The verbose Transit map for {:objects {} :refs {}} is canonical text. + (t/is (= "{\"~:objects\":{},\"~:refs\":{}}" + (fingerprint/snapshot-text {:objects {} :refs {}}))) + (t/is (= {:algorithm :sha-256 + :digest "ceb57067506683f5a0e80615bc9dd6f6d16325e333864da705fa4916fbdfe52d"} + (fingerprint/fingerprint {:objects {} :refs {}})))) diff --git a/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs b/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs new file mode 100644 index 0000000000..58852a055b --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/geometry_scenes_test.cljs @@ -0,0 +1,86 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.geometry-scenes-test + (:require + [app.common.types.path :as path] + [app.common.uuid :as uuid] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + ;; These 3 register the scenes, do not remove + [benches.render-wasm.scenes.masks] + [benches.render-wasm.scenes.paths] + [benches.render-wasm.scenes.plus] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :as helpers])) + +(t/deftest curved-paths-use-canonical-content-and-stable-order + (let [params {:count 12 :width 1920 :height 1080 :seed 42} + build (:build (decl/registered-scene :paths)) + scene (build params) + again (build params) + shapes (helpers/children-of scene uuid/zero) + first (first shapes) + content (vec (:content first))] + (t/is (= 12 (count shapes))) + (t/is (= scene again) "same scene parameters and seed reproduce content and ids") + (t/is (path/content? (:content first))) + (t/is (= [:move-to :curve-to :line-to :close-path] + (mapv :command content))) + (t/is (pos? (:width (:selrect first)))) + (t/is (pos? (:height (:selrect first)))) + (t/is (= (mapv :id shapes) (mapv :id (rest (snapshot/upload-order scene))))) + (t/is (= scene (snapshot/validate! scene))))) + +(t/deftest plus-paths-have-two-crossing-open-lines + (let [build (:build (decl/registered-scene :plus)) + scene (build {:count 8 :width 1920 :height 1080 :seed 42}) + first (first (helpers/children-of scene uuid/zero)) + segments (vec (:content first)) + [a b c d] segments] + (t/is (= 8 (count (helpers/children-of scene uuid/zero)))) + (t/is (= [:move-to :line-to :move-to :line-to] + (mapv :command segments))) + (t/is (= (get-in a [:params :y]) (get-in b [:params :y])) + "the first line is horizontal") + (t/is (< (get-in a [:params :x]) (get-in b [:params :x]))) + (t/is (= (get-in c [:params :x]) (get-in d [:params :x])) + "the second line is vertical") + (t/is (< (get-in c [:params :y]) (get-in d [:params :y]))) + (t/is (seq (:strokes first))) + (t/is (= scene (snapshot/validate! scene))))) + +(t/deftest mask-child-order-and-descendants-are-reachable + (let [build (:build (decl/registered-scene :masks)) + scene (build {:width 1920 :height 1080 :seed 42}) + groups (helpers/children-of scene uuid/zero) + group (first groups) + children (helpers/children-of scene (:id group)) + mask (first children) + uploaded (snapshot/upload-order scene)] + (t/is (= 24 (count groups))) + (t/is (= 241 (count (:objects scene))) "root, groups and their nine children") + (t/is (= (:id group) (get-in scene [:refs [:mask 0]]))) + (t/is (true? (:masked-group group))) + (t/is (= :circle (:type mask)) "the first child is the mask") + (t/is (= 9 (count children))) + (t/is (every? #(= (:id group) (:parent-id %)) children)) + (t/is (= (:selrect mask) (:selrect group))) + (t/is (= uuid/zero (:id (first uploaded)))) + (t/is (= (:id group) (:id (second uploaded)))) + (t/is (= (:id mask) (:id (nth uploaded 2)))) + (t/is (= (count (:objects scene)) (count uploaded))) + (t/is (= scene (snapshot/validate! scene))))) + +(t/deftest camera-cases-share-each-scene-seed + (doseq [scene-id [:paths :plus :masks]] + (let [selected (->> (cases/collect-cases {:master-seed 42}) + (filter #(= scene-id (:scene %))) + vec)] + (t/is (= #{"load" "pan" "zoom"} (set (map (comp name :id) selected)))) + (t/is (= 1 (count (set (map :scene-seed selected))))) + (t/is (every? #(not (contains? % :run!)) selected))))) diff --git a/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs new file mode 100644 index 0000000000..8a1a8641a5 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs @@ -0,0 +1,1142 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.lifecycle-test + "Tests the browser bridge with fake DOM and renderer boundaries. + + It covers bridge input, ownership, disposal and `load-scene` with fake + canvases and a fake Emscripten factory. Progressive frames use an optional + clock at the browser boundary." + (:require + [app.common.render-wasm.wasm :as wasm] + [app.common.transit :as transit] + [benches.render-wasm.browser.bridge :as browser] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.measurement :as measurement] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [benches.render-wasm.runtime.camera :as camera] + [cljs.test :as t :include-macros true])) + +;; Forward declaration: entry-test helpers below build args against the +;; fake Emscripten factory URL defined alongside the fake-DOM harness. +(declare factory-url) + +;; Partial-init unwinding of `init-context` is covered in +;; `frontend-tests.render-wasm.webgl-test`, which also pins the success, +;; nil-context and registration-failure cases. + +(defn- register-phase-case + [options run] + (let [params (:params (decl/registered-case :rects/load))] + (decl/defcase :rects/lifecycle + (-> (merge {:params params + :view {:scale 1 :x 0 :y 0} + :context :fresh} + options) + (update :params #(merge params %))) + [rtx] + (run rtx)))) + +(defn- load-case + "Returns the collected rectangle case for entry tests." + [] + (first (cases/collect-cases {:master-seed 42 + :filter (if (decl/registered-case :rects/lifecycle) + "rects/lifecycle" + "rects/load")}))) + +(defn- scene-args + "Transit request for `browser/load-scene`. `overrides` change the + collected case descriptor." + [seed overrides] + (let [base (or (some #(when (= (:id %) (:id overrides)) %) + (cases/collect-cases {:master-seed 42})) + (load-case))] + (transit/encode-str + {:case (merge base {:scene-seed seed} overrides) + :module-url (factory-url) + :wasm-url "./fake.wasm"}))) + +(defn- expect-failed-args + "Calls `load-scene` with `args` (never touching DOM/FFI: validation runs + first) and asserts a failure map. Calls `done` when finished." + [args check-phase? done] + (-> (.then (browser/load-scene args) + (fn [result] + (let [m (transit/decode-str result)] + (t/is (= "failed" (:status m))) + (when check-phase? + (t/is (= "invalid-args" (:phase m)))) + (done)))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))) + +(t/deftest invalid-seed-fails-before-timers + (t/async done + (expect-failed-args (scene-args -1 {}) true done))) + +(t/deftest missing-seed-fails-before-timers + (t/async done + (expect-failed-args (transit/encode-str {:case (dissoc (load-case) :scene-seed)}) true done))) + +(t/deftest missing-case-fails-before-timers + (t/async done + (expect-failed-args (transit/encode-str {}) true done))) + +(t/deftest malformed-transit-fails-before-timers + (t/async done + (expect-failed-args "not transit" true done))) + +(t/deftest invalid-view-fails-before-timers + (t/async done + (expect-failed-args + (scene-args 42 {:view {:scale -1 :x 0 :y 0}}) + true + done))) + +(t/deftest invalid-viewport-fails-before-timers + (t/async done + (expect-failed-args + (scene-args 42 {:view {:scale 1 :x 0 :y 0 + :viewport {:width -5}}}) + true + done))) + +(t/deftest missing-compiled-body-fails-before-claim + (t/async done + (expect-failed-args (scene-args 42 {:id :rects/missing}) true done))) + +(t/deftest ^:async wrong-scene-body-rejects-the-load-request + (let [id :rects/wrong-scene + descriptor (load-case)] + (decl/register-case! {:id id :scene :other :ns "lifecycle-test" + :run! (fn [_] nil)}) + (try + (let [request (transit/encode-str {:case (assoc descriptor :id id)}) + result (transit/decode-str (await (browser/prepare-scene-request request)))] + (t/is (= "failed" (:status result))) + (t/is (= "invalid-args" (:phase result)))) + (finally + (decl/unregister-case! id))))) + +(t/deftest non-finite-view-values-fail-before-timers + (t/async done + (expect-failed-args (scene-args 42 {:view {:scale js/Infinity :x 0 :y 0}}) + true + (fn [] + (expect-failed-args (scene-args 42 {:view {:scale 1 :x js/NaN :y 0}}) + true + (fn [] + (expect-failed-args (scene-args 42 {:view {:scale 1 :x 0 :y js/Infinity}}) + true + done))))))) + +(t/deftest viewport-defaults-apply-at-use + (t/is (= {:width 1920 :height 1080 :dpr 2} + (decl/resolve-viewport {:scale 1 :x 0 :y 0}))) + (t/is (= {:width 800 :height 600 :dpr 2} + (decl/resolve-viewport {:scale 1 :x 0 :y 0 + :viewport {:width 800 :height 600}})))) + +(t/deftest dispose-resets-lifecycle-state + (set! wasm/internal-module #js {}) + (set! wasm/context-initialized? true) + (reset! wasm/context-lost? true) + (set! wasm/gl-context-handle 7) + (set! wasm/gl-context #js {}) + (set! wasm/canvas #js {}) + (let [removed (atom false) + fake #js {:remove (fn [] (reset! removed true))}] + (reset! @#'browser/canvas* fake) + (let [result (browser/dispose!)] + (t/is (= "disposed" (unchecked-get result "status"))) + (t/is (nil? wasm/internal-module)) + (t/is (false? wasm/context-initialized?)) + (t/is (false? @wasm/context-lost?)) + (t/is (nil? wasm/gl-context-handle)) + (t/is (nil? wasm/gl-context)) + (t/is (nil? wasm/canvas)) + (t/is (nil? @@#'browser/canvas*)) + (t/is (true? @removed) "canvas is removed from the DOM"))) + (t/testing "double dispose is idempotent" + (let [result (browser/dispose!)] + (t/is (= "disposed" (unchecked-get result "status"))) + (t/is (nil? wasm/internal-module)))) + (t/testing "dispose bumps the owner epoch" + (let [before @@#'browser/owner-epoch*] + (browser/dispose!) + (t/is (< before @@#'browser/owner-epoch*))))) + +(t/deftest stale-work-never-disposes-the-newer-owner + (let [guard @#'browser/guard-current! + terminate @#'browser/terminal-failure + epoch @@#'browser/owner-epoch*] + (t/testing "guard passes for the current owner" + (t/is (nil? (guard epoch "module-init")))) + (t/testing "guard throws stale once superseded" + (browser/dispose!) + (let [thrown (try + (guard epoch "module-init") + nil + (catch :default cause + cause))] + (t/is (errors/stale? thrown) + "the marker lives in ex-data, never in message text"))) + (t/testing "terminal stale resolves without disposal" + (let [before @@#'browser/owner-epoch* + result (terminate epoch + (ex-info "superseded" + {::errors/stale true + :phase "upload"}) + "upload") + after @@#'browser/owner-epoch*] + (t/is (= "stale" (:status result))) + (t/is (= before after) "no epoch bump: the newer owner is untouched"))) + (t/testing "terminal failure keeps its phase and disposes the current owner" + (let [current @@#'browser/owner-epoch* + result (terminate current (ex-info "upload failed" {:phase "upload"}) "aborted")] + (t/is (= "failed" (:status result))) + (t/is (= "upload" (:phase result))) + (t/is (< current @@#'browser/owner-epoch*) "current owner disposed"))) + (t/testing "phaseless failures use the fallback phase" + (set! wasm/internal-module nil) + (let [current @@#'browser/owner-epoch* + result (terminate current (ex-info "boom" {}) "aborted")] + (t/is (= "failed" (:status result))) + (t/is (= "aborted" (:phase result))) + (t/is (< current @@#'browser/owner-epoch*) "current owner disposed"))))) + +(t/deftest bridge-exposes-executor-entries + (doseq [key ["ping" "loadScene" "prepareScene" "runPrepared" "dispose"]] + (t/is (fn? (unchecked-get browser/bridge key)) + (str key " is callable through the browser ABI"))) + (t/is (= "ok" (unchecked-get (.ping browser/bridge) "status")))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Through-entry fake harness. +;; +;; The DOM/module boundary the fakes cover does not change with the entry: +;; fake DOM, fake Emscripten factory, call recording. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- record! + "Appends `event` (a vector) to the shared call log." + [event] + (.push (unchecked-get js/globalThis "__benchCalls") (clj->js event))) + +(defn- fake-canvas + "Numbered fake canvas identified by `id`. Removal and context loss are + recorded in the shared call log. The context reports a drawing-buffer size + distinct from any canvas size, so tests prove `read-graphics` reads it from + the context rather than the canvas attributes." + ([id] + (fake-canvas id nil)) + ([id {:keys [vendor renderer]}] + (let [debug-ext (when (and vendor renderer) + #js {:UNMASKED_VENDOR_WEBGL 1 :UNMASKED_RENDERER_WEBGL 2}) + lose-ext #js {:loseContext (fn [] (record! ["lose" id]) nil)} + context #js {:getExtension (fn [name] + (case name + "WEBGL_lose_context" lose-ext + "WEBGL_debug_renderer_info" debug-ext + nil)) + :getParameter (fn [parameter] (case parameter 1 vendor 2 renderer nil)) + :getContextAttributes (fn [] + #js {"antialias" false + "depth" true + "stencil" false + "alpha" false + "preserveDrawingBuffer" false}) + :drawingBufferWidth 1234 + :drawingBufferHeight 5678}] + #js {:id id + :width 0 + :height 0 + :style #js {} + :getContext (fn [_ _] context) + :addEventListener (fn [_ _] nil) + :remove (fn [] (record! ["remove" id]) nil)}))) + +(defn- install-fake-dom! + "Installs fake DOM, import, clock and scenario globals and returns a + `restore!` thunk that restores their previous values. Created canvases are + numbered from 1 per installation. + The import polyfill mirrors + `resources/polyfills/dynamicImport.js`, which ships with the app HTML but + not with the unit-test bundle." + [] + (let [prev-document (unchecked-get js/globalThis "document") + prev-window (unchecked-get js/globalThis "window") + prev-import (unchecked-get js/globalThis "dynamicImport") + prev-performance (unchecked-get js/globalThis "performance") + prev-now-fn (unchecked-get prev-performance "now") + prev-now-desc (js/Object.getOwnPropertyDescriptor prev-performance "now") + prev-now (unchecked-get js/globalThis "__benchNow") + prev-script (unchecked-get js/globalThis "__benchRenderScript") + prev-calls (unchecked-get js/globalThis "__benchCalls") + prev-factory (unchecked-get js/globalThis "__benchFactoryCalls") + prev-throw (unchecked-get js/globalThis "__benchThrowIn") + prev-gate (unchecked-get js/globalThis "__benchGateRaf") + prev-raf (unchecked-get js/globalThis "__benchRafCb") + next-id (atom 0) + document #js {:createElement (fn [_] + (let [id (swap! next-id inc)] + (record! ["create" id]) + (fake-canvas id))) + :body #js {:appendChild (fn [canvas] + (record! ["append" (unchecked-get canvas "id")]) + canvas)} + :querySelectorAll (fn [_] #js [])} + window #js {:requestAnimationFrame + (fn [cb] + ;; Park the first frame while `__benchGateRaf` + ;; is set so a second load can supersede a live, + ;; active owner; later frames run synchronously. + (if (and (unchecked-get js/globalThis "__benchGateRaf") + (nil? (unchecked-get js/globalThis "__benchRafCb"))) + (unchecked-set js/globalThis "__benchRafCb" cb) + (if-some [clock (unchecked-get js/globalThis "__benchNow")] + (let [timestamp (+ clock 16)] + (unchecked-set js/globalThis "__benchNow" timestamp) + (cb timestamp)) + (cb 0))) + nil)} + fake-now (fn [] + (if-some [clock (unchecked-get js/globalThis "__benchNow")] + clock + (.call prev-now-fn prev-performance))) + dynamic-import (fn [url] (js/eval (str "import(" (pr-str url) ")")))] + (unchecked-set js/globalThis "document" document) + (unchecked-set js/globalThis "window" window) + ;; Patch `now` on the real object: shadow hangs the compiled + ;; `performance.*` namespaces off the global `performance` object, so + ;; replacing that global hides every `benches.render-wasm.*` var. + (js/Object.defineProperty prev-performance "now" + #js {:value fake-now :configurable true :writable true}) + (unchecked-set js/globalThis "dynamicImport" dynamic-import) + (fn restore! [] + (unchecked-set js/globalThis "document" prev-document) + (unchecked-set js/globalThis "window" prev-window) + (if (undefined? prev-now-desc) + (js-delete prev-performance "now") + (js/Object.defineProperty prev-performance "now" prev-now-desc)) + (unchecked-set js/globalThis "__benchNow" prev-now) + (unchecked-set js/globalThis "__benchRenderScript" prev-script) + (unchecked-set js/globalThis "__benchCalls" prev-calls) + (unchecked-set js/globalThis "__benchFactoryCalls" prev-factory) + (unchecked-set js/globalThis "__benchThrowIn" prev-throw) + (unchecked-set js/globalThis "__benchGateRaf" prev-gate) + (unchecked-set js/globalThis "__benchRafCb" prev-raf) + (unchecked-set js/globalThis "dynamicImport" prev-import)))) + +(def ^:private factory-source + "ES module stubbing the Emscripten factory for entry tests. Records every + renderer call in `globalThis.__benchCalls`; `globalThis.__benchThrowIn` + names one module fn or an array of module fns that throw. + `globalThis.__benchRenderScript` holds optional [frame-type, duration-ms] + pairs that advance `__benchNow`; absent + a script, `_render` immediately returns Full. The instance trap answers + `undefined` for `then` and non-strings: promise assimilation would treat the + instance as a never-settling thenable and hang the load. A real 16 MiB + heap with bump allocation backs `_alloc_bytes`/`HEAPU8`, so the real batch + upload assembles bytes instead of throwing on stub returns." + (str "export default function (opts) {" + "globalThis.__benchFactoryCalls.push(opts || null);" + "var rec = globalThis.__benchCalls;" + "function push(e) { rec.push(e); }" + "var GL = {" + "registerContext: function () { push(['register']); return 7; }," + "makeContextCurrent: function () {}," + "deleteContext: function (h) { push(['delete', h]); }" + "};" + "var heapBuf = new ArrayBuffer(16777216);" + "var heapNext = 1024;" + "var base = {" + "GL: GL," + "HEAPU8: new Uint8Array(heapBuf)," + "HEAPU32: new Uint32Array(heapBuf)," + "HEAP32: new Int32Array(heapBuf)," + "HEAPF32: new Float32Array(heapBuf)," + "_alloc_bytes: function (size) { var p = heapNext; heapNext += ((size + 3) & ~3); return p; }," + "_free_bytes: function () {}," + "_read_error_code: function () { return 0; }," + "_render: function (timestamp, flags) {" + "push(['render', timestamp, flags]);" + "var script = globalThis.__benchRenderScript;" + "if (script && script.length) {" + "var next = script.shift(); globalThis.__benchNow += next[1]; return next[0];" + "}" + "return 2;" + "}," + "_clean_up: function () { push(['clean']); return 0; }" + "};" + "var inst = new Proxy(base, {" + "get: function (t, p) {" + "if (typeof p !== 'string') return undefined;" + "if (p === 'then') return undefined;" + "if (p in t) return t[p];" + "return function () {" + "push(['call', p]);" + "var faults = globalThis.__benchThrowIn;" + "if (Array.isArray(faults) ? faults.includes(p) : faults === p) throw new Error('fake ' + p + ' failure');" + "return 0;" + "};" + "}" + "});" + "return Promise.resolve(inst);" + "}")) + +(defn- factory-url + "Data-URL Emscripten factory for entry tests. Fully hermetic: no + filesystem or network access." + [] + (str "data:text/javascript," (js/encodeURIComponent factory-source))) + +(defn- reset-bench-globals! + "Resets the recording and scenario flags the fake factory reads." + [] + (unchecked-set js/globalThis "__benchCalls" #js []) + (unchecked-set js/globalThis "__benchFactoryCalls" #js []) + (unchecked-set js/globalThis "__benchThrowIn" nil) + (unchecked-set js/globalThis "__benchGateRaf" false) + (unchecked-set js/globalThis "__benchRafCb" nil) + (unchecked-set js/globalThis "__benchNow" nil) + (unchecked-set js/globalThis "__benchRenderScript" nil)) + +(defn- read-calls + "Call log as Clojure data." + [] + (js->clj (unchecked-get js/globalThis "__benchCalls"))) + +(defn- effect-count + "Counts recorded effects without filtering any returned evidence." + [calls effect] + (count (filter #(= effect (first %)) calls))) + +(defn- load-result + "Decodes the bridge's Transit result for `k`. A rejected bridge arrives + as a `threw` map so the test fails with the cause attached." + [args k] + (-> (browser/load-scene args) + (.then (fn [result] + (t/is (string? result) "loadScene returns Transit") + (k (transit/decode-str result)))) + (.catch (fn [cause] (k {:status "threw" :cause (str cause)}))))) + +(defn- with-entry-env + "Installs the fake DOM, resets bench globals, then calls `f` with a + `cleanup` thunk that disposes the owner and restores the previous globals. + Call `cleanup` after asserting, before `done`." + [f] + (let [restore-dom! (install-fake-dom!) + cleanup (fn [] + (try + (browser/dispose!) + (finally + (decl/unregister-case! :rects/lifecycle) + (restore-dom!))))] + (reset-bench-globals!) + (try + (register-phase-case {} camera/render-sync*) + (f cleanup) + (catch :default cause + (cleanup) + (throw cause))))) + +(t/deftest ^:async preparations-record-the-context-identity-and-software-status + (doseq [[identity software] [[{:vendor "NVIDIA" :renderer "ANGLE (NVIDIA, Vulkan)"} false] + [{:vendor "Google" :renderer "ANGLE (SwiftShader)"} true] + [{:vendor "Mesa" :renderer "llvmpipe (LLVM)"} true] + [nil false]]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (set! (.-createElement js/document) (fn [_] (fake-canvas 1 identity))) + (let [response (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + graphics (get-in response [:preparation :effective-graphics])] + (t/is (= "ok" (:status response))) + (t/is (= (or (:vendor identity) "unknown") (:vendor graphics))) + (t/is (= (or (:renderer identity) "unknown") (:renderer graphics))) + (t/is (= software (:software graphics)))) + (finally + (cleanup)))))))) + +(t/deftest successful-load-reaches-full + (t/async done + (with-entry-env + (fn [cleanup] + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "ok" (:status m))) + (t/is (= 1 (count (get-in m [:preparation :diagnostics :setup-render :slices])))) + (t/is (= [::camera/render-sync] + (mapv :command (get-in m [:phases 0 :events])))) + (t/is (= 1001 (:shapes (:scene m))) "the canonical scene uploads whole") + (let [graphics (:effective-graphics m)] + (t/is (= 1234 (:drawing-buffer-width graphics)) + "drawing buffer width comes from the context") + (t/is (= 5678 (:drawing-buffer-height graphics)) + "drawing buffer height comes from the context") + (t/is (false? (:antialias graphics)) + "context attributes pass through") + (t/is (true? (:depth graphics)) + "context attributes pass through")) + (let [calls (read-calls)] + (t/is (= 1 (effect-count calls "create")) "one canvas per load") + (t/is (zero? (effect-count calls "remove")) "success keeps the owner live")) + (cleanup) + (done))))))) + +(t/deftest preparation-retains-progressive-frame-evidence + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchNow" 0) + (unchecked-set js/globalThis "__benchRenderScript" + #js [#js [wasm/FRAME_TYPE_PARTIAL 2] + #js [wasm/FRAME_TYPE_VIEWPORT_READY 3] + #js [wasm/FRAME_TYPE_FULL 5]]) + (-> (.loadScene browser/bridge (scene-args 7 {})) + (.then (fn [text] + (t/is (string? text) "the exported loadScene returns Transit") + (let [{:keys [status preparation scene]} (transit/decode-str text) + {:keys [slices viewport-ready-ms full-ms]} + (get-in preparation [:diagnostics :setup-render]) + retained slices + render-calls (filterv #(= "render" (first %)) (read-calls))] + (t/is (= "ok" status)) + (t/is (= {:shapes 1001 :seed 7} scene) + "the real canonical case prepares and uploads the whole scene") + (t/is (= 3 (count retained)) "all three slices cross Transit") + (t/is (= [wasm/FRAME_TYPE_PARTIAL + wasm/FRAME_TYPE_VIEWPORT_READY + wasm/FRAME_TYPE_FULL] + (mapv :frame-type retained))) + (t/is (= [0 wasm/FRAME_TYPE_PARTIAL wasm/FRAME_TYPE_VIEWPORT_READY] + (mapv :flags retained)) + "ViewportReady becomes the next flags and the drain continues to Full") + (t/is (= [16 34 53] (mapv :timestamp retained)) + "each continuation waits for the next frame") + (t/is (= [2 3 5] (mapv :duration-ms retained))) + (t/is (= render-calls + (mapv (fn [{:keys [timestamp flags]}] + ["render" timestamp flags]) + retained)) + "decoded slices retain every WASM call's timestamp and flags in order") + (t/is (= 37 viewport-ready-ms + (+ (:timestamp (nth retained 1)) + (:duration-ms (nth retained 1)))) + "viewport completion ends with the ViewportReady slice") + (t/is (= 58 full-ms + (+ (:timestamp (nth retained 2)) + (:duration-ms (nth retained 2)))) + "Full completion ends with the final slice") + (t/is (< viewport-ready-ms full-ms)) + (t/is (= [{:phase :preparation + :preparation-id (:preparation-id preparation) + :outcome :completed :events []}] + (:phases preparation)))) + (cleanup) + (done))) + (.catch (fn [cause] + (t/is false (str "exported progressive load failed: " cause)) + (cleanup) + (done)))))))) + +(t/deftest invalid-args-keep-the-current-owner + (t/async done + (with-entry-env + (fn [cleanup] + (load-result (scene-args 7 {}) + (fn [loaded] + (t/is (= "ok" (:status loaded))) + (let [owner @@#'browser/canvas* + epoch @@#'browser/owner-epoch*] + (load-result (transit/encode-str {:seed -1}) + (fn [invalid] + (t/is (= "invalid-args" (:phase invalid))) + (t/is (identical? owner @@#'browser/canvas*)) + (t/is (= epoch @@#'browser/owner-epoch*)) + (t/is (zero? (effect-count (read-calls) "remove"))) + (cleanup) + (done)))))))))) + +(t/deftest compiled-pan-finishes-at-its-final-render + (t/async done + (with-entry-env + (fn [cleanup] + (register-phase-case {:context :reuse} + #(camera/pan* % {:dx 10 :dy 5 :steps 2 :settle-ms 0})) + (load-result (scene-args 7 {}) + (fn [m] + (let [phase (first (:phases m)) + event (first (:events phase)) + trace (:trace event)] + (t/is (= "ok" (:status m))) + (t/is (= :main (:phase phase))) + (t/is (= ::camera/pan (:command event))) + (t/is (= 2 (count (:cached-slices trace)))) + (t/is (= 0 (:settling-requested-ms trace))) + (t/is (= [4] (mapv :flags (:slices trace)))) + (t/is (= [2] (mapv :frame-type (:slices trace)))) + (t/is (= 2 (effect-count (read-calls) "render")) + "preparation and camera finalization each render once")) + (cleanup) + (done))))))) + +(t/deftest ^:async reused-context-prepares-once-and-resets-each-main + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [views (atom []) + options {:dy 0 :steps 1 :settle-ms 0} + pan (fn [phase dx rtx] + (swap! views conj [phase (get-in rtx [:view :x])]) + (camera/pan* rtx (assoc options :dx dx)))] + (register-phase-case {:context :reuse + :preparation {:callback #(pan :preparation 5 %)} + :reset #(pan :reset -10 %)} + #(pan :main 10 %)) + (let [prepared (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + prep (:preparation prepared) + request (transit/encode-str {:preparation-id (:preparation-id prep)}) + first (transit/decode-str (await (browser/run-prepared request))) + second (transit/decode-str (await (browser/run-prepared request)))] + (t/is (= ["ok" "ok" "ok"] (mapv :status [prepared first second]))) + (t/is (= [[:preparation 0] [:main 5] [:reset 15] [:main 5] [:reset 15]] @views)) + (t/is (= [:preparation] (mapv :phase (:phases prep)))) + (t/is (= [::camera/pan] (mapv :command (get-in prep [:phases 0 :events])))) + (t/is (= prep (:preparation first) (:preparation second))) + (doseq [execution [first second]] + (t/is (= [:main :reset] (mapv :phase (:phases execution)))) + (t/is (= [:completed :completed] (mapv :outcome (:phases execution)))) + (t/is (= [[::camera/pan] [::camera/pan]] + (mapv #(mapv :command (:events %)) (:phases execution))))) + (t/is (= 1 (effect-count (read-calls) "create"))) + (t/is (= 6 (effect-count (read-calls) "render")) + "the callback, initial Full, and four main/reset gestures render") + (t/is (= 1 (count (filter #(= ["call" "_init_shapes_pool"] %) (read-calls))))))) + (finally + (cleanup))))))) + +(t/deftest ^:async reused-preparation-keeps-its-original-case-deadline + (await + (with-entry-env + (^:async fn [cleanup] + (try + (unchecked-set js/globalThis "__benchNow" 0) + (register-phase-case {:context :reuse :preparation {:render :none}} camera/render-sync*) + (let [request (-> (scene-args 7 {}) transit/decode-str + (assoc :timeout-ms 100) transit/encode-str) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + id (get-in prepared [:preparation :preparation-id])] + (unchecked-set js/globalThis "__benchNow" 60) + (let [first (transit/decode-str (await (browser/run-prepared + (transit/encode-str {:preparation-id id :timeout-ms 10000}))))] + (t/is (= "ok" (:status first)))) + (unchecked-set js/globalThis "__benchNow" 101) + (let [expired (transit/decode-str (await (browser/run-prepared + (transit/encode-str {:preparation-id id}))))] + (t/is (= "failed" (:status expired))) + (t/is (= "deadline" (:phase expired))))) + (finally + (cleanup))))))) + +(t/deftest ^:async fresh-contexts-prepare-and-reset-once + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [visits (atom []) + work (fn [phase rtx] + (swap! visits conj phase) + (camera/render-sync* rtx))] + (register-phase-case {:preparation {:callback #(work :preparation %)} + :reset #(work :reset %)} + #(work :main %)) + (dotimes [_ 2] + (let [prepared (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + prep (:preparation prepared) + request (transit/encode-str {:preparation-id (:preparation-id prep)}) + evidence (transit/decode-str (await (browser/run-prepared request))) + repeated (transit/decode-str (await (browser/run-prepared request)))] + (t/is (= "ok" (:status evidence))) + (t/is (= [:main :reset] (mapv :phase (:phases evidence)))) + (t/is (= [::camera/render-sync] (mapv :command (get-in prep [:phases 0 :events])))) + (t/is (= "invalid-args" (:phase repeated))))) + (t/is (= [:preparation :main :reset :preparation :main :reset] @visits)) + (t/is (= 2 (effect-count (read-calls) "create"))) + (t/is (= 2 (effect-count (read-calls) "render")))) + (finally + (cleanup))))))) + +(t/deftest ^:async preparation-policy-keeps-selected-events-and-empty-main + (doseq [context [:fresh :reuse] + render [nil :none]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (unchecked-set js/globalThis "__benchNow" 0) + (register-phase-case + {:params {:count 2} + :context context + :preparation (cond-> {:callback #(-> % (camera/render-sync*) (camera/render-sync*))} + render (assoc :render render))} + identity) + (let [evidence (transit/decode-str (await (browser/load-scene (scene-args 7 {})))) + prep (:preparation evidence) + block (first (:phases prep))] + (t/is (= "ok" (:status evidence))) + (t/is (= :completed (:outcome block))) + (t/is (= [{:command ::camera/render-sync :metrics {:command/elapsed [0]}} + {:command ::camera/render-sync :metrics {:command/elapsed [0]}}] + (:events block))) + (t/is (= [{:phase :main :preparation-id (:preparation-id prep) + :outcome :completed :events []}] + (:phases evidence))) + (t/is (= (if (= render :none) 0 1) (effect-count (read-calls) "render")))) + (finally + (cleanup)))))))) + +(t/deftest ^:async reused-context-carries-view-without-a-reset + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [views (atom [])] + (register-phase-case {:params {:count 2} :context :reuse} + (fn [rtx] + (swap! views conj (:view rtx)) + (camera/pan rtx {:dx 10 :dy 5 :steps 1 :settle-ms 0}))) + (let [prepared (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + request (transit/encode-str {:preparation-id (get-in prepared [:preparation :preparation-id])}) + first (transit/decode-str (await (browser/run-prepared request))) + second (transit/decode-str (await (browser/run-prepared request)))] + (t/is (= ["ok" "ok"] (mapv :status [first second]))) + (t/is (= [{:scale 1 :x 0 :y 0} {:scale 1 :x 10 :y 5}] @views)) + (t/is (= [[] []] (mapv #(get-in % [:phases 0 :events]) [first second]))))) + (finally + (cleanup))))))) + +(t/deftest ^:async diagnostic-encoding-keeps-phase-evidence-and-releases-its-owner + (await + (with-entry-env + (^:async fn [cleanup] + (try + (register-phase-case + {:params {:count 2} :context :reuse} + (fn [rtx] + (-> (camera/render-sync* rtx) + (.then (fn [_] + (throw (ex-info "main fixture failed" + {:command "fixture" :path "scene.transit" + :partial {:safe {:elapsed [0 js/Infinity js/NaN]} + :handle (fn [] nil)}}))))))) + (let [request (transit/encode-str (assoc (transit/decode-str (scene-args 7 {})) + :preparation-id "encoding")) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + response (await (browser/run-prepared (transit/encode-str {:preparation-id "encoding"}))) + evidence (transit/decode-str response) + attempt (measurement/attempt :rects/lifecycle "encoding" false evidence) + saved (-> (result/create-run + {:run-id "encoding" :started-at "start" + :plan {:cases [(load-case)] :warmups 0 :repetitions 1} + :metadata {:git {:sha nil :dirty nil} :build {:functional {}} + :environment {} :scored? true}}) + (result/record-preparation (assoc (:preparation prepared) :warmup? false)) + (result/record-attempt attempt) + (result/record-case-outcome :rects/lifecycle {:outcome :failed}) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode)] + (t/is (= "failed" (:status evidence))) + (t/is (= "main" (:phase evidence))) + (t/is (= "main fixture failed" (:message evidence))) + (t/is (= "main fixture failed" (get-in evidence [:cause 0 :message]))) + (t/is (= "fixture" (:command evidence))) + (t/is (= "scene.transit" (:path evidence))) + (t/is (= [::camera/render-sync] (mapv :command (get-in evidence [:phases 0 :events])))) + (t/is (= (:phases evidence) (:phases attempt))) + (t/is (= (:preparation prepared) (:preparation evidence))) + (t/is (js/Number.isNaN (get-in saved [:attempts 0 :partial :safe :elapsed 2]))) + (t/is (some #(= :encode-diagnostic (:operation %)) (get-in saved [:attempts 0 :failure :errors]))) + (t/is (= "invalid-args" + (:phase (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "encoding"})))))))) + (register-phase-case {:params {:count 2}} camera/render-sync*) + (t/is (= "ok" (:status (transit/decode-str (await (browser/load-scene (scene-args 8 {}))))))) + (finally + (cleanup))))))) + +(t/deftest ^:async malformed-preparation-partial-keeps-the-original-failure-and-setup + (await + (with-entry-env + (^:async fn [cleanup] + (try + (register-phase-case + {:params {:count 2} + :preparation {:callback (fn [rtx] + (.then (camera/render-sync* rtx) + (fn [_] (throw (ex-info "preparation failed" {:partial 42})))))}} + camera/render-sync*) + (let [request (transit/encode-str (assoc (transit/decode-str (scene-args 7 {})) + :preparation-id "malformed")) + data (transit/decode-str (await (browser/prepare-scene-request request)))] + (t/is (= "preparation failed" (:message data))) + (t/is (= "preparation" (:phase data))) + (t/is (= [:preparation] (mapv :phase (:phases data)))) + (t/is (number? (get-in data [:partial :setup :upload-ms]))) + (t/is (some #(= :encode-diagnostic (:operation %)) (:errors data))) + (t/is (= "invalid-args" + (:phase (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "malformed"})))))))) + (finally (cleanup))))))) + +(t/deftest ^:async response-encoding-failure-retains-completed-preparation-work + (await + (with-entry-env + (^:async fn [cleanup] + (let [create-element (.-createElement js/document)] + (try + (set! (.-createElement js/document) + (fn [_] + (let [canvas (fake-canvas 1 nil) + gl (.getContext canvas "webgl2")] + (aset gl "drawingBufferWidth" (fn [] nil)) + canvas))) + (let [request (transit/encode-str (assoc (transit/decode-str (scene-args 7 {})) + :preparation-id "fallback")) + data (transit/decode-str (await (browser/prepare-scene-request request)))] + (t/is (= "failed" (:status data))) + (t/is (= :encode-response (:operation data))) + (t/is (= [:preparation] (mapv :phase (:phases data)))) + (t/is (= :completed (get-in data [:phases 0 :outcome]))) + (t/is (seq (:cause data))) + (t/is (= "invalid-args" + (:phase (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "fallback"})))))))) + (set! (.-createElement js/document) create-element) + (t/is (= "ok" (:status (transit/decode-str (await (browser/load-scene (scene-args 8 {}))))))) + (finally + (set! (.-createElement js/document) create-element) + (cleanup)))))))) + +(t/deftest ^:async phase-failure-keeps-raw-evidence-and-stops-the-context + (doseq [context [:fresh :reuse] + failed-phase [:preparation :main :reset]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (unchecked-set js/globalThis "__benchNow" 0) + (let [visits (atom []) + work (fn [phase rtx] + (swap! visits conj phase) + (-> (camera/render-sync* rtx) + (.then (fn [next-rtx] + (when (= phase failed-phase) + (throw (js/Error. "phase fixture failed"))) + next-rtx))))] + (register-phase-case {:params {:count 2} :context context + :preparation {:callback #(work :preparation %)} + :reset #(work :reset %)} + #(work :main %)) + (let [request (-> (transit/decode-str (scene-args 7 {})) + (assoc :preparation-id "phase-failure") + (transit/encode-str)) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + run (transit/encode-str {:preparation-id "phase-failure"}) + evidence (if (= failed-phase :preparation) + prepared + (transit/decode-str (await (browser/run-prepared run)))) + stopped (transit/decode-str (await (browser/run-prepared run))) + blocks (:phases evidence) + diagnostic (errors/project-diagnostic evidence) + preparation (if (= failed-phase :preparation) + {:preparation-id "phase-failure" :case-id :rects/lifecycle + :warmup? false :diagnostics (get-in evidence [:partial :setup]) + :phases blocks :failure diagnostic} + (assoc (:preparation prepared) :warmup? false)) + saved (-> (cond-> + (-> (result/create-run + {:run-id "phase-failure" :started-at "start" + :plan {:cases [(load-case)] :warmups 0 :repetitions 1} + :metadata {:git {:sha nil :dirty nil} :build {:functional {}} + :environment {} :scored? true}}) + (result/record-preparation preparation)) + (or (= context :fresh) (not= failed-phase :preparation)) + (result/record-attempt + (measurement/attempt :rects/lifecycle "phase-failure" false + (if (= failed-phase :preparation) + (dissoc evidence :phases) + evidence)))) + (result/record-case-outcome :rects/lifecycle + {:outcome :failed}) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + stored (if (= failed-phase :preparation) + (first (:preparations saved)) + (first (:attempts saved)))] + (t/is (= "failed" (:status evidence))) + (t/is (= (name failed-phase) (:phase evidence))) + (t/is (= "phase fixture failed" (:message evidence))) + (t/is (= (:cause evidence) (get-in stored [:failure :cause]))) + (t/is (= blocks (:phases stored))) + (t/is (= "phase-failure" (:preparation-id stored))) + (t/is (= 1 (count (re-seq #"phase fixture failed" + (format/format-run (summary/summarize saved)))))) + (t/is (= (case failed-phase + :preparation [:preparation] + :main [:preparation :main] + :reset [:preparation :main :reset]) + @visits)) + (t/is (= (if (= failed-phase :reset) [:main :reset] [failed-phase]) + (mapv :phase blocks))) + (t/is (= :failed (:outcome (last blocks)))) + (t/is (= (repeat (count blocks) + [{:command ::camera/render-sync :metrics {:command/elapsed [0]}}]) + (map :events blocks))) + (when (= failed-phase :preparation) + (let [setup (get-in evidence [:partial :setup])] + (t/is (number? (:module-ms setup))) + (t/is (number? (:graphics-ms setup))) + (t/is (number? (:upload-ms setup))) + (t/is (string? (get-in setup [:fingerprint :digest]))))) + (when-not (= failed-phase :preparation) + (t/is (= (:preparation prepared) (:preparation evidence))) + (t/is (= :completed (get-in evidence [:preparation :phases 0 :outcome])))) + (t/is (= "invalid-args" (:phase stopped))) + (t/is (= 1 (effect-count (read-calls) "remove")))) + (register-phase-case {:params {:count 2}} camera/render-sync*) + (let [next-case (transit/decode-str (await (browser/load-scene (scene-args 8 {}))))] + (t/is (= "ok" (:status next-case))) + (t/is (= [::camera/render-sync] + (mapv :command (get-in next-case [:phases 0 :events])))))) + (finally + (cleanup)))))))) + +(t/deftest ^:async superseded-phase-failure-keeps-its-own-preparation + (doseq [failed-phase [:main :reset]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (let [entered (volatile! nil) + reject (volatile! nil) + started (js/Promise. (fn [resolve _reject] (vreset! entered resolve))) + work (fn [phase rtx] + (-> (camera/render-sync* rtx) + (.then (fn [next-rtx] + (if (= phase failed-phase) + (js/Promise. (fn [_resolve fail] + (vreset! reject fail) + (@entered nil))) + next-rtx)))))] + (register-phase-case {:params {:count 2} :context :reuse + :reset #(work :reset %)} + #(work :main %)) + (let [request (-> (transit/decode-str (scene-args 7 {})) + (assoc :preparation-id "first") + (transit/encode-str)) + prepared (transit/decode-str (await (browser/prepare-scene-request request))) + first-run (browser/run-prepared (transit/encode-str {:preparation-id "first"}))] + (await started) + (register-phase-case {:params {:count 2}} camera/render-sync*) + (let [next-request (-> (transit/decode-str (scene-args 8 {})) + (assoc :preparation-id "second") + (transit/encode-str)) + next-prepared (transit/decode-str (await (browser/prepare-scene-request next-request)))] + (@reject (js/Error. "late phase failure")) + (let [failed (transit/decode-str (await first-run)) + next-run (transit/decode-str + (await (browser/run-prepared (transit/encode-str {:preparation-id "second"}))))] + (t/is (= "failed" (:status failed))) + (t/is (= (name failed-phase) (:phase failed))) + (t/is (= (:preparation prepared) (:preparation failed))) + (t/is (= (if (= failed-phase :main) [:main] [:main :reset]) + (mapv :phase (:phases failed)))) + (t/is (= :failed (:outcome (last (:phases failed))))) + (t/is (= (repeat (count (:phases failed)) "first") + (map :preparation-id (:phases failed)))) + (t/is (= ["ok" "ok"] (mapv :status [next-prepared next-run]))) + (t/is (= "second" (get-in next-run [:preparation :preparation-id]))))))) + (finally + (cleanup)))))))) + +(t/deftest rejected-camera-body-releases-owner + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" "_render_from_cache") + (register-phase-case {:context :reuse} + #(camera/pan* % {:dx 10 :dy 5 :steps 2 :settle-ms 0})) + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "main" (:phase m))) + (t/is (= 1 (effect-count (read-calls) "remove"))) + (t/is (nil? @@#'browser/canvas*)) + (cleanup) + (done))))))) + +(t/deftest rejected-import-resolves-module-init-failure + (t/async done + ;; Needs the entry env for the import polyfill, even though the import + ;; itself fails before touching the DOM. + (with-entry-env + (fn [cleanup] + (load-result (transit/encode-str + {:case (assoc (load-case) :scene-seed 7) + :module-url "data:text/javascript,this is not valid javascript((("}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "module-init" (:phase m))) + (t/is (= {} (get-in m [:partial :setup]))) + (cleanup) + (done))))))) + +(defn- assert-wasm-cause + "Shared cause assertions for WASM failures through `load-scene`: plain-data + vector, wire contract, failing fn name, fake message and WASM code." + [cause fn-name message-pattern] + (t/is (vector? cause) "cause is plain-data vector") + (t/is (decl/transit-round-trips? cause) "cause satisfies wire contract") + (t/is (some #(= fn-name (:fn %)) cause) "cause names the failing WASM fn") + (t/is (some #(and (string? (:message %)) + (re-find message-pattern (:message %))) + cause) + "fake message preserved") + (t/is (some #(let [code (:code %)] + (and (some? code) + (re-find #"wasm-critical" (str code)))) + cause) + "WASM code preserved")) + +(t/deftest set-browser-failure-releases-registered-context + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" "_set_browser") + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "graphics-init" (:phase m))) + (t/is (= {} (get-in m [:partial :setup]))) + (t/is (= "graphics init failed" (:message m)) + "top message stays phase-labeled") + (assert-wasm-cause (:cause m) + "_set_browser" + #"fake _set_browser failure") + (let [calls (read-calls)] + (t/is (= 1 (effect-count calls "register"))) + (t/is (= 1 (effect-count calls "clean")) "renderer state is released") + (t/is (some #{["delete" 7]} calls) "the registered handle is deleted") + (t/is (= 1 (effect-count calls "lose")) "the browser context is released") + (t/is (= 1 (effect-count calls "remove")) "the canvas is removed")) + (t/is (nil? @@#'browser/canvas*) "no owner left behind") + (cleanup) + (done))))))) + +(t/deftest upload-failure-keeps-wasm-cause + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" "_set_view") + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "upload" (:phase m))) + (t/is (= "upload failed" (:message m)) + "top message stays phase-labeled") + (assert-wasm-cause (:cause m) + "_set_view" + #"fake _set_view failure") + (let [setup (get-in m [:partial :setup])] + (t/is (number? (:module-ms setup))) + (t/is (number? (:graphics-ms setup))) + (t/is (string? (get-in setup [:fingerprint :digest])))) + (cleanup) + (done))))))) + +(t/deftest ^:async upload-retains-batch-and-end-loading-failures-in-order + (doseq [functions [["_set_shapes_batch" "_end_loading"] ["_end_loading"]]] + (await + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchThrowIn" (clj->js functions)) + (load-result (scene-args 7 {}) + (fn [m] + (t/is (= "failed" (:status m))) + (t/is (= "upload failed" (:message m))) + (t/is (= (mapv {"_set_shapes_batch" :serialize-shapes "_end_loading" :end-loading} functions) + (mapv :operation (:errors m)))) + (t/is (= functions (mapv #(get-in % [:cause 0 :fn]) (:errors m)))) + (t/is (decl/transit-round-trips? m)) + (cleanup)))))))) + +(t/deftest nested-stale-marker-does-not-resolve-stale + (let [terminate @#'browser/terminal-failure + current @@#'browser/owner-epoch* + nested (ex-info "upload failed" + {:phase "upload"} + (ex-info "superseded" + {::errors/stale true + :phase "upload"})) + result (terminate current nested "aborted") + m result] + (t/is (= "failed" (:status m)) "only the top level decides staleness") + (t/is (= "upload" (:phase m))) + (t/is (vector? (:cause m)) "cause still rendered"))) + +(t/deftest overlapping-load-keeps-the-newer-owner + (t/async done + (with-entry-env + (fn [cleanup] + (unchecked-set js/globalThis "__benchGateRaf" true) + (let [args (fn [seed] (scene-args seed {})) + first-load (browser/load-scene (args 7))] + ;; Microtasks drain before this macrotask: the first load parked on + ;; its first frame with a live canvas. Macrotask ordering + ;; (never wall timing) sequences the assertions. + (js/setTimeout + (fn [] + (t/is (some? (unchecked-get js/globalThis "__benchRafCb")) + "first load is waiting for a frame") + (let [second-load (browser/load-scene (args 8))] + (js/setTimeout + (fn [] + ;; The second load completed; release the parked frame so + ;; the first load observes its superseded epoch. + (if-some [parked (unchecked-get js/globalThis "__benchRafCb")] + (parked 0) + (t/is false "first load never parked on a frame")) + (-> (js/Promise.all #js [first-load second-load]) + (.then (fn [results] + (let [[stale ok] (mapv transit/decode-str (array-seq results))] + (t/is (= "stale" (:status stale)) "superseded load resolves stale") + (t/is (= "ok" (:status ok)) "newer owner completes") + (t/is (= 8 (:seed (:scene ok))) "the completer is the second load") + (let [calls (read-calls)] + (t/is (= 1 (effect-count calls "remove")) "superseded canvas is removed") + (t/is (some #{["remove" 1]} calls)) + (t/is (nil? (some #{["remove" 2]} calls)) "newer canvas survives")) + (t/is (some? @@#'browser/canvas*) "the newer owner stays live") + (t/is (true? wasm/context-initialized?))) + (cleanup) + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (cleanup) + (done))))) + 0))) + 0)))))) diff --git a/frontend/test/performance_tests/render_wasm/log_test.cljs b/frontend/test/performance_tests/render_wasm/log_test.cljs new file mode 100644 index 0000000000..d12468acc7 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/log_test.cljs @@ -0,0 +1,161 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.log-test + (:require + ["fs" :as fs] + ["node:stream" :as stream] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [benches.render-wasm.node.log :as log] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "bench-log-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- capture + [] + (let [text (atom "") + out (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))})] + {:stream out :text text})) + +(t/deftest levels-filter-console-and-retain-every-line-in-the-file + (doseq [[level expected] [[:warn "! warning\n✖ failure\n"] + [:info "Phase\n attempt 1/1 completed\n! warning\n✖ failure\n"] + [:debug "Phase\n attempt 1/1 completed\n· raw child\n! warning\n✖ failure\n"]]] + (let [stdout (capture) + stderr (capture) + file (path/join @temporary-root (name level))] + (log/configure {:level level :color :never + :stdout (:stream stdout) :stderr (:stream stderr)}) + (binding [log/*log-path* file] + (log/info "Phase") + (log/info " attempt 1/1 completed") + (log/debug "raw child") + (log/warn "warning") + (log/error {:message "failure" :tail? false}) + (log/result "Run saved")) + (t/is (= "Run saved\n" @(:text stdout))) + (t/is (= (str/replace expected "✖ failure\n" (str "✖ failure\n log " file "\n")) + @(:text stderr))) + (t/is (= (str "Phase\n attempt 1/1 completed\nraw child\nwarning\n✖ failure\n log " + file "\nRun saved\n") + (fs/readFileSync file "utf8")))))) + +(t/deftest colors-apply-to-console-headings-and-preserve-indented-progress + (doseq [[mode tty no-color term colored?] + [[:always false "1" "dumb" true] [:never true "" "xterm" false] + [:auto false "" "xterm" false] [:auto true "" "xterm" true] + [:auto true "1" "xterm" false] [:auto true "" "dumb" false]]] + (let [stdout (capture) + stderr (capture) + file (path/join @temporary-root (str (random-uuid))) + env (.-env js/process) + old-no-color (.-NO_COLOR env) + old-term (.-TERM env)] + (try + (set! (.-NO_COLOR env) no-color) + (set! (.-TERM env) term) + (set! (.-isTTY ^js (:stream stderr)) tty) + (log/configure {:color mode :stdout (:stream stdout) :stderr (:stream stderr)}) + (binding [log/*log-path* file] + (log/info "Phase\n progress") + (log/result "Output /tmp/run")) + (t/is (= colored? (str/includes? @(:text stderr) "\u001b["))) + (t/is (= "Phase\n progress\n" + (str/replace @(:text stderr) #"\u001b\[[0-9;]*m" ""))) + (t/is (= "Output /tmp/run\n" @(:text stdout))) + (t/is (= "Phase\n progress\nOutput /tmp/run\n" (fs/readFileSync file "utf8"))) + (finally + (if (nil? old-no-color) (js-delete env "NO_COLOR") (set! (.-NO_COLOR env) old-no-color)) + (if (nil? old-term) (js-delete env "TERM") (set! (.-TERM env) old-term))))))) + +(t/deftest ^:async nested-tasks-route-lines-to-their-own-files-on-success-and-failure + (let [parent (path/join @temporary-root "comparison.log") + child (path/join @temporary-root "build.log") + output (capture)] + (log/configure {:level :warn :stdout (:stream output) :stderr (:stream output)}) + (binding [log/*log-path* parent] + (log/debug "comparison before") + (await ((^:async fn [] + (binding [log/*log-path* child] + (await (timers/setImmediate)) + (log/debug "child success"))))) + (log/debug "comparison between") + (try + (await ((^:async fn [] + (binding [log/*log-path* child] + (await (timers/setImmediate)) + (log/debug "child failure") + (throw (js/Error. "failed")))))) + (catch :default _ (log/debug "comparison after")))) + (t/is (= "comparison before\ncomparison between\ncomparison after\n" + (fs/readFileSync parent "utf8"))) + (t/is (= "child success\nchild failure\n" (fs/readFileSync child "utf8"))))) + +(t/deftest console-diagnostics-escape-controls-and-retain-the-log-evidence + (let [file (path/join @temporary-root "controls.log") + output (capture) + message (str "work failed\u001b\u0007" (apply str (repeat 2000 "x")))] + (log/configure {:color :never :stderr (:stream output)}) + (binding [log/*log-path* file] + (log/warn message)) + (t/is (= (str message "\n") (fs/readFileSync file "utf8"))) + (t/is (str/includes? @(:text output) "\\u001b\\u0007")) + (t/is (< (count @(:text output)) 1100)) + (doseq [separator ["" "\n"]] + (reset! (:text output) "") + (fs/writeFileSync file (str (apply str (repeat 2000 "\u0007")) separator "FATAL: compiler failed\n") "utf8") + (log/error {:message "tail evidence" :tail-log file}) + (t/is (str/includes? @(:text output) "FATAL: compiler failed")) + (t/is (every? #(let [code (.charCodeAt % 0)] + (or (= code 10) (and (>= code 32) (or (< code 127) (> code 159))))) + (seq @(:text output))))))) + +(t/deftest error-names-tail-source-and-bounds-evidence-before-appending + (let [file (path/join @temporary-root "comparison.log") + child (path/join @temporary-root "build.log") + output (capture)] + (fs/writeFileSync child (str "excluded\n" (apply str (repeat 9000 "x")) "\n" + (str/join "\n" (map #(str "line-" %) (range 30)))) "utf8") + (log/configure {:color :never :stderr (:stream output)}) + (binding [log/*log-path* file] + (log/error {:message "build failed" :fields [[:phase :renderer] [:build "id"]] + :tail-log child})) + (let [expected (str "✖ build failed\n phase :renderer\n build id\n" + " log " file "\n tail-log " child "\n tail\n" + (str/join "\n" (map #(str " line-" %) (range 10 30))) "\n")] + (t/is (= expected @(:text output))) + (t/is (= expected (fs/readFileSync file "utf8")))) + (fs/writeFileSync child (apply str (repeat 10000 "z")) "utf8") + (reset! (:text output) "") + (binding [log/*log-path* file] + (log/error {:message "bounded" :tail-log child})) + (t/is (= 8192 (count (re-find #"z+$" (str/trim @(:text output)))))) + (fs/writeFileSync file "last line\n" "utf8") + (reset! (:text output) "") + (binding [log/*log-path* file] + (log/error {:message "same source"})) + (t/is (= (str "✖ same source\n log " file "\n tail\n last line\n") + @(:text output))))) + +(t/deftest error-reporting-survives-unreadable-tail-and-file-append-failure + (let [output (capture)] + (log/configure {:color :never :stderr (:stream output)}) + (binding [log/*log-path* @temporary-root] + (log/error {:message "original failure" :tail-log (path/join @temporary-root "missing")})) + (t/is (str/starts-with? @(:text output) "✖ original failure\n")) + (t/is (str/includes? @(:text output) @temporary-root)))) diff --git a/frontend/test/performance_tests/render_wasm/merge_test.cljs b/frontend/test/performance_tests/render_wasm/merge_test.cljs new file mode 100644 index 0000000000..8e4ee8e042 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/merge_test.cljs @@ -0,0 +1,219 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.merge-test + (:require + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true])) + +(defn- metric + [value] + {:command :fixture/work :metrics {:command/elapsed [value]}}) + +(defn- case-definition + [case-id] + {:id case-id + :scene :case + :scene-description "merge scene" + :scene-seed 7 + :params {:count 10} + :view {:scale 1 :x 0 :y 0} + :context :fresh + :preparation {}}) + +(defn- preparation + [preparation-id case-id] + {:preparation-id preparation-id + :case-id case-id :warmup? false + :fingerprint {:algorithm :sha-256 + :digest (apply str (repeat 64 "a"))} + :effective-graphics {:webgl2 true} + :diagnostics {} + :phases [{:phase :preparation :preparation-id preparation-id :outcome :completed :events []}]}) + +(defn- attempt + [case-id preparation-id warmup?] + {:case-id case-id + :preparation-id preparation-id + :warmup? warmup? + :outcome :completed + :phases [{:phase :main :preparation-id preparation-id + :outcome :completed :events [(metric 1.0)]}]}) + +(defn- make-run + [{:keys [run-id cases warmups repetitions git-sha build environment analysis + timeout-ms started-at ended-at termination preparations] + :or {git-sha "same-sha" + build {:mode :release} + environment {:node "v24"} + timeout-ms 30000 + started-at "2026-10-26T00:00:00.000Z" + ended-at "2026-10-26T00:01:00.000Z" + termination {:reason :completed}}}] + (let [run (result/create-run + {:run-id run-id + :started-at started-at + :plan {:cases (mapv case-definition cases) + :warmups warmups + :repetitions repetitions + :timeout-policy {:case-ms timeout-ms}} + :metadata {:git {:sha git-sha :dirty false} + :build {:functional build} + :environment environment + :scored? true} + :analysis analysis}) + run (reduce + (fn [run case-id] + (let [options (get preparations case-id) + warmup-id (or (:warmup options) + (str run-id "-" (name case-id))) + measured-id (or (:measured options) warmup-id)] + (-> run + (result/record-preparation (preparation warmup-id case-id)) + (cond-> (not= warmup-id measured-id) + (result/record-preparation (preparation measured-id case-id))) + (as-> run + (reduce (fn [run _] + (result/record-attempt + run (attempt case-id warmup-id true))) + run (range warmups)) + (reduce (fn [run _] + (result/record-attempt + run (attempt case-id measured-id false))) + run (range repetitions)))))) + run cases) + run (if (= :completed (:reason termination)) + (reduce #(result/record-case-outcome %1 %2 {:outcome :completed}) run cases) + run)] + (result/finish-run run ended-at termination))) + +(t/deftest merges-measured-attempts-and-updates-the-plan + (let [base (make-run {:run-id "base" + :cases [:case/load :case/pan] + :warmups 1 + :repetitions 2 + :started-at "2026-10-26T00:00:00.000Z" + :ended-at "2026-10-26T00:01:00.000Z"}) + additional (make-run {:run-id "additional" + :cases [:case/load :case/pan] + :warmups 1 + :repetitions 3 + :started-at "2026-10-26T00:02:00.000Z" + :ended-at "2026-10-26T00:03:00.000Z"}) + merged (result/merge-runs base additional {:run-id "merged"})] + (t/is (= "merged" (:run-id merged))) + (t/is (= 1 (get-in merged [:plan :warmups]))) + (t/is (= 5 (get-in merged [:plan :repetitions]))) + (t/is (= 12 (count (:attempts merged)))) + (t/is (= (range 12) (map :attempt-id (:attempts merged)))) + (t/is (= [true false false true false false false false false false false false] + (map :warmup? (:attempts merged)))) + (t/is (every? #(every? zero? (vals %)) (vals (:remaining merged)))) + (t/is (= "2026-10-26T00:03:00.000Z" (:ended-at merged))) + (t/is (= 2 (get-in merged [:metadata :diagnostics :merge + :additional-warmups-excluded]))) + (t/is (= merged (result/check-run merged))) + (t/is (= (:attempts merged) + (:attempts (result/decode (result/encode merged))))))) + +(t/deftest excludes-additional-warmup-only-preparations + (let [base (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1}) + additional (make-run {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :preparations + {:case/load {:warmup "warm-prep" + :measured "measured-prep"}}}) + merged (result/merge-runs base additional {:run-id "merged"}) + preparation-ids (map :preparation-id (:preparations merged))] + (t/is (some #{"base-load"} preparation-ids)) + (t/is (some #{"measured-prep"} preparation-ids)) + (t/is (not (some #{"warm-prep"} preparation-ids))) + (t/is (= 3 (count (:attempts merged)))) + (t/is (= "measured-prep" (:preparation-id (last (:attempts merged))))))) + +(t/deftest rejects-incompatible-runs + (let [base (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1})] + (doseq [[label options message] + [["case" {:cases [:case/pan]} #"different case definitions"] + ["timeout" {:timeout-ms 1000} #"different timeout policy"] + ["warmups" {:warmups 0} #"different warmup count"] + ["analysis" {:analysis {:draws 1000}} #"different analysis settings"] + ["git" {:git-sha "other"} #"different git provenance"] + ["build" {:build {:mode :debug}} #"different build facts"] + ["environment" {:environment {:node "v25"}} + #"different runtime environment"]]] + (t/testing label + (let [additional (make-run (merge {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1} + options))] + (t/is (thrown-with-msg? js/Error message + (result/merge-runs base additional + {:run-id "merged"})))))))) + +(t/deftest rejects-non-completed-runs + (let [completed (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1}) + interrupted (make-run {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :termination {:reason :interrupted}})] + (t/is (thrown-with-msg? js/Error #"only completed runs can be merged" + (result/merge-runs completed interrupted + {:run-id "merged"}))) + (t/is (thrown-with-msg? js/Error #"only completed runs can be merged" + (result/merge-runs interrupted completed + {:run-id "merged"}))))) + +(t/deftest rejects-duplicate-preparation-ids + (let [base (make-run {:run-id "base" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :preparations + {:case/load {:warmup "shared" + :measured "shared"}}}) + additional (make-run {:run-id "additional" + :cases [:case/load] + :warmups 1 + :repetitions 1 + :preparations + {:case/load {:warmup "shared" + :measured "shared"}}})] + (t/is (thrown-with-msg? js/Error #"duplicate preparation ids" + (result/merge-runs base additional + {:run-id "merged"}))))) + +(t/deftest quantity-prose-does-not-change-merge-compatibility + (let [base (make-run {:run-id "base" :cases [:case/load] :warmups 0 :repetitions 1}) + additional (-> (make-run {:run-id "additional" :cases [:case/load] :warmups 0 :repetitions 1}) + (assoc-in [:quantities :command/elapsed :description] "Elapsed command work.")) + merged (result/merge-runs base additional {:run-id "merged"})] + (t/is (= 2 (get-in merged [:plan :repetitions]))) + (t/is (= merged (result/decode (result/encode merged)))) + (let [malformed (assoc-in additional [:quantities :command/elapsed :description] 42) + report (summary/summarize malformed)] + (t/is (= :malformed-quantity + (get-in report [:cases 0 :metrics [:main :fixture/work :command/elapsed] :diagnostics 0 :reason]))) + (t/is (thrown-with-msg? js/Error #"malformed quantity descriptions" + (result/merge-runs base malformed {:run-id "invalid"})))) + (t/is (thrown-with-msg? js/Error #"different quantity units" + (result/merge-runs base + (assoc-in additional [:quantities :command/elapsed :unit] :seconds) + {:run-id "invalid"}))))) diff --git a/frontend/test/performance_tests/render_wasm/outcome_test.cljs b/frontend/test/performance_tests/render_wasm/outcome_test.cljs new file mode 100644 index 0000000000..e9ad786d34 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/outcome_test.cljs @@ -0,0 +1,47 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.outcome-test + (:require + [benches.render-wasm.node.outcome :as outcome] + [cljs.test :as t :include-macros true])) + +(t/deftest failed-reports-the-cause-with-exit-code-1-and-its-log-path + (let [cause (ex-info "boom" {:log-path "build.log"}) + result (outcome/failed cause {:home :h})] + (t/is (= [cause] (:errors result))) + (t/is (= 1 (:exit-code result))) + (t/is (= "build.log" (:log-path result))) + (t/is (= :h (:home result))))) + +(t/deftest failed-prefers-the-log-path-given-in-data + (let [result (outcome/failed (ex-info "boom" {:log-path "build.log"}) {:log-path "run.log"})] + (t/is (= "run.log" (:log-path result))))) + +(t/deftest add-error-appends-and-keeps-interruption-exit-codes + (doseq [[before after] [[0 1] [1 1] [130 130] [143 143]]] + (let [first-cause (js/Error. "first") + second-cause (js/Error. "second") + result (-> {:errors [first-cause] :exit-code before} + (outcome/add-error second-cause))] + (t/is (= [first-cause second-cause] (:errors result))) + (t/is (= after (:exit-code result)))))) + +(t/deftest publish-returns-the-outcome-when-printing-succeeds + (let [printed (atom nil) + result (outcome/publish {:exit-code 0} #(reset! printed :done))] + (t/is (= :done @printed)) + (t/is (= {:exit-code 0} result)))) + +(t/deftest publish-adds-a-failed-print-to-the-outcome + (let [cause (js/Error. "stream closed") + result (outcome/publish {:exit-code 0} #(throw cause))] + (t/is (= [cause] (:errors result))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest interruption-code-maps-signal-names + (t/is (= 143 (outcome/interruption-code "SIGTERM"))) + (t/is (= 130 (outcome/interruption-code "SIGINT")))) diff --git a/frontend/test/performance_tests/render_wasm/process_test.cljs b/frontend/test/performance_tests/render_wasm/process_test.cljs new file mode 100644 index 0000000000..65862e7bb8 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/process_test.cljs @@ -0,0 +1,615 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.process-test + "Real Node/POSIX processes, files and Linux flock ownership." + (:require + ["child_process" :as child-process] + ["fs" :as fs] + ["node:module" :as node-module] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.process :as process] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private temp-root (atom nil)) +(def ^:private test-grace-ms 100) + +(defn- file + [name] + (path/join @temp-root name)) + +(defn- pause + [ms] + (js/Promise. (fn [resolve _] (js/setTimeout resolve ms)))) + +(defn- within + "A broken lifecycle must fail the test rather than stall the test runner." + ([promise] (within promise 5000)) + ([promise timeout-ms] + (let [timer (volatile! nil)] + (-> (js/Promise.race + #js [promise + (js/Promise. + (fn [_ reject] + (vreset! timer + (js/setTimeout + #(reject (ex-info "Process test stalled" {:type :test-watchdog})) + timeout-ms))))]) + (.finally #(js/clearTimeout @timer)))))) + +(defn- ^:async with-live-group + [faults f] + (let [^js native-fs ((node-module/createRequire (path/join (.cwd js/process) "package.json")) "fs") + kill (.-kill js/process) + read-dir (.-readdirSync native-fs) + read-file (.-readFileSync native-fs) + group (atom nil) + signals (atom []) + probed (atom false) + fake-pid "2147483646"] + (set! (.-kill js/process) + (fn [pid signal] + (if (neg? pid) + (do + (reset! group (- pid)) + (when-not (:keep-child faults) + (try + (.call kill js/process pid signal) + (catch :default cause + (when-not (= "ESRCH" (.-code cause)) (throw cause))))) + (when (string? signal) + (swap! signals conj {:signal signal :time-ms (js/Date.now)})) + (when-let [cause (get faults signal)] (throw cause)) + true) + (.call kill js/process pid signal)))) + (set! (.-readdirSync native-fs) + (fn [location & args] + (if (and (= "/proc" location) @group) + (if (and (:probe faults) (compare-and-set! probed false true)) + (throw (:probe faults)) + (.concat (apply read-dir location args) #js [fake-pid])) + (apply read-dir location args)))) + (set! (.-readFileSync native-fs) + (fn [location & args] + (if (and @group (= location (str "/proc/" fake-pid "/stat"))) + (str fake-pid " (unresolved) S 0 " @group " " @group " 0") + (apply read-file location args)))) + (node-module/syncBuiltinESMExports) + (try + (await (f signals)) + (finally + (when (and (:keep-child faults) @group) + (try + (.call kill js/process (- @group) "SIGKILL") + (catch :default cause + (when-not (= "ESRCH" (.-code cause)) (throw cause))))) + (set! (.-kill js/process) kill) + (set! (.-readdirSync native-fs) read-dir) + (set! (.-readFileSync native-fs) read-file) + (node-module/syncBuiltinESMExports))))) + +(defn- ^:async wait-for + [pred] + (let [limit (process/deadline 3000)] + (loop [] + (when-not (pred) + (when (zero? (process/remaining limit)) + (throw (ex-info "Process test did not become ready" {:type :test-watchdog}))) + (await (pause 10)) + (recur))))) + +(defn- process-status + [pid] + (try + (let [stat (fs/readFileSync (str "/proc/" pid "/stat") "utf8") + fields (str/split (str/trim (subs stat (inc (.lastIndexOf stat ")")))) #"\s+")] + {:state (first fields) + :group (js/Number (nth fields 2))}) + (catch :default cause + (if (contains? #{"ENOENT" "ESRCH"} (.-code cause)) + nil + (throw cause))))) + +(defn- running? + [pid] + (when-let [status (process-status pid)] + ;; An orphan can remain as a zombie until init reaps it. It owns no pipes + ;; or locks and cannot run; kill(pid, 0) alone does not distinguish it. + (not (contains? #{"Z" "X" "x"} (:state status))))) + +(defn- read-pid + [name] + (js/Number (fs/readFileSync (file name) "utf8"))) + +(t/use-fixtures + :each + {:before + (fn [] + (reset! temp-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-wasm-process-test-")))) + :after + (fn [] + (t/async done + (let [cleanup (^:async fn [] + (try + (let [pids (->> (array-seq (fs/readdirSync @temp-root)) + (filter #(str/ends-with? % ".pid")) + (mapv read-pid))] + ;; Also clean up when an assertion or watchdog exposes a regression. + ;; Signal individual recorded PIDs so a missing detached flag cannot + ;; make test teardown kill the test runner's own process group. + (doseq [pid pids] + (when (running? pid) + (try + (.kill js/process pid "SIGKILL") + (catch :default cause + (when-not (= "ESRCH" (.-code cause)) + (throw cause)))))) + (await (wait-for #(not-any? running? pids)))) + (finally + (try + (fs/rmSync @temp-root #js {:recursive true :force true}) + (finally + (reset! temp-root nil))))))] + (.then (cleanup) + (fn [_] (done)) + (fn [cause] + (try + (t/is false (str "Process test cleanup failed: " cause)) + (finally + (done))))))))}) + +(defn- node-source + [pid-file source] + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(" (pr-str pid-file) ", String(process.pid));\n" + source)) + +(defn- node-options + [source overrides] + (merge {:command (.-execPath js/process) + :args ["-e" (node-source "root.pid" source)] + :cwd @temp-root + :deadline (process/deadline 3000) + :termination-grace-ms test-grace-ms} + overrides)) + +(t/deftest ^:async unresolved-group-termination-is-bounded-and-fatal + (await + (with-live-group + {} + (^:async fn [signals] + (let [started (js/Date.now) + cause (try + (await (within (process/run (node-options "process.exit(0);" {:termination-grace-ms 20})) 8000)) + nil + (catch :default cause cause)) + data (ex-data cause)] + (t/is (errors/fatal? cause)) + (t/is (= :process-unresolved (:type data))) + (t/is (= (.-execPath js/process) (:command data))) + (t/is (integer? (:pid data))) + (t/is (= (:pid data) (:pgid data))) + (t/is (= ["SIGTERM" "SIGKILL"] (:signals data))) + (t/is (= ["SIGTERM" "SIGKILL"] (mapv :signal @signals))) + (let [killed-at (some #(when (= "SIGKILL" (:signal %)) (:time-ms %)) @signals)] + (t/is (and (number? killed-at) (>= (- (js/Date.now) killed-at) 5000)))) + (t/is (< (- (js/Date.now) started) 8000)) + (t/is (not (running? (read-pid "root.pid"))))))))) + + +(t/deftest ^:async unresolved-owner-retains-a-live-lock-and-all-failures + (let [lock-file (file "unresolved.lock") + messages ["output handler failed" "TERM denied" "probe unavailable" "KILL denied"]] + (await + (with-live-group + {"SIGTERM" (js/Error. (nth messages 1)) + :probe (js/Error. (nth messages 2)) + "SIGKILL" (js/Error. (nth messages 3)) + :keep-child true} + (^:async fn [signals] + (let [options (node-options "" + {:command "flock" + :args ["--exclusive" lock-file (.-execPath js/process) "-e" + (node-source "root.pid" "process.stdout.write('ready'); setInterval(() => {}, 1000);")] + :deadline (process/deadline 30000) + :termination-grace-ms 20 + :on-output (fn [& _] (throw (js/Error. (first messages))))}) + cause (try + (await (within (process/run options) 8000)) + nil + (catch :default cause cause)) + entries (errors/get-errors cause) + wire (-> (errors/evidence :process cause) transit/encode-str transit/decode-str)] + (t/is (process/unresolved? cause)) + (t/is (errors/fatal? cause)) + (t/is (= [:process-output :process-signal :process-probe :process-signal :process-unresolved] + (mapv #(-> % :error ex-data :type) entries))) + (doseq [[entry message] (map vector (:errors wire) messages)] + (t/is (some #(= message (:message %)) (:cause entry)))) + (t/is (true? (:unresolved-process wire))) + (t/is (true? (:fatal wire))) + (t/is (= ["SIGTERM" "SIGKILL"] (mapv :signal @signals))) + (t/is (running? (read-pid "root.pid"))) + (t/is (= 1 (.-status (child-process/spawnSync "flock" #js ["--nonblock" lock-file "true"] + #js {:timeout 1000})))))))) + (await (wait-for #(zero? (.-status (child-process/spawnSync "flock" #js ["--nonblock" lock-file "true"] + #js {:timeout 1000}))))))) + + +(t/deftest ^:async input-stream-errors-have-one-occurrence-each-with-line-output + (let [^js native ((node-module/createRequire (path/join (.cwd js/process) "package.json")) "child_process") + spawn (.-spawn native)] + (set! (.-spawn native) + (fn [& args] + (let [^js child (apply spawn args)] + (.once (.-stdout child) "data" + (fn [_] + (.emit (.-stdout child) "error" (js/Error. "first read failed")) + (.emit (.-stdout child) "error" (js/Error. "second read failed")))) + child))) + (node-module/syncBuiltinESMExports) + (try + (let [cause (try + (await (within (process/run + (node-options "process.stdout.write('ready'); setInterval(() => {}, 1000);" + {:on-line (fn [& _] nil)})))) + nil + (catch :default cause cause)) + wire (errors/evidence :process cause)] + (t/is (= [:process-output :process-output] + (mapv #(-> % :error ex-data :type) (errors/get-errors cause)))) + (doseq [[entry message] (map vector (:errors wire) ["first read failed" "second read failed"])] + (t/is (some #(= message (:message %)) (:cause entry))))) + (finally + (set! (.-spawn native) spawn) + (node-module/syncBuiltinESMExports))))) + +(defn- ^:async rejected + [promise] + (try + (let [result (await (within promise))] + (t/is false (str "Expected a process failure, got " (pr-str result))) + nil) + (catch :default cause + (t/is (some? (ex-data cause)) "failures carry ex-info data") + (ex-data cause)))) + +(defn- stubborn-descendant + [] + (let [grandchild (node-source + "grandchild.pid" + (str "process.on('SIGTERM', () => fs.writeFileSync('grandchild.term', 'seen'));\n" + "fs.writeFileSync('grandchild.ready', 'ready');\n" + "setInterval(() => {}, 1000);\n"))] + (node-source + "descendant.pid" + (str "process.on('SIGTERM', () => fs.writeFileSync('descendant.term', 'seen'));\n" + "require('node:child_process').spawn(process.execPath,\n" + " ['-e', " (pr-str grandchild) "], {stdio: ['ignore', 'inherit', 'inherit']});\n" + "const ready = setInterval(() => {\n" + " if (fs.existsSync('grandchild.ready')) {\n" + " clearInterval(ready); fs.writeFileSync('descendant.ready', 'ready');\n" + " process.stdout.write('descendant-ready\\n');\n" + " }\n" + "}, 10);\n" + "setInterval(() => fs.appendFileSync('heartbeat', '.'), 10);\n")))) + +(defn- spawn-descendant + [] + (str "const child = require('node:child_process').spawn(process.execPath,\n" + " ['-e', " (pr-str (stubborn-descendant)) "],\n" + " {stdio: ['ignore', 'inherit', 'inherit']});\n")) + +(defn- ^:async assert-tree-stopped + [] + (t/is (not (running? (read-pid "root.pid"))) "the owner has stopped") + (t/is (not (running? (read-pid "descendant.pid"))) "the descendant has stopped at settlement") + (t/is (not (running? (read-pid "grandchild.pid"))) "the grandchild has stopped at settlement") + (t/is (= "seen" (fs/readFileSync (file "descendant.term") "utf8")) + "the owned group receives SIGTERM before SIGKILL") + (t/is (= "seen" (fs/readFileSync (file "grandchild.term") "utf8")) + "SIGTERM also reaches the grandchild") + (let [heartbeat (when (fs/existsSync (file "heartbeat")) + (fs/readFileSync (file "heartbeat") "utf8"))] + (await (pause 30)) + (t/is (= heartbeat (when (fs/existsSync (file "heartbeat")) + (fs/readFileSync (file "heartbeat") "utf8"))) + "no descendant writes after the promise settles"))) + +(t/deftest ^:async deadlines-use-date-now-and-clamp-at-zero + (let [before (js/Date.now) + limit (process/deadline 20)] + (t/is (<= (+ before 20) limit (+ (js/Date.now) 20))) + (t/is (<= 0 (process/remaining limit) 20)) + (await (pause 30)) + (t/is (zero? (process/remaining limit))) + (t/is (zero? (process/remaining (- before 1000)))))) + +(t/deftest ^:async uses-real-cwd-literal-args-env-and-closed-stdin + (let [source (str "process.stdin.resume();\n" + "process.stdin.on('end', () => {\n" + " const fact = {cwd: process.cwd(), args: process.argv.slice(1),\n" + " value: process.env.RENDERER_PROCESS_TEST, path: process.env.PATH || null};\n" + " fs.writeFileSync('fact.json', JSON.stringify(fact));\n" + " process.stdout.write(JSON.stringify(fact));\n" + " process.stderr.write('stdin-ended\\n');\n" + "});\n") + args ["space ; $(touch unexpected)" "--literal"]] + (doseq [env [{:RENDERER_PROCESS_TEST "map value"} + #js {:RENDERER_PROCESS_TEST "map value"}]] + (let [options (node-options source {:env env}) + result (await (within (process/run (update options :args into args)))) + fact (js->clj (js/JSON.parse (:stdout result)) :keywordize-keys true)] + (t/is (= 0 (:exit-code result))) + (t/is (= "stdin-ended\n" (:stderr result))) + (t/is (= {:cwd @temp-root + :args args + :value "map value" + :path (or (.-PATH (.-env js/process)) nil)} + fact)) + (t/is (= (:stdout result) (fs/readFileSync (file "fact.json") "utf8"))) + (t/is (not (fs/existsSync (file "unexpected"))) "arguments do not pass through a shell"))))) + +(t/deftest ^:async routes-both-streams-before-the-child-can-finish + (let [seen (atom {:stdout "" :stderr ""}) + source (str "process.stdout.write('ready\\n');\n" + "process.stderr.write('warning\\n');\n" + "const timer = setInterval(() => {\n" + " if (fs.existsSync('ack')) {\n" + " clearInterval(timer); fs.writeFileSync('done', 'yes');\n" + " process.stdout.write('complete\\n');\n" + " }\n" + "}, 10);\n") + result (await + (within + (process/run + (node-options + source + {:on-output + (fn [text stream] + (swap! seen update stream str text) + (when (str/includes? (:stdout @seen) "ready\n") + (fs/writeFileSync (file "ack") "yes")))}))))] + (t/is (= {:exit-code 0 :stdout "ready\ncomplete\n" :stderr "warning\n"} result)) + (t/is (= (select-keys result [:stdout :stderr]) @seen)) + (t/is (= "yes" (fs/readFileSync (file "done") "utf8"))))) + +(t/deftest ^:async lines-keep-streams-separate-and-flush-final-fragments + (let [seen (atom {:stdout [] :stderr []}) + source (str "process.stdout.write('out-'); process.stderr.write('err-');" + "setTimeout(() => { process.stdout.write('done\\r');" + "process.stderr.write('done\\nlast-err');" + "setTimeout(() => process.stdout.write('\\nlast-out'), 20); }, 20);") + result (await (process/run + (node-options source + {:capture-output? false + :on-line (fn [line stream] + (swap! seen update stream conj line))})))] + (t/is (= {:stdout ["out-done" "last-out"] :stderr ["err-done" "last-err"]} @seen)) + (t/is (= {:exit-code 0 :stdout "" :stderr ""} result)))) + +(t/deftest ^:async decodes-utf8-across-real-pipe-chunks + (let [source (str "const bytes = Buffer.from('Ω😀');\n" + "process.stdout.write(bytes.subarray(0, 1));\n" + "setTimeout(() => process.stdout.write(bytes.subarray(1)), 20);\n") + result (await (within (process/run (node-options source {}))))] + (t/is (= "Ω😀" (:stdout result))))) + +(t/deftest ^:async streams-large-output-without-retaining-it + (let [sizes (atom {:stdout 0 :stderr 0}) + source (str "process.stdout.write('x'.repeat(256 * 1024));\n" + "process.stderr.write('y'.repeat(128 * 1024));\n") + result (await + (within + (process/run + (node-options source + {:capture-output? false + :on-output (fn [text stream] + (swap! sizes update stream + (count text)))}))))] + (t/is (= {:exit-code 0 :stdout "" :stderr ""} result)) + (t/is (= {:stdout (* 256 1024) :stderr (* 128 1024)} @sizes)))) + +(t/deftest ^:async nonzero-exit-keeps-code-command-and-output + (let [options (node-options + "process.stdout.write('fact\\n'); process.stderr.write('failed\\n'); process.exitCode = 17;" + {}) + data (await (rejected (process/run options)))] + (t/is (= :process-exit (:type data))) + (t/is (= (:command options) (:command data))) + (t/is (= 17 (:exit-code data))) + (t/is (nil? (:signal data))) + (t/is (= "fact\n" (:stdout data))) + (t/is (= "failed\n" (:stderr data))))) + +(t/deftest ^:async spawn-failure-keeps-os-evidence + (let [command (file "no-such-command") + data (await (rejected (process/run (node-options "" {:command command}))))] + (t/is (= :process-spawn (:type data))) + (t/is (= command (:command data))) + (t/is (= "ENOENT" (:code data))) + (t/is (nil? (:exit-code data))) + (t/is (nil? (:signal data))))) + +(t/deftest ^:async expired-deadline-does-not-start-work + (let [data (await (rejected + (process/run + (node-options "fs.writeFileSync('started', 'yes');" + {:deadline (- (js/Date.now) 1)}))))] + (t/is (= :process-timeout (:type data))) + (t/is (= (.-execPath js/process) (:command data))) + (t/is (not (fs/existsSync (file "started")))) + (t/is (not (fs/existsSync (file "root.pid")))))) + +(t/deftest ^:async already-aborted-signal-does-not-start-work + (let [controller (js/AbortController.)] + (.abort controller "already-cancelled") + (let [data (await (rejected + (process/run + (node-options "fs.writeFileSync('started', 'yes');" + {:signal (.-signal controller)}))))] + (t/is (= :process-cancelled (:type data))) + (t/is (= "already-cancelled" (:reason data))) + (t/is (not (fs/existsSync (file "started")))) + (t/is (not (fs/existsSync (file "root.pid"))))))) + +(t/deftest ^:async timeout-kills-the-owner-and-a-term-resistant-descendant + (let [limit (process/deadline 1000) + data (await + (rejected + (process/run + (node-options (str (spawn-descendant) "setInterval(() => {}, 1000);") + {:deadline limit}))))] + (t/is (= :process-timeout (:type data))) + (t/is (= limit (:deadline data))) + (t/is (= "SIGTERM" (:signal data))) + (t/is (fs/existsSync (file "descendant.ready"))) + (await (assert-tree-stopped)))) + +(t/deftest ^:async cancellation-kills-the-whole-owned-group + (let [controller (js/AbortController.) + output (atom "") + groups (atom nil) + data (await + (rejected + (process/run + (node-options + (str (spawn-descendant) "setInterval(() => {}, 1000);") + {:signal (.-signal controller) + :on-output + (fn [text stream] + (when (= :stdout stream) + (swap! output str text) + (when (str/includes? @output "descendant-ready\n") + (reset! groups + {:root-pid (read-pid "root.pid") + :root (:group (process-status (read-pid "root.pid"))) + :descendant (:group (process-status (read-pid "descendant.pid"))) + :grandchild (:group (process-status (read-pid "grandchild.pid"))) + :runner (:group (process-status (.-pid js/process)))}) + (.abort controller "cancel-tree"))))}))))] + (t/is (= :process-cancelled (:type data))) + (t/is (= "cancel-tree" (:reason data))) + (t/is (= "SIGTERM" (:signal data))) + (t/is (= (:root-pid @groups) (:root @groups) (:descendant @groups) (:grandchild @groups)) + "the root leads the group containing all generations") + (t/is (not= (:runner @groups) (:root @groups)) "the group is separate from the runner") + (await (assert-tree-stopped)))) + + +(t/deftest ^:async default-cancellation-produces-portable-error-evidence + (let [controller (js/AbortController.) + cause (try + (await (within (process/run + (node-options "process.stdout.write('ready'); setInterval(() => {}, 1000);" + {:signal (.-signal controller) + :on-output (fn [& _] (.abort controller))})))) + nil + (catch :default cause cause)) + evidence (errors/evidence :process cause)] + (t/is (= :process-cancelled (:type (ex-data cause)))) + (t/is (= evidence (-> evidence transit/encode-str transit/decode-str))))) + +(t/deftest ^:async output-callback-failure-cleans-the-owned-group + (doseq [callback [:on-output :on-line]] + (let [seen (atom "") + data (await + (rejected + (process/run + (node-options + (str (spawn-descendant) "setInterval(() => {}, 1000);") + {callback + (fn [text stream] + (when (= :stdout stream) + (swap! seen str text) + (when (str/includes? @seen "descendant-ready") + (throw (js/Error. "Output consumer failed")))))}))))] + (t/is (= :process-output (:type data))) + (t/is (= :stdout (:stream data))) + (t/is (= "SIGTERM" (:signal data))) + (t/is (str/includes? (:stdout data) "descendant-ready\n")) + (await (assert-tree-stopped))))) + +(t/deftest ^:async successful-owner-exit-cleans-descendants-that-hold-the-pipes + (let [source (str (spawn-descendant) + "child.unref();\n" + "const timer = setInterval(() => {\n" + " if (fs.existsSync('descendant.ready')) {\n" + " clearInterval(timer); process.stdout.write('root-complete\\n');\n" + " }\n" + "}, 10);\n") + result (await (within (process/run (node-options source {}))))] + (t/is (= 0 (:exit-code result))) + (t/is (str/includes? (:stdout result) "root-complete\n")) + (t/is (str/includes? (:stdout result) "descendant-ready\n")) + (await (assert-tree-stopped)))) + +(t/deftest ^:async nonzero-owner-exit-also-cleans-descendants-that-hold-the-pipes + (let [source (str (spawn-descendant) + "child.unref();\n" + "const timer = setInterval(() => {\n" + " if (fs.existsSync('descendant.ready')) {\n" + " clearInterval(timer); process.stderr.write('root-failed\\n');\n" + " process.exitCode = 17;\n" + " }\n" + "}, 10);\n") + data (await (rejected (process/run (node-options source {}))))] + (t/is (= :process-exit (:type data))) + (t/is (= 17 (:exit-code data))) + (t/is (= "root-failed\n" (:stderr data))) + (await (assert-tree-stopped)))) + +(t/deftest ^:async lock-wait-times-out-and-cancelled-holder-releases-inherited-flock + (let [lock (file "build.lock") + controller (js/AbortController.) + holder (node-source + "holder.pid" + (str "process.on('SIGTERM', () => fs.writeFileSync('holder.term', 'seen'));\n" + "fs.writeFileSync('holder.ready', 'yes');\n" + "setInterval(() => {}, 1000);\n")) + held (rejected + (process/run + (node-options "" + {:command "flock" + :args ["--exclusive" lock (.-execPath js/process) "-e" holder] + :signal (.-signal controller)})))] + (try + (await (wait-for #(fs/existsSync (file "holder.ready")))) + (let [limit (process/deadline 150) + data (await + (rejected + (process/run + (node-options "" + {:command "flock" + :args ["--exclusive" lock (.-execPath js/process) "-e" + (node-source "waiter.pid" "fs.writeFileSync('waiter-acquired', 'yes');")] + :deadline limit}))))] + (t/is (= :process-timeout (:type data))) + (t/is (= limit (:deadline data))) + (t/is (not (fs/existsSync (file "waiter-acquired")))) + (t/is (running? (read-pid "holder.pid")) "the first owner still holds the lock")) + (.abort controller "release-lock") + (let [data (await held)] + (t/is (= :process-cancelled (:type data))) + (t/is (not (running? (read-pid "holder.pid"))))) + (let [result (await + (within + (process/run + (node-options "" + {:command "flock" + :args ["--exclusive" "--nonblock" lock (.-execPath js/process) "-e" + (node-source "probe.pid" "fs.writeFileSync('probe-acquired', 'yes');")]}))))] + (t/is (= 0 (:exit-code result))) + (t/is (= "yes" (fs/readFileSync (file "probe-acquired") "utf8"))) + (t/is (not (fs/existsSync (file "waiter-acquired")))) + (t/is (= "seen" (fs/readFileSync (file "holder.term") "utf8")))) + (finally + (.abort controller "test-cleanup") + (await held))))) diff --git a/frontend/test/performance_tests/render_wasm/protocol_test.cljs b/frontend/test/performance_tests/render_wasm/protocol_test.cljs new file mode 100644 index 0000000000..f17621553c --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/protocol_test.cljs @@ -0,0 +1,417 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.protocol-test + "Tests render drains and camera sessions with a controllable clock. + A drain renders until Full completion. A camera session moves a view through + cached previews, a settling wait and final rendering to Full." + (:require + [app.common.render-wasm.wasm :as wasm] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.runtime.protocol :as protocol] + [cljs.test :as t :include-macros true])) + +(defn- fake-clock + "Returns a controllable clock in milliseconds. + `:now` reads it; `:advance!` adds the supplied duration." + [start] + (let [t (atom start)] + {:now (fn [] @t) + :advance! (fn [ms] (swap! t + ms) @t)})) + +(defn- drain-hooks + "Returns hooks that simulate rendering and record flags in `calls`. + `:render` returns the frame types from `script` in order. Each `:frame` adds + 16 ms to the clock; `:sleep` adds the requested wait. Both return Promises." + [{:keys [clock script calls check]}] + (let [remaining (atom (vec script))] + {:now (:now clock) + :frame (fn [] + ((:advance! clock) 16) + (js/Promise.resolve ((:now clock)))) + :sleep (fn [ms] + ((:advance! clock) ms) + (js/Promise.resolve nil)) + :check (or check (fn [] nil)) + :render (fn [_timestamp flags] + (swap! calls conj flags) + (let [next (first @remaining)] + (swap! remaining rest) + next)) + :render-from-cache (fn [] 0) + :set-view (fn [_view] nil) + :set-view-start (fn [] nil) + :set-view-end (fn [] nil)})) + +(t/deftest ^:async completion-keeps-the-renderer-endpoint + (doseq [camera [false true]] + (let [clock (fake-clock 1000) + hooks (assoc (drain-hooks {:clock clock :script [] :calls (atom [])}) + :render (fn [_ _] + ((:advance! clock) 7) + (js/queueMicrotask #((:advance! clock) 500)) + wasm/FRAME_TYPE_FULL)) + result (await + (if camera + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms 0}) + (protocol/animate-view! {:to {:scale 1 :x 10 :y 5} :steps 1}) + (protocol/finish-camera!)) + (protocol/drain hooks {:flags 0 :origin 1000 :immediate true}))) + end (if camera 1023 1007)] + (t/is (= end (:end-ms result))) + (t/is (= (+ end 500) ((:now clock)))) + (if camera + (do + (t/is (= 23 (:interact-ms result))) + (t/is (= 7 (:last-input-to-full-ms result))) + (t/is (= 7 (:time-to-full-ms result)))) + (t/is (= 7 (:full-ms result))))))) + +(t/deftest immediate-full-reports-equal-boundaries + (t/async done + (let [clock (fake-clock 1000) + calls (atom []) + hooks (drain-hooks {:clock clock :script [2] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 1000 :immediate true}) + (.then (fn [{:keys [slices viewport-ready-ms full-ms]}] + (t/is (= 1 (count slices))) + (t/is (= {:timestamp 1000 + :flags 0 + :frame-type 2 + :duration-ms 0} + (first slices))) + (t/is (= full-ms viewport-ready-ms) + "immediate Full supplies both boundaries") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest progressive-frames-propagate-flags + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1 3 2] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate false}) + (.then (fn [{:keys [slices viewport-ready-ms full-ms]}] + (t/is (= [0 1 3] @calls) + "the previous frame type becomes the next flags") + (t/is (= [1 3 2] (mapv :frame-type slices))) + (t/is (< viewport-ready-ms full-ms) + "ViewportReady lands before Full") + (t/is (= (:timestamp (nth slices 1)) (+ 16 16)) + "later frames wait for rAF") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest partial-jumps-straight-to-full + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1 2] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [{:keys [slices viewport-ready-ms full-ms]}] + (t/is (= [0 1] @calls)) + (t/is (= [1 2] (mapv :frame-type slices))) + (t/is (= viewport-ready-ms full-ms) + "Full as the first completion supplies both boundaries") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest unknown-frame-types-reject + (t/async done + ;; One async context chaining the bad values: a `t/async` inside + ;; `doseq` would never run (the doseq returns nil to the runner). + (letfn [(step [remaining] + (if (empty? remaining) + (done) + (let [bad (first remaining) + clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [bad] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [_] + (t/is false (str "must reject for frame type " bad)) + (done))) + (.catch (fn [cause] + (t/is (= :benches.render-wasm.runtime.protocol/unexpected-frame-type + (:type (ex-data cause))) + (str "frame type " bad)) + (step (rest remaining))))))))] + (step [0 7 js/NaN])))) + +(t/deftest cancellation-stops-further-renders + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + fired (atom 0) + hooks (drain-hooks {:clock clock + :script [1 1 1 1] + :calls calls + :check (fn [] + (swap! fired inc) + (when (> @fired 2) + (throw (ex-info "canceled" {:phase "test"}))))})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [_] + (t/is false "must reject once check throws") + (done))) + (.catch (fn [cause] + (t/is (= "canceled" (ex-message cause))) + (t/is (= 1 (count @calls)) "cancellation stops before the next render") + (t/is (= 3 @fired) "guard fires once before and once after the render") + (done))))))) + +(t/deftest progressive-failure-retains-completed-call-evidence + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1 7] :calls calls})] + (-> (protocol/drain hooks {:flags 0 :origin 0 :immediate true}) + (.then (fn [_] (t/is false "unknown frame must fail") (done))) + (.catch (fn [cause] + (t/is (= [1 7] (mapv :frame-type (get-in (ex-data cause) [:partial :slices])))) + (t/is (= [0 0] (mapv :duration-ms (get-in (ex-data cause) [:partial :slices])))) + (done))))))) + +(t/deftest ^:async unreadable-error-message-keeps-render-evidence + (let [clock (fake-clock 0) + calls (atom []) + hooks (drain-hooks {:clock clock :script [1] :calls calls}) + render (:render hooks) + cause (ex-info "unreadable" {} (js/Error. "device lost")) + _ (js/Object.defineProperty cause "message" + #js {:get (fn [] (throw (js/Error. "message getter failed")))}) + hooks (assoc hooks :render (fn [timestamp flags] + (if (seq @calls) + (throw cause) + (render timestamp flags)))) + result (try + (await (protocol/drain hooks {:flags 0 :origin 0 :immediate true})) + nil + (catch :default error error))] + (t/is (= [1] (mapv :frame-type (get-in (ex-data result) [:partial :slices])))) + (t/is (some #(= "device lost" (:message %)) (errors/describe-cause result))))) + +(t/deftest camera-failure-retains-completed-previews-and-authored-pauses + (t/async done + (let [clock (fake-clock 0) + calls (atom []) + previews (atom 0) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls calls}) + :render-from-cache (fn [] + (if (= 1 (swap! previews inc)) + ((:advance! clock) 3) + (throw (ex-info "preview failed" {}))))) + view {:scale 1 :x 0 :y 0}] + (-> (protocol/start-camera! {:hooks hooks :view view} {:settle-ms 0}) + (protocol/preview-view! view) + (protocol/sleep! 7) + (protocol/preview-view! (assoc view :x 1)) + (.then (fn [_] (t/is false "second preview must fail") (done))) + (.catch (fn [cause] + (t/is (= [3] (mapv :duration-ms (get-in (ex-data cause) [:partial :cached-slices])))) + (t/is (= [{:requested-ms 7 :actual-ms 7}] + (get-in (ex-data cause) [:partial :authored-sleeps]))) + (done))))))) + +(t/deftest rejected-pause-retains-earlier-camera-evidence + (t/async done + (let [clock (fake-clock 0) + sleeps (atom 0) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls (atom [])}) + :render-from-cache (fn [] ((:advance! clock) 3)) + :sleep (fn [ms] + (if (= 1 (swap! sleeps inc)) + (do ((:advance! clock) ms) (js/Promise.resolve nil)) + (js/Promise.reject (ex-info "pause failed" {}))))) + view {:scale 1 :x 0 :y 0}] + (-> (protocol/start-camera! {:hooks hooks :view view} {:settle-ms 0}) + (protocol/preview-view! view) + (protocol/sleep! 7) + (protocol/sleep! 13) + (.then (fn [_] (t/is false "second pause must fail") (done))) + (.catch (fn [cause] + (t/is (= "pause failed" (ex-message cause))) + (t/is (= [3] (mapv :duration-ms (get-in (ex-data cause) [:partial :cached-slices])))) + (t/is (= [{:requested-ms 7 :actual-ms 7}] + (get-in (ex-data cause) [:partial :authored-sleeps]))) + (done))))))) + +(t/deftest restore-drains-with-sync-tiles-immediately + (t/async done + (let [clock (fake-clock 500) + calls (atom []) + order (atom []) + hooks (assoc (drain-hooks {:clock clock :script [1 3 2] :calls calls}) + :set-view-start (fn [] (swap! order conj :start) nil) + :set-view (fn [view] + (swap! order conj [:view view]) + nil) + :set-view-end (fn [] (swap! order conj :end) nil)) + view {:scale 1 :x 10 :y 20}] + (-> (protocol/restore hooks view) + (.then (fn [{:keys [slices]}] + (t/is (= [:start [:view view] :end] @order) + "camera resets before the drain") + (t/is (= [4 1 3] @calls) + "restore drains with SyncTiles first") + (t/is (= [1 3 2] (mapv :frame-type slices))) + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest camera-session-times-gesture-settle-and-final-drain + (t/async done + (let [clock (fake-clock 0) + renders (atom []) + cached (atom []) + views (atom []) + hooks {:now (:now clock) + :frame (fn [] + ((:advance! clock) 16) + (js/Promise.resolve ((:now clock)))) + :sleep (fn [ms] + ((:advance! clock) ms) + (js/Promise.resolve nil)) + :check (fn [] nil) + :render (fn [_timestamp flags] + (swap! renders conj flags) + 2) + :render-from-cache (fn [] + (swap! cached conj true) + ((:advance! clock) 3) + nil) + :set-view (fn [view] + (swap! views conj view) + nil) + :set-view-start (fn [] nil) + :set-view-end (fn [] + ((:advance! clock) 5) + nil)} + frames [{:scale 1 :x 0 :y 0} {:scale 1 :x 100 :y 0}]] + (-> (protocol/start-camera! {:hooks hooks :view (first frames)} {:settle-ms 100}) + (protocol/preview-view! (first frames)) + (protocol/preview-view! (second frames)) + (protocol/finish-camera!) + (.then (fn [m] + (t/is (= frames @views) "every gesture frame sets the view") + (t/is (= 2 (count @cached)) "one cached preview per frame") + (t/is (= 2 (count (:cached-slices m)))) + (t/is (every? #(contains? % :timestamp) (:cached-slices m))) + (t/is (every? #(contains? % :duration-ms) (:cached-slices m)) + "cached slices keep their own shape") + (t/is (= 100 (:settling-requested-ms m))) + (t/is (= 100 (:settling-actual-ms m)) + "fake sleep advances the clock exactly") + (t/is (= 5 (:set-view-end-ms m))) + (t/is (= [4] @renders) "final drain starts with SyncTiles") + (t/is (>= (:time-to-full-ms m) (:set-view-end-ms m)) + "finalization includes _set_view_end") + (t/is (= (:time-to-viewport-ready-ms m) (:time-to-full-ms m)) + "immediate final drain gives equal boundaries") + (t/is (= 108 (:last-input-to-full-ms m)) + "last input at 35, Full at 143: settle + set_view_end + drain") + (t/is (= [wasm/FRAME_TYPE_FULL] (mapv :frame-type (:slices m))) + "frame is full") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest camera-session-chains-promises-and-records-authored-sleep + (t/async done + (let [clock (fake-clock 0) + views (atom []) + events (atom []) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls (atom [])}) + :frame (fn [] + (swap! events conj :frame) + ((:advance! clock) 16) + (js/Promise.resolve ((:now clock)))) + :set-view (fn [view] + (swap! views conj view) + ((:advance! clock) 2)) + :render-from-cache (fn [] + (swap! events conj :cached) + ((:advance! clock) 3)) + :set-view-end (fn [] + (swap! events conj :end) + ((:advance! clock) 5))) + base {:scale 1 :x 0 :y 0} + target {:scale 2 :x 100 :y 40}] + (-> (protocol/start-camera! {:hooks hooks :view base} {:settle-ms 100}) + (protocol/animate-view! {:to target :steps 3}) + (protocol/sleep! 25) + (protocol/finish-camera!) + (.then (fn [m] + (t/is (apply < (map :x @views)) + "intermediate camera views advance toward the target") + (t/is (= target (last @views))) + (t/is (= [:frame :cached :frame :cached :frame :cached :end] @events)) + (t/is (= 3 (count (:cached-slices m)))) + (t/is (= [5 5 5] (mapv :view-and-preview-ms (:cached-slices m))) + "call span also includes the view update") + (t/is (= [3 3 3] (mapv :duration-ms (:cached-slices m)))) + (t/is (= [{:requested-ms 25 :actual-ms 25}] (:authored-sleeps m))) + (t/is (= 88 (:active-ms m)) + "three rAF waits, view/cache calls and authored pause") + (t/is (= 100 (:settling-actual-ms m))) + (t/is (= 193 (:interact-ms m)) + "active, settle and camera end are one span") + (done))) + (.catch (fn [cause] + (t/is false (str "must resolve, threw: " cause)) + (done))))))) + +(t/deftest camera-session-rejects-hook-errors + (t/async done + (let [clock (fake-clock 0) + hooks (assoc (drain-hooks {:clock clock :script [2] :calls (atom [])}) + :render-from-cache (fn [] (throw (ex-info "cache failed" {:phase "preview"}))))] + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms 100}) + (protocol/preview-view! {:scale 1 :x 10 :y 0}) + (protocol/finish-camera!) + (.then (fn [_] + (t/is false "preview error must reject") + (done))) + (.catch (fn [cause] + (t/is (= "cache failed" (ex-message cause))) + (done))))))) + +(t/deftest invalid-settle-ms-rejects + (t/async done + (let [hooks (drain-hooks {:clock (fake-clock 0) :script [2] :calls (atom [])})] + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms -1}) + (.then (fn [_] + (t/is false "invalid settle-ms must reject") + (done))) + (.catch (fn [cause] + (t/is (= ::protocol/invalid-settle-ms (:type (ex-data cause)))) + (done))))))) + +(t/deftest invalid-sleep-rejects-with-camera-evidence + (t/async done + (let [hooks (drain-hooks {:clock (fake-clock 0) :script [2] :calls (atom [])})] + (-> (protocol/start-camera! {:hooks hooks :view {:scale 1 :x 0 :y 0}} + {:settle-ms 0}) + (protocol/sleep! -1) + (.then (fn [_] + (t/is false "invalid sleep must reject") + (done))) + (.catch (fn [cause] + (t/is (= ::protocol/invalid-sleep (:type (ex-data cause)))) + (t/is (= {:cached-slices [] :authored-sleeps []} + (:partial (ex-data cause)))) + (done))))))) diff --git a/frontend/test/performance_tests/render_wasm/report_test.cljs b/frontend/test/performance_tests/render_wasm/report_test.cljs new file mode 100644 index 0000000000..48cda88851 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/report_test.cljs @@ -0,0 +1,624 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.report-test + (:require + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.measurement :as measurement] + [benches.render-wasm.report.analysis :as method] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def saved-case + {:id :saved/pan :scene :saved :scene-seed 42 :params {:count 10} + :view {:scale 1 :x 0 :y 0} :context :reuse :preparation {}}) + +(def metadata + {:git {:sha "abc" :dirty false} + :build {:functional {:mode :release :features []}} + :environment {:browser "Chromium" :node "v24"} :scored? true}) + +(defn- event + [command values] + {:command command :metrics {:command/elapsed values}}) + +(defn- block + [phase prep events] + {:phase phase :preparation-id prep :outcome :completed :events events}) + +(defn- preparation + [id events] + {:preparation-id id :case-id :saved/pan :warmup? false + :fingerprint {:algorithm :sha-256 :digest (apply str (repeat 64 "a"))} + :effective-graphics {:renderer "hardware"} + :diagnostics {:upload-ms 123} + :phases [(block :preparation id events)]}) + +(defn- new-run + [warmups repetitions context] + (result/create-run {:run-id "run" :started-at "start" + :plan {:cases [(assoc saved-case :context context)] + :warmups warmups :repetitions repetitions} + :metadata metadata :analysis {:draws 50}})) + +(defn- add-attempt + [run prep warmup? outcome phases] + (result/record-attempt + run (measurement/attempt :saved/pan prep warmup? + (cond-> {:status (if (= :completed outcome) "ok" "failed") + :phases phases} + (= :failed outcome) (assoc :phase (name (:phase (last phases))) + :message "renderer failed"))))) + +(defn- sample-run + [values] + (reduce (fn [run value] + (add-attempt run :prep false :completed + [(block :main :prep [(event :test/pan [value])])])) + (result/record-preparation (new-run 0 (count values) :reuse) + (preparation :prep [])) values)) + +(defn- error-type + [f] + (try (f) nil (catch :default cause (:type (ex-data cause))))) + +(def main-pan [:main :test/pan :command/elapsed]) +(def main-global [:main :global :command/elapsed]) + +(defn- approximately? + [a b] + (< (js/Math.abs (- a b)) 1e-9)) + +(t/deftest mathematical-vectors + (let [settings (method/resolve-settings {:min-p95 1 :min-p99 1}) + stats (method/statistics [1 2 3 4 10] settings)] + (t/is (= 4 (get-in stats [:mean :value]))) + (t/is (= 3 (get-in stats [:median :value]))) + (t/is (= [1 10] [(get-in stats [:min :value]) (get-in stats [:max :value])])) + (t/is (approximately? (js/Math.sqrt 12.5) (get-in stats [:sample-sd :value]))) + (t/is (= 1 (get-in stats [:mad :value]))) + (t/is (approximately? 8.8 (get-in stats [:p95 :value]))) + (t/is (approximately? 9.76 (get-in stats [:p99 :value]))) + (t/is (= 1.75 (method/quantile [4 2 1 3] 0.25))) + (t/is (= 2.5 (method/quantile [1 2 3 4] 0.5))) + (t/is (= [1 4] [(method/quantile [4 1 3 2] 0) (method/quantile [4 1 3 2] 1)])))) + +(t/deftest numeric-arguments-throw-while-unavailable-results-retain-reasons + (doseq [probability [-0.1 1.1 js/NaN js/Infinity js/-Infinity nil false "0.5"]] + (t/is (= ::method/invalid-probability + (error-type #(method/quantile [1 2] probability))))) + (doseq [values [[js/NaN] [js/Infinity] [js/-Infinity] [nil] ["1"] 42 {} #{1 2}]] + (t/is (= ::method/invalid-observations (error-type #(method/quantile values 0.5))))) + (doseq [override [{:draws 0} {:seed -1} {:seed 4294967296} {:confidence 1} + {:confidence js/NaN} {:min-ci 0} {:unknown 1}]] + (t/is (= ::method/invalid-settings (error-type #(method/resolve-settings override))))) + (t/is (nil? (method/quantile [] 0.5))) + (t/is (nil? (method/quantile nil 0.5))) + (t/is (= {:reason :no-valid-observations} (:median (method/statistics [] method/defaults))))) + +(t/deftest calculated-numbers-and-overflow-have-independent-result-maps + (let [stats (method/statistics [4] method/defaults) + changes (method/changes [1e-308] [1e308] (method/resolve-settings {:draws 5 :min-ci 1}))] + (t/is (= {:value 4} (:median stats))) + (t/is (= {:value 0} (:mad stats))) + (t/is (= {:reason :insufficient-observations :minimum 2} (:sample-sd stats))) + (t/is (= {:value 1e308} (:absolute changes))) + (t/is (= {:reason :undefined-estimate} (:percentage changes))) + (t/is (= {:lower 1e308 :upper 1e308} (:absolute-ci changes))) + (t/is (= {:reason :undefined-bootstrap-estimate} (:percentage-ci changes))))) + +(t/deftest empty-singleton-and-support-thresholds + (let [settings (method/resolve-settings nil)] + (t/is (= :no-valid-observations (get-in (method/statistics [] settings) [:median :reason]))) + (t/is (= :insufficient-observations (get-in (method/statistics [7] settings) [:sample-sd :reason]))) + (t/is (= 0 (get-in (method/statistics [7] settings) [:mad :value]))) + (t/is (= :insufficient-observations (get-in (method/statistics (vec (repeat 9 7)) settings) [:median-ci :reason]))) + (t/is (= {:lower 7 :upper 7} (:median-ci (method/statistics (vec (repeat 10 7)) settings)))) + (t/is (= :insufficient-observations (get-in (method/statistics (vec (repeat 99 7)) settings) [:p95 :reason]))) + (t/is (= {:value 7} (:p95 (method/statistics (vec (repeat 100 7)) settings)))) + (t/is (= :insufficient-observations (get-in (method/statistics (vec (repeat 499 7)) settings) [:p99 :reason]))) + (t/is (= {:value 7} (:p99 (method/statistics (vec (repeat 500 7)) settings)))))) + +(t/deftest deterministic-bootstrap-and-independent-difference-vectors + (let [settings (method/resolve-settings {:draws 100 :seed 123 :confidence 0.8}) + stats (method/statistics (vec (range 1 11)) settings) + changes (method/changes (vec (range 1 11)) (vec (range 6 16)) settings)] + (t/is (= 5.5 (get-in stats [:median :value]))) + (t/is (= {:lower 3.5 :upper 7} (:median-ci stats))) + (t/is (= 5 (get-in changes [:absolute :value]))) + (t/is (approximately? 2.45 (get-in changes [:absolute-ci :lower]))) + (t/is (= 8 (get-in changes [:absolute-ci :upper]))) + (t/is (approximately? 31.201923076923077 (get-in changes [:percentage-ci :lower]))) + (t/is (approximately? 215.35714285714295 (get-in changes [:percentage-ci :upper]))) + (t/is (= stats (method/statistics (vec (range 1 11)) settings))))) + +(t/deftest zero-baseline-percentages-and-undefined-draws + (let [settings (method/resolve-settings {:draws 100 :min-ci 1 :seed 0})] + (t/is (= :zero-baseline (get-in (method/changes [0 0] [1 2] settings) [:percentage :reason]))) + (t/is (= :zero-baseline (get-in (method/changes [0 0] [1 2] settings) [:percentage-ci :reason]))) + (t/is (= :zero-bootstrap-baseline (get-in (method/changes [0 1] [2 3] settings) [:percentage-ci :reason]))) + (t/is (= :no-valid-observations (get-in (method/changes [] [1] settings) [:absolute :reason]))))) + +(t/deftest repeated-commands-keep-invocations-values-and-statistical-n-distinct + (let [phases [(block :main :prep + [(assoc-in (event :test/pan [2]) [:metrics :renderer/call-elapsed] [1 2]) + (assoc-in (event :test/zoom [4]) [:metrics :renderer/call-elapsed] [4]) + (assoc-in (event :test/pan [3]) [:metrics :renderer/call-elapsed] [3])])] + run (-> (new-run 1 2 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep true :completed phases) + (add-attempt :prep false :completed phases) + (add-attempt :prep false :completed phases) + (result/record-case-outcome :saved/pan {:outcome :completed}) + (result/finish-run "end" {:reason :completed}) + result/encode result/decode) + report (first (:cases (summary/summarize run))) + pan (get-in report [:metrics [:main :test/pan :renderer/call-elapsed]]) + global (get-in report [:metrics [:main :global :renderer/call-elapsed]])] + (t/is (= 3 (count (get-in run [:attempts 1 :phases 0 :events])))) + (t/is (= {:valid 2 :invalid 0 :absent 0 :failed 0 :warmup 1 :unattempted 0 + :invocations 4 :raw-values 6} (:counts pan))) + (t/is (= 2 (get-in pan [:statistics :count]))) + (t/is (= 6 (get-in pan [:statistics :median :value]))) + (t/is (= 10 (get-in global [:statistics :median :value]))) + (t/is (= 6 (get-in global [:counts :invocations]))) + (t/is (= 8 (get-in global [:counts :raw-values]))) + (t/is (str/includes? (format/format-run (summary/summarize run)) "invocations=4 raw-values=6 n=2")))) + +(t/deftest phase-eligibility-keeps-preparation-after-reset-failure + (let [run (-> (new-run 1 3 :reuse) + (result/record-preparation (preparation :prep [(event :test/setup [8])])) + (add-attempt :prep true :completed [(block :main :prep []) (block :reset :prep [])]) + (add-attempt :prep false :completed + [(block :main :prep [(event :test/pan [2])]) + (block :reset :prep [(event :test/restore [3])])]) + (add-attempt :prep false :failed + [(block :main :prep [(event :test/pan [100])]) + (assoc (block :reset :prep [(event :test/restore [200])]) :outcome :failed)]) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + report (first (:cases (summary/summarize run))) + prep (get-in report [:metrics [:preparation :test/setup :command/elapsed]])] + (t/is (= [0 1 2] (mapv :attempt-id (:attempts run)))) + (t/is (= {:warmups 0 :repetitions 1} (:remaining report))) + (t/is (= 8 (get-in prep [:statistics :median :value]))) + (t/is (= 1 (get-in prep [:statistics :count]))) + (t/is (= :insufficient-observations (get-in prep [:statistics :median-ci :reason]))) + (t/is (= 0 (get-in prep [:counts :warmup]))) + (t/is (= [2] (:values (summary/observations run :saved/pan main-pan)))) + (t/is (= [3] (:values (summary/observations run :saved/pan [:reset :test/restore :command/elapsed])))) + (t/is (= {:executions 3 :completed 3 :failed 0 :warmup 1} (get-in report [:phases :main]))) + (t/is (= {:executions 3 :completed 2 :failed 1 :warmup 1} (get-in report [:phases :reset]))))) + +(t/deftest fresh-preparations-follow-their-own-slot-eligibility + (let [run (-> (new-run 1 2 :fresh) + (result/record-preparation (assoc (preparation :warm [(event :test/setup [900])]) :warmup? true)) + (add-attempt :warm true :completed [(block :main :warm [])]) + (result/record-preparation (preparation :first [(event :test/setup [5])])) + (add-attempt :first false :completed [(block :main :first [])]) + (result/record-preparation (preparation :second [(event :test/setup [7])])) + (add-attempt :second false :failed [(assoc (block :main :second []) :outcome :failed)])) + obs (summary/observations run :saved/pan [:preparation :test/setup :command/elapsed])] + (t/is (= [5 7] (:values obs))) + (t/is (= 1 (get-in obs [:counts :warmup]))) + (t/is (= 2 (get-in obs [:counts :valid]))))) + +(t/deftest fresh-warmup-preparation-is-excluded-before-main-delivery + (let [run (-> (new-run 1 1 :fresh) + (result/record-preparation + (assoc (preparation :prep [(event :test/setup [9])]) :warmup? true)) + result/encode result/decode) + report (get-in (summary/summarize run) + [:cases 0 :metrics [:preparation :test/setup :command/elapsed]])] + (t/is (= 1 (get-in report [:counts :warmup]))) + (t/is (= :no-valid-observations (get-in report [:statistics :median :reason]))))) + +(t/deftest failed-preparation-excludes-a-completed-callback-from-observations + (let [run (-> (new-run 0 1 :reuse) + (result/record-preparation + (assoc (preparation :prep [(event :test/setup [9])]) + :failure {:phase "preparation" :message "initial Full failed"}))) + report (first (:cases (summary/summarize run))) + metric (get-in report [:metrics [:preparation :test/setup :command/elapsed]])] + (t/is (= {:executions 1 :completed 1 :failed 0 :warmup 0} (get-in report [:phases :preparation]))) + (t/is (= 1 (get-in metric [:counts :failed]))) + (t/is (= :no-valid-observations (get-in metric [:statistics :median :reason]))))) + +(t/deftest invalid-members-propagate-only-to-dependent-quantities + (let [bad (assoc-in (event :test/pan [0]) [:metrics :renderer/call-elapsed] [1 js/NaN]) + good (assoc-in (event :test/zoom [0]) [:metrics :renderer/call-elapsed] [2]) + run (-> (new-run 0 1 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep false :completed [(block :main :prep [bad good])])) + report (first (:cases (summary/summarize run)))] + (t/is (= [0] (:values (summary/observations run :saved/pan main-global)))) + (t/is (= [2] (:values (summary/observations run :saved/pan [:main :test/zoom :renderer/call-elapsed])))) + (doseq [scope [:test/pan :global]] + (let [metric (get-in report [:metrics [:main scope :renderer/call-elapsed]])] + (t/is (= 1 (get-in metric [:counts :invalid]))) + (t/is (= :non-finite (get-in metric [:diagnostics 0 :reason]))) + (t/is (= :no-valid-observations (get-in metric [:statistics :median :reason]))))))) + +(t/deftest raw-special-values-and-partial-failures-survive-transit + (let [values [0 -1 js/NaN js/Infinity js/-Infinity] + initial (-> (new-run 0 6 :reuse) + (result/record-preparation (preparation :prep []))) + run (-> (reduce (fn [run value] + (add-attempt run :prep false :completed + [(block :main :prep [(event :test/pan [value])])])) + initial values) + (result/record-attempt + (measurement/attempt :saved/pan :prep false + {:status "failed" :phase "main" :message "render failed" + :cause [{:message "WASM trap"}] + :partial {:clock js/NaN} + :phases [(assoc (block :main :prep [(event :test/pan [99])]) + :outcome :failed)]})) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + raw (mapv #(get-in % [:phases 0 :events 0 :metrics :command/elapsed 0]) (:attempts run)) + obs (summary/observations run :saved/pan main-pan)] + (t/is (= [0 -1] (subvec raw 0 2))) + (t/is (js/Number.isNaN (nth raw 2))) + (t/is (= [js/Infinity js/-Infinity] (subvec raw 3 5))) + (t/is (= [{:message "WASM trap"}] (get-in run [:attempts 5 :failure :cause]))) + (t/is (js/Number.isNaN (get-in run [:attempts 5 :partial :clock]))) + (t/is (= 99 (nth raw 5))) + (t/is (= [0] (:values obs))) + (t/is (= 1 (get-in obs [:counts :failed]))) + (t/is (= 4 (get-in obs [:counts :invalid]))))) + +(t/deftest empty-blocks-support-preparation-only-and-reset-only-cases + (doseq [phase [:preparation :reset]] + (let [run (-> (new-run 0 1 :reuse) + (result/record-preparation + (preparation :prep (if (= :preparation phase) [(event :test/work [2])] []))) + (add-attempt :prep false :completed + [(block :main :prep []) + (block :reset :prep (if (= :reset phase) [(event :test/work [2])] []))]) + (result/record-case-outcome :saved/pan {:outcome :completed}) + (result/finish-run "end" {:reason :completed}) + result/encode result/decode)] + (t/is (= [2] (:values (summary/observations run :saved/pan [phase :global :command/elapsed])))))) + (let [empty-run (-> (new-run 0 1 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep false :completed [(block :main :prep [])]))] + (t/is (= ::result/invalid-record + (error-type #(result/finish-run empty-run "end" {:reason :completed}))))) + (let [failed (-> (new-run 0 1 :reuse) + (result/record-preparation {:preparation-id :prep :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] + :failure {:phase "preparation" :message "setup failed"}}) + (result/finish-run "end" {:reason :preparation-error}))] + (t/is (str/includes? (format/format-run (summary/summarize (result/decode (result/encode failed)))) + "setup failed")))) + +(t/deftest comparison-reports-preparation-failures-once-per-source + (let [failure {:phase "preparation" :message "setup failed" + :errors [{:phase "upload" :operation :serialize-shapes :message "setup failed"} + {:phase "upload" :operation :end-loading :message "ending load failed"}]} + run (-> (new-run 0 2 :fresh) + (result/record-preparation (preparation :ready [])) + (add-attempt :ready false :completed [(block :main :ready [(event :test/work [2])])]) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/record-attempt (measurement/attempt :saved/pan :failed false + (assoc failure :status "failed" :phases []))) + (result/record-case-outcome :saved/pan {:outcome :failed :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + (assoc-in [:metadata :diagnostics :screenshots] + {"saved/pan" {:status :skipped :reason :no-completed-measured-attempt}}) + result/encode result/decode) + candidate (-> run + (assoc :run-id "candidate") + (assoc-in [:metadata :diagnostics :screenshots] + {"saved/pan" {:status :ok :file "candidate-pan.png"}}) + result/encode result/decode) + compared (-> (compare/compare-runs run candidate + {:allow-all-differences true}) + result/encode result/decode) + text (format/format-comparison compared)] + (t/is (= 2 (count (re-seq #"setup failed" text)))) + (t/is (= 2 (count (re-seq #"ending load failed" text)))) + (t/is (str/includes? text "median change")) + (t/is (= 2 (count (re-seq #"Preparation :failed" text)))) + (t/is (str/includes? text "Baseline screenshots\n \"saved/pan\": skipped; reason :no-completed-measured-attempt")) + (t/is (str/includes? text "Candidate screenshots\n \"saved/pan\": ok\n File \"candidate-pan.png\"")))) + +(t/deftest collected-failures-print-a-long-message-once-and-retain-enclosing-context + (let [message (str "failure-marker " (apply str (repeat 1400 "x"))) + failure {:phase "preparation" :operation :prepare-scene :path "scene.transit" :message message + :errors [{:phase "upload" :operation :serialize-shapes + :message (subs message 0 1024)}]} + run (-> (new-run 0 1 :fresh) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (= 1 (count (re-seq #"failure-marker" text)))) + (t/is (str/includes? text ":prepare-scene \"preparation\"")) + (t/is (str/includes? text "Path \"scene.transit\"")) + (t/is (str/includes? text ":serialize-shapes \"upload\"")))) + +(t/deftest unresolved-process-details-survive-the-report-entry-limit + (let [failure (errors/evidence + :cleanup + (errors/combine + (conj (mapv (fn [index] {:phase "cleanup" :error (js/Error. (str "cleanup " index))}) + (range 8)) + {:phase "cleanup" :operation :stop-process + :error (errors/fatal-error "Compiler group may still be running" + {:unresolved-process true :pid 4242 :pgid 4242 + :signals ["SIGTERM" "SIGKILL"]} nil)}))) + run (-> (new-run 0 1 :fresh) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "pgid 4242")) + (t/is (str/includes? text "SIGKILL")) + (t/is (str/includes? text "unresolved-process true")) + (t/is (str/includes? text "1 more errors recorded")))) + +(t/deftest equal-diagnostics-keep-failures-from-distinct-preparations + (let [failure {:phase "renderer" :message "shared renderer failure"}] + (doseq [phases [[] + [(block :main :ready []) + (assoc (block :reset :ready []) :outcome :failed)]]] + (t/testing (str "attempt phases " (mapv :phase phases)) + (let [run (-> (new-run 0 2 :fresh) + (result/record-preparation (preparation :ready [])) + (result/record-attempt + (measurement/attempt :saved/pan :ready false + (assoc failure :status "failed" :phases phases))) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/record-case-outcome :saved/pan {:outcome :failed :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "Preparation :failed (case :saved/pan): \"renderer\": \"shared renderer failure\"")) + (t/is (str/includes? text "Attempt 0 (preparation :ready) (case :saved/pan): \"renderer\": \"shared renderer failure\"")) + (t/is (= 2 (count (re-seq #"shared renderer failure" text))))))))) + +(t/deftest equal-diagnostics-keep-failures-from-distinct-cases + (let [failure {:phase "preparation" :message "shared setup failure"}] + (doseq [harness? [false true]] + (t/testing (if harness? "harness failure in another case" "outcome failure in another case") + (let [run (-> (result/create-run + {:run-id "run" :started-at "start" + :plan {:cases [saved-case (assoc saved-case :id :saved/other)] + :warmups 0 :repetitions 1} + :metadata (assoc-in metadata [:diagnostics :failures] + (if harness? + [(assoc failure :case-id :saved/other :stage :preparation)] + [])) + :analysis {:draws 50}}) + (result/record-preparation {:preparation-id :failed :case-id :saved/pan :warmup? false + :diagnostics {} :phases [] :failure failure}) + (result/record-case-outcome :saved/pan {:outcome :failed :failure failure}) + (result/record-case-outcome :saved/other {:outcome :failed :failure failure}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "Preparation :failed (case :saved/pan): \"preparation\": \"shared setup failure\"")) + (t/is (str/includes? text (str (if harness? "Harness 0 :preparation" "Case outcome") + " (case :saved/other): \"preparation\": \"shared setup failure\""))) + (t/is (= 2 (count (re-seq #"shared setup failure" text))))))))) + +(t/deftest ambiguous-screenshot-matches-report-the-warning-and-each-harness-failure + (let [failure {:phase "screenshot" :message "screenshot deadline exceeded" + :cause [{:message "browser deadline"}] + :stage :case-deadline :case-id :saved/pan}] + (doseq [[label case-ids failures] + [["multiple deadlines" [:saved/pan] [failure failure]] + ["distinct IDs with the same displayed name" [:saved/pan "saved/pan"] [failure]]]] + (t/testing label + (let [run (-> (result/create-run + {:run-id "run" :started-at "start" + :plan {:cases (mapv #(assoc saved-case :id %) case-ids) + :warmups 0 :repetitions 1} + :metadata (assoc metadata :diagnostics + {:failures failures + :screenshots {"saved/pan" {:status :failed :file "saved-pan.png" + :message (:message failure) + :cause (:cause failure)}}}) + :analysis {:draws 50}}) + (result/finish-run "end" {:reason :failed :failure failure}) + result/encode result/decode) + text (format/format-run (summary/summarize run))] + (t/is (str/includes? text "\"saved/pan\": warning; \"screenshot deadline exceeded\"")) + (t/is (str/includes? text "File \"saved-pan.png\"")) + (doseq [index (range (count failures))] + (t/is (str/includes? text (str "Harness " index " :case-deadline (case :saved/pan): \"screenshot\": \"screenshot deadline exceeded\"")))) + (t/is (= (inc (count failures)) (count (re-seq #"screenshot deadline exceeded" text))))))))) + +(t/deftest offline-comparison-uses-one-analysis-and-saved-quantity-units + (let [base (-> (sample-run [1 2 3]) result/encode result/decode) + candidate (-> (sample-run [3 4 5]) + (assoc-in [:analysis :seed] 99) + (assoc-in [:quantities :command/elapsed :description] "Reworded description") + result/encode result/decode) + comparison (-> (compare/compare-runs base candidate {:analysis {:draws 20}}) + result/encode result/decode)] + (t/is (= 2 (get-in comparison [:pairs 0 :metrics main-pan :absolute :value]))) + (t/is (= 20 (get-in comparison [:analysis :draws]))) + (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? (format/format-comparison comparison) "median change 2")) + (let [changed (assoc-in candidate [:quantities :command/elapsed :unit] :seconds)] + (t/is (= ::compare/incompatible (error-type #(compare/compare-runs base changed)))) + (t/is (= 2 (get-in (compare/compare-runs base changed {:allow-all-differences true}) + [:pairs 0 :metrics main-pan :absolute :value])))))) + +(t/deftest unknown-and-malformed-quantities-produce-diagnostics + (doseq [[description reason] [[nil :unknown-quantity] [{:unit 2} :malformed-quantity]]] + (let [run (cond-> (sample-run [1]) + (nil? description) (update :quantities dissoc :command/elapsed) + (some? description) (assoc-in [:quantities :command/elapsed] description)) + report (get-in (summary/summarize (result/decode (result/encode run))) + [:cases 0 :metrics main-pan])] + (t/is (= reason (get-in report [:diagnostics 0 :reason]))) + (t/is (= :no-valid-observations (get-in report [:statistics :median :reason])))))) + +(defn- feature-run + [features selection] + (-> (sample-run [1]) + (assoc-in [:metadata :build :functional :features] features) + (assoc-in [:metadata :build :provenance :feature-selection] selection))) + +(t/deftest automatic-feature-permission-checks-effective-and-requested-evidence + (let [baseline (feature-run ["base" "old"] {:default-features? false :features ["base"]}) + candidate (feature-run ["base" "toggle" "transitive"] + {:default-features? false :features ["base" "toggle"]}) + expected {:feature "toggle" :baseline-features ["base" "old"] + :candidate-features ["base" "toggle" "transitive"] + :added ["toggle" "transitive"] :removed ["old"]} + edge (assoc expected :from "0" :to "1") + report (compare/compare-runs baseline candidate {:expected-feature-change expected :edge edge})] + (t/is (= :feature-toggle (get-in report [:pairs 0 :mismatches 0 :rule]))) + (t/is (= report (result/decode (result/encode report)))) + (doseq [invalid [nil (assoc expected :feature "base") (assoc expected :feature :toggle) + (assoc expected :baseline-features ["base"]) + (assoc expected :candidate-features ["base" "toggle"]) + (assoc expected :added ["toggle"]) (assoc expected :removed []) + (assoc expected :added #{"toggle" "transitive"}) + (dissoc expected :baseline-features)]] + (t/is (= ::compare/incompatible + (error-type #(compare/compare-runs baseline candidate {:expected-feature-change invalid}))))) + (doseq [selection [nil {} {:default-features? true :features ["base" "toggle"]} + {:default-features? false :features ["toggle"]} + {:default-features? false :features #{"base" "toggle"}}]] + (t/is (= ::compare/incompatible + (error-type #(compare/compare-runs + baseline (assoc-in candidate [:metadata :build :provenance :feature-selection] selection) + {:expected-feature-change expected}))))) + (doseq [bad-edge [(dissoc edge :removed) (assoc edge :added []) (assoc edge :feature :toggle)]] + (t/is (= ::compare/invalid-edge + (error-type #(compare/compare-runs baseline candidate {:edge bad-edge}))))) + (t/is (= ::compare/invalid-edge + (error-type #(compare/compare-runs baseline candidate + {:edge edge :expected-feature-change (assoc expected :added [])})))))) + +(t/deftest missing-nil-and-empty-effective-features-keep-distinct-meaning + (doseq [before [::missing nil []] after [::missing nil []]] + (let [with-features (fn [features] + (cond-> (sample-run [1]) + (= ::missing features) (update-in [:metadata :build :functional] dissoc :features) + (not= ::missing features) (assoc-in [:metadata :build :functional :features] features))) + baseline (-> (with-features before) result/encode result/decode) + candidate (-> (with-features after) result/encode result/decode) + compared (compare/compare-runs baseline candidate {:allow-build-differences true})] + (t/is (= (if (= before after) [] [:effective-features]) + (mapv :field (get-in compared [:pairs 0 :mismatches]))))))) + +(t/deftest comparison-boundaries-validate-permissions-rules-and-intervals + (let [base (sample-run [1]) + candidate (assoc-in (sample-run [2]) [:metadata :build :functional :mode] :debug) + compared (compare/compare-runs base candidate {:allow-build-differences true}) + path [:pairs 0 :metrics main-pan]] + (doseq [bad [(dissoc compared :permissions) + (assoc-in compared [:permissions :allow-build-differences] :yes) + (update compared :permissions dissoc :allow-scene-differences) + (assoc-in compared [:pairs 0 :mismatches 0 :rule] :other) + (assoc-in compared [:pairs 0 :mismatches 0 :waived?] false) + (update-in compared [:pairs 0 :mismatches 0] dissoc :rule) + (assoc-in compared (conj path :absolute :value) js/NaN) + (assoc-in compared (conj path :absolute-ci) {:lower 2 :upper 1}) + (assoc-in compared [:pairs 0 :baseline :metrics main-pan :counts :valid] -1)]] + (t/is (= ::result/invalid-record (error-type #(result/decode (transit/encode-str bad)))))))) + +(t/deftest scene-permission-requires-complete-consistent-preparation-evidence + (let [base (sample-run [1]) + changed (result/record-preparation base + (assoc-in (preparation :other []) [:fingerprint :digest] + (apply str (repeat 64 "b")))) + report (compare/compare-runs base changed {:allow-all-differences true})] + (t/is (= ::compare/incompatible + (error-type #(compare/compare-runs base changed {:allow-scene-differences true})))) + (t/is (some #(= [:fingerprint :incomplete-or-varying-evidence] (:field %)) + (get-in report [:pairs 0 :mismatches]))))) + +(t/deftest matching-missing-preparation-facts-produce-incomplete-evidence + (let [run (-> (new-run 0 1 :reuse) + (result/record-preparation + (-> (preparation :prep [(event :test/setup [9])]) + (dissoc :fingerprint :effective-graphics) + (assoc :failure {:phase "preparation" :message "setup failed"}))))] + (doseq [options [{} {:allow-scene-differences true}]] + (t/is (= ::compare/incompatible (error-type #(compare/compare-runs run run options))))) + (let [report (compare/compare-runs run run {:allow-all-differences true})] + (t/is (= #{[:fingerprint :incomplete-or-varying-evidence] + [:effective-graphics :incomplete-or-varying-evidence]} + (set (map :field (get-in report [:pairs 0 :mismatches])))))))) + +(t/deftest saved-boundaries-refuse-corrupt-identities-and-phase-history + (let [run (sample-run [1])] + (doseq [bad [(assoc-in run [:attempts 0 :attempt-id] 3) + (assoc-in run [:attempts 0 :phases] []) + (assoc-in run [:preparations 0 :phases] []) + (assoc-in run [:preparations 0 :failure] {:phase "preparation" :message "setup failed"}) + (update-in run [:attempts 0] assoc :outcome :failed + :phases [(assoc (block :main :prep []) :outcome :failed) + (block :reset :prep [])]) + (assoc-in run [:attempts 0 :preparation-id] :unknown) + (assoc-in run [:attempts 0 :phases 0 :preparation-id] :unknown) + (assoc-in run [:attempts 0 :phases 0 :events 0 :metrics :command/elapsed] []) + (assoc-in run [:attempts 0 :phases 0 :events 0 :metrics :command/elapsed] [1 2]) + (assoc-in run [:remaining :saved/pan :repetitions] 2)]] + (t/is (= ::result/invalid-record + (error-type #(result/decode (transit/encode-str bad)))))))) + +(t/deftest saved-failures-reject-malformed-diagnostics + (let [diagnostic {:phase "main" :message "renderer failed"} + failed-preparation (-> (new-run 0 1 :reuse) + (result/record-preparation (assoc (preparation :prep []) :failure diagnostic)) + (result/finish-run "end" {:reason :failed}) + result/encode result/decode) + failed-attempt (-> (new-run 0 1 :reuse) + (result/record-preparation (preparation :prep [])) + (add-attempt :prep false :failed [(assoc (block :main :prep []) :outcome :failed)]) + (result/record-case-outcome :saved/pan {:outcome :failed :failure diagnostic}) + (result/finish-run "end" {:reason :failed :failure diagnostic}) + (assoc-in [:metadata :diagnostics :failures] [diagnostic]) + result/encode result/decode)] + (doseq [[run path] [[failed-preparation [:preparations 0 :failure]] + [failed-attempt [:attempts 0 :failure]] + [failed-attempt [:case-outcomes :saved/pan :failure]] + [failed-attempt [:termination :failure]] + [failed-attempt [:metadata :diagnostics :failures 0]]] + invalid [{:phase 42 :message "bad location"} + {:phase "main" :message nil} + {:phase "main" :message "bad cause" :cause [42]} + {:phase "main" :message "bad error collection" + :errors [{:phase "cleanup" :message "nested collection" :errors []}]}]] + (t/is (= ::result/invalid-record + (error-type #(result/decode (transit/encode-str (assoc-in run path invalid))))))))) + +(t/deftest case-outcomes-validate-completion-and-close-the-recording-boundary + (let [run (sample-run [1]) + completed (result/record-case-outcome run :saved/pan {:outcome :completed}) + ended (result/finish-run completed "end" {:reason :completed})] + (t/is (= ended (result/decode (result/encode ended)))) + (doseq [bad [(assoc ended :case-outcomes {}) + (-> run + (assoc-in [:plan :repetitions] 2) + (assoc-in [:remaining :saved/pan :repetitions] 1) + (assoc-in [:case-outcomes :saved/pan] {:outcome :completed})) + (assoc-in ended [:case-outcomes :unknown] {:outcome :completed})]] + (t/is (= ::result/invalid-record (error-type #(result/decode (transit/encode-str bad)))))) + (t/is (= ::result/invalid-record + (error-type #(result/record-preparation completed (preparation :other []))))) + (let [stopped (-> run + (assoc-in [:plan :repetitions] 2) + (assoc-in [:remaining :saved/pan :repetitions] 1) + (result/record-case-outcome :saved/pan {:outcome :failed}))] + (t/is (thrown-with-msg? js/Error #"case already terminated" + (add-attempt stopped :prep false :completed [(block :main :prep [])])))))) diff --git a/frontend/test/performance_tests/render_wasm/request_test.cljs b/frontend/test/performance_tests/render_wasm/request_test.cljs new file mode 100644 index 0000000000..0229b9503e --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/request_test.cljs @@ -0,0 +1,208 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.request-test + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli.request :as request] + [cljs.test :as t :include-macros true] + [performance-tests.render-wasm.test-helpers :refer [failure-data]])) + +(def ^:private console-defaults + {:level :info :color :auto}) + +(def ^:private measurement-defaults + {:seed 42 :filter nil :warmups 3 :repetitions 10 + :timeout-ms 30000 :startup-timeout-ms 30000 :output-dir nil + :screenshots false :require-gpu false}) + +(def ^:private comparison-defaults + {:allow-build-differences false :allow-scene-differences false + :allow-all-differences false}) + +(def ^:private existing-run + {:command "run" :console console-defaults + :options (assoc measurement-defaults + :build {:kind :existing :id "ready-build"} + :remove false)}) + +(def ^:private default-cases + [[{:command "build"} + {:command "build" :console console-defaults + :options {:default-features true :features [] :build-timeout-ms 300000}}] + [{:command "run" :console {} :options {:build {:kind :prepare}}} + {:command "run" :console console-defaults + :options (assoc measurement-defaults + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000} + :remove false)}] + [{:command "run" :options {:build {:kind :existing :id "ready-build"}}} + existing-run] + [{:command "ab" :options {:features ["render-cache"]}} + {:command "ab" :console console-defaults + :options (merge measurement-defaults comparison-defaults + {:features ["render-cache"] :max-features 3 + :confirm false :dry-run false :build-timeout-ms 300000})}] + [{:command "compare" + :options {:baseline-path "base.transit" :candidate-path "candidate.transit"}} + {:command "compare" :console console-defaults + :options (assoc comparison-defaults + :baseline-path "base.transit" + :candidate-path "candidate.transit" :case-pair nil)}] + [{:command "merge" + :options {:base-path "base.transit" :additional-path "extra.transit"}} + {:command "merge" :console console-defaults + :options {:base-path "base.transit" :additional-path "extra.transit" + :output-dir nil}}] + [{:command "purge" :options {:target :runs}} + {:command "purge" :console console-defaults :options {:target :runs}}] + [{:command "help"} + {:command "help" :console console-defaults :options {:topic nil}}]]) + +(t/deftest defaults-produce-complete-command-requests + (doseq [[partial expected] default-cases] + (t/testing (pr-str partial) + (t/is (= expected (request/check (request/apply-defaults partial))))))) + +(t/deftest explicit-values-and-literal-text-survive-resolution + (let [partial {:command "run" :console {:level :warn :color :never} + :options {:seed 0 :warmups 0 :filter " " :output-dir " ./runs " + :build {:kind :prepare :default-features false + :features []}}} + expected {:command "run" :console {:level :warn :color :never} + :options (assoc measurement-defaults + :seed 0 :warmups 0 :filter " " :output-dir " ./runs " + :build {:kind :prepare :default-features false + :features [] :build-timeout-ms 300000} + :remove false)}] + (t/is (= expected (request/check (request/apply-defaults partial))))) + (let [partial {:command "compare" + :options {:baseline-path " ./base " :candidate-path " ./candidate " + :case-pair [" rects/load " "other/load"]}}] + (t/is (= (assoc partial :console console-defaults + :options (merge comparison-defaults (:options partial))) + (request/check (request/apply-defaults partial))))) + (t/is (= existing-run + (request/check + (request/apply-defaults (assoc-in existing-run [:options :warmups] nil)))))) + +(t/deftest required-selections-and-paths-have-no-defaults + (doseq [partial [{} + {:command "unknown"} + {:command "run"} + {:command "run" :options {:build {}}} + {:command "run" :options {:build {:kind :unknown}}} + {:command "run" :options {:build {:kind :existing}}} + {:command "ab"} + {:command "compare"} + {:command "compare" :options {:baseline-path "base"}} + {:command "compare" :options {:candidate-path "candidate"}} + {:command "merge" :options {:base-path "base"}} + {:command "merge" :options {:additional-path "additional"}} + {:command "purge"}]] + (t/testing (pr-str partial) + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (request/apply-defaults partial))))))))) + +(t/deftest receiver-requires-complete-data-and-reports-the-field + (doseq [[_ complete] default-cases] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (dissoc complete :console))))))) + (doseq [incomplete [(dissoc existing-run :options) + (update existing-run :console dissoc :level) + (update existing-run :options dissoc :warmups) + (assoc-in existing-run [:options :warmups] nil) + (assoc-in existing-run [:options :remove] nil) + (assoc-in existing-run [:console :level] nil)]] + (let [data (failure-data #(request/check incomplete))] + (t/is (= ::request/invalid-request (:type data))) + (t/is (seq (:errors (::sm/explain data)))))) + (let [data (failure-data #(request/check (assoc-in existing-run [:options :warmups] -1)))] + (t/is (some #(= [:options :warmups] (:in %)) (:errors (::sm/explain data)))))) + +(t/deftest receiver-rejects-extra-fields-at-every-map-boundary + (doseq [[path value] [[[:extra] true] + [[:console :stream] "stdout"] + [[:options :browser] {}] + [[:options :build :features] []] + [[:options :build :default-features] false] + [[:options :build :build-timeout-ms] 1000]]] + (let [input (assoc-in existing-run path value) + resolved (request/apply-defaults input)] + (t/is (= input resolved)) + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check resolved))))))) + (doseq [partial [{:command "run" :options {:build {:kind :prepare :id "ready-build"}}} + {:command "ab" :options {:features ["cache"] :default-features false}} + {:command "ab" :options {:features ["cache"] :remove false}} + {:command "ab" :options {:features ["cache"] :build {:kind :prepare}}}]] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (request/apply-defaults partial)))))))) + +(t/deftest numeric-constraints-match-seeds-counts-and-node-timers + (doseq [[field values] [[:seed [0 4294967295]] + [:warmups [0 js/Number.MAX_SAFE_INTEGER]] + [:repetitions [1 js/Number.MAX_SAFE_INTEGER]] + [:timeout-ms [1 2147483647]] + [:startup-timeout-ms [1 2147483647]]]] + (doseq [value values] + (let [input (assoc-in existing-run [:options field] value)] + (t/is (= input (request/check input)))))) + (doseq [[field values] [[:seed [-1 4294967296 1.5 "42" js/NaN js/Infinity]] + [:warmups [-1 (inc js/Number.MAX_SAFE_INTEGER) 0.5 "3"]] + [:repetitions [0 (inc js/Number.MAX_SAFE_INTEGER) 1.5]] + [:timeout-ms [0 2147483648 1.5 "30000"]] + [:startup-timeout-ms [0 2147483648]]]] + (doseq [value values] + (t/testing (str field " " value) + (let [input (assoc-in existing-run [:options field] value) + resolved (request/apply-defaults input)] + (when (string? value) + (t/is (= value (get-in resolved [:options field])))) + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check resolved))))))))) + (doseq [command ["build" "ab"] + value [0 2147483648 1.5]] + (let [partial {:command command :options {:features ["cache"] :build-timeout-ms value}}] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check (request/apply-defaults partial))))))))) + +(t/deftest command-values-use-their-domain-constraints + (doseq [partial [{:command "run" :options {:build {:kind :existing :id "../build"}}} + {:command "run" :options {:build {:kind :existing :id ""}}} + {:command "build" :options {:features ["bad feature"]}} + {:command "build" :options {:features [nil]}} + {:command "ab" :options {:features []}} + {:command "ab" :options {:features ["cache"] :max-features 0}} + {:command "ab" :options {:features ["cache"] :max-features 7}} + {:command "ab" :options {:features ["cache"] :max-features 1.5}} + {:command "compare" :options {:baseline-path " " :candidate-path "c"}} + {:command "merge" :options {:base-path "b" :additional-path ""}} + {:command "merge" :options {:base-path "b" :additional-path "a" + :output-dir " "}} + {:command "purge" :options {:target :unknown}} + {:command "help" :options {:topic "unknown"}} + {:command "help" :console {:level :error}} + {:command "help" :console {:color true}}]] + (t/testing (pr-str partial) + (let [resolved (request/apply-defaults partial)] + (t/is (= ::request/invalid-request + (:type (failure-data #(request/check resolved)))))))) + (doseq [pair [["only"] ["one" "two" "three"] ["" "two"] ["one" " "]]] + (t/is (= ::request/invalid-request + (:type (failure-data + #(request/check + (request/apply-defaults + {:command "compare" :options {:baseline-path "b" :candidate-path "c" + :case-pair pair}}))))))) + (doseq [filter-text [nil "" " " " rects "]] + (let [input (assoc-in existing-run [:options :filter] filter-text)] + (t/is (= input (request/check (request/apply-defaults input)))))) + (doseq [target [:builds :runs :comparisons :all]] + (t/is (= target + (get-in (request/check + (request/apply-defaults {:command "purge" :options {:target target}})) + [:options :target]))))) diff --git a/frontend/test/performance_tests/render_wasm/run_test.cljs b/frontend/test/performance_tests/render_wasm/run_test.cljs new file mode 100644 index 0000000000..ee71a21f8f --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/run_test.cljs @@ -0,0 +1,1483 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.run-test + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.node.cli.deferred :as deferred] + [benches.render-wasm.node.cli.request :as request] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runner.browser :as browser] + [benches.render-wasm.node.runner.feature-comparison :as feature-comparison] + [benches.render-wasm.node.runner.run :as run] + [benches.render-wasm.node.runner.server :as server] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.report.format :as format] + [benches.render-wasm.report.summarize :as summary] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-run-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- output-dir + [] + (path/join @temporary-root (str "out-" (random-uuid)))) + +(defn- build-dir + [] + (let [dir (path/join @temporary-root (str "build-" (random-uuid)))] + (fs/mkdirSync dir #js {:recursive true}) + (fs/writeFileSync (path/join dir "index.html") "") + dir)) + +(defn- selected-build + [dir id] + {:id id + :dir dir + :log-path (path/join dir "build.log") + :manifest {:type :renderer-build + :id id + :status :ready + :created-at "2026-10-27T00:00:00.000Z" + :updated-at "2026-10-27T00:00:00.000Z" + :functional {:mode :release + :default-features? true + :features [] + :cljs-profile-marks false} + :provenance {:git {:sha "build-revision" :dirty false}}}}) + +(defn- collected + [id context] + {:id id + :scene :case + :scene-description "test scene" + :scene-seed 1 + :params {} + :view {:scale 1 :x 0 :y 0} + :context context + :preparation {}}) + +(defn- next-count + [state key] + (inc (get @state key 0))) + +(defn- check-page + [page] + (when-not (and (map? page) (some? (:id page))) + (throw (ex-info "browser service requires the page object" + {:phase :browser-page-startup})))) + +(def ^:private hardware-graphics + {:webgl2 true :vendor "NVIDIA" :renderer "ANGLE (NVIDIA, Vulkan, NVIDIA)" :software false}) + +(defn- fake-browser + [{:keys [launch-failure new-page-failure close-page-failure + prepare-failure run-failure screenshot-failure + interruption controller effective-graphics events close-failure] + :or {effective-graphics (constantly {}) + events [{:command :fixture/work :metrics {:command/elapsed [1]}}]}}] + (let [state (atom {:launches 0 :pages 0 :close-pages 0 :closes 0 + :host nil :runs {} :preparations {}})] + {:state state + :service + {:launch + (fn [_] + (let [n (next-count state :launches)] + (swap! state assoc :launches n) + (when (and launch-failure (launch-failure n)) + (throw (ex-info "browser launch failed" {:phase :browser-startup}))) + {:id n :version "FakeBrowser/1" :browser #js {}})) + :new-page + (fn [owner host _] + (when-not (and (string? (:base-url host)) + (some? (get-in host [:build :id]))) + (throw (ex-info "new-page requires the opened host" + {:phase :browser-page-startup}))) + (swap! state assoc :host host) + (let [n (next-count state :pages)] + (swap! state assoc :pages n) + (when (and new-page-failure (new-page-failure n)) + (throw (ex-info "browser page failed" {:phase :browser-page-startup}))) + {:page {:id n :browser-id (:id owner)} :context {}})) + :close-page + (fn [_] + (let [n (next-count state :close-pages)] + (swap! state assoc :close-pages n) + (when (and close-page-failure (close-page-failure n)) + (throw (ex-info "page close failed" {:phase :browser-page-close}))))) + :close + (fn [_] + (let [n (:closes (swap! state update :closes inc))] + (when (and close-failure (close-failure n)) + (throw (ex-info "browser shutdown failed" {:phase :browser-close}))))) + :connected? + (fn [_] true) + :prepare + (fn [page request _] + (check-page page) + (let [{:keys [preparation-id] case-desc :case} (transit/decode-str request) + case-id (:id case-desc) + failure (when prepare-failure + (prepare-failure case-id))] + (case failure + :throw + (throw (ex-info "scene preparation failed" {:phase :scene-build})) + + :response + {:status "failed" :phase "scene-build" :message "scene rejected"} + + (let [preparation {:preparation-id preparation-id + :case-id case-id + :fingerprint {:algorithm :sha-256 + :digest (apply str (repeat 64 "a"))} + :effective-graphics (effective-graphics case-id (:id page)) + :diagnostics {:browser-id (:browser-id page)} + :phases [{:phase :preparation :preparation-id preparation-id + :outcome :completed :events []}]}] + (swap! state assoc-in [:preparations preparation-id] case-id) + {:status "ok" :preparation preparation})))) + :run + (fn [page request _] + (check-page page) + (let [{:keys [preparation-id]} (transit/decode-str request) + case-id (get-in @state [:preparations preparation-id]) + n (inc (get-in @state [:runs case-id] 0)) + _ (swap! state assoc-in [:runs case-id] n) + signal (when interruption (interruption case-id n))] + (when signal + (let [reason (ex-info "run interrupted" + {:type ::interrupted :signal signal})] + (.abort controller reason) + (throw reason))) + (case (when run-failure (run-failure case-id n)) + :throw + (throw (ex-info "attempt failed" {:phase "interact"})) + + :response + {:status "failed" :phase "interact" :message "attempt rejected"} + + {:status "ok" + :phases [{:phase :main :preparation-id preparation-id :outcome :completed + :events events}]}))) + :screenshot + (fn [page file _] + (check-page page) + (when (and screenshot-failure (screenshot-failure file)) + (throw (ex-info "screenshot failed" {:phase :screenshot}))) + (fs/writeFileSync file "png"))}})) + +(defn- options + [fake out] + {:output-dir out + :seed 42 + :filter nil + :warmups 0 + :repetitions 1 + :timeout-ms 5000 + :startup-timeout-ms 5000 + :screenshots false + :remove false + :browser (:service fake)}) + +(defn- execute + ([fake cases opts] + (execute fake cases opts (js/AbortController.))) + ([_fake cases opts controller] + (run/execute (artifacts/frontend-root) + (runs/create-home {:root (artifacts/frontend-root) :run-id "test-run" + :output-dir (:output-dir opts)}) + (selected-build (build-dir) "test-build") + cases opts (.-signal controller)))) + +(defn- attempts-for + [run case-id] + (filterv #(= case-id (:case-id %)) (:attempts run))) + +(t/deftest ^:async unmeasured-successful-work-saves-a-terminal-failure + (let [fake (fake-browser {:events []}) + response (await (execute fake [(collected :case/unmeasured :reuse)] + (options fake (path/join @temporary-root "unmeasured")))) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))] + (t/is (= 1 (:exit-code response))) + (t/is (= :failed (get-in saved [:termination :reason]))) + (t/is (= :no-measurements (get-in saved [:case-outcomes :case/unmeasured :failure :reason]))) + (t/is (= :completed (get-in saved [:attempts 0 :outcome]))))) + +(defn- preparations-for + [run case-id] + (filterv #(= case-id (:case-id %)) (:preparations run))) + +(defn- saved-record + [result] + (result/decode (fs/readFileSync (get-in result [:home :record-path]) "utf8"))) + +(t/deftest ^:async retained-run-keeps-all-progress-levels-in-its-log + (doseq [level [:info :warn :debug]] + (let [root @temporary-root + cases [(collected :case/a :fresh) (collected :case/b :reuse)] + build (artifacts/create-build root {:default-features? true :features []} + {:git {:sha "fixture" :dirty false}}) + stdout (atom "") + stderr (atom "") + fake (fake-browser {}) + out (output-dir)] + (doseq [file [(:index-path build) (path/join (:browser-dir build) "browser.js") + (path/join (:renderer-dir build) "render-wasm.js") + (path/join (:renderer-dir build) "render-wasm.wasm") + (path/join (:renderer-dir build) "shared.js")]] + (fs/writeFileSync file "fixture" "utf8")) + (fs/writeFileSync (:cases-path build) + (str "module.exports = {buildIdentity() {return " + (js/JSON.stringify (transit/encode-str {:build-id (:id build)})) + ";}, collectCases() {return " + (js/JSON.stringify (transit/encode-str {:build-id (:id build) :cases cases})) + ";}};") "utf8") + (artifacts/write-manifest build (assoc (:manifest build) :status :ready)) + (log/configure {:level level :color :never + :stdout #js {:write #(swap! stdout str %)} + :stderr #js {:write #(swap! stderr str %)}}) + (let [resolved (:options (request/apply-defaults + {:command "run" + :options (assoc (dissoc (options fake out) :browser) + :build {:kind :existing :id (:id build)} + :warmups 1 :repetitions 2)})) + response (await (run/run root resolved nil {:browser (:service fake)})) + saved (saved-record response) + text (fs/readFileSync (get-in response [:home :log-path]) "utf8")] + (t/is (= 0 (:exit-code response))) + (t/is (= (:id build) (get-in saved [:metadata :build :provenance :build-id]))) + (t/is (str/includes? text "▸ Case case/a (fresh)\n")) + (doseq [case-text (rest (str/split text #"▸ Case "))] + (t/is (str/includes? case-text " attempt 1/3 completed (warmup)\n")) + (t/is (str/includes? case-text " attempt 2/3 completed\n")) + (t/is (str/includes? case-text " attempt 3/3 completed\n"))) + (t/is (str/includes? text "Browser FakeBrowser/1")) + (t/is (= (not= :warn level) (str/includes? @stderr "▸ Case case/a"))) + (t/is (= (= :debug level) (str/includes? @stderr "Browser FakeBrowser/1"))))))) + +(t/deftest ^:async fresh-and-reuse-cases-follow-their-context-policies + (let [fake (fake-browser {}) + cases [(collected :case/fresh :fresh) (collected :case/reuse :reuse)] + opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2) + result (await (execute fake cases opts)) + saved (saved-record result) + run (:run result) + fresh (attempts-for run :case/fresh) + reuse (attempts-for run :case/reuse)] + (t/is (= saved run)) + (t/is (= 6 (count (:attempts run)))) + (t/is (= 3 (count (preparations-for run :case/fresh)))) + (t/is (= 1 (count (preparations-for run :case/reuse)))) + (t/is (= 3 (count (distinct (map :preparation-id fresh))))) + (t/is (= 1 (count (distinct (map :preparation-id reuse))))) + (t/is (every? #(= :completed (:outcome %)) (:attempts run))) + (t/is (every? #(every? zero? (vals %)) (vals (:remaining run)))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (zero? (:exit-code result))) + (t/is (empty? (get-in run [:metadata :diagnostics :screenshots]))) + (t/is (empty? (filter #(str/ends-with? % ".png") + (array-seq (fs/readdirSync (get-in result [:home :dir])))))))) + +(t/deftest ^:async page-startup-failure-abandons-the-case-and-keeps-its-slots + (doseq [context [:fresh :reuse]] + (let [fake (fake-browser {:new-page-failure #(= 1 %)}) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/a context) + (collected :case/b :fresh)] + opts)) + run (:run result) + state @(:state fake)] + (t/is (empty? (attempts-for run :case/a))) + (t/is (empty? (preparations-for run :case/a))) + (t/is (= {:warmups 0 :repetitions 2} + (get-in run [:remaining :case/a]))) + (t/is (= [:completed :completed] + (mapv :outcome (attempts-for run :case/b)))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :browser-page (get-in run [:termination :failure :stage]))) + (t/is (= 1 (:exit-code result))) + (t/is (= 2 (:launches state))) + (t/is (= 2 (:closes state)))))) + +(t/deftest ^:async failed-attempts-stop-the-case-and-keep-pending-slots + (doseq [failure [:throw :response]] + (let [fake (fake-browser {:run-failure (fn [case-id n] + (when (and (= case-id :case/fresh) (= n 1)) + failure))}) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/fresh :fresh)] opts)) + run (:run result) + tries (attempts-for run :case/fresh)] + (t/is (= [:failed] (mapv :outcome tries))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/fresh]))) + (t/is (= :failed (get-in run [:case-outcomes :case/fresh :outcome]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))) + (t/is (= 1 (count (preparations-for run :case/fresh))))))) + +(t/deftest ^:async reuse-preparation-failure-leaves-slots-and-continues-to-the-next-case + (doseq [kind [:throw :response]] + (let [fake (fake-browser {:prepare-failure #(when (= % :case/reuse) kind)}) + opts (assoc (options fake (output-dir)) :screenshots true :repetitions 2) + result (await (execute fake [(collected :case/reuse :reuse) + (collected :case/fresh :fresh)] opts)) + run (:run result) + shots (get-in run [:metadata :diagnostics :screenshots])] + (t/is (empty? (attempts-for run :case/reuse))) + (t/is (= {:warmups 0 :repetitions 2} (get-in run [:remaining :case/reuse]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for run :case/fresh)))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))) + (t/is (= :skipped (get-in shots ["case/reuse" :status]))) + (t/is (= :ok (get-in shots ["case/fresh" :status]))) + (t/is (= 1 (count (re-seq (re-pattern (if (= kind :throw) + "scene preparation failed" "scene rejected")) + (:report result))))) + (let [failure (last (get-in run [:metadata :diagnostics :failures]))] + (t/is (= :preparation (:stage failure))) + (t/is (= :case/reuse (:case-id failure))) + (t/is (= "scene-build" (:phase failure))))))) + +(t/deftest ^:async fresh-preparation-failure-consumes-its-slot-and-stops-the-case + (doseq [kind [:throw :response]] + (let [preparations (atom 0) + fake (fake-browser {:prepare-failure #(when (and (= % :case/fresh) + (= 1 (swap! preparations inc))) + kind)}) + result (await (execute fake [(collected :case/fresh :fresh)] + (assoc (options fake (output-dir)) :repetitions 2 :screenshots true))) + run (:run result) + tries (attempts-for run :case/fresh)] + (t/is (= [:failed] (mapv :outcome tries))) + (t/is (= (:preparation-id (first (preparations-for run :case/fresh))) + (:preparation-id (first tries)))) + (t/is (= "scene-build" (get-in (first tries) [:failure :phase]))) + (t/is (= 1 (count (preparations-for run :case/fresh)))) + (t/is (= :skipped (get-in run [:metadata :diagnostics :screenshots "case/fresh" :status]))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/fresh]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (count (re-seq (re-pattern (if (= kind :throw) + "scene preparation failed" "scene rejected")) + (:report result))))) + (t/is (str/includes? (:report result) "no-completed-measured-attempt")) + (t/is (= (:report result) (format/format-run (summary/summarize (saved-record result))))) + (t/is (= 1 (:exit-code result)))))) + +(t/deftest ^:async reuse-attempt-failure-leaves-pending-slots-and-continues-to-the-next-case + (doseq [kind [:throw :response]] + (let [fake (fake-browser {:run-failure (fn [case-id n] + (when (and (= case-id :case/reuse) (= n 1)) kind))}) + opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2 :screenshots true) + result (await (execute fake [(collected :case/reuse :reuse) + (collected :case/fresh :fresh)] opts)) + run (saved-record result)] + (t/is (= [:failed] (mapv :outcome (attempts-for run :case/reuse)))) + (t/is (= {:warmups 0 :repetitions 2} (get-in run [:remaining :case/reuse]))) + (t/is (= [:completed :completed :completed] (mapv :outcome (attempts-for run :case/fresh)))) + (t/is (= {:status :skipped :reason :no-completed-measured-attempt} + (get-in run [:metadata :diagnostics :screenshots "case/reuse"]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result)))))) + +(t/deftest ^:async final-attempt-failure-records-why-the-screenshot-was-skipped + (doseq [context [:fresh :reuse] + failure [:preparation :attempt] + :when (or (= context :fresh) (= failure :attempt))] + (let [fake (fake-browser (if (= failure :preparation) + {:prepare-failure (constantly :response)} + {:run-failure (fn [_ _] :response)})) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a context)] opts)) + run (saved-record result)] + (t/is (= [:failed] (mapv :outcome (:attempts run)))) + (t/is (= {:status :skipped :reason :no-completed-measured-attempt} + (get-in run [:metadata :diagnostics :screenshots "case/a"]))) + (t/is (= :failed (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result)))))) + +(t/deftest ^:async browser-close-failure-abandons-the-case-and-relaunches-for-the-next + (let [fake (fake-browser {:close-page-failure #(= % 1)}) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/a :fresh) + (collected :case/b :fresh)] opts)) + run (:run result) + state @(:state fake)] + (t/is (= 2 (:launches state))) + (t/is (= 1 (count (attempts-for run :case/a)))) + (t/is (= :completed (:outcome (first (attempts-for run :case/a))))) + (t/is (= :failed (get-in run [:case-outcomes :case/a :outcome]))) + (t/is (= 1 (get-in run [:remaining :case/a :repetitions]))) + (t/is (= 2 (count (attempts-for run :case/b)))) + (t/is (every? #(= :completed (:outcome %)) (attempts-for run :case/b))) + (t/is (every? zero? (vals (get-in run [:remaining :case/b])))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async page-close-failure-after-the-final-slot-is-a-harness-failure + (let [fake (fake-browser {:close-page-failure #(= % 1)}) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + run (:run result)] + (t/is (= :failed (get-in run [:termination :reason]))) + (t/is (every? zero? (vals (get-in run [:remaining :case/a])))) + (t/is (= :browser-close + (:stage (last (get-in run [:metadata :diagnostics :failures]))))) + (t/is (= :browser-close (get-in run [:termination :failure :stage]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async initial-browser-launch-failure-aborts-before-any-case + (let [fake (fake-browser {:launch-failure #(= % 1)}) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + run (:run result) + state @(:state fake)] + (t/is (= 1 (:launches state))) + (t/is (empty? (:attempts run))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/a]))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :browser-launch (:stage (last (get-in run [:metadata :diagnostics :failures]))))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async browser-process-details-survive-saved-run-report + (let [context {:command "chromium" :exit-code 17 :signal "SIGKILL" + :endpoint "ws://127.0.0.1:1234/devtools/browser/owner"} + fake (assoc-in (fake-browser {}) [:service :launch] + (fn [_] + (throw (ex-info "browser launch failed" context)))) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + saved (saved-record result)] + (t/is (= 1 (:exit-code result))) + (t/is (= context (select-keys (get-in saved [:termination :failure]) (keys context)))) + (t/is (= context (select-keys (first (get-in saved [:metadata :diagnostics :failures])) + (keys context)))) + (t/is (str/includes? (:report result) "chromium")) + (t/is (str/includes? (:report result) "SIGKILL")) + (t/is (str/includes? (:report result) (:endpoint context))))) + +(t/deftest ^:async hardware-preparations-complete-fresh-and-reuse-cases + (let [fake (fake-browser {:effective-graphics (constantly hardware-graphics)}) + opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1 :repetitions 2) + result (await (execute fake [(collected :case/fresh :fresh) + (collected :case/reuse :reuse)] opts)) + run (saved-record result)] + (t/is (= 0 (:exit-code result))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= 6 (count (:attempts run)))) + (t/is (every? #(= hardware-graphics (:effective-graphics %)) (:preparations run))) + (t/is (true? (get-in run [:metadata :diagnostics :require-gpu?]))))) + +(t/deftest ^:async required-gpu-failures-preserve-evidence-and-all-attempt-slots + (doseq [context [:fresh :reuse] + graphics [(assoc hardware-graphics :renderer "ANGLE (SwiftShader)" :software true) + (assoc hardware-graphics :webgl2 false) + (assoc hardware-graphics :renderer nil) + (assoc hardware-graphics :renderer " UnKnOwN ") + (assoc hardware-graphics :vendor "") + {}]] + (let [fake (fake-browser {:effective-graphics (constantly graphics)}) + opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1) + result (await (execute fake [(collected :case/a context) + (collected :case/b :reuse)] opts)) + run (saved-record result) + failure (get-in run [:termination :failure]) + state @(:state fake)] + (t/is (= 1 (:exit-code result))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= [graphics] (mapv :effective-graphics (:preparations run)))) + (t/is (= [] (:attempts run))) + (t/is (= {:case/a {:warmups 1 :repetitions 1} + :case/b {:warmups 1 :repetitions 1}} (:remaining run))) + (t/is (= :gpu (:stage failure))) + (t/is (= graphics (get-in failure [:partial :graphics]))) + (t/is (str/includes? (:message failure) (pr-str (:renderer graphics)))) + (t/is (str/includes? (:message failure) "./manage.sh run-devenv --gpu")) + (t/is (str/includes? (:message failure) "pnpm run render-bench")) + (t/is (= 1 (:closes state)))))) + +(t/deftest ^:async graphics-rejection-preserves-earlier-attempts-and-stops-the-run + (doseq [context [:fresh :reuse]] + (let [software (assoc hardware-graphics :renderer "SwiftShader" :software true) + fake (fake-browser {:effective-graphics (fn [_ page-id] + (if (= page-id 1) hardware-graphics software))}) + opts (assoc (options fake (output-dir)) :require-gpu true :repetitions 2) + result (await (execute fake [(collected :case/a context) + (collected :case/b context)] opts)) + run (saved-record result) + failure (get-in run [:termination :failure])] + (t/is (= 1 (:exit-code result))) + (t/is (= :gpu (:stage failure))) + (t/is (= software (get-in failure [:partial :graphics]))) + (t/is (= (if (= context :fresh) 1 2) (count (:attempts run)))) + (t/is (every? #(= :completed (:outcome %)) (:attempts run))) + (t/is (= [hardware-graphics software] (mapv :effective-graphics (:preparations run)))) + (t/is (= {:case/a {:warmups 0 :repetitions (if (= context :fresh) 1 0)} + :case/b {:warmups 0 :repetitions 2}} (:remaining run))) + (t/is (= 1 (:closes @(:state fake))))))) + +(t/deftest ^:async sigint-and-sigterm-persist-partial-interrupted-records + (doseq [context [:fresh :reuse] + [signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + fake (fake-browser {:controller controller + :interruption (fn [case-id n] + (when (and (= case-id :case/a) (= n 1)) + signal))}) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a context) + (collected :case/b context)] + opts controller)) + run (:run result) + saved (saved-record result) + shots (get-in run [:metadata :diagnostics :screenshots])] + (t/is (= saved run)) + (t/is (= :interrupted (get-in run [:termination :reason]))) + (t/is (= signal (get-in run [:termination :signal]))) + (t/is (= code (:exit-code result))) + (t/is (= 1 (count (attempts-for run :case/a)))) + (t/is (empty? (attempts-for run :case/b))) + (t/is (= 1 (get-in run [:remaining :case/b :repetitions]))) + (t/is (= :skipped (get-in shots ["case/b" :status]))) + (t/is (= :not-run (get-in shots ["case/b" :reason]))) + (t/is (pos? (:closes @(:state fake))))))) + +(t/deftest ^:async interruption-before-an-attempt-preserves-unspent-slots + (doseq [context [:fresh :reuse] + method [:launch :new-page :prepare] + [reason signal code] [["SIGINT" "SIGINT" 130] + ["SIGTERM" "SIGTERM" 143] + [:SIGTERM "SIGTERM" 143] + [(ex-info "stopped" {:signal "SIGTERM"}) "SIGTERM" 143] + [(js/Error. "stopped") "SIGINT" 130]]] + (let [controller (js/AbortController.) + base (fake-browser {}) + stop (^:async fn [& _] + (let [cause (ex-info "interrupted setup" {:signal signal :phase :preparation})] + (.abort controller reason) + (throw cause))) + fake (assoc base :service (assoc (:service base) method stop)) + opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2) + result (await (execute fake [(collected :case/a context) + (collected :case/b context)] opts controller)) + run (saved-record result) + consumed (if (and (= method :prepare) (= context :fresh)) 1 0)] + (t/is (= :interrupted (get-in run [:termination :reason]))) + (t/is (= signal (get-in run [:termination :signal]))) + (t/is (= code (:exit-code result))) + (t/is (= consumed (count (:attempts run)))) + (t/is (= {:warmups (- 1 consumed) :repetitions 2} (get-in run [:remaining :case/a]))) + (t/is (= {:warmups 1 :repetitions 2} (get-in run [:remaining :case/b]))) + (t/is (= (if (and (= method :prepare) (= context :reuse)) [:preparation] []) + (mapv :stage (get-in run [:metadata :diagnostics :failures]))))))) + +(t/deftest ^:async phase-and-output-failures-retain-evidence-and-stop-the-case + (doseq [context [:fresh :reuse] + phase [:preparation :main :reset :invalid-preparation :contradictory-preparation :invalid-main :invalid-reset]] + (let [base (fake-browser {:prepare-failure (when (= :preparation phase) + #(when (= :case/a %) :response))}) + services (:service base) + fake (assoc base :service + (assoc services + :prepare (fn [page request bounds] + (let [response ((:prepare services) page request bounds)] + (if (= :case/a (get-in (transit/decode-str request) [:case :id])) + (case phase + :invalid-preparation (update response :preparation dissoc :fingerprint) + :contradictory-preparation + (-> response + (assoc-in [:preparation :failure] + {:phase "preparation" :message "callback failed"}) + (assoc-in [:preparation :phases 0 :outcome] :failed)) + response) + response))) + :run (fn [page request bounds] + (let [id (:preparation-id (transit/decode-str request)) + case-id (get-in @(:state base) [:preparations id]) + block {:phase :main :preparation-id id :outcome :completed + :events [{:command :fixture/work :metrics {:command/elapsed [1]}}]}] + (if (= :case/a case-id) + (case phase + :main {:status "failed" :phase "main" :message "main failed" + :cause [{:message "WASM trap" :code "E_RENDER"}] + :reason :renderer-rejected :deadline-exceeded false + :partial {:completed-call 7} + :phases [(assoc block :outcome :failed)]} + :reset {:status "failed" :phase "reset" :message "reset failed" + :cause [{:message "WASM trap" :code "E_RENDER"}] + :reason :renderer-rejected :deadline-exceeded false + :partial {:completed-call 7} + :phases [block (assoc block :phase :reset :outcome :failed)]} + :invalid-main {:status "ok" :phases []} + :invalid-reset {:status "ok" :phases [block {:phase :reset}]}) + ((:run services) page request bounds)))))) + response (await (execute fake [(collected :case/a context) (collected :case/b context)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response) + prep-failed? (#{:preparation :invalid-preparation :contradictory-preparation} phase) + attempts (attempts-for saved :case/a)] + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= (if (and prep-failed? (= :reuse context)) [] [:failed]) (mapv :outcome attempts))) + (t/is (= {:warmups 0 :repetitions (if (and prep-failed? (= :reuse context)) 2 1)} + (get-in saved [:remaining :case/a]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for saved :case/b)))) + (when (#{:main :reset} phase) + (t/is (= {:cause [{:message "WASM trap" :code "E_RENDER"}] + :reason :renderer-rejected :deadline-exceeded false} + (select-keys (:failure (first attempts)) [:cause :reason :deadline-exceeded]))) + (t/is (= {:completed-call 7} (:partial (first attempts))))) + (when (= :contradictory-preparation phase) + (t/is (= "invalid-output" (get-in (first (preparations-for saved :case/a)) [:failure :phase])))) + (when (= :invalid-reset phase) + (t/is (= [:main] (mapv :phase (:phases (first attempts))))) + (t/is (= {:phase :reset} (get-in (first attempts) [:partial :response :phases 1])))) + (when (#{:invalid-main :invalid-reset} phase) + (t/is (= "invalid-output" (get-in (first attempts) [:failure :phase]))))))) + +(t/deftest ^:async malformed-bridge-diagnostics-stop-the-case-with-delivered-evidence + (doseq [stage [:prepare :run] + evidence [nil "unexpected response" + {:status "unknown" :phase "main" :message "unexpected status"} + {:status "failed" :phase "main"} + {:status "failed" :phase "main" :message "bad cause" :cause [:invalid]}]] + (let [base (fake-browser {}) + service (get-in base [:service stage]) + delivered (atom nil) + fake (assoc-in base [:service stage] + (fn [page request bounds] + (let [response (service page request bounds) + params (transit/decode-str request) + case-id (if (= :prepare stage) + (get-in params [:case :id]) + (get-in @(:state base) [:preparations (:preparation-id params)]))] + (if (= :case/a case-id) + (let [phases (if (= :prepare stage) + (get-in response [:preparation :phases]) + (conj (:phases response) {:phase :reset})) + rejected (cond-> evidence + (map? evidence) (assoc :phases phases))] + (reset! delivered rejected) + (transit/decode-str (transit/encode-str rejected))) + response)))) + response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response) + failed (if (= :prepare stage) + (first (preparations-for saved :case/a)) + (first (attempts-for saved :case/a)))] + (t/is (= 1 (:exit-code response))) + (t/is (= "invalid-output" (get-in failed [:failure :phase]))) + (t/is (= "invalid bridge response" (get-in failed [:failure :message]))) + (t/is (= @delivered (if (= :prepare stage) + (get-in failed [:failure :partial :response]) + (get-in failed [:partial :response])))) + (t/is (= (if (map? evidence) [(first (:phases @delivered))] []) (:phases failed))) + (t/is (= {:warmups 0 :repetitions (if (= :prepare stage) 2 1)} + (get-in saved [:remaining :case/a]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for saved :case/b)))) + (t/is (str/includes? (:report response) "invalid bridge response"))))) + +(t/deftest ^:async stale-bridge-results-stop-the-case-with-ownership-diagnostics + (doseq [stage [:prepare :run]] + (let [base (fake-browser {}) + fake (assoc-in base [:service stage] (constantly {:status "stale"})) + response (await (execute fake [(collected :case/a :reuse)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response) + failure (get-in saved [:case-outcomes :case/a :failure])] + (t/is (= 1 (:exit-code response))) + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= "stale" (:status failure))) + (t/is (= (if (= :prepare stage) "preparation" "attempt") (:phase failure))) + (t/is (= "browser context was superseded" (:message failure))) + (t/is (= {:warmups 0 :repetitions (if (= :prepare stage) 2 1)} + (get-in saved [:remaining :case/a])))))) + +(t/deftest ^:async runner-terminates-a-blocked-page-and-completes-the-next-case + (let [dir (build-dir) + owners (atom []) + prepared (atom {}) + attempts (atom {}) + identity-text (.stringify js/JSON (transit/encode-str {:build-id "test-build"})) + services (assoc browser/service + :launch (^:async fn [options] + (when-let [previous (last @owners)] + (t/is (false? (.isConnected ^js (:browser previous)))) + (t/is (not (fs/existsSync (:user-data-dir previous))))) + (let [owner (await (browser/launch options))] + (swap! owners conj owner) + owner)) + :prepare (fn [_ request _] + (let [{:keys [preparation-id] definition :case} (transit/decode-str request)] + (swap! prepared assoc preparation-id (:id definition)) + {:status "ok" + :preparation {:preparation-id preparation-id :case-id (:id definition) + :fingerprint {:algorithm :sha-256 :digest (apply str (repeat 64 "a"))} + :effective-graphics {} :diagnostics {} + :phases [{:phase :preparation :preparation-id preparation-id + :outcome :completed :events []}]}})) + :run (^:async fn [page request bounds] + (let [id (:preparation-id (transit/decode-str request)) + case-id (get @prepared id) + n (get (swap! attempts update case-id (fnil inc 0)) case-id)] + (await (browser/within + #(.evaluate ^js page + (if (and (= :case/a case-id) (= n 2)) + "while (true) {}" "1")) + bounds)) + {:status "ok" + :phases [{:phase :main :preparation-id id :outcome :completed + :events [{:command :fixture/work :metrics {:command/elapsed [1]}}]}]})))] + (fs/writeFileSync (path/join dir "index.html") + (str "")) + (try + (let [opts (assoc (options nil (output-dir)) :browser services :repetitions 2 + :timeout-ms 3000 :startup-timeout-ms 30000) + response (await (run/execute (artifacts/frontend-root) + (runs/create-home {:root (artifacts/frontend-root) :run-id "blocked-case" + :output-dir (:output-dir opts)}) + (selected-build dir "test-build") + [(collected :case/a :reuse) (collected :case/b :reuse)] opts nil)) + saved (saved-record response)] + (t/is (= [:completed :failed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for saved :case/b)))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome]))) + (t/is (= 2 (count @owners))) + (t/is (not= (:user-data-dir (first @owners)) (:user-data-dir (second @owners)))) + (doseq [owner @owners] + (t/is (false? (.isConnected ^js (:browser owner)))) + (t/is (not (fs/existsSync (:user-data-dir owner)))))) + (finally + (doseq [owner @owners] + (await (browser/close owner))))))) + +(t/deftest ^:async opening-and-screenshot-work-share-the-case-deadline + (doseq [stage [:page :screenshot]] + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {}) + spend (fn [page bounds] + (when (= 1 (:id page)) + (swap! clock + 300) + (when (>= @clock (:deadline bounds)) + (throw (ex-info "browser work timed out" {:phase (name stage)}))))) + fake (if (= :page stage) + (assoc-in base [:service :new-page] + (fn [owner host bounds] + (let [opened ((get-in base [:service :new-page]) owner host bounds)] + (spend (:page opened) bounds) + opened))) + (assoc-in base [:service :screenshot] + (fn [page file bounds] + (spend page bounds) + ((get-in base [:service :screenshot]) page file bounds))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 250 :screenshots true))) + saved (saved-record response)] + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome]))) + (t/is (= {:warmups 0 :repetitions (if (= :page stage) 1 0)} + (get-in saved [:remaining :case/a]))) + (when (= :screenshot stage) + (t/is (= [:completed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= 1 (count (re-seq #"browser work timed out" (:report response))))) + (t/is (str/includes? (:report response) "\"case/a\": failed; see Harness 0")) + (t/is (str/includes? (:report response) "File \"case-a.png\"")))) + (finally + (set! (.-now js/Date) system-now)))))) + +(t/deftest ^:async failed-browser-disposal-stops-later-cases + (let [fake (fake-browser {:run-failure (fn [_ _] :response) :close-failure #(= 1 %)}) + response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :repetitions 2))) + saved (saved-record response)] + (t/is (= [:case/a] (mapv :case-id (:attempts saved)))) + (t/is (= {:warmups 0 :repetitions 2} (get-in saved [:remaining :case/b]))) + (t/is (= :browser-close (get-in saved [:termination :failure :stage]))))) + +(t/deftest ^:async distinct-harness-failures-with-equal-diagnostics-appear-separately + (let [fake (fake-browser {:run-failure (fn [_ _] :response) + :close-failure (constantly true)}) + response (await (execute fake [(collected :case/a :reuse)] + (options fake (output-dir)))) + saved (saved-record response) + report (:report response)] + (t/is (= [:browser-close :cleanup] (mapv :stage (get-in saved [:metadata :diagnostics :failures])))) + (t/is (str/includes? report "Harness 0")) + (t/is (str/includes? report "Harness 1")) + (t/is (str/includes? report ":browser-close")) + (t/is (str/includes? report ":cleanup")) + (t/is (= 2 (count (re-seq #"browser shutdown failed" report)))) + (t/is (= 1 (count (re-seq #"attempt rejected" report)))))) + +(t/deftest ^:async failed-startup-cleanup-aborts-even-after-the-case-deadline + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {:run-failure (fn [_ _] :response)}) + fake (assoc-in base [:service :launch] + (fn [bounds] + (let [owner ((get-in base [:service :launch]) bounds)] + (when (= 2 (:id owner)) + (reset! clock (inc (:deadline bounds))) + (throw (ex-info "browser shutdown failed after startup error" + {:type ::browser/startup-cleanup}))) + owner)))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse) + (collected :case/c :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 250))) + saved (saved-record response)] + (t/is (= :failed (get-in saved [:case-outcomes :case/b :outcome]))) + (t/is (= {:warmups 0 :repetitions 1} (get-in saved [:remaining :case/c]))) + (t/is (= :browser-launch (get-in saved [:termination :failure :stage])))) + (finally + (set! (.-now js/Date) system-now))))) + +(t/deftest ^:async cleanup-time-does-not-change-the-case-failure + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {:new-page-failure #(= 1 %)}) + fake (assoc-in base [:service :close] + (fn [owner] + (swap! clock + 1000) + ((get-in base [:service :close]) owner)))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 500))) + saved (saved-record response)] + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome])))) + (finally + (set! (.-now js/Date) system-now))))) + +(t/deftest ^:async service-cleanup-preserves-the-originating-failure-classification + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {:new-page-failure #(= 1 %)}) + fake (assoc-in base [:service :new-page] + (fn [owner host bounds] + (try + ((get-in base [:service :new-page]) owner host bounds) + (catch :default cause + (let [annotated (errors/with-context cause "page startup failed" + {::browser/deadline-exceeded false})] + (swap! clock + 1000) + (throw annotated))))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a :reuse) (collected :case/b :reuse)] + (assoc (options fake (output-dir)) :timeout-ms 250))) + saved (saved-record response)] + (t/is (= :failed (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= :completed (get-in saved [:case-outcomes :case/b :outcome])))) + (finally + (set! (.-now js/Date) system-now))))) + +(t/deftest ^:async case-expiry-keeps-delivered-attempts-and-continues-in-a-new-browser + (doseq [context [:fresh :reuse]] + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {}) + active-case (atom nil) + spend (fn [duration bounds] + (when (= :case/a @active-case) + (swap! clock + duration) + (when (>= @clock (:deadline bounds)) + (throw (ex-info "browser work timed out" {:phase (:phase bounds)}))))) + services (:service base) + fake (assoc base :service + (assoc services + :prepare (fn [page request bounds] + (reset! active-case (get-in (transit/decode-str request) [:case :id])) + (spend 25 bounds) + ((:prepare services) page request bounds)) + :run (fn [page request bounds] + (spend 110 bounds) + ((:run services) page request bounds))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [response (await (execute fake [(collected :case/a context) (collected :case/b context)] + (assoc (options fake (output-dir)) + :warmups 1 :repetitions 3 :timeout-ms 350))) + saved (saved-record response) + first-prep (first (preparations-for saved :case/a)) + next-prep (first (preparations-for saved :case/b))] + (t/is (= [:completed :completed :failed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= {:warmups 0 :repetitions 1} (get-in saved [:remaining :case/a]))) + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= [:completed :completed :completed :completed] + (mapv :outcome (attempts-for saved :case/b)))) + (t/is (not= (get-in first-prep [:diagnostics :browser-id]) + (get-in next-prep [:diagnostics :browser-id])))) + (finally + (set! (.-now js/Date) system-now)))))) + +(t/deftest ^:async preparation-and-rendering-share-the-case-budget + (doseq [context [:fresh :reuse]] + (let [system-now (.-now js/Date) + clock (atom (system-now)) + base (fake-browser {}) + work (^:async fn [bounds delegate page request] + (await (js/Promise.resolve nil)) + (swap! clock + 150) + (when (>= @clock (:deadline bounds)) + (throw (ex-info "browser work timed out" {:phase (:phase bounds)}))) + (delegate page request bounds)) + fake (assoc base :service + (assoc (:service base) + :prepare (fn [page request bounds] + (work bounds (:prepare (:service base)) page request)) + :run (fn [page request bounds] + (work bounds (:run (:service base)) page request))))] + (try + (set! (.-now js/Date) (fn [] @clock)) + (let [result (await (execute fake [(collected :case/a context)] + (assoc (options fake (output-dir)) :timeout-ms 250))) + saved (saved-record result)] + (t/is (= :failed (:outcome (first (:attempts saved))))) + (t/is (= :timed-out (get-in saved [:case-outcomes :case/a :outcome]))) + (t/is (= 1 (:exit-code result)))) + (finally + (set! (.-now js/Date) system-now)))))) + +(t/deftest ^:async browser-disconnection-leaves-pending-slots-and-relaunches-for-the-next-case + (let [base (fake-browser {}) + disconnected (atom nil) + new-page (:new-page (:service base)) + service (assoc (:service base) + :connected? (fn [owner] + (compare-and-set! disconnected nil (:browser owner)) + (not (identical? @disconnected (:browser owner)))) + :new-page (fn [owner host options] + (when (identical? @disconnected (:browser owner)) + (throw (js/Error. "browser disconnected"))) + (new-page owner host options))) + fake (assoc base :service service) + opts (assoc (options fake (output-dir)) :repetitions 2) + result (await (execute fake [(collected :case/a :fresh) (collected :case/b :fresh)] opts)) + run (saved-record result)] + (t/is (= [:completed] (mapv :outcome (attempts-for run :case/a)))) + (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/a]))) + (t/is (= [:completed :completed] (mapv :outcome (attempts-for run :case/b)))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async provenance-timeout-saves-the-unattempted-run + (let [bin (path/join @temporary-root "bin") + prior (.-PATH (.-env js/process)) + _ (fs/mkdirSync bin) + _ (fs/writeFileSync (path/join bin "git") + (str "#!" (.-execPath js/process) "\nsetInterval(() => {}, 1000);\n") + #js {:mode 8r755}) + fake (fake-browser {})] + (try + (set! (.-PATH (.-env js/process)) (str bin path/delimiter prior)) + (let [result (await (execute fake [(collected :case/a :fresh)] + (assoc (options fake (output-dir)) :startup-timeout-ms 50))) + run (saved-record result)] + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :runner-git (get-in run [:termination :failure :stage]))) + (t/is (= {:case/a {:warmups 0 :repetitions 1}} (:remaining run))) + (t/is (= [] (:attempts run))) + (t/is (= 1 (:exit-code result)))) + (finally + (set! (.-PATH (.-env js/process)) prior))))) + +(t/deftest ^:async unavailable-build-directory-saves-a-server-startup-failure + (let [root @temporary-root + selected (selected-build (build-dir) "unavailable-build") + home (runs/create-home {:root root :run-id "unavailable-build" :output-dir (output-dir)}) + _ (fs/renameSync (:dir selected) (str (:dir selected) ".unavailable")) + fake (fake-browser {}) + result (await (run/execute root home selected [(collected :case/a :fresh)] + (options fake (:dir home)) (.-signal (js/AbortController.)))) + run (saved-record result)] + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= :server-startup (get-in run [:termination :failure :stage]))) + (t/is (= {:case/a {:warmups 0 :repetitions 1}} (:remaining run))) + (t/is (= [] (:attempts run))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async failed-browser-relaunch-aborts-the-pending-cases + (let [fake (fake-browser {:new-page-failure #(= % 1) :launch-failure #(= % 2)}) + cases [(collected :case/a :fresh) (collected :case/b :reuse) (collected :case/c :fresh)] + result (await (execute fake cases (options fake (output-dir)))) + run (saved-record result)] + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= [] (:attempts run))) + (t/is (= [:browser-page :browser-launch] (mapv :stage (get-in run [:metadata :diagnostics :failures])))) + (t/is (= {:case/a {:warmups 0 :repetitions 1} + :case/b {:warmups 0 :repetitions 1} + :case/c {:warmups 0 :repetitions 1}} (:remaining run))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async interruption-exit-status-takes-precedence-over-finalization-errors + (doseq [failure [:cleanup :record] + [signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + out (output-dir) + base (fake-browser {:controller controller :interruption (fn [_ _] signal)}) + close (:close (:service base)) + fake (assoc base :service + (assoc (:service base) :close + (fn [owner] + (close owner) + (if (= failure :cleanup) + (throw (js/Error. "cleanup rejected")) + (let [record (path/join out "run.transit")] + (fs/renameSync record (str record ".saved")) + (fs/mkdirSync record)))))) + result (await (execute fake [(collected :case/a :fresh)] (options fake out) controller))] + (t/is (= code (:exit-code result))) + (t/is (= :interrupted (get-in result [:run :termination :reason]))) + (t/is (= signal (get-in result [:run :termination :signal]))) + (t/is (seq (:errors result)))))) + +(t/deftest ^:async log-failure-still-closes-pages-browsers-and-hosts + (let [out (output-dir) + base (fake-browser {}) + prepare (:prepare (:service base)) + service (assoc (:service base) + :prepare + (fn [page request bounds] + (let [response (prepare page request bounds)] + (fs/renameSync (path/join out "run.log") + (path/join out "run.log.saved")) + (fs/mkdirSync (path/join out "run.log")) + response))) + fake (assoc base :service service) + result (await (execute fake [(collected :case/a :fresh)] + (options fake out))) + state @(:state fake) + host (:host state)] + (t/is (some errors/fatal? (:errors result))) + (t/is (= 1 (:exit-code result))) + (t/is (empty? (attempts-for (:run result) :case/a))) + (t/is (= 1 (:close-pages state))) + (t/is (= 1 (:closes state))) + (t/is (false? (.-listening ^js (:server host)))))) + +(t/deftest ^:async storage-failure-survives-recovery-and-cleanup-error-takes-precedence + (doseq [cleanup-fails [false true]] + (let [out (output-dir) + file (path/join out "run.log") + saved (str file ".saved") + base (fake-browser {}) + prepare (:prepare (:service base)) + close (:close (:service base)) + service (assoc (:service base) + :prepare (fn [page request bounds] + (let [response (prepare page request bounds)] + (fs/renameSync file saved) + (fs/mkdirSync file) + response)) + :close (fn [owner] + (close owner) + (fs/rmdirSync file) + (fs/renameSync saved file) + (when cleanup-fails + (throw (js/Error. "cleanup rejected"))))) + fake (assoc base :service service) + result (await (execute fake [(collected :case/a :fresh)] (options fake out))) + run (saved-record result)] + (t/is (= 1 (:exit-code result))) + (t/is (some errors/fatal? (:errors result))) + (t/is (str/includes? (ex-message (first (:errors result))) + (if cleanup-fails "cleanup rejected" "cannot append benchmark log"))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (str/includes? (fs/readFileSync file "utf8") "Run incomplete"))))) + +(t/deftest ^:async cleanup-retains-page-browser-and-server-failures-in-the-saved-report + (let [base (fake-browser {:close-page-failure (constantly true) :close-failure (constantly true)}) + fake (assoc-in base [:service :prepare] + (fn [& _] (throw (errors/fatal-error "preparation transport failed" {} nil)))) + close server/close] + (await (mock/with-mocks* + {server/close (fn [host] + (.then (close host) (fn [_] (throw (js/Error. "server disposal failed")))))} + (let [response (await (execute fake [(collected :case/a :fresh)] (options fake (output-dir)))) + saved (saved-record response) + failure (get-in saved [:termination :failure])] + (t/is (= 1 (:exit-code response))) + (t/is (= ["page close failed" "browser shutdown failed" "server disposal failed"] + (mapv :message (:errors failure)))) + (t/is (str/includes? (:report response) "server disposal failed")) + (t/is (str/includes? (:report response) "browser shutdown failed"))))))) + +(t/deftest ^:async cleanup-error-keeps-completed-attempt-evidence-and-exits-nonzero + (let [base (fake-browser {}) + fake (assoc base :service (assoc (:service base) :close + (fn [_] (throw (js/Error. "cleanup rejected"))))) + result (await (execute fake [(collected :case/a :fresh)] (options fake (output-dir)))) + run (saved-record result)] + (t/is (= [:completed] (mapv :outcome (:attempts run)))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= :cleanup (get-in run [:termination :failure :stage]))) + (t/is (= "cleanup rejected" (ex-message (first (:errors result))))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async interruption-during-shutdown-is-persisted + (doseq [[signal code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [controller (js/AbortController.) + base (fake-browser {}) + close (:close (:service base)) + service (assoc (:service base) + :close + (fn [owner] + (when-not (.-aborted (.-signal controller)) + (.abort controller signal)) + (close owner))) + fake (assoc base :service service) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)) + controller)) + run (:run result)] + (t/is (= :interrupted (get-in run [:termination :reason]))) + (t/is (= signal (get-in run [:termination :signal]))) + (t/is (= code (:exit-code result))) + (t/is (= run (saved-record result))) + (t/is (= 1 (:closes @(:state fake))))))) + +(t/deftest ^:async screenshots-record-files-and-failures-as-diagnostics + (doseq [context [:fresh :reuse]] + (let [fake (fake-browser {}) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a context)] opts)) + run (:run result) + shot (get-in run [:metadata :diagnostics :screenshots "case/a"])] + (t/is (= :ok (:status shot))) + (t/is (= "case-a.png" (:file shot))) + (t/is (fs/existsSync (path/join (get-in result [:home :dir]) "case-a.png"))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (str/includes? (:report result) "Screenshots")) + (t/is (str/includes? (:report result) "case-a.png")) + (t/is (zero? (:exit-code result))))) + (let [fake (fake-browser {:screenshot-failure (constantly true)}) + opts (assoc (options fake (output-dir)) :screenshots true) + result (await (execute fake [(collected :case/a :fresh)] opts)) + run (:run result) + shot (get-in run [:metadata :diagnostics :screenshots "case/a"])] + (t/is (= :failed (:status shot))) + (t/is (= "case-a.png" (:file shot))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (str/includes? (:report result) "\"case/a\": warning")) + (t/is (= 1 (count (re-seq #"screenshot failed" (:report result))))) + (t/is (str/includes? (:report result) "case-a.png")) + (t/is (zero? (:exit-code result))))) + +(t/deftest ^:async metadata-records-build-and-controller-provenance-and-budgets + (let [fake (fake-browser {}) + opts (assoc (options fake (output-dir)) + :timeout-ms 1234 :build-timeout-ms 5678 :startup-timeout-ms 9012) + result (await (execute fake [(collected :case/a :fresh)] opts)) + run (:run result) + diag (get-in run [:metadata :diagnostics])] + (t/is (= {:case-ms 1234} (get-in run [:plan :timeout-policy]))) + (t/is (= {:sha "build-revision" :dirty false} (get-in run [:metadata :git]))) + (t/is (= {:mode :release :default-features? true :features [] :cljs-profile-marks false} + (get-in run [:metadata :build :functional]))) + (t/is (= "test-build" (get-in run [:metadata :build :provenance :build-id]))) + (t/is (true? (get-in run [:metadata :scored?]))) + (t/is (= "FakeBrowser/1" (get-in run [:metadata :environment :browser]))) + (t/is (= 1234 (:timeout-ms diag))) + (t/is (= 5678 (:build-timeout-ms diag))) + (t/is (= 9012 (:startup-timeout-ms diag))) + (t/is (= 42 (:seed diag))) + (t/is (= 0 (:warmups diag))) + (t/is (= 1 (:repetitions diag))) + (t/is (false? (:screenshots? diag))) + (t/is (map? (:runner-git diag))) + (t/is (= :unverified (:runtime-helper-identity diag))))) + +(t/deftest ^:async run-metadata-copies-only-saved-build-environment-evidence + (doseq [environment [::missing {} {"CFLAGS" "" "NODE_ENV" "recorded-build-value"}]] + (let [fake (fake-browser {}) + root (artifacts/frontend-root) + opts (options fake (output-dir)) + build (cond-> (selected-build (build-dir) "test-build") + (not= ::missing environment) (assoc-in [:manifest :environment] environment)) + home (runs/create-home {:root root :run-id "test-run" :output-dir (:output-dir opts)}) + controller (js/AbortController.) + executed (await (run/execute root home build [(collected :case/a :fresh)] + opts (.-signal controller))) + saved (saved-record executed) + facts (get-in saved [:metadata :build])] + (t/is (zero? (:exit-code executed))) + (t/is (= saved (:run executed))) + (if (= ::missing environment) + (t/is (not (contains? facts :environment))) + (t/is (= environment (:environment facts))))))) + +(t/deftest ^:async remove-deletes-the-used-build-after-the-record-is-saved + (let [root @temporary-root + id "removable-build" + dir (path/join (artifacts/managed-root root) "builds" id) + _ (fs/mkdirSync dir #js {:recursive true}) + _ (fs/writeFileSync (path/join dir "index.html") "") + fake (fake-browser {}) + out (path/join @temporary-root "remove-run") + home (runs/create-home {:root root :run-id "remove-run" + :output-dir out :remove true}) + result (await (run/execute root home (selected-build dir id) + [(collected :case/a :fresh)] + (assoc (options fake out) :remove true) + (.-signal (js/AbortController.))))] + (t/is (fs/existsSync (get-in result [:home :record-path]))) + (t/is (fs/existsSync dir)) + (t/is (not (fs/existsSync (path/join dir "index.html")))) + (t/is (= :completed (get-in result [:run :termination :reason]))) + (t/is (zero? (:exit-code result))))) + + +(t/deftest ^:async unresolved-failure-retains-build-and-earlier-results-on-interruption + (doseq [signal [nil "SIGINT" "SIGTERM"]] + (let [root @temporary-root + id (str "unresolved-" (or signal "failure")) + dir (path/join (artifacts/managed-root root) "builds" id) + index (path/join dir "index.html") + controller (js/AbortController.) + failure (errors/fatal-error "Command group may still be running" + {:type :process-unresolved :unresolved-process true + :pid 4242 :pgid 4242 :signals ["SIGTERM" "SIGKILL"]} nil) + base (fake-browser {}) + prepare (get-in base [:service :prepare]) + fake (assoc-in base [:service :prepare] + (fn [page request bounds] + (if (= :case/b (get-in (transit/decode-str request) [:case :id])) + (do + (when signal (.abort controller signal)) + (throw failure)) + (prepare page request bounds)))) + out (path/join root id) + home (runs/create-home {:root root :run-id id :output-dir out :remove true})] + (fs/mkdirSync dir #js {:recursive true}) + (fs/writeFileSync index "retained renderer") + (let [response (await (run/execute root home (selected-build dir id) + [(collected :case/a :reuse) (collected :case/b :reuse) (collected :case/c :reuse)] + (assoc (options fake out) :remove true) (.-signal controller))) + saved (saved-record response)] + (t/is (= (case signal "SIGINT" 130 "SIGTERM" 143 1) (:exit-code response))) + (t/is (some process/unresolved? (:errors response))) + (t/is (= [:completed] (mapv :outcome (attempts-for saved :case/a)))) + (t/is (= 1 (get-in saved [:remaining :case/c :repetitions]))) + (t/is (str/includes? (:report response) "pgid 4242")) + (t/is (str/includes? (:report response) "SIGKILL")) + (t/is (= "retained renderer" (when (fs/existsSync index) (fs/readFileSync index "utf8")))) + (t/is (false? (.-listening ^js (get-in @(:state fake) [:host :server])))))))) + +(t/deftest ^:async failed-final-record-write-retains-the-selected-build + (let [root @temporary-root + id "retained-after-write-failure" + dir (path/join (artifacts/managed-root root) "builds" id) + _ (fs/mkdirSync dir #js {:recursive true}) + _ (fs/writeFileSync (path/join dir "index.html") "") + out (output-dir) + record (path/join out "run.transit") + prior (path/join out "run.before-finish.transit") + base (fake-browser {}) + close (:close (:service base)) + fake (assoc base :service + (assoc (:service base) :close + (fn [owner] + (close owner) + ;; Make the final atomic rename fail after all + ;; attempts have reached the saved record. + (fs/renameSync record prior) + (fs/mkdirSync record)))) + home (runs/create-home {:root root :run-id "failed-write" + :output-dir out :remove true}) + response (await (run/execute root home (selected-build dir id) + [(collected :case/a :fresh)] + (assoc (options fake out) :remove true) + (.-signal (js/AbortController.)))) + saved (result/decode (fs/readFileSync prior "utf8"))] + (t/is (= :running (get-in saved [:termination :reason]))) + (t/is (= 1 (count (:attempts saved)))) + (t/is (every? zero? (vals (get-in saved [:remaining :case/a])))) + (t/is (seq (:errors response))) + (t/is (= 1 (:exit-code response))) + (t/is (= "" + (fs/readFileSync (path/join dir "index.html") "utf8"))))) + +(defn- prepare-comparison-build + [{:keys [selection on-created]}] + (let [dir (build-dir) + build (-> (selected-build dir (str (random-uuid))) + (assoc :manifest-path (path/join dir "build.transit")) + (assoc-in [:manifest :functional :default-features?] false) + (assoc-in [:manifest :functional :features] (:features selection)) + (assoc-in [:manifest :provenance :feature-selection] selection))] + (on-created build) + (js/Promise.resolve build))) + +(t/deftest ^:async feature-comparison-completes-with-managed-output + (let [root @temporary-root + cases [(collected :case/a :fresh)] + fake (fake-browser {}) + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir nil + :options (dissoc (options nil nil) :browser)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal] + (run/execute root home selected cases + (assoc opts :browser (:service fake)) signal))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))] + (t/is (= 0 (:exit-code response))) + (t/is (= [:completed :completed] (mapv :status (:configurations saved)))) + (doseq [configuration (:configurations saved)] + (let [run (result/decode (fs/readFileSync (get-in configuration [:run :record-path]) "utf8"))] + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= 1 (count (:attempts run)))))))) + +(t/deftest ^:async failed-feature-configuration-prints-its-saved-cause-and-log + (let [root @temporary-root + cases [(collected :case/a :fresh)] + fake (fake-browser {:close-page-failure #(= % 1)}) + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir nil + :options (assoc (dissoc (options nil nil) :browser) :filter "case/")} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal] + (run/execute root home selected cases + (assoc opts :browser (:service fake)) signal))}})) + home (:home response) + saved (result/decode (fs/readFileSync (:record-path home) "utf8")) + config (first (:configurations saved)) + request (request/apply-defaults {:command "ab" :console {:color :never} + :options {:features ["a"] :filter "case/"}})] + (t/is (= 1 (:exit-code response))) + (t/is (empty? (:errors response))) + (t/is (= [:failed :completed] (mapv :status (:configurations saved)))) + (t/is (= :browser-close (get-in config [:failure :stage]))) + (t/is (every? #(= :completed (:status %)) (:edges saved))) + (await + (mock/with-mocks* + {feature-comparison/prepare (fn [_ _ _] (js/Promise.resolve prepared)) + feature-comparison/execute (fn [_ _ _ _] (js/Promise.resolve response))} + (t/is (= 1 (await (deferred/execute (transit/encode-str request))))) + (let [log (fs/readFileSync (:log-path home) "utf8") + position (str/last-index-of log (str "Comparison " (:comparison-id home) "\n")) + report (subs log (or position 0))] + (t/is (some? position)) + (t/is (str/includes? report "Failures\n Configuration \"0\":")) + (t/is (= 1 (count (re-seq #"page close failed" report)))) + (t/is (str/includes? report (get-in config [:run :record-path]))) + (t/is (str/includes? report (get-in config [:run :log-path]))) + (t/is (str/includes? report (:record-path home)))))))) + +(t/deftest ^:async gpu-rejection-stops-later-matrix-configurations + (doseq [[hardware-first close-fails] [[false false] [true false] [true true]]] + (let [root @temporary-root + out (output-dir) + cases [(collected :case/a :fresh)] + software (assoc hardware-graphics :renderer "SwiftShader" :software true) + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir out + :options (assoc (dissoc (options nil out) :browser) + :require-gpu true :repetitions 2)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal] + (let [fake (fake-browser + {:close-failure (when close-fails (constantly true)) + :effective-graphics + (fn [_ page-id] + (if (and hardware-first (= page-id 1)) + hardware-graphics + software))})] + (run/execute root home selected cases + (assoc opts :browser (:service fake)) signal)))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8")) + run (result/decode (fs/readFileSync (get-in saved [:configurations 0 :run :record-path]) "utf8")) + failure (some #(when (= :gpu (:stage %)) %) + (get-in run [:metadata :diagnostics :failures]))] + (t/is (= 1 (:exit-code response))) + (t/is (= (if hardware-first 1 0) (count (:attempts run)))) + (t/is (= software (get-in failure [:partial :graphics]))) + (t/is (= (if close-fails :cleanup :gpu) (get-in run [:termination :failure :stage]))) + (t/is (= :failed (get-in saved [:configurations 0 :status]))) + (t/is (= :gpu (get-in saved [:configurations 0 :failure :stage]))) + (t/is (= :unattempted (get-in saved [:configurations 1 :status])))))) + +(t/deftest ^:async comparison-stops-after-run-record-or-log-storage-fails + (doseq [filename ["run.transit" "run.log"]] + (let [executed (atom nil) + root @temporary-root + out (output-dir) + cases [(collected :case/a :fresh)] + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir out + :options (dissoc (options nil out) :browser)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (^:async fn [root home selected cases opts signal] + (let [base (fake-browser {}) + close (:close (:service base)) + file (path/join (:dir home) filename) + service (assoc (:service base) :close + (fn [owner] + (close owner) + (fs/renameSync file (str file ".saved")) + (fs/mkdirSync file))) + outcome (await (run/execute root home selected cases + (assoc opts :browser service) signal))] + (reset! executed outcome) + outcome))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8")) + evidence (path/join out "configs" "0" + (if (= filename "run.transit") "run.transit.saved" "run.transit"))] + (t/is (= 1 (count (:attempts (result/decode (fs/readFileSync evidence "utf8")))))) + (t/is (= 1 (:exit-code response))) + (t/is (re-find #"Case case/a" (or (:report @executed) ""))) + (t/is (= :failed (get-in saved [:configurations 0 :status]))) + (t/is (= :unattempted (get-in saved [:configurations 1 :status])))))) diff --git a/frontend/test/performance_tests/render_wasm/runner.cljs b/frontend/test/performance_tests/render_wasm/runner.cljs new file mode 100644 index 0000000000..d8ecaba150 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/runner.cljs @@ -0,0 +1,229 @@ +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.runner + (:require + [app.common.logging :as l] + [cljs.test :as t] + [clojure.string :as str] + [clojure.tools.cli :refer [parse-opts]] + [goog.object :as gobj] + [performance-tests.render-wasm.artifacts-test] + [performance-tests.render-wasm.boolean-test] + [performance-tests.render-wasm.browser-test] + [performance-tests.render-wasm.build-test] + [performance-tests.render-wasm.builder-test] + [performance-tests.render-wasm.cli-test] + [performance-tests.render-wasm.commands-test] + [performance-tests.render-wasm.compare-test] + [performance-tests.render-wasm.containers-test] + [performance-tests.render-wasm.contracts-test] + [performance-tests.render-wasm.errors-test] + [performance-tests.render-wasm.execution-test] + [performance-tests.render-wasm.feature-comparison-test] + [performance-tests.render-wasm.fingerprint-test] + [performance-tests.render-wasm.geometry-scenes-test] + [performance-tests.render-wasm.lifecycle-test] + [performance-tests.render-wasm.log-test] + [performance-tests.render-wasm.merge-test] + [performance-tests.render-wasm.outcome-test] + [performance-tests.render-wasm.process-test] + [performance-tests.render-wasm.protocol-test] + [performance-tests.render-wasm.report-test] + [performance-tests.render-wasm.request-test] + [performance-tests.render-wasm.run-test] + [performance-tests.render-wasm.runs-test] + [performance-tests.render-wasm.scene-snapshot-test] + [performance-tests.render-wasm.server-test])) + +(enable-console-print!) + +(defmethod t/report [:cljs.test/default :begin-test-var] [m] + (let [v (:var m)] + (println (str " ▸ " (:ns (meta v)) "/" (:name (meta v)))))) + +(defmethod t/report [:cljs.test/default :end-run-tests] [m] + (if (cljs.test/successful? m) + (.exit js/process 0) + (.exit js/process 1))) + +(def test-namespaces + ['performance-tests.render-wasm.artifacts-test + 'performance-tests.render-wasm.boolean-test + 'performance-tests.render-wasm.browser-test + 'performance-tests.render-wasm.build-test + 'performance-tests.render-wasm.builder-test + 'performance-tests.render-wasm.cli-test + 'performance-tests.render-wasm.commands-test + 'performance-tests.render-wasm.compare-test + 'performance-tests.render-wasm.containers-test + 'performance-tests.render-wasm.contracts-test + 'performance-tests.render-wasm.errors-test + 'performance-tests.render-wasm.execution-test + 'performance-tests.render-wasm.feature-comparison-test + 'performance-tests.render-wasm.fingerprint-test + 'performance-tests.render-wasm.geometry-scenes-test + 'performance-tests.render-wasm.lifecycle-test + 'performance-tests.render-wasm.log-test + 'performance-tests.render-wasm.merge-test + 'performance-tests.render-wasm.outcome-test + 'performance-tests.render-wasm.process-test + 'performance-tests.render-wasm.protocol-test + 'performance-tests.render-wasm.report-test + 'performance-tests.render-wasm.request-test + 'performance-tests.render-wasm.run-test + 'performance-tests.render-wasm.runs-test + 'performance-tests.render-wasm.scene-snapshot-test + 'performance-tests.render-wasm.server-test]) + +(assert (every? find-ns-obj test-namespaces) + "test-namespaces contains a namespace that isn't required in runner.cljs") + +;; This runner mirrors frontend-tests.runner for the bench-only +;; :performance-tests target. Both runners need forwarded CLI args, focused +;; namespace/var execution, fixture preservation, and app log-level +;; setup. The logic stays local to avoid cross-target test classpath +;; coupling. +(def ^:private log-levels + #{:trace :debug :info :warn :error}) + +(def cli-options + [["-f" "--focus FOCUS" "Run one test namespace or one test var, e.g. performance-tests.render-wasm.contracts-test"] + ["-l" "--log-level LEVEL" "Set app logger level: trace|debug|info|warn|error" + :parse-fn keyword + :validate [log-levels "must be one of trace, debug, info, warn, error"]] + ["-h" "--help"]]) + +(defn- argv + [] + (let [args (->> (.-argv js/process) + (array-seq) + (drop 2))] + ;; `pnpm run test:renderer-benchmarks -- --focus ...` forwards the separator + ;; to the node process, so drop one leading `--` before handing args to + ;; tools.cli. + (cond-> args + (= "--" (first args)) rest))) + +(defn- usage + [summary] + (str "Usage: pnpm run test:renderer-benchmarks -- [options]\n\n" + "Options:\n" + summary "\n\n" + "Focus examples:\n" + " pnpm run test:renderer-benchmarks -- --focus performance-tests.render-wasm.contracts-test\n" + " pnpm run test:renderer-benchmarks -- --focus performance-tests.render-wasm.contracts-test/some-test-var\n\n" + "Log level example (quiets app logging during the run):\n" + " pnpm run test:renderer-benchmarks -- --focus performance-tests.render-wasm.contracts-test --log-level warn")) + +(defn- fail! + [message] + (js/console.error message) + (.exit js/process 1)) + +(defn- parse-focus + [focus] + (let [[ns-name test-name & extra] (str/split focus #"/")] + (cond + (or (str/blank? ns-name) (seq extra)) + (fail! (str "Invalid --focus value: " focus)) + + (some? test-name) + {:ns (symbol ns-name) :test test-name} + + :else + {:ns (symbol ns-name)}))) + +(defn- fixture-value + [ns-obj fixture-name] + (let [value (gobj/get ns-obj (munge fixture-name))] + (when-not (undefined? value) + value))) + +(defn- ns-test-vars + [ns-sym] + (when-let [ns-obj (find-ns-obj ns-sym)] + (->> (js-keys ns-obj) + (keep (fn [key] + (some-> (gobj/get ns-obj key) + (.-cljs$lang$var)))) + (filter (comp :test meta)) + (sort-by (comp :line meta))))) + +(defn- ns-fixtures + [ns-sym vars] + (when-let [ns-obj (find-ns-obj ns-sym)] + (let [ns-key (or (some-> vars first meta :ns) ns-sym) + once-fixtures (fixture-value ns-obj "cljs-test-once-fixtures") + each-fixtures (fixture-value ns-obj "cljs-test-each-fixtures")] + {:once (when once-fixtures {ns-key once-fixtures}) + :each (when each-fixtures {ns-key each-fixtures})}))) + +(defn- selected-tests + [{:keys [ns test]}] + (when-not (some #{ns} test-namespaces) + (fail! (str "Unknown test namespace: " ns))) + (let [vars (vec (ns-test-vars ns))] + (when (empty? vars) + (fail! (str "No tests found in namespace: " ns))) + (if test + (let [test-sym (symbol test) + test-var (some #(when (= test-sym (:name (meta %))) %) vars)] + (if test-var + {:vars [test-var] + :fixtures (ns-fixtures ns [test-var])} + (fail! (str "Unknown test var: " ns "/" test)))) + {:vars vars + :fixtures (ns-fixtures ns vars)}))) + +(defn- merge-fixtures + [fixtures] + {:once (apply merge (keep :once fixtures)) + :each (apply merge (keep :each fixtures))}) + +(defn- run-test-vars! + [tests] + (let [vars (vec (mapcat :vars tests)) + fixtures (merge-fixtures (map :fixtures tests)) + env (assoc (t/empty-env) + :once-fixtures (:once fixtures) + :each-fixtures (:each fixtures)) + summary (volatile! {:test 0 :pass 0 :fail 0 :error 0 :type :summary})] + + (t/set-env! env) + + (t/run-block + (concat (t/test-vars-block vars) + [(fn [] + (vswap! summary + (partial merge-with +) + (:report-counters (t/get-current-env)))) + (fn [] + (t/report @summary) + (t/report (assoc @summary :type :end-run-tests)))])))) + +(defn- run-focused-test! + [focus] + (run-test-vars! [(selected-tests (parse-focus focus))])) + +(defn init + [] + (let [{:keys [options errors summary]} (parse-opts (argv) cli-options)] + (cond + (seq errors) + (fail! (str/join "\n" errors)) + + (:help options) + (do + (println (usage summary)) + (.exit js/process 0)) + + :else + (do + (l/setup! {:app (or (:log-level options) :warn)}) + + (if (:focus options) + (run-focused-test! (:focus options)) + (run-test-vars! (map #(selected-tests {:ns %}) test-namespaces))))))) diff --git a/frontend/test/performance_tests/render_wasm/runs_test.cljs b/frontend/test/performance_tests/render_wasm/runs_test.cljs new file mode 100644 index 0000000000..a0af15f0b0 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/runs_test.cljs @@ -0,0 +1,165 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.runs-test + (:require + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.node.storage.artifacts :as artifacts] + [benches.render-wasm.node.storage.runs :as runs] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root + (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-runs-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- failure + [f] + (try + (f) + nil + (catch :default error error))) + +(defn- sample-run + [run-id] + (result/create-run + {:run-id run-id + :started-at "2026-10-26T00:00:00.000Z" + :plan {:cases [{:id :rects/load}] + :warmups 0 + :repetitions 1 + :timeout-policy {:case-ms 30000}} + :metadata {:git {:sha "abc" :dirty false} + :build {:functional {:mode :release}} + :environment {:node "v24.0.0"} + :scored? true}})) + +(t/deftest managed-home-lives-under-the-frontend-root + (let [home (runs/create-home {:root @temporary-root :run-id "run-1"})] + (t/is (= (path/join (artifacts/managed-root @temporary-root) "runs" "run-1") + (:dir home))) + (t/is (.isDirectory (fs/statSync (:dir home)))) + (t/is (= "" (fs/readFileSync (:log-path home) "utf8"))) + (t/is (not (fs/existsSync (:record-path home)))))) + +(t/deftest explicit-home-must-not-exist-and-resolves-from-caller-cwd + (let [previous (.cwd js/process) + relative "explicit-run"] + (try + (.chdir js/process @temporary-root) + (let [home (runs/create-home {:root @temporary-root + :run-id "run-2" + :output-dir relative})] + (t/is (= (path/join @temporary-root relative) (:dir home))) + (t/is (.isDirectory (fs/statSync (:dir home)))) + (t/is (not (fs/existsSync (path/join (artifacts/managed-root @temporary-root) + "runs" "run-2"))))) + (finally + (.chdir js/process previous)))) + (let [directory (path/join @temporary-root "already-there")] + (fs/mkdirSync directory) + (fs/writeFileSync (path/join directory "sentinel") "keep") + (let [error (failure #(runs/create-home {:root @temporary-root + :run-id "run-3" + :output-dir directory}))] + (t/is (= ::artifacts/invalid-home (:type (ex-data error)))) + (t/is (= "output directory already exists" (ex-message error))) + (t/is (= "keep" (fs/readFileSync (path/join directory "sentinel") "utf8")))))) + +(t/deftest record-writes-are-atomic-and-invalid-records-leave-the-saved-file + (let [home (runs/create-home {:root @temporary-root :run-id "run-4"}) + run (sample-run "run-4")] + (runs/write-record! home run) + (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8")))) + (t/is (empty? (filter #(str/ends-with? % ".tmp") + (array-seq (fs/readdirSync (:dir home)))))) + (let [error (failure #(runs/write-record! home (assoc-in run [:metadata :scored?] "yes")))] + (t/is (= ::result/invalid-record (:type (ex-data error)))) + (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8"))))))) + +(t/deftest record-read-failures-retain-file-context-and-cause + (let [file (path/join @temporary-root "invalid.transit")] + (fs/writeFileSync file (transit/encode-str (dissoc (sample-run "invalid") :plan))) + (let [error (failure #(runs/get-record file))] + (t/is (= {:path file :operation :read-record :type ::result/invalid-record} + (select-keys (ex-data error) [:path :operation :type]))) + (t/is (= ::result/invalid-record (:type (ex-data (ex-cause error))))) + (t/is (= (ex-message error) (ex-message (ex-cause error)))))) + (let [error (failure #(runs/get-record @temporary-root))] + (t/is (= {:path @temporary-root :operation :read-record} + (select-keys (ex-data error) [:path :operation]))) + (t/is (= "EISDIR" (.-code (ex-cause error)))))) + +(t/deftest screenshot-paths-stay-in-the-run-home + (let [home (runs/create-home {:root @temporary-root :run-id "run-5"}) + shot (runs/screenshot-path home :rects/load)] + (t/is (= "rects-load.png" (:file shot))) + (t/is (= (path/join (:dir home) "rects-load.png") (:path shot))) + (t/is (= (path/dirname (:record-path home)) (path/dirname (:path shot)))))) + +(t/deftest malformed-ids-and-empty-paths-fail-at-the-home-boundary + (doseq [id ["../escape" "" "a/b" "a\\b"]] + (t/is (thrown-with-msg? js/Error #"invalid run id" + (runs/create-home {:root @temporary-root :run-id id})))) + (t/is (thrown-with-msg? js/Error #"nonempty path" + (artifacts/check-output-dir " ")))) + +(t/deftest home-creation-reports-an-invalid-parent + (let [parent (path/join @temporary-root "file") + dir (path/join parent "run")] + (fs/writeFileSync parent "occupied") + (let [error (failure #(runs/create-home {:root @temporary-root + :run-id "run-6" :output-dir dir}))] + (t/is (= ::artifacts/invalid-home (:type (ex-data error)))) + (t/is (= (path/resolve dir) (:path (ex-data error)))) + (t/is (= "ENOTDIR" (.-code (ex-cause error)))) + (t/is (= (ex-message (ex-cause error)) (:cause (ex-data error)))) + (t/is (str/includes? (ex-message error) "cannot inspect output directory")) + (t/is (str/includes? (ex-message error) "ENOTDIR"))))) + +(t/deftest home-creation-wraps-log-failures + (let [dir (path/join @temporary-root "occupied-log")] + (fs/mkdirSync dir) + (fs/writeFileSync (path/join dir "run.log") "") + (let [error (failure #(artifacts/create-home dir "run" "cannot create run home"))] + (t/is (= ::artifacts/invalid-home (:type (ex-data error)))) + (t/is (= "cannot create run home" (ex-message error))) + (t/is (= dir (:path (ex-data error)))) + (t/is (= "EEXIST" (.-code (ex-cause error)))) + (t/is (= (ex-message (ex-cause error)) (:cause (ex-data error))))))) + +(t/deftest explicit-home-creates-missing-parents + (let [out (path/join @temporary-root "missing" "parents" "run") + home (runs/create-home {:root @temporary-root :run-id "nested" + :output-dir out})] + (t/is (.isDirectory (fs/statSync out))) + (t/is (= "" (fs/readFileSync (:log-path home) "utf8"))) + (t/is (thrown-with-msg? js/Error #"already exists" + (runs/create-home {:root @temporary-root :run-id "again" + :output-dir out}))))) + +(t/deftest remove-rejects-output-inside-builds-before-creating-it + (let [builds (path/join (artifacts/managed-root @temporary-root) "builds") + alias (path/join @temporary-root "build-alias")] + (fs/mkdirSync builds #js {:recursive true}) + (fs/symlinkSync builds alias "dir") + (doseq [out [(path/join builds "direct") + (path/join alias "missing" "nested")]] + (t/is (thrown-with-msg? js/Error #"--remove.*inside builds" + (runs/create-home {:root @temporary-root :run-id "blocked" + :output-dir out :remove true}))) + (t/is (not (fs/existsSync out)))) + (let [out (path/join @temporary-root "builds-other" "allowed")] + (runs/create-home {:root @temporary-root :run-id "allowed" + :output-dir out :remove true}) + (t/is (fs/existsSync (path/join out "run.log")))))) diff --git a/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs b/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs new file mode 100644 index 0000000000..08f6070de3 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/scene_snapshot_test.cljs @@ -0,0 +1,316 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.scene-snapshot-test + (:require + [app.common.schema :as sm] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [benches.render-wasm.snapshot :as snapshot] + [cljs.test :as t :include-macros true])) + +(defn- sample-snapshot + "Simple tiny scene: root frame with one rectangle." + [] + (let [rect-id (uuid/custom 1 1) + rect (cts/setup-shape {:id rect-id :type :rect :name "Rect" + :x 32 :y 48 :width 200 :height 120}) + root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 1920 :height 1080 + :shapes [rect-id]})] + {:objects {uuid/zero root, rect-id rect} + :refs {:rect rect-id}})) + +(defn- nested-instance + "Root with two frames; frame A with two rectangles, frame B with one." + [] + (let [a-id (uuid/custom 2 1) + b-id (uuid/custom 2 2) + a1-id (uuid/custom 2 3) + a2-id (uuid/custom 2 4) + b1-id (uuid/custom 2 5) + frame-a (cts/setup-shape {:id a-id :type :frame :name "A" + :x 0 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [a1-id a2-id]}) + frame-b (cts/setup-shape {:id b-id :type :frame :name "B" + :x 200 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [b1-id]}) + rect-a1 (cts/setup-shape {:id a1-id :type :rect :name "A1" + :x 10 :y 10 :width 10 :height 10 + :parent-id a-id :frame-id a-id}) + rect-a2 (cts/setup-shape {:id a2-id :type :rect :name "A2" + :x 30 :y 10 :width 10 :height 10 + :parent-id a-id :frame-id a-id}) + rect-b1 (cts/setup-shape {:id b1-id :type :rect :name "B1" + :x 210 :y 10 :width 10 :height 10 + :parent-id b-id :frame-id b-id}) + root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 1920 :height 1080 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [a-id b-id]})] + {:objects {uuid/zero root + a-id frame-a + b-id frame-b + a1-id rect-a1 + a2-id rect-a2 + b1-id rect-b1} + :refs {:a a-id + :a1 a1-id + :b b-id}})) + +(defn- validation-error + "Returns the ex-data of the validation failure, or nil when it passes." + [instance] + (try + (snapshot/validate! instance) + nil + (catch :default cause + (ex-data cause)))) + +(t/deftest canonical-example-is-valid + (t/testing "validate! returns the same instance" + (let [instance (sample-snapshot)] + (t/is (identical? instance (snapshot/validate! instance))))) + + (t/testing "the envelope schema accepts the example" + (t/is (true? (sm/validate snapshot/schema:snapshot (sample-snapshot)))))) + +(t/deftest nested-hierarchy-is-valid + (let [instance (nested-instance)] + (t/is (identical? instance (snapshot/validate! instance))))) + +(t/deftest root-only-instance-is-valid + (let [root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 100 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :shapes []}) + instance {:objects {uuid/zero root} :refs {}}] + (t/is (identical? instance (snapshot/validate! instance))) + (t/is (= [uuid/zero] (mapv :id (snapshot/upload-order instance)))))) + +(t/deftest upload-order-starts-at-root + (let [instance (sample-snapshot) + ids (mapv :id (snapshot/upload-order instance))] + (t/is (= [uuid/zero (snapshot/ref-id instance :rect)] ids)))) + +(t/deftest upload-order-is-parent-before-child + (let [instance (nested-instance) + order (snapshot/upload-order instance) + names (mapv :name order) + position (into {} (map-indexed (fn [index shape] [(:id shape) index]) order))] + (t/is (= ["Root Frame" "A" "A1" "A2" "B" "B1"] names)) + (t/is (every? (fn [shape] + (or (= uuid/zero (:id shape)) + (< (position (:parent-id shape)) + (position (:id shape))))) + order)))) + +(t/deftest upload-order-follows-flex-reverse + (let [frame-id (uuid/custom 8 1) + a-id (uuid/custom 8 2) + b-id (uuid/custom 8 3) + frame (cts/setup-shape {:id frame-id :type :frame :name "Flex" + :x 0 :y 0 :width 200 :height 100 + :parent-id uuid/zero :frame-id uuid/zero + :layout :flex :layout-flex-dir :row-reverse + :shapes [a-id b-id]}) + shape-a (cts/setup-shape {:id a-id :type :rect :name "A" + :x 0 :y 0 :width 10 :height 10 + :parent-id frame-id :frame-id frame-id}) + shape-b (cts/setup-shape {:id b-id :type :rect :name "B" + :x 20 :y 0 :width 10 :height 10 + :parent-id frame-id :frame-id frame-id}) + root (cts/setup-shape {:id uuid/zero :type :frame :name "Root Frame" + :x 0 :y 0 :width 1920 :height 1080 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [frame-id]}) + instance {:objects {uuid/zero root + frame-id frame + a-id shape-a + b-id shape-b} + :refs {}}] + (t/is (identical? instance (snapshot/validate! instance))) + (t/is (= ["Root Frame" "Flex" "B" "A"] + (mapv :name (snapshot/upload-order instance)))))) + +(t/deftest ref-id-resolves-labels + (let [instance (sample-snapshot)] + (t/is (= (uuid/custom 1 1) (snapshot/ref-id instance :rect))) + (t/is (nil? (snapshot/ref-id instance :missing))))) + +(t/deftest missing-root-is-rejected + (let [instance (sample-snapshot) + data (validation-error (update instance :objects dissoc uuid/zero))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest non-frame-root-is-rejected + (let [instance (sample-snapshot) + data (validation-error (assoc-in instance [:objects uuid/zero :type] :rect))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest root-with-non-zero-parent-id-is-rejected + (let [instance (sample-snapshot) + data (validation-error + (assoc-in instance [:objects uuid/zero :parent-id] (uuid/custom 9 9)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest object-key-must-match-shape-id + (let [instance (sample-snapshot) + rect (get-in instance [:objects (snapshot/ref-id instance :rect)]) + data (validation-error (assoc-in instance [:objects (uuid/custom 7 7)] rect))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= (:id rect) (:shape-id data))))) + +(t/deftest missing-child-is-rejected + (let [instance (sample-snapshot) + bogus (uuid/custom 3 3) + data (validation-error + (assoc-in instance [:objects uuid/zero :shapes] [bogus]))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= bogus (:child data))))) + +(t/deftest child-not-listed-by-parent-is-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects uuid/zero :shapes] []))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "object is not reachable from the root" (:hint data))) + (t/is (= rect-id (:id data))))) + +(t/deftest mismatched-parent-id-is-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error + (assoc-in instance [:objects rect-id :parent-id] (uuid/custom 9 7)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "shape :shapes lists a child with a different :parent-id" (:hint data))) + (t/is (= rect-id (:child data))))) + +(t/deftest duplicate-child-ids-are-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error + (assoc-in instance [:objects uuid/zero :shapes] [rect-id rect-id]))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "an object is reachable more than once" (:hint data))) + (t/is (= rect-id (:id data))))) + +(t/deftest shape-listing-itself-is-rejected + (let [instance (sample-snapshot) + data (validation-error + (assoc-in instance [:objects uuid/zero :shapes] [uuid/zero]))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "root must not be a descendant of itself" (:hint data))) + (t/is (= uuid/zero (:id data))))) + +(t/deftest detached-cycle-is-rejected + (let [c1-id (uuid/custom 5 1) + c2-id (uuid/custom 5 2) + shape1 (cts/setup-shape {:id c1-id :type :rect :name "C1" + :x 0 :y 0 :width 10 :height 10 + :parent-id c2-id :frame-id c1-id + :shapes [c2-id]}) + shape2 (cts/setup-shape {:id c2-id :type :rect :name "C2" + :x 0 :y 0 :width 10 :height 10 + :parent-id c1-id :frame-id c2-id + :shapes [c1-id]}) + data (validation-error + (assoc (sample-snapshot) :objects + (assoc (get (sample-snapshot) :objects) + c1-id shape1 + c2-id shape2)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "object is not reachable from the root" (:hint data))) + (t/is (contains? #{c1-id c2-id} (:id data))))) + +(t/deftest reachable-cycle-is-rejected + (let [a-id (uuid/custom 4 1) + b-id (uuid/custom 4 2) + shape-a (cts/setup-shape {:id a-id :type :rect :name "A" + :x 0 :y 0 :width 10 :height 10 + :parent-id uuid/zero :frame-id uuid/zero + :shapes [b-id]}) + shape-b (cts/setup-shape {:id b-id :type :rect :name "B" + :x 0 :y 0 :width 10 :height 10 + :parent-id a-id :frame-id uuid/zero + :shapes [a-id]}) + instance (-> (sample-snapshot) + (assoc-in [:objects uuid/zero :shapes] [a-id]) + (assoc-in [:objects a-id] shape-a) + (assoc-in [:objects b-id] shape-b)) + data (validation-error instance)] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "shape :shapes lists a child with a different :parent-id" (:hint data))) + (t/is (= a-id (:child data))))) + +(t/deftest non-vector-children-are-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects rect-id :shapes] 5))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= "shape :shapes must be a vector" (:hint data))) + (t/is (= rect-id (:id data))))) + +(t/deftest frame-id-mismatch-is-rejected + (let [instance (nested-instance) + a1-id (snapshot/ref-id instance :a1) + data (validation-error + (assoc-in instance [:objects a1-id :frame-id] uuid/zero))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= :invalid-frame (:code data))) + (t/is (= a1-id (:shape-id data))))) + +(t/deftest missing-ref-target-is-rejected + (let [instance (sample-snapshot) + data (validation-error + (assoc-in instance [:refs :rect] (uuid/custom 6 6)))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= :rect (:label data))))) + +(t/deftest non-uuid-ref-value-is-rejected + (let [data (validation-error (assoc-in (sample-snapshot) [:refs :rect] "not-a-uuid"))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + +(t/deftest missing-refs-key-is-rejected + (let [data (validation-error (dissoc (sample-snapshot) :refs))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + +(t/deftest non-canonical-shape-is-rejected + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects rect-id] {:id rect-id}))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + +(t/deftest malformed-envelopes-are-rejected + (t/testing "empty objects map" + (let [data (validation-error {:objects {} :refs {}})] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (= uuid/zero (:id data))))) + + (t/testing "nil instance" + (let [data (validation-error nil)] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + + (t/testing "nil shape value" + (let [instance (sample-snapshot) + rect-id (snapshot/ref-id instance :rect) + data (validation-error (assoc-in instance [:objects rect-id] nil))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data))))) + + (t/testing "nil ref value" + (let [data (validation-error (assoc-in (sample-snapshot) [:refs :rect] nil))] + (t/is (= ::snapshot/invalid-snapshot (:type data))) + (t/is (some? (::sm/explain data)))))) diff --git a/frontend/test/performance_tests/render_wasm/server_test.cljs b/frontend/test/performance_tests/render_wasm/server_test.cljs new file mode 100644 index 0000000000..8dcb7c8d79 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/server_test.cljs @@ -0,0 +1,233 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.server-test + (:require + ["fs" :as fs] + ["http" :as http] + ["net" :as net] + ["os" :as os] + ["path" :as path] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.runner.server :as server] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) +(def ^:private wasm-bytes [0 97 115 109 1 0 0 0]) + +(defn- write-file + [file content] + (fs/mkdirSync (path/dirname file) #js {:recursive true}) + (fs/writeFileSync file content)) + +(t/use-fixtures :each + {:before (fn [] + (let [root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-http-")) + dir (path/join root "selected")] + (reset! temporary-root root) + (write-file (path/join dir "index.html") "selected") + (write-file (path/join dir "browser" "browser.js") "export const selected = true;") + (write-file (path/join dir "cases.cjs") "module.exports = {};") + (write-file (path/join dir "renderer" "render-wasm.wasm") + (js/Buffer.from (to-array wasm-bytes))) + (write-file (path/join root "selected-outside" "secret.txt") "private sibling") + (write-file (path/join root "outside.txt") "private file"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- build + [] + {:id "selected-http-build" :dir (path/join @temporary-root "selected")}) + +(defn- request + [host request-path method] + (js/Promise. + (fn [resolve reject] + (let [address (js/URL. (:base-url host)) + req (http/request + ;; Supplying a raw path avoids URL normalization hiding traversal. + #js {:hostname (.-hostname address) + :port (.-port address) + :path request-path + :method method + :agent false} + (fn [response] + (let [chunks (atom [])] + (.on ^js response "data" #(swap! chunks conj %)) + (.once ^js response "error" reject) + (.once ^js response "end" + #(resolve {:status (.-statusCode response) + :headers (js->clj (.-headers response)) + :body (js/Buffer.concat (to-array @chunks))})))))] + (.once ^js req "error" reject) + (.setTimeout ^js req 3000 #(.destroy ^js req (js/Error. "test request timed out"))) + (.end ^js req))))) + +(defn- ^:async request-failure + [host] + (try + (await (request host "/" "GET")) + nil + (catch :default error error))) + +(t/deftest ^:async serves-only-the-selected-build-with-head-and-artifact-mime-types + (let [selected (build) + host (await (server/open selected))] + (try + (t/is (= selected (:build host))) + (t/is (re-matches #"http://127\.0\.0\.1:[0-9]+" (:base-url host))) + (let [index (await (request host "/?unused=query" "GET")) + browser (await (request host "/browser/browser.js" "GET")) + cjs (await (request host "/cases.cjs" "GET")) + wasm (await (request host "/renderer/render-wasm.wasm" "GET")) + head (await (request host "/renderer/render-wasm.wasm" "HEAD"))] + (t/is (= 200 (:status index))) + (t/is (= "selected" (.toString (:body index) "utf8"))) + (t/is (= "text/html; charset=utf-8" (get-in index [:headers "content-type"]))) + (t/is (= "export const selected = true;" (.toString (:body browser) "utf8"))) + (t/is (= "application/javascript" (get-in browser [:headers "content-type"]))) + (t/is (= "application/javascript" (get-in cjs [:headers "content-type"]))) + (t/is (= "application/wasm" (get-in wasm [:headers "content-type"]))) + (t/is (zero? (js/Buffer.compare (:body wasm) (js/Buffer.from (to-array wasm-bytes))))) + (t/is (= 200 (:status head))) + (t/is (= "8" (get-in head [:headers "content-length"]))) + (t/is (zero? (.-length (:body head))))) + (t/is (= 404 (:status (await (request host "/selected-outside/secret.txt" "GET"))))) + (t/is (= 405 (:status (await (request host "/" "POST"))))) + (finally + (await (server/close host)))))) + +(t/deftest ^:async rejects-raw-traversal-and-outside-file-and-directory-symlinks + (let [selected (build) + dir (:dir selected)] + (fs/symlinkSync (path/join @temporary-root "outside.txt") (path/join dir "escape.txt")) + (fs/symlinkSync (path/join @temporary-root "selected-outside") (path/join dir "linked") "dir") + (let [host (await (server/open selected))] + (try + (doseq [request-path ["/../selected-outside/secret.txt" + "/%2e%2e/selected-outside/secret.txt" + "/%2e%2e%2fselected-outside%2fsecret.txt" + "/%5c..%5cselected-outside%5csecret.txt" + "/escape.txt" "/linked/secret.txt"]] + (let [response (await (request host request-path "GET"))] + (t/is (= 403 (:status response)) request-path) + (t/is (= "forbidden" (.toString (:body response) "utf8")) request-path))) + (t/is (= 400 (:status (await (request host "/%zz" "GET"))))) + (t/is (= 403 (:status (await (request host "/%00" "GET"))))) + (t/is (= 404 (:status (await (request host "/missing-file" "GET"))))) + (t/is (= 404 (:status (await (request host "/browser" "GET"))))) + (finally + (await (server/close host))))))) + +(defn- connect + [host] + (js/Promise. + (fn [resolve reject] + (let [address (js/URL. (:base-url host)) + socket (net/createConnection #js {:host "127.0.0.1" :port (js/Number (.-port address))})] + (.once ^js socket "error" reject) + (.once ^js socket "connect" #(resolve socket)))))) + +(defn- bounded + [promise] + (js/Promise. + (fn [resolve reject] + (let [timer (js/setTimeout #(reject (js/Error. "test shutdown timed out")) 3000)] + (.then promise + (fn [value] (js/clearTimeout timer) (resolve value)) + (fn [error] (js/clearTimeout timer) (reject error))))))) + +(t/deftest ^:async close-destroys-owned-connections-and-keeps-other-servers-live + (let [host (await (server/open (build)))] + (try + (let [other (await (server/open (build)))] + (try + (let [socket (await (connect host)) + closed (js/Promise. (fn [resolve _] (.once ^js socket "close" resolve)))] + (try + (await (bounded (js/Promise.all (to-array [(server/close host) (server/close host)])))) + (await (bounded closed)) + (t/is (some? (await (request-failure host)))) + (let [response (await (request other "/" "GET"))] + (t/is (= 200 (:status response))) + (t/is (= "selected" (.toString (:body response) "utf8")))) + (await (server/close host)) + (finally + (.destroy ^js socket)))) + (finally + (await (server/close other))))) + (finally + (await (server/close host)))))) + +(defn- ^:async startup-failure + [opening] + (try + (let [host (await opening)] + (await (server/close host)) + nil) + (catch :default error error))) + +(t/deftest ^:async startup-abort-retains-a-server-close-failure + (let [controller (js/AbortController.) + close server/close + close-error (js/Error. "server disposal failed")] + (.abort controller) + (await (mock/with-mocks* + {server/close (fn [host] + (.then (close host) (fn [_] (throw close-error))))} + (let [cause (await (bounded (startup-failure (server/open (build) {:signal (.-signal controller)})))) + entries (errors/get-errors cause)] + (t/is (= ::server/aborted (:type (ex-data cause)))) + (t/is (= ["renderer server startup aborted" "server disposal failed"] + (mapv #(errors/message-of (:error %)) entries)))))))) + +(t/deftest ^:async startup-rejects-already-aborted-and-in-flight-signals + (let [already (js/AbortController.)] + (.abort already) + (let [error (await (startup-failure (server/open (build) {:signal (.-signal already)})))] + (t/is (= ::server/aborted (:type (ex-data error)))))) + (let [controller (js/AbortController.) + opening (server/open (build) {:signal (.-signal controller)})] + (.abort controller) + (let [error (await (startup-failure opening))] + (t/is (= ::server/aborted (:type (ex-data error)))))) + (let [host (await (server/open (build) {:startup-timeout-ms 3000}))] + (try + (t/is (= 200 (:status (await (request host "/" "GET"))))) + (finally + (await (server/close host)))))) + + +(defn- ^:async with-listen + "Calls f while http.Server#listen runs `listen` and returns the promise f returns." + [listen f] + (let [proto (.-prototype http/Server)] + (set! (.-listen proto) listen) + (try + (await (f)) + (finally + (js-delete proto "listen"))))) + +(t/deftest ^:async startup-timeout-rejects-with-the-configured-limit + (let [never-listens (fn [& _] nil) + error (await (with-listen never-listens + #(startup-failure (server/open (build) {:startup-timeout-ms 50}))))] + (t/is (= ::server/startup-timeout (:type (ex-data error)))) + (t/is (= 50 (:timeout-ms (ex-data error)))))) + +(t/deftest ^:async startup-rejects-with-the-listen-failure + (let [thrown (js/Error. "listen threw") + emitted (js/Error. "server emitted error") + throws (fn [& _] (throw thrown)) + emits (fn [& _] (this-as ^js self (js/process.nextTick #(.emit self "error" emitted))))] + (doseq [[failure listen] [[thrown throws] [emitted emits]]] + (let [error (await (with-listen listen #(startup-failure (server/open (build)))))] + (t/is (identical? failure error) (ex-message failure)))))) + +(t/deftest ^:async startup-rejects-when-the-build-directory-is-missing + (let [missing (path/join @temporary-root "missing") + error (await (startup-failure (server/open {:id "missing-build" :dir missing})))] + (t/is (= "ENOENT" (.-code ^js error))))) diff --git a/frontend/test/performance_tests/render_wasm/test_helpers.cljs b/frontend/test/performance_tests/render_wasm/test_helpers.cljs new file mode 100644 index 0000000000..4609e8f079 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/test_helpers.cljs @@ -0,0 +1,49 @@ +(ns performance-tests.render-wasm.test-helpers + "Shared helpers for renderer benchmark scene tests. + + These helpers assert nothing about shapes or the renderer. They only + capture errors, read root children in order, and run raw scopes so each + test namespace tells a complete story without copying bodies." + (:require + [app.common.uuid :as uuid] + [benches.render-wasm.builder :as builder])) + +(defn failure-data + "Returns the ex-data of the thrown failure, or nil when `f` passes." + [f] + (try + (f) + nil + (catch :default cause + (ex-data cause)))) + +(defn child-ids + "Ids listed in the :shapes vector of `id`." + [snapshot id] + (get-in snapshot [:objects id :shapes])) + +(defn children-of + "Child shapes of `id` in order." + [snapshot id] + (mapv (:objects snapshot) (child-ids snapshot id))) + +(defn root-shapes + "Root children of a scene snapshot in creation order." + [snapshot] + (children-of snapshot uuid/zero)) + +(defn run-scope + "Runs `f` inside a raw scope and returns the scope state, so tests can + inspect partial builds without calling finish!." + [params f] + (let [state (builder/start params)] + (binding [builder/*state* state + builder/*defaults* {}] + (f)) + state)) + +(defn selrect + "Selrect of `shape` as an [x y width height] tuple." + [shape] + [(:x (:selrect shape)) (:y (:selrect shape)) + (:width (:selrect shape)) (:height (:selrect shape))]) diff --git a/render-wasm/scripts/cargo-build.cjs b/render-wasm/scripts/cargo-build.cjs index 035703a03c..248c9ae2cf 100644 --- a/render-wasm/scripts/cargo-build.cjs +++ b/render-wasm/scripts/cargo-build.cjs @@ -15,6 +15,38 @@ async function main() { // A failed invocation must not leave facts from an earlier build. fs.rmSync(infoFile, { force: true }); + // This helper inherits the same shell setup as Cargo. Keep only build inputs, + // excluding credentials, invocation identities and artifact destinations. + const environmentKeys = new Set([ + "RUSTFLAGS", + "RUSTC_WRAPPER", + "CARGO_TARGET_DIR", + "CARGO_HOME", + "CC", + "CFLAGS", + "CXXFLAGS", + "LDFLAGS", + "PENPOT_WASM_FUNCTION_NAMES", + "CARGO_ENCODED_RUSTFLAGS", + "CARGO_BUILD_TARGET", + "BUILD_MODE", + "EMCC_CFLAGS", + "EM_INITIAL_HEAP", + "EM_MEMORY_GROWTH_GEOMETRIC_STEP", + "EM_MALLOC", + "EM_CACHE", + "NODE_ENV", + "CARGO_NET_OFFLINE", + ]); + const environment = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => + environmentKeys.has(key) || + key.startsWith("CARGO_PROFILE_") || + key.startsWith("EMSDK"), + ), + ); + const artifacts = []; const buildScripts = new Map(); let succeeded = false; @@ -99,6 +131,7 @@ async function main() { package_id: artifact.package_id, features: artifact.features, profile: artifact.profile, + environment, out_dir: outDir, artifact_dir: path.dirname(path.resolve(jsFiles[0])), shared_file: sharedFile, diff --git a/render-wasm/scripts/cargo-build.test.cjs b/render-wasm/scripts/cargo-build.test.cjs index 01a080aa42..f36acb088e 100644 --- a/render-wasm/scripts/cargo-build.test.cjs +++ b/render-wasm/scripts/cargo-build.test.cjs @@ -97,11 +97,12 @@ function cargoOutput(records) { return records.map((record) => JSON.stringify(record)).join("\n") + "\n"; } -function run(f, records) { +function run(f, records, env = {}) { return spawnSync(process.execPath, [helper, f.info, f.manifest], { cwd: f.root, input: cargoOutput(records), encoding: "utf8", + env, }); } @@ -126,6 +127,7 @@ for (const fresh of [false, true]) { package_id: f.artifact.package_id, features: f.artifact.features, profile: f.artifact.profile, + environment: {}, out_dir: f.outDir, artifact_dir: f.artifactDir, shared_file: path.join(f.outDir, "render_wasm_shared.js"), @@ -137,6 +139,127 @@ for (const fresh of [false, true]) { }); } +test("records only allowlisted values from the helper process environment", (t) => { + const f = fixture(t); + const environment = { + RUSTFLAGS: "-C target-feature=+simd128", + RUSTC_WRAPPER: "", + CARGO_TARGET_DIR: "/tmp/target with spaces", + CARGO_HOME: "/opt/cargo", + CC: "emcc", + CFLAGS: "-O3\n-g0", + CXXFLAGS: "-std=c++17", + LDFLAGS: "-sALLOW_MEMORY_GROWTH=1", + PENPOT_WASM_FUNCTION_NAMES: "1", + CARGO_ENCODED_RUSTFLAGS: "-C\u001fopt-level=3", + CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", + BUILD_MODE: "release", + EMCC_CFLAGS: "-O3 -sASSERTIONS=0", + CARGO_PROFILE_RELEASE_LTO: "true", + CARGO_PROFILE_CUSTOM_OPT_LEVEL: "z", + EMSDK: "/opt/emsdk", + EMSDK_NODE: "/opt/emsdk/node/bin/node", + EM_INITIAL_HEAP: "268435456", + EM_MEMORY_GROWTH_GEOMETRIC_STEP: "0.8", + EM_MALLOC: "dlmalloc", + EM_CACHE: "/tmp/emsdk_cache", + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + }; + const result = run(f, [f.artifact, f.buildScript, finished], { + ...environment, + SECRET_TOKEN: "must not be saved", + PATH: "/private/path", + HOME: "/private/home", + CARGO_REGISTRIES_PRIVATE_TOKEN: "must not be saved", + CARGO_PROFILE: "not the prefix", + RUSTFLAGS_EXTRA: "not an exact key", + EM_UNRELATED: "not allowlisted", + VERSION_TAG: "build-identity", + RENDER_BENCH_BUILD_ID: "build-identity", + FRONTEND_DEST: "/per-build/renderer", + EXPORT_DEST: "/per-build/export", + FRONTEND_SHARED_DEST: "/per-build/shared.js", + EXPORT_SHARED_DEST: "/per-build/export-shared.js", + RENDER_WASM_SHARED_ARTIFACT: "/per-build/shared.js", + RENDER_WASM_BUILD_INFO: f.info, + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(f.info, "utf8")).environment, + environment, + ); +}); + +test("preserves unset versus empty environment values", (t) => { + const f = fixture(t); + for (const environment of [{}, { CFLAGS: "" }, { CFLAGS: "-O3" }]) { + const result = run(f, [f.artifact, f.buildScript, finished], environment); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(f.info, "utf8")).environment, + environment, + ); + } +}); + +test("captures exported setup values after sourcing the actual build environment", (t) => { + const f = fixture(t); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + 'source "$1" frontend --offline; export EMSDK_NODE="/setup/node"; CFLAGS="not exported"; shift; exec "$@"', + "cargo-build-test", + path.resolve(__dirname, "..", "_build_env"), + process.execPath, + helper, + f.info, + f.manifest, + ], + { + cwd: f.root, + input: cargoOutput([f.artifact, f.buildScript, finished]), + encoding: "utf8", + env: { + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + RUSTFLAGS: "-C debuginfo=0", + RUSTC_WRAPPER: "", + EMCC_CFLAGS: "must be replaced by setup", + EM_INITIAL_HEAP: "1", + SECRET_TOKEN: "must not be saved", + VERSION_TAG: "per-build-identity", + FRONTEND_DEST: "/per-build/renderer", + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + const environment = JSON.parse(fs.readFileSync(f.info, "utf8")).environment; + assert.deepEqual(environment, { + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + RUSTFLAGS: "-C debuginfo=0 -C target-feature=+simd128", + RUSTC_WRAPPER: "", + CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", + CARGO_TARGET_DIR: "target/frontend", + BUILD_MODE: "release", + CARGO_PROFILE_DIR: "release", + EMSDK_NODE: "/setup/node", + EM_INITIAL_HEAP: "268435456", + EM_MEMORY_GROWTH_GEOMETRIC_STEP: "0.8", + EM_MALLOC: "dlmalloc", + EM_CACHE: "/tmp/emsdk_cache", + EMCC_CFLAGS: environment.EMCC_CFLAGS, + }); + assert.match(environment.EMCC_CFLAGS, /^-O3 -sASSERTIONS=0 /); + assert.match(environment.EMCC_CFLAGS, /-sINITIAL_HEAP=268435456/); + assert.match(environment.EMCC_CFLAGS, /-sMEMORY_GROWTH_GEOMETRIC_STEP=0\.8/); + assert.match(environment.EMCC_CFLAGS, /-sMALLOC=dlmalloc/); +}); + test("records an empty actual feature set", (t) => { const f = fixture(t); const result = run(f, [