github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-10-29 05:50:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

🚧 WIP 25.1 Cleanup.

Browse Source 
More consistent args.
Avoid importing unnecessary namespaces.
Improve file write error handling.

AI-assisted-by: deepseek-flash, gpt-6.1-sol
This commit is contained in:
Miguel de Benito Delgado 2026-10-27 12:14:36 +01:00
parent 036236400d
commit c14ca9e508
22 changed files with 582 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
frontend/package.json
View File

@ -20,7 +20,7 @@
"build:storybook:assets": "node ./scripts/build-storybook-assets.js",
"build:storybook:cljs": "clojure -M:dev:shadow-cljs compile storybook",
"build:wasm": "../render-wasm/build frontend",
"build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-runner",
"build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-commands bench-render-wasm-runner",
"build:app:libs": "node ./scripts/build-libs.js",
"build:app:main": "clojure -M:dev:shadow-cljs release main worker",
"build:app:worker": "clojure -M:dev:shadow-cljs release worker",

5
frontend/performance/benches/render_wasm/node/browser.cljs
View File

@ -15,6 +15,7 @@
[app.common.transit :as transit]
[benches.render-wasm.failures :as failures]
[benches.render-wasm.identity :as identity]
[benches.render-wasm.node.options :as options]
[benches.render-wasm.node.process :as process]))
(def launch-settings
@ -211,7 +212,7 @@
Startup failures include information about endpoint and profile."
[{:keys [signal startup-timeout-ms deadline]
:or {startup-timeout-ms process/default-startup-timeout-ms}}]
:or {startup-timeout-ms options/default-startup-timeout-ms}}]
(let [deadline (or deadline (process/deadline startup-timeout-ms))
budget (process/remaining deadline)]
(when (and signal (.-aborted ^js signal))
@ -278,7 +279,7 @@
Failures close the started page/context before rejecting."
[owner host {:keys [viewport signal startup-timeout-ms deadline]
:or {startup-timeout-ms process/default-startup-timeout-ms}}]
:or {startup-timeout-ms options/default-startup-timeout-ms}}]
(let [deadline (or deadline (process/deadline startup-timeout-ms))
bounds {:deadline deadline :signal signal :phase :browser-page-startup}
^js browser (:browser owner)

28
frontend/performance/benches/render_wasm/node/build.cljs
View File

@ -20,28 +20,10 @@
[benches.render-wasm.identity :as identity]
[benches.render-wasm.node.artifacts :as artifacts]
[benches.render-wasm.node.catalog :as catalog]
[benches.render-wasm.node.options :as options]
[benches.render-wasm.node.process :as process]
[clojure.string :as str]))
(def diagnostic-features #{"stats" "profile" "profile-macros" "profile-raf"})
(defn feature-selection
"Validates measurement feature selection before allocating an attempt.
Cargo validates functional feature names during the retained build."
[{:keys [default-features? features]
:or {default-features? true features []}}]
(when-not (and (boolean? default-features?) (vector? features)
(every? #(and (string? %) (re-matches #"[A-Za-z0-9_-]+" %)) features))
(throw (ex-info "invalid Cargo feature selection" {:type ::invalid-features})))
(when-let [selected (seq (filter diagnostic-features features))]
(throw (ex-info "measurement builds cannot enable diagnostic Cargo features"
{:type ::diagnostic-features :features (vec selected)})))
{:default-features? default-features? :features (vec (sort (distinct features)))})
(defn parse-feature-list
[text]
(:features (feature-selection {:features (mapv str/trim (str/split text #"," -1))})))
(defn ^:async git-provenance
"Captures revision and dirty state through the same bounded child boundary."
[root bounds]
@ -90,11 +72,11 @@
Storage errors have no recorded ID. Later errors carry :build-id and :phase.
A retry always creates a new attempt."
[{:keys [root features build-timeout-ms signal on-created]
:or {build-timeout-ms process/default-build-timeout-ms}}]
:or {build-timeout-ms options/default-build-timeout-ms}}]
(when-not (and (js/Number.isSafeInteger build-timeout-ms) (pos? build-timeout-ms))
(throw (ex-info "build timeout must be a positive integer" {:type ::invalid-timeout})))
(let [root (or root (artifacts/frontend-root))
selection (feature-selection features)
selection (options/feature-selection features)
deadline (process/deadline build-timeout-ms)
bounds {:deadline deadline :signal signal}
initial (artifacts/create-build root selection {:git {:sha nil :dirty nil}
@ -124,8 +106,8 @@
"RENDER_WASM_SHARED_ARTIFACT" (path/join (:renderer-dir build) "shared.js")
"RENDER_WASM_BUILD_INFO" facts-path}})))
(let [facts (js->clj (js/JSON.parse (fs/readFileSync facts-path "utf8")) :keywordize-keys true)
resolved (:features (feature-selection {:features (:features facts)
:default-features? (:default-features? selection)}))]
resolved (:features (options/feature-selection {:features (:features facts)
:default-features? (:default-features? selection)}))]
(when-not (and (= "3" (get-in facts [:profile :opt_level]))
(false? (get-in facts [:profile :debug_assertions])))
(throw (ex-info "Cargo did not produce an optimized renderer"

89
frontend/performance/benches/render_wasm/node/commands.cljs Normal file
View File

@ -0,0 +1,89 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns benches.render-wasm.node.commands
"Execution commands loaded after CLI validation."
(:require
[app.common.transit :as transit]
[benches.render-wasm.node.artifacts :as artifacts]
[benches.render-wasm.node.build :as build]
[benches.render-wasm.node.process :as process]
[benches.render-wasm.node.run :as run]))
(enable-console-print!)
(defn- print-error
[cause]
(.error js/console (or (ex-message cause) (str cause)))
(when-let [data (ex-data cause)]
(.error js/console (pr-str data))))
(defn- ^:async run-command
[root options]
(let [controller (js/AbortController.)
signal (.-signal controller)
interrupted (atom nil)
remove-signals (process/on-signals
(fn [name]
(when-not (.-aborted signal)
(reset! interrupted name)
(.abort controller
(ex-info "run interrupted"
{:type ::interrupted :signal name})))))]
(try
(let [result (await (run/run root options signal
#(.write (.-stderr js/process) %)
#(println %)))]
(when-let [error (:error result)]
(print-error error))
(:exit-code result))
(catch :default cause
(print-error cause)
(case @interrupted
"SIGINT" 130
"SIGTERM" 143
1))
(finally
(remove-signals)))))
(defn- ^:async build-command
[root options]
(let [controller (js/AbortController.)
remove-signals (process/on-signals #(.abort controller %))]
(try
(await (build/prepare {:root root
:features {:default-features? (:default-features options)
:features (or (:features options) [])}
:build-timeout-ms (:build-timeout-ms options)
:signal (.-signal controller)
:on-created #(println (:id %))}))
0
(finally (remove-signals)))))
(defn ^:async execute
"Accepts a validated Transit command from the CLI and returns its exit code.
Each compiled module owns its CLJS values; only the string crosses modules.
Execution errors are printed here before returning a numeric exit code."
[request]
(try
(let [{:keys [command options]} (transit/decode-str request)
root (artifacts/frontend-root)]
(case command
"purge"
(do
(artifacts/purge root (cond (:only-builds options) :builds
(:only-runs options) :runs
:else :all))
0)
"build"
(await (build-command root options))
"run"
(await (run-command root options))))
(catch :default cause
(print-error cause)
1)))

8
frontend/performance/benches/render_wasm/node/compare.cljs
View File

@ -24,18 +24,18 @@
(defn execute
"Reads and compares two saved run files.
options: :compare-cases is one exact A,B selector; A belongs to baseline and
options: :cases is one exact A,B selector; A belongs to baseline and
B to the candidate. :force bypasses compatibility checks. Returns the
validated `comparison` and its formatted text."
[baseline-path candidate-path {:keys [compare-cases force]}]
[baseline-path candidate-path {:keys [cases force]}]
(let [baseline (read-run baseline-path)
candidate (read-run candidate-path)
comparison (result/check-record
(compare/compare-runs
baseline candidate
(cond-> {:force (boolean force)}
compare-cases
cases
(assoc :pairs (compare/selector-pair baseline candidate
compare-cases)))))]
cases)))))]
{:comparison comparison
:text (format/format-comparison comparison)}))

32
frontend/performance/benches/render_wasm/node/options.cljs Normal file
View File

@ -0,0 +1,32 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns benches.render-wasm.node.options
"Command defaults and feature validation."
(:require
[clojure.string :as str]))
(def default-build-timeout-ms 300000)
(def default-startup-timeout-ms 30000)
(def ^:private diagnostic-features #{"stats" "profile" "profile-macros" "profile-raf"})
(defn feature-selection
"Validates measurement feature selection before allocating an attempt.
Cargo validates functional feature names during the retained build."
[{:keys [default-features? features]
:or {default-features? true features []}}]
(when-not (and (boolean? default-features?) (vector? features)
(every? #(and (string? %) (re-matches #"[A-Za-z0-9_-]+" %)) features))
(throw (ex-info "invalid Cargo feature selection" {:type ::invalid-features})))
(when-let [selected (seq (filter diagnostic-features features))]
(throw (ex-info "measurement builds cannot enable diagnostic Cargo features"
{:type ::diagnostic-features :features (vec selected)})))
{:default-features? default-features? :features (vec (sort (distinct features)))})
(defn parse-feature-list
[text]
(:features (feature-selection {:features (mapv str/trim (str/split text #"," -1))})))

2
frontend/performance/benches/render_wasm/node/process.cljs
View File

@ -11,8 +11,6 @@
["fs" :as fs]
[clojure.string :as str]))
(def default-build-timeout-ms 300000)
(def default-startup-timeout-ms 30000)
(def default-termination-grace-ms 5000)
(defn deadline

7
frontend/performance/benches/render_wasm/node/run.cljs
View File

@ -702,12 +702,13 @@
(catch :default record-cause
(safe-log state (str "run: could not record cleanup failure: "
(message-of record-cause) "\n")))))
(let [run (try
(let [saved-run (try
(finish-run state)
(catch :default cause
(reset! error cause)
@(:run-ref state)))
removal-error (when (:remove? opts)
nil))
run (or saved-run @(:run-ref state))
removal-error (when (and (:remove? opts) saved-run)
(try
(remove-build root selected-build)
nil

92
frontend/performance/benches/render_wasm/node/runner.cljs
View File

@ -5,13 +5,12 @@
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns benches.render-wasm.node.runner
"Build preparation, serial runs, offline comparison and managed cleanup."
"CLI parsing, offline comparison and deferred execution commands."
(:require
[benches.render-wasm.node.artifacts :as artifacts]
[benches.render-wasm.node.build :as build]
["path" :as path]
[app.common.transit :as transit]
[benches.render-wasm.node.compare :as compare]
[benches.render-wasm.node.process :as process]
[benches.render-wasm.node.run :as run]
[benches.render-wasm.node.options :as options]
[clojure.string :as str]
[clojure.tools.cli :refer [parse-opts]]))
@ -49,7 +48,7 @@
[["-h" "--help" "Print this help"]
[nil "--[no-]default-features" "Enable Cargo defaults" :default true]
[nil "--features FEATURES" "Comma-separated functional Cargo features"
:parse-fn build/parse-feature-list]
:parse-fn options/parse-feature-list]
[nil "--build ID" "Reuse a ready build" :id :build-id
:validate nonempty]
[nil "--filter SUBSTR" "Select cases whose id contains SUBSTR"]
@ -62,10 +61,10 @@
[nil "--timeout-ms MS" "Attempt deadline" :default 30000
:parse-fn parse-number :validate positive-timeout]
[nil "--build-timeout-ms MS" "Complete preparation deadline"
:default process/default-build-timeout-ms
:default options/default-build-timeout-ms
:parse-fn parse-number :validate positive-timeout]
[nil "--startup-timeout-ms MS" "Host/browser readiness deadline"
:default process/default-startup-timeout-ms
:default options/default-startup-timeout-ms
:parse-fn parse-number :validate positive-timeout]
[nil "--output-dir DIR" "Explicit run home; must not exist"
:validate nonempty]
@ -73,7 +72,7 @@
[nil "--screenshots" "Capture one screenshot per case"]
[nil "--only-builds" "Purge build attempts"]
[nil "--only-runs" "Purge managed runs"]
[nil "--compare-cases A,B" "Compare one explicit baseline,candidate pair"]
[nil "--cases A,B" "Select one baseline,candidate pair"]
[nil "--force" "Bypass comparison compatibility checks"]])
(defn usage
@ -81,7 +80,7 @@
(str "Usage: runner build [options]\n"
" runner purge [--only-builds | --only-runs]\n"
" runner run [options]\n"
" runner compare BASELINE CANDIDATE [--compare-cases A,B] [--force]\n\n"
" runner compare BASELINE CANDIDATE [--cases A,B] [--force]\n\n"
"build prints one attempt ID after storage initialization. Exit 0 means\n"
"ready; errors retain the build.transit and build.log for that ID.\n"
"run prints one run ID on stdout. Progress, failures and the summary go\n"
@ -139,7 +138,7 @@
#{"--only-builds" "--only-runs"})
(def ^:private compare-flags
#{"--compare-cases" "--force"})
#{"--cases" "--force"})
(defn- invalid-args
[message]
@ -223,76 +222,23 @@
(when-let [data (ex-data cause)]
(.error js/console (pr-str data))))
(defn- ^:async run-command
[root options]
(let [controller (js/AbortController.)
signal (.-signal controller)
interrupted (atom nil)
remove-signals (process/on-signals
(fn [name]
(when-not (.-aborted signal)
(reset! interrupted name)
(.abort controller
(ex-info "run interrupted"
{:type ::interrupted :signal name})))))]
(try
(let [result (await (run/run root options signal
#(.write (.-stderr js/process) %)
#(println %)))]
(when-let [error (:error result)]
(print-error error))
(set! (.-exitCode js/process) (:exit-code result)))
(catch :default cause
(print-error cause)
(set! (.-exitCode js/process)
(case @interrupted
"SIGINT" 130
"SIGTERM" 143
1)))
(finally
(remove-signals)))))
(defn- ^:async build-command
[root options]
(let [controller (js/AbortController.)
remove-signals (process/on-signals #(.abort controller %))]
(try
(await (build/prepare {:root root
:features {:default-features? (:default-features options)
:features (or (:features options) [])}
:build-timeout-ms (:build-timeout-ms options)
:signal (.-signal controller)
:on-created #(println (:id %))}))
(finally (remove-signals)))))
(defn- ^:async compare-command
[arguments options]
(let [{:keys [text]} (compare/execute (second arguments) (nth arguments 2)
(select-keys options [:compare-cases :force]))]
(println text)))
(defn ^:async -main
[& args]
(try
(let [{:keys [command arguments options summary]} (parse-args args)
root (artifacts/frontend-root)]
(let [{:keys [command arguments options summary] :as request} (parse-args args)]
(cond
(:help options)
(println (usage summary))
(= command "purge")
(artifacts/purge root (cond (:only-builds options) :builds
(:only-runs options) :runs
:else :all))
(= command "build")
(await (build-command root options))
(= command "run")
(await (run-command root options))
(= command "compare")
(await (compare-command arguments options))))
(println (:text (compare/execute (second arguments) (nth arguments 2)
(select-keys options [:cases :force]))))
:else
(let [^js commands (js/require (path/join js/__dirname "commands.cjs"))]
(set! (.-exitCode js/process)
(await (.execute commands
(transit/encode-str (select-keys request [:command :options]))))))))
(catch :default cause
(print-error cause)
(when (= ::invalid-args (:type (ex-data cause)))

3
frontend/performance/benches/render_wasm/node/runs.cljs
View File

@ -68,8 +68,7 @@
{:dir dir
:record-path record-path
:log-path log-path
:run-id run-id
:managed? (some? managed)}))
:run-id run-id}))
(defn write-record!
"Validates and atomically replaces the saved RunRecord."

4
frontend/performance/benches/render_wasm/node/server.cljs
View File

@ -10,7 +10,7 @@
["fs" :as fs]
["http" :as http]
["path" :as path]
[benches.render-wasm.node.process :as process]
[benches.render-wasm.node.options :as options]
[clojure.string :as str]))
(def ^:private mime
@ -119,7 +119,7 @@
has a 30000 ms default timeout and accepts an optional AbortSignal. Abort,
timeout and listen errors close the owned server before rejecting."
([build] (open build {}))
([build {:keys [startup-timeout-ms signal] :or {startup-timeout-ms process/default-startup-timeout-ms}}]
([build {:keys [startup-timeout-ms signal] :or {startup-timeout-ms options/default-startup-timeout-ms}}]
(js/Promise.
(fn [resolve reject]
(let [root (fs/realpathSync (:dir build))

45
frontend/performance/benches/render_wasm/report/compare.cljs
View File

@ -36,14 +36,14 @@
[baseline candidate selector]
(let [names (str/split selector #"," -1)]
(when-not (and (= 2 (count names)) (every? seq names))
(throw (ex-info "--compare-cases requires one exact A,B pair" {:type ::invalid-selector})))
(throw (ex-info "--cases requires one exact A,B pair" {:type ::invalid-selector})))
[[(resolve-case baseline (first names)) (resolve-case candidate (second names))]]))
(defn- difference
"Returns one compatibility difference when recorded values disagree."
[field baseline candidate waived?]
[field baseline candidate]
(when (not= baseline candidate)
{:field field :baseline baseline :candidate candidate :waived? waived?}))
{:field field :baseline baseline :candidate candidate :waived? false}))
(defn- preparations
"Returns chronological preparation evidence for an exact saved case."
@ -63,40 +63,40 @@
(defn- evidence-differences
"Checks cross-run equality plus missing or varying preparation evidence."
[field baseline candidate waived?]
[field baseline candidate]
(concat
(when-let [d (difference field baseline candidate waived?)] [d])
(when-let [d (difference field baseline candidate)] [d])
(when (or (not= 1 (count baseline)) (not= 1 (count candidate)))
[{:field [field :incomplete-or-varying-evidence]
:baseline baseline :candidate candidate :waived? waived?}])))
:baseline baseline :candidate candidate :waived? false}])))
(defn- pair-differences
"Checks saved inputs, graphics, environment, build, sampling and metric definitions."
[baseline candidate b c shared explicit?]
[baseline candidate b c shared]
(let [bdef (some #(when (= b (:id %)) %) (get-in baseline [:plan :cases]))
cdef (some #(when (= c (:id %)) %) (get-in candidate [:plan :cases]))]
(vec
(concat
(keep identity
[(difference :execution (execution-settings bdef) (execution-settings cdef) explicit?)
[(difference :execution (execution-settings bdef) (execution-settings cdef))
(difference :environment (get-in baseline [:metadata :environment])
(get-in candidate [:metadata :environment]) false)
(get-in candidate [:metadata :environment]))
(difference :functional-build (get-in baseline [:metadata :build :functional])
(get-in candidate [:metadata :build :functional]) false)
(get-in candidate [:metadata :build :functional]))
(difference :warmups (get-in baseline [:plan :warmups])
(get-in candidate [:plan :warmups]) false)
(get-in candidate [:plan :warmups]))
(difference :timeout-policy (get-in baseline [:plan :timeout-policy])
(get-in candidate [:plan :timeout-policy]) false)
(get-in candidate [:plan :timeout-policy]))
(difference :scored? (get-in baseline [:metadata :scored?])
(get-in candidate [:metadata :scored?]) false)])
(get-in candidate [:metadata :scored?]))])
(evidence-differences :fingerprint (preparation-facts baseline b :fingerprint)
(preparation-facts candidate c :fingerprint) explicit?)
(preparation-facts candidate c :fingerprint))
(evidence-differences :effective-graphics (preparation-facts baseline b :effective-graphics)
(preparation-facts candidate c :effective-graphics) false)
(preparation-facts candidate c :effective-graphics))
(mapcat (fn [metric-id]
(let [bdefs (summary/definitions baseline b metric-id)
cdefs (summary/definitions candidate c metric-id)]
(evidence-differences [:metric metric-id] (set bdefs) (set cdefs) false)))
(evidence-differences [:metric metric-id] (set bdefs) (set cdefs))))
shared)))))
(defn- side-report
@ -106,6 +106,7 @@
:definition (some #(when (= id (:id %)) %) (get-in run [:plan :cases]))
:preparations (preparations run id)
:remaining (get-in run [:remaining id])
:failed-attempts (summary/get-failed-attempts run id)
:metrics (into {} (map (fn [metric-id]
[metric-id (summary/summarize-metric run id metric-id analysis force)])) shared)})
@ -116,11 +117,11 @@
c (resolve-case candidate c)
candidate-ids (set (summary/metric-ids candidate c))
shared (filterv #(contains? candidate-ids %) (summary/metric-ids baseline b))
mismatches (pair-differences baseline candidate b c shared explicit?)]
mismatches (pair-differences baseline candidate b c shared)]
(when (empty? shared)
(throw (ex-info "selected cases have no shared metric identifiers"
{:type ::no-shared-metrics :baseline b :candidate c})))
(when (and (not force) (some #(not (:waived? %)) mismatches))
(when (and (not force) (seq mismatches))
(throw (ex-info "incompatible saved runs"
{:type ::incompatible :baseline b :candidate c :mismatches mismatches})))
{:explicit? explicit?
@ -145,12 +146,12 @@
Inputs must already satisfy the `:runrecord` contract. Encoding validates the
returned comparison at the boundary.
Default pairs are over identical identifiers. This can be overriden by picking
explicit pairs, or by bypassing all checks.
Default pairs use identical identifiers. Explicit pairs select cases without
changing compatibility checks.
- `:pairs` accepts an extensible vector of explicit [baseline candidate] pairs.
- `:force` allows comparing any two records, but obviusly only compares metrics
common to both.
- `:force` allows all compatibility mismatches, retaining them in the report.
Only shared metrics are compared, using raw values without conversions.
- `:analysis` resolves from method defaults, independent of each run's defaults."
([baseline candidate] (compare-runs baseline candidate {}))
([baseline candidate {:keys [pairs force analysis]}]

17
frontend/performance/benches/render_wasm/report/format.cljs
View File

@ -50,6 +50,14 @@
" MAD " (estimate->text (:mad statistics))
" p95 " (estimate->text (:p95 statistics)) " p99 " (estimate->text (:p99 statistics))))))
(defn- case-status->text
"Displays pending slots and failed attempts, including excluded warmups."
[{:keys [remaining failed-attempts]}]
(str " remaining " (pr-str remaining)
" failed-warmups=" (count (filter :warmup? failed-attempts))
(when (seq failed-attempts)
(str "\n failed attempts " (pr-str failed-attempts)))))
(defn format-run
"Formats a summary without browser, registry or renderer access."
[{:keys [run-id metadata analysis termination source-coverage interpretation cases]}]
@ -59,16 +67,17 @@
"\n" source-coverage "\n" interpretation
(when (some #(get-in % [:effective-graphics :software])
(mapcat :preparations cases)) "\nSOFTWARE RENDERER recorded")
(apply str (map (fn [{:keys [case-id remaining metrics]}]
(apply str (map (fn [{:keys [case-id metrics] :as case-report}]
(str "\n\nCase " (result/case-name case-id)
" remaining " (pr-str remaining)
(case-status->text case-report)
(apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics))))
cases))))
(defn- side->text
"Displays one comparison side's definitions, preparation facts and observations."
[label {:keys [case-id definition preparations metrics]}]
[label {:keys [case-id definition preparations metrics] :as side}]
(str "\n" label " " (result/case-name case-id)
(case-status->text side)
"\n case " (pr-str definition)
"\n preparations " (pr-str preparations)
(apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics))))
@ -83,7 +92,9 @@
(get-in pair [:candidate :preparations]))) pairs))
"SOFTWARE RENDERER recorded\n")
"Baseline " (pr-str (:run-id baseline)) " Git " (pr-str (get-in baseline [:metadata :git]))
"\nBaseline termination " (pr-str (:termination baseline))
"\nCandidate " (pr-str (:run-id candidate)) " Git " (pr-str (get-in candidate [:metadata :git]))
"\nCandidate termination " (pr-str (:termination candidate))
"\nAnalysis " (pr-str analysis) "\n" source-coverage "\n" interpretation
(apply str
(map (fn [{:keys [baseline candidate mismatches metrics]}]

9
frontend/performance/benches/render_wasm/report/summarize.cljs
View File

@ -26,6 +26,11 @@
[run case-id]
(filterv #(= case-id (:case-id %)) (:attempts run)))
(defn get-failed-attempts
"Returns failed warmup and measured attempts with their saved evidence."
[run case-id]
(filterv #(= :failed (:outcome %)) (case-attempts run case-id)))
(defn metric-ids
"Discovers dynamic metric identifiers in first-appearance order, including failures."
[run case-id]
@ -57,7 +62,8 @@
:unattempted counts remaining measured repetitions; case :remaining also
retains pending warmups. :warmup counts executed warmup attempts.
Failed measured attempts count as failed even with partial metrics.
Failed warmups count as warmup attempts."
Failed warmups count as warmup attempts; case :failed-attempts retains
their failures separately from measured observation counts."
[run case-id metric-id]
(reduce
(fn [out {:keys [warmup? outcome metrics]}]
@ -109,6 +115,7 @@
:definition definition
:preparations (filterv #(= id (:case-id %)) (:preparations run))
:remaining (get-in run [:remaining id])
:failed-attempts (get-failed-attempts run id)
:metrics (into {} (map (fn [metric-id]
[metric-id (summarize-metric run id metric-id analysis)]))
(metric-ids run id))})

18
frontend/performance/benches/render_wasm/result.cljs
View File

@ -145,6 +145,7 @@
[:case-id some?] [:definition [:map [:id some?]]]
[:preparations [:vector schema:preparation]]
[:remaining [:map {:closed true} [:warmups [:int {:min 0}]] [:repetitions [:int {:min 0}]]]]
[:failed-attempts [:vector schema:attempt]]
[:metrics [:map-of :any schema:metric-report]]])
(def schema:provenance
@ -229,6 +230,15 @@
"measured attempt precedes planned warmups" {:case-id case-id})
(update-in remaining path dec)))
(defn- check-completion
"Requires all planned attempts to succeed before a run can be completed."
[run termination]
(when (= :completed (:reason termination))
(require-valid! (every? #(every? zero? (vals %)) (vals (:remaining run)))
"completed run still has unattempted slots" {})
(require-valid! (not-any? #(= :failed (:outcome %)) (:attempts run))
"completed run has failed attempts" {})))
(defn check-run
"Validates schema3 and chronological accounting at a saved-data boundary.
Unsupported versions fail without invoking a legacy reader or case registry."
@ -257,9 +267,7 @@
(require-valid! (= (:invalid-reason entry)
(or (invalid-reason entry) (:invalid-reason entry)))
"metric validity disagrees with raw value" {:metric-id id})))
(when (= :completed (get-in run [:termination :reason]))
(require-valid! (every? #(every? zero? (vals %)) (vals left))
"completed run still has unattempted slots" {})))
(check-completion run (:termination run)))
run)
(defn create-run
@ -318,9 +326,7 @@
(check-open! run)
(require-valid! (sm/validate schema:termination termination) "invalid termination" {})
(require-valid! (not= :running (:reason termination)) "termination needs an end reason" {})
(when (= :completed (:reason termination))
(require-valid! (every? #(every? zero? (vals %)) (vals (:remaining run)))
"completed run still has unattempted slots" {}))
(check-completion run termination)
(assoc run :ended-at ended-at :termination termination))
(defn check-record

18
frontend/shadow-cljs.edn
View File

@ -212,6 +212,24 @@
:elide-asserts true
:anon-fn-naming-policy :off}}}
:bench-render-wasm-commands
{:target :node-library
:output-to "target/performance/render-wasm/commands.cjs"
:output-dir "target/performance/render-wasm/commands/"
:exports {:execute benches.render-wasm.node.commands/execute}
:devtools {:autoload false}
:compiler-options
{:output-feature-set :es2020
:output-wrapper false
:warnings {:fn-deprecated false}}
:release
{:compiler-options
{:fn-invoke-direct true
:elide-asserts true
:anon-fn-naming-policy :off}}}
:bench-render-wasm-browser
{:target :esm
:output-dir #shadow/env ["RENDER_BENCH_BROWSER_DIR" :default "target/performance/render-wasm/browser/"]

241
frontend/test/performance_tests/render_wasm/cli_integration.cjs
View File

@ -78,6 +78,10 @@ function createFrontend(t) {
fs.writeFileSync(path.join(root, "deps.edn"), "{}\n");
fs.mkdirSync(managed, { recursive: true });
fs.copyFileSync(bootstrap, runner);
fs.copyFileSync(
path.join(path.dirname(bootstrap), "commands.cjs"),
path.join(managed, "commands.cjs"),
);
fs.symlinkSync(
path.join(frontendRoot, "node_modules"),
path.join(root, "node_modules"),
@ -100,13 +104,17 @@ function createFrontend(t) {
async function runCli(fixture, args) {
try {
const output = await execute(process.execPath, [fixture.runner, ...args], {
cwd: os.tmpdir(),
encoding: "utf8",
timeout: 15000,
killSignal: "SIGKILL",
maxBuffer: 1024 * 1024,
});
const output = await execute(
process.execPath,
[...(fixture.nodeArgs ?? []), fixture.runner, ...args],
{
cwd: os.tmpdir(),
encoding: "utf8",
timeout: 15000,
killSignal: "SIGKILL",
maxBuffer: 1024 * 1024,
},
);
return { exitCode: 0, ...output };
} catch (error) {
// A timeout, signal or spawn failure must fail the test, rather than
@ -285,17 +293,13 @@ function runFixtureText({
k("attempts"),
[attempt],
k("remaining"),
map([
k(caseId),
map([k("warmups"), 0, k("repetitions"), 0]),
]),
map([k(caseId), map([k("warmups"), 0, k("repetitions"), 0])]),
k("termination"),
map([k("reason"), k("completed")]),
]);
return transit.writer("json").write(record);
}
function tree(directory) {
return fs
.readdirSync(directory, { withFileTypes: true })
@ -521,76 +525,161 @@ fs.writeFileSync(${JSON.stringify(marker)}, "ran");
},
);
test("help lists run and compare without touching managed storage", testOptions, async (t) => {
const fixture = createFrontend(t);
const original = tree(fixture.managed);
test(
"help and compare work without a frontend checkout or execution dependencies",
testOptions,
async (t) => {
const temporary = fs.mkdtempSync(
path.join(os.tmpdir(), "render-wasm-offline-"),
);
t.after(() => fs.rmSync(temporary, { recursive: true, force: true }));
const runner = path.join(temporary, "runner.cjs");
const guard = path.join(temporary, "guard.cjs");
fs.copyFileSync(bootstrap, runner);
fs.symlinkSync(
path.join(frontendRoot, "node_modules"),
path.join(temporary, "node_modules"),
"dir",
);
fs.writeFileSync(
guard,
`
const Module = require("node:module");
const load = Module._load;
Module._load = function (id, ...args) {
if (["playwright", "http", "node:http", "child_process", "node:child_process"].includes(id)) {
throw new Error("Offline command imported execution dependency: " + id);
}
return load.call(this, id, ...args);
};
`,
);
const baseline = path.join(temporary, "baseline.transit");
const candidate = path.join(temporary, "candidate.transit");
fs.writeFileSync(
baseline,
runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }),
);
fs.writeFileSync(
candidate,
runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }),
);
const fixture = { runner, nodeArgs: ["--require", guard] };
const result = await runCli(fixture, ["--help"]);
const help = await runCli(fixture, ["--help"]);
assert.equal(help.exitCode, 0, help.stderr);
assert.match(help.stdout, /runner compare/);
const comparison = await runCli(fixture, ["compare", baseline, candidate]);
assert.equal(comparison.exitCode, 0, comparison.stderr);
assert.match(comparison.stdout, /median change/);
assert.equal(fs.existsSync(path.join(temporary, "commands.cjs")), false);
},
);
assert.equal(result.exitCode, 0);
assert.match(result.stdout, /runner run/);
assert.match(result.stdout, /runner compare/);
assert.match(result.stdout, /--compare-cases/);
assert.deepEqual(tree(fixture.managed), original);
});
test(
"help lists run and compare without touching managed storage",
testOptions,
async (t) => {
const fixture = createFrontend(t);
const original = tree(fixture.managed);
const result = await runCli(fixture, ["--help"]);
test("compare reads saved run files without starting resources", testOptions, async (t) => {
const fixture = createFrontend(t);
const baseline = path.join(fixture.managed, "baseline.transit");
const candidate = path.join(fixture.managed, "candidate.transit");
fs.writeFileSync(
baseline,
runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }),
);
fs.writeFileSync(
candidate,
runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }),
);
const original = tree(fixture.managed);
assert.equal(result.exitCode, 0);
assert.match(result.stdout, /runner run/);
assert.match(result.stdout, /runner compare/);
assert.match(result.stdout, /--cases/);
assert.deepEqual(tree(fixture.managed), original);
},
);
const result = await runCli(fixture, ["compare", baseline, candidate]);
test(
"compare reads saved run files without starting resources",
testOptions,
async (t) => {
const fixture = createFrontend(t);
const baseline = path.join(fixture.managed, "baseline.transit");
const candidate = path.join(fixture.managed, "candidate.transit");
fs.writeFileSync(
baseline,
runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }),
);
fs.writeFileSync(
candidate,
runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }),
);
const original = tree(fixture.managed);
assert.equal(result.exitCode, 0);
assert.match(result.stdout, /Baseline/);
assert.match(result.stdout, /case\/load/);
assert.match(result.stdout, /median change/);
assert.deepEqual(tree(fixture.managed), original);
});
const result = await runCli(fixture, ["compare", baseline, candidate]);
test("compare supports explicit pairs and force from the CLI", testOptions, async (t) => {
const fixture = createFrontend(t);
const baseline = path.join(fixture.managed, "baseline.transit");
const candidate = path.join(fixture.managed, "candidate.transit");
fs.writeFileSync(
baseline,
runFixtureText({ runId: "baseline", caseId: "a/pan", value: 10 }),
);
fs.writeFileSync(
candidate,
runFixtureText({
runId: "candidate",
caseId: "b/pan",
value: 12,
environment: { node: "v25" },
}),
);
const original = tree(fixture.managed);
assert.equal(result.exitCode, 0);
assert.match(result.stdout, /Baseline/);
assert.match(result.stdout, /case\/load/);
assert.match(result.stdout, /median change/);
assert.deepEqual(tree(fixture.managed), original);
},
);
const incompatible = await runCli(fixture, ["compare", baseline, candidate]);
assert.equal(incompatible.exitCode, 1);
assert.deepEqual(tree(fixture.managed), original);
test(
"compare supports explicit pairs and force from the CLI",
testOptions,
async (t) => {
const fixture = createFrontend(t);
const baseline = path.join(fixture.managed, "baseline.transit");
const candidate = path.join(fixture.managed, "candidate.transit");
fs.writeFileSync(
baseline,
runFixtureText({ runId: "baseline", caseId: "a/pan", value: 10 }),
);
fs.writeFileSync(
candidate,
runFixtureText({
runId: "candidate",
caseId: "b/pan",
value: 12,
environment: { node: "v25" },
}),
);
const original = tree(fixture.managed);
const forced = await runCli(fixture, [
"compare",
baseline,
candidate,
"--compare-cases",
"a/pan,b/pan",
"--force",
]);
assert.equal(forced.exitCode, 0);
assert.match(forced.stdout, /FORCED COMPARISON/);
assert.match(forced.stdout, /Pair a\/pan,b\/pan/);
assert.deepEqual(tree(fixture.managed), original);
});
const incompatible = await runCli(fixture, [
"compare",
baseline,
candidate,
]);
assert.equal(incompatible.exitCode, 1);
assert.deepEqual(tree(fixture.managed), original);
const forced = await runCli(fixture, [
"compare",
baseline,
candidate,
"--cases",
"a/pan,b/pan",
"--force",
]);
assert.equal(forced.exitCode, 0);
assert.match(forced.stdout, /FORCED COMPARISON/);
assert.match(forced.stdout, /Pair a\/pan,b\/pan/);
assert.deepEqual(tree(fixture.managed), original);
fs.writeFileSync(
candidate,
runFixtureText({
runId: "candidate",
caseId: "a/pan",
value: 12,
environment: { node: "v25" },
}),
);
const automatic = await runCli(fixture, [
"compare",
baseline,
candidate,
"--force",
]);
assert.equal(automatic.exitCode, 0);
assert.match(automatic.stdout, /FORCED COMPARISON/);
assert.match(automatic.stdout, /Pair a\/pan,a\/pan/);
},
);

22
frontend/test/performance_tests/render_wasm/compare_test.cljs
View File

@ -111,15 +111,29 @@
(finally
(.chdir js/process previous)))))
(t/deftest compare-cases-selects-one-explicit-baseline-candidate-pair
(t/deftest cases-selects-one-explicit-baseline-candidate-pair
(let [baseline (write-run (path/join @temporary-root "baseline.transit")
(make-run "baseline" :a/pan 10))
candidate (write-run (path/join @temporary-root "candidate.transit")
(make-run "candidate" :b/pan 12 :scene :other))
(make-run "candidate" :b/pan 12))
{:keys [text]} (compare/execute baseline candidate
{:compare-cases "a/pan,b/pan"})]
{:cases "a/pan,b/pan"})]
(t/is (str/includes? text "Pair a/pan,b/pan"))))
(t/deftest cases-requires-force-for-different-scenes
(let [baseline (write-run (path/join @temporary-root "baseline.transit")
(make-run "baseline" :a/load 10))
candidate (write-run (path/join @temporary-root "candidate.transit")
(make-run "candidate" :b/load 12 :scene :other))
selector {:cases "a/load,b/load"}]
(t/is (= ::report-compare/incompatible
(:type (ex-data (failure #(compare/execute baseline candidate selector))))))
(let [{:keys [comparison text]}
(compare/execute baseline candidate (assoc selector :force true))]
(t/is (true? (:forced? comparison)))
(t/is (str/includes? text "MISMATCHES"))
(t/is (str/includes? text "Pair a/load,b/load")))))
(t/deftest force-bypasses-differences-and-displays-them
(let [baseline (write-run (path/join @temporary-root "baseline.transit")
(make-run "baseline" :case/load 10))
@ -140,7 +154,7 @@
(doseq [selector ["case/load" "case/load,other/load,third/load" ""]]
(t/is (= ::report-compare/invalid-selector
(:type (ex-data (failure #(compare/execute baseline candidate
{:compare-cases selector})))))))
{:cases selector})))))))
(fs/writeFileSync (path/join @temporary-root "broken.transit") "not transit")
(t/is (thrown? js/Error (compare/execute baseline
(path/join @temporary-root "broken.transit")

127
frontend/test/performance_tests/render_wasm/report_test.cljs
View File

@ -174,6 +174,100 @@
(t/is (= ::result/invalid-record (:type (error-data #(result/finish-run run "end" {:reason :completed})))))
(t/is (= ::result/invalid-record (:type (error-data #(result/record-attempt ended {})))))))
(t/deftest completed-runs-reject-failed-warmups-and-measured-attempts
(doseq [failed-warmup? [true false]]
(let [run (-> (result/create-run {:run-id "failed" :started-at "start"
:plan {:cases [saved-case] :warmups 1 :repetitions 1}
:metadata metadata})
(result/record-preparation preparation)
(result/record-attempt {:case-id :saved/pan :preparation-id :prep/one
:warmup? true
:outcome (if failed-warmup? :failed :completed)
:metrics {metric-id (entry 1000)} :trace {}})
(result/record-attempt {:case-id :saved/pan :preparation-id :prep/one
:warmup? false
:outcome (if failed-warmup? :completed :failed)
:metrics {metric-id (entry 2)} :trace {}}))
forged (assoc run :ended-at "end" :termination {:reason :completed})
failed (result/finish-run run "end" {:reason :failed})]
(t/is (= {:warmups 0 :repetitions 0} (get-in run [:remaining :saved/pan])))
(t/is (= ::result/invalid-record
(:type (error-data #(result/finish-run run "end" {:reason :completed})))))
(t/is (= ::result/invalid-record (:type (error-data #(result/check-run forged)))))
(t/is (= ::result/invalid-record
(:type (error-data #(result/decode (transit/encode-str forged))))))
(t/is (= failed (-> failed result/encode result/decode)))))
(let [completed (result/finish-run (run-with [2] {:warmups 1}) "end" {:reason :completed})]
(t/is (= completed (-> completed result/encode result/decode)))))
(t/deftest reports-show-failed-warmups-without-changing-measured-counts
(let [failed {:case-id :saved/pan :preparation-id :prep/one :warmup? true
:outcome :failed :metrics {metric-id (entry 1000)}
:trace {:render-slices [3]}
:failure {:stage :warmup :message "warmup render failed"}
:partial {:phase :render-full}}
base (-> (run-with [1 2] {:warmups 1})
(result/finish-run "end" {:reason :completed})
result/encode result/decode)
other (-> (result/create-run {:run-id "candidate" :started-at "start"
:plan {:cases [saved-case] :warmups 1 :repetitions 2}
:metadata metadata})
(result/record-preparation preparation)
(result/record-attempt failed)
(result/record-attempt {:case-id :saved/pan :preparation-id :prep/one
:warmup? false :outcome :completed
:metrics {metric-id (entry 2)} :trace {}})
(result/record-attempt {:case-id :saved/pan :preparation-id :prep/one
:warmup? false :outcome :completed
:metrics {metric-id (entry 4)} :trace {}})
(result/finish-run "end" {:reason :failed
:failure {:message "warmup render failed"}})
result/encode result/decode)
report (summary/summarize other)
comparison (-> (compare/compare-runs base other) result/encode result/decode)
counts {:valid 2 :invalid 0 :absent 0 :failed 0 :unattempted 0 :warmup 1}
evidence [(assoc failed :ordinal 0)]
text (format/format-comparison comparison)]
(t/is (= evidence (get-in report [:cases 0 :failed-attempts])))
(t/is (= evidence (get-in comparison [:pairs 0 :candidate :failed-attempts])))
(t/is (= counts (get-in report [:cases 0 :metrics metric-id :counts])))
(t/is (= counts (get-in comparison [:pairs 0 :candidate :metrics metric-id :counts])))
(t/is (= 3 (get-in comparison [:pairs 0 :candidate :metrics metric-id :statistics :median :value])))
(t/is (str/includes? text "Baseline termination {:reason :completed}"))
(t/is (str/includes? text "Candidate termination {:reason :failed"))
(doseq [output [text (format/format-run report)]]
(t/is (str/includes? output "failed-warmups=1"))
(t/is (str/includes? output "warmup render failed"))
(t/is (str/includes? output ":phase :render-full"))
(t/is (str/includes? output ":render-slices [3]")))))
(t/deftest comparisons-display-running-interrupted-and-preparation-failure-status
(let [base (-> (run-with [1] {:repetitions 2}) result/encode result/decode)]
(doseq [termination [{:reason :running}
{:reason :interrupted :signal "SIGINT"}
{:reason :preparation-error :failure {:message "cannot prepare renderer"}}]]
(let [other (if (= :running (:reason termination))
base
(result/finish-run base "end" termination))
comparison (-> (compare/compare-runs other other) result/encode result/decode)
text (format/format-comparison comparison)]
(t/is (str/includes? text (str "Baseline termination " (pr-str termination))))
(t/is (str/includes? text (str "Candidate termination " (pr-str termination))))
(t/is (str/includes? text "remaining {:warmups 0, :repetitions 1}"))))
(let [empty (result/create-run {:run-id "empty" :started-at "start"
:plan {:cases [saved-case] :warmups 0 :repetitions 2}
:metadata metadata})
stopped (-> empty
(result/finish-run "end" {:reason :preparation-error
:failure {:message "cannot prepare renderer"}})
result/encode result/decode)
report (summary/summarize stopped)
text (format/format-run report)]
(t/is (str/includes? text ":reason :preparation-error"))
(t/is (str/includes? text "cannot prepare renderer"))
(t/is (str/includes? text "remaining {:warmups 0, :repetitions 2}"))
(t/is (empty? (get-in report [:cases 0 :metrics]))))))
(t/deftest invalid-missing-and-failed-evidence
(let [run (run-with [1 js/NaN js/Infinity js/-Infinity 0 nil :absent :failed])
obs (summary/observations run :saved/pan metric-id)]
@ -230,7 +324,7 @@
saved (-> run
(result/record-attempt failed)
(result/record-attempt attempt)
(result/finish-run "end" {:reason :completed})
(result/finish-run "end" {:reason :failed})
result/encode
result/decode)
report (summary/summarize saved)]
@ -407,10 +501,12 @@
[:timeout-policy (assoc-in (run-with [3]) [:plan :timeout-policy :attempt-ms] 100)]]]
(let [changed (-> changed result/encode result/decode)
rejected (error-data #(compare/compare-runs base changed))
selected (error-data #(compare/compare-runs base changed {:pairs [[:saved/pan :saved/pan]]}))
forced (compare/compare-runs base changed {:force true})
mismatches (get-in forced [:pairs 0 :mismatches])
text (format/format-comparison forced)]
(t/is (= ::compare/incompatible (:type rejected)))
(t/is (= rejected selected))
(t/is (:forced? forced))
(t/is (= [field] (mapv :field mismatches)))
(t/is (= (:mismatches rejected) mismatches))
@ -419,7 +515,7 @@
(t/is (str/includes? text "MISMATCHES"))
(t/is (str/includes? text (str ":field " (pr-str field))))))))
(t/deftest explicit-pairs-keep-non-case-checks-and-exact-names
(t/deftest explicit-pairs-select-cases-without-waiving-compatibility
(let [base (run-with [1])
other (-> (run-with [2] {:definition (assoc saved-case :id :different/zoom :scene :different
:operation {:steps 8} :batch-size 20
@ -428,12 +524,20 @@
result/encode
result/decode)
pairs (compare/selector-pair base other "saved/pan,different/zoom")
comparison (compare/compare-runs base other {:pairs pairs})
rejected (error-data #(compare/compare-runs base other {:pairs pairs}))
comparison (-> (compare/compare-runs base other {:pairs pairs :force true})
result/encode result/decode)
execution (first (filter #(= :execution (:field %)) (get-in comparison [:pairs 0 :mismatches])))
execution-keys [:scene :operation :batch-size :completion :preparation]]
(t/is (= [[:saved/pan :different/zoom]] pairs))
(t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value])))
(t/is (every? :waived? (get-in comparison [:pairs 0 :mismatches])))
(t/is (= ::compare/incompatible (:type rejected)))
(t/is (= #{:execution :fingerprint} (set (map :field (:mismatches rejected)))))
(t/is (= (:mismatches rejected) (get-in comparison [:pairs 0 :mismatches])))
(t/is (every? #(false? (:waived? %)) (get-in comparison [:pairs 0 :mismatches])))
(t/is (str/includes? (format/format-comparison comparison) "FORCED COMPARISON"))
(t/is (str/includes? (format/format-comparison comparison) ":field :execution"))
(t/is (str/includes? (format/format-comparison comparison) ":field :fingerprint"))
(t/is (= {:scene :saved :operation {:steps 3}
:batch-size 1 :completion :render-full :preparation {}}
(select-keys (:baseline execution) execution-keys)))
@ -451,14 +555,25 @@
(t/is (= ::compare/incompatible (:type rejected)))
(t/is (:forced? forced))
(t/is (= (:mismatches rejected) mismatches))
(t/is (= [{:field :scored? :baseline true :candidate false :waived? false}]
(vec (remove :waived? mismatches))))
(t/is (= #{:execution :fingerprint :scored?} (set (map :field mismatches))))
(t/is (every? #(false? (:waived? %)) mismatches))
(t/is (str/includes? text "FORCED COMPARISON"))
(t/is (str/includes? text "MISMATCHES"))
(t/is (str/includes? text ":field :scored?")))
(t/is (= ["saved/pan"] (:available (error-data #(compare/resolve-case base "pan")))))
(t/is (= ::compare/invalid-selector (:type (error-data #(compare/selector-pair base other "saved/pan")))))))
(t/deftest explicit-pairs-allow-unlike-names-with-matching-execution-and-fingerprints
(let [base (-> (run-with [1]) result/encode result/decode)
other (-> (run-with [2] {:definition (assoc saved-case :id :different/zoom)})
result/encode result/decode)
comparison (-> (compare/compare-runs base other {:pairs [[:saved/pan :different/zoom]]})
result/encode result/decode)]
(t/is (false? (:forced? comparison)))
(t/is (true? (get-in comparison [:pairs 0 :explicit?])))
(t/is (empty? (get-in comparison [:pairs 0 :mismatches])))
(t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value])))))
(t/deftest force-preserves-metric-definitions-and-excludes-failed-values
(let [base (run-with [1 :failed])
changed (assoc-in (run-with [2]) [:attempts 0 :metrics metric-id]

31
frontend/test/performance_tests/render_wasm/run_test.cljs
View File

@ -468,3 +468,34 @@
(t/is (not (fs/existsSync dir)))
(t/is (= :completed (get-in result [:run :termination :reason])))
(t/is (zero? (:exit-code result)))))
(t/deftest ^:async failed-final-record-write-retains-the-selected-build
(let [root @temporary-root
id "retained-after-write-failure"
dir (path/join (artifacts/managed-root root) "builds" id)
_ (fs/mkdirSync dir #js {:recursive true})
_ (fs/writeFileSync (path/join dir "index.html") "<!doctype html>")
out (output-dir)
record (path/join out "run.transit")
prior (path/join out "run.before-finish.transit")
base (fake-browser {})
close (:close (:service base))
fake (assoc base :service
(assoc (:service base) :close
(fn [owner]
(close owner)
;; Make the final atomic rename fail after all
;; attempts have reached the saved record.
(fs/renameSync record prior)
(fs/mkdirSync record))))
response (await (run/execute root (selected-build dir id)
[(collected :case/a :fresh)]
(assoc (options fake out) :remove true)
(.-signal (js/AbortController.)) nil nil))
saved (result/decode (fs/readFileSync prior "utf8"))]
(t/is (= :running (get-in saved [:termination :reason])))
(t/is (= 1 (count (:attempts saved))))
(t/is (every? zero? (vals (get-in saved [:remaining :case/a]))))
(t/is (some? (:error response)))
(t/is (= 1 (:exit-code response)))
(t/is (fs/existsSync dir))))

20
frontend/test/performance_tests/render_wasm/runner_test.cljs
View File

@ -6,7 +6,7 @@
(ns performance-tests.render-wasm.runner-test
(:require
[benches.render-wasm.node.process :as process]
[benches.render-wasm.node.options :as options]
[benches.render-wasm.node.runner :as runner]
[cljs.test :as t :include-macros true]
[clojure.string :as str]))
@ -15,7 +15,7 @@
(let [{:keys [command options]} (runner/parse-args ["build"])]
(t/is (= "build" command))
(t/is (true? (:default-features options)))
(t/is (= process/default-build-timeout-ms (:build-timeout-ms options))))
(t/is (= options/default-build-timeout-ms (:build-timeout-ms options))))
(let [{:keys [command options]} (runner/parse-args
["--" "build" "--no-default-features"
"--features" "functional_b,functional-a,functional_b"
@ -78,7 +78,7 @@
(t/is (str/includes? help "runner run"))
(t/is (str/includes? help "runner compare"))
(t/is (str/includes? help "--build-timeout-ms"))
(t/is (str/includes? help "--compare-cases"))))
(t/is (str/includes? help "--cases"))))
(t/is (thrown-with-msg? js/Error #"expected build, purge, run or compare"
(runner/parse-args [])))
(doseq [args [["unknown"] ["build" "purge"] ["build" "--unknown"]
@ -130,24 +130,28 @@
(runner/parse-args ["compare" "baseline.transit" "candidate.transit"])]
(t/is (= "compare" command))
(t/is (= ["compare" "baseline.transit" "candidate.transit"] arguments))
(t/is (nil? (:compare-cases options)))
(t/is (nil? (:cases options)))
(t/is (not (:force options))))
(let [{:keys [options]}
(runner/parse-args ["compare" "a.transit" "b.transit"
"--compare-cases" "rects/pan,paths/pan" "--force"])]
(t/is (= "rects/pan,paths/pan" (:compare-cases options)))
"--cases" "rects/pan,paths/pan" "--force"])]
(t/is (= "rects/pan,paths/pan" (:cases options)))
(t/is (true? (:force options))))
(let [{:keys [options]}
(runner/parse-args ["compare" "a.transit" "b.transit" "--force"])]
(t/is (nil? (:cases options)))
(t/is (true? (:force options)))))
(t/deftest commands-reject-flags-owned-by-other-commands
(doseq [[args message]
[[["run" "--only-builds"] #"purge category flags require purge"]
[["run" "--compare-cases" "a,b"] #"compare options require compare"]
[["run" "--cases" "a,b"] #"compare options require compare"]
[["compare" "a" "b" "--seed" "1"] #"run options require run"]
[["compare" "a" "b" "--features" "functional"] #"build options require build"]
[["build" "--build" "retained"] #"run options require run"]
[["build" "--only-runs"] #"purge category flags require purge"]
[["purge" "--seed" "1"] #"run options require run"]
[["purge" "--compare-cases" "a,b"] #"compare options require compare"]]]
[["purge" "--cases" "a,b"] #"compare options require compare"]]]
(t/is (thrown-with-msg? js/Error message (runner/parse-args args)))))
(t/deftest numeric-and-path-options-reject-invalid-bounds

2
frontend/test/performance_tests/render_wasm/runs_test.cljs
View File

@ -47,7 +47,6 @@
(let [home (runs/create-home {:root @temporary-root :run-id "run-1"})]
(t/is (= (path/join (artifacts/managed-root @temporary-root) "runs" "run-1")
(:dir home)))
(t/is (true? (:managed? home)))
(t/is (.isDirectory (fs/statSync (:dir home))))
(t/is (= "" (fs/readFileSync (:log-path home) "utf8")))
(t/is (not (fs/existsSync (:record-path home))))))
@ -61,7 +60,6 @@
:run-id "run-2"
:output-dir relative})]
(t/is (= (path/join @temporary-root relative) (:dir home)))
(t/is (false? (:managed? home)))
(t/is (.isDirectory (fs/statSync (:dir home))))
(t/is (not (fs/existsSync (path/join (artifacts/managed-root @temporary-root)
"runs" "run-2")))))