From c14ca9e50801d9377ab67a56c2a14beabd5c2cd8 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Tue, 27 Oct 2026 12:14:36 +0100 Subject: [PATCH] :construction: WIP 25.1 Cleanup. More consistent args. Avoid importing unnecessary namespaces. Improve file write error handling. AI-assisted-by: deepseek-flash, gpt-6.1-sol --- frontend/package.json | 2 +- .../benches/render_wasm/node/browser.cljs | 5 +- .../benches/render_wasm/node/build.cljs | 28 +- .../benches/render_wasm/node/commands.cljs | 89 +++++++ .../benches/render_wasm/node/compare.cljs | 8 +- .../benches/render_wasm/node/options.cljs | 32 +++ .../benches/render_wasm/node/process.cljs | 2 - .../benches/render_wasm/node/run.cljs | 7 +- .../benches/render_wasm/node/runner.cljs | 92 ++----- .../benches/render_wasm/node/runs.cljs | 3 +- .../benches/render_wasm/node/server.cljs | 4 +- .../benches/render_wasm/report/compare.cljs | 45 ++-- .../benches/render_wasm/report/format.cljs | 17 +- .../benches/render_wasm/report/summarize.cljs | 9 +- .../benches/render_wasm/result.cljs | 18 +- frontend/shadow-cljs.edn | 18 ++ .../render_wasm/cli_integration.cjs | 241 ++++++++++++------ .../render_wasm/compare_test.cljs | 22 +- .../render_wasm/report_test.cljs | 127 ++++++++- .../render_wasm/run_test.cljs | 31 +++ .../render_wasm/runner_test.cljs | 20 +- .../render_wasm/runs_test.cljs | 2 - 22 files changed, 582 insertions(+), 240 deletions(-) create mode 100644 frontend/performance/benches/render_wasm/node/commands.cljs create mode 100644 frontend/performance/benches/render_wasm/node/options.cljs diff --git a/frontend/package.json b/frontend/package.json index dee7cfe58d..38eb0983e8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,7 +20,7 @@ "build:storybook:assets": "node ./scripts/build-storybook-assets.js", "build:storybook:cljs": "clojure -M:dev:shadow-cljs compile storybook", "build:wasm": "../render-wasm/build frontend", - "build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-runner", + "build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-commands bench-render-wasm-runner", "build:app:libs": "node ./scripts/build-libs.js", "build:app:main": "clojure -M:dev:shadow-cljs release main worker", "build:app:worker": "clojure -M:dev:shadow-cljs release worker", diff --git a/frontend/performance/benches/render_wasm/node/browser.cljs b/frontend/performance/benches/render_wasm/node/browser.cljs index 5c94a37d97..110c2bef15 100644 --- a/frontend/performance/benches/render_wasm/node/browser.cljs +++ b/frontend/performance/benches/render_wasm/node/browser.cljs @@ -15,6 +15,7 @@ [app.common.transit :as transit] [benches.render-wasm.failures :as failures] [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.options :as options] [benches.render-wasm.node.process :as process])) (def launch-settings @@ -211,7 +212,7 @@ Startup failures include information about endpoint and profile." [{:keys [signal startup-timeout-ms deadline] - :or {startup-timeout-ms process/default-startup-timeout-ms}}] + :or {startup-timeout-ms options/default-startup-timeout-ms}}] (let [deadline (or deadline (process/deadline startup-timeout-ms)) budget (process/remaining deadline)] (when (and signal (.-aborted ^js signal)) @@ -278,7 +279,7 @@ Failures close the started page/context before rejecting." [owner host {:keys [viewport signal startup-timeout-ms deadline] - :or {startup-timeout-ms process/default-startup-timeout-ms}}] + :or {startup-timeout-ms options/default-startup-timeout-ms}}] (let [deadline (or deadline (process/deadline startup-timeout-ms)) bounds {:deadline deadline :signal signal :phase :browser-page-startup} ^js browser (:browser owner) diff --git a/frontend/performance/benches/render_wasm/node/build.cljs b/frontend/performance/benches/render_wasm/node/build.cljs index 4cd9ec7acb..32244a6ecf 100644 --- a/frontend/performance/benches/render_wasm/node/build.cljs +++ b/frontend/performance/benches/render_wasm/node/build.cljs @@ -20,28 +20,10 @@ [benches.render-wasm.identity :as identity] [benches.render-wasm.node.artifacts :as artifacts] [benches.render-wasm.node.catalog :as catalog] + [benches.render-wasm.node.options :as options] [benches.render-wasm.node.process :as process] [clojure.string :as str])) -(def diagnostic-features #{"stats" "profile" "profile-macros" "profile-raf"}) - -(defn feature-selection - "Validates measurement feature selection before allocating an attempt. - Cargo validates functional feature names during the retained build." - [{:keys [default-features? features] - :or {default-features? true features []}}] - (when-not (and (boolean? default-features?) (vector? features) - (every? #(and (string? %) (re-matches #"[A-Za-z0-9_-]+" %)) features)) - (throw (ex-info "invalid Cargo feature selection" {:type ::invalid-features}))) - (when-let [selected (seq (filter diagnostic-features features))] - (throw (ex-info "measurement builds cannot enable diagnostic Cargo features" - {:type ::diagnostic-features :features (vec selected)}))) - {:default-features? default-features? :features (vec (sort (distinct features)))}) - -(defn parse-feature-list - [text] - (:features (feature-selection {:features (mapv str/trim (str/split text #"," -1))}))) - (defn ^:async git-provenance "Captures revision and dirty state through the same bounded child boundary." [root bounds] @@ -90,11 +72,11 @@ Storage errors have no recorded ID. Later errors carry :build-id and :phase. A retry always creates a new attempt." [{:keys [root features build-timeout-ms signal on-created] - :or {build-timeout-ms process/default-build-timeout-ms}}] + :or {build-timeout-ms options/default-build-timeout-ms}}] (when-not (and (js/Number.isSafeInteger build-timeout-ms) (pos? build-timeout-ms)) (throw (ex-info "build timeout must be a positive integer" {:type ::invalid-timeout}))) (let [root (or root (artifacts/frontend-root)) - selection (feature-selection features) + selection (options/feature-selection features) deadline (process/deadline build-timeout-ms) bounds {:deadline deadline :signal signal} initial (artifacts/create-build root selection {:git {:sha nil :dirty nil} @@ -124,8 +106,8 @@ "RENDER_WASM_SHARED_ARTIFACT" (path/join (:renderer-dir build) "shared.js") "RENDER_WASM_BUILD_INFO" facts-path}}))) (let [facts (js->clj (js/JSON.parse (fs/readFileSync facts-path "utf8")) :keywordize-keys true) - resolved (:features (feature-selection {:features (:features facts) - :default-features? (:default-features? selection)}))] + resolved (:features (options/feature-selection {:features (:features facts) + :default-features? (:default-features? selection)}))] (when-not (and (= "3" (get-in facts [:profile :opt_level])) (false? (get-in facts [:profile :debug_assertions]))) (throw (ex-info "Cargo did not produce an optimized renderer" diff --git a/frontend/performance/benches/render_wasm/node/commands.cljs b/frontend/performance/benches/render_wasm/node/commands.cljs new file mode 100644 index 0000000000..c428101dfd --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/commands.cljs @@ -0,0 +1,89 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.commands + "Execution commands loaded after CLI validation." + (:require + [app.common.transit :as transit] + [benches.render-wasm.node.artifacts :as artifacts] + [benches.render-wasm.node.build :as build] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.run :as run])) + +(enable-console-print!) + +(defn- print-error + [cause] + (.error js/console (or (ex-message cause) (str cause))) + (when-let [data (ex-data cause)] + (.error js/console (pr-str data)))) + +(defn- ^:async run-command + [root options] + (let [controller (js/AbortController.) + signal (.-signal controller) + interrupted (atom nil) + remove-signals (process/on-signals + (fn [name] + (when-not (.-aborted signal) + (reset! interrupted name) + (.abort controller + (ex-info "run interrupted" + {:type ::interrupted :signal name})))))] + (try + (let [result (await (run/run root options signal + #(.write (.-stderr js/process) %) + #(println %)))] + (when-let [error (:error result)] + (print-error error)) + (:exit-code result)) + (catch :default cause + (print-error cause) + (case @interrupted + "SIGINT" 130 + "SIGTERM" 143 + 1)) + (finally + (remove-signals))))) + +(defn- ^:async build-command + [root options] + (let [controller (js/AbortController.) + remove-signals (process/on-signals #(.abort controller %))] + (try + (await (build/prepare {:root root + :features {:default-features? (:default-features options) + :features (or (:features options) [])} + :build-timeout-ms (:build-timeout-ms options) + :signal (.-signal controller) + :on-created #(println (:id %))})) + 0 + (finally (remove-signals))))) + +(defn ^:async execute + "Accepts a validated Transit command from the CLI and returns its exit code. + Each compiled module owns its CLJS values; only the string crosses modules. + Execution errors are printed here before returning a numeric exit code." + [request] + (try + (let [{:keys [command options]} (transit/decode-str request) + root (artifacts/frontend-root)] + (case command + "purge" + (do + (artifacts/purge root (cond (:only-builds options) :builds + (:only-runs options) :runs + :else :all)) + 0) + + "build" + (await (build-command root options)) + + "run" + (await (run-command root options)))) + (catch :default cause + (print-error cause) + 1))) diff --git a/frontend/performance/benches/render_wasm/node/compare.cljs b/frontend/performance/benches/render_wasm/node/compare.cljs index 1aaaef0bc9..70a8d4259b 100644 --- a/frontend/performance/benches/render_wasm/node/compare.cljs +++ b/frontend/performance/benches/render_wasm/node/compare.cljs @@ -24,18 +24,18 @@ (defn execute "Reads and compares two saved run files. - options: :compare-cases is one exact A,B selector; A belongs to baseline and + options: :cases is one exact A,B selector; A belongs to baseline and B to the candidate. :force bypasses compatibility checks. Returns the validated `comparison` and its formatted text." - [baseline-path candidate-path {:keys [compare-cases force]}] + [baseline-path candidate-path {:keys [cases force]}] (let [baseline (read-run baseline-path) candidate (read-run candidate-path) comparison (result/check-record (compare/compare-runs baseline candidate (cond-> {:force (boolean force)} - compare-cases + cases (assoc :pairs (compare/selector-pair baseline candidate - compare-cases)))))] + cases)))))] {:comparison comparison :text (format/format-comparison comparison)})) diff --git a/frontend/performance/benches/render_wasm/node/options.cljs b/frontend/performance/benches/render_wasm/node/options.cljs new file mode 100644 index 0000000000..af67582677 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/options.cljs @@ -0,0 +1,32 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.options + "Command defaults and feature validation." + (:require + [clojure.string :as str])) + +(def default-build-timeout-ms 300000) +(def default-startup-timeout-ms 30000) + +(def ^:private diagnostic-features #{"stats" "profile" "profile-macros" "profile-raf"}) + +(defn feature-selection + "Validates measurement feature selection before allocating an attempt. + Cargo validates functional feature names during the retained build." + [{:keys [default-features? features] + :or {default-features? true features []}}] + (when-not (and (boolean? default-features?) (vector? features) + (every? #(and (string? %) (re-matches #"[A-Za-z0-9_-]+" %)) features)) + (throw (ex-info "invalid Cargo feature selection" {:type ::invalid-features}))) + (when-let [selected (seq (filter diagnostic-features features))] + (throw (ex-info "measurement builds cannot enable diagnostic Cargo features" + {:type ::diagnostic-features :features (vec selected)}))) + {:default-features? default-features? :features (vec (sort (distinct features)))}) + +(defn parse-feature-list + [text] + (:features (feature-selection {:features (mapv str/trim (str/split text #"," -1))}))) diff --git a/frontend/performance/benches/render_wasm/node/process.cljs b/frontend/performance/benches/render_wasm/node/process.cljs index 9dd3c94524..f2db7c631d 100644 --- a/frontend/performance/benches/render_wasm/node/process.cljs +++ b/frontend/performance/benches/render_wasm/node/process.cljs @@ -11,8 +11,6 @@ ["fs" :as fs] [clojure.string :as str])) -(def default-build-timeout-ms 300000) -(def default-startup-timeout-ms 30000) (def default-termination-grace-ms 5000) (defn deadline diff --git a/frontend/performance/benches/render_wasm/node/run.cljs b/frontend/performance/benches/render_wasm/node/run.cljs index 5f05b7b17c..c5df581063 100644 --- a/frontend/performance/benches/render_wasm/node/run.cljs +++ b/frontend/performance/benches/render_wasm/node/run.cljs @@ -702,12 +702,13 @@ (catch :default record-cause (safe-log state (str "run: could not record cleanup failure: " (message-of record-cause) "\n"))))) - (let [run (try + (let [saved-run (try (finish-run state) (catch :default cause (reset! error cause) - @(:run-ref state))) - removal-error (when (:remove? opts) + nil)) + run (or saved-run @(:run-ref state)) + removal-error (when (and (:remove? opts) saved-run) (try (remove-build root selected-build) nil diff --git a/frontend/performance/benches/render_wasm/node/runner.cljs b/frontend/performance/benches/render_wasm/node/runner.cljs index 885622dfa8..5716ce5fe5 100644 --- a/frontend/performance/benches/render_wasm/node/runner.cljs +++ b/frontend/performance/benches/render_wasm/node/runner.cljs @@ -5,13 +5,12 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns benches.render-wasm.node.runner - "Build preparation, serial runs, offline comparison and managed cleanup." + "CLI parsing, offline comparison and deferred execution commands." (:require - [benches.render-wasm.node.artifacts :as artifacts] - [benches.render-wasm.node.build :as build] + ["path" :as path] + [app.common.transit :as transit] [benches.render-wasm.node.compare :as compare] - [benches.render-wasm.node.process :as process] - [benches.render-wasm.node.run :as run] + [benches.render-wasm.node.options :as options] [clojure.string :as str] [clojure.tools.cli :refer [parse-opts]])) @@ -49,7 +48,7 @@ [["-h" "--help" "Print this help"] [nil "--[no-]default-features" "Enable Cargo defaults" :default true] [nil "--features FEATURES" "Comma-separated functional Cargo features" - :parse-fn build/parse-feature-list] + :parse-fn options/parse-feature-list] [nil "--build ID" "Reuse a ready build" :id :build-id :validate nonempty] [nil "--filter SUBSTR" "Select cases whose id contains SUBSTR"] @@ -62,10 +61,10 @@ [nil "--timeout-ms MS" "Attempt deadline" :default 30000 :parse-fn parse-number :validate positive-timeout] [nil "--build-timeout-ms MS" "Complete preparation deadline" - :default process/default-build-timeout-ms + :default options/default-build-timeout-ms :parse-fn parse-number :validate positive-timeout] [nil "--startup-timeout-ms MS" "Host/browser readiness deadline" - :default process/default-startup-timeout-ms + :default options/default-startup-timeout-ms :parse-fn parse-number :validate positive-timeout] [nil "--output-dir DIR" "Explicit run home; must not exist" :validate nonempty] @@ -73,7 +72,7 @@ [nil "--screenshots" "Capture one screenshot per case"] [nil "--only-builds" "Purge build attempts"] [nil "--only-runs" "Purge managed runs"] - [nil "--compare-cases A,B" "Compare one explicit baseline,candidate pair"] + [nil "--cases A,B" "Select one baseline,candidate pair"] [nil "--force" "Bypass comparison compatibility checks"]]) (defn usage @@ -81,7 +80,7 @@ (str "Usage: runner build [options]\n" " runner purge [--only-builds | --only-runs]\n" " runner run [options]\n" - " runner compare BASELINE CANDIDATE [--compare-cases A,B] [--force]\n\n" + " runner compare BASELINE CANDIDATE [--cases A,B] [--force]\n\n" "build prints one attempt ID after storage initialization. Exit 0 means\n" "ready; errors retain the build.transit and build.log for that ID.\n" "run prints one run ID on stdout. Progress, failures and the summary go\n" @@ -139,7 +138,7 @@ #{"--only-builds" "--only-runs"}) (def ^:private compare-flags - #{"--compare-cases" "--force"}) + #{"--cases" "--force"}) (defn- invalid-args [message] @@ -223,76 +222,23 @@ (when-let [data (ex-data cause)] (.error js/console (pr-str data)))) -(defn- ^:async run-command - [root options] - (let [controller (js/AbortController.) - signal (.-signal controller) - interrupted (atom nil) - remove-signals (process/on-signals - (fn [name] - (when-not (.-aborted signal) - (reset! interrupted name) - (.abort controller - (ex-info "run interrupted" - {:type ::interrupted :signal name})))))] - (try - (let [result (await (run/run root options signal - #(.write (.-stderr js/process) %) - #(println %)))] - (when-let [error (:error result)] - (print-error error)) - (set! (.-exitCode js/process) (:exit-code result))) - (catch :default cause - (print-error cause) - (set! (.-exitCode js/process) - (case @interrupted - "SIGINT" 130 - "SIGTERM" 143 - 1))) - (finally - (remove-signals))))) - -(defn- ^:async build-command - [root options] - (let [controller (js/AbortController.) - remove-signals (process/on-signals #(.abort controller %))] - (try - (await (build/prepare {:root root - :features {:default-features? (:default-features options) - :features (or (:features options) [])} - :build-timeout-ms (:build-timeout-ms options) - :signal (.-signal controller) - :on-created #(println (:id %))})) - (finally (remove-signals))))) - -(defn- ^:async compare-command - [arguments options] - (let [{:keys [text]} (compare/execute (second arguments) (nth arguments 2) - (select-keys options [:compare-cases :force]))] - (println text))) - (defn ^:async -main [& args] (try - (let [{:keys [command arguments options summary]} (parse-args args) - root (artifacts/frontend-root)] + (let [{:keys [command arguments options summary] :as request} (parse-args args)] (cond (:help options) (println (usage summary)) - (= command "purge") - (artifacts/purge root (cond (:only-builds options) :builds - (:only-runs options) :runs - :else :all)) - - (= command "build") - (await (build-command root options)) - - (= command "run") - (await (run-command root options)) - (= command "compare") - (await (compare-command arguments options)))) + (println (:text (compare/execute (second arguments) (nth arguments 2) + (select-keys options [:cases :force])))) + + :else + (let [^js commands (js/require (path/join js/__dirname "commands.cjs"))] + (set! (.-exitCode js/process) + (await (.execute commands + (transit/encode-str (select-keys request [:command :options])))))))) (catch :default cause (print-error cause) (when (= ::invalid-args (:type (ex-data cause))) diff --git a/frontend/performance/benches/render_wasm/node/runs.cljs b/frontend/performance/benches/render_wasm/node/runs.cljs index 8ed2e0b824..00480a1852 100644 --- a/frontend/performance/benches/render_wasm/node/runs.cljs +++ b/frontend/performance/benches/render_wasm/node/runs.cljs @@ -68,8 +68,7 @@ {:dir dir :record-path record-path :log-path log-path - :run-id run-id - :managed? (some? managed)})) + :run-id run-id})) (defn write-record! "Validates and atomically replaces the saved RunRecord." diff --git a/frontend/performance/benches/render_wasm/node/server.cljs b/frontend/performance/benches/render_wasm/node/server.cljs index 792f605fdd..281cb84080 100644 --- a/frontend/performance/benches/render_wasm/node/server.cljs +++ b/frontend/performance/benches/render_wasm/node/server.cljs @@ -10,7 +10,7 @@ ["fs" :as fs] ["http" :as http] ["path" :as path] - [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.options :as options] [clojure.string :as str])) (def ^:private mime @@ -119,7 +119,7 @@ has a 30000 ms default timeout and accepts an optional AbortSignal. Abort, timeout and listen errors close the owned server before rejecting." ([build] (open build {})) - ([build {:keys [startup-timeout-ms signal] :or {startup-timeout-ms process/default-startup-timeout-ms}}] + ([build {:keys [startup-timeout-ms signal] :or {startup-timeout-ms options/default-startup-timeout-ms}}] (js/Promise. (fn [resolve reject] (let [root (fs/realpathSync (:dir build)) diff --git a/frontend/performance/benches/render_wasm/report/compare.cljs b/frontend/performance/benches/render_wasm/report/compare.cljs index 461b9cb2a5..f9b694c293 100644 --- a/frontend/performance/benches/render_wasm/report/compare.cljs +++ b/frontend/performance/benches/render_wasm/report/compare.cljs @@ -36,14 +36,14 @@ [baseline candidate selector] (let [names (str/split selector #"," -1)] (when-not (and (= 2 (count names)) (every? seq names)) - (throw (ex-info "--compare-cases requires one exact A,B pair" {:type ::invalid-selector}))) + (throw (ex-info "--cases requires one exact A,B pair" {:type ::invalid-selector}))) [[(resolve-case baseline (first names)) (resolve-case candidate (second names))]])) (defn- difference "Returns one compatibility difference when recorded values disagree." - [field baseline candidate waived?] + [field baseline candidate] (when (not= baseline candidate) - {:field field :baseline baseline :candidate candidate :waived? waived?})) + {:field field :baseline baseline :candidate candidate :waived? false})) (defn- preparations "Returns chronological preparation evidence for an exact saved case." @@ -63,40 +63,40 @@ (defn- evidence-differences "Checks cross-run equality plus missing or varying preparation evidence." - [field baseline candidate waived?] + [field baseline candidate] (concat - (when-let [d (difference field baseline candidate waived?)] [d]) + (when-let [d (difference field baseline candidate)] [d]) (when (or (not= 1 (count baseline)) (not= 1 (count candidate))) [{:field [field :incomplete-or-varying-evidence] - :baseline baseline :candidate candidate :waived? waived?}]))) + :baseline baseline :candidate candidate :waived? false}]))) (defn- pair-differences "Checks saved inputs, graphics, environment, build, sampling and metric definitions." - [baseline candidate b c shared explicit?] + [baseline candidate b c shared] (let [bdef (some #(when (= b (:id %)) %) (get-in baseline [:plan :cases])) cdef (some #(when (= c (:id %)) %) (get-in candidate [:plan :cases]))] (vec (concat (keep identity - [(difference :execution (execution-settings bdef) (execution-settings cdef) explicit?) + [(difference :execution (execution-settings bdef) (execution-settings cdef)) (difference :environment (get-in baseline [:metadata :environment]) - (get-in candidate [:metadata :environment]) false) + (get-in candidate [:metadata :environment])) (difference :functional-build (get-in baseline [:metadata :build :functional]) - (get-in candidate [:metadata :build :functional]) false) + (get-in candidate [:metadata :build :functional])) (difference :warmups (get-in baseline [:plan :warmups]) - (get-in candidate [:plan :warmups]) false) + (get-in candidate [:plan :warmups])) (difference :timeout-policy (get-in baseline [:plan :timeout-policy]) - (get-in candidate [:plan :timeout-policy]) false) + (get-in candidate [:plan :timeout-policy])) (difference :scored? (get-in baseline [:metadata :scored?]) - (get-in candidate [:metadata :scored?]) false)]) + (get-in candidate [:metadata :scored?]))]) (evidence-differences :fingerprint (preparation-facts baseline b :fingerprint) - (preparation-facts candidate c :fingerprint) explicit?) + (preparation-facts candidate c :fingerprint)) (evidence-differences :effective-graphics (preparation-facts baseline b :effective-graphics) - (preparation-facts candidate c :effective-graphics) false) + (preparation-facts candidate c :effective-graphics)) (mapcat (fn [metric-id] (let [bdefs (summary/definitions baseline b metric-id) cdefs (summary/definitions candidate c metric-id)] - (evidence-differences [:metric metric-id] (set bdefs) (set cdefs) false))) + (evidence-differences [:metric metric-id] (set bdefs) (set cdefs)))) shared))))) (defn- side-report @@ -106,6 +106,7 @@ :definition (some #(when (= id (:id %)) %) (get-in run [:plan :cases])) :preparations (preparations run id) :remaining (get-in run [:remaining id]) + :failed-attempts (summary/get-failed-attempts run id) :metrics (into {} (map (fn [metric-id] [metric-id (summary/summarize-metric run id metric-id analysis force)])) shared)}) @@ -116,11 +117,11 @@ c (resolve-case candidate c) candidate-ids (set (summary/metric-ids candidate c)) shared (filterv #(contains? candidate-ids %) (summary/metric-ids baseline b)) - mismatches (pair-differences baseline candidate b c shared explicit?)] + mismatches (pair-differences baseline candidate b c shared)] (when (empty? shared) (throw (ex-info "selected cases have no shared metric identifiers" {:type ::no-shared-metrics :baseline b :candidate c}))) - (when (and (not force) (some #(not (:waived? %)) mismatches)) + (when (and (not force) (seq mismatches)) (throw (ex-info "incompatible saved runs" {:type ::incompatible :baseline b :candidate c :mismatches mismatches}))) {:explicit? explicit? @@ -145,12 +146,12 @@ Inputs must already satisfy the `:runrecord` contract. Encoding validates the returned comparison at the boundary. - Default pairs are over identical identifiers. This can be overriden by picking - explicit pairs, or by bypassing all checks. + Default pairs use identical identifiers. Explicit pairs select cases without + changing compatibility checks. - `:pairs` accepts an extensible vector of explicit [baseline candidate] pairs. - - `:force` allows comparing any two records, but obviusly only compares metrics - common to both. + - `:force` allows all compatibility mismatches, retaining them in the report. + Only shared metrics are compared, using raw values without conversions. - `:analysis` resolves from method defaults, independent of each run's defaults." ([baseline candidate] (compare-runs baseline candidate {})) ([baseline candidate {:keys [pairs force analysis]}] diff --git a/frontend/performance/benches/render_wasm/report/format.cljs b/frontend/performance/benches/render_wasm/report/format.cljs index 3c04e32b2e..1c5d6b4064 100644 --- a/frontend/performance/benches/render_wasm/report/format.cljs +++ b/frontend/performance/benches/render_wasm/report/format.cljs @@ -50,6 +50,14 @@ " MAD " (estimate->text (:mad statistics)) " p95 " (estimate->text (:p95 statistics)) " p99 " (estimate->text (:p99 statistics)))))) +(defn- case-status->text + "Displays pending slots and failed attempts, including excluded warmups." + [{:keys [remaining failed-attempts]}] + (str " remaining " (pr-str remaining) + " failed-warmups=" (count (filter :warmup? failed-attempts)) + (when (seq failed-attempts) + (str "\n failed attempts " (pr-str failed-attempts))))) + (defn format-run "Formats a summary without browser, registry or renderer access." [{:keys [run-id metadata analysis termination source-coverage interpretation cases]}] @@ -59,16 +67,17 @@ "\n" source-coverage "\n" interpretation (when (some #(get-in % [:effective-graphics :software]) (mapcat :preparations cases)) "\nSOFTWARE RENDERER recorded") - (apply str (map (fn [{:keys [case-id remaining metrics]}] + (apply str (map (fn [{:keys [case-id metrics] :as case-report}] (str "\n\nCase " (result/case-name case-id) - " remaining " (pr-str remaining) + (case-status->text case-report) (apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics)))) cases)))) (defn- side->text "Displays one comparison side's definitions, preparation facts and observations." - [label {:keys [case-id definition preparations metrics]}] + [label {:keys [case-id definition preparations metrics] :as side}] (str "\n" label " " (result/case-name case-id) + (case-status->text side) "\n case " (pr-str definition) "\n preparations " (pr-str preparations) (apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics)))) @@ -83,7 +92,9 @@ (get-in pair [:candidate :preparations]))) pairs)) "SOFTWARE RENDERER recorded\n") "Baseline " (pr-str (:run-id baseline)) " Git " (pr-str (get-in baseline [:metadata :git])) + "\nBaseline termination " (pr-str (:termination baseline)) "\nCandidate " (pr-str (:run-id candidate)) " Git " (pr-str (get-in candidate [:metadata :git])) + "\nCandidate termination " (pr-str (:termination candidate)) "\nAnalysis " (pr-str analysis) "\n" source-coverage "\n" interpretation (apply str (map (fn [{:keys [baseline candidate mismatches metrics]}] diff --git a/frontend/performance/benches/render_wasm/report/summarize.cljs b/frontend/performance/benches/render_wasm/report/summarize.cljs index 1007cd72f2..8861537076 100644 --- a/frontend/performance/benches/render_wasm/report/summarize.cljs +++ b/frontend/performance/benches/render_wasm/report/summarize.cljs @@ -26,6 +26,11 @@ [run case-id] (filterv #(= case-id (:case-id %)) (:attempts run))) +(defn get-failed-attempts + "Returns failed warmup and measured attempts with their saved evidence." + [run case-id] + (filterv #(= :failed (:outcome %)) (case-attempts run case-id))) + (defn metric-ids "Discovers dynamic metric identifiers in first-appearance order, including failures." [run case-id] @@ -57,7 +62,8 @@ :unattempted counts remaining measured repetitions; case :remaining also retains pending warmups. :warmup counts executed warmup attempts. Failed measured attempts count as failed even with partial metrics. - Failed warmups count as warmup attempts." + Failed warmups count as warmup attempts; case :failed-attempts retains + their failures separately from measured observation counts." [run case-id metric-id] (reduce (fn [out {:keys [warmup? outcome metrics]}] @@ -109,6 +115,7 @@ :definition definition :preparations (filterv #(= id (:case-id %)) (:preparations run)) :remaining (get-in run [:remaining id]) + :failed-attempts (get-failed-attempts run id) :metrics (into {} (map (fn [metric-id] [metric-id (summarize-metric run id metric-id analysis)])) (metric-ids run id))}) diff --git a/frontend/performance/benches/render_wasm/result.cljs b/frontend/performance/benches/render_wasm/result.cljs index 6cf671acda..603ba17de7 100644 --- a/frontend/performance/benches/render_wasm/result.cljs +++ b/frontend/performance/benches/render_wasm/result.cljs @@ -145,6 +145,7 @@ [:case-id some?] [:definition [:map [:id some?]]] [:preparations [:vector schema:preparation]] [:remaining [:map {:closed true} [:warmups [:int {:min 0}]] [:repetitions [:int {:min 0}]]]] + [:failed-attempts [:vector schema:attempt]] [:metrics [:map-of :any schema:metric-report]]]) (def schema:provenance @@ -229,6 +230,15 @@ "measured attempt precedes planned warmups" {:case-id case-id}) (update-in remaining path dec))) +(defn- check-completion + "Requires all planned attempts to succeed before a run can be completed." + [run termination] + (when (= :completed (:reason termination)) + (require-valid! (every? #(every? zero? (vals %)) (vals (:remaining run))) + "completed run still has unattempted slots" {}) + (require-valid! (not-any? #(= :failed (:outcome %)) (:attempts run)) + "completed run has failed attempts" {}))) + (defn check-run "Validates schema3 and chronological accounting at a saved-data boundary. Unsupported versions fail without invoking a legacy reader or case registry." @@ -257,9 +267,7 @@ (require-valid! (= (:invalid-reason entry) (or (invalid-reason entry) (:invalid-reason entry))) "metric validity disagrees with raw value" {:metric-id id}))) - (when (= :completed (get-in run [:termination :reason])) - (require-valid! (every? #(every? zero? (vals %)) (vals left)) - "completed run still has unattempted slots" {}))) + (check-completion run (:termination run))) run) (defn create-run @@ -318,9 +326,7 @@ (check-open! run) (require-valid! (sm/validate schema:termination termination) "invalid termination" {}) (require-valid! (not= :running (:reason termination)) "termination needs an end reason" {}) - (when (= :completed (:reason termination)) - (require-valid! (every? #(every? zero? (vals %)) (vals (:remaining run))) - "completed run still has unattempted slots" {})) + (check-completion run termination) (assoc run :ended-at ended-at :termination termination)) (defn check-record diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index b7d46b06cb..12a50246ac 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -212,6 +212,24 @@ :elide-asserts true :anon-fn-naming-policy :off}}} + :bench-render-wasm-commands + {:target :node-library + :output-to "target/performance/render-wasm/commands.cjs" + :output-dir "target/performance/render-wasm/commands/" + :exports {:execute benches.render-wasm.node.commands/execute} + :devtools {:autoload false} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false + :warnings {:fn-deprecated false}} + + :release + {:compiler-options + {:fn-invoke-direct true + :elide-asserts true + :anon-fn-naming-policy :off}}} + :bench-render-wasm-browser {:target :esm :output-dir #shadow/env ["RENDER_BENCH_BROWSER_DIR" :default "target/performance/render-wasm/browser/"] diff --git a/frontend/test/performance_tests/render_wasm/cli_integration.cjs b/frontend/test/performance_tests/render_wasm/cli_integration.cjs index 09b7e65971..3eed938318 100644 --- a/frontend/test/performance_tests/render_wasm/cli_integration.cjs +++ b/frontend/test/performance_tests/render_wasm/cli_integration.cjs @@ -78,6 +78,10 @@ function createFrontend(t) { fs.writeFileSync(path.join(root, "deps.edn"), "{}\n"); fs.mkdirSync(managed, { recursive: true }); fs.copyFileSync(bootstrap, runner); + fs.copyFileSync( + path.join(path.dirname(bootstrap), "commands.cjs"), + path.join(managed, "commands.cjs"), + ); fs.symlinkSync( path.join(frontendRoot, "node_modules"), path.join(root, "node_modules"), @@ -100,13 +104,17 @@ function createFrontend(t) { async function runCli(fixture, args) { try { - const output = await execute(process.execPath, [fixture.runner, ...args], { - cwd: os.tmpdir(), - encoding: "utf8", - timeout: 15000, - killSignal: "SIGKILL", - maxBuffer: 1024 * 1024, - }); + const output = await execute( + process.execPath, + [...(fixture.nodeArgs ?? []), fixture.runner, ...args], + { + cwd: os.tmpdir(), + encoding: "utf8", + timeout: 15000, + killSignal: "SIGKILL", + maxBuffer: 1024 * 1024, + }, + ); return { exitCode: 0, ...output }; } catch (error) { // A timeout, signal or spawn failure must fail the test, rather than @@ -285,17 +293,13 @@ function runFixtureText({ k("attempts"), [attempt], k("remaining"), - map([ - k(caseId), - map([k("warmups"), 0, k("repetitions"), 0]), - ]), + map([k(caseId), map([k("warmups"), 0, k("repetitions"), 0])]), k("termination"), map([k("reason"), k("completed")]), ]); return transit.writer("json").write(record); } - function tree(directory) { return fs .readdirSync(directory, { withFileTypes: true }) @@ -521,76 +525,161 @@ fs.writeFileSync(${JSON.stringify(marker)}, "ran"); }, ); -test("help lists run and compare without touching managed storage", testOptions, async (t) => { - const fixture = createFrontend(t); - const original = tree(fixture.managed); +test( + "help and compare work without a frontend checkout or execution dependencies", + testOptions, + async (t) => { + const temporary = fs.mkdtempSync( + path.join(os.tmpdir(), "render-wasm-offline-"), + ); + t.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + const runner = path.join(temporary, "runner.cjs"); + const guard = path.join(temporary, "guard.cjs"); + fs.copyFileSync(bootstrap, runner); + fs.symlinkSync( + path.join(frontendRoot, "node_modules"), + path.join(temporary, "node_modules"), + "dir", + ); + fs.writeFileSync( + guard, + ` + const Module = require("node:module"); + const load = Module._load; + Module._load = function (id, ...args) { + if (["playwright", "http", "node:http", "child_process", "node:child_process"].includes(id)) { + throw new Error("Offline command imported execution dependency: " + id); + } + return load.call(this, id, ...args); + }; + `, + ); + const baseline = path.join(temporary, "baseline.transit"); + const candidate = path.join(temporary, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }), + ); + const fixture = { runner, nodeArgs: ["--require", guard] }; - const result = await runCli(fixture, ["--help"]); + const help = await runCli(fixture, ["--help"]); + assert.equal(help.exitCode, 0, help.stderr); + assert.match(help.stdout, /runner compare/); + const comparison = await runCli(fixture, ["compare", baseline, candidate]); + assert.equal(comparison.exitCode, 0, comparison.stderr); + assert.match(comparison.stdout, /median change/); + assert.equal(fs.existsSync(path.join(temporary, "commands.cjs")), false); + }, +); - assert.equal(result.exitCode, 0); - assert.match(result.stdout, /runner run/); - assert.match(result.stdout, /runner compare/); - assert.match(result.stdout, /--compare-cases/); - assert.deepEqual(tree(fixture.managed), original); -}); +test( + "help lists run and compare without touching managed storage", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const original = tree(fixture.managed); + const result = await runCli(fixture, ["--help"]); -test("compare reads saved run files without starting resources", testOptions, async (t) => { - const fixture = createFrontend(t); - const baseline = path.join(fixture.managed, "baseline.transit"); - const candidate = path.join(fixture.managed, "candidate.transit"); - fs.writeFileSync( - baseline, - runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), - ); - fs.writeFileSync( - candidate, - runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }), - ); - const original = tree(fixture.managed); + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /runner run/); + assert.match(result.stdout, /runner compare/); + assert.match(result.stdout, /--cases/); + assert.deepEqual(tree(fixture.managed), original); + }, +); - const result = await runCli(fixture, ["compare", baseline, candidate]); +test( + "compare reads saved run files without starting resources", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }), + ); + const original = tree(fixture.managed); - assert.equal(result.exitCode, 0); - assert.match(result.stdout, /Baseline/); - assert.match(result.stdout, /case\/load/); - assert.match(result.stdout, /median change/); - assert.deepEqual(tree(fixture.managed), original); -}); + const result = await runCli(fixture, ["compare", baseline, candidate]); -test("compare supports explicit pairs and force from the CLI", testOptions, async (t) => { - const fixture = createFrontend(t); - const baseline = path.join(fixture.managed, "baseline.transit"); - const candidate = path.join(fixture.managed, "candidate.transit"); - fs.writeFileSync( - baseline, - runFixtureText({ runId: "baseline", caseId: "a/pan", value: 10 }), - ); - fs.writeFileSync( - candidate, - runFixtureText({ - runId: "candidate", - caseId: "b/pan", - value: 12, - environment: { node: "v25" }, - }), - ); - const original = tree(fixture.managed); + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /Baseline/); + assert.match(result.stdout, /case\/load/); + assert.match(result.stdout, /median change/); + assert.deepEqual(tree(fixture.managed), original); + }, +); - const incompatible = await runCli(fixture, ["compare", baseline, candidate]); - assert.equal(incompatible.exitCode, 1); - assert.deepEqual(tree(fixture.managed), original); +test( + "compare supports explicit pairs and force from the CLI", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "a/pan", value: 10 }), + ); + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "b/pan", + value: 12, + environment: { node: "v25" }, + }), + ); + const original = tree(fixture.managed); - const forced = await runCli(fixture, [ - "compare", - baseline, - candidate, - "--compare-cases", - "a/pan,b/pan", - "--force", - ]); - assert.equal(forced.exitCode, 0); - assert.match(forced.stdout, /FORCED COMPARISON/); - assert.match(forced.stdout, /Pair a\/pan,b\/pan/); - assert.deepEqual(tree(fixture.managed), original); -}); + const incompatible = await runCli(fixture, [ + "compare", + baseline, + candidate, + ]); + assert.equal(incompatible.exitCode, 1); + assert.deepEqual(tree(fixture.managed), original); + + const forced = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--cases", + "a/pan,b/pan", + "--force", + ]); + assert.equal(forced.exitCode, 0); + assert.match(forced.stdout, /FORCED COMPARISON/); + assert.match(forced.stdout, /Pair a\/pan,b\/pan/); + assert.deepEqual(tree(fixture.managed), original); + + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "a/pan", + value: 12, + environment: { node: "v25" }, + }), + ); + const automatic = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--force", + ]); + assert.equal(automatic.exitCode, 0); + assert.match(automatic.stdout, /FORCED COMPARISON/); + assert.match(automatic.stdout, /Pair a\/pan,a\/pan/); + }, +); diff --git a/frontend/test/performance_tests/render_wasm/compare_test.cljs b/frontend/test/performance_tests/render_wasm/compare_test.cljs index 723411dbac..e24a8119b6 100644 --- a/frontend/test/performance_tests/render_wasm/compare_test.cljs +++ b/frontend/test/performance_tests/render_wasm/compare_test.cljs @@ -111,15 +111,29 @@ (finally (.chdir js/process previous))))) -(t/deftest compare-cases-selects-one-explicit-baseline-candidate-pair +(t/deftest cases-selects-one-explicit-baseline-candidate-pair (let [baseline (write-run (path/join @temporary-root "baseline.transit") (make-run "baseline" :a/pan 10)) candidate (write-run (path/join @temporary-root "candidate.transit") - (make-run "candidate" :b/pan 12 :scene :other)) + (make-run "candidate" :b/pan 12)) {:keys [text]} (compare/execute baseline candidate - {:compare-cases "a/pan,b/pan"})] + {:cases "a/pan,b/pan"})] (t/is (str/includes? text "Pair a/pan,b/pan")))) +(t/deftest cases-requires-force-for-different-scenes + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :a/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :b/load 12 :scene :other)) + selector {:cases "a/load,b/load"}] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate selector)))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate (assoc selector :force true))] + (t/is (true? (:forced? comparison))) + (t/is (str/includes? text "MISMATCHES")) + (t/is (str/includes? text "Pair a/load,b/load"))))) + (t/deftest force-bypasses-differences-and-displays-them (let [baseline (write-run (path/join @temporary-root "baseline.transit") (make-run "baseline" :case/load 10)) @@ -140,7 +154,7 @@ (doseq [selector ["case/load" "case/load,other/load,third/load" ""]] (t/is (= ::report-compare/invalid-selector (:type (ex-data (failure #(compare/execute baseline candidate - {:compare-cases selector}))))))) + {:cases selector}))))))) (fs/writeFileSync (path/join @temporary-root "broken.transit") "not transit") (t/is (thrown? js/Error (compare/execute baseline (path/join @temporary-root "broken.transit") diff --git a/frontend/test/performance_tests/render_wasm/report_test.cljs b/frontend/test/performance_tests/render_wasm/report_test.cljs index d2254c8c40..2910b46b22 100644 --- a/frontend/test/performance_tests/render_wasm/report_test.cljs +++ b/frontend/test/performance_tests/render_wasm/report_test.cljs @@ -174,6 +174,100 @@ (t/is (= ::result/invalid-record (:type (error-data #(result/finish-run run "end" {:reason :completed}))))) (t/is (= ::result/invalid-record (:type (error-data #(result/record-attempt ended {}))))))) +(t/deftest completed-runs-reject-failed-warmups-and-measured-attempts + (doseq [failed-warmup? [true false]] + (let [run (-> (result/create-run {:run-id "failed" :started-at "start" + :plan {:cases [saved-case] :warmups 1 :repetitions 1} + :metadata metadata}) + (result/record-preparation preparation) + (result/record-attempt {:case-id :saved/pan :preparation-id :prep/one + :warmup? true + :outcome (if failed-warmup? :failed :completed) + :metrics {metric-id (entry 1000)} :trace {}}) + (result/record-attempt {:case-id :saved/pan :preparation-id :prep/one + :warmup? false + :outcome (if failed-warmup? :completed :failed) + :metrics {metric-id (entry 2)} :trace {}})) + forged (assoc run :ended-at "end" :termination {:reason :completed}) + failed (result/finish-run run "end" {:reason :failed})] + (t/is (= {:warmups 0 :repetitions 0} (get-in run [:remaining :saved/pan]))) + (t/is (= ::result/invalid-record + (:type (error-data #(result/finish-run run "end" {:reason :completed}))))) + (t/is (= ::result/invalid-record (:type (error-data #(result/check-run forged))))) + (t/is (= ::result/invalid-record + (:type (error-data #(result/decode (transit/encode-str forged)))))) + (t/is (= failed (-> failed result/encode result/decode))))) + (let [completed (result/finish-run (run-with [2] {:warmups 1}) "end" {:reason :completed})] + (t/is (= completed (-> completed result/encode result/decode))))) + +(t/deftest reports-show-failed-warmups-without-changing-measured-counts + (let [failed {:case-id :saved/pan :preparation-id :prep/one :warmup? true + :outcome :failed :metrics {metric-id (entry 1000)} + :trace {:render-slices [3]} + :failure {:stage :warmup :message "warmup render failed"} + :partial {:phase :render-full}} + base (-> (run-with [1 2] {:warmups 1}) + (result/finish-run "end" {:reason :completed}) + result/encode result/decode) + other (-> (result/create-run {:run-id "candidate" :started-at "start" + :plan {:cases [saved-case] :warmups 1 :repetitions 2} + :metadata metadata}) + (result/record-preparation preparation) + (result/record-attempt failed) + (result/record-attempt {:case-id :saved/pan :preparation-id :prep/one + :warmup? false :outcome :completed + :metrics {metric-id (entry 2)} :trace {}}) + (result/record-attempt {:case-id :saved/pan :preparation-id :prep/one + :warmup? false :outcome :completed + :metrics {metric-id (entry 4)} :trace {}}) + (result/finish-run "end" {:reason :failed + :failure {:message "warmup render failed"}}) + result/encode result/decode) + report (summary/summarize other) + comparison (-> (compare/compare-runs base other) result/encode result/decode) + counts {:valid 2 :invalid 0 :absent 0 :failed 0 :unattempted 0 :warmup 1} + evidence [(assoc failed :ordinal 0)] + text (format/format-comparison comparison)] + (t/is (= evidence (get-in report [:cases 0 :failed-attempts]))) + (t/is (= evidence (get-in comparison [:pairs 0 :candidate :failed-attempts]))) + (t/is (= counts (get-in report [:cases 0 :metrics metric-id :counts]))) + (t/is (= counts (get-in comparison [:pairs 0 :candidate :metrics metric-id :counts]))) + (t/is (= 3 (get-in comparison [:pairs 0 :candidate :metrics metric-id :statistics :median :value]))) + (t/is (str/includes? text "Baseline termination {:reason :completed}")) + (t/is (str/includes? text "Candidate termination {:reason :failed")) + (doseq [output [text (format/format-run report)]] + (t/is (str/includes? output "failed-warmups=1")) + (t/is (str/includes? output "warmup render failed")) + (t/is (str/includes? output ":phase :render-full")) + (t/is (str/includes? output ":render-slices [3]"))))) + +(t/deftest comparisons-display-running-interrupted-and-preparation-failure-status + (let [base (-> (run-with [1] {:repetitions 2}) result/encode result/decode)] + (doseq [termination [{:reason :running} + {:reason :interrupted :signal "SIGINT"} + {:reason :preparation-error :failure {:message "cannot prepare renderer"}}]] + (let [other (if (= :running (:reason termination)) + base + (result/finish-run base "end" termination)) + comparison (-> (compare/compare-runs other other) result/encode result/decode) + text (format/format-comparison comparison)] + (t/is (str/includes? text (str "Baseline termination " (pr-str termination)))) + (t/is (str/includes? text (str "Candidate termination " (pr-str termination)))) + (t/is (str/includes? text "remaining {:warmups 0, :repetitions 1}")))) + (let [empty (result/create-run {:run-id "empty" :started-at "start" + :plan {:cases [saved-case] :warmups 0 :repetitions 2} + :metadata metadata}) + stopped (-> empty + (result/finish-run "end" {:reason :preparation-error + :failure {:message "cannot prepare renderer"}}) + result/encode result/decode) + report (summary/summarize stopped) + text (format/format-run report)] + (t/is (str/includes? text ":reason :preparation-error")) + (t/is (str/includes? text "cannot prepare renderer")) + (t/is (str/includes? text "remaining {:warmups 0, :repetitions 2}")) + (t/is (empty? (get-in report [:cases 0 :metrics])))))) + (t/deftest invalid-missing-and-failed-evidence (let [run (run-with [1 js/NaN js/Infinity js/-Infinity 0 nil :absent :failed]) obs (summary/observations run :saved/pan metric-id)] @@ -230,7 +324,7 @@ saved (-> run (result/record-attempt failed) (result/record-attempt attempt) - (result/finish-run "end" {:reason :completed}) + (result/finish-run "end" {:reason :failed}) result/encode result/decode) report (summary/summarize saved)] @@ -407,10 +501,12 @@ [:timeout-policy (assoc-in (run-with [3]) [:plan :timeout-policy :attempt-ms] 100)]]] (let [changed (-> changed result/encode result/decode) rejected (error-data #(compare/compare-runs base changed)) + selected (error-data #(compare/compare-runs base changed {:pairs [[:saved/pan :saved/pan]]})) forced (compare/compare-runs base changed {:force true}) mismatches (get-in forced [:pairs 0 :mismatches]) text (format/format-comparison forced)] (t/is (= ::compare/incompatible (:type rejected))) + (t/is (= rejected selected)) (t/is (:forced? forced)) (t/is (= [field] (mapv :field mismatches))) (t/is (= (:mismatches rejected) mismatches)) @@ -419,7 +515,7 @@ (t/is (str/includes? text "MISMATCHES")) (t/is (str/includes? text (str ":field " (pr-str field)))))))) -(t/deftest explicit-pairs-keep-non-case-checks-and-exact-names +(t/deftest explicit-pairs-select-cases-without-waiving-compatibility (let [base (run-with [1]) other (-> (run-with [2] {:definition (assoc saved-case :id :different/zoom :scene :different :operation {:steps 8} :batch-size 20 @@ -428,12 +524,20 @@ result/encode result/decode) pairs (compare/selector-pair base other "saved/pan,different/zoom") - comparison (compare/compare-runs base other {:pairs pairs}) + rejected (error-data #(compare/compare-runs base other {:pairs pairs})) + comparison (-> (compare/compare-runs base other {:pairs pairs :force true}) + result/encode result/decode) execution (first (filter #(= :execution (:field %)) (get-in comparison [:pairs 0 :mismatches]))) execution-keys [:scene :operation :batch-size :completion :preparation]] (t/is (= [[:saved/pan :different/zoom]] pairs)) (t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))) - (t/is (every? :waived? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (= ::compare/incompatible (:type rejected))) + (t/is (= #{:execution :fingerprint} (set (map :field (:mismatches rejected))))) + (t/is (= (:mismatches rejected) (get-in comparison [:pairs 0 :mismatches]))) + (t/is (every? #(false? (:waived? %)) (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? (format/format-comparison comparison) "FORCED COMPARISON")) + (t/is (str/includes? (format/format-comparison comparison) ":field :execution")) + (t/is (str/includes? (format/format-comparison comparison) ":field :fingerprint")) (t/is (= {:scene :saved :operation {:steps 3} :batch-size 1 :completion :render-full :preparation {}} (select-keys (:baseline execution) execution-keys))) @@ -451,14 +555,25 @@ (t/is (= ::compare/incompatible (:type rejected))) (t/is (:forced? forced)) (t/is (= (:mismatches rejected) mismatches)) - (t/is (= [{:field :scored? :baseline true :candidate false :waived? false}] - (vec (remove :waived? mismatches)))) + (t/is (= #{:execution :fingerprint :scored?} (set (map :field mismatches)))) + (t/is (every? #(false? (:waived? %)) mismatches)) (t/is (str/includes? text "FORCED COMPARISON")) (t/is (str/includes? text "MISMATCHES")) (t/is (str/includes? text ":field :scored?"))) (t/is (= ["saved/pan"] (:available (error-data #(compare/resolve-case base "pan"))))) (t/is (= ::compare/invalid-selector (:type (error-data #(compare/selector-pair base other "saved/pan"))))))) +(t/deftest explicit-pairs-allow-unlike-names-with-matching-execution-and-fingerprints + (let [base (-> (run-with [1]) result/encode result/decode) + other (-> (run-with [2] {:definition (assoc saved-case :id :different/zoom)}) + result/encode result/decode) + comparison (-> (compare/compare-runs base other {:pairs [[:saved/pan :different/zoom]]}) + result/encode result/decode)] + (t/is (false? (:forced? comparison))) + (t/is (true? (get-in comparison [:pairs 0 :explicit?]))) + (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))))) + (t/deftest force-preserves-metric-definitions-and-excludes-failed-values (let [base (run-with [1 :failed]) changed (assoc-in (run-with [2]) [:attempts 0 :metrics metric-id] diff --git a/frontend/test/performance_tests/render_wasm/run_test.cljs b/frontend/test/performance_tests/render_wasm/run_test.cljs index 54837e118e..cac383c18f 100644 --- a/frontend/test/performance_tests/render_wasm/run_test.cljs +++ b/frontend/test/performance_tests/render_wasm/run_test.cljs @@ -468,3 +468,34 @@ (t/is (not (fs/existsSync dir))) (t/is (= :completed (get-in result [:run :termination :reason]))) (t/is (zero? (:exit-code result))))) + +(t/deftest ^:async failed-final-record-write-retains-the-selected-build + (let [root @temporary-root + id "retained-after-write-failure" + dir (path/join (artifacts/managed-root root) "builds" id) + _ (fs/mkdirSync dir #js {:recursive true}) + _ (fs/writeFileSync (path/join dir "index.html") "") + out (output-dir) + record (path/join out "run.transit") + prior (path/join out "run.before-finish.transit") + base (fake-browser {}) + close (:close (:service base)) + fake (assoc base :service + (assoc (:service base) :close + (fn [owner] + (close owner) + ;; Make the final atomic rename fail after all + ;; attempts have reached the saved record. + (fs/renameSync record prior) + (fs/mkdirSync record)))) + response (await (run/execute root (selected-build dir id) + [(collected :case/a :fresh)] + (assoc (options fake out) :remove true) + (.-signal (js/AbortController.)) nil nil)) + saved (result/decode (fs/readFileSync prior "utf8"))] + (t/is (= :running (get-in saved [:termination :reason]))) + (t/is (= 1 (count (:attempts saved)))) + (t/is (every? zero? (vals (get-in saved [:remaining :case/a])))) + (t/is (some? (:error response))) + (t/is (= 1 (:exit-code response))) + (t/is (fs/existsSync dir)))) diff --git a/frontend/test/performance_tests/render_wasm/runner_test.cljs b/frontend/test/performance_tests/render_wasm/runner_test.cljs index fcb33c3ed5..5bbd2fd4aa 100644 --- a/frontend/test/performance_tests/render_wasm/runner_test.cljs +++ b/frontend/test/performance_tests/render_wasm/runner_test.cljs @@ -6,7 +6,7 @@ (ns performance-tests.render-wasm.runner-test (:require - [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.options :as options] [benches.render-wasm.node.runner :as runner] [cljs.test :as t :include-macros true] [clojure.string :as str])) @@ -15,7 +15,7 @@ (let [{:keys [command options]} (runner/parse-args ["build"])] (t/is (= "build" command)) (t/is (true? (:default-features options))) - (t/is (= process/default-build-timeout-ms (:build-timeout-ms options)))) + (t/is (= options/default-build-timeout-ms (:build-timeout-ms options)))) (let [{:keys [command options]} (runner/parse-args ["--" "build" "--no-default-features" "--features" "functional_b,functional-a,functional_b" @@ -78,7 +78,7 @@ (t/is (str/includes? help "runner run")) (t/is (str/includes? help "runner compare")) (t/is (str/includes? help "--build-timeout-ms")) - (t/is (str/includes? help "--compare-cases")))) + (t/is (str/includes? help "--cases")))) (t/is (thrown-with-msg? js/Error #"expected build, purge, run or compare" (runner/parse-args []))) (doseq [args [["unknown"] ["build" "purge"] ["build" "--unknown"] @@ -130,24 +130,28 @@ (runner/parse-args ["compare" "baseline.transit" "candidate.transit"])] (t/is (= "compare" command)) (t/is (= ["compare" "baseline.transit" "candidate.transit"] arguments)) - (t/is (nil? (:compare-cases options))) + (t/is (nil? (:cases options))) (t/is (not (:force options)))) (let [{:keys [options]} (runner/parse-args ["compare" "a.transit" "b.transit" - "--compare-cases" "rects/pan,paths/pan" "--force"])] - (t/is (= "rects/pan,paths/pan" (:compare-cases options))) + "--cases" "rects/pan,paths/pan" "--force"])] + (t/is (= "rects/pan,paths/pan" (:cases options))) + (t/is (true? (:force options)))) + (let [{:keys [options]} + (runner/parse-args ["compare" "a.transit" "b.transit" "--force"])] + (t/is (nil? (:cases options))) (t/is (true? (:force options))))) (t/deftest commands-reject-flags-owned-by-other-commands (doseq [[args message] [[["run" "--only-builds"] #"purge category flags require purge"] - [["run" "--compare-cases" "a,b"] #"compare options require compare"] + [["run" "--cases" "a,b"] #"compare options require compare"] [["compare" "a" "b" "--seed" "1"] #"run options require run"] [["compare" "a" "b" "--features" "functional"] #"build options require build"] [["build" "--build" "retained"] #"run options require run"] [["build" "--only-runs"] #"purge category flags require purge"] [["purge" "--seed" "1"] #"run options require run"] - [["purge" "--compare-cases" "a,b"] #"compare options require compare"]]] + [["purge" "--cases" "a,b"] #"compare options require compare"]]] (t/is (thrown-with-msg? js/Error message (runner/parse-args args))))) (t/deftest numeric-and-path-options-reject-invalid-bounds diff --git a/frontend/test/performance_tests/render_wasm/runs_test.cljs b/frontend/test/performance_tests/render_wasm/runs_test.cljs index 08bec87267..201c7a29f9 100644 --- a/frontend/test/performance_tests/render_wasm/runs_test.cljs +++ b/frontend/test/performance_tests/render_wasm/runs_test.cljs @@ -47,7 +47,6 @@ (let [home (runs/create-home {:root @temporary-root :run-id "run-1"})] (t/is (= (path/join (artifacts/managed-root @temporary-root) "runs" "run-1") (:dir home))) - (t/is (true? (:managed? home))) (t/is (.isDirectory (fs/statSync (:dir home)))) (t/is (= "" (fs/readFileSync (:log-path home) "utf8"))) (t/is (not (fs/existsSync (:record-path home)))))) @@ -61,7 +60,6 @@ :run-id "run-2" :output-dir relative})] (t/is (= (path/join @temporary-root relative) (:dir home))) - (t/is (false? (:managed? home))) (t/is (.isDirectory (fs/statSync (:dir home)))) (t/is (not (fs/existsSync (path/join (artifacts/managed-root @temporary-root) "runs" "run-2")))))