From 43b5e473cfc2d31e02805562e0d3bad89ba55a35 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Wed, 28 Oct 2026 15:26:19 +0100 Subject: [PATCH] :construction: WIP 26, A/B tests and a bunch of fixes AI-assisted-by: deepseek-flash, gpt-6.1-sol --- .../benches/render_wasm/feature_plan.cljs | 76 ++ .../benches/render_wasm/node/artifacts.cljs | 65 +- .../benches/render_wasm/node/build.cljs | 35 +- .../benches/render_wasm/node/commands.cljs | 48 +- .../benches/render_wasm/node/compare.cljs | 22 +- .../render_wasm/node/feature_comparison.cljs | 732 ++++++++++++++++ .../render_wasm/node/feature_plan.cljs | 73 ++ .../benches/render_wasm/node/run.cljs | 30 +- .../benches/render_wasm/node/runner.cljs | 74 +- .../benches/render_wasm/report/compare.cljs | 197 ++++- .../benches/render_wasm/report/format.cljs | 22 +- .../benches/render_wasm/report/summarize.cljs | 12 +- .../benches/render_wasm/result.cljs | 165 +++- .../render_wasm/artifacts_test.cljs | 71 ++ .../render_wasm/build_test.cljs | 55 ++ .../render_wasm/cli_integration.cjs | 440 +++++++++- .../render_wasm/compare_test.cljs | 116 ++- .../render_wasm/feature_comparison_test.cljs | 822 ++++++++++++++++++ .../render_wasm/report_test.cljs | 528 ++++++++++- .../render_wasm/run_test.cljs | 69 +- .../performance_tests/render_wasm/runner.cljs | 2 + .../render_wasm/runner_test.cljs | 112 ++- render-wasm/scripts/cargo-build.cjs | 33 + render-wasm/scripts/cargo-build.test.cjs | 125 ++- 24 files changed, 3718 insertions(+), 206 deletions(-) create mode 100644 frontend/performance/benches/render_wasm/feature_plan.cljs create mode 100644 frontend/performance/benches/render_wasm/node/feature_comparison.cljs create mode 100644 frontend/performance/benches/render_wasm/node/feature_plan.cljs create mode 100644 frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs diff --git a/frontend/performance/benches/render_wasm/feature_plan.cljs b/frontend/performance/benches/render_wasm/feature_plan.cljs new file mode 100644 index 0000000000..44082890e4 --- /dev/null +++ b/frontend/performance/benches/render_wasm/feature_plan.cljs @@ -0,0 +1,76 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.feature-plan + "Requested-feature configurations and comparison edges." + (:require + [benches.render-wasm.node.options :as options] + [clojure.string :as str])) + +(def ^:const default-max-features 3) + +(defn check-manifest + "Checks names and empty declared defaults against Cargo metadata's feature + definitions. Cargo owns activation; these definitions are not resolved here." + [features definitions] + (let [features (:features (options/feature-selection {:features features}))] + (when-not (and (map? definitions) + (every? string? (keys definitions)) + (every? #(and (vector? %) (every? string? %)) (vals definitions))) + (throw (ex-info "invalid Cargo feature definitions" {:type ::invalid-manifest}))) + (when (seq (get definitions "default")) + (throw (ex-info "ab requires empty declared Cargo defaults" {:type ::nonempty-defaults}))) + (when-let [unknown (seq (remove #(contains? definitions %) features))] + (throw (ex-info "unknown requested Cargo features" + {:type ::unknown-features :features (vec unknown)}))) + features)) + +(defn plan + "Plans sorted, MSB-first reflected Gray configurations and all off-to-on + edges. Each selection disables Cargo defaults. Execution needs confirmation + above eight configurations or with an empty filter; dry-run needs none." + [features {:keys [max-features confirm dry-run filter] + :or {max-features default-max-features confirm false dry-run false}}] + (let [features (:features (options/feature-selection {:features features})) + n (count features)] + (when-not (and (int? max-features) (<= 1 max-features 6)) + (throw (ex-info "max-features must be an integer from 1 to 6" {:type ::invalid-limit}))) + (when-not (and (boolean? confirm) (boolean? dry-run) + (or (nil? filter) (string? filter))) + (throw (ex-info "invalid feature planning options" {:type ::invalid-options}))) + (when (zero? n) + (throw (ex-info "ab requires at least one requested feature" {:type ::missing-features}))) + (when (> n max-features) + (throw (ex-info "requested features exceed max-features" + {:type ::too-many-features :count n :max-features max-features}))) + (when (and (not dry-run) (not confirm) + (or (> (bit-shift-left 1 n) 8) (str/blank? filter))) + (throw (ex-info "ab requires --confirm above eight configurations or with an empty filter" + {:type ::confirmation-required}))) + (let [slug (fn [bits] + (str/join (map #(if (bit-test bits %) "1" "0") + (reverse (range n))))) + gray-values (mapv #(bit-xor % (bit-shift-right % 1)) + (range (bit-shift-left 1 n))) + configurations (mapv (fn [bits] + {:slug (slug bits) + :selection {:default-features? false + :features (into [] + (keep-indexed + (fn [i feature] + (when (bit-test bits (- n i 1)) + feature))) + features)}}) + gray-values) + edges (into [] + (for [bits gray-values + [i feature] (map-indexed vector features) + :let [bit (- n i 1)] + :when (not (bit-test bits bit))] + {:from (slug bits) + :to (slug (bit-set bits bit)) + :feature feature}))] + {:features features :configurations configurations :edges edges}))) diff --git a/frontend/performance/benches/render_wasm/node/artifacts.cljs b/frontend/performance/benches/render_wasm/node/artifacts.cljs index 35bc476448..f2d52f55f2 100644 --- a/frontend/performance/benches/render_wasm/node/artifacts.cljs +++ b/frontend/performance/benches/render_wasm/node/artifacts.cljs @@ -5,12 +5,11 @@ ;; Copyright (c) KALEIDOS INC (ns benches.render-wasm.node.artifacts - "Managed renderer builds under frontend/target/performance/render-wasm. + "Outputs of renderer builds managed under + frontend/target/performance/render-wasm. - create-build writes the initial manifest and log before returning. A build - helper can then publish its :on-created output, compile into the returned - paths, and write a :ready or :failed manifest. Failed builds retain their - files and logs until an idle caller explicitly removes them." + Here, 'managed' means that the runner owns the build directory and its + contents, which can be deleted by means of the `purge` command." (:require ["fs" :as fs] ["path" :as path] @@ -28,6 +27,7 @@ [:status [:enum :building :ready :failed]] [:created-at :string] [:updated-at :string] + [:environment {:optional true} [:map-of :string :string]] [:functional :map] [:provenance :map]]) @@ -123,9 +123,12 @@ (defn create-build "Allocates a UUID build and saves its :building build.transit and empty log. - root is the frontend checkout. selection is {:default-features? boolean - :features vector}; provenance is {:git {:sha string-or-nil - :dirty boolean-or-nil}}. This function does not compile or remove builds." + - root: the frontend checkout. + - selection: {:default-features? boolean :features vector} + - provenance: {:git {:sha string-or-nil :dirty boolean-or-nil}}. + + Returns a build reference. + " [root selection provenance] (let [id (str (random-uuid)) now (.toISOString (js/Date.)) @@ -272,19 +275,55 @@ (recur (into pending (map #(path/join dir %) names)) (conj visited real))))))))) -(defn purge - "Unlinks known outputs from :builds, :runs or both (:all). +(defn- purge-feature-comparisons + "Unlinks feature-comparison records, configuration run evidence and edge + reports from the output directory. - The caller must be idle: no build, server or run may still own these paths. - Follows directory symlinks; keeps directories and unrelated files." + Keeps directories, builds and unrelated files." + [root] + (loop [pending [root] + visited #{}] + (when-let [dir (peek pending)] + (let [pending (pop pending) + real (when (directory? dir) (fs/realpathSync dir))] + (if (or (nil? real) (contains? visited real)) + (recur pending visited) + (let [parent (path/basename (path/dirname dir)) + base (path/basename dir) + names (array-seq (fs/readdirSync dir))] + (when (= "feature-comparisons" parent) + (unlink-if-present (path/join dir "feature-comparison.transit"))) + (when (= "configs" parent) + (unlink-if-present (path/join dir "run.log")) + (unlink-if-present (path/join dir "run.transit"))) + (when (= "comparisons" base) + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".transit")] + (unlink-if-present (path/join dir file)))) + (doseq [file names + :when (str/ends-with? (str/lower-case file) ".png")] + (unlink-if-present (path/join dir file))) + (recur (into pending (map #(path/join dir %) names)) + (conj visited real)))))))) + +(defn purge + "Unlinks known outputs from :builds, :runs, :comparisons or all of them (:all). + + The caller must be idle: no build, server, run or comparison may still own + these paths. Follows directory symlinks; keeps directories and unrelated + files." [root category] (let [managed (managed-root root)] (case category :builds (purge-builds (path/join managed "builds")) :runs (purge-runs (path/join managed "runs")) + :comparisons (purge-feature-comparisons + (path/join managed "feature-comparisons")) :all (do (purge-builds (path/join managed "builds")) - (purge-runs (path/join managed "runs"))) + (purge-runs (path/join managed "runs")) + (purge-feature-comparisons + (path/join managed "feature-comparisons"))) (throw (ex-info "invalid renderer artifact category" {:type ::invalid-category :category category}))))) diff --git a/frontend/performance/benches/render_wasm/node/build.cljs b/frontend/performance/benches/render_wasm/node/build.cljs index 32244a6ecf..05ce1ca724 100644 --- a/frontend/performance/benches/render_wasm/node/build.cljs +++ b/frontend/performance/benches/render_wasm/node/build.cljs @@ -66,17 +66,17 @@ (defn ^:async prepare "Returns a ready build reference after compilation and identity validation. - Options: frontend :root, :features, :build-timeout-ms (300000), AbortSignal - :signal, and optional :on-created callback. The callback runs once after - directory, log and initial manifest creation. CLI callers print the ID there. - Storage errors have no recorded ID. Later errors carry :build-id and :phase. - A retry always creates a new attempt." - [{:keys [root features build-timeout-ms signal on-created] + Options: frontend :root, :selection or :features, :build-timeout-ms + (300000), AbortSignal :signal, and optional :on-created callback. The callback + runs once after directory, log and initial manifest creation. CLI callers + print the ID there. Storage errors have no recorded ID. Later errors carry + :build-id and :phase. A retry always creates a new attempt." + [{:keys [root features selection build-timeout-ms signal on-created] :or {build-timeout-ms options/default-build-timeout-ms}}] (when-not (and (js/Number.isSafeInteger build-timeout-ms) (pos? build-timeout-ms)) (throw (ex-info "build timeout must be a positive integer" {:type ::invalid-timeout}))) (let [root (or root (artifacts/frontend-root)) - selection (options/feature-selection features) + selection (options/feature-selection (or selection features)) deadline (process/deadline build-timeout-ms) bounds {:deadline deadline :signal signal} initial (artifacts/create-build root selection {:git {:sha nil :dirty nil} @@ -105,18 +105,21 @@ "FRONTEND_SHARED_DEST" (path/join root "src" "app" "render_wasm" "api" "shared.js") "RENDER_WASM_SHARED_ARTIFACT" (path/join (:renderer-dir build) "shared.js") "RENDER_WASM_BUILD_INFO" facts-path}}))) - (let [facts (js->clj (js/JSON.parse (fs/readFileSync facts-path "utf8")) :keywordize-keys true) - resolved (:features (options/feature-selection {:features (:features facts) + (let [facts (js/JSON.parse (fs/readFileSync facts-path "utf8")) + profile (js->clj (.-profile facts) :keywordize-keys true) + resolved (:features (options/feature-selection {:features (js->clj (.-features facts)) :default-features? (:default-features? selection)}))] - (when-not (and (= "3" (get-in facts [:profile :opt_level])) - (false? (get-in facts [:profile :debug_assertions]))) + (when-not (and (= "3" (:opt_level profile)) + (false? (:debug_assertions profile))) (throw (ex-info "Cargo did not produce an optimized renderer" - {:type ::renderer-profile :profile (:profile facts)}))) + {:type ::renderer-profile :profile profile}))) (swap! build-ref #(artifacts/write-manifest - % (-> (:manifest %) - (assoc-in [:functional :features] resolved) - (assoc-in [:functional :default-features?] (boolean (some #{"default"} resolved))) - (assoc-in [:provenance :cargo-profile] (:profile facts))))))) + % (cond-> (-> (:manifest %) + (assoc-in [:functional :features] resolved) + (assoc-in [:functional :default-features?] (boolean (some #{"default"} resolved))) + (assoc-in [:provenance :cargo-profile] profile)) + (js/Object.hasOwn facts "environment") + (assoc :environment (js->clj (.-environment facts)))))))) (reset! phase :cljs) (let [build @build-ref] (await (process/run (merge bounds diff --git a/frontend/performance/benches/render_wasm/node/commands.cljs b/frontend/performance/benches/render_wasm/node/commands.cljs index c428101dfd..6e8734f1a4 100644 --- a/frontend/performance/benches/render_wasm/node/commands.cljs +++ b/frontend/performance/benches/render_wasm/node/commands.cljs @@ -10,6 +10,8 @@ [app.common.transit :as transit] [benches.render-wasm.node.artifacts :as artifacts] [benches.render-wasm.node.build :as build] + [benches.render-wasm.node.feature-comparison :as feature-comparison] + [benches.render-wasm.node.feature-plan :as feature-plan] [benches.render-wasm.node.process :as process] [benches.render-wasm.node.run :as run])) @@ -63,6 +65,46 @@ 0 (finally (remove-signals))))) +(defn- ^:async ab-command + [root options] + (let [controller (js/AbortController.) + signal (.-signal controller) + interrupted (atom nil) + remove-signals (process/on-signals + (fn [name] + (when-not (.-aborted signal) + (reset! interrupted name) + (.abort controller name))))] + (try + (let [prepared (await (feature-plan/prepare root options signal))] + (if (:dry-run options) + (do + (.write (.-stdout js/process) (feature-plan/format-plan prepared)) + 0) + (let [result (await (feature-comparison/execute + root prepared signal + {:on-created + (fn [home record] + (println (:comparison-id record)) + (println (:dir home))) + :on-progress + #(.write (.-stderr js/process) %)}))] + (when-let [error (:error result)] + (print-error error)) + (or (case @interrupted + "SIGINT" 130 + "SIGTERM" 143 + nil) + (:exit-code result))))) + (catch :default cause + (print-error cause) + (case @interrupted + "SIGINT" 130 + "SIGTERM" 143 + 1)) + (finally + (remove-signals))))) + (defn ^:async execute "Accepts a validated Transit command from the CLI and returns its exit code. Each compiled module owns its CLJS values; only the string crosses modules. @@ -76,6 +118,7 @@ (do (artifacts/purge root (cond (:only-builds options) :builds (:only-runs options) :runs + (:only-comparisons options) :comparisons :else :all)) 0) @@ -83,7 +126,10 @@ (await (build-command root options)) "run" - (await (run-command root options)))) + (await (run-command root options)) + + "ab" + (await (ab-command root options)))) (catch :default cause (print-error cause) 1))) diff --git a/frontend/performance/benches/render_wasm/node/compare.cljs b/frontend/performance/benches/render_wasm/node/compare.cljs index cb349279e1..247d5bb9b0 100644 --- a/frontend/performance/benches/render_wasm/node/compare.cljs +++ b/frontend/performance/benches/render_wasm/node/compare.cljs @@ -24,16 +24,28 @@ (defn execute "Reads and compares two saved run files. - options: :cases is one exact A,B selector; A belongs to baseline and - B to the candidate. :force bypasses compatibility checks. Returns the - validated `comparison` and its formatted text." - [baseline-path candidate-path {:keys [cases force]}] + Runs are only compared if the build options and scene configuration match. + These can be overriden via flags. + + It is also possible to restrict comparison to specific scene ids. + + Options: + - `:cases` is one A,B selector, where A is a baseline an B the candidate. + - The three `:allow-*-differences` options select build, scene or all + compatibility permissions. + + Returns the validated `comparison` with its selected permissions, retained + differences and formatted text." + [baseline-path candidate-path {:keys [cases allow-build-differences + allow-scene-differences allow-all-differences]}] (let [baseline (read-run baseline-path) candidate (read-run candidate-path) comparison (result/check-record (compare/compare-runs baseline candidate - (cond-> {:force (boolean force)} + (cond-> {:allow-build-differences (boolean allow-build-differences) + :allow-scene-differences (boolean allow-scene-differences) + :allow-all-differences (boolean allow-all-differences)} cases (assoc :pairs (compare/selector-pair baseline candidate cases)))))] diff --git a/frontend/performance/benches/render_wasm/node/feature_comparison.cljs b/frontend/performance/benches/render_wasm/node/feature_comparison.cljs new file mode 100644 index 0000000000..2bc37903f9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/feature_comparison.cljs @@ -0,0 +1,732 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.feature-comparison + "Serial feature A/B orchestration and persistence of comparisons." + (:require + ["fs" :as fs] + ["node:timers/promises" :as timers] + ["path" :as path] + [benches.render-wasm.failures :as failures] + [benches.render-wasm.node.artifacts :as artifacts] + [benches.render-wasm.node.build :as build] + [benches.render-wasm.node.catalog :as catalog] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.run :as run] + [benches.render-wasm.node.runs :as runs] + [benches.render-wasm.report.compare :as compare] + [benches.render-wasm.result :as result] + [clojure.set :as set])) + +(def ^:private record-file "feature-comparison.transit") + +(def ^:private warnings + ["Timing values do not determine success." + "Intervals are exploratory; the comparison records no p-value verdicts."]) + +(def default-services + {:prepare-build build/prepare + :collect-catalog catalog/collect + :create-run-home runs/create-home + :execute-run run/execute}) + +(defn- now + [] + (.toISOString (js/Date.))) + +(defn- invalid-home + [message data] + (throw (ex-info message (assoc data :type ::invalid-home)))) + +(defn- check-id + [id] + (when-not (and (string? id) (re-matches #"[A-Za-z0-9][A-Za-z0-9_-]{0,127}" id)) + (invalid-home "invalid feature comparison id" {:id id})) + id) + +(defn create-home + "Creates the feature-comparison home directory + + The default lives under the managed feature-comparisons directory. + An explicit directory resolves from the caller's cwd and must not exist. + Paths inside the managed builds tree are rejected." + [root comparison-id output-dir] + (check-id comparison-id) + (let [default-dir (path/join (artifacts/managed-root root) + "feature-comparisons" comparison-id) + dir (runs/check-output-dir (or output-dir default-dir))] + (when (artifacts/inside-builds? root dir) + (invalid-home "feature comparison cannot use a directory inside builds" {:path dir})) + (try + (fs/mkdirSync dir #js {:recursive true}) + (catch :default cause + (invalid-home "cannot create feature comparison home" + {:path dir :cause (or (ex-message cause) (str cause))}))) + {:dir dir + :record-path (path/join dir record-file)})) + +(defn write-record + "Validates schema:feature-comparison and atomically replaces the saved file." + [home record] + (let [record (result/check-record record)] + (files/atomic-write (:record-path home) (result/encode record)) + record)) + +(defn- permissions + [options] + {:allow-build-differences (boolean (:allow-build-differences options)) + :allow-scene-differences (boolean (:allow-scene-differences options)) + :allow-all-differences (boolean (:allow-all-differences options))}) + +(defn- data-of + [cause] + (try + (ex-data cause) + (catch :default _ + nil))) + +(defn- signal-name + [signal] + (when (and signal (.-aborted ^js signal)) + (let [reason (.-reason ^js signal)] + (cond + (string? reason) reason + (keyword? reason) (name reason) + :else (or (:signal (data-of reason)) "SIGINT"))))) + +(defn- signal-exit + [signal] + (if (= "SIGTERM" signal) 143 130)) + +(defn- message-of + [cause] + (or (ex-message cause) (str cause))) + +(defn- failure-evidence + [phase cause] + {:phase (name phase) + :message (message-of cause) + :cause (failures/describe-cause cause)}) + +(defn- storage-error + [cause] + (ex-info "cannot persist feature comparison" + {:type ::storage-error :cause (message-of cause)} + cause)) + +(defn- storage-error? + [cause] + (loop [cause cause + depth 0] + (cond + (nil? cause) false + (= ::storage-error (:type (data-of cause))) true + (>= depth 8) false + :else (recur (ex-cause cause) (inc depth))))) + +(defn- change-config + [record slug f] + (update record :configurations + (fn [configurations] + (mapv (fn [configuration] + (if (= slug (:slug configuration)) + (f configuration) + configuration)) + configurations)))) + +(defn- change-edge + [record from to f] + (update record :edges + (fn [edges] + (mapv (fn [edge] + (if (and (= from (:from edge)) (= to (:to edge))) + (f edge) + edge)) + edges)))) + +(defn- config-by-slug + [record slug] + (some #(when (= slug (:slug %)) %) (:configurations record))) + +(defn- persist + [state update] + (let [next (assoc (update @(:record-ref state)) :updated-at (now)) + saved (try + (write-record (:home state) next) + (catch :default cause + (throw (storage-error cause))))] + (reset! (:record-ref state) saved) + (reset! (:saved-ref state) saved) + saved)) + +(defn- initial-record + [prepared comparison-id started-at] + (let [plan (:plan prepared) + options (:options prepared)] + {:type :feature-comparison + :comparison-id comparison-id + :started-at started-at + :updated-at started-at + :status :running + :options options + :permissions (permissions options) + :cases (:cases prepared) + :features (:features plan) + :configurations (mapv #(assoc % :status :planned) (:configurations plan)) + :edges (mapv #(assoc % :status :planned) (:edges plan)) + :warnings warnings})) + +(defn- progress + [state text] + (when-let [on-progress (:on-progress state)] + (on-progress text))) + +(defn- fail-config + [state slug phase cause] + (persist state + (fn [record] + (change-config record slug + #(-> % (assoc :status :failed + :failure (failure-evidence phase cause)) + (dissoc :reason)))))) + +(defn- interrupt-config + [state slug signal failure] + (persist state + (fn [record] + (change-config record slug + #(cond-> (assoc % :status :failed :reason signal) + (some? failure) (assoc :failure failure)))))) + +(defn- mark-configs-unattempted + [state from-index reason] + (loop [index from-index] + (when (< index (count (:configurations @(:record-ref state)))) + (let [configuration (nth (:configurations @(:record-ref state)) index)] + (when (= :planned (:status configuration)) + (persist state + (fn [record] + (change-config record (:slug configuration) + #(assoc % :status :unattempted :reason reason))))) + (recur (inc index)))))) + +(defn- mark-edge + [state edge status data] + (persist state + (fn [record] + (change-edge record (:from edge) (:to edge) + #(merge % (assoc data :status status)))))) + +(defn- mark-edges-unattempted + [state reason] + (doseq [edge (:edges @(:record-ref state))] + (when (= :planned (:status edge)) + (mark-edge state edge :unattempted {:reason reason})))) + +(defn- finish + [state status exit-code error interrupted?] + (let [record (persist state #(assoc % :status status :ended-at (now)))] + {:comparison record + :home (:home state) + :exit-code exit-code + :error error + :interrupted? interrupted?})) + +(defn- finish-root + [state error] + (let [record @(:record-ref state) + completed (and (every? #(= :completed (:status %)) (:configurations record)) + (every? #(= :completed (:status %)) (:edges record)))] + (if completed + (finish state :completed 0 error false) + (finish state :failed 1 error false)))) + +(defn- interrupt-comparison + [state signal error] + (let [reason (str "interrupted by " signal)] + (mark-configs-unattempted state 0 reason) + (mark-edges-unattempted state reason) + (finish state :interrupted (signal-exit signal) error true))) + +(defn- ^:async prepare-build + [state slug selection] + (try + (let [selected (await ((:prepare-build (:services state)) + {:root (:root state) + :selection selection + :build-timeout-ms (get-in state [:options :build-timeout-ms]) + :signal (:signal state) + :on-created + (fn [build] + (persist state + (fn [record] + (change-config + record slug + #(-> % + (assoc :status :building + :build (select-keys build + [:id :manifest-path :log-path])) + (dissoc :failure :reason))))))}))] + {:build selected}) + (catch :default cause + (if (storage-error? cause) + (throw cause) + {:error cause})))) + +(defn- load-run + [path] + (when-not (and (string? path) (fs/existsSync path)) + (throw (ex-info "run record is missing" {:type ::missing-run :path path}))) + (let [record (result/decode (fs/readFileSync path "utf8"))] + (when-not (= :run-record (:type record)) + (throw (ex-info "expected a saved run record" {:type ::invalid-run :path path}))) + record)) + +(defn- ^:async execute-config + [state configuration] + (let [slug (:slug configuration) + selection (:selection configuration) + signal (:signal state) + options (:options state) + cases (:cases state) + root (:root state) + home (:home state) + services (:services state)] + (if-let [interrupted (signal-name signal)] + (do + (interrupt-config state slug interrupted nil) + {:interrupted? true :signal interrupted}) + (do + (persist state + (fn [record] + (change-config record slug + #(-> % (assoc :status :building) + (dissoc :failure :reason))))) + (let [prepared (await (prepare-build state slug selection))] + (cond + (:error prepared) + (if-let [interrupted (signal-name signal)] + (do + (interrupt-config state slug interrupted + (failure-evidence :build (:error prepared))) + {:interrupted? true :signal interrupted :error (:error prepared)}) + (do + (fail-config state slug :build (:error prepared)) + {:stop? true + :reason (str "not attempted after build failure in configuration " slug) + :error (:error prepared)})) + + (signal-name signal) + (let [interrupted (signal-name signal)] + (interrupt-config state slug interrupted nil) + {:interrupted? true :signal interrupted}) + + :else + (let [selected (:build prepared) + manifest (:manifest selected) + effective (get-in manifest [:functional :features]) + saved-selection (get-in manifest [:provenance :feature-selection]) + valid-features? (and (vector? effective) (every? string? effective))] + (if-not (and (= selection saved-selection) valid-features?) + (let [cause (ex-info "renderer build manifest disagrees with the planned selection" + {:type ::invalid-build + :selection selection + :saved-selection saved-selection})] + (fail-config state slug :build cause) + {:stop? true + :reason (str "not attempted after build failure in configuration " slug) + :error cause}) + (do + (persist state + (fn [record] + (change-config record slug + #(assoc % :status :ready + :effective-features effective)))) + (progress state (str "comparison: configuration " slug " built\n")) + (let [catalog-result + (try + (let [collected ((:collect-catalog services) + selected + {:master-seed (:seed options) + :filter (:filter options)})] + (if (= collected cases) + {:cases collected} + {:mismatch true})) + (catch :default cause + (if (storage-error? cause) + (throw cause) + {:error cause})))] + (cond + (:error catalog-result) + (if-let [interrupted (signal-name signal)] + (do + (interrupt-config state slug interrupted + (failure-evidence :catalog (:error catalog-result))) + {:interrupted? true :signal interrupted :error (:error catalog-result)}) + (do + (fail-config state slug :catalog (:error catalog-result)) + {:stop? true + :reason (str "not attempted after catalog failure in configuration " slug) + :error (:error catalog-result)})) + + (signal-name signal) + (let [interrupted (signal-name signal)] + (interrupt-config state slug interrupted nil) + {:interrupted? true :signal interrupted}) + + (:mismatch catalog-result) + (let [cause (ex-info "build catalog disagrees with the prepared cases" + {:type ::catalog-mismatch + :configuration slug})] + (fail-config state slug :catalog cause) + {:stop? true + :reason (str "not attempted after catalog mismatch in configuration " slug) + :error cause}) + + :else + (let [run-id (str (random-uuid)) + run-home-result (try + {:home ((:create-run-home services) + {:root root + :run-id run-id + :output-dir (path/join (:dir home) "configs" slug) + :options options})} + (catch :default cause + (if (storage-error? cause) + (throw cause) + {:error cause})))] + (if-let [cause (:error run-home-result)] + (do + (fail-config state slug :run-home cause) + {:stop? true + :reason (str "not attempted after run-home failure in configuration " slug) + :error cause}) + (let [run-home (:home run-home-result)] + (persist state + (fn [record] + (change-config + record slug + #(assoc % :status :running + :run {:run-id (:run-id run-home) + :record-path (:record-path run-home) + :log-path (:log-path run-home)})))) + (progress state (str "comparison: configuration " slug " running\n")) + (let [run-result + (try + (await ((:execute-run services) + root run-home selected cases + (assoc options :build-id (:id selected)) + signal + (fn [text] (progress state text)) + (fn [id] + (progress state + (str "comparison: run " id " started\n"))))) + (catch :default cause + (if (storage-error? cause) + (throw cause) + {:error cause})))] + (if-let [run-record (:run run-result)] + (let [returned-home (:home run-result) + run-home (or returned-home run-home) + exit-code (:exit-code run-result) + termination (get-in run-record [:termination :reason]) + attempts (:attempts run-record) + evidence-error + (or (:storage-error run-result) + (try + (when-not (= (:run-id run-record) + (:run-id (load-run (:record-path run-home)))) + (throw (ex-info "saved run identity disagrees with execution result" + {:type ::invalid-run + :configuration slug}))) + nil + (catch :default cause cause))) + run-error (or evidence-error (:error run-result)) + interrupted (or (signal-name signal) + (= :interrupted termination)) + completed (and (nil? evidence-error) + (= :completed termination) + (zero? exit-code)) + run-failure (cond + (some? evidence-error) + (failure-evidence :run-evidence evidence-error) + + (map? (get-in run-record [:termination :failure])) + (get-in run-record [:termination :failure]) + + (some? run-error) + (failure-evidence :run run-error)) + reason (cond + interrupted + (or (signal-name signal) + (get-in run-record [:termination :signal]) + "SIGINT") + + (not completed) + (when-not run-failure "run did not complete")) + run-ref {:run-id (:run-id run-record) + :record-path (:record-path run-home) + :log-path (:log-path run-home) + :exit-code exit-code}] + (if completed + (do + (persist state + (fn [record] + (change-config + record slug + #(-> % (assoc :status :completed :run run-ref) + (dissoc :failure :reason))))) + (if (empty? attempts) + (let [cause (ex-info "configuration run made no attempts" + {:type ::zero-attempts + :configuration slug})] + {:stop? true + :reason (str "not attempted after a run with no attempts in configuration " + slug) + :error cause}) + (do + (progress state + (str "comparison: configuration " slug " completed\n")) + {:stop? false}))) + (do + (persist state + (fn [record] + (change-config + record slug + #(cond-> (assoc % :status :failed :run run-ref) + (some? reason) (assoc :reason reason) + (some? run-failure) (assoc :failure run-failure))))) + (cond + interrupted + {:interrupted? true :signal reason :error run-error} + + (some? evidence-error) + {:stop? true + :reason (str "not attempted after run evidence failure in configuration " slug) + :error evidence-error} + + :else + (if (empty? attempts) + (let [cause (ex-info "configuration run made no attempts" + {:type ::zero-attempts + :configuration slug})] + {:stop? true + :reason (str "not attempted after a run with no attempts in configuration " + slug) + :error cause}) + (do + (progress state + (str "comparison: configuration " slug " failed\n")) + {:stop? false :error run-error})))))) + (let [cause (ex-info "run executor returned no record" + {:type ::missing-run})] + (fail-config state slug :run cause) + {:stop? true + :reason (str "not attempted after run failure in configuration " slug) + :error cause}))))))))))))))))) + +(defn- sorted-features + [features] + (when-not (and (vector? features) (every? string? features)) + (throw (ex-info "run metadata lacks an effective feature vector" + {:type ::missing-features}))) + (vec (sort (distinct features)))) + +(defn- edge-facts + [baseline candidate edge] + (let [baseline-features (sorted-features + (get-in baseline [:metadata :build :functional :features])) + candidate-features (sorted-features + (get-in candidate [:metadata :build :functional :features])) + baseline-set (set baseline-features) + candidate-set (set candidate-features)] + {:from (:from edge) + :to (:to edge) + :feature (:feature edge) + :baseline-features baseline-features + :candidate-features candidate-features + :added (vec (sort (set/difference candidate-set baseline-set))) + :removed (vec (sort (set/difference baseline-set candidate-set)))})) + +(defn- save-comparison + [path comparison] + (let [record (result/check-record comparison) + text (result/encode record)] + (try + (files/atomic-write path text) + (catch :default cause + (throw (storage-error cause)))))) + +(defn- unacknowledged-count + [comparison] + (reduce + 0 (map (fn [pair] (count (remove :waived? (:mismatches pair)))) + (:pairs comparison)))) + +(defn- analyze-edge + [state edge] + (try + (let [record @(:record-ref state) + baseline-config (config-by-slug record (:from edge)) + candidate-config (config-by-slug record (:to edge)) + baseline (load-run (get-in baseline-config [:run :record-path])) + candidate (load-run (get-in candidate-config [:run :record-path])) + facts (edge-facts baseline candidate edge) + expected (select-keys facts [:feature :baseline-features + :candidate-features :added :removed]) + comparison (compare/compare-runs + baseline candidate + {:allow-build-differences + (get-in record [:permissions :allow-build-differences]) + :allow-scene-differences + (get-in record [:permissions :allow-scene-differences]) + :allow-all-differences + (get-in record [:permissions :allow-all-differences]) + :expected-feature-change expected + :edge facts + :on-unallowed-difference :record}) + path (path/join (get-in state [:home :dir]) + "comparisons" + (str (:from edge) "__" (:to edge) ".transit"))] + (save-comparison path comparison) + {:path path + :unacknowledged (unacknowledged-count comparison)}) + (catch :default cause + (if (storage-error? cause) + {:storage-error cause} + {:error cause})))) + +(defn- ^:async run-edges + [state error] + (if-let [interrupted (signal-name (:signal state))] + (interrupt-comparison state interrupted error) + (let [comparisons (path/join (get-in state [:home :dir]) "comparisons")] + (try + (fs/mkdirSync comparisons #js {:recursive true}) + (catch :default cause + (throw (storage-error cause)))) + (let [outcome + (loop [edges (:edges @(:record-ref state)) + error error] + (await (timers/setImmediate)) + (if-let [interrupted (signal-name (:signal state))] + (do + (mark-edges-unattempted state (str "interrupted by " interrupted)) + {:interrupted interrupted :error error}) + (if-let [edge (first edges)] + (let [result (analyze-edge state edge)] + (cond + (:storage-error result) + (do + (mark-edge state edge :failed + {:failure (failure-evidence :comparison-storage + (:storage-error result))}) + (mark-edges-unattempted state + "not attempted after comparison storage failure") + {:storage (:storage-error result)}) + + (:error result) + (do + (mark-edge state edge :failed + {:failure (failure-evidence :comparison (:error result))}) + (recur (rest edges) (:error result))) + + (pos? (:unacknowledged result)) + (do + (mark-edge state edge :failed + {:path (:path result) + :failure {:unacknowledged-differences (:unacknowledged result)}}) + (recur (rest edges) error)) + + :else + (do + (mark-edge state edge :completed {:path (:path result)}) + (recur (rest edges) error)))) + {:done true :error error})))] + (cond + (:interrupted outcome) + (interrupt-comparison state (:interrupted outcome) (:error outcome)) + + (:storage outcome) + (finish state :failed 1 (:storage outcome) false) + + :else + (finish-root state (:error outcome))))))) + +(defn- ^:async run-comparisons + [state] + (loop [index 0] + (if (>= index (count (:configurations @(:record-ref state)))) + (if-let [interrupted (signal-name (:signal state))] + (interrupt-comparison state interrupted nil) + (run-edges state nil)) + (if-let [interrupted (signal-name (:signal state))] + (interrupt-comparison state interrupted nil) + (let [configuration (nth (:configurations @(:record-ref state)) index) + outcome (await (execute-config state configuration))] + (cond + (:interrupted? outcome) + (interrupt-comparison state (:signal outcome) (:error outcome)) + + (:stop? outcome) + (do + (mark-configs-unattempted state (inc index) (:reason outcome)) + (run-edges state (:error outcome))) + + :else + (recur (inc index)))))))) + +(defn ^:async execute + "Executes a prepared feature comparison and owns builds, runs and reports. + + - prepared: node.feature-plan/prepare output. + - callbacks: accepts :on-created fn(home record), :on-progress fn(string) and + an optional :services map with :prepare-build, :collect-catalog, + :create-run-home and :execute-run overrides. + + Operational failures are returned as :error. The saved record reflects the + last successful atomic write." + [root prepared signal callbacks] + (let [callbacks (or callbacks {}) + services (merge default-services (:services callbacks)) + comparison-id (str (random-uuid)) + home (try + (create-home root comparison-id (:output-dir prepared)) + (catch :default cause + {:error cause}))] + (if-let [cause (:error home)] + {:comparison nil + :home nil + :exit-code 1 + :error cause + :interrupted? false} + (let [started (now) + initial (initial-record prepared comparison-id started) + record-ref (atom initial) + saved-ref (atom nil) + state {:root root + :home home + :options (:options prepared) + :cases (:cases prepared) + :signal signal + :services services + :on-created (:on-created callbacks) + :on-progress (:on-progress callbacks) + :record-ref record-ref + :saved-ref saved-ref}] + (try + (let [saved (write-record home initial)] + (reset! record-ref saved) + (reset! saved-ref saved)) + (when-let [on-created (:on-created state)] + (on-created home @saved-ref)) + (let [result (await (run-comparisons state))] + result) + (catch :default cause + {:comparison @saved-ref + :home home + :exit-code 1 + :error cause + :interrupted? false})))))) diff --git a/frontend/performance/benches/render_wasm/node/feature_plan.cljs b/frontend/performance/benches/render_wasm/node/feature_plan.cljs new file mode 100644 index 0000000000..7b41b839c9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/feature_plan.cljs @@ -0,0 +1,73 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns benches.render-wasm.node.feature-plan + "Plan comparisons based on cargo metadata and cases collected." + (:require + ["fs" :as fs] + ["path" :as path] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.feature-plan :as feature-plan] + [benches.render-wasm.node.options :as options] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.node.runs :as runs] + [clojure.string :as str])) + +(defn- get-root-features + [cwd text] + (let [metadata (js->clj (js/JSON.parse text)) + manifest (fs/realpathSync (path/join cwd "Cargo.toml")) + package (first (filter #(= manifest (fs/realpathSync + (path/resolve cwd (get % "manifest_path")))) + (get metadata "packages")))] + (when-not package + (throw (ex-info "Cargo metadata does not contain the renderer manifest" + {:type ::missing-package :manifest-path manifest}))) + (get package "features"))) + +(defn ^:async prepare + "Accepts the frontend root and resolved CLI options. Reads Cargo metadata in + render-wasm with the inherited environment, checks an optional fresh output + path, and collects selected case data once." + [root opts signal] + (let [opts (merge {:max-features feature-plan/default-max-features + :confirm false :dry-run false :seed 42 + :allow-build-differences false + :allow-scene-differences false + :allow-all-differences false} + opts + {:default-features false}) + plan (feature-plan/plan (:features opts) opts) + output-dir (runs/check-output-dir (:output-dir opts)) + cwd (path/resolve root ".." "render-wasm") + metadata (await (process/run + {:command "cargo" + :args ["metadata" "--offline" "--locked" "--no-deps" "--format-version" "1"] + :cwd cwd + :deadline (process/deadline (:build-timeout-ms opts options/default-build-timeout-ms)) + :signal signal}))] + (feature-plan/check-manifest (:features plan) (get-root-features cwd (:stdout metadata))) + {:plan plan + :cases (cases/collect-cases {:master-seed (:seed opts) :filter (:filter opts)}) + :output-dir output-dir + :options (assoc opts :features (:features plan) :output-dir output-dir)})) + +(defn format-plan + "Formats prepared data without collecting again or reading the filesystem." + [{:keys [plan cases options]}] + (str "Feature comparison plan\n" + "Features: " (str/join ", " (:features plan)) "\n" + "Configurations (Cargo defaults disabled):\n" + (str/join "\n" (map #(str " " (:slug %) ": " + (str/join ", " (get-in % [:selection :features]))) + (:configurations plan))) "\n" + "Edges:\n" + (str/join "\n" (map #(str " " (:from %) " -> " (:to %) ": " (:feature %)) + (:edges plan))) "\n" + "Cases: " (count cases) "\n" + "Attempts per case: " (:warmups options) " warmups, " (:repetitions options) " measured\n" + "Timeouts (ms): attempt " (:timeout-ms options) + ", build " (:build-timeout-ms options) ", startup " (:startup-timeout-ms options) "\n")) diff --git a/frontend/performance/benches/render_wasm/node/run.cljs b/frontend/performance/benches/render_wasm/node/run.cljs index 653b6a5389..77e2df5002 100644 --- a/frontend/performance/benches/render_wasm/node/run.cljs +++ b/frontend/performance/benches/render_wasm/node/run.cljs @@ -67,7 +67,7 @@ (when (and signal (.-aborted ^js signal)) (let [reason (.-reason ^js signal) data (ex-data reason)] - {:signal (or (:signal data) "SIGINT") + {:signal (if (string? reason) reason (or (:signal data) "SIGINT")) :cause reason}))) (defn- interrupted? @@ -76,7 +76,11 @@ (defn- log [state text] - (runs/append-log! (:home state) text) + (try + (runs/append-log! (:home state) text) + (catch :default cause + (reset! (:storage-error-ref state) cause) + (throw cause))) (when-let [progress (:on-progress state)] (progress text))) @@ -90,7 +94,11 @@ (defn- persist [state] - (runs/write-record! (:home state) @(:run-ref state))) + (try + (runs/write-record! (:home state) @(:run-ref state)) + (catch :default cause + (reset! (:storage-error-ref state) cause) + (throw cause)))) (defn- record-preparation [state preparation] @@ -618,9 +626,11 @@ (defn- metadata [root selected-build opts] {:git (git-facts selected-build) - :build {:functional (or (get-in selected-build [:manifest :functional]) {}) - :provenance (assoc (get-in selected-build [:manifest :provenance]) - :build-id (:id selected-build))} + :build (cond-> {:functional (or (get-in selected-build [:manifest :functional]) {}) + :provenance (assoc (get-in selected-build [:manifest :provenance]) + :build-id (:id selected-build))} + (contains? (:manifest selected-build) :environment) + (assoc :environment (get-in selected-build [:manifest :environment]))) :environment (environment root selected-build) :scored? true :diagnostics {:seed (:seed opts) @@ -651,7 +661,8 @@ Options are resolved CLI options. root is the frontend checkout. The record exists before on-created runs, so a browser startup failure still leaves a valid partial record. Returns {:run schema:run :home home :exit-code n - :error cause-or-nil}." + :error cause-or-nil :storage-error cause-or-nil}. A storage error reports any + failed record or log write, including a failure followed by a successful write." [root home selected-build cases options signal on-progress on-created] (let [opts (assoc (normalize-options options) :run-id (:run-id home)) run (result/create-run @@ -670,6 +681,7 @@ :on-progress on-progress :home home :run-ref (atom run) + :storage-error-ref (atom nil) :owner-ref (atom nil) :page-ref (atom nil) :host-ref (atom nil) @@ -732,6 +744,7 @@ exit (if (= :interrupted reason) base (if (or @error + @(:storage-error-ref state) (and (= :completed reason) @(:harness-failed? state))) 1 @@ -739,7 +752,8 @@ {:run run :home home :exit-code exit - :error @error}))))) + :error @error + :storage-error @(:storage-error-ref state)}))))) (defn ^:async run "Prepares or loads a build, collects matching cases and executes one run. diff --git a/frontend/performance/benches/render_wasm/node/runner.cljs b/frontend/performance/benches/render_wasm/node/runner.cljs index c1bebf3e1a..24192dd391 100644 --- a/frontend/performance/benches/render_wasm/node/runner.cljs +++ b/frontend/performance/benches/render_wasm/node/runner.cljs @@ -9,6 +9,7 @@ (:require ["path" :as path] [app.common.transit :as transit] + [benches.render-wasm.feature-plan :as feature-plan] [benches.render-wasm.node.compare :as compare] [benches.render-wasm.node.options :as options] [clojure.string :as str] @@ -49,6 +50,12 @@ [nil "--[no-]default-features" "Enable Cargo defaults" :default true] [nil "--features FEATURES" "Comma-separated functional Cargo features" :parse-fn options/parse-feature-list] + [nil "--max-features N" "ab feature limit (1..6)" + :default feature-plan/default-max-features + :parse-fn parse-number + :validate [#(and (js/Number.isSafeInteger %) (<= 1 % 6)) "must be an integer from 1 to 6"]] + [nil "--confirm" "Confirm ab with >8 configurations or an empty filter" :default false] + [nil "--dry-run" "Validate and print the ab plan without executing" :default false] [nil "--build ID" "Reuse a ready build" :id :build-id :validate nonempty] [nil "--filter SUBSTR" "Select cases whose id contains SUBSTR"] @@ -72,20 +79,40 @@ [nil "--screenshots" "Capture one screenshot per case"] [nil "--only-builds" "Purge build attempts"] [nil "--only-runs" "Purge managed runs"] + [nil "--only-comparisons" "Purge managed feature comparisons"] [nil "--cases A,B" "Select one baseline,candidate pair"] - [nil "--force" "Bypass comparison compatibility checks"]]) + [nil "--allow-build-differences" "Allow recorded build differences"] + [nil "--allow-scene-differences" "Allow scene inputs and fingerprint content differences"] + [nil "--allow-all-differences" "Allow all comparison compatibility differences"]]) (defn usage [summary] (str "Usage: runner build [options]\n" - " runner purge [--only-builds | --only-runs]\n" + " runner purge [--only-builds | --only-runs | --only-comparisons]\n" " runner run [options]\n" - " runner compare BASELINE CANDIDATE [--cases A,B] [--force]\n\n" + " runner ab --features A,B [--filter SUBSTR] [--max-features N]\n" + " [--confirm] [--dry-run] [run and comparison options]\n" + " runner compare BASELINE CANDIDATE [--cases A,B]\n" + " [--allow-build-differences] [--allow-scene-differences]\n" + " [--allow-all-differences]\n\n" "build prints one attempt ID after storage initialization. Exit 0 means\n" "ready; errors retain the build.transit and build.log for that ID.\n" "run prints one run ID on stdout. Progress, failures and the summary go\n" "to stderr. Builds and managed runs live under target/performance/render-wasm.\n" - "compare reads saved schema:run records and starts no build, host or browser.\n\n" + "compare reads saved schema:run records and starts no build, host or browser.\n" + "Build and scene permissions combine; operation and runtime checks stay strict.\n" + "All-differences permits every compatibility difference. Reports retain them.\n\n" + "ab uses every on/off configuration with Cargo defaults disabled; declared\n" + "defaults must be empty. --max-features defaults to 3 (range 1..6).\n" + "Use --confirm above eight configurations or with an empty filter.\n" + "--dry-run checks metadata, cases and output freshness without confirmation,\n" + "output creation, compilation or browser startup. ab accepts run budgets,\n" + "timeouts, seed, filter, output and screenshots, plus comparison permissions.\n" + "It rejects --build, --remove, --cases and --[no-]default-features.\n" + "ab prints the comparison UUID and output path on stdout; progress and\n" + "failures go to stderr. --dry-run prints only the plan without creating\n" + "the comparison home. Managed output lives under\n" + "feature-comparisons//feature-comparison.transit.\n\n" "Preparation and runs require an idle checkout with dependencies installed.\n" "Keep renderer watchers stopped until preparation ends. purge assumes no\n" "experiment is using the managed paths.\n\n" @@ -135,10 +162,14 @@ "--startup-timeout-ms" "--output-dir" "--remove" "--screenshots"}) (def ^:private purge-flags - #{"--only-builds" "--only-runs"}) + #{"--only-builds" "--only-runs" "--only-comparisons"}) + +(def ^:private ab-flags + #{"--max-features" "--confirm" "--dry-run"}) (def ^:private compare-flags - #{"--cases" "--force"}) + #{"--cases" "--allow-build-differences" "--allow-scene-differences" + "--allow-all-differences"}) (defn- invalid-args [message] @@ -150,6 +181,8 @@ (defn- check-command-flags! [command flags] + (when (and (not= command "ab") (some-flags? flags ab-flags)) + (invalid-args "ab planning options require ab")) (let [build? (some-flags? flags build-flags) run? (some-flags? flags run-flags) purge? (some-flags? flags purge-flags) @@ -172,6 +205,13 @@ (when purge? (invalid-args "purge category flags require purge")) (when compare? (invalid-args "compare options require compare"))) + "ab" + (do + (when purge? (invalid-args "purge category flags require purge")) + (when (some-flags? flags ["--build" "--remove" "--cases" + "--default-features" "--no-default-features"]) + (invalid-args "ab rejects --build, --remove, --cases and --[no-]default-features"))) + "compare" (do (when purge? (invalid-args "purge category flags require purge")) @@ -201,19 +241,25 @@ (invalid-args (str/join "\n" errors))) (when-not (:help options) (when-not (and (= 1 (count (take 1 arguments))) - (contains? #{"build" "purge" "run" "compare"} command)) - (invalid-args "expected build, purge, run or compare; try --help")) + (contains? #{"build" "purge" "run" "ab" "compare"} command)) + (invalid-args "expected build, purge, run, ab or compare; try --help")) (check-command-flags! command flags) (check-arguments! command arguments) - (when (and (= command "purge") (:only-builds options) (:only-runs options)) - (invalid-args "--only-builds and --only-runs are mutually exclusive")) + (when (and (= command "purge") + (> (count (filter options [:only-builds :only-runs :only-comparisons])) 1)) + (invalid-args "--only-builds, --only-runs and --only-comparisons are mutually exclusive")) (when (and (= command "run") (:build-id options) (some-flags? flags ["--default-features" "--no-default-features" "--features"])) - (invalid-args "--build cannot be combined with feature-selection flags"))) + (invalid-args "--build cannot be combined with feature-selection flags")) + (when (= command "ab") + (try + (feature-plan/plan (:features options []) options) + (catch :default cause + (invalid-args (ex-message cause)))))) {:command command :arguments arguments - :options options + :options (cond-> options (= command "ab") (assoc :default-features false)) :summary summary})) (defn- print-error @@ -232,7 +278,9 @@ (= command "compare") (println (:text (compare/execute (second arguments) (nth arguments 2) - (select-keys options [:cases :force])))) + (select-keys options [:cases :allow-build-differences + :allow-scene-differences + :allow-all-differences])))) :else (let [^js commands (js/require (path/join js/__dirname "commands.cjs"))] diff --git a/frontend/performance/benches/render_wasm/report/compare.cljs b/frontend/performance/benches/render_wasm/report/compare.cljs index ab5744aee5..996a69ac41 100644 --- a/frontend/performance/benches/render_wasm/report/compare.cljs +++ b/frontend/performance/benches/render_wasm/report/compare.cljs @@ -7,13 +7,13 @@ (ns benches.render-wasm.report.compare "Offline comparisons of paired cases from two saved schema:run records. - By default, comparisons are made between matching cases with equal settings. - It is however possible to to explicitly select pairs allowing different case - definitions or to entirely deactivate compatibility checks." + Matching case identifiers select default pairs; explicit pairs only select + cases. Permissions acknowledge recorded differences without hiding them." (:require [benches.render-wasm.report.analysis :as analysis] [benches.render-wasm.report.summarize :as summary] [benches.render-wasm.result :as result] + [clojure.set :as set] [clojure.string :as str])) (defn resolve-case @@ -31,7 +31,7 @@ {:type ::unknown-case :selector selector :available (mapv result/case-name ids)}))))) (defn selector-pair - "Resolves the prototype CLI's single A,B selector into the extensible pair list. + "Resolves a single A,B comparison selector into an extensible pair list. A belongs to baseline; B belongs to candidate. Force is a separate option." [baseline candidate selector] (let [names (str/split selector #"," -1)] @@ -55,10 +55,15 @@ [run id field] (set (map field (preparations run id)))) -(defn- execution-settings - "Selects declared execution inputs; descriptive text remains provenance." +(defn- scene-settings + "Selects scene identity and generation inputs, separate from execution." [definition] - (select-keys definition [:scene :scene-seed :params :view :context :completion + (select-keys definition [:scene :scene-seed :params])) + +(defn- execution-settings + "Selects operation and measurement settings that scene permission cannot ignore." + [definition] + (select-keys definition [:view :context :completion :batch-size :preparation :operation])) (defn- evidence-differences @@ -70,6 +75,20 @@ [{:field [field :incomplete-or-varying-evidence] :baseline baseline :candidate candidate :waived? false}]))) +(defn- feature-names? + "Checks the vector-of-strings shape of recorded or expected feature names." + [features] + (and (vector? features) (every? string? features))) + +(defn- get-effective-feature-evidence + "Returns comparison values and eligibility for automatic feature acknowledgment. + Missing, nil and explicitly empty features remain distinct. Collections compare + as sets, but only saved vectors of strings qualify for automatic acknowledgment." + [run] + (let [features (get-in run [:metadata :build :functional :features] ::missing-features)] + {:value (if (coll? features) (set features) features) + :valid (feature-names? features)})) + (defn- pair-differences "Checks saved inputs, graphics, environment, build, sampling and metric definitions." [baseline candidate b c shared] @@ -78,11 +97,16 @@ (vec (concat (keep identity - [(difference :execution (execution-settings bdef) (execution-settings cdef)) + [(difference :scene (scene-settings bdef) (scene-settings cdef)) + (difference :execution (execution-settings bdef) (execution-settings cdef)) (difference :environment (get-in baseline [:metadata :environment]) (get-in candidate [:metadata :environment])) - (difference :functional-build (get-in baseline [:metadata :build :functional]) - (get-in candidate [:metadata :build :functional])) + (difference :effective-features (:value (get-effective-feature-evidence baseline)) + (:value (get-effective-feature-evidence candidate))) + (difference :functional-build (dissoc (get-in baseline [:metadata :build :functional]) :features) + (dissoc (get-in candidate [:metadata :build :functional]) :features)) + (difference :build-environment (get-in baseline [:metadata :build :environment]) + (get-in candidate [:metadata :build :environment])) (difference :warmups (get-in baseline [:plan :warmups]) (get-in candidate [:plan :warmups])) (difference :timeout-policy (get-in baseline [:plan :timeout-policy]) @@ -99,40 +123,102 @@ (evidence-differences [:metric metric-id] (set bdefs) (set cdefs)))) shared))))) +(defn- feature-change-evidence? + "Checks feature facts against saved effective features and requested selections. + Both selections must record one off-to-on requested toggle with equal default + policy. Effective transitive additions/removals remain unrestricted, but must + match the supplied facts." + [baseline candidate {:keys [feature baseline-features candidate-features added removed]}] + (let [b (get-effective-feature-evidence baseline) + c (get-effective-feature-evidence candidate) + b-request (get-in baseline [:metadata :build :provenance :feature-selection]) + c-request (get-in candidate [:metadata :build :provenance :feature-selection])] + (and (:valid b) (:valid c) + (string? feature) + (every? feature-names? + [baseline-features candidate-features added removed]) + (map? b-request) (map? c-request) + (boolean? (:default-features? b-request)) + (boolean? (:default-features? c-request)) + (= (:default-features? b-request) (:default-features? c-request)) + (feature-names? (:features b-request)) + (feature-names? (:features c-request)) + (not (contains? (set (:features b-request)) feature)) + (= (conj (set (:features b-request)) feature) (set (:features c-request))) + (= (:value b) (set baseline-features)) + (= (:value c) (set candidate-features)) + (= (set/difference (:value c) (:value b)) (set added)) + (= (set/difference (:value b) (:value c)) (set removed))))) + +(defn- expected-feature-change? + [baseline candidate expected] + (and (feature-change-evidence? baseline candidate expected) + (contains? (set (:added expected)) (:feature expected)))) + +(defn- acknowledge-difference + "Retains a mismatch and records its applicable rule, if any. + Automatic feature evidence and scoped permissions take precedence over all. + Fingerprint content is scene data; incomplete/varying evidence is not." + [{:keys [field] :as mismatch} permissions expected-feature-change?] + (let [rule (cond + (and (= :effective-features field) expected-feature-change?) :feature-toggle + (and (:allow-build-differences permissions) + (contains? #{:effective-features :functional-build :build-environment} field)) :build + (and (:allow-scene-differences permissions) + (contains? #{:scene :fingerprint} field)) :scene + (:allow-all-differences permissions) :all)] + (cond-> mismatch + rule (assoc :waived? true :rule rule)))) + (defn- side-report "Reports one side with saved definitions, setup evidence and sample accounting." - [run id shared analysis force] + [run id shared analysis allow-all-differences] {:case-id id :definition (some #(when (= id (:id %)) %) (get-in run [:plan :cases])) :preparations (preparations run id) :remaining (get-in run [:remaining id]) :failed-attempts (summary/get-failed-attempts run id) :metrics (into {} (map (fn [metric-id] - [metric-id (summary/summarize-metric run id metric-id analysis force)])) shared)}) + [metric-id (summary/summarize-metric run id metric-id analysis allow-all-differences)])) shared)}) + +(defn- metric-changes + "Keeps diagnostic changes unavailable for differing or varying definitions. + Only all-differences permits comparing their valid raw values without conversion." + [baseline candidate b c id mismatches allow-all-differences analysis] + (if (and (not allow-all-differences) + (some #(contains? #{[:metric id] [[:metric id] :incomplete-or-varying-evidence]} + (:field %)) mismatches)) + {:absolute {:reason :inconsistent-metric-definitions} + :percentage {:reason :inconsistent-metric-definitions} + :absolute-ci {:reason :inconsistent-metric-definitions} + :percentage-ci {:reason :inconsistent-metric-definitions}} + (analysis/changes (:values (summary/observations baseline b id)) + (:values (summary/observations candidate c id)) analysis))) (defn- compare-pair "Compares shared numeric observations after resolving the compatibility policy." - [baseline candidate [b c] explicit? force analysis] - (let [b (resolve-case baseline b) - c (resolve-case candidate c) - candidate-ids (set (summary/metric-ids candidate c)) - shared (filterv #(contains? candidate-ids %) (summary/metric-ids baseline b)) - mismatches (pair-differences baseline candidate b c shared)] + [baseline candidate [b c] explicit? permissions expected-feature-change? on-unallowed-difference analysis] + (let [b (resolve-case baseline b) + c (resolve-case candidate c) + candidate-ids (set (summary/metric-ids candidate c)) + shared (filterv #(contains? candidate-ids %) (summary/metric-ids baseline b)) + mismatches (mapv #(acknowledge-difference % permissions expected-feature-change?) + (pair-differences baseline candidate b c shared)) + allow-all-differences (:allow-all-differences permissions)] (when (empty? shared) (throw (ex-info "selected cases have no shared metric identifiers" {:type ::no-shared-metrics :baseline b :candidate c}))) - (when (and (not force) (seq mismatches)) + (when (and (not= :record on-unallowed-difference) (some #(not (:waived? %)) mismatches)) (throw (ex-info "incompatible saved runs" {:type ::incompatible :baseline b :candidate c :mismatches mismatches}))) {:explicit? explicit? :mismatches mismatches - :baseline (side-report baseline b shared analysis force) - :candidate (side-report candidate c shared analysis force) + :baseline (side-report baseline b shared analysis allow-all-differences) + :candidate (side-report candidate c shared analysis allow-all-differences) :metrics (into {} (map (fn [id] - [id (analysis/changes (:values (summary/observations baseline b id)) - (:values (summary/observations candidate c id)) - analysis)])) + [id (metric-changes baseline candidate b c id mismatches + allow-all-differences analysis)])) shared)})) (defn- provenance @@ -143,32 +229,63 @@ (defn compare-runs "Creates a schema:comparison from constructed or decoded schema:run records. - Inputs must already satisfy schema:run. Encoding validates the - returned comparison at the boundary. + Inputs must already satisfy schema:run. Encoding validates the returned + comparison at the boundary. Default pairs use identical identifiers. Explicit pairs select cases without changing compatibility checks. - `:pairs` accepts an extensible vector of explicit [baseline candidate] pairs. - - `:force` allows all compatibility mismatches, retaining them in the report. - Only shared metrics are compared, using raw values without conversions. + - `:allow-build-differences` acknowledges build settings and effective features. + - `:allow-scene-differences` acknowledges scene inputs and fingerprint content. + It never acknowledges incomplete or varying evidence or execution settings. + - `:allow-all-differences` acknowledges any compatibility mismatch and permits + mixed-definition statistics on valid raw observations, without conversions. + Scoped permissions combine by union. Every mismatch retains its values, + legacy `:waived?` field and, only when acknowledged, its `:rule`. + - `:expected-feature-change` is planner evidence with string `:feature` and + vectors `:baseline-features`, `:candidate-features`, `:added`, `:removed`. + Automatic acknowledgment requires saved effective vectors of strings, exact + sets/deltas with that feature added, and saved requested selections differing + only by that toggle, with the same recorded default-feature policy. + - `:edge` attaches checked feature-comparison context: `:from`, `:to`, and the five expected + feature fields. When both options are supplied, their feature facts must agree. + The ordinary encoding boundary validates the saved edge's shape. + - Internal `:on-unallowed-difference :record` returns diagnostic reports instead + of throwing for unacknowledged differences. It keeps strict statistics. - `:analysis` resolves from method defaults, independent of each run's defaults." ([baseline candidate] (compare-runs baseline candidate {})) - ([baseline candidate {:keys [pairs force analysis]}] - (let [explicit? (some? pairs) + ([baseline candidate {:keys [pairs allow-build-differences allow-scene-differences + allow-all-differences expected-feature-change edge + on-unallowed-difference analysis]}] + (let [explicit? (some? pairs) candidate-ids (set (map :id (get-in candidate [:plan :cases]))) - pairs (or pairs (->> (get-in baseline [:plan :cases]) - (map :id) - (filter #(contains? candidate-ids %)) - (mapv #(vector % %)))) - analysis (analysis/resolve-settings analysis)] + pairs (or pairs (->> (get-in baseline [:plan :cases]) + (map :id) + (filter #(contains? candidate-ids %)) + (mapv #(vector % %)))) + analysis (analysis/resolve-settings analysis) + permissions {:allow-build-differences (boolean allow-build-differences) + :allow-scene-differences (boolean allow-scene-differences) + :allow-all-differences (boolean allow-all-differences)} + expected? (expected-feature-change? baseline candidate expected-feature-change)] + (when (and (some? edge) (not (feature-change-evidence? baseline candidate edge))) + (throw (ex-info "edge context disagrees with saved run feature evidence" + {:type ::invalid-edge :edge edge}))) + (when (and (some? edge) (some? expected-feature-change) + (not= expected-feature-change + (select-keys edge [:feature :baseline-features :candidate-features :added :removed]))) + (throw (ex-info "edge context disagrees with expected feature facts" + {:type ::invalid-edge :edge edge :expected-feature-change expected-feature-change}))) (when (or (empty? pairs) (not (every? #(and (vector? %) (= 2 (count %))) pairs))) (throw (ex-info "comparison needs saved case pairs" {:type ::invalid-pairs :baseline (mapv (comp result/case-name :id) (get-in baseline [:plan :cases])) :candidate (mapv (comp result/case-name :id) (get-in candidate [:plan :cases]))}))) - {:type :comparison :analysis analysis - :baseline (provenance baseline) :candidate (provenance candidate) - :forced? (boolean force) - :source-coverage summary/source-coverage :interpretation summary/interpretation - :pairs (mapv #(compare-pair baseline candidate % explicit? force analysis) pairs)}))) + (cond-> {:type :comparison :analysis analysis + :baseline (provenance baseline) :candidate (provenance candidate) + :permissions permissions + :source-coverage summary/source-coverage :interpretation summary/interpretation + :pairs (mapv #(compare-pair baseline candidate % explicit? permissions expected? + on-unallowed-difference analysis) pairs)} + (some? edge) (assoc :edge edge))))) diff --git a/frontend/performance/benches/render_wasm/report/format.cljs b/frontend/performance/benches/render_wasm/report/format.cljs index 1c5d6b4064..438d4c305c 100644 --- a/frontend/performance/benches/render_wasm/report/format.cljs +++ b/frontend/performance/benches/render_wasm/report/format.cljs @@ -8,7 +8,7 @@ "Text formatting for run summaries and saved comparisons. Text leads with observed median, bootstrap interval and distinct counts. - Full statistics and forced differences retain units and observation bases." + Full statistics and acknowledged differences retain units and observation bases." (:require [benches.render-wasm.result :as result] [clojure.string :as str])) @@ -83,10 +83,17 @@ (apply str (map (fn [[id metric]] (str "\n" (metric->text id metric))) metrics)))) (defn format-comparison - "Formats changes, exploratory intervals and every recorded mismatch. - Forced output displays both saved definitions and original metric definitions." - [{:keys [baseline candidate analysis forced? source-coverage interpretation pairs]}] - (str (when forced? "FORCED COMPARISON: compatibility differences bypassed\n") + "Formats selected permissions, acknowledged/unacknowledged differences and changes. + Both saved definitions remain visible. Edge labels describe actual feature + evidence, including equal effective sets, without predicting a timing effect." + [{:keys [baseline candidate analysis permissions edge source-coverage interpretation pairs]}] + (str "Permissions " (pr-str permissions) "\n" + (when edge + (str "Edge " (:from edge) " -> " (:to edge) " feature " (:feature edge) + (when (= (set (:baseline-features edge)) (set (:candidate-features edge))) + " (effective feature sets identical)") + "\n effective features " (pr-str (select-keys edge [:baseline-features :candidate-features :added :removed])) + "\n")) (when (some #(get-in % [:effective-graphics :software]) (mapcat (fn [pair] (concat (get-in pair [:baseline :preparations]) (get-in pair [:candidate :preparations]))) pairs)) @@ -100,7 +107,10 @@ (map (fn [{:keys [baseline candidate mismatches metrics]}] (str "\n\nPair " (result/case-name (:case-id baseline)) "," (result/case-name (:case-id candidate)) - (when (seq mismatches) (str "\nMISMATCHES " (pr-str mismatches))) + (apply str + (map (fn [mismatch] + (str "\n" (if (:waived? mismatch) "ACKNOWLEDGED" "UNACKNOWLEDGED") + " DIFFERENCE " (pr-str mismatch))) mismatches)) (side->text "Baseline" baseline) (side->text "Candidate" candidate) (str/join "\n" diff --git a/frontend/performance/benches/render_wasm/report/summarize.cljs b/frontend/performance/benches/render_wasm/report/summarize.cljs index d0963c4d7e..44721d475e 100644 --- a/frontend/performance/benches/render_wasm/report/summarize.cljs +++ b/frontend/performance/benches/render_wasm/report/summarize.cljs @@ -83,16 +83,18 @@ (case-attempts run case-id))) (defn summarize-metric - "Computes statistics for one dynamic metric with its definitions and counts." + "Computes statistics for one dynamic metric with its definitions and counts. + The optional allow-all-differences flag permits mixed-definition statistics; + scoped comparison permissions never change this summary policy." ([run case-id metric-id analysis] (summarize-metric run case-id metric-id analysis false)) - ([run case-id metric-id analysis force?] + ([run case-id metric-id analysis allow-all-differences] (let [{:keys [values counts]} (observations run case-id metric-id) - definitions (definitions run case-id metric-id) - mixed? (> (count (observation-definitions run case-id metric-id)) 1)] + definitions (definitions run case-id metric-id) + mixed? (> (count (observation-definitions run case-id metric-id)) 1)] {:definitions definitions :counts counts :mixed-definitions? mixed? - :statistics (if (and mixed? (not force?)) + :statistics (if (and mixed? (not allow-all-differences)) {:reason :inconsistent-metric-definitions :count (count values)} (analysis/statistics values analysis))}))) diff --git a/frontend/performance/benches/render_wasm/result.cljs b/frontend/performance/benches/render_wasm/result.cljs index f67f427016..c6423af73a 100644 --- a/frontend/performance/benches/render_wasm/result.cljs +++ b/frontend/performance/benches/render_wasm/result.cljs @@ -10,6 +10,7 @@ [app.common.schema :as sm] [app.common.transit :as transit] [benches.render-wasm.codec :as codec] + [benches.render-wasm.feature-plan :as feature-plan] [benches.render-wasm.report.analysis :as analysis])) (def schema:basis @@ -70,7 +71,10 @@ "Git, functional build settings, environment and diagnostic provenance." [:map {:closed true} [:git [:map [:sha [:maybe string?]] [:dirty [:maybe boolean?]]]] - [:build [:map [:functional map?] [:provenance {:optional true} map?]]] + [:build [:map + [:functional map?] + [:provenance {:optional true} map?] + [:environment {:optional true} [:map-of :string :string]]]] [:environment map?] [:scored? boolean?] [:diagnostics {:optional true} map?]]) @@ -152,21 +156,77 @@ [:run-id some?] [:started-at string?] [:ended-at [:maybe string?]] [:metadata schema:metadata] [:termination schema:termination]]) +(def schema:permissions + [:map {:closed true} + [:allow-build-differences boolean?] + [:allow-scene-differences boolean?] + [:allow-all-differences boolean?]]) + +(def schema:feature-comparison + "Saved feature comparison progress. Case data and options remain ordinary + extensible maps; configuration/edge states and retained references are typed." + [:map {:closed true} + [:type [:= :feature-comparison]] + [:comparison-id string?] + [:started-at string?] + [:updated-at string?] + [:ended-at {:optional true} string?] + [:status [:enum :planned :running :completed :failed :interrupted]] + [:options :map] + [:permissions schema:permissions] + [:cases [:vector [:map [:id some?]]]] + [:features [:vector string?]] + [:configurations + [:vector + [:map + [:slug string?] + [:selection [:map {:closed true} + [:default-features? boolean?] + [:features [:vector string?]]]] + [:status [:enum :planned :building :ready :running :completed :failed :unattempted]] + [:build {:optional true} [:map [:id string?] [:manifest-path string?] [:log-path string?]]] + [:run {:optional true} [:map [:run-id string?] [:record-path string?] [:log-path string?] + [:exit-code {:optional true} :int]]] + [:effective-features {:optional true} [:vector string?]] + [:failure {:optional true} :map] + [:reason {:optional true} string?]]]] + [:edges + [:vector + [:map + [:from string?] [:to string?] [:feature string?] + [:status [:enum :planned :completed :failed :unattempted]] + [:path {:optional true} string?] + [:failure {:optional true} :map] + [:reason {:optional true} string?]]]] + [:warnings [:vector string?]]]) + (def schema:comparison - "schema:comparison retains resolved analysis, differences and per-side reports." + "Resolved analysis, selected permissions, every mismatch and per-side reports. + Every acknowledged difference has a rule; unacknowledged differences have none. + Optional edge context records requested toggles and Cargo's actual effective + sets/deltas, including aliases." [:map {:closed true} [:type [:= :comparison]] [:analysis analysis/schema:settings] [:baseline schema:provenance] [:candidate schema:provenance] - [:forced? boolean?] + [:permissions schema:permissions] + [:edge {:optional true} + [:map {:closed true} + [:from string?] [:to string?] [:feature string?] + [:baseline-features [:vector string?]] [:candidate-features [:vector string?]] + [:added [:vector string?]] [:removed [:vector string?]]]] [:source-coverage string?] [:interpretation string?] [:pairs [:vector [:map {:closed true} [:explicit? boolean?] - [:mismatches [:vector [:map {:closed true} - [:field :any] [:baseline :any] [:candidate :any] [:waived? boolean?]]]] + [:mismatches [:vector + [:and + [:map {:closed true} + [:field :any] [:baseline :any] [:candidate :any] [:waived? boolean?] + [:rule {:optional true} [:enum :feature-toggle :build :scene :all]]] + [:fn #(= (:waived? %) (contains? % :rule))]]]] [:baseline schema:comparison-side] [:candidate schema:comparison-side] [:metrics [:map-of :any [:map {:closed true} [:absolute schema:estimate] [:percentage schema:estimate] @@ -322,14 +382,107 @@ (check-completion run termination) (assoc run :ended-at ended-at :termination termination)) +(defn- check-feature-configuration + "Requires evidence that matches each saved configuration's progress." + [{:keys [status build run effective-features failure reason]}] + (when (#{:ready :running :completed} status) + (require-valid! (some? build) "feature comparison configuration needs a build" {:status status})) + (when (#{:ready :completed} status) + (require-valid! (some? effective-features) + "feature comparison configuration needs effective features" {:status status})) + (when (#{:running :completed} status) + (require-valid! (some? run) "feature comparison configuration needs a run" {:status status})) + (when (= :completed status) + (require-valid! (some? (:exit-code run)) + "completed feature comparison configuration needs a run exit code" {}) + (require-valid! (zero? (:exit-code run)) + "completed feature comparison configuration needs a successful run" {})) + (when (= :failed status) + (require-valid! (or (some? failure) (some? reason)) + "failed feature comparison configuration needs a failure or reason" {})) + (when (= :unattempted status) + (require-valid! (some? reason) + "unattempted feature comparison configuration needs a reason" {}))) + +(defn- check-feature-edge + "Requires a path for completed analysis and a reason for unattempted work." + [{:keys [status path failure reason]}] + (case status + :completed + (require-valid! (some? path) "completed feature comparison edge needs a report path" {}) + + :failed + (require-valid! (or (some? failure) (some? reason)) + "failed feature comparison edge needs a failure or reason" {}) + + :unattempted + (require-valid! (some? reason) + "unattempted feature comparison edge needs a reason" {}) + + nil)) + +(defn check-feature-comparison + "Validates saved feature-comparison progress against its planned experiment. + Configurations and edges must match the pure planner after status evidence is + attached. Terminal records cannot retain active work, and completed work must + retain its build, run, effective-feature and report evidence." + [record] + (require-valid! (sm/validate schema:feature-comparison record) + "invalid schema:feature-comparison" {}) + (let [features (:features record) + configurations (:configurations record) + edges (:edges record) + expected (feature-plan/plan features + {:max-features (count features) + :dry-run true + :confirm true}) + terminal? (contains? #{:completed :failed :interrupted} (:status record)) + active? (comp #{:planned :building :ready :running} :status)] + (require-valid! (= features (:features expected)) + "feature comparison features disagree with normalized selection" {}) + (require-valid! (= (mapv #(select-keys % [:slug :selection]) configurations) + (:configurations expected)) + "feature comparison configurations disagree with planned features" {}) + (require-valid! (= (mapv #(select-keys % [:from :to :feature]) edges) + (:edges expected)) + "feature comparison edges disagree with planned configurations" {}) + (require-valid! (= (count (:cases record)) (count (set (map :id (:cases record))))) + "duplicate feature comparison case" {}) + (when terminal? + (require-valid! (some? (:ended-at record)) + "terminal feature comparison needs an end time" {}) + (require-valid! (not-any? active? configurations) + "terminal feature comparison still has active configurations" {}) + (require-valid! (not-any? active? edges) + "terminal feature comparison still has active edges" {})) + (when-not terminal? + (require-valid! (nil? (:ended-at record)) + "unfinished feature comparison has an end time" {})) + (when (= :planned (:status record)) + (require-valid! (every? (comp #{:planned} :status) configurations) + "planned feature comparison has started configurations" {}) + (require-valid! (every? (comp #{:planned} :status) edges) + "planned feature comparison has started edges" {})) + (when (= :completed (:status record)) + (require-valid! (every? (comp #{:completed} :status) configurations) + "completed feature comparison has unfinished configurations" {}) + (require-valid! (every? (comp #{:completed} :status) edges) + "completed feature comparison has unfinished edges" {})) + (doseq [configuration configurations] + (check-feature-configuration configuration)) + (doseq [edge edges] + (check-feature-edge edge))) + record) + (defn check-record - "Validates schema:run or schema:comparison at a Transit boundary." + "Validates a run, comparison or feature-comparison at a Transit boundary." [record] (case (:type record) :run-record (check-run record) :comparison (do (require-valid! (sm/validate schema:comparison record) "invalid schema:comparison" {}) record) + :feature-comparison (check-feature-comparison record) (throw (ex-info "unsupported benchmark format" {:type ::unsupported-format})))) (defn encode diff --git a/frontend/test/performance_tests/render_wasm/artifacts_test.cljs b/frontend/test/performance_tests/render_wasm/artifacts_test.cljs index ac827bd2bd..d8b2ce3ec1 100644 --- a/frontend/test/performance_tests/render_wasm/artifacts_test.cljs +++ b/frontend/test/performance_tests/render_wasm/artifacts_test.cljs @@ -241,6 +241,23 @@ (t/is (= ::artifacts/unsupported-manifest (:type (ex-data (failure #(artifacts/load-build @temporary-root (:id build))))))))) +(t/deftest build-environment-roundtrips-without-inventing-missing-evidence + (t/is (not (contains? (:manifest (create-build)) :environment))) + (doseq [environment [{} {"CFLAGS" "" "RUSTFLAGS" "-C\u001fopt-level=3"}]] + (let [build (complete-files (create-build)) + ready (artifacts/write-manifest build (assoc (:manifest build) + :status :ready :environment environment)) + saved (artifacts/load-build @temporary-root (:id ready))] + (t/is (= environment (get-in saved [:manifest :environment])))))) + +(t/deftest build-environment-rejects-non-string-keys-and-values + (doseq [environment [nil [] {:CFLAGS ""} {"CFLAGS" nil} {"CFLAGS" 0}]] + (let [build (create-build) + saved (fs/readFileSync (:manifest-path build) "utf8") + cause (failure #(artifacts/write-manifest build (assoc (:manifest build) :environment environment)))] + (t/is (= ::artifacts/invalid-manifest (:type (ex-data cause)))) + (t/is (= saved (fs/readFileSync (:manifest-path build) "utf8")))))) + (t/deftest ids-cannot-select-or-delete-outside-paths (let [build (ready-build) outside (path/join @temporary-root "outside.txt")] @@ -329,6 +346,60 @@ (t/is (= "unmanaged" (fs/readFileSync outside "utf8"))) (artifacts/purge root category)))) +(t/deftest comparison-purge-unlinks-only-feature-comparison-evidence + (doseq [category [:comparisons :all]] + (let [root (fs/mkdtempSync (path/join @temporary-root (str (name category) "-"))) + managed (artifacts/managed-root root) + build-file (path/join managed "builds" "saved" "build.transit") + ab-dir (path/join managed "feature-comparisons" "ab-1") + config-dir (path/join ab-dir "configs" "00") + edge-dir (path/join ab-dir "comparisons") + run-dir (path/join managed "runs" "saved") + outside (path/join root "unmanaged" "feature-comparison.transit") + outputs [["feature-comparison.transit" ab-dir] + ["run.log" config-dir] + ["run.transit" config-dir] + ["shot.PNG" config-dir] + ["00__10.transit" edge-dir]] + kept [["notes.txt" ab-dir] + ["keep.txt" config-dir] + ["keep.txt" edge-dir]]] + (doseq [[file dir] outputs + :let [path (path/join dir file)]] + (write-file path "comparison output")) + (doseq [[file dir] kept] + (write-file (path/join dir file) "unrelated")) + (write-file (path/join managed "feature-comparisons" "keep.txt") + "unrelated root") + (write-file (path/join managed "feature-comparisons" "ab-1" "directory.PNG" + "keep.txt") + "keep directory") + (write-file build-file "retained build") + (write-file (path/join run-dir "run.transit") "standalone run") + (write-file outside "unmanaged") + (artifacts/purge root category) + (doseq [[file dir] outputs] + (t/is (not (fs/existsSync (path/join dir file))) (str category " " file))) + (doseq [[file dir] kept] + (t/is (fs/existsSync (path/join dir file)) (str category " " file))) + (t/is (= "unrelated root" + (fs/readFileSync (path/join managed "feature-comparisons" "keep.txt") + "utf8"))) + (t/is (fs/existsSync (path/join ab-dir "directory.PNG" "keep.txt"))) + (t/is (.isDirectory (fs/statSync ab-dir))) + (t/is (.isDirectory (fs/statSync config-dir))) + (t/is (.isDirectory (fs/statSync edge-dir))) + (t/is (= (= category :comparisons) + (fs/existsSync (path/join run-dir "run.transit"))) + (str category " standalone run")) + (t/is (= (= category :comparisons) (fs/existsSync build-file)) + (str category " retained build")) + (when (= category :comparisons) + (t/is (= "standalone run" + (fs/readFileSync (path/join run-dir "run.transit") "utf8"))) + (t/is (= "retained build" (fs/readFileSync build-file "utf8")))) + (t/is (= "unmanaged" (fs/readFileSync outside "utf8")))))) + (t/deftest purge-follows-category-and-nested-directory-symlinks-and-stops-cycles (let [managed (artifacts/managed-root @temporary-root)] (fs/mkdirSync managed #js {:recursive true}) diff --git a/frontend/test/performance_tests/render_wasm/build_test.cljs b/frontend/test/performance_tests/render_wasm/build_test.cljs index 15ec9ca827..e004c6ff4b 100644 --- a/frontend/test/performance_tests/render_wasm/build_test.cljs +++ b/frontend/test/performance_tests/render_wasm/build_test.cljs @@ -118,6 +118,61 @@ (t/is false (str "Unexpected test error: " cause)) (done)))) +(t/deftest ^:async renderer-facts-retain-string-environment-keys-and-missing-evidence + (let [bin (path/join @temporary-root "bin") + original-path (.. js/process -env -PATH)] + (fs/mkdirSync bin) + (fs/writeFileSync (path/join bin "clojure") "#!/bin/sh\nexit 24\n" #js {:mode 493}) + (try + (set! (.. js/process -env -PATH) (str bin path/delimiter original-path)) + (doseq [environment [::missing {} {"RUSTFLAGS" "-C\u001fopt-level=3" "CFLAGS" ""}]] + (let [facts (cond-> {:features [] :profile {:opt_level "3" :debug_assertions false}} + (not= ::missing environment) (assoc :environment environment))] + (install-renderer + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, " + (js/JSON.stringify (js/JSON.stringify (clj->js facts))) ");\n")) + (let [cause (await (rejected (build/prepare {:root (frontend-root) + :build-timeout-ms 10000 + :on-created record-created}))) + manifest (read-manifest (retained-build cause))] + (t/is (= :cljs (:phase (ex-data cause)))) + (t/is (= 24 (:exit-code (ex-data (ex-cause cause))))) + (if (= ::missing environment) + (t/is (not (contains? manifest :environment))) + (t/is (= environment (:environment manifest))))))) + (finally + (set! (.. js/process -env -PATH) original-path))))) + +(t/deftest ^:async explicit-selection-preserves-disabled-cargo-defaults + (let [bin (path/join @temporary-root "bin") + original-path (.. js/process -env -PATH)] + (fs/mkdirSync bin) + (fs/writeFileSync (path/join bin "clojure") "#!/bin/sh\nexit 24\n" #js {:mode 493}) + (try + (set! (.. js/process -env -PATH) (str bin path/delimiter original-path)) + (install-renderer + (str "const fs = require('node:fs');\n" + "fs.writeFileSync(process.env.RENDER_WASM_BUILD_INFO, " + (js/JSON.stringify + (js/JSON.stringify + (clj->js {:features ["a"] + :profile {:opt_level "3" :debug_assertions false}}))) + ");\n")) + (let [cause (await (rejected (build/prepare + {:root (frontend-root) + :selection {:default-features? false :features ["a"]} + :build-timeout-ms 10000 + :on-created record-created}))) + manifest (read-manifest (retained-build cause))] + (t/is (= :cljs (:phase (ex-data cause)))) + (t/is (= {:default-features? false :features ["a"]} + (get-in manifest [:provenance :feature-selection]))) + (t/is (= ["a"] (get-in manifest [:functional :features]))) + (t/is (false? (get-in manifest [:functional :default-features?])))) + (finally + (set! (.. js/process -env -PATH) original-path))))) + (t/deftest real-git-outside-a-repository-has-unknown-provenance (t/async done (complete diff --git a/frontend/test/performance_tests/render_wasm/cli_integration.cjs b/frontend/test/performance_tests/render_wasm/cli_integration.cjs index 0210d8965d..0beb422e43 100644 --- a/frontend/test/performance_tests/render_wasm/cli_integration.cjs +++ b/frontend/test/performance_tests/render_wasm/cli_integration.cjs @@ -97,8 +97,11 @@ function createFrontend(t) { managed, runner, compiler: path.join(renderer, "build"), + rendererRoot: renderer, builds: path.join(managed, "builds"), runs: path.join(managed, "runs"), + comparisons: path.join(managed, "feature-comparisons"), + env: {}, }; } @@ -113,6 +116,7 @@ async function runCli(fixture, args) { timeout: 15000, killSignal: "SIGKILL", maxBuffer: 1024 * 1024, + env: { ...process.env, ...fixture.env }, }, ); return { exitCode: 0, ...output }; @@ -172,11 +176,73 @@ function seedRuns(fixture) { } } +function seedComparisons(fixture) { + const id = randomUUID(); + const directory = path.join(fixture.comparisons, id); + const config = path.join(directory, "configs", "00"); + const edges = path.join(directory, "comparisons"); + fs.mkdirSync(config, { recursive: true }); + fs.mkdirSync(edges, { recursive: true }); + fs.mkdirSync(path.join(config, "shot.PNG"), { recursive: true }); + fs.writeFileSync( + path.join(directory, "feature-comparison.transit"), + "comparison\n", + ); + fs.writeFileSync(path.join(directory, "keep.txt"), "keep\n"); + for (const file of ["run.transit", "run.log", "case-load.PNG"]) { + fs.writeFileSync(path.join(config, file), "configuration output\n"); + } + fs.writeFileSync(path.join(config, "keep.txt"), "keep\n"); + fs.writeFileSync(path.join(config, "shot.PNG", "keep.txt"), "keep\n"); + for (const file of ["00__10.transit", "keep.txt"]) { + fs.writeFileSync(path.join(edges, file), "edge output\n"); + } + fs.writeFileSync(path.join(fixture.comparisons, "keep.txt"), "keep\n"); +} + +function installCargoMetadata(fixture, definitions) { + const manifest = path.join(fixture.rendererRoot, "Cargo.toml"); + const bin = path.join(path.dirname(fixture.rendererRoot), "fixture-bin"); + const metadata = path.join(fixture.rendererRoot, "metadata.json"); + const cargo = path.join(bin, "cargo"); + fs.writeFileSync( + manifest, + "[package]\nname = 'fixture'\nversion = '0.1.0'\n", + ); + fs.writeFileSync( + path.join(fixture.rendererRoot, "Cargo.lock"), + "version = 4\n", + ); + fs.mkdirSync(bin, { recursive: true }); + fs.writeFileSync( + metadata, + JSON.stringify({ + packages: [{ manifest_path: manifest, features: definitions }], + }), + ); + fs.writeFileSync( + cargo, + `#!/usr/bin/env node\nconst fs = require("node:fs");\nprocess.stdout.write(fs.readFileSync(process.env.CARGO_METADATA_FIXTURE, "utf8"));\n`, + { mode: 0o755 }, + ); + fixture.env.PATH = `${bin}${path.delimiter}${process.env.PATH}`; + fixture.env.CARGO_METADATA_FIXTURE = metadata; + return cargo; +} + function runFixtureText({ runId, caseId, value, environment = { node: "v24" }, + mode = "release", + features = [], + scene = "case", + sceneSeed = 7, + count = 10, + fingerprint = "a".repeat(64), + operation = "load", + operationArgs = { distance: 10 }, }) { const k = transit.keyword; const map = (pairs) => transit.map(pairs); @@ -200,7 +266,7 @@ function runFixtureText({ k("case-id"), k(caseId), k("fingerprint"), - map([k("algorithm"), k("sha-256"), k("digest"), "a".repeat(64)]), + map([k("algorithm"), k("sha-256"), k("digest"), fingerprint]), k("effective-graphics"), map([k("webgl2"), true]), k("diagnostics"), @@ -234,7 +300,30 @@ function runFixtureText({ k("plan"), map([ k("cases"), - [map([k("id"), k(caseId)])], + [ + map([ + k("id"), + k(caseId), + k("scene"), + k(scene), + k("scene-seed"), + sceneSeed, + k("params"), + map([k("count"), count]), + k("operation"), + map([ + k("id"), + k(operation), + k("args"), + map( + Object.entries(operationArgs).flatMap(([key, entry]) => [ + k(key), + entry, + ]), + ), + ]), + ]), + ], k("warmups"), 0, k("repetitions"), @@ -247,7 +336,10 @@ function runFixtureText({ k("git"), map([k("sha"), runId, k("dirty"), false]), k("build"), - map([k("functional"), map([k("mode"), k("release")])]), + map([ + k("functional"), + map([k("mode"), k(mode), k("features"), features]), + ]), k("environment"), map( Object.entries(environment).flatMap(([key, entry]) => [k(key), entry]), @@ -371,12 +463,78 @@ test( }, ); +test( + "ab dry-run prints the Gray plan without creating comparison output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [], b: ["a"] }); + const retained = tree(fixture.managed); + + const result = await runCli(fixture, [ + "ab", + "--features", + "b,a", + "--filter", + "rects/load", + "--dry-run", + ]); + + assert.equal(result.exitCode, 0); + assert.equal(result.stderr, ""); + assert.match(result.stdout, /^Feature comparison plan/); + for (const part of [ + "Features: a, b", + "00: ", + "01: b", + "11: a, b", + "10: a", + "00 -> 10: a", + "00 -> 01: b", + "01 -> 11: a", + "10 -> 11: b", + "Cases: 1", + ]) { + assert.ok(result.stdout.includes(part), part); + } + assert.equal(fs.existsSync(fixture.comparisons), false); + assert.deepEqual(tree(fixture.managed), retained); + }, +); + +test( + "ab validates requested features before creating comparison output", + testOptions, + async (t) => { + const fixture = createFrontend(t); + installCargoMetadata(fixture, { default: [], a: [] }); + const retained = tree(fixture.managed); + + const result = await runCli(fixture, [ + "ab", + "--features", + "missing", + "--filter", + "rects/load", + "--dry-run", + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /unknown requested Cargo features/); + assert.equal(fs.existsSync(fixture.comparisons), false); + assert.deepEqual(tree(fixture.managed), retained); + }, +); + test("purge deletes only the selected artifacts", testOptions, async (t) => { const fixture = createFrontend(t); const retained = tree(fixture.managed); seedBuilds(fixture); seedRuns(fixture); + seedComparisons(fixture); const runs = tree(fixture.runs); + const comparisons = tree(fixture.comparisons); assert.deepEqual(await runCli(fixture, ["purge", "--only-builds"]), { exitCode: 0, @@ -389,9 +547,10 @@ test("purge deletes only the selected artifacts", testOptions, async (t) => { ]); } assert.deepEqual(tree(fixture.runs), runs); + assert.deepEqual(tree(fixture.comparisons), comparisons); assert.deepEqual( tree(fixture.managed).filter( - ([name]) => !["runs", "builds"].includes(name), + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), ), retained, ); @@ -407,14 +566,51 @@ test("purge deletes only the selected artifacts", testOptions, async (t) => { assert.deepEqual(fs.readdirSync(path.join(fixture.runs, id)), ["keep.txt"]); } assert.deepEqual(tree(fixture.builds), builds); + assert.deepEqual(tree(fixture.comparisons), comparisons); assert.deepEqual( tree(fixture.managed).filter( - ([name]) => !["runs", "builds"].includes(name), + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), + ), + retained, + ); + + const buildsBeforeComparisonPurge = tree(fixture.builds); + const runsBeforeComparisonPurge = tree(fixture.runs); + assert.deepEqual(await runCli(fixture, ["purge", "--only-comparisons"]), { + exitCode: 0, + stdout: "", + stderr: "", + }); + const [comparisonId] = fs.readdirSync(fixture.comparisons); + const comparison = path.join(fixture.comparisons, comparisonId); + for (const file of [ + "feature-comparison.transit", + "configs/00/run.transit", + "configs/00/run.log", + "configs/00/case-load.PNG", + "comparisons/00__10.transit", + ]) { + assert.equal(fs.existsSync(path.join(comparison, file)), false, file); + } + for (const file of [ + "keep.txt", + "configs/00/keep.txt", + "configs/00/shot.PNG/keep.txt", + "comparisons/keep.txt", + ]) { + assert.equal(fs.existsSync(path.join(comparison, file)), true, file); + } + assert.deepEqual(tree(fixture.builds), buildsBeforeComparisonPurge); + assert.deepEqual(tree(fixture.runs), runsBeforeComparisonPurge); + assert.deepEqual( + tree(fixture.managed).filter( + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), ), retained, ); seedRuns(fixture); + seedComparisons(fixture); assert.deepEqual(await runCli(fixture, ["purge"]), { exitCode: 0, stdout: "", @@ -425,9 +621,23 @@ test("purge deletes only the selected artifacts", testOptions, async (t) => { assert.deepEqual(fs.readdirSync(path.join(root, id)), ["keep.txt"]); } } + for (const id of fs.readdirSync(fixture.comparisons)) { + const directory = path.join(fixture.comparisons, id); + if (id === "keep.txt") continue; + for (const file of [ + "feature-comparison.transit", + "configs/00/run.transit", + "configs/00/run.log", + "configs/00/case-load.PNG", + "comparisons/00__10.transit", + ]) { + assert.equal(fs.existsSync(path.join(directory, file)), false, file); + } + assert.equal(fs.existsSync(path.join(directory, "keep.txt")), true); + } assert.deepEqual( tree(fixture.managed).filter( - ([name]) => !["runs", "builds"].includes(name), + ([name]) => !["runs", "builds", "feature-comparisons"].includes(name), ), retained, ); @@ -535,6 +745,8 @@ test( for (const args of [ ["purge", "--only-builds", "--only-runs"], + ["purge", "--only-builds", "--only-comparisons"], + ["purge", "--only-runs", "--only-comparisons"], ["purge", "--only-ready-builds"], ["purge", "--only-builds", "ready"], ]) { @@ -662,9 +874,36 @@ test( const help = await runCli(fixture, ["--help"]); assert.equal(help.exitCode, 0, help.stderr); assert.match(help.stdout, /runner compare/); + const abHelp = await runCli(fixture, ["ab", "--help"]); + assert.equal(abHelp.exitCode, 0, abHelp.stderr); + assert.match(abHelp.stdout, /runner ab --features/); + assert.match(abHelp.stdout, /--only-comparisons/); + for (const args of [["ab"], ["ab", "--dry-run"]]) { + const invalid = await runCli(fixture, args); + assert.equal(invalid.exitCode, 1, invalid.stderr); + assert.match(invalid.stderr, /at least one requested feature/); + assert.doesNotMatch( + invalid.stderr, + /execution dependency|Cannot find module/, + ); + } const comparison = await runCli(fixture, ["compare", baseline, candidate]); assert.equal(comparison.exitCode, 0, comparison.stderr); assert.match(comparison.stdout, /median change/); + for (const flag of [ + "--allow-build-differences", + "--allow-scene-differences", + "--allow-all-differences", + ]) { + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + flag, + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /median change/); + } assert.equal(fs.existsSync(path.join(temporary, "commands.cjs")), false); }, ); @@ -682,6 +921,10 @@ test( assert.match(result.stdout, /runner run/); assert.match(result.stdout, /runner compare/); assert.match(result.stdout, /--cases/); + assert.match(result.stdout, /--allow-build-differences/); + assert.match(result.stdout, /--allow-scene-differences/); + assert.match(result.stdout, /--allow-all-differences/); + assert.doesNotMatch(result.stdout, /--force/); assert.deepEqual(tree(fixture.managed), original); }, ); @@ -714,7 +957,7 @@ test( ); test( - "compare supports explicit pairs and force from the CLI", + "compare supports explicit pairs and all-differences from the CLI", testOptions, async (t) => { const fixture = createFrontend(t); @@ -743,17 +986,20 @@ test( assert.equal(incompatible.exitCode, 1); assert.deepEqual(tree(fixture.managed), original); - const forced = await runCli(fixture, [ - "compare", - baseline, - candidate, - "--cases", - "a/pan,b/pan", - "--force", + const selected = ["compare", baseline, candidate, "--cases", "a/pan,b/pan"]; + const strict = await runCli(fixture, selected); + assert.equal(strict.exitCode, 1); + assert.match(strict.stderr, /incompatible saved runs/); + + const allowed = await runCli(fixture, [ + ...selected, + "--allow-all-differences", ]); - assert.equal(forced.exitCode, 0); - assert.match(forced.stdout, /FORCED COMPARISON/); - assert.match(forced.stdout, /Pair a\/pan,b\/pan/); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /:allow-all-differences true/); + assert.match(allowed.stdout, /ACKNOWLEDGED DIFFERENCE/); + assert.match(allowed.stdout, /:rule :all/); + assert.match(allowed.stdout, /Pair a\/pan,b\/pan/); assert.deepEqual(tree(fixture.managed), original); fs.writeFileSync( @@ -769,10 +1015,164 @@ test( "compare", baseline, candidate, - "--force", + "--allow-all-differences", ]); - assert.equal(automatic.exitCode, 0); - assert.match(automatic.stdout, /FORCED COMPARISON/); + assert.equal(automatic.exitCode, 0, automatic.stderr); + assert.match(automatic.stdout, /:allow-all-differences true/); + assert.match(automatic.stdout, /ACKNOWLEDGED DIFFERENCE/); assert.match(automatic.stdout, /Pair a\/pan,a\/pan/); }, ); + +test( + "compare applies build and scene permissions separately and together", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + + const build = "--allow-build-differences"; + const scene = "--allow-scene-differences"; + for (const scenario of [ + { + changes: { mode: "debug", features: ["fixture"] }, + allowed: [build], + denied: [[], [scene]], + fields: ["functional-build", "effective-features"], + rules: ["build"], + }, + { + changes: { + scene: "other", + sceneSeed: 8, + count: 20, + fingerprint: "b".repeat(64), + }, + allowed: [scene], + denied: [[], [build]], + fields: ["scene", "fingerprint"], + rules: ["scene"], + }, + { + changes: { mode: "debug", count: 20 }, + allowed: [build, scene], + denied: [[], [build], [scene]], + fields: ["functional-build", "scene"], + rules: ["build", "scene"], + }, + ]) { + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "case/load", + value: 12, + ...scenario.changes, + }), + ); + const original = tree(fixture.managed); + for (const flags of scenario.denied) { + const rejected = await runCli(fixture, [ + "compare", + baseline, + candidate, + ...flags, + ]); + assert.equal(rejected.exitCode, 1, rejected.stderr); + assert.match(rejected.stderr, /incompatible saved runs/); + } + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + ...scenario.allowed, + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.match(allowed.stdout, /:allow-all-differences false/); + assert.match(allowed.stdout, /ACKNOWLEDGED DIFFERENCE/); + for (const flag of scenario.allowed) { + assert.ok(allowed.stdout.includes(`:${flag.slice(2)} true`)); + } + for (const field of scenario.fields) { + assert.ok(allowed.stdout.includes(`:field :${field}`)); + } + for (const rule of scenario.rules) { + assert.ok(allowed.stdout.includes(`:rule :${rule}`)); + } + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test( + "scoped compare permissions keep operation arguments and runtime environment strict", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const baseline = path.join(fixture.managed, "baseline.transit"); + const candidate = path.join(fixture.managed, "candidate.transit"); + fs.writeFileSync( + baseline, + runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }), + ); + for (const [field, changes] of [ + ["execution", { operation: "pan" }], + ["execution", { operationArgs: { distance: 20 } }], + ["environment", { environment: { node: "v25" } }], + ]) { + fs.writeFileSync( + candidate, + runFixtureText({ + runId: "candidate", + caseId: "case/load", + value: 12, + ...changes, + }), + ); + const original = tree(fixture.managed); + const rejected = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-build-differences", + "--allow-scene-differences", + ]); + assert.equal(rejected.exitCode, 1); + assert.match(rejected.stderr, /incompatible saved runs/); + assert.ok(rejected.stderr.includes(`:field :${field}`)); + + const allowed = await runCli(fixture, [ + "compare", + baseline, + candidate, + "--allow-all-differences", + ]); + assert.equal(allowed.exitCode, 0, allowed.stderr); + assert.ok(allowed.stdout.includes(`:field :${field}`)); + assert.match(allowed.stdout, /:rule :all/); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + +test("compare rejects --force without an alias", testOptions, async (t) => { + const fixture = createFrontend(t); + const original = tree(fixture.managed); + for (const flags of [["--force"], ["--force", "--allow-all-differences"]]) { + const rejected = await runCli(fixture, [ + "compare", + "missing-baseline.transit", + "missing-candidate.transit", + ...flags, + ]); + assert.equal(rejected.exitCode, 1); + assert.match(rejected.stderr, /Unknown option.*--force/); + assert.doesNotMatch(rejected.stderr, /ENOENT/); + assert.deepEqual(tree(fixture.managed), original); + } +}); diff --git a/frontend/test/performance_tests/render_wasm/compare_test.cljs b/frontend/test/performance_tests/render_wasm/compare_test.cljs index 10f1c2e014..d2014834e3 100644 --- a/frontend/test/performance_tests/render_wasm/compare_test.cljs +++ b/frontend/test/performance_tests/render_wasm/compare_test.cljs @@ -45,7 +45,7 @@ :preparation {}}) (defn- make-run - [run-id case-id value & {:keys [environment functional fingerprint scene] + [run-id case-id value & {:keys [environment functional fingerprint scene case-settings] :or {environment {:node "v24" :browser "Fake"} functional {:mode :release} fingerprint (apply str (repeat 64 "a")) @@ -54,7 +54,7 @@ run (result/create-run {:run-id run-id :started-at "2026-10-26T00:00:00.000Z" - :plan {:cases [(case-definition case-id scene)] + :plan {:cases [(merge (case-definition case-id scene) case-settings)] :warmups 0 :repetitions 1 :timeout-policy {:attempt-ms 30000}} @@ -103,6 +103,11 @@ (let [{:keys [comparison text]} (compare/execute "baseline.transit" "candidate.transit" {})] (t/is (= :comparison (:type comparison))) + (t/is (= {:allow-build-differences false + :allow-scene-differences false + :allow-all-differences false} + (:permissions comparison))) + (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) (t/is (= 2 (get-in comparison [:pairs 0 :metrics :elapsed :absolute :value]))) (t/is (str/includes? text "Baseline")) (t/is (str/includes? text "Candidate")) @@ -120,21 +125,88 @@ {:cases "a/pan,b/pan"})] (t/is (str/includes? text "Pair a/pan,b/pan")))) -(t/deftest cases-requires-force-for-different-scenes +(t/deftest cases-requires-scene-permission-for-different-scenes (let [baseline (write-run (path/join @temporary-root "baseline.transit") (make-run "baseline" :a/load 10)) candidate (write-run (path/join @temporary-root "candidate.transit") - (make-run "candidate" :b/load 12 :scene :other)) + (make-run "candidate" :b/load 12 :scene :other + :fingerprint (apply str (repeat 64 "b")))) selector {:cases "a/load,b/load"}] (t/is (= ::report-compare/incompatible (:type (ex-data (failure #(compare/execute baseline candidate selector)))))) + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate + (assoc selector :allow-build-differences true))))))) (let [{:keys [comparison text]} - (compare/execute baseline candidate (assoc selector :force true))] - (t/is (true? (:forced? comparison))) - (t/is (str/includes? text "MISMATCHES")) + (compare/execute baseline candidate (assoc selector :allow-scene-differences true))] + (t/is (= {:allow-build-differences false + :allow-scene-differences true + :allow-all-differences false} + (:permissions comparison))) + (t/is (= #{:scene :fingerprint} + (set (map :field (get-in comparison [:pairs 0 :mismatches]))))) + (t/is (every? #(and (:waived? %) (= :scene (:rule %))) + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) (t/is (str/includes? text "Pair a/load,b/load"))))) -(t/deftest force-bypasses-differences-and-displays-them +(t/deftest build-permission-allows-build-differences-only + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :functional {:mode :debug :features ["fixture"]}))] + (doseq [options [{} {:allow-scene-differences true}]] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate options))))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate {:allow-build-differences true})] + (t/is (= {:allow-build-differences true + :allow-scene-differences false + :allow-all-differences false} + (:permissions comparison))) + (t/is (= #{:functional-build :effective-features} + (set (map :field (get-in comparison [:pairs 0 :mismatches]))))) + (t/is (every? #(and (:waived? %) (= :build (:rule %))) + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? text ":rule :build"))))) + +(t/deftest build-and-scene-permissions-combine + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 :scene :other + :functional {:mode :debug}))] + (doseq [options [{:allow-build-differences true} {:allow-scene-differences true}]] + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate options))))))) + (let [{:keys [comparison]} + (compare/execute baseline candidate {:allow-build-differences true + :allow-scene-differences true})] + (t/is (= {:allow-build-differences true + :allow-scene-differences true + :allow-all-differences false} + (:permissions comparison))) + (t/is (= {:scene :scene :functional-build :build} + (into {} (map (juxt :field :rule)) + (get-in comparison [:pairs 0 :mismatches]))))))) + +(t/deftest scoped-permissions-keep-operation-and-runtime-checks-strict + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10))] + (doseq [[field settings] + [[:execution {:case-settings {:operation {:id :pan :args {:distance 20}}}}] + [:environment {:environment {:node "v25" :browser "Fake"}}]]] + (let [candidate (write-run (path/join @temporary-root "candidate.transit") + (apply make-run "candidate" :case/load 12 (mapcat identity settings))) + cause (failure #(compare/execute baseline candidate + {:allow-build-differences true + :allow-scene-differences true}))] + (t/is (= ::report-compare/incompatible (:type (ex-data cause)))) + (t/is (= [field] (mapv :field (:mismatches (ex-data cause))))) + (t/is (every? (comp not :waived?) (:mismatches (ex-data cause)))))))) + +(t/deftest all-permission-allows-runtime-differences-and-displays-them (let [baseline (write-run (path/join @temporary-root "baseline.transit") (make-run "baseline" :case/load 10)) candidate (write-run (path/join @temporary-root "candidate.transit") @@ -142,9 +214,22 @@ :environment {:node "v25" :browser "Fake"}))] (t/is (= ::report-compare/incompatible (:type (ex-data (failure #(compare/execute baseline candidate {})))))) - (let [{:keys [text]} (compare/execute baseline candidate {:force true})] - (t/is (str/starts-with? text "FORCED COMPARISON")) - (t/is (str/includes? text "MISMATCHES"))))) + (t/is (= ::report-compare/incompatible + (:type (ex-data (failure #(compare/execute baseline candidate {:force true})))))) + (let [{:keys [comparison text]} + (compare/execute baseline candidate {:allow-all-differences true})] + (t/is (= {:allow-build-differences false + :allow-scene-differences false + :allow-all-differences true} + (:permissions comparison))) + (t/is (= [{:field :environment + :baseline {:node "v24" :browser "Fake"} + :candidate {:node "v25" :browser "Fake"} + :waived? true :rule :all}] + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/starts-with? text "Permissions ")) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) + (t/is (str/includes? text ":rule :all"))))) (t/deftest invalid-selectors-and-unreadable-records-fail-at-the-boundary (let [baseline (write-run (path/join @temporary-root "baseline.transit") @@ -161,10 +246,11 @@ {}))) (fs/writeFileSync (path/join @temporary-root "malformed.transit") (transit/encode-str (dissoc (make-run "malformed" :case/load 12) :plan))) - (t/is (= ::result/invalid-record - (:type (ex-data (failure #(compare/execute baseline - (path/join @temporary-root "malformed.transit") - {})))))) + (doseq [options [{} {:allow-all-differences true}]] + (t/is (= ::result/invalid-record + (:type (ex-data (failure #(compare/execute baseline + (path/join @temporary-root "malformed.transit") + options))))))) (t/is (thrown? js/Error (compare/execute baseline (path/join @temporary-root "missing.transit") {}))))) diff --git a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs new file mode 100644 index 0000000000..9e439d8f08 --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs @@ -0,0 +1,822 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns performance-tests.render-wasm.feature-comparison-test + (:require + ["fs" :as fs] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [benches.render-wasm.cases :as cases] + [benches.render-wasm.declarations :as decl] + [benches.render-wasm.feature-plan :as feature-plan] + [benches.render-wasm.node.feature-comparison :as feature-comparison] + [benches.render-wasm.node.feature-plan :as node-plan] + [benches.render-wasm.node.files :as files] + [benches.render-wasm.node.process :as process] + [benches.render-wasm.result :as result] + [cljs.test :as t :include-macros true] + [clojure.string :as str] + [frontend-tests.helpers.mock :as mock])) + +(def ^:private temporary-root (atom nil)) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-feature-plan-"))) + :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) + +(defn- frontend-root [] (path/join @temporary-root "frontend")) +(defn- renderer-root [] (path/join @temporary-root "render-wasm")) + +(defn- write-cargo-fixture + [] + (fs/mkdirSync (frontend-root)) + (fs/mkdirSync (renderer-root)) + (fs/writeFileSync (path/join (renderer-root) "Cargo.toml") + (str "[package]\nname = 'feature-plan-fixture'\nversion = '0.1.0'\nedition = '2021'\n" + "[lib]\npath = 'lib.rs'\n[features]\ndefault = []\na = []\nb = ['a']\n")) + (fs/writeFileSync (path/join (renderer-root) "lib.rs") "") + (fs/writeFileSync (path/join (renderer-root) "Cargo.lock") + "version = 4\n[[package]]\nname = 'feature-plan-fixture'\nversion = '0.1.0'\n")) + +(defn- planning-options + [] + {:features ["b" "a"] :filter "rects/load" :seed 42 :dry-run true + :warmups 0 :repetitions 1 :timeout-ms 2000 + :build-timeout-ms 10000 :startup-timeout-ms 3000}) + +(defn- metadata-output + [definitions] + {:stdout (js/JSON.stringify + (clj->js {"packages" [{"manifest_path" (path/join (frontend-root) "Cargo.toml") + "features" {"default" ["wrong-package"]}} + {"manifest_path" (path/join (renderer-root) "Cargo.toml") + "features" definitions}]}))}) + +(defn- error-message + [promise] + (.then promise (fn [_] nil) (fn [cause] (ex-message cause)))) + +(t/deftest one-feature-has-an-explicit-off-and-on-selection + (t/is (= {:features ["a"] + :configurations [{:slug "0" :selection {:default-features? false :features []}} + {:slug "1" :selection {:default-features? false :features ["a"]}}] + :edges [{:from "0" :to "1" :feature "a"}]} + (feature-plan/plan ["a"] {:filter "rects/load"})))) + +(t/deftest two-feature-order-and-all-edges-are-msb-first + (t/is (= {:features ["a" "b"] + :configurations [{:slug "00" :selection {:default-features? false :features []}} + {:slug "01" :selection {:default-features? false :features ["b"]}} + {:slug "11" :selection {:default-features? false :features ["a" "b"]}} + {:slug "10" :selection {:default-features? false :features ["a"]}}] + :edges [{:from "00" :to "10" :feature "a"} + {:from "00" :to "01" :feature "b"} + {:from "01" :to "11" :feature "a"} + {:from "10" :to "11" :feature "b"}]} + (feature-plan/plan ["b" "a" "a"] {:filter "rects/load"})))) + +(t/deftest three-feature-order-and-complete-graph + (let [{:keys [configurations edges]} (feature-plan/plan ["c" "a" "b"] {:filter "rects/load"}) + by-slug (into {} (map (juxt :slug #(set (get-in % [:selection :features])))) configurations)] + (t/is (= ["000" "001" "011" "010" "110" "111" "101" "100"] (mapv :slug configurations))) + (t/is (= 12 (count (set edges)))) + (doseq [{:keys [from to feature]} edges] + (t/is (not (contains? (by-slug from) feature))) + (t/is (= (conj (by-slug from) feature) (by-slug to)))))) + +(t/deftest upper-limit-still-emits-every-edge-once + (let [{:keys [configurations edges]} + (feature-plan/plan ["a" "b" "c" "d" "e" "f"] {:max-features 6 :confirm true})] + (t/is (= 64 (count (set configurations)))) + (t/is (= 192 (count (set edges)))) + (t/is (every? false? (map #(get-in % [:selection :default-features?]) configurations))))) + +(t/deftest invalid-inputs-fail-before-planning + (doseq [features [nil '("a") "a" [nil] [1] [""] ["a b"] ["a/b"] ["a.b"] [" a"]]] + (t/is (thrown? js/Error (feature-plan/plan features {:dry-run true})) (pr-str features))) + (t/is (thrown-with-msg? js/Error #"at least one" (feature-plan/plan [] {:dry-run true}))) + (doseq [limit [nil 0 7 -1 1.5 "3" js/NaN js/Infinity]] + (t/is (thrown-with-msg? js/Error #"integer from 1 to 6" + (feature-plan/plan ["a"] {:max-features limit :dry-run true})))) + (doseq [opts [{:confirm nil} {:confirm 1} {:dry-run nil} {:dry-run "true"} {:filter 1}]] + (t/is (thrown-with-msg? js/Error #"invalid feature planning options" + (feature-plan/plan ["a"] opts)))) + (t/is (thrown-with-msg? js/Error #"exceed max-features" + (feature-plan/plan ["a" "b" "c" "d"] {:dry-run true})))) + +(t/deftest confirmation-applies-to-execution-only + (doseq [filter [nil "" " "]] + (t/is (thrown-with-msg? js/Error #"requires --confirm" (feature-plan/plan ["a"] {:filter filter}))) + (t/is (= 2 (count (:configurations (feature-plan/plan ["a"] {:filter filter :dry-run true})))))) + (t/is (= 8 (count (:configurations (feature-plan/plan ["a" "b" "c"] {:filter "rects"}))))) + (t/is (thrown-with-msg? js/Error #"requires --confirm" + (feature-plan/plan ["a" "b" "c" "d"] {:max-features 4 :filter "rects"}))) + (doseq [opts [{:confirm true} {:dry-run true}]] + (t/is (= 16 (count (:configurations + (feature-plan/plan ["a" "b" "c" "d"] (assoc opts :max-features 4)))))))) + +(t/deftest manifest-validation-leaves-activation-to-cargo + (t/is (= ["a" "b"] (feature-plan/check-manifest ["b" "a"] + {"default" [] "a" [] "b" ["a" "dep:library"]}))) + (t/is (= ["a"] (feature-plan/check-manifest ["a"] {"a" []}))) + (t/is (thrown-with-msg? js/Error #"unknown requested" + (feature-plan/check-manifest ["missing"] {"default" [] "a" []}))) + (t/is (thrown-with-msg? js/Error #"empty declared Cargo defaults" + (feature-plan/check-manifest ["a"] {"default" ["a"] "a" []}))) + (doseq [definitions [nil [] {"a" "b"} {:a []} {"a" [1]}]] + (t/is (thrown-with-msg? js/Error #"invalid Cargo feature definitions" + (feature-plan/check-manifest ["a"] definitions)))) + (doseq [feature ["stats" "profile" "profile-macros" "profile-raf"]] + (t/is (thrown-with-msg? js/Error #"diagnostic Cargo features" + (feature-plan/plan [feature] {:dry-run true}))) + (t/is (thrown-with-msg? js/Error #"diagnostic Cargo features" + (feature-plan/check-manifest [feature] {"default" [] feature []})))) + ;; A declared alias is allowed here. Actual Cargo-reported diagnostics are + ;; rejected later by build validation, not by a second activation resolver. + (t/is (= ["alias"] (feature-plan/check-manifest ["alias"] {"default" [] "alias" ["stats"]})))) + +(t/deftest ^:async real-offline-metadata-and-collection-create-no-output + (write-cargo-fixture) + (let [output (path/join @temporary-root "new" "comparison") + opts (assoc (planning-options) :output-dir output) + before (vec (fs/readdirSync (renderer-root))) + ready (await (node-plan/prepare (frontend-root) opts nil))] + (t/is (= ["a" "b"] (get-in ready [:plan :features]))) + (t/is (= (cases/collect-cases {:master-seed 42 :filter "rects/load"}) (:cases ready))) + (t/is (= (assoc opts :features ["a" "b"] :default-features false :max-features 3 :confirm false + :allow-build-differences false :allow-scene-differences false :allow-all-differences false) + (:options ready))) + (t/is (= output (:output-dir ready))) + (t/is (not (fs/existsSync (path/join @temporary-root "new")))) + (t/is (= before (vec (fs/readdirSync (renderer-root))))) + (t/is (empty? (fs/readdirSync (frontend-root)))) + (let [text (node-plan/format-plan ready)] + (doseq [part ["a, b" "00" "01" "11" "10" "00 -> 10: a" "Cases: 1" + "0 warmups, 1 measured" "attempt 2000, build 10000, startup 3000"]] + (t/is (str/includes? text part)))))) + +(t/deftest ^:async metadata-selects-the-package-by-manifest-not-array-position + (write-cargo-fixture) + (fs/writeFileSync (path/join (frontend-root) "Cargo.toml") "decoy") + (await + (mock/with-mocks* + {process/run (fn [request] + (t/is (= "cargo" (:command request))) + (t/is (= ["metadata" "--offline" "--locked" "--no-deps" "--format-version" "1"] + (:args request))) + (t/is (= (renderer-root) (:cwd request))) + (t/is (not (contains? request :env))) + (js/Promise.resolve (metadata-output {"default" [] "a" [] "b" ["a"]})))} + (let [ready (await (node-plan/prepare (frontend-root) (planning-options) nil))] + (t/is (= ["a" "b"] (get-in ready [:plan :features]))))))) + +(t/deftest ^:async collected-custom-case-data-and-order-survive-planning + (write-cargo-fixture) + (decl/register-scene! {:id :feature-plan-fixture :ns "feature-comparison-test" + :description "Planning data fixture" + :params-schema [:map [:count :int]] + :build (fn [_] (throw (js/Error. "planning must not construct a scene")))}) + (doseq [id [:feature-plan-fixture/second :feature-plan-fixture/first]] + (decl/register-case! {:id id :scene :feature-plan-fixture :ns "feature-comparison-test" + :params {:count 17} :view {:scale 2 :x 10 :y -5} :context :reuse + :completion :await :batch-size 7 + :operation {:id :custom/work :args {:amount 3}} + :preparation {:reset :outside-timing :data [1 2]} + :run! (fn [_] (throw (js/Error. "planning must not run an operation")))})) + (try + (let [ready (await (node-plan/prepare (frontend-root) + (assoc (planning-options) :filter "feature-plan-fixture/") nil)) + data (:cases ready)] + (t/is (= [:feature-plan-fixture/second :feature-plan-fixture/first] (mapv :id data))) + (t/is (= (cases/collect-cases {:master-seed 42 :filter "feature-plan-fixture/"}) data)) + (t/is (= {:params {:count 17} :view {:scale 2 :x 10 :y -5} :context :reuse + :completion :await :batch-size 7 :operation {:id :custom/work :args {:amount 3}} + :preparation {:reset :outside-timing :data [1 2]}} + (select-keys (first data) [:params :view :context :completion :batch-size :operation :preparation])))) + (finally + (decl/unregister-case! :feature-plan-fixture/second) + (decl/unregister-case! :feature-plan-fixture/first) + (decl/unregister-scene! :feature-plan-fixture)))) + +(t/deftest ^:async planning-propagates-manifest-and-collector-errors + (write-cargo-fixture) + (fs/writeFileSync (path/join (frontend-root) "Cargo.toml") "decoy") + (doseq [[definitions message] [[{"default" [] "a" []} "unknown requested Cargo features"] + [{"default" ["a"] "a" [] "b" []} "ab requires empty declared Cargo defaults"]]] + (await (mock/with-mocks* + {process/run (fn [_] (js/Promise.resolve (metadata-output definitions)))} + (t/is (= message (await (error-message + (node-plan/prepare (frontend-root) (planning-options) nil)))))))) + (t/is (= "no benchmark cases match the selection" + (await (error-message (node-plan/prepare (frontend-root) + (assoc (planning-options) :filter "missing-case") nil)))))) + +(t/deftest ^:async invalid-input-and-existing-output-fail-before-cargo + (write-cargo-fixture) + (await + (mock/with-mocks* + {process/run (fn [_] (throw (js/Error. "unexpected Cargo invocation")))} + (t/is (= "output directory already exists" + (await (error-message (node-plan/prepare (frontend-root) + (assoc (planning-options) :output-dir (frontend-root)) nil))))) + (t/is (= "ab requires at least one requested feature" + (await (error-message (node-plan/prepare (frontend-root) + (assoc (planning-options) :features []) nil)))))))) + +(t/deftest ^:async cancellation-reaches-the-process-boundary + (write-cargo-fixture) + (let [controller (js/AbortController.)] + (.abort controller "SIGINT") + (t/is (str/includes? (await (error-message + (node-plan/prepare (frontend-root) (planning-options) (.-signal controller)))) + "cancelled")))) + +(defn- planned-record + [] + (let [plan (feature-plan/plan ["a" "b"] {:dry-run true})] + {:type :feature-comparison :comparison-id "ab-id" + :started-at "start" :updated-at "start" :status :planned + :options (planning-options) + :permissions {:allow-build-differences false :allow-scene-differences true :allow-all-differences false} + :cases (cases/collect-cases {:master-seed 42 :filter "rects/load"}) + :features (:features plan) + :configurations (mapv #(assoc % :status :planned) (:configurations plan)) + :edges (mapv #(assoc % :status :planned) (:edges plan)) + :warnings ["Timing values do not determine success."]})) + +(defn- completed-configuration + [configuration] + (assoc configuration + :status :completed + :effective-features ["transitive"] + :build {:id "build-id" :manifest-path "/build.transit" :log-path "/build.log"} + :run {:run-id "run-id" :record-path "/run.transit" :log-path "/run.log" + :exit-code 0})) + +(defn- completed-record + [] + (let [record (planned-record)] + (assoc record + :status :completed :updated-at "end" :ended-at "end" + :configurations (mapv completed-configuration (:configurations record)) + :edges (mapv #(assoc % :status :completed :path "/comparison.transit") + (:edges record))))) + +(defn- terminal-record + [status] + (let [record (planned-record)] + (assoc record + :status status :updated-at "end" :ended-at "end" + :configurations + (mapv (fn [[index configuration]] + (if (zero? index) + (assoc (completed-configuration configuration) + :status :failed :failure {:phase :run} + :reason "SIGINT") + (assoc configuration :status :unattempted + :reason "stopped after failure"))) + (map-indexed vector (:configurations record))) + :edges + (mapv (fn [[index edge]] + (if (zero? index) + (assoc edge :status :failed :path "/comparison.transit" + :failure {:message "partial"}) + (assoc edge :status :unattempted + :reason "stopped after failure"))) + (map-indexed vector (:edges record)))))) + +(defn- running-record + [] + (-> (planned-record) + (assoc :status :running) + (assoc-in [:configurations 0 :status] :building) + (assoc-in [:configurations 0 :build] + {:id "build-id" :manifest-path "/build.transit" :log-path "/build.log"}))) + +(t/deftest feature-comparison-record-round-trips-plan-and-partial-evidence + (let [record (terminal-record :interrupted)] + (t/is (= (planned-record) (result/decode (result/encode (planned-record))))) + (t/is (= record (result/decode (result/encode record)))))) + +(t/deftest feature-comparison-progress-must-match-saved-evidence + (doseq [record [(planned-record) (running-record) (completed-record) + (terminal-record :failed) (terminal-record :interrupted)]] + (t/is (= record (result/check-record record)))) + (let [completed (completed-record) + failed (terminal-record :failed) + running (running-record)] + (doseq [[record message] + [[(assoc (planned-record) :status :completed) + #"terminal feature comparison needs an end time"] + [(assoc-in completed [:configurations 0 :status] :planned) + #"still has active configurations"] + [(assoc-in completed [:edges 0 :status] :planned) #"still has active edges"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :build) configurations))) + #"configuration needs a build"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :effective-features) configurations))) + #"configuration needs effective features"] + [(update completed :configurations + (fn [configurations] + (mapv #(dissoc % :run) configurations))) + #"configuration needs a run"] + [(update completed :configurations + (fn [configurations] + (mapv #(update % :run dissoc :exit-code) configurations))) + #"run exit code"] + [(assoc-in completed [:configurations 0 :run :exit-code] 1) + #"successful run"] + [(update-in completed [:edges 0] dissoc :path) + #"edge needs a report path"] + [(update-in failed [:configurations 0] dissoc :failure :reason) + #"configuration needs a failure or reason"] + [(update-in failed [:configurations 1] dissoc :reason) + #"configuration needs a reason"] + [(update-in failed [:edges 0] dissoc :failure :reason) + #"edge needs a failure"] + [(update-in failed [:edges 1] dissoc :reason) + #"edge needs a reason"] + [(assoc running :ended-at "end") #"unfinished feature comparison has an end time"] + [(assoc-in (planned-record) [:configurations 0 :status] :building) + #"started configurations"] + [(assoc-in (planned-record) [:edges 0 :status] :completed) + #"started edges"] + [(assoc (planned-record) :features ["b" "a"]) + #"features disagree with normalized selection"] + [(update (planned-record) :configurations + #(assoc % 1 (first %))) + #"configurations disagree with planned features"] + [(update (planned-record) :configurations pop) + #"configurations disagree with planned features"] + [(assoc-in (planned-record) [:edges 0 :to] "11") + #"edges disagree with planned configurations"] + [(update (planned-record) :cases conj (first (:cases (planned-record)))) + #"duplicate feature comparison case"]]] + (t/is (thrown-with-msg? js/Error message (result/check-record record)) + (pr-str message))))) + +(t/deftest feature-comparison-current-state-shapes-are-checked + (doseq [[location value] [[[:comparison-id] 1] [[:updated-at] nil] [[:status] :ready] + [[:ended-at] nil] [[:features] [:a]] [[:cases] [{}]] + [[:permissions :allow-scene-differences] nil] + [[:configurations 0 :status] :interrupted] + [[:configurations 0 :selection :features] "a"] + [[:configurations 0 :selection :default-features?] nil] + [[:configurations 0 :build] {:id "id"}] + [[:configurations 0 :run] {:run-id "id" :record-path "p" :log-path "l" :exit-code "0"}] + [[:configurations 0 :failure] "error"] + [[:configurations 0 :effective-features] [:a]] + [[:edges 0 :status] :running] [[:edges 0 :path] 1] [[:warnings] [nil]]]] + (t/is (thrown-with-msg? js/Error #"invalid schema:feature-comparison" + (result/check-record (assoc-in (planned-record) location value))) + (pr-str location)))) + +(defn- create-frontend-root + [] + (fs/mkdirSync (frontend-root) #js {:recursive true})) + +(defn- comparison-output-dir + [] + (path/join @temporary-root "output" "comparison")) + +(defn- fixture-case + [] + {:id :fixture/comparison-case}) + +(defn- comparison-prepared + ([features] (comparison-prepared features {})) + ([features overrides] + (let [plan (feature-plan/plan features {:dry-run true}) + output (or (:output-dir overrides) (comparison-output-dir)) + opts (merge {:default-features false + :filter "fixture/" + :seed 42 + :warmups 0 + :repetitions 1 + :timeout-ms 1000 + :build-timeout-ms 5000 + :startup-timeout-ms 1000 + :allow-build-differences false + :allow-scene-differences false + :allow-all-differences false} + overrides + {:features (:features plan) + :output-dir output})] + {:plan plan + :cases [(fixture-case)] + :output-dir output + :options opts}))) + +(defn- fake-build + [selection extra-features] + {:id (str (random-uuid)) + :manifest-path (path/join @temporary-root "builds" "build.transit") + :log-path (path/join @temporary-root "builds" "build.log") + :manifest {:functional {:mode :release + :features (vec (concat (:features selection) extra-features))} + :provenance {:feature-selection selection}}}) + +(defn- build-service + [{:keys [extra-features fail-on inspect] + :or {extra-features (constantly []) + fail-on (constantly false)}}] + (fn [request] + (when inspect (inspect request)) + (let [selection (get request :selection) + build (fake-build selection (extra-features selection))] + (when-let [on-created (:on-created request)] + (on-created build)) + (if (fail-on selection) + (js/Promise.reject (js/Error. "fixture build failure")) + (js/Promise.resolve build))))) + +(defn- metric-entry + [value] + (result/metric {:value value + :unit :ms + :semantics :duration + :basis {:kind :attempt-total} + :zero-policy :positive})) + +(defn- fingerprint + [character] + {:algorithm :sha-256 + :digest (apply str (repeat 64 character))}) + +(defn- preparation + [case-id digest] + {:preparation-id (str (random-uuid)) + :case-id case-id + :fingerprint (fingerprint digest) + :effective-graphics {:backend :fixture} + :diagnostics {}}) + +(defn- fake-run-record + [selected-build run-home cases options {:keys [kind digest metric-value] + :or {kind :completed digest "a" metric-value 10}}] + (let [run (result/create-run + {:run-id (:run-id run-home) + :started-at (.toISOString (js/Date.)) + :plan {:cases cases + :warmups 0 + :repetitions 1 + :timeout-policy {:attempt-ms (:timeout-ms options)}} + :metadata {:git {:sha nil :dirty nil} + :build {:functional (get-in selected-build [:manifest :functional]) + :provenance (get-in selected-build [:manifest :provenance])} + :environment {} + :scored? true}}) + failed? (= kind :failed) + run (if (= kind :zero-attempts) + run + (reduce + (fn [run case] + (let [case-id (:id case)] + (if failed? + (result/record-attempt + run + {:case-id case-id + :preparation-id nil + :warmup? false + :outcome :failed + :metrics {} + :trace {} + :failure {:message "fixture failure"}}) + (let [prep (preparation case-id digest)] + (-> run + (result/record-preparation prep) + (result/record-attempt + {:case-id case-id + :preparation-id (:preparation-id prep) + :warmup? false + :outcome :completed + :metrics {:fixture/total-ms (metric-entry metric-value)} + :trace {}})))))) + run + cases)) + termination (case kind + :failed {:reason :failed :failure {:message "fixture failure"}} + :zero-attempts {:reason :incomplete} + :interrupted {:reason :interrupted :signal "SIGINT"} + {:reason :completed})] + (result/finish-run run (.toISOString (js/Date.)) termination))) + +(defn- execute-service + [spec-or-fn] + (fn [_ run-home selected-build cases options _signal _on-progress on-created] + (when on-created (on-created (:run-id run-home))) + (let [spec (if (fn? spec-or-fn) (spec-or-fn selected-build) spec-or-fn) + run (fake-run-record selected-build run-home cases options spec) + exit-code (case (:kind spec) + :failed 1 + :zero-attempts 1 + :interrupted 130 + 0)] + (files/atomic-write (:record-path run-home) + (result/encode (result/check-record run))) + (js/Promise.resolve {:run run :home run-home :exit-code exit-code :error nil})))) + +(defn- comparison-services + [{:keys [prepare-build collect-catalog execute-run]}] + {:prepare-build prepare-build + :collect-catalog (or collect-catalog (fn [_ _] [(fixture-case)])) + :execute-run execute-run}) + +(t/deftest create-home-creates-only-a-fresh-home + (create-frontend-root) + (let [home (feature-comparison/create-home (frontend-root) "comparison-a" nil)] + (t/is (= (path/join (frontend-root) "target" "performance" "render-wasm" + "feature-comparisons" "comparison-a") + (:dir home))) + (t/is (fs/existsSync (:dir home))) + (t/is (not (fs/existsSync (:record-path home))))) + (let [existing (path/join @temporary-root "existing")] + (fs/mkdirSync existing) + (t/is (thrown-with-msg? js/Error #"already exists" + (feature-comparison/create-home (frontend-root) + "comparison-b" existing)))) + (let [builds (path/join (frontend-root) "target" "performance" "render-wasm" "builds") + dir (path/join builds "build-a" "nested")] + (fs/mkdirSync (path/join builds "build-a") #js {:recursive true}) + (t/is (thrown-with-msg? js/Error #"inside builds" + (feature-comparison/create-home (frontend-root) + "comparison-c" dir))))) + +(t/deftest ^:async initial-record-is-saved-before-the-first-build + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record (path/join (:output-dir prepared) "feature-comparison.transit") + observed (atom nil) + created (atom nil) + services (comparison-services + {:prepare-build (build-service + {:inspect (fn [_] + (reset! observed + (result/decode (fs/readFileSync record "utf8")))) + :fail-on (constantly true)}) + :execute-run (execute-service {})}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services services + :on-created (fn [home saved] (reset! created [home saved]))}))] + (t/is (= :running (get-in @created [1 :status]))) + (t/is (every? #(= :planned (:status %)) (get-in @created [1 :configurations]))) + (t/is (= (:dir (first @created)) (:dir (:home result)))) + (t/is (= :running (:status @observed))) + (t/is (= :building (get-in @observed [:configurations 0 :status]))) + (t/is (nil? (get-in @observed [:configurations 0 :build]))) + (t/is (= :planned (get-in @observed [:configurations 1 :status]))) + (t/is (= 1 (:exit-code result))))) + +(t/deftest ^:async completed-configurations-and-edge-return-zero + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run (execute-service {})})})) + record (:comparison result) + edge (first (:edges record)) + path (:path edge) + comparison (result/decode (fs/readFileSync path "utf8"))] + (t/is (= 0 (:exit-code result))) + (t/is (= :completed (:status record))) + (t/is (every? #(= :completed (:status %)) (:configurations record))) + (t/is (every? #(= :completed (:status %)) (:edges record))) + (t/is (= ["a"] (get-in record [:configurations 1 :effective-features]))) + (t/is (= :comparison (:type comparison))) + (t/is (= {:from "0" + :to "1" + :feature "a" + :baseline-features [] + :candidate-features ["a"] + :added ["a"] + :removed []} + (:edge comparison))) + (t/is (= record (result/decode (fs/readFileSync (get-in result [:home :record-path]) + "utf8")))))) + +(t/deftest ^:async attempted-failed-run-continues-and-failed-edge-returns-one + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (execute-service + (fn [build] + (if (seq (get-in build [:manifest :functional :features])) + {} + {:kind :failed})))})})) + record (:comparison result) + edge (first (:edges record))] + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (:status record))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= :completed (get-in record [:configurations 1 :status]))) + (t/is (fs/existsSync (get-in record [:configurations 0 :run :record-path]))) + (t/is (= :failed (:status edge))) + (t/is (not (contains? edge :path))) + (t/is (str/includes? (get-in edge [:failure :message]) "no shared metric")))) + +(t/deftest ^:async zero-attempts-stop-later-configurations + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + base (execute-service {:kind :zero-attempts}) + calls (atom 0) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @calls)) + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (str/includes? (get-in record [:configurations 1 :reason]) "no attempts")) + (t/is (= :failed (get-in record [:edges 0 :status]))))) + +(t/deftest ^:async build-failure-stops-and-missing-endpoint-fails + (create-frontend-root) + (let [prepared (comparison-prepared ["a" "b"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service + {:fail-on #(= ["b"] (:features %))}) + :execute-run (execute-service {})})})) + record (:comparison result) + edge (first (:edges record))] + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (:status record))) + (t/is (= :completed (get-in record [:configurations 0 :status]))) + (t/is (= :failed (get-in record [:configurations 1 :status]))) + (t/is (= :unattempted (get-in record [:configurations 2 :status]))) + (t/is (= :unattempted (get-in record [:configurations 3 :status]))) + (t/is (= :failed (:status edge))) + (t/is (not (contains? edge :path))) + (t/is (str/includes? (get-in edge [:failure :message]) "run record is missing")))) + +(t/deftest ^:async catalog-mismatch-stops-before-a-run-home + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + calls (atom 0) + base (execute-service {}) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :collect-catalog (fn [_ _] [{:id :other/case}]) + :execute-run execute})})) + record (:comparison result)] + (t/is (zero? @calls)) + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (get-in record [:configurations 0 :status]))) + (t/is (= "catalog" (get-in record [:configurations 0 :failure :phase]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))))) + +(t/deftest ^:async unacknowledged-differences-save-a-failed-edge-report + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run + (execute-service + (fn [build] + (if (seq (get-in build [:manifest :functional :features])) + {:digest "b"} + {:digest "a"})))})})) + record (:comparison result) + edge (first (:edges record)) + saved (result/decode (fs/readFileSync (:path edge) "utf8"))] + (t/is (= 1 (:exit-code result))) + (t/is (= :failed (:status record))) + (t/is (= :failed (:status edge))) + (t/is (fs/existsSync (:path edge))) + (t/is (= {:unacknowledged-differences 1} (:failure edge))) + (t/is (= :comparison (:type saved))) + (t/is (= ["a"] (get-in saved [:edge :added]))) + (t/is (some #(and (= :fingerprint (:field %)) (not (:waived? %))) + (get-in saved [:pairs 0 :mismatches]))))) + +(t/deftest ^:async interruption-skips-comparisons-and-returns-the-signal-exit + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + controller (js/AbortController.) + base (execute-service + (fn [build] + (when (empty? (get-in build [:manifest :functional :features])) + (.abort controller "SIGINT")) + {:kind :interrupted})) + calls (atom 0) + execute (fn [& args] + (swap! calls inc) + (apply base args)) + result (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @calls)) + (t/is (= 130 (:exit-code result))) + (t/is (true? (:interrupted? result))) + (t/is (= :interrupted (:status record))) + (t/is (some? (:ended-at record))) + (t/is (= "SIGINT" (get-in record [:configurations 0 :reason]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (= :unattempted (get-in record [:edges 0 :status]))) + (t/is (not (fs/existsSync (path/join (:dir (:home result)) "comparisons")))))) + +(t/deftest ^:async interruption-between-configurations-stops-before-the-next-build + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + controller (js/AbortController.) + builds (atom 0) + base (execute-service {}) + execute (fn [& args] + (let [result (apply base args)] + (.abort controller "SIGTERM") + result)) + result (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build + (build-service + {:inspect (fn [_] (swap! builds inc))}) + :execute-run execute})})) + record (:comparison result)] + (t/is (= 1 @builds)) + (t/is (= 143 (:exit-code result))) + (t/is (= :interrupted (:status record))) + (t/is (= :completed (get-in record [:configurations 0 :status]))) + (t/is (= :unattempted (get-in record [:configurations 1 :status]))) + (t/is (str/includes? (get-in record [:configurations 1 :reason]) "SIGTERM")) + (t/is (= :unattempted (get-in record [:edges 0 :status]))) + (t/is (not (fs/existsSync (path/join (:dir (:home result)) "comparisons")))))) + +(t/deftest ^:async persistence-failure-retains-the-last-saved-state + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + record-path (path/join (:output-dir prepared) "feature-comparison.transit") + prepare (fn [request] + (fs/rmSync record-path) + (fs/mkdirSync record-path) + (let [build (fake-build (:selection request) [])] + ((:on-created request) build) + (js/Promise.resolve build))) + result (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build prepare + :execute-run (execute-service {})})})) + record (:comparison result)] + (t/is (= 1 (:exit-code result))) + (t/is (some? (:error result))) + (t/is (= :running (:status record))) + (t/is (= :building (get-in record [:configurations 0 :status]))) + (t/is (nil? (get-in record [:configurations 0 :build]))) + (t/is (= :planned (get-in record [:configurations 1 :status]))))) + +(t/deftest ^:async pending-real-signals-stop-edge-analysis + (create-frontend-root) + (doseq [[signal exit-code] [["SIGINT" 130] ["SIGTERM" 143]]] + (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root signal)}) + controller (js/AbortController.) + remove (process/on-signals #(.abort controller %)) + execute (execute-service + (fn [build] + (when (seq (get-in build [:manifest :functional :features])) + (.kill js/process (.-pid js/process) signal)) + {}))] + (try + (let [response (await (feature-comparison/execute + (frontend-root) prepared (.-signal controller) + {:services (comparison-services + {:prepare-build (build-service {}) + :execute-run execute})}))] + (await (timers/setImmediate)) + (let [saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))] + (t/is (= exit-code (:exit-code response))) + (t/is (= :interrupted (:status saved))) + (t/is (every? #(= :unattempted (:status %)) (:edges saved))))) + (finally + (remove)))))) diff --git a/frontend/test/performance_tests/render_wasm/report_test.cljs b/frontend/test/performance_tests/render_wasm/report_test.cljs index 648bb20de6..0e5d937f77 100644 --- a/frontend/test/performance_tests/render_wasm/report_test.cljs +++ b/frontend/test/performance_tests/render_wasm/report_test.cljs @@ -76,6 +76,13 @@ (reduce #(add %1 %2 false) (reduce #(add %1 %2 true) run (repeat warmups 1000)) values)))) +(defn- run-with-features + "Adds independent effective-feature evidence and a saved requested selection." + [values features selection] + (run-with values {:facts (-> metadata + (assoc-in [:build :functional :features] features) + (assoc-in [:build :provenance :feature-selection] selection))})) + (defn- approximately? "Compares computed numbers with a tolerance for floating point arithmetic." [a b] @@ -458,16 +465,16 @@ (t/is (= 1 (get-in report [:cases 0 :metrics :bad :counts :invalid]))) (t/is (= :no-valid-observations (get-in report [:cases 0 :metrics :bad :statistics :median :reason]))))) -(t/deftest definition-drift-is-visible-and-force-compares-raw-values - (let [run (assoc-in (run-with [1 2]) [:attempts 1 :metrics metric-id :basis] - {:kind :per-call-average :calls 2}) - report (summary/summarize run) - forced (compare/compare-runs run run {:force true})] +(t/deftest definition-drift-is-visible-and-all-differences-compares-raw-values + (let [run (assoc-in (run-with [1 2]) [:attempts 1 :metrics metric-id :basis] + {:kind :per-call-average :calls 2}) + report (summary/summarize run) + comparison (compare/compare-runs run run {:allow-all-differences true})] (t/is (= :inconsistent-metric-definitions (get-in report [:cases 0 :metrics metric-id :statistics :reason]))) (t/is (= ::compare/incompatible (:type (error-data #(compare/compare-runs run run))))) - (t/is (= 0 (get-in forced [:pairs 0 :metrics metric-id :absolute :value]))) - (t/is (= 1.5 (get-in forced [:pairs 0 :baseline :metrics metric-id :statistics :median :value]))) - (t/is (true? (get-in forced [:pairs 0 :baseline :metrics metric-id :mixed-definitions?]))))) + (t/is (= 0 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))) + (t/is (= 1.5 (get-in comparison [:pairs 0 :baseline :metrics metric-id :statistics :median :value]))) + (t/is (true? (get-in comparison [:pairs 0 :baseline :metrics metric-id :mixed-definitions?]))))) (t/deftest failed-invalid-and-warmup-definitions-do-not-suppress-valid-statistics (let [run (-> (run-with [1 :failed js/NaN] {:warmups 1}) @@ -492,7 +499,7 @@ (t/is (= 1.5 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))) (t/is (= comparison (-> comparison result/encode result/decode))))) -(t/deftest strict-and-forced-compatibility +(t/deftest strict-and-all-differences-compatibility (let [base (run-with [1 2])] (doseq [[field changed] [[:fingerprint (run-with [3] {:prep (assoc-in preparation [:fingerprint :digest] (apply str (repeat 64 "b")))})] @@ -503,24 +510,29 @@ [:execution (run-with [3] {:definition (assoc saved-case :completion :await)})] [:execution (run-with [3] {:definition (assoc saved-case :preparation {:settle-ms 25})})] [:environment (run-with [3] {:facts (assoc-in metadata [:environment :browser] "other")})] - [:functional-build (run-with [3] {:facts (assoc-in metadata [:build :functional :features] #{:other})})] + [:effective-features (run-with [3] {:facts (assoc-in metadata [:build :functional :features] #{:other})})] + [:functional-build (run-with [3] {:facts (assoc-in metadata [:build :functional :mode] :debug)})] + [:scene (run-with [3] {:definition (assoc saved-case :scene :other)})] + [:scene (run-with [3] {:definition (assoc saved-case :scene-seed 43)})] + [:scene (run-with [3] {:definition (assoc saved-case :params {:count 20})})] [:warmups (run-with [3] {:warmups 1})] [:effective-graphics (run-with [3] {:prep (assoc preparation :effective-graphics {:renderer "software" :software true})})] [:timeout-policy (assoc-in (run-with [3]) [:plan :timeout-policy :attempt-ms] 100)]]] (let [changed (-> changed result/encode result/decode) rejected (error-data #(compare/compare-runs base changed)) selected (error-data #(compare/compare-runs base changed {:pairs [[:saved/pan :saved/pan]]})) - forced (compare/compare-runs base changed {:force true}) - mismatches (get-in forced [:pairs 0 :mismatches]) - text (format/format-comparison forced)] + comparison (compare/compare-runs base changed {:allow-all-differences true}) + mismatches (get-in comparison [:pairs 0 :mismatches]) + text (format/format-comparison comparison)] (t/is (= ::compare/incompatible (:type rejected))) (t/is (= rejected selected)) - (t/is (:forced? forced)) + (t/is (= {:allow-build-differences false :allow-scene-differences false :allow-all-differences true} + (:permissions comparison))) (t/is (= [field] (mapv :field mismatches))) - (t/is (= (:mismatches rejected) mismatches)) - (t/is (every? #(false? (:waived? %)) mismatches)) - (t/is (str/includes? text "FORCED COMPARISON")) - (t/is (str/includes? text "MISMATCHES")) + (t/is (= (mapv #(assoc % :waived? true :rule :all) (:mismatches rejected)) mismatches)) + (t/is (every? #(and (false? (:waived? %)) (not (contains? % :rule))) (:mismatches rejected))) + (t/is (str/includes? text "Permissions")) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) (t/is (str/includes? text (str ":field " (pr-str field)))))))) (t/deftest explicit-pairs-select-cases-without-waiving-compatibility @@ -533,23 +545,24 @@ result/decode) pairs (compare/selector-pair base other "saved/pan,different/zoom") rejected (error-data #(compare/compare-runs base other {:pairs pairs})) - comparison (-> (compare/compare-runs base other {:pairs pairs :force true}) + comparison (-> (compare/compare-runs base other {:pairs pairs :allow-all-differences true}) result/encode result/decode) execution (first (filter #(= :execution (:field %)) (get-in comparison [:pairs 0 :mismatches]))) - execution-keys [:scene :operation :batch-size :completion :preparation]] + execution-keys [:operation :batch-size :completion :preparation]] (t/is (= [[:saved/pan :different/zoom]] pairs)) (t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))) (t/is (= ::compare/incompatible (:type rejected))) - (t/is (= #{:execution :fingerprint} (set (map :field (:mismatches rejected))))) - (t/is (= (:mismatches rejected) (get-in comparison [:pairs 0 :mismatches]))) - (t/is (every? #(false? (:waived? %)) (get-in comparison [:pairs 0 :mismatches]))) - (t/is (str/includes? (format/format-comparison comparison) "FORCED COMPARISON")) + (t/is (= #{:scene :execution :fingerprint} (set (map :field (:mismatches rejected))))) + (t/is (= (mapv #(assoc % :waived? true :rule :all) (:mismatches rejected)) + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (every? :waived? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (str/includes? (format/format-comparison comparison) "ACKNOWLEDGED DIFFERENCE")) (t/is (str/includes? (format/format-comparison comparison) ":field :execution")) (t/is (str/includes? (format/format-comparison comparison) ":field :fingerprint")) - (t/is (= {:scene :saved :operation {:steps 3} + (t/is (= {:operation {:steps 3} :batch-size 1 :completion :render-full :preparation {}} (select-keys (:baseline execution) execution-keys))) - (t/is (= {:scene :different :operation {:steps 8} + (t/is (= {:operation {:steps 8} :batch-size 20 :completion :call :preparation {:settle-ms 25}} (select-keys (:candidate execution) execution-keys))) (t/is (= ::compare/incompatible @@ -557,16 +570,15 @@ {:pairs pairs}))))) (let [unscored (assoc-in other [:metadata :scored?] false) rejected (error-data #(compare/compare-runs base unscored {:pairs pairs})) - forced (compare/compare-runs base unscored {:pairs pairs :force true}) - mismatches (get-in forced [:pairs 0 :mismatches]) - text (format/format-comparison forced)] + comparison (compare/compare-runs base unscored {:pairs pairs :allow-all-differences true}) + mismatches (get-in comparison [:pairs 0 :mismatches]) + text (format/format-comparison comparison)] (t/is (= ::compare/incompatible (:type rejected))) - (t/is (:forced? forced)) - (t/is (= (:mismatches rejected) mismatches)) - (t/is (= #{:execution :fingerprint :scored?} (set (map :field mismatches)))) - (t/is (every? #(false? (:waived? %)) mismatches)) - (t/is (str/includes? text "FORCED COMPARISON")) - (t/is (str/includes? text "MISMATCHES")) + (t/is (true? (get-in comparison [:permissions :allow-all-differences]))) + (t/is (= (mapv #(assoc % :waived? true :rule :all) (:mismatches rejected)) mismatches)) + (t/is (= #{:scene :execution :fingerprint :scored?} (set (map :field mismatches)))) + (t/is (every? :waived? mismatches)) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE")) (t/is (str/includes? text ":field :scored?"))) (t/is (= ["saved/pan"] (:available (error-data #(compare/resolve-case base "pan"))))) (t/is (= ::compare/invalid-selector (:type (error-data #(compare/selector-pair base other "saved/pan"))))))) @@ -577,23 +589,24 @@ result/encode result/decode) comparison (-> (compare/compare-runs base other {:pairs [[:saved/pan :different/zoom]]}) result/encode result/decode)] - (t/is (false? (:forced? comparison))) + (t/is (= {:allow-build-differences false :allow-scene-differences false :allow-all-differences false} + (:permissions comparison))) (t/is (true? (get-in comparison [:pairs 0 :explicit?]))) (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) (t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))))) -(t/deftest force-preserves-metric-definitions-and-excludes-failed-values +(t/deftest all-differences-preserves-metric-definitions-and-excludes-failed-values (let [base (run-with [1 :failed]) changed (assoc-in (run-with [2]) [:attempts 0 :metrics metric-id] (entry 2 {:unit :seconds :basis {:kind :per-call-average :calls 50}})) - comparison (compare/compare-runs base changed {:force true})] + comparison (compare/compare-runs base changed {:allow-all-differences true})] (t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))) (t/is (= :seconds (get-in comparison [:pairs 0 :candidate :metrics metric-id :definitions 0 :unit]))) (t/is (= 1 (get-in comparison [:pairs 0 :baseline :metrics metric-id :counts :failed]))) (t/is (= ::compare/no-shared-metrics (:type (error-data #(compare/compare-runs base (assoc-in changed [:attempts 0 :metrics] {:other (entry 2)}) - {:force true}))))))) + {:allow-all-differences true}))))))) (t/deftest comparison-boundary-validates-nested-evidence (let [run (run-with [1 2]) @@ -609,12 +622,445 @@ (assoc-in comparison [:analysis :confidence] 1)]] (t/is (= ::result/invalid-record (:type (error-data #(result/decode (transit/encode-str changed))))))))) -(t/deftest force-keeps-invalid-measurements-unusable +(t/deftest all-differences-keeps-invalid-measurements-unusable (let [baseline (run-with [js/NaN js/Infinity :absent :failed] {:repetitions 5}) candidate (run-with [2] {:facts (assoc-in metadata [:environment :cpu] "other")}) - comparison (compare/compare-runs baseline candidate {:force true})] + comparison (compare/compare-runs baseline candidate {:allow-all-differences true})] (t/is (= {:valid 0 :invalid 2 :absent 1 :failed 1 :unattempted 1 :warmup 0} (get-in comparison [:pairs 0 :baseline :metrics metric-id :counts]))) (t/is (= :no-valid-observations (get-in comparison [:pairs 0 :metrics metric-id :absolute :reason]))) (t/is (= :no-valid-observations (get-in comparison [:pairs 0 :metrics metric-id :percentage-ci :reason]))) (t/is (= comparison (result/decode (result/encode comparison)))))) + +(t/deftest build-permission-acknowledges-only-recorded-build-differences + (let [baseline (run-with [1])] + (doseq [[key value field] [[:mode :debug :functional-build] + [:features ["other"] :effective-features] + [:default-features? false :functional-build] + [:instrumentation {:stats true} :functional-build] + [:env {"PROFILE" "1"} :functional-build]]] + (let [candidate (run-with [2] {:facts (assoc-in metadata [:build :functional key] value)}) + comparison (compare/compare-runs baseline candidate {:allow-build-differences true}) + mismatches (get-in comparison [:pairs 0 :mismatches])] + (t/is (= ::compare/incompatible (:type (error-data #(compare/compare-runs baseline candidate))))) + (t/is (= ::compare/incompatible + (:type (error-data #(compare/compare-runs baseline candidate {:allow-scene-differences true}))))) + (t/is (= [field] (mapv :field mismatches))) + (t/is (every? #(and (:waived? %) (= :build (:rule %))) mismatches)) + (t/is (= {:allow-build-differences true :allow-scene-differences false :allow-all-differences false} + (:permissions comparison))) + (t/is (= comparison (-> comparison result/encode result/decode))))))) + +(t/deftest run-build-environment-validates-strings-and-preserves-absence + (let [missing (-> (run-with [1]) result/encode result/decode)] + (t/is (not (contains? (get-in missing [:metadata :build]) :environment))) + (doseq [environment [{} {"CFLAGS" "" "RUSTFLAGS" "-C\u001fopt-level=3"}]] + (let [run (run-with [1] {:facts (assoc-in metadata [:build :environment] environment)})] + (t/is (= environment (get-in (-> run result/encode result/decode) [:metadata :build :environment]))))) + (doseq [environment [nil [] {:CFLAGS ""} {"CFLAGS" nil} {"CFLAGS" 0}]] + (t/is (= ::result/invalid-record + (:type (error-data #(result/decode + (transit/encode-str + (assoc-in missing [:metadata :build :environment] environment)))))))))) + +(t/deftest build-environment-differences-require-build-or-all-permission + (doseq [[before after] [[::missing {}] [{} ::missing] + [{} {"CFLAGS" ""}] + [{"CFLAGS" "-O2"} {"CFLAGS" "-O3"}]]] + (let [baseline (run-with [1] {:facts (cond-> metadata + (not= ::missing before) (assoc-in [:build :environment] before))}) + candidate (run-with [2] {:facts (cond-> metadata + (not= ::missing after) (assoc-in [:build :environment] after))})] + (t/is (empty? (get-in (compare/compare-runs baseline baseline) [:pairs 0 :mismatches]))) + (doseq [permissions [{} {:allow-scene-differences true}]] + (t/is (= ::compare/incompatible + (:type (error-data #(compare/compare-runs baseline candidate permissions)))))) + (doseq [[permissions rule] [[{:allow-build-differences true} :build] + [{:allow-all-differences true} :all]]] + (let [comparison (compare/compare-runs baseline candidate permissions)] + (t/is (= [{:field :build-environment + :baseline (when-not (= ::missing before) before) + :candidate (when-not (= ::missing after) after) + :waived? true :rule rule}] + (get-in comparison [:pairs 0 :mismatches]))) + (t/is (= comparison (-> comparison result/encode result/decode)))))))) + +(t/deftest feature-toggle-does-not-acknowledge-build-environment-changes + (let [baseline (assoc-in (run-with-features [1] [] {:default-features? false :features []}) + [:metadata :build :environment] {"CFLAGS" "-O2"}) + candidate (assoc-in (run-with-features [2] ["toggle"] {:default-features? false :features ["toggle"]}) + [:metadata :build :environment] {"CFLAGS" "-O3"}) + expected {:feature "toggle" :baseline-features [] :candidate-features ["toggle"] + :added ["toggle"] :removed []} + diagnostic (compare/compare-runs baseline candidate {:expected-feature-change expected + :on-unallowed-difference :record})] + (t/is (= ::compare/incompatible + (:type (error-data #(compare/compare-runs baseline candidate {:expected-feature-change expected}))))) + (t/is (= [[:effective-features true :feature-toggle] [:build-environment false nil]] + (mapv (juxt :field :waived? :rule) (get-in diagnostic [:pairs 0 :mismatches])))) + (let [allowed (compare/compare-runs baseline candidate {:expected-feature-change expected + :allow-build-differences true})] + (t/is (= [[:effective-features true :feature-toggle] [:build-environment true :build]] + (mapv (juxt :field :waived? :rule) (get-in allowed [:pairs 0 :mismatches]))))))) + +(t/deftest scene-permission-acknowledges-scene-inputs-and-fingerprint-content + (let [baseline (run-with [1])] + (doseq [[field options] [[:scene {:definition (assoc saved-case :scene :other)}] + [:scene {:definition (assoc saved-case :scene-seed 43)}] + [:scene {:definition (assoc saved-case :params {:count 2000})}] + [:fingerprint {:prep (assoc-in preparation [:fingerprint :digest] + (apply str (repeat 64 "b")))}]]] + (let [candidate (run-with [2] options) + comparison (compare/compare-runs baseline candidate {:allow-scene-differences true}) + mismatches (get-in comparison [:pairs 0 :mismatches])] + (t/is (= ::compare/incompatible (:type (error-data #(compare/compare-runs baseline candidate))))) + (t/is (= ::compare/incompatible + (:type (error-data #(compare/compare-runs baseline candidate {:allow-build-differences true}))))) + (t/is (= [field] (mapv :field mismatches))) + (t/is (every? #(and (:waived? %) (= :scene (:rule %))) mismatches)) + (t/is (= 1 (get-in comparison [:pairs 0 :metrics metric-id :absolute :value]))) + (t/is (= comparison (-> comparison result/encode result/decode))))))) + +(t/deftest build-and-scene-permissions-combine-by-union + (let [baseline (run-with [1]) + candidate (run-with [2] {:definition (assoc saved-case :id :other/load :params {:count 2000}) + :facts (assoc-in metadata [:build :functional :mode] :debug) + :prep (assoc-in preparation [:fingerprint :digest] (apply str (repeat 64 "b")))}) + pairs [[:saved/pan :other/load]] + options {:pairs pairs :allow-build-differences true :allow-scene-differences true} + comparison (compare/compare-runs baseline candidate options) + mismatches (get-in comparison [:pairs 0 :mismatches])] + (doseq [permissions [{} {:allow-build-differences true} {:allow-scene-differences true}]] + (t/is (= ::compare/incompatible + (:type (error-data #(compare/compare-runs baseline candidate (assoc permissions :pairs pairs))))))) + (t/is (= #{[:scene :scene] [:functional-build :build] [:fingerprint :scene]} + (set (map (juxt :field :rule) mismatches)))) + (t/is (every? :waived? mismatches)) + (t/is (= {:allow-build-differences true :allow-scene-differences true :allow-all-differences false} + (:permissions comparison))) + (t/is (= comparison (-> comparison result/encode result/decode))))) + +(t/deftest scoped-permissions-retain-execution-runtime-and-measurement-checks + (let [baseline (run-with [1]) + scoped {:allow-build-differences true :allow-scene-differences true}] + (doseq [[field candidate] + [[:execution (run-with [2] {:definition (assoc saved-case :operation {:steps 30})})] + [:execution (run-with [2] {:definition (assoc saved-case :operation {:steps 3 :args {:distance 20}})})] + [:execution (run-with [2] {:definition (assoc saved-case :view {:scale 2 :x 0 :y 0})})] + [:execution (run-with [2] {:definition (assoc saved-case :context :fresh)})] + [:execution (run-with [2] {:definition (assoc saved-case :completion :call)})] + [:execution (run-with [2] {:definition (assoc saved-case :batch-size 10)})] + [:execution (run-with [2] {:definition (assoc saved-case :preparation {:settle-ms 25})})] + [:environment (run-with [2] {:facts (assoc-in metadata [:environment :cpu] "other")})] + [:effective-graphics (run-with [2] {:prep (assoc-in preparation [:effective-graphics :dpr] 1)})] + [:warmups (run-with [2] {:warmups 1})] + [:timeout-policy (assoc-in (run-with [2]) [:plan :timeout-policy :attempt-ms] 100)] + [:scored? (run-with [2] {:facts (assoc metadata :scored? false)})] + [[:metric metric-id] (assoc-in (run-with [2]) [:attempts 0 :metrics metric-id :unit] :seconds)]]] + (let [candidate (-> candidate result/encode result/decode) + rejected (error-data #(compare/compare-runs baseline candidate scoped)) + comparison (compare/compare-runs baseline candidate (assoc scoped :allow-all-differences true))] + (t/is (= ::compare/incompatible (:type rejected))) + (t/is (= [field] (mapv :field (:mismatches rejected)))) + (t/is (every? #(and (false? (:waived? %)) (not (contains? % :rule))) (:mismatches rejected))) + (t/is (= (mapv #(assoc % :waived? true :rule :all) (:mismatches rejected)) + (get-in comparison [:pairs 0 :mismatches]))))))) + +(t/deftest scene-permission-never-waives-incomplete-or-varying-evidence + (let [baseline (run-with [1]) + varying (result/record-preparation + (run-with [2]) + (-> preparation + (assoc :preparation-id :prep/two) + (assoc-in [:fingerprint :digest] (apply str (repeat 64 "b"))))) + missing (-> (run-with [:failed]) + (assoc :preparations []) + (assoc-in [:attempts 0 :preparation-id] nil) + (assoc-in [:attempts 0 :metrics metric-id] (entry 2))) + scoped {:allow-scene-differences true :allow-build-differences true}] + (doseq [candidate [varying missing]] + (let [candidate (-> candidate result/encode result/decode) + rejected (error-data #(compare/compare-runs baseline candidate scoped)) + diagnostic (compare/compare-runs baseline candidate (assoc scoped :on-unallowed-difference :record)) + mismatches (get-in diagnostic [:pairs 0 :mismatches]) + evidence (first (filter #(= [:fingerprint :incomplete-or-varying-evidence] (:field %)) mismatches))] + (t/is (= ::compare/incompatible (:type rejected))) + (t/is (= (:mismatches rejected) mismatches)) + (t/is (false? (:waived? evidence))) + (t/is (not (contains? evidence :rule))) + (t/is (= :scene (:rule (first (filter #(= :fingerprint (:field %)) mismatches))))) + (t/is (every? :waived? + (get-in (compare/compare-runs baseline candidate {:allow-all-differences true}) + [:pairs 0 :mismatches]))) + (t/is (= diagnostic (-> diagnostic result/encode result/decode))))))) + +(t/deftest diagnostic-record-retains-unacknowledged-differences-and-selected-rules + (let [baseline (run-with [1]) + candidate (run-with [2] {:definition (assoc saved-case :params {:count 2000} :operation {:steps 30}) + :facts (assoc-in metadata [:build :functional :mode] :debug)}) + options {:allow-build-differences true :on-unallowed-difference :record} + diagnostic (compare/compare-runs baseline candidate options) + mismatches (get-in diagnostic [:pairs 0 :mismatches]) + text (format/format-comparison diagnostic)] + (t/is (= (:mismatches (error-data #(compare/compare-runs baseline candidate (dissoc options :on-unallowed-difference)))) + mismatches)) + (t/is (= [[:scene false nil] [:execution false nil] [:functional-build true :build]] + (mapv (juxt :field :waived? :rule) mismatches))) + (t/is (= 1 (get-in diagnostic [:pairs 0 :metrics metric-id :absolute :value]))) + (t/is (str/includes? text "Permissions {:allow-build-differences true")) + (t/is (str/includes? text "ACKNOWLEDGED DIFFERENCE {:field :functional-build")) + (t/is (str/includes? text "UNACKNOWLEDGED DIFFERENCE {:field :scene")) + (t/is (str/includes? text ":rule :build")) + (t/is (= diagnostic (-> diagnostic result/encode result/decode))))) + +(t/deftest diagnostic-record-keeps-strict-definition-statistics + (let [baseline (run-with [1 2]) + mixed (assoc-in (run-with [3 4]) [:attempts 1 :metrics metric-id :basis] + {:kind :per-call-average :calls 2}) + differing (assoc-in (run-with [3 4]) [:attempts 0 :metrics metric-id :unit] :seconds) + single-definition (-> (run-with [3 4]) + (assoc-in [:attempts 0 :metrics metric-id :unit] :seconds) + (assoc-in [:attempts 1 :metrics metric-id :unit] :seconds))] + (doseq [candidate [mixed differing single-definition]] + (let [diagnostic (compare/compare-runs baseline candidate {:allow-build-differences true + :allow-scene-differences true + :on-unallowed-difference :record}) + allowed (compare/compare-runs baseline candidate {:allow-all-differences true}) + changes (get-in diagnostic [:pairs 0 :metrics metric-id])] + (t/is (every? #(= {:reason :inconsistent-metric-definitions} %) (vals changes))) + (t/is (= 2 (get-in diagnostic [:pairs 0 :candidate :metrics metric-id :counts :valid]))) + (t/is (every? #(false? (:waived? %)) (get-in diagnostic [:pairs 0 :mismatches]))) + (t/is (= 2 (get-in allowed [:pairs 0 :metrics metric-id :absolute :value]))) + (t/is (= 3.5 (get-in allowed [:pairs 0 :candidate :metrics metric-id :statistics :median :value]))) + (if (= candidate single-definition) + (t/is (= 3.5 (get-in diagnostic [:pairs 0 :candidate :metrics metric-id :statistics :median :value]))) + (t/is (= :inconsistent-metric-definitions + (get-in diagnostic [:pairs 0 :candidate :metrics metric-id :statistics :reason])))) + (t/is (= diagnostic (-> diagnostic result/encode result/decode))))))) + +(t/deftest feature-toggle-acknowledges-only-exact-effective-feature-evidence + (let [baseline (run-with-features [1] ["base" "old"] {:default-features? false :features ["base"]}) + candidate (run-with-features [2] ["base" "toggle" "transitive"] + {:default-features? false :features ["base" "toggle"]}) + expected {:feature "toggle" :baseline-features ["base" "old"] + :candidate-features ["base" "toggle" "transitive"] + :added ["toggle" "transitive"] :removed ["old"]} + edge (assoc expected :from "0" :to "1") + comparison (compare/compare-runs baseline candidate {:expected-feature-change expected :edge edge}) + mismatch (get-in comparison [:pairs 0 :mismatches 0])] + (t/is (= :effective-features (:field mismatch))) + (t/is (true? (:waived? mismatch))) + (t/is (= :feature-toggle (:rule mismatch))) + (t/is (= edge (:edge comparison))) + (t/is (= {:allow-build-differences false :allow-scene-differences false :allow-all-differences false} + (:permissions comparison))) + (t/is (= comparison (-> comparison result/encode result/decode))) + (doseq [invalid [nil + (assoc expected :feature "base") + (assoc expected :feature :toggle) + (assoc expected :baseline-features ["base"]) + (assoc expected :candidate-features ["base" "toggle"]) + (assoc expected :added ["toggle"]) + (assoc expected :removed []) + (assoc expected :added #{"toggle" "transitive"}) + (dissoc expected :baseline-features)]] + (let [options {:expected-feature-change invalid} + rejected (error-data #(compare/compare-runs baseline candidate options)) + diagnostic (compare/compare-runs baseline candidate (assoc options :on-unallowed-difference :record))] + (t/is (= ::compare/incompatible (:type rejected))) + (t/is (false? (get-in diagnostic [:pairs 0 :mismatches 0 :waived?]))) + (t/is (not (contains? (get-in diagnostic [:pairs 0 :mismatches 0]) :rule))))) + (let [changed (-> candidate + (assoc-in [:metadata :build :functional :mode] :debug) + (assoc-in [:plan :cases 0 :params] {:count 2000}) + (assoc-in [:plan :cases 0 :operation] {:steps 30})) + diagnostic (compare/compare-runs baseline changed {:expected-feature-change expected + :on-unallowed-difference :record})] + (t/is (= [[:scene false nil] [:execution false nil] + [:effective-features true :feature-toggle] [:functional-build false nil]] + (mapv (juxt :field :waived? :rule) (get-in diagnostic [:pairs 0 :mismatches]))))))) + +(t/deftest edge-context-roundtrips-and-labels-identical-effective-sets + (let [run (run-with-features [1] ["base" "alias"] {:default-features? false :features ["base"]}) + candidate (run-with-features [2] ["alias" "base"] {:default-features? false :features ["base" "toggle"]}) + expected {:feature "toggle" :baseline-features ["base" "alias"] + :candidate-features ["alias" "base"] :added [] :removed []} + edge (assoc expected :from "0" :to "1") + comparison (compare/compare-runs run candidate {:expected-feature-change expected :edge edge})] + (t/is (empty? (get-in comparison [:pairs 0 :mismatches]))) + (t/is (= edge (:edge comparison))) + (t/is (= comparison (-> comparison result/encode result/decode))) + (t/is (str/includes? (format/format-comparison comparison) "Edge 0 -> 1 feature toggle")) + (t/is (str/includes? (format/format-comparison comparison) "effective feature sets identical")) + (let [contradictory (assoc edge :candidate-features ["base" "alias" "toggle"] :added ["toggle"])] + (doseq [opts [{:edge contradictory} + {:edge contradictory + :expected-feature-change (dissoc contradictory :from :to)}]] + (t/is (= ::compare/invalid-edge + (:type (error-data #(compare/compare-runs run candidate opts))))))))) + +(t/deftest comparison-schema-validates-permissions-rules-and-edge-context + (let [baseline (run-with [1]) + candidate (run-with [2] {:facts (assoc-in metadata [:build :functional :mode] :debug)}) + comparison (compare/compare-runs baseline candidate {:allow-build-differences true}) + edge {:from "0" :to "1" :feature "toggle" :baseline-features [] + :candidate-features ["toggle"] :added ["toggle"] :removed []}] + (doseq [changed [(dissoc comparison :permissions) + (assoc-in comparison [:permissions :allow-build-differences] :yes) + (update comparison :permissions dissoc :allow-scene-differences) + (assoc comparison :forced? true) + (assoc-in comparison [:pairs 0 :mismatches 0 :rule] :other) + (assoc-in comparison [:pairs 0 :mismatches 0 :rule] nil) + (assoc-in comparison [:pairs 0 :mismatches 0 :waived?] false) + (update-in comparison [:pairs 0 :mismatches 0] dissoc :rule) + (assoc comparison :edge (dissoc edge :removed)) + (assoc comparison :edge (assoc edge :feature :toggle)) + (assoc comparison :edge (assoc edge :added #{"toggle"}))]] + (t/is (= ::result/invalid-record (:type (error-data #(result/decode (transit/encode-str changed))))))) + (doseq [rule [:feature-toggle :build :scene :all]] + (let [record (-> comparison (assoc :edge edge) (assoc-in [:pairs 0 :mismatches 0 :rule] rule))] + (t/is (= record (-> record result/encode result/decode))) + (t/is (= ::result/invalid-record + (:type (error-data #(result/decode (transit/encode-str + (update-in record [:pairs 0 :mismatches 0] dissoc :rule))))))))))) + +(t/deftest every-permission-keeps-invalid-measurements-and-no-shared-metrics-unusable + (let [baseline (run-with [js/NaN js/Infinity 0 -1 :absent :failed] {:warmups 1 :repetitions 7}) + candidate (run-with [2] {:warmups 1})] + (doseq [permissions [{} + {:allow-build-differences true} + {:allow-scene-differences true} + {:allow-build-differences true :allow-scene-differences true} + {:allow-all-differences true} + {:on-unallowed-difference :record}]] + (let [comparison (compare/compare-runs baseline candidate permissions)] + (t/is (= {:valid 0 :invalid 4 :absent 1 :failed 1 :unattempted 1 :warmup 1} + (get-in comparison [:pairs 0 :baseline :metrics metric-id :counts]))) + (t/is (= :no-valid-observations (get-in comparison [:pairs 0 :metrics metric-id :absolute :reason]))) + (t/is (= :no-valid-observations (get-in comparison [:pairs 0 :metrics metric-id :percentage-ci :reason]))) + (t/is (= comparison (-> comparison result/encode result/decode))) + (t/is (= ::compare/no-shared-metrics + (:type (error-data #(compare/compare-runs baseline + (update candidate :attempts + (fn [attempts] + (mapv (fn [attempt] + (assoc attempt :metrics {:other (entry 2)})) attempts))) + permissions))))))))) + +(t/deftest missing-nil-and-empty-effective-feature-evidence-stay-distinct-through-transit + (let [empty-run (run-with-features [1] [] {:default-features? false :features []}) + missing (-> empty-run + (update-in [:metadata :build :functional] dissoc :features) + result/encode result/decode) + nil-run (-> empty-run + (assoc-in [:metadata :build :functional :features] nil) + result/encode result/decode)] + (t/is (empty? (get-in (compare/compare-runs empty-run empty-run) [:pairs 0 :mismatches]))) + (doseq [[baseline candidate bfeatures cfeatures] + [[missing empty-run ::compare/missing-features #{}] + [nil-run empty-run nil #{}] + [missing nil-run ::compare/missing-features nil] + [empty-run missing #{} ::compare/missing-features] + [empty-run nil-run #{} nil]]] + (let [rejected (error-data #(compare/compare-runs baseline candidate)) + diagnostic (compare/compare-runs baseline candidate {:on-unallowed-difference :record}) + mismatch {:field :effective-features :baseline bfeatures :candidate cfeatures :waived? false}] + (t/is (= ::compare/incompatible (:type rejected))) + (t/is (= [mismatch] (:mismatches rejected))) + (t/is (= [mismatch] (get-in diagnostic [:pairs 0 :mismatches]))) + (t/is (= diagnostic (-> diagnostic result/encode result/decode))))))) + +(t/deftest automatic-feature-acknowledgment-requires-saved-effective-vectors-of-strings + (let [baseline (run-with-features [1] [] {:default-features? false :features []}) + candidate (run-with-features [2] ["toggle"] {:default-features? false :features ["toggle"]}) + expected {:feature "toggle" :baseline-features [] :candidate-features ["toggle"] + :added ["toggle"] :removed []}] + (t/is (= :feature-toggle + (get-in (compare/compare-runs baseline candidate {:expected-feature-change expected}) + [:pairs 0 :mismatches 0 :rule]))) + (doseq [[b c] [[(update-in baseline [:metadata :build :functional] dissoc :features) candidate] + [(assoc-in baseline [:metadata :build :functional :features] nil) candidate] + [(assoc-in baseline [:metadata :build :functional :features] #{}) candidate] + [baseline (assoc-in candidate [:metadata :build :functional :features] #{"toggle"})] + [baseline (assoc-in candidate [:metadata :build :functional :features] [:toggle])]]] + (let [b (-> b result/encode result/decode) + c (-> c result/encode result/decode) + options {:expected-feature-change expected} + rejected (error-data #(compare/compare-runs b c options)) + diagnostic (compare/compare-runs b c (assoc options :on-unallowed-difference :record))] + (t/is (= ::compare/incompatible (:type rejected))) + (t/is (= [:effective-features] (mapv :field (get-in diagnostic [:pairs 0 :mismatches])))) + (t/is (false? (get-in diagnostic [:pairs 0 :mismatches 0 :waived?]))) + (t/is (not (contains? (get-in diagnostic [:pairs 0 :mismatches 0]) :rule))) + (t/is (= diagnostic (-> diagnostic result/encode result/decode))))))) + +(t/deftest automatic-feature-acknowledgment-checks-saved-requested-selections + (let [b-request {:default-features? false :features ["base"]} + c-request {:default-features? false :features ["base" "toggle"]} + expected {:feature "toggle" :baseline-features ["base" "old"] + :candidate-features ["base" "toggle" "transitive"] + :added ["toggle" "transitive"] :removed ["old"]}] + (doseq [default-features? [false true]] + (let [baseline (run-with-features [1] (:baseline-features expected) + (assoc b-request :default-features? default-features?)) + candidate (run-with-features [2] (:candidate-features expected) + {:default-features? default-features? :features ["toggle" "base"]}) + comparison (compare/compare-runs (-> baseline result/encode result/decode) + (-> candidate result/encode result/decode) + {:expected-feature-change expected})] + (t/is (= [:feature-toggle] (mapv :rule (get-in comparison [:pairs 0 :mismatches])))) + (t/is (every? :waived? (get-in comparison [:pairs 0 :mismatches]))))) + (doseq [[b c] [[::missing c-request] + [b-request ::missing] + [nil c-request] + [b-request nil] + [{} c-request] + [b-request {}] + [[] c-request] + [(dissoc b-request :default-features?) c-request] + [b-request (dissoc c-request :default-features?)] + [(assoc b-request :default-features? :false) c-request] + [b-request (assoc c-request :default-features? true)] + [(dissoc b-request :features) c-request] + [b-request (dissoc c-request :features)] + [(assoc b-request :features #{"base"}) c-request] + [b-request (assoc c-request :features ["base" :toggle])] + [c-request c-request] + [b-request b-request] + [b-request (assoc c-request :features ["base" "extra" "toggle"])] + [(assoc b-request :features ["base" "old"]) c-request]]] + (let [baseline (cond-> (run-with-features [1] (:baseline-features expected) b) + (= ::missing b) (update-in [:metadata :build :provenance] dissoc :feature-selection)) + candidate (cond-> (run-with-features [2] (:candidate-features expected) c) + (= ::missing c) (update-in [:metadata :build :provenance] dissoc :feature-selection)) + baseline (-> baseline result/encode result/decode) + candidate (-> candidate result/encode result/decode) + options {:expected-feature-change expected} + rejected (error-data #(compare/compare-runs baseline candidate options)) + diagnostic (compare/compare-runs baseline candidate (assoc options :on-unallowed-difference :record))] + (t/is (= ::compare/incompatible (:type rejected))) + (t/is (= [:effective-features] (mapv :field (get-in diagnostic [:pairs 0 :mismatches])))) + (t/is (false? (get-in diagnostic [:pairs 0 :mismatches 0 :waived?]))) + (t/is (not (contains? (get-in diagnostic [:pairs 0 :mismatches 0]) :rule))) + (t/is (= diagnostic (-> diagnostic result/encode result/decode))))))) + +(t/deftest edge-option-retains-context-and-rejects-conflicting-expected-facts + (let [baseline (run-with-features [1] [] {:default-features? false :features []}) + candidate (run-with-features [2] ["toggle" "transitive"] {:default-features? false :features ["toggle"]}) + expected {:feature "toggle" :baseline-features [] :candidate-features ["toggle" "transitive"] + :added ["toggle" "transitive"] :removed []} + edge (assoc expected :from "0" :to "1") + comparison (compare/compare-runs baseline candidate {:expected-feature-change expected :edge edge})] + (t/is (= edge (:edge comparison))) + (t/is (= comparison (-> comparison result/encode result/decode))) + (t/is (str/includes? (format/format-comparison comparison) "Edge 0 -> 1 feature toggle")) + (t/is (not (str/includes? (format/format-comparison comparison) "effective feature sets identical"))) + (doseq [changed [(assoc edge :feature "other") + (assoc edge :baseline-features ["base"]) + (assoc edge :candidate-features ["toggle"]) + (assoc edge :added ["toggle"]) + (assoc edge :removed ["base"])]] + (t/is (= ::compare/invalid-edge + (:type (error-data #(compare/compare-runs baseline candidate + {:expected-feature-change expected :edge changed + :on-unallowed-difference :record})))))))) diff --git a/frontend/test/performance_tests/render_wasm/run_test.cljs b/frontend/test/performance_tests/render_wasm/run_test.cljs index b1fd73ea7e..51fce2cd5c 100644 --- a/frontend/test/performance_tests/render_wasm/run_test.cljs +++ b/frontend/test/performance_tests/render_wasm/run_test.cljs @@ -10,7 +10,9 @@ ["os" :as os] ["path" :as path] [app.common.transit :as transit] + [benches.render-wasm.feature-plan :as feature-plan] [benches.render-wasm.node.artifacts :as artifacts] + [benches.render-wasm.node.feature-comparison :as feature-comparison] [benches.render-wasm.node.run :as run] [benches.render-wasm.node.runs :as runs] [benches.render-wasm.result :as result] @@ -395,9 +397,7 @@ :close (fn [owner] (when-not (.-aborted (.-signal controller)) - (.abort controller - (ex-info "run interrupted" - {:type ::interrupted :signal signal}))) + (.abort controller signal)) (close owner))) fake (assoc base :service service) result (await (execute fake [(collected :case/a :fresh)] @@ -455,6 +455,25 @@ (t/is (map? (:runner-git diag))) (t/is (= :unverified (:runtime-helper-identity diag))))) +(t/deftest ^:async run-metadata-copies-only-saved-build-environment-evidence + (doseq [environment [::missing {} {"CFLAGS" "" "NODE_ENV" "recorded-build-value"}]] + (let [fake (fake-browser {}) + root (artifacts/frontend-root) + opts (options fake (output-dir)) + build (cond-> (selected-build (build-dir) "test-build") + (not= ::missing environment) (assoc-in [:manifest :environment] environment)) + home (runs/create-home {:root root :run-id (:run-id opts) :output-dir (:output-dir opts)}) + controller (js/AbortController.) + executed (await (run/execute root home build [(collected :case/a :fresh)] + opts (.-signal controller) nil nil)) + saved (saved-record executed) + facts (get-in saved [:metadata :build])] + (t/is (zero? (:exit-code executed))) + (t/is (= saved (:run executed))) + (if (= ::missing environment) + (t/is (not (contains? facts :environment))) + (t/is (= environment (:environment facts))))))) + (t/deftest ^:async remove-deletes-the-used-build-after-the-record-is-saved (let [root @temporary-root id "removable-build" @@ -508,3 +527,47 @@ (t/is (= 1 (:exit-code response))) (t/is (= "" (fs/readFileSync (path/join dir "index.html") "utf8"))))) + +(t/deftest ^:async comparison-stops-after-run-record-or-log-storage-fails + (doseq [filename ["run.transit" "run.log"]] + (let [root @temporary-root + out (output-dir) + cases [(collected :case/a :fresh)] + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir out + :options (dissoc (options nil out) :run-id :browser)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build + (fn [{:keys [selection on-created]}] + (let [dir (build-dir) + build (-> (selected-build dir (str (random-uuid))) + (assoc :manifest-path (path/join dir "build.transit")) + (assoc-in [:manifest :functional :default-features?] false) + (assoc-in [:manifest :functional :features] (:features selection)) + (assoc-in [:manifest :provenance :feature-selection] selection))] + (on-created build) + (js/Promise.resolve build))) + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal on-progress on-created] + (let [base (fake-browser {}) + close (:close (:service base)) + file (path/join (:dir home) filename) + service (assoc (:service base) :close + (fn [owner] + (close owner) + (fs/renameSync file (str file ".saved")) + (fs/mkdirSync file)))] + (run/execute root home selected cases (assoc opts :browser service) + signal on-progress on-created)))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8")) + evidence (path/join out "configs" "0" + (if (= filename "run.transit") "run.transit.saved" "run.transit"))] + (t/is (= 1 (count (:attempts (result/decode (fs/readFileSync evidence "utf8")))))) + (t/is (= 1 (:exit-code response))) + (t/is (= :failed (get-in saved [:configurations 0 :status]))) + (t/is (= :unattempted (get-in saved [:configurations 1 :status])))))) diff --git a/frontend/test/performance_tests/render_wasm/runner.cljs b/frontend/test/performance_tests/render_wasm/runner.cljs index feefeee772..4cca5bc713 100644 --- a/frontend/test/performance_tests/render_wasm/runner.cljs +++ b/frontend/test/performance_tests/render_wasm/runner.cljs @@ -18,6 +18,7 @@ [performance-tests.render-wasm.compare-test] [performance-tests.render-wasm.containers-test] [performance-tests.render-wasm.contracts-test] + [performance-tests.render-wasm.feature-comparison-test] [performance-tests.render-wasm.fingerprint-test] [performance-tests.render-wasm.geometry-scenes-test] [performance-tests.render-wasm.lifecycle-test] @@ -50,6 +51,7 @@ 'performance-tests.render-wasm.compare-test 'performance-tests.render-wasm.containers-test 'performance-tests.render-wasm.contracts-test + 'performance-tests.render-wasm.feature-comparison-test 'performance-tests.render-wasm.fingerprint-test 'performance-tests.render-wasm.geometry-scenes-test 'performance-tests.render-wasm.lifecycle-test diff --git a/frontend/test/performance_tests/render_wasm/runner_test.cljs b/frontend/test/performance_tests/render_wasm/runner_test.cljs index 5bbd2fd4aa..37d53cf298 100644 --- a/frontend/test/performance_tests/render_wasm/runner_test.cljs +++ b/frontend/test/performance_tests/render_wasm/runner_test.cljs @@ -78,8 +78,12 @@ (t/is (str/includes? help "runner run")) (t/is (str/includes? help "runner compare")) (t/is (str/includes? help "--build-timeout-ms")) - (t/is (str/includes? help "--cases")))) - (t/is (thrown-with-msg? js/Error #"expected build, purge, run or compare" + (t/is (str/includes? help "--cases")) + (t/is (str/includes? help "--allow-build-differences")) + (t/is (str/includes? help "--allow-scene-differences")) + (t/is (str/includes? help "--allow-all-differences")) + (t/is (not (str/includes? help "--force"))))) + (t/is (thrown-with-msg? js/Error #"expected build, purge, run, ab or compare" (runner/parse-args []))) (doseq [args [["unknown"] ["build" "purge"] ["build" "--unknown"] ["run" "extra"] ["compare" "only-one"]]] @@ -131,16 +135,31 @@ (t/is (= "compare" command)) (t/is (= ["compare" "baseline.transit" "candidate.transit"] arguments)) (t/is (nil? (:cases options))) - (t/is (not (:force options)))) + (t/is (not-any? options [:allow-build-differences :allow-scene-differences + :allow-all-differences]))) + (doseq [[flag option] [["--allow-build-differences" :allow-build-differences] + ["--allow-scene-differences" :allow-scene-differences] + ["--allow-all-differences" :allow-all-differences]] + selectors [[] ["--cases" "rects/pan,paths/pan"]]] + (let [{:keys [options]} + (runner/parse-args (into ["compare" "a.transit" "b.transit" flag] selectors))] + (t/is (= (when (seq selectors) "rects/pan,paths/pan") (:cases options))) + (t/is (true? (get options option))) + (t/is (not-any? options (remove #{option} [:allow-build-differences + :allow-scene-differences + :allow-all-differences]))))) (let [{:keys [options]} (runner/parse-args ["compare" "a.transit" "b.transit" - "--cases" "rects/pan,paths/pan" "--force"])] - (t/is (= "rects/pan,paths/pan" (:cases options))) - (t/is (true? (:force options)))) - (let [{:keys [options]} - (runner/parse-args ["compare" "a.transit" "b.transit" "--force"])] - (t/is (nil? (:cases options))) - (t/is (true? (:force options))))) + "--allow-build-differences" "--allow-scene-differences"])] + (t/is (true? (:allow-build-differences options))) + (t/is (true? (:allow-scene-differences options))) + (t/is (not (:allow-all-differences options))))) + +(t/deftest force-is-rejected-without-an-alias + (doseq [args [["compare" "a.transit" "b.transit" "--force"] + ["compare" "a.transit" "b.transit" "--force" "--allow-all-differences"] + ["compare" "a.transit" "b.transit" "--cases" "a,b" "--force"]]] + (t/is (thrown? js/Error (runner/parse-args args))))) (t/deftest commands-reject-flags-owned-by-other-commands (doseq [[args message] @@ -152,7 +171,12 @@ [["build" "--only-runs"] #"purge category flags require purge"] [["purge" "--seed" "1"] #"run options require run"] [["purge" "--cases" "a,b"] #"compare options require compare"]]] - (t/is (thrown-with-msg? js/Error message (runner/parse-args args))))) + (t/is (thrown-with-msg? js/Error message (runner/parse-args args)))) + (doseq [command ["build" "purge" "run"] + flag ["--allow-build-differences" "--allow-scene-differences" + "--allow-all-differences"]] + (t/is (thrown-with-msg? js/Error #"compare options require compare" + (runner/parse-args [command flag]))))) (t/deftest numeric-and-path-options-reject-invalid-bounds (doseq [args [["run" "--seed" "4294967296"] @@ -172,8 +196,8 @@ (t/deftest option-looking-values-and-filenames-are-not-command-flags (let [parsed (runner/parse-args ["compare" "--" "--seed" "candidate.transit"])] (t/is (= ["compare" "--seed" "candidate.transit"] (:arguments parsed)))) - (let [parsed (runner/parse-args ["run" "--filter" "--force"])] - (t/is (= "--force" (:filter (:options parsed)))))) + (let [parsed (runner/parse-args ["run" "--filter" "--allow-all-differences"])] + (t/is (= "--allow-all-differences" (:filter (:options parsed)))))) (t/deftest numeric-options-reject-blank-values (doseq [args [["run" "--seed" ""] @@ -181,3 +205,65 @@ ["run" "--repetitions" ""] ["run" "--timeout-ms" " "]]] (t/is (thrown? js/Error (runner/parse-args args))))) + +(t/deftest ab-accepts-run-budgets-and-comparison-permissions + (let [{:keys [command options]} + (runner/parse-args ["ab" "--features" " b,a,b " "--filter" "rects/load" + "--seed" "7" "--warmups" "0" "--repetitions" "2" + "--timeout-ms" "1000" "--build-timeout-ms" "2000" + "--startup-timeout-ms" "3000" "--output-dir" "new-home" + "--screenshots" "--allow-build-differences" + "--allow-scene-differences" "--allow-all-differences"])] + (t/is (= "ab" command)) + (t/is (= {:features ["a" "b"] :filter "rects/load" :seed 7 + :warmups 0 :repetitions 2 :timeout-ms 1000 :build-timeout-ms 2000 + :startup-timeout-ms 3000 :output-dir "new-home" :screenshots true + :allow-build-differences true :allow-scene-differences true :allow-all-differences true + :max-features 3 :confirm false :dry-run false :default-features false} + options)))) + +(t/deftest ab-rejects-missing-features-and-incompatible-flags-before-execution + (t/is (thrown-with-msg? js/Error #"at least one" (runner/parse-args ["ab" "--dry-run"]))) + (doseq [args [["--build" "id"] ["--remove"] ["--cases" "a,b"] + ["--default-features"] ["--no-default-features"]]] + (t/is (thrown-with-msg? js/Error #"ab rejects" + (runner/parse-args (into ["ab" "--features" "a" "--dry-run"] args))))) + (doseq [features ["" "a," "profile" "a/b"]] + (t/is (thrown? js/Error (runner/parse-args ["ab" "--features" features "--dry-run"])))) + (t/is (thrown-with-msg? js/Error #"purge category flags require purge" + (runner/parse-args ["ab" "--features" "a" "--only-comparisons"]))) + (t/is (thrown? js/Error (runner/parse-args ["ab" "extra" "--features" "a" "--dry-run"])))) + +(t/deftest ab-limits-and-confirmation-apply-at-the-parser-boundary + (t/is (thrown-with-msg? js/Error #"requires --confirm" (runner/parse-args ["ab" "--features" "a"]))) + (t/is (:confirm (:options (runner/parse-args ["ab" "--features" "a" "--confirm"])))) + (t/is (:dry-run (:options (runner/parse-args ["ab" "--features" "a" "--dry-run"])))) + (t/is (thrown-with-msg? js/Error #"exceed max-features" + (runner/parse-args ["ab" "--features" "a,b,c,d" "--dry-run"]))) + (t/is (thrown-with-msg? js/Error #"requires --confirm" + (runner/parse-args ["ab" "--features" "a,b,c,d" "--max-features" "4" "--filter" "rects"]))) + (doseq [flag ["--confirm" "--dry-run"]] + (t/is (= 4 (:max-features (:options (runner/parse-args ["ab" "--features" "a,b,c,d" "--max-features" "4" flag])))))) + (doseq [limit ["" " " "0" "7" "-1" "1.5" "NaN" "Infinity" "3x"]] + (t/is (thrown? js/Error (runner/parse-args ["ab" "--features" "a" "--max-features" limit "--dry-run"]))))) + +(t/deftest planning-flags-are-only-for-ab-and-help-explains-the-contract + (doseq [command [["build"] ["run"] ["purge"] ["compare" "before" "after"]] + flags [["--dry-run"] ["--confirm"] ["--max-features" "2"]]] + (t/is (thrown-with-msg? js/Error #"ab planning options require ab" + (runner/parse-args (into command flags))))) + (let [{:keys [summary]} (runner/parse-args ["ab" "--help"]) + help (runner/usage summary)] + (doseq [part ["runner ab --features" "--only-comparisons" "--max-features" + "--confirm" "--dry-run" "Cargo defaults disabled" + "feature-comparisons//feature-comparison.transit"]] + (t/is (str/includes? help part))))) + +(t/deftest comparison-purge-is-exclusive-and-only-for-purge + (t/is (true? (get-in (runner/parse-args ["purge" "--only-comparisons"]) [:options :only-comparisons]))) + (doseq [other ["--only-builds" "--only-runs"]] + (t/is (thrown-with-msg? js/Error #"mutually exclusive" + (runner/parse-args ["purge" "--only-comparisons" other])))) + (doseq [command [["build"] ["run"] ["compare" "before" "after"]]] + (t/is (thrown-with-msg? js/Error #"purge category flags require purge" + (runner/parse-args (conj command "--only-comparisons")))))) diff --git a/render-wasm/scripts/cargo-build.cjs b/render-wasm/scripts/cargo-build.cjs index 035703a03c..248c9ae2cf 100644 --- a/render-wasm/scripts/cargo-build.cjs +++ b/render-wasm/scripts/cargo-build.cjs @@ -15,6 +15,38 @@ async function main() { // A failed invocation must not leave facts from an earlier build. fs.rmSync(infoFile, { force: true }); + // This helper inherits the same shell setup as Cargo. Keep only build inputs, + // excluding credentials, invocation identities and artifact destinations. + const environmentKeys = new Set([ + "RUSTFLAGS", + "RUSTC_WRAPPER", + "CARGO_TARGET_DIR", + "CARGO_HOME", + "CC", + "CFLAGS", + "CXXFLAGS", + "LDFLAGS", + "PENPOT_WASM_FUNCTION_NAMES", + "CARGO_ENCODED_RUSTFLAGS", + "CARGO_BUILD_TARGET", + "BUILD_MODE", + "EMCC_CFLAGS", + "EM_INITIAL_HEAP", + "EM_MEMORY_GROWTH_GEOMETRIC_STEP", + "EM_MALLOC", + "EM_CACHE", + "NODE_ENV", + "CARGO_NET_OFFLINE", + ]); + const environment = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => + environmentKeys.has(key) || + key.startsWith("CARGO_PROFILE_") || + key.startsWith("EMSDK"), + ), + ); + const artifacts = []; const buildScripts = new Map(); let succeeded = false; @@ -99,6 +131,7 @@ async function main() { package_id: artifact.package_id, features: artifact.features, profile: artifact.profile, + environment, out_dir: outDir, artifact_dir: path.dirname(path.resolve(jsFiles[0])), shared_file: sharedFile, diff --git a/render-wasm/scripts/cargo-build.test.cjs b/render-wasm/scripts/cargo-build.test.cjs index e757c7cd95..6a988129f2 100644 --- a/render-wasm/scripts/cargo-build.test.cjs +++ b/render-wasm/scripts/cargo-build.test.cjs @@ -97,11 +97,12 @@ function cargoOutput(records) { return records.map((record) => JSON.stringify(record)).join("\n") + "\n"; } -function run(f, records) { +function run(f, records, env = {}) { return spawnSync(process.execPath, [helper, f.info, f.manifest], { cwd: f.root, input: cargoOutput(records), encoding: "utf8", + env, }); } @@ -126,6 +127,7 @@ for (const fresh of [false, true]) { package_id: f.artifact.package_id, features: f.artifact.features, profile: f.artifact.profile, + environment: {}, out_dir: f.outDir, artifact_dir: f.artifactDir, shared_file: path.join(f.outDir, "render_wasm_shared.js"), @@ -137,6 +139,127 @@ for (const fresh of [false, true]) { }); } +test("records only allowlisted values from the helper process environment", (t) => { + const f = fixture(t); + const environment = { + RUSTFLAGS: "-C target-feature=+simd128", + RUSTC_WRAPPER: "", + CARGO_TARGET_DIR: "/tmp/target with spaces", + CARGO_HOME: "/opt/cargo", + CC: "emcc", + CFLAGS: "-O3\n-g0", + CXXFLAGS: "-std=c++17", + LDFLAGS: "-sALLOW_MEMORY_GROWTH=1", + PENPOT_WASM_FUNCTION_NAMES: "1", + CARGO_ENCODED_RUSTFLAGS: "-C\u001fopt-level=3", + CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", + BUILD_MODE: "release", + EMCC_CFLAGS: "-O3 -sASSERTIONS=0", + CARGO_PROFILE_RELEASE_LTO: "true", + CARGO_PROFILE_CUSTOM_OPT_LEVEL: "z", + EMSDK: "/opt/emsdk", + EMSDK_NODE: "/opt/emsdk/node/bin/node", + EM_INITIAL_HEAP: "268435456", + EM_MEMORY_GROWTH_GEOMETRIC_STEP: "0.8", + EM_MALLOC: "dlmalloc", + EM_CACHE: "/tmp/emsdk_cache", + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + }; + const result = run(f, [f.artifact, f.buildScript, finished], { + ...environment, + SECRET_TOKEN: "must not be saved", + PATH: "/private/path", + HOME: "/private/home", + CARGO_REGISTRIES_PRIVATE_TOKEN: "must not be saved", + CARGO_PROFILE: "not the prefix", + RUSTFLAGS_EXTRA: "not an exact key", + EM_UNRELATED: "not allowlisted", + VERSION_TAG: "build-identity", + RENDER_BENCH_BUILD_ID: "build-identity", + FRONTEND_DEST: "/per-build/renderer", + EXPORT_DEST: "/per-build/export", + FRONTEND_SHARED_DEST: "/per-build/shared.js", + EXPORT_SHARED_DEST: "/per-build/export-shared.js", + RENDER_WASM_SHARED_ARTIFACT: "/per-build/shared.js", + RENDER_WASM_BUILD_INFO: f.info, + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(f.info, "utf8")).environment, + environment, + ); +}); + +test("preserves unset versus empty environment values", (t) => { + const f = fixture(t); + for (const environment of [{}, { CFLAGS: "" }, { CFLAGS: "-O3" }]) { + const result = run(f, [f.artifact, f.buildScript, finished], environment); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(f.info, "utf8")).environment, + environment, + ); + } +}); + +test("captures exported setup values after sourcing the actual build environment", (t) => { + const f = fixture(t); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + 'source "$1" frontend --offline; export EMSDK_NODE="/setup/node"; CFLAGS="not exported"; shift; exec "$@"', + "cargo-build-test", + path.resolve(__dirname, "..", "_build_env"), + process.execPath, + helper, + f.info, + f.manifest, + ], + { + cwd: f.root, + input: cargoOutput([f.artifact, f.buildScript, finished]), + encoding: "utf8", + env: { + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + RUSTFLAGS: "-C debuginfo=0", + RUSTC_WRAPPER: "", + EMCC_CFLAGS: "must be replaced by setup", + EM_INITIAL_HEAP: "1", + SECRET_TOKEN: "must not be saved", + VERSION_TAG: "per-build-identity", + FRONTEND_DEST: "/per-build/renderer", + }, + }, + ); + assert.equal(result.status, 0, result.stderr); + const environment = JSON.parse(fs.readFileSync(f.info, "utf8")).environment; + assert.deepEqual(environment, { + NODE_ENV: "production", + CARGO_NET_OFFLINE: "true", + RUSTFLAGS: "-C debuginfo=0 -C target-feature=+simd128", + RUSTC_WRAPPER: "", + CARGO_BUILD_TARGET: "wasm32-unknown-emscripten", + CARGO_TARGET_DIR: "target/frontend", + BUILD_MODE: "release", + CARGO_PROFILE_DIR: "release", + EMSDK_NODE: "/setup/node", + EM_INITIAL_HEAP: "268435456", + EM_MEMORY_GROWTH_GEOMETRIC_STEP: "0.8", + EM_MALLOC: "dlmalloc", + EM_CACHE: "/tmp/emsdk_cache", + EMCC_CFLAGS: environment.EMCC_CFLAGS, + }); + assert.match(environment.EMCC_CFLAGS, /^-O3 -sASSERTIONS=0 /); + assert.match(environment.EMCC_CFLAGS, /-sINITIAL_HEAP=268435456/); + assert.match(environment.EMCC_CFLAGS, /-sMEMORY_GROWTH_GEOMETRIC_STEP=0\.8/); + assert.match(environment.EMCC_CFLAGS, /-sMALLOC=dlmalloc/); +}); + test("records an empty actual feature set", (t) => { const f = fixture(t); const result = run(f, [