From 67d99366d983054e236ea88fbefada41f1350293 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Mon, 2 Nov 2026 14:48:34 +0100 Subject: [PATCH] :recycle: Command specific parsing AI-assisted-by: gpt-6.1-sol, deepseek-flash --- .../benches/render_wasm/node/cli.cljs | 111 +++++++ .../benches/render_wasm/node/cli/ab.cljs | 42 ++- .../benches/render_wasm/node/cli/build.cljs | 21 +- .../benches/render_wasm/node/cli/compare.cljs | 36 ++- .../benches/render_wasm/node/cli/console.cljs | 20 +- .../benches/render_wasm/node/cli/help.cljs | 16 +- .../benches/render_wasm/node/cli/merge.cljs | 17 +- .../benches/render_wasm/node/cli/purge.cljs | 24 +- .../render_wasm/node/cli/registry.cljs | 33 +++ .../benches/render_wasm/node/cli/run.cljs | 34 +++ .../benches/render_wasm/node/cli/shared.cljs | 58 +++- .../benches/render_wasm/node/request.cljs | 19 +- .../render_wasm/cli_test.cljs | 276 ++++++++++++++++++ .../performance_tests/render_wasm/runner.cljs | 2 + 14 files changed, 680 insertions(+), 29 deletions(-) create mode 100644 frontend/performance/benches/render_wasm/node/cli.cljs create mode 100644 frontend/performance/benches/render_wasm/node/cli/registry.cljs create mode 100644 frontend/test/performance_tests/render_wasm/cli_test.cljs diff --git a/frontend/performance/benches/render_wasm/node/cli.cljs b/frontend/performance/benches/render_wasm/node/cli.cljs new file mode 100644 index 0000000000..d791c581b9 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli.cljs @@ -0,0 +1,111 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns benches.render-wasm.node.cli + "Command-specific argument resolution and help for renderer benchmarks." + (:require + [benches.render-wasm.feature-plan :as feature-plan] + [benches.render-wasm.node.cli.console :as console] + [benches.render-wasm.node.cli.help :as help] + [benches.render-wasm.node.cli.registry :as registry] + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.request :as request] + [clojure.string :as str] + [clojure.tools.cli :as tools])) + +(def ^:private commands + (into {} registry/commands)) + +(defn- get-definition + [topic] + (if (nil? topic) + {:usage help/usage :description help/description :options []} + (get commands topic))) + +(defn- get-declarations + [definition] + (into help/cli-options (concat console/cli-options (:options definition)))) + +(defn- argument-error + [topic message cause] + (let [usage (or (:usage (get-definition topic)) help/usage)] + (ex-info (str message "\nUsage: " usage) + (assoc (ex-data cause) + :type ::invalid-args :cause-type (:type (ex-data cause)) + :command topic :usage usage) + cause))) + +(defn parse-resolve + "Returns a complete request for command-first argv, or throws an argument error. + A leading pnpm forwarding delimiter is accepted." + [argv] + (let [argv (if (= "--" (first argv)) (rest argv) argv) + first-arg (first argv) + topic (when (and first-arg (not (str/starts-with? first-arg "-"))) first-arg) + args (if topic (rest argv) argv)] + (when (and topic (not (contains? commands topic))) + (throw (argument-error topic (str "unknown command: " topic) nil))) + (let [definition (get-definition topic)] + (try + (let [{:keys [options arguments errors]} + (tools/parse-opts args (get-declarations definition) :no-defaults true) + _ (when (seq errors) + (throw (ex-info (str/join "\n" errors) {:type ::shared/invalid-args}))) + console (console/resolve-options options) + partial (if (:help options) + (do + (when (and (nil? topic) (seq arguments)) + (throw (ex-info "a command must precede its options" + {:type ::shared/invalid-args}))) + {:command "help" :console console :options {:topic topic}}) + (do + (when-not topic + (throw (ex-info "a command is required" {:type ::shared/invalid-args}))) + {:command topic :console console + :options ((:resolve definition) + (dissoc options :help :quiet :verbose :color) arguments)})) + resolved (request/check (request/apply-defaults partial))] + (if (and (not= "help" (:command resolved)) (:check definition)) + (update resolved :options (:check definition)) + resolved)) + (catch :default cause + (if (contains? #{::shared/invalid-args ::request/invalid-request + ::feature-plan/too-many-features ::feature-plan/confirmation-required} + (:type (ex-data cause))) + (throw (argument-error topic (ex-message cause) cause)) + (throw cause))))))) + +(defn- get-defaults + [topic] + (let [definition (get-definition topic) + partial {:command (or topic "help") :options (or (:help-options definition) {})} + resolved (request/apply-defaults partial) + options ((or (:help-defaults definition) identity) (:options resolved))] + (merge (:console resolved) options))) + +(defn get-help + "Returns root help for nil or help for a known command name." + [topic] + (let [definition (get-definition topic) + defaults (get-defaults topic) + summary (:summary + (tools/parse-opts + [] (get-declarations definition) + :no-defaults true + :summary-fn + (fn [specs] + (tools/summarize + (mapv (fn [{:keys [id] :as spec}] + (if (contains? defaults id) + (assoc spec :default (get defaults id) + :default-desc (pr-str (get defaults id))) + spec)) + specs)))))] + (str "Usage: " (:usage definition) "\n\n" (:description definition) "\n\n" + (when (nil? topic) + (str (str/join "\n" (map (fn [[_ definition]] (str " " (:usage definition))) registry/commands)) + "\n\n")) + "Options:\n" summary))) diff --git a/frontend/performance/benches/render_wasm/node/cli/ab.cljs b/frontend/performance/benches/render_wasm/node/cli/ab.cljs index 78d9fc3651..abc90ad406 100644 --- a/frontend/performance/benches/render_wasm/node/cli/ab.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/ab.cljs @@ -7,13 +7,49 @@ (ns benches.render-wasm.node.cli.ab "Options for comparing configurations of Cargo features." (:require - [benches.render-wasm.node.cli.shared :as shared])) + [benches.render-wasm.feature-plan :as feature-plan] + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.options :as options])) + +(def schema:max-features + [:int {:min 1 :max 6}]) (def schema:options [:merge shared/schema:measurement shared/schema:comparison-permissions [:map {:closed true} [:features [:vector {:min 1} shared/schema:feature]] - [:max-features {:default 3} [:int {:min 1 :max 6}]] + [:max-features {:default feature-plan/default-max-features} schema:max-features] [:confirm {:default false} :boolean] [:dry-run {:default false} :boolean] - [:build-timeout-ms {:default 300000} shared/schema:timeout-ms]]]) + [:build-timeout-ms {:default options/default-build-timeout-ms} shared/schema:timeout-ms]]]) + +(def ^:const usage "runner ab --features A,B [options]") + +(def ^:const description + "Compare configurations of Cargo features. + Managed output: frontend/target/performance/render-wasm/feature-comparisons//feature-comparison.transit") + +(def cli-options + (into [[nil "--features FEATURES" "Comma-separated functional Cargo features" + :parse-fn options/parse-feature-list] + (shared/number-option "--max-features N" "Maximum number of requested features" + schema:max-features "must be an integer from 1 to 6") + [nil "--confirm" "Confirm above eight configurations or with a blank filter"] + [nil "--dry-run" "Validate and print the plan"] + (shared/build-timeout-option "Deadline for metadata and each configuration build")] + (concat shared/measurement-options shared/comparison-options))) + +(defn resolve-options + "Resolves AB arguments and requires a feature selection." + [options arguments] + (shared/check-arguments arguments 0) + (when-not (contains? options :features) + (throw (ex-info "ab requires at least one requested feature (--features)" + {:type ::shared/invalid-args}))) + options) + +(defn check-options + "Checks feature limits and confirmation for resolved AB options." + [options] + (feature-plan/plan (:features options) options) + options) diff --git a/frontend/performance/benches/render_wasm/node/cli/build.cljs b/frontend/performance/benches/render_wasm/node/cli/build.cljs index 0bccf47ff5..a31d5d25bd 100644 --- a/frontend/performance/benches/render_wasm/node/cli/build.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/build.cljs @@ -7,10 +7,27 @@ (ns benches.render-wasm.node.cli.build "Options for preparing a benchmark build." (:require - [benches.render-wasm.node.cli.shared :as shared])) + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.options :as options])) (def schema:options [:map {:closed true} [:default-features {:default true} :boolean] [:features {:default []} [:vector shared/schema:feature]] - [:build-timeout-ms {:default 300000} shared/schema:timeout-ms]]) + [:build-timeout-ms {:default options/default-build-timeout-ms} shared/schema:timeout-ms]]) + +(def ^:const usage "runner build [options]") + +(def ^:const description "Prepare a renderer build with the selected Cargo features.") + +(def cli-options + [[nil "--[no-]default-features" "Enable Cargo defaults"] + [nil "--features FEATURES" "Comma-separated functional Cargo features" + :parse-fn options/parse-feature-list] + (shared/build-timeout-option "Complete preparation deadline")]) + +(defn resolve-options + "Resolves build arguments and supplied flags." + [options arguments] + (shared/check-arguments arguments 0) + options) diff --git a/frontend/performance/benches/render_wasm/node/cli/compare.cljs b/frontend/performance/benches/render_wasm/node/cli/compare.cljs index ce02839627..a409f2132d 100644 --- a/frontend/performance/benches/render_wasm/node/cli/compare.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/compare.cljs @@ -8,11 +8,43 @@ "Options for comparing saved benchmark records." (:require [app.common.schema :as sm] - [benches.render-wasm.node.cli.shared :as shared])) + [benches.render-wasm.node.cli.shared :as shared] + [clojure.string :as str])) + +(def schema:case-pair + [:tuple ::sm/text ::sm/text]) (def schema:options [:merge shared/schema:comparison-permissions [:map {:closed true} [:baseline-path ::sm/text] [:candidate-path ::sm/text] - [:case-pair {:default nil} [:maybe [:tuple ::sm/text ::sm/text]]]]]) + [:case-pair {:default nil} [:maybe schema:case-pair]]]]) + +(def ^:const usage "runner compare BASELINE CANDIDATE [options]") + +(def ^:const description + "Compare two saved records. Paths resolve from the caller's working directory. +--cases selects one exact baseline,candidate pair by saved case ID or name.") + +(def ^:private valid-case-pair? + (sm/validator schema:case-pair)) + +(defn- parse-case-pair + [text] + (let [pair (str/split text #"," -1)] + (when-not (valid-case-pair? pair) + (throw (ex-info "--cases requires one exact nonblank A,B pair" + {:type ::shared/invalid-args}))) + pair)) + +(def cli-options + (into [[nil "--cases A,B" "Select one baseline,candidate pair" :id :case-pair + :parse-fn parse-case-pair]] + shared/comparison-options)) + +(defn resolve-options + "Resolves two literal paths and supplied comparison flags." + [options arguments] + (shared/check-arguments arguments 2) + (assoc options :baseline-path (first arguments) :candidate-path (second arguments))) diff --git a/frontend/performance/benches/render_wasm/node/cli/console.cljs b/frontend/performance/benches/render_wasm/node/cli/console.cljs index e4218a41b2..c5ef86b793 100644 --- a/frontend/performance/benches/render_wasm/node/cli/console.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/console.cljs @@ -5,9 +5,27 @@ ;; Copyright (c) KALEIDOS INC (ns benches.render-wasm.node.cli.console - "Console settings carried by resolved benchmark requests.") + "Console settings carried by resolved benchmark requests." + (:require + [benches.render-wasm.node.cli.shared :as shared] + [benches.render-wasm.node.options :as options])) (def schema:console [:map {:closed true} [:level {:default :info} [:enum :info :warn :debug]] [:color {:default :auto} [:enum :auto :always :never]]]) + +(def cli-options + [["-q" "--quiet" "Only warnings, errors and results"] + ["-v" "--verbose" "Show debug details and raw build output"] + [nil "--[no-]color" "Colorize terminal output"]]) + +(defn resolve-options + "Resolves supplied console flags into console settings." + [options] + (when (and (:quiet options) (:verbose options)) + (throw (ex-info "--quiet and --verbose are mutually exclusive" + {:type ::shared/invalid-args}))) + (cond-> {} + (or (:quiet options) (:verbose options)) (assoc :level (options/console-level options)) + (contains? options :color) (assoc :color (options/color-mode options)))) diff --git a/frontend/performance/benches/render_wasm/node/cli/help.cljs b/frontend/performance/benches/render_wasm/node/cli/help.cljs index 82be6c0d3c..bb7ebe802c 100644 --- a/frontend/performance/benches/render_wasm/node/cli/help.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/help.cljs @@ -5,9 +5,21 @@ ;; Copyright (c) KALEIDOS INC (ns benches.render-wasm.node.cli.help - "Options for benchmark CLI help.") + "Options for benchmark CLI help." + (:require + [benches.render-wasm.node.cli.registry :as registry])) (def schema:options [:map {:closed true} [:topic {:default nil} - [:maybe [:enum "build" "run" "ab" "compare" "merge" "purge"]]]]) + [:maybe (into [:enum] registry/command-names)]]]) + +(def ^:const usage "runner COMMAND [options]") + +(def ^:const description + "Renderer benchmarks. Use COMMAND --help for command options. +Numeric options accept JavaScript number syntax, including exponents, radix +prefixes and surrounding whitespace, within each option's integer bounds.") + +(def cli-options + [["-h" "--help" "Print help"]]) diff --git a/frontend/performance/benches/render_wasm/node/cli/merge.cljs b/frontend/performance/benches/render_wasm/node/cli/merge.cljs index 8bf773c2f0..0ba0fa4ea2 100644 --- a/frontend/performance/benches/render_wasm/node/cli/merge.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/merge.cljs @@ -7,10 +7,25 @@ (ns benches.render-wasm.node.cli.merge "Options for merging saved benchmark records." (:require - [app.common.schema :as sm])) + [app.common.schema :as sm] + [benches.render-wasm.node.cli.shared :as shared])) (def schema:options [:map {:closed true} [:base-path ::sm/text] [:additional-path ::sm/text] [:output-dir {:default nil} [:maybe ::sm/text]]]) + +(def ^:const usage "runner merge BASE ADDITIONAL [options]") + +(def ^:const description + "Merge two saved records. Paths resolve from the caller's working directory.") + +(def cli-options + [shared/output-option]) + +(defn resolve-options + "Resolves two literal paths and the supplied output directory." + [options arguments] + (shared/check-arguments arguments 2) + (assoc options :base-path (first arguments) :additional-path (second arguments))) diff --git a/frontend/performance/benches/render_wasm/node/cli/purge.cljs b/frontend/performance/benches/render_wasm/node/cli/purge.cljs index 8920765f1d..0d777a3f9b 100644 --- a/frontend/performance/benches/render_wasm/node/cli/purge.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/purge.cljs @@ -5,8 +5,28 @@ ;; Copyright (c) KALEIDOS INC (ns benches.render-wasm.node.cli.purge - "Options for purging managed benchmark artifacts.") + "Options for purging managed benchmark artifacts." + (:require + [benches.render-wasm.node.cli.shared :as shared])) + +(def targets [:builds :runs :comparisons :all]) (def schema:options [:map {:closed true} - [:target [:enum :builds :runs :comparisons :all]]]) + [:target (into [:enum] targets)]]) + +(def ^:const usage "runner purge builds|runs|comparisons|all [options]") + +(def ^:const description "Purge files in one explicit category of managed benchmark artifacts.") + +(def cli-options []) + +(defn resolve-options + "Resolves the required artifact category." + [options arguments] + (shared/check-arguments arguments 1) + (let [target (some #(when (= (name %) (first arguments)) %) targets)] + (when-not target + (throw (ex-info "purge requires builds, runs, comparisons or all" + {:type ::shared/invalid-args}))) + (assoc options :target target))) diff --git a/frontend/performance/benches/render_wasm/node/cli/registry.cljs b/frontend/performance/benches/render_wasm/node/cli/registry.cljs new file mode 100644 index 0000000000..bc70470991 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/cli/registry.cljs @@ -0,0 +1,33 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns benches.render-wasm.node.cli.registry + "Ordered command definitions for parsing, request schemas and help." + (:require + [benches.render-wasm.node.cli.ab :as ab] + [benches.render-wasm.node.cli.build :as build] + [benches.render-wasm.node.cli.compare :as compare] + [benches.render-wasm.node.cli.merge :as merge] + [benches.render-wasm.node.cli.purge :as purge] + [benches.render-wasm.node.cli.run :as run])) + +(def commands + [["build" {:usage build/usage :description build/description + :options build/cli-options :resolve build/resolve-options :schema build/schema:options}] + ["run" {:usage run/usage :description run/description + :options run/cli-options :resolve run/resolve-options :schema run/schema:options + :help-options {:build {:kind :prepare}} :help-defaults run/get-help-defaults}] + ["ab" {:usage ab/usage :description ab/description + :options ab/cli-options :resolve ab/resolve-options :schema ab/schema:options + :check ab/check-options}] + ["compare" {:usage compare/usage :description compare/description + :options compare/cli-options :resolve compare/resolve-options :schema compare/schema:options}] + ["merge" {:usage merge/usage :description merge/description + :options merge/cli-options :resolve merge/resolve-options :schema merge/schema:options}] + ["purge" {:usage purge/usage :description purge/description + :options purge/cli-options :resolve purge/resolve-options :schema purge/schema:options}]]) + +(def command-names (mapv first commands)) diff --git a/frontend/performance/benches/render_wasm/node/cli/run.cljs b/frontend/performance/benches/render_wasm/node/cli/run.cljs index 4a156de47a..c9232c8081 100644 --- a/frontend/performance/benches/render_wasm/node/cli/run.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/run.cljs @@ -7,6 +7,7 @@ (ns benches.render-wasm.node.cli.run "Options for a benchmark run and its selected build source." (:require + [app.common.schema :as sm] [benches.render-wasm.node.cli.build :as build] [benches.render-wasm.node.cli.shared :as shared])) @@ -26,3 +27,36 @@ [:map {:closed true} [:build schema:build] [:remove {:default false} :boolean]]]) + +(def ^:const usage "runner run [options]") + +(def ^:const description + "Run cases using existing or new builds. +Managed output: frontend/target/performance/render-wasm/runs//run.transit") + +(def cli-options + (into [[nil "--build ID" "Reuse a ready build" :id :build-id + :validate [(sm/validator shared/schema:build-id) "must be a benchmark artifact ID"]]] + (concat build/cli-options shared/measurement-options + [[nil "--remove" "Remove the used build after shutdown"]]))) + +(defn resolve-options + "Selects an existing build or a preparation branch from supplied flags." + [options arguments] + (shared/check-arguments arguments 0) + (let [preparation (select-keys options [:default-features :features :build-timeout-ms])] + (when (and (contains? options :build-id) (seq preparation)) + (throw (ex-info "--build cannot be combined with --features, --[no-]default-features or --build-timeout-ms" + {:type ::shared/invalid-args}))) + (assoc (dissoc options :build-id :default-features :features :build-timeout-ms) + :build (if (contains? options :build-id) + {:kind :existing :id (:build-id options)} + (assoc preparation :kind :prepare))))) + +(defn get-help-defaults + "Returns CLI defaults for run options with a preparation source." + [options] + (assoc (dissoc options :build) + :default-features (get-in options [:build :default-features]) + :features (get-in options [:build :features]) + :build-timeout-ms (get-in options [:build :build-timeout-ms]))) diff --git a/frontend/performance/benches/render_wasm/node/cli/shared.cljs b/frontend/performance/benches/render_wasm/node/cli/shared.cljs index 6ec5af0ad4..3401af6232 100644 --- a/frontend/performance/benches/render_wasm/node/cli/shared.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/shared.cljs @@ -10,7 +10,8 @@ [app.common.schema :as sm] [benches.render-wasm.node.artifacts :as artifacts] [benches.render-wasm.node.options :as options] - [benches.render-wasm.random :as random])) + [benches.render-wasm.random :as random] + [clojure.string :as str])) (def schema:nonnegative-count [:int {:min 0 :max js/Number.MAX_SAFE_INTEGER}]) @@ -38,7 +39,7 @@ [:warmups {:default 3} schema:nonnegative-count] [:repetitions {:default 10} schema:positive-count] [:timeout-ms {:default 30000} schema:timeout-ms] - [:startup-timeout-ms {:default 30000} schema:timeout-ms] + [:startup-timeout-ms {:default options/default-startup-timeout-ms} schema:timeout-ms] [:output-dir {:default nil} [:maybe ::sm/text]] [:screenshots {:default false} :boolean] [:require-gpu {:default false} :boolean]]) @@ -48,3 +49,56 @@ [:allow-build-differences {:default false} :boolean] [:allow-scene-differences {:default false} :boolean] [:allow-all-differences {:default false} :boolean]]) + +(defn- parse-number + [text] + (if (str/blank? text) js/NaN (js/Number text))) + +(defn number-option + "Declares a numeric option with the given value constraint. + Values accept JavaScript number syntax, including exponents, radix prefixes + and surrounding whitespace." + [flag description schema message] + [nil flag description + :parse-fn parse-number + :validate [(sm/validator schema) message]]) + +(def output-option + [nil "--output-dir DIR" "Explicit output home; must not exist" + :validate [(sm/validator ::sm/text) "must be a nonblank path"]]) + +(def measurement-options + [(number-option "--seed N" "Master scene seed" schema:seed + "must be a uint32 integer") + [nil "--filter SUBSTR" "Select cases whose id contains SUBSTR"] + (number-option "--warmups N" "Warmup attempts per case" schema:nonnegative-count + "must be a nonnegative JavaScript-safe integer") + (number-option "--repetitions N" "Measured attempts per case" schema:positive-count + "must be a positive JavaScript-safe integer") + (number-option "--timeout-ms MS" "Attempt deadline" schema:timeout-ms + "must be a positive integer within Node's timer range") + (number-option "--startup-timeout-ms MS" "Host/browser readiness deadline" schema:timeout-ms + "must be a positive integer within Node's timer range") + output-option + [nil "--screenshots" "Capture one screenshot per case"] + [nil "--require-gpu" "Fail when GPU rendering is unavailable"]]) + +(def comparison-options + [[nil "--allow-build-differences" "Allow recorded build differences"] + [nil "--allow-scene-differences" "Allow scene inputs and fingerprint content differences"] + [nil "--allow-all-differences" "Allow all comparison compatibility differences"]]) + +(defn build-timeout-option + "Declares a preparation deadline with the given description." + [description] + (number-option "--build-timeout-ms MS" description schema:timeout-ms + "must be a positive integer within Node's timer range")) + +(defn check-arguments + "Checks the number of positional arguments." + [arguments expected] + (when-not (= expected (count arguments)) + (throw (ex-info (str "expected " expected " positional " + (if (= expected 1) "argument" "arguments") + ", received " (count arguments)) + {:type ::invalid-args :arguments arguments})))) diff --git a/frontend/performance/benches/render_wasm/node/request.cljs b/frontend/performance/benches/render_wasm/node/request.cljs index 916ef842b2..d7e62adad6 100644 --- a/frontend/performance/benches/render_wasm/node/request.cljs +++ b/frontend/performance/benches/render_wasm/node/request.cljs @@ -8,14 +8,9 @@ "Resolved benchmark requests, sender defaults and receiver validation." (:require [app.common.schema :as sm] - [benches.render-wasm.node.cli.ab :as ab] - [benches.render-wasm.node.cli.build :as build] - [benches.render-wasm.node.cli.compare :as compare] [benches.render-wasm.node.cli.console :as console] [benches.render-wasm.node.cli.help :as help] - [benches.render-wasm.node.cli.merge :as merge] - [benches.render-wasm.node.cli.purge :as purge] - [benches.render-wasm.node.cli.run :as run] + [benches.render-wasm.node.cli.registry :as registry] [malli.transform :as mt])) (defn- command-schema @@ -26,14 +21,10 @@ [:options {:default {}} options]]) (def schema:request - [:multi {:dispatch :command} - ["build" (command-schema "build" build/schema:options)] - ["run" (command-schema "run" run/schema:options)] - ["ab" (command-schema "ab" ab/schema:options)] - ["compare" (command-schema "compare" compare/schema:options)] - ["merge" (command-schema "merge" merge/schema:options)] - ["purge" (command-schema "purge" purge/schema:options)] - ["help" (command-schema "help" help/schema:options)]]) + (into [:multi {:dispatch :command}] + (map (fn [[command definition]] + [command (command-schema command (:schema definition))])) + (conj registry/commands ["help" {:schema help/schema:options}]))) (def ^:private default-decoder (delay (sm/decoder schema:request (mt/default-value-transformer)))) diff --git a/frontend/test/performance_tests/render_wasm/cli_test.cljs b/frontend/test/performance_tests/render_wasm/cli_test.cljs new file mode 100644 index 0000000000..312cb824be --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/cli_test.cljs @@ -0,0 +1,276 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns performance-tests.render-wasm.cli-test + (:require + [app.common.schema :as sm] + [benches.render-wasm.node.cli :as cli] + [benches.render-wasm.node.request :as request] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private measurement-defaults + {:seed 42 :filter nil :warmups 3 :repetitions 10 + :timeout-ms 30000 :startup-timeout-ms 30000 :output-dir nil + :screenshots false :require-gpu false}) + +(def ^:private comparison-defaults + {:allow-build-differences false :allow-scene-differences false + :allow-all-differences false}) + +(defn- resolved + [command options] + {:command command :console {:level :info :color :auto} :options options}) + +(defn- failure + [argv] + (try + (cli/parse-resolve argv) + nil + (catch :default cause cause))) + +(t/deftest commands-resolve-complete-requests + (doseq [[argv expected] + [[["build"] + (resolved "build" {:default-features true :features [] :build-timeout-ms 300000})] + [["run"] + (resolved "run" (assoc measurement-defaults :remove false + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000}))] + [["run" "--build" "ready-build"] + (resolved "run" (assoc measurement-defaults :remove false + :build {:kind :existing :id "ready-build"}))] + [["ab" "--features" "cache" "--dry-run"] + (resolved "ab" (merge measurement-defaults comparison-defaults + {:features ["cache"] :max-features 3 :confirm false + :dry-run true :build-timeout-ms 300000}))] + [["compare" "base" "candidate"] + (resolved "compare" (assoc comparison-defaults :baseline-path "base" + :candidate-path "candidate" :case-pair nil))] + [["compare" "base" "candidate" "--allow-build-differences" + "--allow-scene-differences" "--allow-all-differences"] + (resolved "compare" {:baseline-path "base" :candidate-path "candidate" :case-pair nil + :allow-build-differences true :allow-scene-differences true + :allow-all-differences true})] + [["ab" "--features" "cache" "--filter" "rects" "--require-gpu" "--screenshots" + "--allow-build-differences" "--allow-scene-differences" "--allow-all-differences"] + (resolved "ab" (assoc measurement-defaults :filter "rects" :require-gpu true + :screenshots true :features ["cache"] :max-features 3 + :confirm false :dry-run false :build-timeout-ms 300000 + :allow-build-differences true :allow-scene-differences true + :allow-all-differences true))] + [["merge" "base" "extra"] + (resolved "merge" {:base-path "base" :additional-path "extra" :output-dir nil})] + [["merge" " ./base " " ./extra " "--output-dir" " ./merged "] + (resolved "merge" {:base-path " ./base " :additional-path " ./extra " + :output-dir " ./merged "})] + [["purge" "comparisons"] (resolved "purge" {:target :comparisons})] + [["--help"] (resolved "help" {:topic nil})]]] + (t/testing (pr-str argv) + (t/is (= expected (cli/parse-resolve argv)))))) + +(t/deftest console-flags-compose-with-every-command + (doseq [argv [["build"] ["run"] ["ab" "--features" "cache" "--dry-run"] + ["compare" "base" "candidate"] ["merge" "base" "extra"] + ["purge" "runs"] ["--help"]] + [flags console] [[["-q" "--no-color"] {:level :warn :color :never} + [["-v" "--color"] {:level :debug :color :always}]]]] + (t/is (= console (:console (cli/parse-resolve (into argv flags)))))) + (doseq [argv [["run" "-qv"] ["build" "--quiet" "--verbose"] + ["compare" "--help" "-qv"] ["--help" "-qv"]]] + (let [error (failure argv)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "mutually exclusive"))))) + +(t/deftest explicit-build-and-measurement-values-survive-resolution + (t/is (= (resolved "build" {:default-features false :features ["a" "b"] + :build-timeout-ms 1000}) + (cli/parse-resolve ["--" "build" "--no-default-features" + "--features=b,a,b" "--build-timeout-ms" "1e3"]))) + (t/is (= {:command "run" :console {:level :info :color :never} + :options {:build {:kind :prepare :default-features false + :features ["a" "b"] :build-timeout-ms 5000} + :seed 0 :filter " rects " :warmups 0 :repetitions 2 + :timeout-ms 1000 :startup-timeout-ms 2000 :output-dir " ./runs " + :screenshots true :remove true :require-gpu true}} + (cli/parse-resolve + ["run" "--no-default-features" "--features" " b,a,b " + "--build-timeout-ms" "5000" "--seed" "0" "--filter" " rects " + "--warmups" "0" "--repetitions" "2" "--timeout-ms" "1000" + "--startup-timeout-ms" "2000" "--output-dir" " ./runs " + "--screenshots" "--remove" "--require-gpu" "--no-color"])))) + +(t/deftest retained-build-conflicts-use-supplied-flags + (doseq [flags [["--features" "cache"] ["--default-features"] + ["--no-default-features"] ["--build-timeout-ms=300000"]] + argv [(into ["run" "--build" "ready"] flags) + (into (into ["run"] flags) ["--build=ready"])]] + (t/is (str/includes? (ex-message (failure argv)) "--build cannot be combined"))) + (t/is (= (resolved "run" (assoc measurement-defaults :remove false :warmups 0 + :build {:kind :existing :id "ready"})) + (cli/parse-resolve ["run" "--build" "ready" "--warmups" "0"])))) + +(t/deftest parsers-reject-options-owned-by-other-commands + (doseq [[argv flags] + [[["build"] ["--warmups" "--cases" "--confirm"]] + [["run"] ["--cases" "--dry-run" "--allow-build-differences"]] + [["ab" "--features" "cache" "--dry-run"] + ["--build" "--remove" "--default-features" "--no-default-features" "--cases"]] + [["compare" "base" "candidate"] ["--output-dir" "--seed" "--features"]] + [["merge" "base" "extra"] ["--warmups" "--cases" "--allow-all-differences"]] + [["purge" "runs"] ["--features" "--seed" "--only-runs"]]] + flag flags] + (doseq [args [(conj argv flag) (conj argv "--help" flag)]] + (let [error (failure args)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "Unknown option")))))) + +(t/deftest command-first-syntax-and-positional-arguments-are-checked + (doseq [argv [[] ["unknown" "--help"] ["--help" "run"] ["-q" "build"] + ["build" "extra"] ["run" "extra"] + ["compare"] ["compare" "one"] ["compare" "one" "two" "three"] + ["compare" "" "candidate"] ["merge" "base"] ["merge" "base" " "] + ["run" "--build" "../escape"] ["run" "--output-dir" " "]]] + (t/testing (pr-str argv) + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv))))))) + (doseq [target ["builds" "runs" "comparisons" "all"]] + (t/is (= (resolved "purge" {:target (keyword target)}) + (cli/parse-resolve ["purge" target])))) + (doseq [args [[] ["unknown"] ["runs" "builds"]]] + (let [error (failure (into ["purge"] args))] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "runner purge builds|runs|comparisons|all"))))) + +(t/deftest argument-errors-include-command-and-validation-details + (let [data (ex-data (failure ["unknown" "--help"]))] + (t/is (= "unknown" (:command data))) + (t/is (= "runner COMMAND [options]" (:usage data)))) + (doseq [[argv path] [[["compare" "" "candidate"] [:options :baseline-path]] + [["merge" "base" " "] [:options :additional-path]]]] + (let [data (ex-data (failure argv))] + (t/is (= ::cli/invalid-args (:type data))) + (t/is (= ::request/invalid-request (:cause-type data))) + (t/is (some #(= path (:in %)) (:errors (::sm/explain data))))))) + +(t/deftest literal-paths-selectors-and-option-looking-values-are-preserved + (t/is (= (resolved "compare" (assoc comparison-defaults :baseline-path " ./base " + :candidate-path " ./candidate " + :case-pair [" rects/load " "other/load"])) + (cli/parse-resolve ["compare" " ./base " " ./candidate " + "--cases" " rects/load ,other/load"]))) + (t/is (= (resolved "compare" (assoc comparison-defaults :baseline-path "--seed" + :candidate-path "-candidate" :case-pair nil)) + (cli/parse-resolve ["compare" "--" "--seed" "-candidate"]))) + (doseq [text ["" " " "--allow-all-differences" "--" "-h"]] + (t/is (= (resolved "run" (assoc measurement-defaults :filter text :remove false + :build {:kind :prepare :default-features true + :features [] :build-timeout-ms 300000})) + (cli/parse-resolve ["run" "--filter" text]))))) + +(t/deftest malformed-selector-pairs-fail-before-record-loading + (doseq [text ["" "one" "one,two,three" ",two" "one," " ,two" "one, "]] + (let [error (failure ["compare" "base" "candidate" (str "--cases=" text)])] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) "--cases")))) + (t/is (= ::cli/invalid-args + (:type (ex-data (failure ["compare" "base" "candidate" "--cases"])))))) + +(t/deftest numeric-values-use-integer-and-timer-bounds + (doseq [[flag values] + [["--seed" ["0" "4294967295"]] + ["--warmups" ["0" "9007199254740991"]] + ["--repetitions" ["1" "9007199254740991"]] + ["--timeout-ms" ["1" "2147483647"]] + ["--startup-timeout-ms" ["1" "2147483647"]] + ["--build-timeout-ms" ["1" " 1000 " "1e3" "0x3e8" "0o1750" "0b1111101000" "2147483647"]]] + value values] + (t/is (= (js/Number value) + (let [options (:options (cli/parse-resolve ["run" flag value]))] + (if (= flag "--build-timeout-ms") + (get-in options [:build :build-timeout-ms]) + (get options (keyword (subs flag 2)))))))) + (doseq [[flag values] + [["--seed" ["-1" "4294967296" "1.5"]] + ["--warmups" ["-1" "9007199254740992" "0.5"]] + ["--repetitions" ["0" "9007199254740992" "1.5"]] + ["--timeout-ms" ["0" "2147483648" "1.5"]] + ["--startup-timeout-ms" ["0" "2147483648"]] + ["--build-timeout-ms" ["0" "2147483648" "1.5" "1000ms"]]] + value (concat values ["" " " "NaN" "Infinity"])] + (t/testing (str flag " " value) + (t/is (= ::cli/invalid-args (:type (ex-data (failure ["run" flag value])))))))) + +(t/deftest feature-selection-and-ab-policy-use-domain-rules + (doseq [command ["build" "run" "ab"] + [feature message] [["stats" "diagnostic Cargo features"] + ["profile" "diagnostic Cargo features"] + ["profile-macros" "diagnostic Cargo features"] + ["profile-raf" "diagnostic Cargo features"] + ["" "invalid Cargo feature selection"] + ["a,,b" "invalid Cargo feature selection"] + ["a/b" "invalid Cargo feature selection"]]] + (let [argv (cond-> [command "--features" feature] (= command "ab") (conj "--dry-run")) + error (failure argv)] + (t/is (= ::cli/invalid-args (:type (ex-data error)))) + (t/is (str/includes? (ex-message error) message)))) + (doseq [argv [["ab"] ["ab" "--features" "cache"] + ["ab" "--features" "cache" "--filter" " "] + ["ab" "--features" "a,b,c,d" "--dry-run"] + ["ab" "--features" "a,b,c,d" "--max-features" "4" "--filter" "rects"]]] + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv)))))) + (doseq [limit ["" " " "0" "7" "1.5" "NaN" "Infinity" "3x"]] + (t/is (= ::cli/invalid-args + (:type (ex-data (failure ["ab" "--features" "cache" "--dry-run" + "--max-features" limit])))))) + (doseq [flags [["--confirm"] ["--dry-run"] ["--filter" "rects"]]] + (t/is (= ["a" "b" "c"] + (get-in (cli/parse-resolve (into ["ab" "--features" "c,b,a,c"] flags)) + [:options :features])))) + (let [error (failure ["ab" "--features" "cache"])] + (t/is (str/includes? (ex-message (ex-cause error)) "--confirm")))) + +(t/deftest help-skips-required-command-arguments-and-keeps-syntax-checks + (doseq [command ["build" "run" "ab" "compare" "merge" "purge"]] + (t/is (= (resolved "help" {:topic command}) + (cli/parse-resolve [command "--help"])))) + (t/is (= (assoc (resolved "help" {:topic "compare"}) + :console {:level :debug :color :never}) + (cli/parse-resolve ["compare" "--help" "--verbose" "--no-color"]))) + (doseq [argv [["--help" "--unknown"] ["run" "--help" "--warmups" "bad"] + ["compare" "--help" "--cases="] ["ab" "--help" "--features"]]] + (t/is (= ::cli/invalid-args (:type (ex-data (failure argv))))))) + +(t/deftest help-shows-command-options-and-schema-defaults + (doseq [[topic flags] + [["build" #{"--[no-]default-features" "--features" "--build-timeout-ms"}] + ["run" #{"--build" "--[no-]default-features" "--features" "--build-timeout-ms" + "--seed" "--filter" "--warmups" "--repetitions" "--timeout-ms" + "--startup-timeout-ms" "--output-dir" "--screenshots" "--require-gpu" "--remove"}] + ["ab" #{"--features" "--max-features" "--confirm" "--dry-run" "--build-timeout-ms" + "--seed" "--filter" "--warmups" "--repetitions" "--timeout-ms" + "--startup-timeout-ms" "--output-dir" "--screenshots" "--require-gpu" + "--allow-build-differences" "--allow-scene-differences" "--allow-all-differences"}] + ["compare" #{"--cases" "--allow-build-differences" + "--allow-scene-differences" "--allow-all-differences"}] + ["merge" #{"--output-dir"}] + ["purge" #{}]]] + (let [text (cli/get-help topic)] + (t/is (str/includes? text (str "Usage: runner " topic))) + (t/is (= (into #{"--help" "--quiet" "--verbose" "--[no-]color"} flags) + (set (re-seq #"--(?:\[no-\])?[a-z][a-z-]*" text)))))) + (t/is (= ["build" "run" "ab" "compare" "merge" "purge"] + (mapv second (re-seq #"(?m)^ runner ([a-z]+)" (cli/get-help nil))))) + (doseq [[topic pattern] [["build" #"--build-timeout-ms MS\s+300000"] + ["run" #"--warmups N\s+3"] + ["run" #"--seed N\s+42"] + ["run" #"--repetitions N\s+10"] + ["run" #"--features FEATURES\s+\[\]"] + ["run" #"--\[no-\]default-features\s+true"] + ["run" #"--build-timeout-ms MS\s+300000"] + ["ab" #"--max-features N\s+3"] + ["merge" #"--output-dir DIR\s+nil"] + ["compare" #"--cases A,B\s+nil"]]] + (t/is (re-find pattern (cli/get-help topic))))) diff --git a/frontend/test/performance_tests/render_wasm/runner.cljs b/frontend/test/performance_tests/render_wasm/runner.cljs index 7117c8c137..d55629bffd 100644 --- a/frontend/test/performance_tests/render_wasm/runner.cljs +++ b/frontend/test/performance_tests/render_wasm/runner.cljs @@ -15,6 +15,7 @@ [performance-tests.render-wasm.browser-test] [performance-tests.render-wasm.build-test] [performance-tests.render-wasm.builder-test] + [performance-tests.render-wasm.cli-test] [performance-tests.render-wasm.compare-test] [performance-tests.render-wasm.containers-test] [performance-tests.render-wasm.contracts-test] @@ -53,6 +54,7 @@ 'performance-tests.render-wasm.browser-test 'performance-tests.render-wasm.build-test 'performance-tests.render-wasm.builder-test + 'performance-tests.render-wasm.cli-test 'performance-tests.render-wasm.compare-test 'performance-tests.render-wasm.containers-test 'performance-tests.render-wasm.contracts-test