♻️ Use per-command schemas in benchmark cli

frontend/performance/benches/render_wasm/node/artifacts.cljs

@@ -5,8 +5,7 @@
 ;; Copyright (c) KALEIDOS INC
 
 (ns benches.render-wasm.node.artifacts
-  "Outputs of renderer builds managed under
-  frontend/target/performance/render-wasm.
+  "Managed renderer builds and output homes for benchmark runs and comparisons.
 
   Here, 'managed' means that the runner owns the build directory and its
   contents, which can be deleted by means of the `purge` command."
@@ -16,6 +15,7 @@
    [app.common.schema :as sm]
    [app.common.transit :as transit]
    [benches.render-wasm.codec :as codec]
+   [benches.render-wasm.errors :as errors]
    [benches.render-wasm.node.files :as files]
    [benches.render-wasm.node.time :as time]
    [clojure.string :as str]))
@@ -67,12 +67,82 @@
         :else
         (recur parent)))))
 
+(defn valid-id?
+  "True for IDs of up to 128 letters, digits, underscores or hyphens,
+  starting with a letter or digit."
+  [id]
+  (and (string? id)
+       (boolean (re-matches #"[A-Za-z0-9][A-Za-z0-9_-]{0,127}" id))))
+
 (defn- check-id
   [id]
-  (when-not (files/valid-id? id)
+  (when-not (valid-id? id)
     (throw (ex-info "invalid renderer build id" {:type ::invalid-id :id id})))
   id)
 
+(defn invalid-home
+  "Throws an ::invalid-home exception with `data` and an optional cause."
+  ([message data]
+   (throw (ex-info message (assoc data :type ::invalid-home))))
+  ([message data cause]
+   (throw (ex-info message (assoc data :type ::invalid-home) cause))))
+
+(defn check-home-id
+  "Returns `id` when it is valid, or throws ::invalid-home with `message`."
+  [id message]
+  (when-not (valid-id? id)
+    (invalid-home message {:id id}))
+  id)
+
+(defn check-output-dir
+  "Resolves an explicit user directory from the caller's cwd and checks that it
+  does not exist. Returns nil for nil. Does not create anything."
+  [dir]
+  (when (some? dir)
+    (when (str/blank? dir)
+      (invalid-home "output directory requires a nonempty path" {}))
+    (let [resolved (path/resolve dir)
+          exists?  (try
+                     (fs/lstatSync resolved #js {:throwIfNoEntry false})
+                     (catch :default cause
+                       (let [detail (errors/message-of cause)]
+                         (invalid-home (str "cannot inspect output directory: " detail)
+                                       {:path resolved :cause detail}
+                                       cause))))]
+      (when exists?
+        (invalid-home "output directory already exists" {:path resolved}))
+      resolved)))
+
+(defn- create-home*
+  [dir stem]
+  (let [log-path (path/join dir (str stem ".log"))]
+    (fs/mkdirSync dir #js {:recursive true})
+    (fs/writeFileSync log-path "" #js {:encoding "utf8" :flag "wx"})
+    {:dir dir
+     :log-path log-path
+     :record-path (path/join dir (str stem ".transit"))}))
+
+(defn create-home
+  "Creates `dir` and its parents, reserves an empty log.
+
+  A filesystem failure throws ::invalid-home with `message` and `:path`, and
+  keeps the original cause.
+
+  Args:
+      dir: path to create.
+      stem: names the .log and .transit files.
+      message: used for errors
+
+  Returns:
+      Directory created, log path and record path."
+  [dir stem message]
+  (try
+    (create-home* dir stem)
+    (catch :default cause
+      (invalid-home message
+                    {:path dir :cause (errors/message-of cause)}
+                    cause))))
+
 (defn- checked-path
   [root file]
   (let [resolved (fs/realpathSync file)]

frontend/performance/benches/render_wasm/node/cli/ab.cljs

@@ -0,0 +1,19 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.ab
+  "Options for comparing configurations of Cargo features."
+  (:require
+   [benches.render-wasm.node.cli.shared :as shared]))
+
+(def schema:options
+  [:merge shared/schema:measurement shared/schema:comparison-permissions
+   [:map {:closed true}
+    [:features [:vector {:min 1} shared/schema:feature]]
+    [:max-features {:default 3} [:int {:min 1 :max 6}]]
+    [:confirm {:default false} :boolean]
+    [:dry-run {:default false} :boolean]
+    [:build-timeout-ms {:default 300000} shared/schema:timeout-ms]]])

frontend/performance/benches/render_wasm/node/cli/build.cljs

@@ -0,0 +1,16 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.build
+  "Options for preparing a benchmark build."
+  (:require
+   [benches.render-wasm.node.cli.shared :as shared]))
+
+(def schema:options
+  [:map {:closed true}
+   [:default-features {:default true} :boolean]
+   [:features {:default []} [:vector shared/schema:feature]]
+   [:build-timeout-ms {:default 300000} shared/schema:timeout-ms]])

frontend/performance/benches/render_wasm/node/cli/compare.cljs

@@ -0,0 +1,18 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.compare
+  "Options for comparing saved benchmark records."
+  (:require
+   [app.common.schema :as sm]
+   [benches.render-wasm.node.cli.shared :as shared]))
+
+(def schema:options
+  [:merge shared/schema:comparison-permissions
+   [:map {:closed true}
+    [:baseline-path ::sm/text]
+    [:candidate-path ::sm/text]
+    [:case-pair {:default nil} [:maybe [:tuple ::sm/text ::sm/text]]]]])

frontend/performance/benches/render_wasm/node/cli/console.cljs

@@ -0,0 +1,13 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.console
+  "Console settings carried by resolved benchmark requests.")
+
+(def schema:console
+  [:map {:closed true}
+   [:level {:default :info} [:enum :info :warn :debug]]
+   [:color {:default :auto} [:enum :auto :always :never]]])

frontend/performance/benches/render_wasm/node/cli/help.cljs

@@ -0,0 +1,13 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.help
+  "Options for benchmark CLI help.")
+
+(def schema:options
+  [:map {:closed true}
+   [:topic {:default nil}
+    [:maybe [:enum "build" "run" "ab" "compare" "merge" "purge"]]]])

frontend/performance/benches/render_wasm/node/cli/merge.cljs

@@ -0,0 +1,16 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.merge
+  "Options for merging saved benchmark records."
+  (:require
+   [app.common.schema :as sm]))
+
+(def schema:options
+  [:map {:closed true}
+   [:base-path ::sm/text]
+   [:additional-path ::sm/text]
+   [:output-dir {:default nil} [:maybe ::sm/text]]])

frontend/performance/benches/render_wasm/node/cli/purge.cljs

@@ -0,0 +1,12 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.purge
+  "Options for purging managed benchmark artifacts.")
+
+(def schema:options
+  [:map {:closed true}
+   [:target [:enum :builds :runs :comparisons :all]]])

frontend/performance/benches/render_wasm/node/cli/run.cljs

@@ -0,0 +1,28 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.run
+  "Options for a benchmark run and its selected build source."
+  (:require
+   [benches.render-wasm.node.cli.build :as build]
+   [benches.render-wasm.node.cli.shared :as shared]))
+
+(def schema:build
+  [:multi {:dispatch :kind}
+   [:prepare
+    [:merge build/schema:options
+     [:map {:closed true}
+      [:kind [:= :prepare]]]]]
+   [:existing
+    [:map {:closed true}
+     [:kind [:= :existing]]
+     [:id shared/schema:build-id]]]])
+
+(def schema:options
+  [:merge shared/schema:measurement
+   [:map {:closed true}
+    [:build schema:build]
+    [:remove {:default false} :boolean]]])

frontend/performance/benches/render_wasm/node/cli/shared.cljs

@@ -0,0 +1,50 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.cli.shared
+  "Value constraints and option groups shared by benchmark commands."
+  (:require
+   [app.common.schema :as sm]
+   [benches.render-wasm.node.artifacts :as artifacts]
+   [benches.render-wasm.node.options :as options]
+   [benches.render-wasm.random :as random]))
+
+(def schema:nonnegative-count
+  [:int {:min 0 :max js/Number.MAX_SAFE_INTEGER}])
+
+(def schema:positive-count
+  [:int {:min 1 :max js/Number.MAX_SAFE_INTEGER}])
+
+(def schema:timeout-ms
+  "Positive integer milliseconds within Node's timer range."
+  [:int {:min 1 :max 2147483647}])
+
+(def schema:seed
+  [:fn {:error/message "expected a uint32 seed"} random/seed?])
+
+(def schema:build-id
+  [:fn {:error/message "expected a benchmark artifact ID"} artifacts/valid-id?])
+
+(def schema:feature
+  [:fn {:error/message "expected a Cargo feature name"} options/feature-name?])
+
+(def schema:measurement
+  [:map {:closed true}
+   [:seed {:default 42} schema:seed]
+   [:filter {:default nil} [:maybe :string]]
+   [:warmups {:default 3} schema:nonnegative-count]
+   [:repetitions {:default 10} schema:positive-count]
+   [:timeout-ms {:default 30000} schema:timeout-ms]
+   [:startup-timeout-ms {:default 30000} schema:timeout-ms]
+   [:output-dir {:default nil} [:maybe ::sm/text]]
+   [:screenshots {:default false} :boolean]
+   [:require-gpu {:default false} :boolean]])
+
+(def schema:comparison-permissions
+  [:map {:closed true}
+   [:allow-build-differences {:default false} :boolean]
+   [:allow-scene-differences {:default false} :boolean]
+   [:allow-all-differences {:default false} :boolean]])

frontend/performance/benches/render_wasm/node/feature_comparison.cljs

@@ -43,14 +43,14 @@
   An explicit directory resolves from the caller's cwd and must not exist.
   Paths inside the managed builds tree are rejected."
   [root comparison-id output-dir]
-  (files/check-id comparison-id "invalid feature comparison id")
+  (artifacts/check-home-id comparison-id "invalid feature comparison id")
   (let [default-dir (path/join (artifacts/managed-root root)
                                "feature-comparisons" comparison-id)
-        dir         (files/check-output-dir (or output-dir default-dir))]
+        dir         (artifacts/check-output-dir (or output-dir default-dir))]
     (when (artifacts/inside-builds? root dir)
-      (files/invalid-home "feature comparison cannot use a directory inside builds" {:path dir}))
-    (files/create-home dir "feature-comparison"
-                       "cannot create feature comparison home")))
+      (artifacts/invalid-home "feature comparison cannot use a directory inside builds" {:path dir}))
+    (artifacts/create-home dir "feature-comparison"
+                           "cannot create feature comparison home")))
 
 (defn write-record
   "Validates schema:feature-comparison and atomically replaces the saved file."

frontend/performance/benches/render_wasm/node/feature_plan.cljs

@@ -11,7 +11,7 @@
    ["path" :as path]
    [benches.render-wasm.cases :as cases]
    [benches.render-wasm.feature-plan :as feature-plan]
-   [benches.render-wasm.node.files :as files]
+   [benches.render-wasm.node.artifacts :as artifacts]
    [benches.render-wasm.node.options :as options]
    [benches.render-wasm.node.process :as process]
    [clojure.string :as str]))
@@ -41,7 +41,7 @@
                           opts
                           {:default-features false})
         plan       (feature-plan/plan (:features opts) opts)
-        output-dir (files/check-output-dir (:output-dir opts))
+        output-dir (artifacts/check-output-dir (:output-dir opts))
         cwd        (path/resolve root ".." "render-wasm")
         metadata   (await (process/run
                            {:command "cargo"

frontend/performance/benches/render_wasm/node/files.cljs

@@ -9,16 +9,8 @@
   (:require
    ["fs" :as fs]
    ["path" :as path]
-   [benches.render-wasm.errors :as errors]
    [clojure.string :as str]))
 
-(defn valid-id?
-  "True for IDs of up to 128 letters, digits, underscores or hyphens,
-  starting with a letter or digit."
-  [id]
-  (and (string? id)
-       (boolean (re-matches #"[A-Za-z0-9][A-Za-z0-9_-]{0,127}" id))))
-
 (defn contained?
   "True when `file` is `root` or a path below it."
   [root file]
@@ -27,69 +19,6 @@
          (not= relative "..")
          (not (str/starts-with? relative (str ".." path/sep))))))
 
-(defn invalid-home
-  "Throws an ::invalid-home exception with `data` and an optional cause."
-  ([message data]
-   (throw (ex-info message (assoc data :type ::invalid-home))))
-  ([message data cause]
-   (throw (ex-info message (assoc data :type ::invalid-home) cause))))
-
-(defn check-id
-  "Returns `id` when it is valid, or throws ::invalid-home with `message`."
-  [id message]
-  (when-not (valid-id? id)
-    (invalid-home message {:id id}))
-  id)
-
-(defn check-output-dir
-  "Resolves an explicit user directory from the caller's cwd and checks that it
-  does not exist. Returns nil for nil. Does not create anything."
-  [dir]
-  (when (some? dir)
-    (when (str/blank? dir)
-      (invalid-home "output directory requires a nonempty path" {}))
-    (let [resolved (path/resolve dir)
-          exists?  (try
-                     (fs/lstatSync resolved #js {:throwIfNoEntry false})
-                     (catch :default cause
-                       (let [detail (errors/message-of cause)]
-                         (invalid-home (str "cannot inspect output directory: " detail)
-                                       {:path resolved :cause detail}
-                                       cause))))]
-      (when exists?
-        (invalid-home "output directory already exists" {:path resolved}))
-      resolved)))
-
-(defn- create-home*
-  [dir stem]
-  (let [log-path (path/join dir (str stem ".log"))]
-    (fs/mkdirSync dir #js {:recursive true})
-    (fs/writeFileSync log-path "" #js {:encoding "utf8" :flag "wx"})
-    {:dir dir
-     :log-path log-path
-     :record-path (path/join dir (str stem ".transit"))}))
-
-(defn create-home
-  "Creates `dir` and its parents, reserves an empty log.
-
-  A filesystem failure throws ::invalid-home with `message` and `:path`, and
-  keeps the original cause.
-
-  Args:
-      dir: path to create.
-      stem: names the .log and .transit files.
-      message: used for errors
-
-  Returns:
-      Directory created, log path and record path."
-  [dir stem message]
-  (try
-    (create-home* dir stem)
-    (catch :default cause
-      (invalid-home message
-                    {:path dir :cause (errors/message-of cause)}
-                    cause))))
-
 (defn get-real-path
   "Resolves symlink ancestors even when the final path does not exist yet."
   [file]

frontend/performance/benches/render_wasm/node/options.cljs

@@ -14,13 +14,19 @@
 
 (def ^:private diagnostic-features #{"stats" "profile" "profile-macros" "profile-raf"})
 
+(defn feature-name?
+  "True for a Cargo feature name made of letters, digits, underscores or hyphens."
+  [value]
+  (and (string? value)
+       (boolean (re-matches #"[A-Za-z0-9_-]+" value))))
+
 (defn feature-selection
   "Validates measurement feature selection before allocating an attempt.
   Cargo validates functional feature names during the retained build."
   [{:keys [default-features? features]
     :or {default-features? true features []}}]
   (when-not (and (boolean? default-features?) (vector? features)
-                 (every? #(and (string? %) (re-matches #"[A-Za-z0-9_-]+" %)) features))
+                 (every? feature-name? features))
     (throw (ex-info "invalid Cargo feature selection" {:type ::invalid-features})))
   (when-let [selected (seq (filter diagnostic-features features))]
     (throw (ex-info "measurement builds cannot enable diagnostic Cargo features"

frontend/performance/benches/render_wasm/node/request.cljs

@@ -0,0 +1,50 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns benches.render-wasm.node.request
+  "Resolved benchmark requests, sender defaults and receiver validation."
+  (:require
+   [app.common.schema :as sm]
+   [benches.render-wasm.node.cli.ab :as ab]
+   [benches.render-wasm.node.cli.build :as build]
+   [benches.render-wasm.node.cli.compare :as compare]
+   [benches.render-wasm.node.cli.console :as console]
+   [benches.render-wasm.node.cli.help :as help]
+   [benches.render-wasm.node.cli.merge :as merge]
+   [benches.render-wasm.node.cli.purge :as purge]
+   [benches.render-wasm.node.cli.run :as run]
+   [malli.transform :as mt]))
+
+(defn- command-schema
+  [command options]
+  [:map {:closed true}
+   [:command [:= command]]
+   [:console {:default {}} console/schema:console]
+   [:options {:default {}} options]])
+
+(def schema:request
+  [:multi {:dispatch :command}
+   ["build" (command-schema "build" build/schema:options)]
+   ["run" (command-schema "run" run/schema:options)]
+   ["ab" (command-schema "ab" ab/schema:options)]
+   ["compare" (command-schema "compare" compare/schema:options)]
+   ["merge" (command-schema "merge" merge/schema:options)]
+   ["purge" (command-schema "purge" purge/schema:options)]
+   ["help" (command-schema "help" help/schema:options)]])
+
+(def ^:private default-decoder
+  (delay (sm/decoder schema:request (mt/default-value-transformer))))
+
+(defn apply-defaults
+  "Fills contextual defaults for the selected command and build source."
+  [partial-request]
+  (@default-decoder partial-request))
+
+(def check
+  "Returns a complete valid request, or throws with schema explain data."
+  (sm/check-fn schema:request
+               :hint "invalid renderer benchmark request"
+               :type ::invalid-request))

frontend/performance/benches/render_wasm/node/runs.cljs

@@ -24,12 +24,12 @@
   after the plan exists. An explicit directory must not exist; a managed home
   uses the frontend root regardless of cwd."
   [{:keys [root run-id output-dir remove]}]
-  (files/check-id run-id "invalid run id")
-  (let [dir (files/check-output-dir
+  (artifacts/check-home-id run-id "invalid run id")
+  (let [dir (artifacts/check-output-dir
              (or output-dir (path/join (artifacts/managed-root root) "runs" run-id)))]
     (when (and remove (artifacts/inside-builds? root dir))
-      (files/invalid-home "--remove cannot use an output directory inside builds" {:path dir}))
-    (assoc (files/create-home dir "run" "cannot create run home") :run-id run-id)))
+      (artifacts/invalid-home "--remove cannot use an output directory inside builds" {:path dir}))
+    (assoc (artifacts/create-home dir "run" "cannot create run home") :run-id run-id)))
 
 (defn get-record
   "Reads and validates a saved run record from a path relative to cwd."

frontend/test/performance_tests/render_wasm/cli_integration.cjs

@@ -4,6 +4,20 @@
 //
 // Copyright (c) KALEIDOS SUBSIDIARY SL
 
+// Our CLJS test suite already covers the internals:
+// - runner_test.cljs tests parse-args,
+// - build_test.cljs tests preparation with fake compilers,
+// - feature_comparison_test.cljs tests orchestration.
+// Those compile to an :esm bundle with :runtime :custom and run in-process from
+// target/performance-tests/test.js.
+//
+// It cannot test the assembled executable, so here we test:
+// - exit codes of the real process, including 130/143 for SIGINT/SIGTERM,
+// - stdout and stderr separation as the shell sees it,
+// - that runner.cjs finds commands.cjs next to itself,
+// - that help, compare, and merge never load execution dependencies,
+// - release-compiled file and manifest behavior on the real filesystem,
+
 // Opt-in: build the release runner and the test bundle first:
 //
 //   pnpm run build:renderer-benchmarks:runner

frontend/test/performance_tests/render_wasm/request_test.cljs

@@ -0,0 +1,208 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC
+
+(ns performance-tests.render-wasm.request-test
+  (:require
+   [app.common.schema :as sm]
+   [benches.render-wasm.node.request :as request]
+   [cljs.test :as t :include-macros true]
+   [performance-tests.render-wasm.test-helpers :refer [failure-data]]))
+
+(def ^:private console-defaults
+  {:level :info :color :auto})
+
+(def ^:private measurement-defaults
+  {:seed 42 :filter nil :warmups 3 :repetitions 10
+   :timeout-ms 30000 :startup-timeout-ms 30000 :output-dir nil
+   :screenshots false :require-gpu false})
+
+(def ^:private comparison-defaults
+  {:allow-build-differences false :allow-scene-differences false
+   :allow-all-differences false})
+
+(def ^:private existing-run
+  {:command "run" :console console-defaults
+   :options (assoc measurement-defaults
+                   :build {:kind :existing :id "ready-build"}
+                   :remove false)})
+
+(def ^:private default-cases
+  [[{:command "build"}
+    {:command "build" :console console-defaults
+     :options {:default-features true :features [] :build-timeout-ms 300000}}]
+   [{:command "run" :console {} :options {:build {:kind :prepare}}}
+    {:command "run" :console console-defaults
+     :options (assoc measurement-defaults
+                     :build {:kind :prepare :default-features true
+                             :features [] :build-timeout-ms 300000}
+                     :remove false)}]
+   [{:command "run" :options {:build {:kind :existing :id "ready-build"}}}
+    existing-run]
+   [{:command "ab" :options {:features ["render-cache"]}}
+    {:command "ab" :console console-defaults
+     :options (merge measurement-defaults comparison-defaults
+                     {:features ["render-cache"] :max-features 3
+                      :confirm false :dry-run false :build-timeout-ms 300000})}]
+   [{:command "compare"
+     :options {:baseline-path "base.transit" :candidate-path "candidate.transit"}}
+    {:command "compare" :console console-defaults
+     :options (assoc comparison-defaults
+                     :baseline-path "base.transit"
+                     :candidate-path "candidate.transit" :case-pair nil)}]
+   [{:command "merge"
+     :options {:base-path "base.transit" :additional-path "extra.transit"}}
+    {:command "merge" :console console-defaults
+     :options {:base-path "base.transit" :additional-path "extra.transit"
+               :output-dir nil}}]
+   [{:command "purge" :options {:target :runs}}
+    {:command "purge" :console console-defaults :options {:target :runs}}]
+   [{:command "help"}
+    {:command "help" :console console-defaults :options {:topic nil}}]])
+
+(t/deftest defaults-produce-complete-command-requests
+  (doseq [[partial expected] default-cases]
+    (t/testing (pr-str partial)
+      (t/is (= expected (request/check (request/apply-defaults partial)))))))
+
+(t/deftest explicit-values-and-literal-text-survive-resolution
+  (let [partial  {:command "run" :console {:level :warn :color :never}
+                  :options {:seed 0 :warmups 0 :filter "  " :output-dir " ./runs "
+                            :build {:kind :prepare :default-features false
+                                    :features []}}}
+        expected {:command "run" :console {:level :warn :color :never}
+                  :options (assoc measurement-defaults
+                                  :seed 0 :warmups 0 :filter "  " :output-dir " ./runs "
+                                  :build {:kind :prepare :default-features false
+                                          :features [] :build-timeout-ms 300000}
+                                  :remove false)}]
+    (t/is (= expected (request/check (request/apply-defaults partial)))))
+  (let [partial {:command "compare"
+                 :options {:baseline-path " ./base " :candidate-path " ./candidate "
+                           :case-pair [" rects/load " "other/load"]}}]
+    (t/is (= (assoc partial :console console-defaults
+                    :options (merge comparison-defaults (:options partial)))
+             (request/check (request/apply-defaults partial)))))
+  (t/is (= existing-run
+           (request/check
+            (request/apply-defaults (assoc-in existing-run [:options :warmups] nil))))))
+
+(t/deftest required-selections-and-paths-have-no-defaults
+  (doseq [partial [{}
+                   {:command "unknown"}
+                   {:command "run"}
+                   {:command "run" :options {:build {}}}
+                   {:command "run" :options {:build {:kind :unknown}}}
+                   {:command "run" :options {:build {:kind :existing}}}
+                   {:command "ab"}
+                   {:command "compare"}
+                   {:command "compare" :options {:baseline-path "base"}}
+                   {:command "compare" :options {:candidate-path "candidate"}}
+                   {:command "merge" :options {:base-path "base"}}
+                   {:command "merge" :options {:additional-path "additional"}}
+                   {:command "purge"}]]
+    (t/testing (pr-str partial)
+      (t/is (= ::request/invalid-request
+               (:type (failure-data #(request/check (request/apply-defaults partial)))))))))
+
+(t/deftest receiver-requires-complete-data-and-reports-the-field
+  (doseq [[_ complete] default-cases]
+    (t/is (= ::request/invalid-request
+             (:type (failure-data #(request/check (dissoc complete :console)))))))
+  (doseq [incomplete [(dissoc existing-run :options)
+                      (update existing-run :console dissoc :level)
+                      (update existing-run :options dissoc :warmups)
+                      (assoc-in existing-run [:options :warmups] nil)
+                      (assoc-in existing-run [:options :remove] nil)
+                      (assoc-in existing-run [:console :level] nil)]]
+    (let [data (failure-data #(request/check incomplete))]
+      (t/is (= ::request/invalid-request (:type data)))
+      (t/is (seq (:errors (::sm/explain data))))))
+  (let [data (failure-data #(request/check (assoc-in existing-run [:options :warmups] -1)))]
+    (t/is (some #(= [:options :warmups] (:in %)) (:errors (::sm/explain data))))))
+
+(t/deftest receiver-rejects-extra-fields-at-every-map-boundary
+  (doseq [[path value] [[[:extra] true]
+                        [[:console :stream] "stdout"]
+                        [[:options :browser] {}]
+                        [[:options :build :features] []]
+                        [[:options :build :default-features] false]
+                        [[:options :build :build-timeout-ms] 1000]]]
+    (let [input    (assoc-in existing-run path value)
+          resolved (request/apply-defaults input)]
+      (t/is (= input resolved))
+      (t/is (= ::request/invalid-request
+               (:type (failure-data #(request/check resolved)))))))
+  (doseq [partial [{:command "run" :options {:build {:kind :prepare :id "ready-build"}}}
+                   {:command "ab" :options {:features ["cache"] :default-features false}}
+                   {:command "ab" :options {:features ["cache"] :remove false}}
+                   {:command "ab" :options {:features ["cache"] :build {:kind :prepare}}}]]
+    (t/is (= ::request/invalid-request
+             (:type (failure-data #(request/check (request/apply-defaults partial))))))))
+
+(t/deftest numeric-constraints-match-seeds-counts-and-node-timers
+  (doseq [[field values] [[:seed [0 4294967295]]
+                          [:warmups [0 js/Number.MAX_SAFE_INTEGER]]
+                          [:repetitions [1 js/Number.MAX_SAFE_INTEGER]]
+                          [:timeout-ms [1 2147483647]]
+                          [:startup-timeout-ms [1 2147483647]]]]
+    (doseq [value values]
+      (let [input (assoc-in existing-run [:options field] value)]
+        (t/is (= input (request/check input))))))
+  (doseq [[field values] [[:seed [-1 4294967296 1.5 "42" js/NaN js/Infinity]]
+                          [:warmups [-1 (inc js/Number.MAX_SAFE_INTEGER) 0.5 "3"]]
+                          [:repetitions [0 (inc js/Number.MAX_SAFE_INTEGER) 1.5]]
+                          [:timeout-ms [0 2147483648 1.5 "30000"]]
+                          [:startup-timeout-ms [0 2147483648]]]]
+    (doseq [value values]
+      (t/testing (str field " " value)
+        (let [input    (assoc-in existing-run [:options field] value)
+              resolved (request/apply-defaults input)]
+          (when (string? value)
+            (t/is (= value (get-in resolved [:options field]))))
+          (t/is (= ::request/invalid-request
+                   (:type (failure-data #(request/check resolved)))))))))
+  (doseq [command ["build" "ab"]
+          value [0 2147483648 1.5]]
+    (let [partial {:command command :options {:features ["cache"] :build-timeout-ms value}}]
+      (t/is (= ::request/invalid-request
+               (:type (failure-data #(request/check (request/apply-defaults partial)))))))))
+
+(t/deftest command-values-use-their-domain-constraints
+  (doseq [partial [{:command "run" :options {:build {:kind :existing :id "../build"}}}
+                   {:command "run" :options {:build {:kind :existing :id ""}}}
+                   {:command "build" :options {:features ["bad feature"]}}
+                   {:command "build" :options {:features [nil]}}
+                   {:command "ab" :options {:features []}}
+                   {:command "ab" :options {:features ["cache"] :max-features 0}}
+                   {:command "ab" :options {:features ["cache"] :max-features 7}}
+                   {:command "ab" :options {:features ["cache"] :max-features 1.5}}
+                   {:command "compare" :options {:baseline-path " " :candidate-path "c"}}
+                   {:command "merge" :options {:base-path "b" :additional-path ""}}
+                   {:command "merge" :options {:base-path "b" :additional-path "a"
+                                               :output-dir " "}}
+                   {:command "purge" :options {:target :unknown}}
+                   {:command "help" :options {:topic "unknown"}}
+                   {:command "help" :console {:level :error}}
+                   {:command "help" :console {:color true}}]]
+    (t/testing (pr-str partial)
+      (let [resolved (request/apply-defaults partial)]
+        (t/is (= ::request/invalid-request
+                 (:type (failure-data #(request/check resolved))))))))
+  (doseq [pair [["only"] ["one" "two" "three"] ["" "two"] ["one" " "]]]
+    (t/is (= ::request/invalid-request
+             (:type (failure-data
+                     #(request/check
+                       (request/apply-defaults
+                        {:command "compare" :options {:baseline-path "b" :candidate-path "c"
+                                                      :case-pair pair}})))))))
+  (doseq [filter-text [nil "" " " " rects "]]
+    (let [input (assoc-in existing-run [:options :filter] filter-text)]
+      (t/is (= input (request/check (request/apply-defaults input))))))
+  (doseq [target [:builds :runs :comparisons :all]]
+    (t/is (= target
+             (get-in (request/check
+                      (request/apply-defaults {:command "purge" :options {:target target}}))
+                     [:options :target])))))

frontend/test/performance_tests/render_wasm/runner.cljs

@@ -29,6 +29,7 @@
    [performance-tests.render-wasm.process-test]
    [performance-tests.render-wasm.protocol-test]
    [performance-tests.render-wasm.report-test]
+   [performance-tests.render-wasm.request-test]
    [performance-tests.render-wasm.run-test]
    [performance-tests.render-wasm.runner-test]
    [performance-tests.render-wasm.runs-test]
@@ -66,6 +67,7 @@
    'performance-tests.render-wasm.process-test
    'performance-tests.render-wasm.protocol-test
    'performance-tests.render-wasm.report-test
+   'performance-tests.render-wasm.request-test
    'performance-tests.render-wasm.run-test
    'performance-tests.render-wasm.runner-test
    'performance-tests.render-wasm.runs-test

frontend/test/performance_tests/render_wasm/runs_test.cljs

@@ -10,7 +10,6 @@
    ["os" :as os]
    ["path" :as path]
    [benches.render-wasm.node.artifacts :as artifacts]
-   [benches.render-wasm.node.files :as files]
    [benches.render-wasm.node.runs :as runs]
    [benches.render-wasm.result :as result]
    [cljs.test :as t :include-macros true]
@@ -72,7 +71,7 @@
     (let [error (failure #(runs/create-home {:root @temporary-root
                                              :run-id "run-3"
                                              :output-dir directory}))]
-      (t/is (= ::files/invalid-home (:type (ex-data error))))
+      (t/is (= ::artifacts/invalid-home (:type (ex-data error))))
       (t/is (= "output directory already exists" (ex-message error)))
       (t/is (= "keep" (fs/readFileSync (path/join directory "sentinel") "utf8"))))))
 
@@ -99,7 +98,7 @@
     (t/is (thrown-with-msg? js/Error #"invalid run id"
                             (runs/create-home {:root @temporary-root :run-id id}))))
   (t/is (thrown-with-msg? js/Error #"nonempty path"
-                          (files/check-output-dir "  "))))
+                          (artifacts/check-output-dir "  "))))
 
 (t/deftest home-creation-reports-an-invalid-parent
   (let [parent (path/join @temporary-root "file")
@@ -107,7 +106,7 @@
     (fs/writeFileSync parent "occupied")
     (let [error (failure #(runs/create-home {:root @temporary-root
                                              :run-id "run-6" :output-dir dir}))]
-      (t/is (= ::files/invalid-home (:type (ex-data error))))
+      (t/is (= ::artifacts/invalid-home (:type (ex-data error))))
       (t/is (= (path/resolve dir) (:path (ex-data error))))
       (t/is (= "ENOTDIR" (.-code (ex-cause error))))
       (t/is (= (ex-message (ex-cause error)) (:cause (ex-data error))))
@@ -118,8 +117,8 @@
   (let [dir (path/join @temporary-root "occupied-log")]
     (fs/mkdirSync dir)
     (fs/writeFileSync (path/join dir "run.log") "")
-    (let [error (failure #(files/create-home dir "run" "cannot create run home"))]
-      (t/is (= ::files/invalid-home (:type (ex-data error))))
+    (let [error (failure #(artifacts/create-home dir "run" "cannot create run home"))]
+      (t/is (= ::artifacts/invalid-home (:type (ex-data error))))
       (t/is (= "cannot create run home" (ex-message error)))
       (t/is (= dir (:path (ex-data error))))
       (t/is (= "EEXIST" (.-code (ex-cause error))))