🚧 WIP 25 complete runner, drop pilot

AI-assisted-by: deepseek-flash, gpt-6.1-sol

frontend/package.json

@@ -21,7 +21,6 @@
     "build:storybook:cljs": "clojure -M:dev:shadow-cljs compile storybook",
     "build:wasm": "../render-wasm/build frontend",
     "build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-runner",
-    "build:renderer-benchmarks:pilot": "clojure -M:dev:renderer-bench release bench-render-wasm-pilot",
     "build:app:libs": "node ./scripts/build-libs.js",
     "build:app:main": "clojure -M:dev:shadow-cljs release main worker",
     "build:app:worker": "clojure -M:dev:shadow-cljs release worker",

frontend/performance/benches/render_wasm/browser/bridge.cljs

@@ -93,7 +93,6 @@
    :set-view-end      #(h/call wasm/internal-module "_set_view_end")})
 
 
-
 ;; Monotonic owner generation. Every async continuation compares its
 ;; captured epoch and drops stale results silently; dispose/load bump it.
 (defonce ^:private owner-epoch*
@@ -103,8 +102,8 @@
 (defonce ^:private module-promise*
   (atom nil))
 
-;; Page-scoped canvas owned by the current load. Fresh page per attempt in
-;; the future runner; the pilot loads once.
+;; Page-scoped canvas owned by the current load. :fresh cases load once per
+;; page. :reuse cases keep one preparation across all attempts.
 (defonce ^:private canvas*
   (atom nil))
 

frontend/performance/benches/render_wasm/browser/index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <meta charset="utf-8" />
-    <title>renderer benchmarks</title>
+    <title>Renderer benchmarks</title>
     <style>
       /* The canvas is the page: kill the UA body margin (8px) that would
          leave a gutter around the scene in screenshots. */

frontend/performance/benches/render_wasm/codec.cljs

@@ -8,8 +8,9 @@
   "Verifies that measurements survive transit encoding.
 
   This ns exists mainly to preserve NaNs in the transit encoding of results.
-  We treat two NaN values as the same retained measurement.
-  Unrepresentable data raises an error."
+  We treat two NaN values as the same measurement.
+
+  Any data we cannot represent raises an error."
   (:require
    [app.common.transit :as transit]))
 

frontend/performance/benches/render_wasm/node/artifacts.cljs

@@ -16,6 +16,7 @@
    ["path" :as path]
    [app.common.transit :as transit]
    [benches.render-wasm.codec :as codec]
+   [benches.render-wasm.node.files :as files]
    [clojure.string :as str]))
 
 (def ^:private executable-dir
@@ -30,7 +31,7 @@
 (defn frontend-root
   "Finds the frontend checkout by walking up from the Node executable.
 
-  Supports target/performance/render-wasm/{runner,pilot}.cjs and the ESM
+  Supports target/performance/render-wasm/runner.cjs and the ESM
   target/{performance-tests,tests}/test.js boots, independent of caller cwd."
   []
   (when-not executable-dir
@@ -101,15 +102,6 @@
      :index-path (path/join dir "index.html")
      :manifest manifest}))
 
-(defn- atomic-write
-  [file content]
-  (let [temporary (str file "." (random-uuid) ".tmp")]
-    (try
-      (fs/writeFileSync temporary content #js {:encoding "utf8" :flag "wx"})
-      (fs/renameSync temporary file)
-      (finally
-        (fs/rmSync temporary #js {:force true})))))
-
 (defn create-build
   "Allocates a UUID build and saves its :building build.transit and empty log.
 
@@ -137,7 +129,7 @@
     (fs/mkdirSync (:browser-dir build))
     (fs/mkdirSync (:renderer-dir build))
     (fs/writeFileSync (:log-path build) "" "utf8")
-    (atomic-write (:manifest-path build) (codec/encode-str manifest))
+    (files/atomic-write (:manifest-path build) (codec/encode-str manifest))
     build))
 
 (defn write-manifest
@@ -165,7 +157,7 @@
                          :id (:id build)
                          :status (:status current)})))
       (let [manifest (assoc manifest :updated-at (.toISOString (js/Date.)))]
-        (atomic-write (:manifest-path build) (codec/encode-str manifest))
+        (files/atomic-write (:manifest-path build) (codec/encode-str manifest))
         (assoc build :manifest manifest)))))
 
 (defn- require-file

frontend/performance/benches/render_wasm/node/browser.cljs

@@ -36,8 +36,8 @@
 
 (defn within
   "Calls work while its absolute deadline and optional AbortSignal allow it.
-  Clears the timer/listener on every outcome and handles late rejections. The
-  caller still owns shutdown of the browser running the work."
+  Clears the timer/listener on every outcome and handles late rejections.
+  The caller owns shutdown of the browser running the work."
   [work {:keys [deadline signal phase]}]
   (js/Promise.
    (fn [resolve reject]
@@ -142,8 +142,37 @@
           {:deadline (process/deadline process/default-termination-grace-ms)
            :phase :browser-disconnect}))
 
+(defn ^:async close-page
+  "Closes an owned page and its context within one termination grace.
+  Repeated or already-closed pages resolve without work."
+  [{:keys [page context]}]
+  (when (or page context)
+    (let [deadline (process/deadline process/default-termination-grace-ms)
+          page-failure
+          (try
+            (when (and page (not (.isClosed ^js page)))
+              (await (within #(.close ^js page)
+                             {:deadline deadline :phase :browser-page-close})))
+            nil
+            (catch :default cause cause))
+          context-failure
+          (try
+            (when context
+              (await (within #(.close ^js context)
+                             {:deadline deadline :phase :browser-context-close})))
+            nil
+            (catch :default cause cause))]
+      (when-let [cause (or page-failure context-failure)]
+        (throw cause)))))
+
 (defn- ^:async shutdown-owner
+  "Closes page/context best effort, then kills the owned process group. A page
+  close failure does not mask process/disconnect/profile failures because the
+  group kill releases the same resources."
   [{:keys [user-data-dir] :as owner}]
+  (try
+    (await (close-page owner))
+    (catch :default _ nil))
   (let [process-failure    (try
                              (await (stop-process owner))
                              nil
@@ -172,16 +201,17 @@
           (reset! shutdown closing)
           closing))))
 
-(defn ^:async open
-  "Spawns owned Chromium, attaches by public CDP and checks the retained bridge.
+(defn ^:async launch
+  "Spawns a Chromium instance and attaches via Chrome DevTools Protocol (CDP).
 
-  Options include a resolved {:width :height :dpr} viewport, an optional signal
-  and startup-timeout-ms (30000 by default). An optional absolute deadline lets
-  server/browser startup share a budget. The result holds :page, :context,
-  :browser, :version, :endpoint and ownership fields consumed by close.
-  Startup failures retain endpoint/profile facts after owned shutdown."
-  [host {:keys [viewport signal startup-timeout-ms deadline]
-         :or {startup-timeout-ms process/default-startup-timeout-ms}}]
+  Options include an optional signal, startup-timeout-ms (30000 by default)
+  and an optional absolute deadline shared with server startup. The result
+  owns the process group, user-data directory and CDP connection. It holds no
+  page or context; use new-page for those.
+
+  Startup failures include information about endpoint and profile."
+  [{:keys [signal startup-timeout-ms deadline]
+    :or {startup-timeout-ms process/default-startup-timeout-ms}}]
   (let [deadline (or deadline (process/deadline startup-timeout-ms))
         budget   (process/remaining deadline)]
     (when (and signal (.-aborted ^js signal))
@@ -225,23 +255,8 @@
                                                                              #js {:timeout (max 1 (process/remaining deadline))})]
                                              (reset! (:connection owner) connecting)
                                              connecting))
-                                         bounds))
-              ^js context (await (within #(.newContext browser
-                                                       #js {:viewport #js {:width (:width viewport)
-                                                                           :height (:height viewport)}
-                                                            :deviceScaleFactor (:dpr viewport)})
-                                         bounds))
-              ^js page    (await (within #(.newPage context) bounds))]
-          (await (within #(.goto page (str (:base-url host) "/")
-                                 #js {:timeout (max 1 (process/remaining deadline))})
-                         bounds))
-          (await (within #(.waitForFunction page "window.__benchReady === true" nil
-                                            #js {:timeout (max 1 (process/remaining deadline))})
-                         bounds))
-          (identity/check (get-in host [:build :id])
-                          (transit/decode-str (await (within #(evaluate page "buildIdentity") bounds))))
-          (assoc owner :browser browser :context context :page page
-                 :version (.version browser) :endpoint address))
+                                         bounds))]
+          (assoc owner :browser browser :version (.version browser) :endpoint address))
         (catch :default cause
           (try
             (await (close owner))
@@ -254,3 +269,83 @@
           (throw (ex-info (or (ex-message cause) (str cause))
                           (assoc (ex-data cause) :endpoint @(:address endpoint) :user-data-dir user-data-dir)
                           cause)))))))
+
+(defn ^:async new-page
+  "Creates a context and page in a browser process we own, navigates to host,
+  waits for the bridge and checks its compiled build identity.
+
+  Returns {:page :context}.
+
+  Failures close the started page/context before rejecting."
+  [owner host {:keys [viewport signal startup-timeout-ms deadline]
+               :or {startup-timeout-ms process/default-startup-timeout-ms}}]
+  (let [deadline (or deadline (process/deadline startup-timeout-ms))
+        bounds   {:deadline deadline :signal signal :phase :browser-page-startup}
+        ^js browser (:browser owner)
+        ^js context (await (within #(.newContext browser
+                                                 #js {:viewport #js {:width (:width viewport)
+                                                                     :height (:height viewport)}
+                                                      :deviceScaleFactor (:dpr viewport)})
+                                   bounds))]
+    (try
+      (let [page     (await (within #(.newPage context) bounds))
+            page-own {:page page :context context}]
+        (try
+          (await (within #(.goto ^js page (str (:base-url host) "/")
+                                 #js {:timeout (max 1 (process/remaining deadline))})
+                         bounds))
+          (await (within #(.waitForFunction ^js page "window.__benchReady === true" nil
+                                            #js {:timeout (max 1 (process/remaining deadline))})
+                         bounds))
+          (identity/check (get-in host [:build :id])
+                          (transit/decode-str (await (within #(evaluate page "buildIdentity") bounds))))
+          page-own
+          (catch :default cause
+            (try
+              (await (close-page page-own))
+              (catch :default _ nil))
+            (throw cause))))
+      (catch :default cause
+        (try
+          (await (close-page {:context context}))
+          (catch :default _ nil))
+        (throw cause)))))
+
+(defn connected?
+  "True while the owned CDP connection is live."
+  [{:keys [browser]}]
+  (boolean (and browser (.isConnected ^js browser))))
+
+(defn prepare-scene
+  "Calls prepareScene and decodes its Transit response under bounds."
+  [page request bounds]
+  (.then (within #(evaluate page "prepareScene" request) bounds)
+         transit/decode-str))
+
+(defn run-prepared
+  "Calls runPrepared and decodes its Transit response under bounds."
+  [page request bounds]
+  (.then (within #(evaluate page "runPrepared" request) bounds)
+         transit/decode-str))
+
+(defn screenshot
+  "Writes one PNG under bounds. The timeout belongs to Playwright's call."
+  [page file {:keys [deadline] :as bounds}]
+  (.then (within #(.screenshot ^js page
+                               #js {:path file
+                                    :timeout (max 1 (process/remaining deadline))})
+                 bounds)
+         (fn [_] nil)))
+
+(def service
+  "Interface to the browser used by the serial runner.
+
+  Tests can replace this map with mocks."
+  {:launch launch
+   :new-page new-page
+   :close-page close-page
+   :close close
+   :connected? connected?
+   :prepare prepare-scene
+   :run run-prepared
+   :screenshot screenshot})

frontend/performance/benches/render_wasm/node/build.cljs

@@ -145,7 +145,7 @@
                                     :env {"RENDER_BENCH_BUILD_ID" (:id build)
                                           "RENDER_BENCH_CASES_PATH" (:cases-path build)
                                           "RENDER_BENCH_BROWSER_DIR" (:browser-dir build)}})))
-        (fs/copyFileSync (path/join root "performance" "benches" "render_wasm" "browser" "pilot.html")
+        (fs/copyFileSync (path/join root "performance" "benches" "render_wasm" "browser" "index.html")
                          (:index-path build))
         (reset! phase :identity)
         (check-profile-marks (catalog/build-identity build))

frontend/performance/benches/render_wasm/node/compare.cljs

@@ -0,0 +1,41 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns benches.render-wasm.node.compare
+  "Offline comparison of two saved RunRecords.
+
+  Inputs are user file paths resolved from the caller's cwd. The namespace
+  depends only on record decoding and the shared report pipeline; it starts no
+  build, host, browser or renderer."
+  (:require
+   ["fs" :as fs]
+   ["path" :as path]
+   [benches.render-wasm.report.compare :as compare]
+   [benches.render-wasm.report.format :as format]
+   [benches.render-wasm.result :as result]))
+
+(defn- read-run
+  [file]
+  (result/decode (fs/readFileSync (path/resolve file) "utf8")))
+
+(defn execute
+  "Reads and compares two saved run files.
+
+  options: :compare-cases is one exact A,B selector; A belongs to baseline and
+  B to the candidate. :force bypasses compatibility checks. Returns the
+  validated `comparison` and its formatted text."
+  [baseline-path candidate-path {:keys [compare-cases force]}]
+  (let [baseline   (read-run baseline-path)
+        candidate  (read-run candidate-path)
+        comparison (result/check-record
+                    (compare/compare-runs
+                     baseline candidate
+                     (cond-> {:force (boolean force)}
+                       compare-cases
+                       (assoc :pairs (compare/selector-pair baseline candidate
+                                                            compare-cases)))))]
+    {:comparison comparison
+     :text (format/format-comparison comparison)}))

frontend/performance/benches/render_wasm/node/files.cljs

@@ -0,0 +1,25 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns benches.render-wasm.node.files
+  "Small filesystem helpers shared by benchmark storage writers."
+  (:require
+   ["fs" :as fs]))
+
+(defn atomic-write
+  "Writes (UTF-8) content to a temporary file and renames it over `file`,
+  so readers see either the previous complete file or the new one. The parent
+  directory must exist.
+
+  A failed write or rename leaves the previous file untouched and removes the
+  temporary file."
+  [file content]
+  (let [temporary (str file "." (random-uuid) ".tmp")]
+    (try
+      (fs/writeFileSync temporary content #js {:encoding "utf8" :flag "wx"})
+      (fs/renameSync temporary file)
+      (finally
+        (fs/rmSync temporary #js {:force true})))))

frontend/performance/benches/render_wasm/node/pilot.cljs

@@ -1,335 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS SUBSIDIARY SL
-
-(ns benches.render-wasm.node.pilot
-  "Executes one unscored case using a prepared or retained renderer build.
-  Stdout contains one raw RunRecord schema3 in Transit; build IDs, progress and
-  reports go to stderr. The saved plan always has 0 warmups and 1 attempt.
-  Node/browser summaries must agree outside measurement.
-
-  Build and startup have separate bounds. The case deadline covers preparation
-  and its one attempt; screenshots, reporting and disposal stay outside it.
-  Exit code 0 means the diagnostic completed, not a performance conclusion.
-
-  Example: node target/performance/render-wasm/pilot.cjs --build ID
-           --case rects/pan --seed 42 --warmups 0 --attempts 1"
-  (:require
-   ["module" :as node-module]
-   ["os" :as os]
-   [app.common.transit :as transit]
-   [benches.render-wasm.declarations :as declarations]
-   [benches.render-wasm.failures :as failures]
-   [benches.render-wasm.measurement :as measurement]
-   [benches.render-wasm.node.artifacts :as artifacts]
-   [benches.render-wasm.node.browser :as browser]
-   [benches.render-wasm.node.build :as build]
-   [benches.render-wasm.node.catalog :as catalog]
-   [benches.render-wasm.node.process :as process]
-   [benches.render-wasm.node.server :as server]
-   [benches.render-wasm.random :as random]
-   [benches.render-wasm.report.format :as format]
-   [benches.render-wasm.report.summarize :as summary]
-   [benches.render-wasm.result :as result]
-   [clojure.string :as str]))
-
-(def ^:private default-seed 42)
-(def ^:private default-timeout-ms 30000)
-
-(defn- usage
-  []
-  (str "pilot [--build ID] [--case SCENE/NAME] [--seed N]\n"
-       "      [--timeout-ms N] [--build-timeout-ms N] [--startup-timeout-ms N]\n"
-       "      [--no-default-features] [--features NAME,NAME]\n"
-       "      [--warmups 0] [--attempts 1] [--screenshot PATH] [--help]\n"
-       "Defaults: rects/load, seed 42, timeout 30000, build timeout 300000,\n"
-       "startup timeout 30000, 0 warmups, 1 attempt.\n"
-       "--build cannot be combined with feature-selection flags."))
-
-(defn- invalid-args
-  [message]
-  (throw (ex-info message {:type ::invalid-args})))
-
-(defn- parse-integer
-  [flag value]
-  (when-not (re-matches #"[0-9]+" value)
-    (invalid-args (str flag " requires a nonnegative integer")))
-  (let [number (js/Number value)]
-    (when-not (js/Number.isSafeInteger number)
-      (invalid-args (str flag " requires a safe integer")))
-    number))
-
-(defn- parse-timeout
-  [flag value]
-  (let [number (parse-integer flag value)]
-    (when-not (<= 1 number 2147483647)
-      (invalid-args (str flag " must be between 1 and 2147483647")))
-    number))
-
-(defn parse-args
-  "Parses the diagnostic CLI without loading a catalog or starting resources.
-  Case availability and Cargo features are checked by the selected build APIs."
-  [argv]
-  (loop [args argv
-         seen #{}
-         opts {:case-id :rects/load :seed default-seed
-               :timeout-ms default-timeout-ms
-               :build-timeout-ms process/default-build-timeout-ms
-               :startup-timeout-ms process/default-startup-timeout-ms
-               :warmups 0 :attempts 1
-               :features {:default-features? true :features []}}]
-    (if (empty? args)
-      (do
-        (when-not (:help? opts)
-          (when (and (:build-id opts) (some seen ["--no-default-features" "--features"]))
-            (invalid-args "--build cannot be combined with feature-selection flags"))
-          (when-not (random/seed? (:seed opts))
-            (invalid-args "--seed must be a uint32 integer"))
-          (when-not (and (zero? (:warmups opts)) (= 1 (:attempts opts)))
-            (invalid-args "the diagnostic pilot requires --warmups 0 and --attempts 1")))
-        (cond-> (dissoc opts :features-text)
-          (contains? opts :features-text)
-          (assoc-in [:features :features] (build/parse-feature-list (:features-text opts)))))
-      (let [[flag & more] args]
-        (cond
-          (#{"--help" "-h"} flag)
-          (recur more (conj seen flag) (assoc opts :help? true))
-
-          (= flag "--no-default-features")
-          (recur more (conj seen flag) (assoc-in opts [:features :default-features?] false))
-
-          (#{"--build" "--case" "--seed" "--timeout-ms" "--build-timeout-ms"
-             "--startup-timeout-ms" "--warmups" "--attempts" "--features" "--screenshot"} flag)
-          (let [[value & rest-args] more]
-            (when (or (nil? value) (str/starts-with? value "--"))
-              (invalid-args (str flag " requires a value")))
-            (when (and (#{"--build" "--screenshot"} flag) (str/blank? value))
-              (invalid-args (str flag " requires a nonempty value")))
-            (when (and (= flag "--case") (not (re-matches #"[^/\s]+/[^/\s]+" value)))
-              (invalid-args "--case requires SCENE/NAME"))
-            (recur rest-args (conj seen flag)
-                   (case flag
-                     "--build" (assoc opts :build-id value)
-                     "--case" (assoc opts :case-id (keyword value))
-                     "--seed" (assoc opts :seed (parse-integer flag value))
-                     "--timeout-ms" (assoc opts :timeout-ms (parse-timeout flag value))
-                     "--build-timeout-ms" (assoc opts :build-timeout-ms (parse-timeout flag value))
-                     "--startup-timeout-ms" (assoc opts :startup-timeout-ms (parse-timeout flag value))
-                     "--warmups" (assoc opts :warmups (parse-integer flag value))
-                     "--attempts" (assoc opts :attempts (parse-integer flag value))
-                     "--features" (assoc opts :features-text value)
-                     "--screenshot" (assoc opts :screenshot value))))
-
-          :else
-          (invalid-args (str "unknown pilot option: " flag)))))))
-
-(defn- trace
-  [message]
-  (.error js/console message))
-
-(defn- pilot-metadata
-  "Uses the selected manifest as primary provenance, including on reuse."
-  [selected-build opts]
-  (let [manifest        (:manifest selected-build)
-        cpus            (array-seq (os/cpus))
-        require-package (if (exists? js/require)
-                          js/require
-                          (node-module/createRequire (:cases-path selected-build)))
-        ^js package     (require-package "playwright/package.json")]
-    {:git (merge {:sha nil :dirty nil} (get-in manifest [:provenance :git]))
-     :build {:functional (:functional manifest)
-             :provenance (assoc (:provenance manifest) :build-id (:id selected-build))}
-     :environment {:node (.-version js/process) :playwright (.-version package)
-                   :browser nil :platform (os/platform) :release (os/release) :arch (os/arch)
-                   :cpu {:models (vec (distinct (map #(.-model %) cpus))) :logical-count (count cpus)}
-                   :launch browser/launch-settings}
-     :scored? false
-     :diagnostics (assoc (select-keys opts [:seed :screenshot])
-                         :runner-git {:sha nil :dirty nil}
-                         :runtime-helper-identity :unverified)}))
-
-(defn- complete-record
-  "Retains evidence and fixed slots. A fresh preparation consumes its attempt;
-  a failed warm preparation leaves the attempt unstarted."
-  [run preparation evidence attempted?]
-  (let [id  (get-in run [:plan :cases 0 :id])
-        run (cond-> run preparation (result/record-preparation preparation))
-        run (if attempted?
-              (result/record-attempt run (measurement/attempt id (:preparation-id preparation) false evidence))
-              run)
-        ok? (= "ok" (:status evidence))]
-    (result/finish-run run (.toISOString (js/Date.))
-                       (cond-> {:reason (if ok? :completed (if attempted? :failed :preparation-failed))}
-                         (not ok?) (assoc :failure (select-keys evidence [:status :phase :message :cause])
-                                          :partial (:partial evidence))))))
-
-(defn- failed-record
-  [run preparation attempted? phase cause]
-  (let [evidence {:status "failed" :phase (name phase)
-                  :message (or (ex-message cause) (str cause))
-                  :cause (failures/describe-cause cause)}]
-    (if (= :running (get-in run [:termination :reason]))
-      (complete-record run preparation evidence attempted?)
-      (assoc run :termination {:reason :diagnostic-failed :failure evidence
-                               :prior-termination (:termination run)}))))
-
-(defn- ^:async execute-case
-  [session selected selected-build opts signal preparation-ref attempted-ref]
-  (let [page     (:page session)
-        deadline (process/deadline (:timeout-ms opts))
-        bounds   {:deadline deadline :signal signal :phase :pilot-case}
-        stamp    (str "?build=" (js/encodeURIComponent (:id selected-build)))
-        request  (transit/encode-str {:case selected :timeout-ms (:timeout-ms opts)
-                                      :module-url (str "/renderer/render-wasm.js" stamp)
-                                      :wasm-url (str "/renderer/render-wasm.wasm" stamp)})
-        prepared (transit/decode-str
-                  (await (browser/within
-                          (fn []
-                            (when (= :fresh (:context selected))
-                              (reset! attempted-ref true))
-                            (browser/evaluate page "prepareScene" request))
-                          bounds)))]
-    (if (= "ok" (:status prepared))
-      (do
-        (reset! preparation-ref (:preparation prepared))
-        (transit/decode-str
-         (await (browser/within
-                 (fn []
-                   (reset! attempted-ref true)
-                   (browser/evaluate page "runPrepared"
-                                     (transit/encode-str
-                                      {:preparation-id (get-in prepared [:preparation :preparation-id])
-                                       :timeout-ms (process/remaining deadline)})))
-                 bounds))))
-      prepared)))
-
-(defn- ^:async check-diagnostics
-  [session run opts signal]
-  (let [^js page (:page session)
-        deadline (process/deadline (:startup-timeout-ms opts))
-        bounds   {:deadline deadline :signal signal :phase :pilot-diagnostics}
-        report   (summary/summarize run)]
-    (when-let [shot (:screenshot opts)]
-      (await (browser/within #(.screenshot page #js {:path shot :timeout (process/remaining deadline)})
-                             bounds)))
-    (let [browser-report (transit/decode-str
-                          (await (browser/within #(browser/evaluate page "summarize" (result/encode run))
-                                                 bounds)))]
-      (when-not (= report browser-report)
-        (throw (ex-info "Node/browser summaries differ" {:type ::summary-mismatch})))
-      (trace "pilot: Node/browser summaries agree"))
-    (await (browser/within #(browser/evaluate page "dispose") bounds))))
-
-(defn- ^:async run-pilot
-  [opts]
-  (let [root            (artifacts/frontend-root)
-        controller      (js/AbortController.)
-        signal          (.-signal controller)
-        interrupted-ref (atom nil)
-        remove-signals  (process/on-signals
-                         (fn [name]
-                           (when-not (.-aborted signal)
-                             (reset! interrupted-ref name)
-                             (.abort controller (ex-info "pilot interrupted" {:type ::interrupted :signal name})))))
-        host-ref        (atom nil)
-        session-ref     (atom nil)
-        preparation-ref (atom nil)
-        attempted-ref   (atom false)
-        run-ref         (atom nil)
-        phase-ref       (atom :pilot-build)
-        started-at      (.toISOString (js/Date.))]
-    (letfn [(failed [cause phase]
-              (trace (str "pilot: " (or (ex-message cause) (str cause))))
-              (when-some [data (ex-data cause)]
-                (trace (pr-str data)))
-              (when @run-ref
-                (swap! run-ref failed-record @preparation-ref @attempted-ref phase cause)))]
-      (try
-        (let [selected-build (if-let [id (:build-id opts)]
-                               (do
-                                 (trace (str "pilot: using build " id))
-                                 (artifacts/load-build root id))
-                               (await (build/prepare
-                                       {:root root :features (:features opts)
-                                        :build-timeout-ms (:build-timeout-ms opts) :signal signal
-                                        :on-created #(trace (str "pilot: build " (:id %)))})))
-              _              (reset! phase-ref :pilot-catalog)
-              descriptors    (catalog/collect selected-build
-                                              {:master-seed (:seed opts)
-                                               :filter (result/case-name (:case-id opts))})
-              selected       (or (some #(when (= (:case-id opts) (:id %)) %) descriptors)
-                                 (throw (ex-info "case is absent from the selected build"
-                                                 {:build-id (:id selected-build) :case-id (:case-id opts)})))
-              startup-end    (process/deadline (:startup-timeout-ms opts))]
-          (reset! run-ref
-                  (result/create-run
-                   {:run-id (str "pilot-" started-at) :started-at started-at
-                    :plan {:cases [selected] :warmups 0 :repetitions 1
-                           :timeout-policy {:attempt-ms (:timeout-ms opts) :pilot-case-ms (:timeout-ms opts)
-                                            :build-ms (:build-timeout-ms opts) :startup-ms (:startup-timeout-ms opts)}}
-                    :metadata (pilot-metadata selected-build opts)}))
-          (reset! phase-ref :pilot-provenance)
-          (swap! run-ref assoc-in [:metadata :diagnostics :runner-git]
-                 (await (build/git-provenance root {:deadline startup-end :signal signal})))
-          (reset! phase-ref :pilot-server-startup)
-          (reset! host-ref (await (server/open selected-build
-                                               {:startup-timeout-ms (process/remaining startup-end)
-                                                :signal signal})))
-          (reset! phase-ref :pilot-browser-startup)
-          (reset! session-ref (await (browser/open @host-ref
-                                                   {:viewport (declarations/resolve-viewport (:view selected))
-                                                    :startup-timeout-ms (:startup-timeout-ms opts)
-                                                    :deadline startup-end :signal signal})))
-          (swap! run-ref assoc-in [:metadata :environment :browser] (:version @session-ref))
-          (trace "pilot: retained browser identity verified")
-          (reset! phase-ref :pilot-case)
-          (let [evidence (await (execute-case @session-ref selected selected-build opts signal
-                                              preparation-ref attempted-ref))]
-            (swap! run-ref complete-record @preparation-ref evidence @attempted-ref))
-          (reset! phase-ref :pilot-diagnostics)
-          (await (check-diagnostics @session-ref @run-ref opts signal)))
-        (catch :default cause
-          (failed cause @phase-ref))
-        (finally
-          (try
-            (when @session-ref
-              (try
-                (await (browser/close @session-ref))
-                (catch :default cause
-                  (failed cause :pilot-browser-cleanup))))
-            (when @host-ref
-              (try
-                (await (server/close @host-ref))
-                (catch :default cause
-                  (failed cause :pilot-server-cleanup))))
-            (finally
-              (remove-signals)))))
-      (when (and @run-ref @interrupted-ref)
-        (swap! run-ref update :termination assoc :reason :interrupted :signal @interrupted-ref))
-      (when @run-ref
-        (try
-          (trace (format/format-run (summary/summarize @run-ref)))
-          (catch :default cause
-            (failed cause :pilot-report))))
-      @run-ref)))
-
-(defn ^:async -main
-  "Runs the diagnostic and lets Node drain output after owned shutdown."
-  [& argv]
-  (try
-    (let [opts (parse-args (or argv []))]
-      (if (:help? opts)
-        (do
-          (println (usage))
-          (set! (.-exitCode js/process) 0))
-        (let [run (await (run-pilot opts))]
-          (when run
-            (println (result/encode run)))
-          (set! (.-exitCode js/process) (if (= :completed (get-in run [:termination :reason])) 0 1)))))
-    (catch :default cause
-      (trace (str "pilot: " (or (ex-message cause) (str cause))))
-      (when (= ::invalid-args (:type (ex-data cause)))
-        (trace (usage)))
-      (set! (.-exitCode js/process) 1))))

frontend/performance/benches/render_wasm/node/run.cljs

@@ -0,0 +1,787 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns benches.render-wasm.node.run
+  "Serial renderer benchmark runs over a retained build.
+
+  One run owns one run home, one loopback host and at most one Chromium process.
+  Fresh cases prepare once per slot on a new page/context. Reuse cases prepare
+  once and run every slot on that page/context. A failed attempt consumes its
+  slot. Browser failures leave the current case's remaining slots unattempted
+  and continue with the next case in a new browser. The run record is atomically
+  replaced after every preparation, attempt and diagnostic change."
+  (:require
+   ["module" :as node-module]
+   ["os" :as os]
+   ["path" :as path]
+   [app.common.transit :as transit]
+   [benches.render-wasm.declarations :as declarations]
+   [benches.render-wasm.failures :as failures]
+   [benches.render-wasm.measurement :as measurement]
+   [benches.render-wasm.node.artifacts :as artifacts]
+   [benches.render-wasm.node.browser :as browser]
+   [benches.render-wasm.node.build :as build]
+   [benches.render-wasm.node.catalog :as catalog]
+   [benches.render-wasm.node.process :as process]
+   [benches.render-wasm.node.runs :as runs]
+   [benches.render-wasm.node.server :as server]
+   [benches.render-wasm.report.format :as format]
+   [benches.render-wasm.report.summarize :as summary]
+   [benches.render-wasm.result :as result]))
+
+(defn- now
+  []
+  (.toISOString (js/Date.)))
+
+(defn- message-of
+  [cause]
+  (or (ex-message cause) (str cause)))
+
+(defn- cause-evidence
+  "Plain failure evidence for a rejected browser, server or persistence action."
+  [cause fallback-phase]
+  (let [data  (ex-data cause)
+        phase (or (:phase data) (name fallback-phase))]
+    (cond-> {:status "failed"
+             :phase (if (keyword? phase) (name phase) (str phase))
+             :message (message-of cause)
+             :cause (failures/describe-cause cause)}
+      (:partial data) (assoc :partial (:partial data)))))
+
+(defn- failure-evidence
+  "Projects a bridge failure response into diagnostics evidence."
+  [evidence fallback-phase]
+  (if (map? evidence)
+    (cond-> (select-keys evidence [:status :phase :message :cause :partial])
+      (nil? (:status evidence)) (assoc :status "failed")
+      (nil? (:phase evidence)) (assoc :phase (name fallback-phase)))
+    {:status "failed"
+     :phase (name fallback-phase)
+     :message (str evidence)}))
+
+(defn- interruption
+  [signal]
+  (when (and signal (.-aborted ^js signal))
+    (let [reason (.-reason ^js signal)
+          data   (ex-data reason)]
+      {:signal (or (:signal data) "SIGINT")
+       :cause reason})))
+
+(defn- interrupted?
+  [state]
+  (some? (interruption (:signal state))))
+
+(defn- log
+  [state text]
+  (runs/append-log! (:home state) text)
+  (when-let [progress (:on-progress state)]
+    (progress text)))
+
+(defn- safe-log
+  "Appends runner diagnostics without replacing an error already under handling."
+  [state text]
+  (try
+    (log state text)
+    (catch :default _
+      nil)))
+
+(defn- persist
+  [state]
+  (runs/write-record! (:home state) @(:run-ref state)))
+
+(defn- record-preparation
+  [state preparation]
+  (swap! (:run-ref state) result/record-preparation preparation)
+  (persist state)
+  (log state (str "run: prepared " (result/case-name (:case-id preparation)) "\n")))
+
+(defn- record-attempt
+  [state attempt]
+  (swap! (:run-ref state) result/record-attempt attempt)
+  (persist state)
+  (log state (str "run: " (if (:warmup? attempt) "warmup" "attempt") " "
+                  (result/case-name (:case-id attempt)) " "
+                  (name (:outcome attempt)) "\n")))
+
+(defn- record-failure
+  [state case-id stage evidence]
+  (let [failure (cond-> (assoc (select-keys evidence [:status :phase :message :cause :partial])
+                               :stage stage)
+                  (some? case-id) (assoc :case-id case-id))]
+    (reset! (:harness-failed? state) true)
+    (swap! (:run-ref state) update-in [:metadata :diagnostics :failures]
+           (fnil conj []) failure)
+    (persist state)
+    (log state (str "run: " (name stage) " failed"
+                    (when (some? case-id) (str " for " (result/case-name case-id)))
+                    ": " (:message failure) "\n"))
+    failure))
+
+(defn- record-screenshot-status
+  [state case-id status]
+  (swap! (:run-ref state) assoc-in
+         [:metadata :diagnostics :screenshots (result/case-name case-id)] status)
+  (persist state))
+
+(defn- screenshot-status
+  [state case-id]
+  (get-in @(:run-ref state)
+          [:metadata :diagnostics :screenshots (result/case-name case-id)]))
+
+(defn- record-screenshot-skip
+  [state case-id reason]
+  (when (:screenshots? (:opts state))
+    (when-not (screenshot-status state case-id)
+      (record-screenshot-status state case-id {:status :skipped :reason reason}))))
+
+(defn- ensure-screenshot-status
+  [state case reason]
+  (when (:screenshots? (:opts state))
+    (when-not (screenshot-status state (:id case))
+      (record-screenshot-skip state (:id case) reason))))
+
+(defn- ^:async screenshot-case
+  [state case page]
+  (when (:screenshots? (:opts state))
+    (let [{:keys [file path]} (runs/screenshot-path (:home state) (:id case))
+          opts   (:opts state)
+          bounds {:deadline (process/deadline (:startup-timeout-ms opts))
+                  :signal (:signal state) :phase :screenshot}]
+      (try
+        (await ((get-in state [:services :screenshot]) page path bounds))
+        (record-screenshot-status state (:id case) {:file file :status :ok})
+        (log state (str "run: screenshot " file "\n"))
+        (catch :default cause
+          (record-screenshot-status state (:id case)
+                                    {:file file :status :failed
+                                     :message (message-of cause)
+                                     :cause (failures/describe-cause cause)})
+          (log state (str "run: screenshot failed for "
+                          (result/case-name (:id case)) ": "
+                          (message-of cause) "\n")))))))
+
+(defn- case-slots
+  "Returns each scheduled slot in order: all warmups, then measured slots.
+  True marks a warmup."
+  [opts]
+  (vec (concat (repeat (:warmups opts) true)
+               (repeat (:repetitions opts) false))))
+
+(defn- renderer-url
+  [build-id file]
+  (str "/renderer/" file "?build=" (js/encodeURIComponent build-id)))
+
+(defn- ^:async prepare-case
+  [state case page deadline]
+  (let [opts    (:opts state)
+        build-id (:id (:selected-build state))
+        request (transit/encode-str
+                 {:case case
+                  :preparation-id (str (random-uuid))
+                  :timeout-ms (:timeout-ms opts)
+                  :module-url (renderer-url build-id "render-wasm.js")
+                  :wasm-url (renderer-url build-id "render-wasm.wasm")})]
+    (await ((get-in state [:services :prepare])
+            page request
+            {:deadline deadline :signal (:signal state) :phase :preparation}))))
+
+(defn- ^:async run-attempt
+  [state page preparation-id deadline]
+  (let [request (transit/encode-str
+                 {:preparation-id preparation-id
+                  :timeout-ms (max 1 (process/remaining deadline))})]
+    (await ((get-in state [:services :run])
+            page request
+            {:deadline deadline :signal (:signal state) :phase :attempt}))))
+
+(defn- ^:async recycle-browser
+  [state]
+  (when-let [owner @(:owner-ref state)]
+    (reset! (:owner-ref state) nil)
+    (try
+      (await ((get-in state [:services :close]) owner))
+      (catch :default cause
+        (log state (str "run: browser cleanup failed: " (message-of cause) "\n"))))))
+
+(defn- ^:async close-current-page
+  [state]
+  (let [owner      @(:owner-ref state)
+        connected? (get-in state [:services :connected?])]
+    (if (and owner connected? (not (connected? owner)))
+      (do
+        (reset! (:page-ref state) nil)
+        (await (recycle-browser state))
+        false)
+      (if-let [page @(:page-ref state)]
+        (do
+          (reset! (:page-ref state) nil)
+          (try
+            (await ((get-in state [:services :close-page]) page))
+            true
+            (catch :default cause
+              (record-failure state nil :browser-close (cause-evidence cause :browser-close))
+              (await (recycle-browser state))
+              false)))
+        true))))
+
+(defn- ^:async ensure-browser
+  [state]
+  (or @(:owner-ref state)
+      (let [opts    (:opts state)
+            options (cond-> {:startup-timeout-ms (:startup-timeout-ms opts)
+                             :signal (:signal state)}
+                      (some? @(:launch-deadline state))
+                      (assoc :deadline @(:launch-deadline state)))
+            owner   (await ((get-in state [:services :launch]) options))]
+        (reset! (:owner-ref state) owner)
+        (swap! (:run-ref state) assoc-in [:metadata :environment :browser] (:version owner))
+        (persist state)
+        (log state (str "run: browser " (:version owner) "\n"))
+        owner)))
+
+(defn- ^:async open-page
+  [state case]
+  (if (interrupted? state)
+    :interrupted
+    (let [launch (try
+                   {:owner (await (ensure-browser state))}
+                   (catch :default cause {:error cause}))]
+      (if-let [cause (:error launch)]
+        (if (interrupted? state)
+          :interrupted
+          (do
+            (record-failure state (:id case) :browser-launch
+                            (cause-evidence cause :browser-launch))
+            (reset! (:abort-run? state) true)
+            :abort))
+        (let [opened (try
+                       {:page-owner (await ((get-in state [:services :new-page])
+                                            (:owner launch) @(:host-ref state)
+                                            {:viewport (declarations/resolve-viewport (:view case))
+                                             :startup-timeout-ms (:startup-timeout-ms (:opts state))
+                                             :signal (:signal state)}))}
+                       (catch :default cause {:error cause}))]
+          (if-let [cause (:error opened)]
+            (if (interrupted? state)
+              :interrupted
+              (do
+                (record-failure state (:id case) :browser-page
+                                (cause-evidence cause :browser-page))
+                (await (recycle-browser state))
+                :abandon))
+            (do
+              (reset! (:page-ref state) (:page-owner opened))
+              (:page (:page-owner opened)))))))))
+
+(defn- ^:async run-fresh-slot
+  [state case page warmup? final?]
+  (let [opts     (:opts state)
+        deadline (process/deadline (:timeout-ms opts))
+        case-id  (:id case)
+        prepared (try
+                   {:response (await (prepare-case state case page deadline))}
+                   (catch :default cause {:error cause}))]
+    (if-let [cause (:error prepared)]
+      (let [evidence (cause-evidence cause :preparation)]
+        (record-attempt state (measurement/attempt case-id nil warmup? evidence))
+        (when final? (record-screenshot-skip state case-id :no-completed-measured-attempt))
+        (if (interrupted? state) :interrupted :failed))
+      (let [response (:response prepared)]
+        (if (= "ok" (:status response))
+          (let [preparation (:preparation response)
+                prep-id     (:preparation-id preparation)]
+            (record-preparation state preparation)
+            (let [attempted (try
+                              {:evidence (await (run-attempt state page prep-id deadline))}
+                              (catch :default cause {:error cause}))]
+              (if-let [cause (:error attempted)]
+                (let [evidence (cause-evidence cause :attempt)]
+                  (record-attempt state
+                                  (measurement/attempt case-id prep-id warmup? evidence))
+                  (when final?
+                    (record-screenshot-skip state case-id :no-completed-measured-attempt))
+                  (if (interrupted? state) :interrupted :failed))
+                (let [attempt (measurement/attempt case-id prep-id warmup?
+                                                   (:evidence attempted))]
+                  (record-attempt state attempt)
+                  (if (= :completed (:outcome attempt))
+                    (do
+                      (when final? (await (screenshot-case state case page)))
+                      :ok)
+                    (do
+                      (when final?
+                        (record-screenshot-skip state case-id
+                                                :no-completed-measured-attempt))
+                      :failed))))))
+          (let [evidence (failure-evidence response :preparation)]
+            (record-attempt state (measurement/attempt case-id nil warmup? evidence))
+            (when final? (record-screenshot-skip state case-id :no-completed-measured-attempt))
+            (if (interrupted? state) :interrupted :failed)))))))
+
+(defn- ^:async run-reuse-slot
+  [state case page preparation-id warmup?]
+  (let [deadline (process/deadline (:timeout-ms (:opts state)))
+        result   (try
+                   {:evidence (await (run-attempt state page preparation-id deadline))}
+                   (catch :default cause {:error cause}))]
+    (if-let [cause (:error result)]
+      (let [evidence (cause-evidence cause :attempt)]
+        (record-attempt state
+                        (measurement/attempt (:id case) preparation-id warmup? evidence))
+        (if (interrupted? state) :interrupted :failed))
+      (let [attempt (measurement/attempt (:id case) preparation-id warmup?
+                                         (:evidence result))]
+        (record-attempt state attempt)
+        (if (= :completed (:outcome attempt)) :ok :failed)))))
+
+(defn- ^:async run-fresh-case
+  [state case]
+  (let [slots (case-slots (:opts state))
+        total (count slots)]
+    (loop [index 0]
+      (if (>= index total)
+        (do
+          (ensure-screenshot-status state case :case-abandoned)
+          :done)
+        (let [page (await (open-page state case))]
+          (cond
+            (= :interrupted page)
+            (do
+              (ensure-screenshot-status state case :case-abandoned)
+              :interrupted)
+
+            (= :abort page)
+            (do
+              (ensure-screenshot-status state case :case-abandoned)
+              :abort)
+
+            (= :abandon page)
+            (do
+              (ensure-screenshot-status state case :case-abandoned)
+              :abandon)
+
+            (nil? page)
+            (do
+              (ensure-screenshot-status state case :case-abandoned)
+              :abandon)
+
+            :else
+            (let [final?  (= index (dec total))
+                  outcome (await (run-fresh-slot state case page (nth slots index) final?))
+                  closed? (await (close-current-page state))]
+              (cond
+                (= :interrupted outcome)
+                (do
+                  (ensure-screenshot-status state case :case-abandoned)
+                  :interrupted)
+
+                (not closed?)
+                (do
+                  (ensure-screenshot-status state case :case-abandoned)
+                  :abandon)
+
+                :else
+                (recur (inc index))))))))))
+
+(defn- ^:async run-reuse-case
+  [state case]
+  (let [page (await (open-page state case))]
+    (cond
+      (= :interrupted page)
+      (do (ensure-screenshot-status state case :case-abandoned) :interrupted)
+
+      (= :abort page) :abort
+
+      (= :abandon page)
+      (do (ensure-screenshot-status state case :case-abandoned) :abandon)
+
+      (nil? page)
+      (do (ensure-screenshot-status state case :case-abandoned) :abandon)
+
+      :else
+      (let [opts     (:opts state)
+            deadline (process/deadline (:timeout-ms opts))
+            prepared (try
+                       {:response (await (prepare-case state case page deadline))}
+                       (catch :default cause {:error cause}))]
+        (if (or (:error prepared)
+                (not= "ok" (:status (:response prepared))))
+          (do
+            (record-failure state (:id case) :preparation
+                            (if-let [cause (:error prepared)]
+                              (cause-evidence cause :preparation)
+                              (failure-evidence (:response prepared) :preparation)))
+            (ensure-screenshot-status state case :no-completed-measured-attempt)
+            (await (close-current-page state))
+            (if (interrupted? state) :interrupted :abandon))
+          (let [preparation (:preparation (:response prepared))
+                prep-id     (:preparation-id preparation)
+                slots       (case-slots opts)
+                total       (count slots)]
+            (record-preparation state preparation)
+            (let [outcome
+                  (loop [index 0]
+                    (if (>= index total)
+                      :done
+                      (let [warmup? (nth slots index)
+                            result  (await (run-reuse-slot state case page prep-id warmup?))]
+                        (cond
+                          (= :ok result)
+                          (if (= index (dec total))
+                            (do (await (screenshot-case state case page)) :done)
+                            (recur (inc index)))
+
+                          (= index (dec total))
+                          (do
+                            (record-screenshot-skip state (:id case)
+                                                    :no-completed-measured-attempt)
+                            result)
+
+                          :else result))))
+                  closed? (await (close-current-page state))]
+              (ensure-screenshot-status state case :case-abandoned)
+              (cond
+                (not closed?) :abandon
+                (= outcome :interrupted) :interrupted
+                :else outcome))))))))
+
+(defn- ^:async launch-initial
+  [state]
+  (let [launched (try
+                   (await (ensure-browser state))
+                   true
+                   (catch :default cause
+                     (if (interrupted? state)
+                       false
+                       (do
+                         (record-failure state nil :browser-launch
+                                         (cause-evidence cause :browser-launch))
+                         false))))]
+    (reset! (:launch-deadline state) nil)
+    launched))
+
+(defn- ^:async run-session
+  [state]
+  (let [opts             (:opts state)
+        startup-deadline (process/deadline (:startup-timeout-ms opts))]
+    (reset! (:launch-deadline state) startup-deadline)
+    (try
+      (let [git (await (build/git-provenance (:root state)
+                                             {:deadline startup-deadline
+                                              :signal (:signal state)}))]
+        (swap! (:run-ref state) assoc-in [:metadata :diagnostics :runner-git] git)
+        (persist state))
+      (catch :default cause
+        (if (interrupted? state)
+          (throw cause)
+          (do
+            (record-failure state nil :runner-git (cause-evidence cause :runner-git))
+            (reset! (:abort-run? state) true)))))
+    (when-not @(:abort-run? state)
+      (try
+        (let [remaining (process/remaining startup-deadline)]
+          (when (zero? remaining)
+            (throw (ex-info "server startup timed out"
+                            {:type ::startup-timeout :phase :server-startup})))
+          (reset! (:host-ref state)
+                  (await (server/open (:selected-build state)
+                                      {:startup-timeout-ms remaining
+                                       :signal (:signal state)}))))
+        (catch :default cause
+          (if (interrupted? state)
+            (throw cause)
+            (do
+              (record-failure state nil :server-startup
+                              (cause-evidence cause :server-startup))
+              (reset! (:abort-run? state) true))))))
+    (when (and (not @(:abort-run? state)) (not (interrupted? state)))
+      (when-not (await (launch-initial state))
+        (reset! (:abort-run? state) true)))
+    (when (and (not @(:abort-run? state)) (not (interrupted? state)))
+      (loop [cases (:cases state)]
+        (when (and (seq cases)
+                   (not @(:abort-run? state))
+                   (not (interrupted? state)))
+          (let [case    (first cases)
+                outcome (if (= :fresh (:context case))
+                          (await (run-fresh-case state case))
+                          (await (run-reuse-case state case)))]
+            (when (#{:abort :interrupted} outcome)
+              (reset! (:abort-run? state) true))
+            (recur (rest cases))))))))
+
+(defn- finish-run
+  [state]
+  (let [run           @(:run-ref state)
+        interrupted   (interruption (:signal state))
+        all-attempted? (every? (fn [[_ slots]]
+                                 (and (zero? (:warmups slots))
+                                      (zero? (:repetitions slots))))
+                               (:remaining run))
+        failed?       (some #(= :failed (:outcome %)) (:attempts run))
+        reason        (cond
+                        interrupted :interrupted
+                        all-attempted? (if failed? :failed :completed)
+                        :else :incomplete)
+        failures      (get-in run [:metadata :diagnostics :failures])
+        attempt-failure (some #(when (:failure %) (:failure %))
+                              (reverse (:attempts run)))
+        termination   (cond-> {:reason reason}
+                        interrupted
+                        (assoc :signal (:signal interrupted))
+
+                        (and (seq failures)
+                             (or (not= :completed reason)
+                                 @(:harness-failed? state)))
+                        (assoc :failure (last failures))
+
+                        (and (not= :completed reason)
+                             (empty? failures)
+                             (some? attempt-failure))
+                        (assoc :failure attempt-failure))]
+    (when (:screenshots? (:opts state))
+      (swap! (:run-ref state) update-in [:metadata :diagnostics :screenshots]
+             (fn [screenshots]
+               (reduce (fn [saved case]
+                         (let [name (result/case-name (:id case))]
+                           (if (contains? saved name)
+                             saved
+                             (assoc saved name {:status :skipped :reason :not-run}))))
+                       (or screenshots {})
+                       (:cases state)))))
+    (swap! (:run-ref state) result/finish-run (now) termination)
+    (persist state)
+    @(:run-ref state)))
+
+(defn- ^:async cleanup
+  [state]
+  (let [page-failure
+        (try
+          (when-let [page @(:page-ref state)]
+            (reset! (:page-ref state) nil)
+            (await ((get-in state [:services :close-page]) page)))
+          nil
+          (catch :default cause cause))
+        browser-failure
+        (try
+          (when-let [owner @(:owner-ref state)]
+            (reset! (:owner-ref state) nil)
+            (await ((get-in state [:services :close]) owner)))
+          nil
+          (catch :default cause cause))
+        server-failure
+        (try
+          (when-let [host @(:host-ref state)]
+            (reset! (:host-ref state) nil)
+            (await (server/close host)))
+          nil
+          (catch :default cause cause))
+        failure (or page-failure browser-failure server-failure)]
+    (when failure
+      (safe-log state (str "run: cleanup failed: " (message-of failure) "\n")))
+    failure))
+
+(defn- remove-build
+  [root selected-build]
+  (artifacts/remove-builds root [(:id selected-build)]))
+
+(defn- git-facts
+  [selected-build]
+  (merge {:sha nil :dirty nil}
+         (get-in selected-build [:manifest :provenance :git])))
+
+(defn- playwright-version
+  [root]
+  (try
+    (let [require-package (if (exists? js/require)
+                            js/require
+                            (node-module/createRequire (path/join root "package.json")))]
+      (.-version (require-package "playwright/package.json")))
+    (catch :default _ nil)))
+
+(defn- environment
+  [root _selected-build]
+  (let [cpus (array-seq (os/cpus))]
+    {:node (.-version js/process)
+     :playwright (playwright-version root)
+     :browser nil
+     :platform (os/platform)
+     :release (os/release)
+     :arch (os/arch)
+     :cpu {:models (vec (distinct (map #(.-model %) cpus)))
+           :logical-count (count cpus)}
+     :launch browser/launch-settings}))
+
+(defn- metadata
+  [root selected-build opts]
+  {:git (git-facts selected-build)
+   :build {:functional (or (get-in selected-build [:manifest :functional]) {})
+           :provenance (assoc (get-in selected-build [:manifest :provenance])
+                              :build-id (:id selected-build))}
+   :environment (environment root selected-build)
+   :scored? true
+   :diagnostics {:seed (:seed opts)
+                 :filter (:filter opts)
+                 :warmups (:warmups opts)
+                 :repetitions (:repetitions opts)
+                 :timeout-ms (:timeout-ms opts)
+                 :build-timeout-ms (:build-timeout-ms opts)
+                 :startup-timeout-ms (:startup-timeout-ms opts)
+                 :screenshots? (boolean (:screenshots? opts))
+                 :runner-git {:sha nil :dirty nil}
+                 :runtime-helper-identity :unverified
+                 :screenshots {}
+                 :failures []}})
+
+(defn- normalize-options
+  [options]
+  (-> options
+      (assoc :screenshots? (boolean (or (:screenshots? options)
+                                        (:screenshots options)))
+             :remove? (boolean (or (:remove? options)
+                                   (:remove options))))
+      (dissoc :screenshots :remove)))
+
+(defn ^:async execute
+  "Executes a prepared selection and owns the run home, host and browser.
+
+  Options are resolved CLI options. root is the frontend checkout. The record
+  exists before on-created runs, so a browser startup failure still leaves a
+  valid partial record. Returns {:run RunRecord :home home :exit-code n
+  :error cause-or-nil}."
+  [root selected-build cases options signal on-progress on-created]
+  (let [opts  (-> (normalize-options options)
+                  (assoc :run-id (or (:run-id options) (str (random-uuid)))))
+        home  (runs/create-home {:root root
+                                 :run-id (:run-id opts)
+                                 :output-dir (:output-dir opts)})
+        run   (result/create-run
+               {:run-id (:run-id opts)
+                :started-at (now)
+                :plan {:cases cases
+                       :warmups (:warmups opts)
+                       :repetitions (:repetitions opts)
+                       :timeout-policy {:attempt-ms (:timeout-ms opts)}}
+                :metadata (metadata root selected-build opts)})
+        state {:root root
+               :selected-build selected-build
+               :cases cases
+               :opts opts
+               :signal signal
+               :on-progress on-progress
+               :home home
+               :run-ref (atom run)
+               :owner-ref (atom nil)
+               :page-ref (atom nil)
+               :host-ref (atom nil)
+               :launch-deadline (atom nil)
+               :abort-run? (atom false)
+               :harness-failed? (atom false)
+               :services (merge browser/service (:browser opts))}]
+    (persist state)
+    (when on-created (on-created (:run-id opts)))
+    (let [error (atom nil)]
+      (try
+        (await (run-session state))
+        (catch :default cause
+          (reset! error cause)
+          (when-not (interrupted? state)
+            (try
+              (record-failure state nil :runner (cause-evidence cause :runner))
+              (catch :default record-cause
+                (safe-log state (str "run: could not record failure: "
+                                     (message-of record-cause) "\n")))))))
+      (when-let [cleanup-error (await (cleanup state))]
+        (reset! error cleanup-error)
+        (try
+          (record-failure state nil :cleanup
+                          (cause-evidence cleanup-error :cleanup))
+          (catch :default record-cause
+            (safe-log state (str "run: could not record cleanup failure: "
+                                 (message-of record-cause) "\n")))))
+      (let [run           (try
+                            (finish-run state)
+                            (catch :default cause
+                              (reset! error cause)
+                              @(:run-ref state)))
+            removal-error (when (:remove? opts)
+                            (try
+                              (remove-build root selected-build)
+                              nil
+                              (catch :default cause cause)))
+            report-error  (try
+                            (log state (str "\n"
+                                            (format/format-run (summary/summarize run))
+                                            "\n"))
+                            nil
+                            (catch :default cause
+                              (safe-log state (str "run: report failed: "
+                                                   (message-of cause) "\n"))
+                              cause))]
+        (when removal-error
+          (reset! error removal-error))
+        (when report-error
+          (reset! error report-error))
+        (let [reason      (get-in run [:termination :reason])
+              signal-exit (if (= "SIGTERM" (get-in run [:termination :signal]))
+                            143 130)
+              base        (cond
+                            (= :completed reason) 0
+                            (= :interrupted reason) signal-exit
+                            :else 1)
+              exit        (if (= :interrupted reason)
+                            base
+                            (if (or @error
+                                    (and (= :completed reason)
+                                         @(:harness-failed? state)))
+                              1
+                              base))]
+          {:run run
+           :home home
+           :exit-code exit
+           :error @error})))))
+
+(defn ^:async run
+  "Prepares or loads a build, collects matching cases and executes one run.
+
+  User output paths resolve before any build work. A build or catalog failure
+  before the plan exists creates no run record and rethrows with build facts."
+  [root options signal on-progress on-created]
+  (let [opts       (normalize-options options)
+        output-dir (runs/check-output-dir (:output-dir opts))
+        run-id     (str (random-uuid))
+        opts       (assoc opts :run-id run-id :output-dir output-dir)]
+    (try
+      (let [selected-build
+            (if-let [id (:build-id opts)]
+              (do
+                (when on-progress (on-progress (str "run: using build " id "\n")))
+                (artifacts/load-build root id))
+              (await (build/prepare
+                      {:root root
+                       :features {:default-features? (:default-features opts true)
+                                  :features (or (:features opts) [])}
+                       :build-timeout-ms (:build-timeout-ms opts)
+                       :signal signal
+                       :on-created #(when on-progress
+                                      (on-progress (str "run: build " (:id %) "\n")))})))
+            cases (try
+                    (catalog/collect selected-build
+                                     {:master-seed (:seed opts)
+                                      :filter (:filter opts)})
+                    (catch :default cause
+                      (throw (ex-info (message-of cause)
+                                      (assoc (ex-data cause)
+                                             :build-id (:id selected-build)
+                                             :log-path (:log-path selected-build))
+                                      cause))))]
+        (await (execute root selected-build cases opts signal on-progress on-created)))
+      (catch :default cause
+        (when on-progress
+          (on-progress (str "run: " (message-of cause) "\n"))
+          (when-let [data (ex-data cause)]
+            (on-progress (str (pr-str data) "\n"))))
+        (throw cause)))))

frontend/performance/benches/render_wasm/node/runner.cljs

@@ -5,82 +5,296 @@
 ;; Copyright (c) KALEIDOS SUBSIDIARY SL
 
 (ns benches.render-wasm.node.runner
-  "Build preparation and managed artifact cleanup for renderer benchmarks."
+  "Build preparation, serial runs, offline comparison and managed cleanup."
   (:require
    [benches.render-wasm.node.artifacts :as artifacts]
    [benches.render-wasm.node.build :as build]
+   [benches.render-wasm.node.compare :as compare]
    [benches.render-wasm.node.process :as process]
+   [benches.render-wasm.node.run :as run]
    [clojure.string :as str]
    [clojure.tools.cli :refer [parse-opts]]))
 
 (enable-console-print!)
 
+(def ^:private max-timeout-ms 2147483647)
+
+(def ^:private positive-timeout
+  [(fn [n] (and (js/Number.isSafeInteger n)
+                (pos? n)
+                (<= n max-timeout-ms)))
+   "Must be a positive integer up to 2147483647"])
+
+(def ^:private nonnegative-integer
+  [(fn [n] (and (js/Number.isSafeInteger n) (not (neg? n))))
+   "Must be a nonnegative integer"])
+
+(def ^:private positive-integer
+  [(fn [n] (and (js/Number.isSafeInteger n) (pos? n)))
+   "Must be a positive integer"])
+
+(def ^:private uint32
+  [(fn [n] (and (js/Number.isSafeInteger n) (<= 0 n 4294967295)))
+   "Must be a uint32 integer"])
+
+(def ^:private nonempty
+  [(fn [s] (not (str/blank? s))) "Must be a nonempty value"])
+
+(defn- parse-number
+  "Parses a numeric CLI value. Blank input stays invalid instead of becoming zero."
+  [text]
+  (if (str/blank? text) js/NaN (js/Number text)))
+
 (def cli-options
   [["-h" "--help" "Print this help"]
    [nil "--[no-]default-features" "Enable Cargo defaults" :default true]
    [nil "--features FEATURES" "Comma-separated functional Cargo features"
     :parse-fn build/parse-feature-list]
+   [nil "--build ID" "Reuse a ready build" :id :build-id
+    :validate nonempty]
+   [nil "--filter SUBSTR" "Select cases whose id contains SUBSTR"]
+   [nil "--seed N" "Master scene seed" :default 42
+    :parse-fn parse-number :validate uint32]
+   [nil "--warmups N" "Warmup attempts per case" :default 3
+    :parse-fn parse-number :validate nonnegative-integer]
+   [nil "--repetitions N" "Measured attempts per case" :default 10
+    :parse-fn parse-number :validate positive-integer]
+   [nil "--timeout-ms MS" "Attempt deadline" :default 30000
+    :parse-fn parse-number :validate positive-timeout]
    [nil "--build-timeout-ms MS" "Complete preparation deadline"
-    :default process/default-build-timeout-ms :parse-fn js/Number
-    :validate [#(and (js/Number.isSafeInteger %) (pos? %)) "Must be a positive integer"]]
+    :default process/default-build-timeout-ms
+    :parse-fn parse-number :validate positive-timeout]
+   [nil "--startup-timeout-ms MS" "Host/browser readiness deadline"
+    :default process/default-startup-timeout-ms
+    :parse-fn parse-number :validate positive-timeout]
+   [nil "--output-dir DIR" "Explicit run home; must not exist"
+    :validate nonempty]
+   [nil "--remove" "Remove the used build after shutdown"]
+   [nil "--screenshots" "Capture one screenshot per case"]
    [nil "--only-builds" "Purge build attempts"]
-   [nil "--only-runs" "Purge managed runs"]])
+   [nil "--only-runs" "Purge managed runs"]
+   [nil "--compare-cases A,B" "Compare one explicit baseline,candidate pair"]
+   [nil "--force" "Bypass comparison compatibility checks"]])
 
 (defn usage
   [summary]
   (str "Usage: runner build [options]\n"
-       "       runner purge [--only-builds | --only-runs]\n\n"
+       "       runner purge [--only-builds | --only-runs]\n"
+       "       runner run [options]\n"
+       "       runner compare BASELINE CANDIDATE [--compare-cases A,B] [--force]\n\n"
        "build prints one attempt ID after storage initialization. Exit 0 means\n"
        "ready; errors retain the build.transit and build.log for that ID.\n"
-       "Progress goes to stderr. Preparation requires an idle checkout with\n"
-       "dependencies installed. Keep renderer watchers stopped until it ends.\n\n"
-       "purge requires an idle machine and deletes managed builds and runs.\n\n"
+       "run prints one run ID on stdout. Progress, failures and the summary go\n"
+       "to stderr. Builds and managed runs live under target/performance/render-wasm.\n"
+       "compare reads saved RunRecords and starts no build, host or browser.\n\n"
+       "Preparation and runs require an idle checkout with dependencies installed.\n"
+       "Keep renderer watchers stopped until preparation ends. purge assumes no\n"
+       "experiment is using the managed paths.\n\n"
        summary))
 
+(def ^:private value-flags
+  "Long options that consume the next argument when no value follows an equals sign."
+  (into #{}
+        (keep (fn [[_ long-option]]
+                (when (and long-option (re-find #"\s" long-option))
+                  (first (str/split long-option #"\s+")))))
+        cli-options))
+
+(defn- supplied-flags
+  [args]
+  (loop [tokens   (seq args)
+         flags    #{}
+         value?   false
+         operand? false]
+    (if-let [token (first tokens)]
+      (cond
+        value?
+        (recur (next tokens) flags false operand?)
+
+        (and (not operand?) (= "--" token))
+        (recur (next tokens) flags false true)
+
+        operand?
+        (recur (next tokens) flags false true)
+
+        (str/starts-with? token "--")
+        (let [[flag inline-value] (str/split token #"=" 2)]
+          (recur (next tokens)
+                 (conj flags flag)
+                 (and (contains? value-flags flag) (nil? inline-value))
+                 false))
+
+        :else
+        (recur (next tokens) flags false false))
+      flags)))
+
+(def ^:private build-flags
+  #{"--default-features" "--no-default-features" "--features" "--build-timeout-ms"})
+
+(def ^:private run-flags
+  #{"--build" "--filter" "--seed" "--warmups" "--repetitions" "--timeout-ms"
+    "--startup-timeout-ms" "--output-dir" "--remove" "--screenshots"})
+
+(def ^:private purge-flags
+  #{"--only-builds" "--only-runs"})
+
+(def ^:private compare-flags
+  #{"--compare-cases" "--force"})
+
+(defn- invalid-args
+  [message]
+  (throw (ex-info message {:type ::invalid-args})))
+
+(defn- some-flags?
+  [flags group]
+  (boolean (some flags group)))
+
+(defn- check-command-flags!
+  [command flags]
+  (let [build?   (some-flags? flags build-flags)
+        run?     (some-flags? flags run-flags)
+        purge?   (some-flags? flags purge-flags)
+        compare? (some-flags? flags compare-flags)]
+    (case command
+      "build"
+      (do
+        (when purge? (invalid-args "purge category flags require purge"))
+        (when run? (invalid-args "run options require run"))
+        (when compare? (invalid-args "compare options require compare")))
+
+      "purge"
+      (do
+        (when build? (invalid-args "build options require build"))
+        (when run? (invalid-args "run options require run"))
+        (when compare? (invalid-args "compare options require compare")))
+
+      "run"
+      (do
+        (when purge? (invalid-args "purge category flags require purge"))
+        (when compare? (invalid-args "compare options require compare")))
+
+      "compare"
+      (do
+        (when purge? (invalid-args "purge category flags require purge"))
+        (when build? (invalid-args "build options require build"))
+        (when run? (invalid-args "run options require run"))))))
+
+(defn- check-arguments!
+  [command arguments]
+  (let [extra (rest arguments)]
+    (case command
+      "compare"
+      (when-not (= 2 (count extra))
+        (invalid-args "compare requires BASELINE and CANDIDATE paths"))
+
+      (when (seq extra)
+        (invalid-args (str command " takes no positional arguments"))))))
+
 (defn parse-args
+  "Parses and validates one runner command without loading a catalog, build,
+  server, browser or renderer. Options are resolved before any run home."
   [args]
   (let [args (cond-> args (= "--" (first args)) rest)
         {:keys [arguments options errors summary]} (parse-opts args cli-options)
         command (first arguments)
-        flags   (into #{} (comp (filter #(str/starts-with? % "--"))
-                                (map #(first (str/split % #"=" 2)))) args)]
+        flags   (supplied-flags args)]
     (when (seq errors)
-      (throw (ex-info (str/join "\n" errors) {:type ::invalid-args})))
+      (invalid-args (str/join "\n" errors)))
     (when-not (:help options)
-      (when-not (and (= 1 (count arguments)) (contains? #{"build" "purge"} command))
-        (throw (ex-info "expected build or purge; try --help" {:type ::invalid-args})))
-      (when (and (:only-builds options) (:only-runs options))
-        (throw (ex-info "--only-builds and --only-runs are mutually exclusive" {:type ::invalid-args})))
-      (when (and (= command "build") (some flags ["--only-builds" "--only-runs"]))
-        (throw (ex-info "purge category flags require purge" {:type ::invalid-args})))
-      (when (and (= command "purge")
-                 (some flags ["--features" "--default-features" "--no-default-features" "--build-timeout-ms"]))
-        (throw (ex-info "build options require build" {:type ::invalid-args}))))
-    {:command command :options options :summary summary}))
+      (when-not (and (= 1 (count (take 1 arguments)))
+                     (contains? #{"build" "purge" "run" "compare"} command))
+        (invalid-args "expected build, purge, run or compare; try --help"))
+      (check-command-flags! command flags)
+      (check-arguments! command arguments)
+      (when (and (= command "purge") (:only-builds options) (:only-runs options))
+        (invalid-args "--only-builds and --only-runs are mutually exclusive"))
+      (when (and (= command "run")
+                 (:build-id options)
+                 (some-flags? flags ["--default-features" "--no-default-features" "--features"]))
+        (invalid-args "--build cannot be combined with feature-selection flags")))
+    {:command command
+     :arguments arguments
+     :options options
+     :summary summary}))
+
+(defn- print-error
+  [cause]
+  (.error js/console (or (ex-message cause) (str cause)))
+  (when-let [data (ex-data cause)]
+    (.error js/console (pr-str data))))
+
+(defn- ^:async run-command
+  [root options]
+  (let [controller   (js/AbortController.)
+        signal       (.-signal controller)
+        interrupted  (atom nil)
+        remove-signals (process/on-signals
+                        (fn [name]
+                          (when-not (.-aborted signal)
+                            (reset! interrupted name)
+                            (.abort controller
+                                    (ex-info "run interrupted"
+                                             {:type ::interrupted :signal name})))))]
+    (try
+      (let [result (await (run/run root options signal
+                                   #(.write (.-stderr js/process) %)
+                                   #(println %)))]
+        (when-let [error (:error result)]
+          (print-error error))
+        (set! (.-exitCode js/process) (:exit-code result)))
+      (catch :default cause
+        (print-error cause)
+        (set! (.-exitCode js/process)
+              (case @interrupted
+                "SIGINT" 130
+                "SIGTERM" 143
+                1)))
+      (finally
+        (remove-signals)))))
+
+(defn- ^:async build-command
+  [root options]
+  (let [controller     (js/AbortController.)
+        remove-signals (process/on-signals #(.abort controller %))]
+    (try
+      (await (build/prepare {:root root
+                             :features {:default-features? (:default-features options)
+                                        :features (or (:features options) [])}
+                             :build-timeout-ms (:build-timeout-ms options)
+                             :signal (.-signal controller)
+                             :on-created #(println (:id %))}))
+      (finally (remove-signals)))))
+
+(defn- ^:async compare-command
+  [arguments options]
+  (let [{:keys [text]} (compare/execute (second arguments) (nth arguments 2)
+                                        (select-keys options [:compare-cases :force]))]
+    (println text)))
 
 (defn ^:async -main
   [& args]
   (try
-    (let [{:keys [command options summary]} (parse-args args)
+    (let [{:keys [command arguments options summary]} (parse-args args)
           root (artifacts/frontend-root)]
       (cond
-        (:help options) (println (usage summary))
+        (:help options)
+        (println (usage summary))
+
         (= command "purge")
         (artifacts/purge root (cond (:only-builds options) :builds
                                     (:only-runs options) :runs
                                     :else :all))
+
         (= command "build")
-        (let [controller (js/AbortController.)
-              remove-signals (process/on-signals #(.abort controller %))]
-          (try
-            (await (build/prepare {:root root
-                                   :features {:default-features? (:default-features options)
-                                              :features (or (:features options) [])}
-                                   :build-timeout-ms (:build-timeout-ms options)
-                                   :signal (.-signal controller)
-                                   :on-created #(println (:id %))}))
-            (finally (remove-signals))))))
+        (await (build-command root options))
+
+        (= command "run")
+        (await (run-command root options))
+
+        (= command "compare")
+        (await (compare-command arguments options))))
     (catch :default cause
-      (.error js/console (or (ex-message cause) (str cause)))
-      (when-let [data (ex-data cause)] (.error js/console (pr-str data)))
+      (print-error cause)
+      (when (= ::invalid-args (:type (ex-data cause)))
+        (.error js/console "Try --help for usage."))
       (set! (.-exitCode js/process) 1))))

frontend/performance/benches/render_wasm/node/runs.cljs

@@ -0,0 +1,93 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns benches.render-wasm.node.runs
+  "Manages saved run destinations, RunRecord persistence and diagnostic logs.
+
+  Runs are saved under <frontend>/target/performance/render-wasm/runs.
+
+  This can be overriden in the runner with `--output-dir PATH`. If done, this
+  path is not included among those purged when cleaning up."
+  (:require
+   ["fs" :as fs]
+   ["path" :as path]
+   [benches.render-wasm.node.artifacts :as artifacts]
+   [benches.render-wasm.node.files :as files]
+   [benches.render-wasm.result :as result]
+   [clojure.string :as str]))
+
+(defn- invalid-home
+  [message data]
+  (throw (ex-info message (assoc data :type ::invalid-home))))
+
+(defn- check-id
+  [id]
+  (when-not (and (string? id) (re-matches #"[A-Za-z0-9][A-Za-z0-9_-]{0,127}" id))
+    (invalid-home "invalid run id" {:id id}))
+  id)
+
+(defn check-output-dir
+  "Resolves an explicit user directory from the caller's cwd and checks that it
+  does not exist. Nil selects the managed default. Does not create anything."
+  [dir]
+  (when (some? dir)
+    (when (str/blank? dir)
+      (invalid-home "output directory requires a nonempty path" {}))
+    (let [resolved (path/resolve dir)]
+      (when (fs/existsSync resolved)
+        (invalid-home "output directory already exists" {:path resolved}))
+      resolved)))
+
+(defn create-home
+  "Creates one run home and its empty log. The record is written separately
+  after the plan exists. An explicit directory must not exist; a managed home
+  uses the frontend root regardless of cwd."
+  [{:keys [root run-id output-dir]}]
+  (check-id run-id)
+  (let [explicit   (check-output-dir output-dir)
+        managed    (when-not explicit (artifacts/managed-root root))
+        runs-dir   (when managed (path/join managed "runs"))
+        dir        (or explicit (path/join runs-dir run-id))
+        record-path (path/join dir "run.transit")
+        log-path    (path/join dir "run.log")]
+    (when managed
+      (fs/mkdirSync runs-dir #js {:recursive true}))
+    (try
+      (fs/mkdirSync dir)
+      (catch :default cause
+        (invalid-home "cannot create run home"
+                      {:path dir :cause (or (ex-message cause) (str cause))})))
+    (try
+      (fs/writeFileSync log-path "" #js {:encoding "utf8" :flag "wx"})
+      (catch :default cause
+        (invalid-home "cannot create run log"
+                      {:path log-path :cause (or (ex-message cause) (str cause))})))
+    {:dir dir
+     :record-path record-path
+     :log-path log-path
+     :run-id run-id
+     :managed? (some? managed)}))
+
+(defn write-record!
+  "Validates and atomically replaces the saved RunRecord."
+  [home run]
+  (files/atomic-write (:record-path home) (result/encode run)))
+
+(defn append-log!
+  "Appends runner diagnostics to the run log. Callers keep lines terminated."
+  [home text]
+  (fs/appendFileSync (:log-path home) (str text) "utf8"))
+
+(defn screenshot-file
+  "Returns the relative PNG filename for a case, replacing path separators."
+  [case-id]
+  (str (str/replace (result/case-name case-id) #"[/\\]" "-") ".png"))
+
+(defn screenshot-path
+  "Returns the relative filename and absolute path under the run home."
+  [home case-id]
+  (let [file (screenshot-file case-id)]
+    {:file file :path (path/join (:dir home) file)}))

frontend/performance/benches/render_wasm/result.cljs

@@ -5,12 +5,7 @@
 ;; Copyright (c) KALEIDOS SUBSIDIARY SL
 
 (ns benches.render-wasm.result
-  "Raw benchmark records retain scheduled work and its chronological evidence.
-  RunRecord schema3 stores a saved plan, provenance, preparations and attempts.
-  A preparation owns a scene fingerprint and setup diagnostics. An attempt uses
-  one scheduled slot, even on failure. Metric entries declare value, unit,
-  semantics, observation basis and zero policy. Comparison schema2 stores an
-  analysis of two saved runs. Everything is encoded with transit."
+  "Schemas for experiments and reports."
   (:require
    [app.common.schema :as sm]
    [app.common.transit :as transit]

frontend/shadow-cljs.edn

@@ -212,24 +212,6 @@
      :elide-asserts true
      :anon-fn-naming-policy :off}}}
 
-  :bench-render-wasm-pilot
-  {:target :node-script
-   :output-to "target/performance/render-wasm/pilot.cjs"
-   :output-dir "target/performance/render-wasm/pilot/"
-   :main benches.render-wasm.node.pilot/-main
-   :devtools {:autoload false}
-
-   :compiler-options
-   {:output-feature-set :es2020
-    :output-wrapper false
-    :warnings {:fn-deprecated false}}
-
-   :release
-   {:compiler-options
-    {:fn-invoke-direct true
-     :elide-asserts true
-     :anon-fn-naming-policy :off}}}
-
   :bench-render-wasm-browser
   {:target :esm
    :output-dir #shadow/env ["RENDER_BENCH_BROWSER_DIR" :default "target/performance/render-wasm/browser/"]

frontend/test/frontend_tests/runner.cljs

@@ -175,7 +175,6 @@
 
 (def test-namespaces
   ['frontend-tests.basic-shapes-test
-   'frontend-tests.benches.fixture-test
    'frontend-tests.code-gen-style-test
    'frontend-tests.composable-tests.comp.sync-test
    'frontend-tests.copy-as-svg-test

frontend/test/performance_tests/render_wasm/browser_test.cljs

@@ -74,24 +74,36 @@
   (t/is (= "ECONNREFUSED" (await (socket-status endpoint))) "the owned CDP listener is gone"))
 
 (defn- ^:async close-fixture
-  [owner host]
+  [owner page host]
   (try
+    (when page
+      (try
+        (await (browser/close-page page))
+        (catch :default _ nil)))
     (when owner
-      (await (browser/close owner)))
+      (try
+        (await (browser/close owner))
+        (catch :default _ nil)))
     (finally
       (await (server/close host)))))
 
 (t/deftest ^:async successful-close-removes-the-profile-and-cdp-listener
   (let [host      (await (open-host build-id true))
-        owner-ref (atom nil)]
+        owner-ref (atom nil)
+        page-ref  (atom nil)]
     (try
-      (let [owner (await (browser/open host {:viewport viewport :startup-timeout-ms 30000}))]
-        (reset! owner-ref owner)
+      (let [owner (await (browser/launch {:startup-timeout-ms 30000}))
+            _     (reset! owner-ref owner)
+            page  (await (browser/new-page owner host {:viewport viewport
+                                                       :startup-timeout-ms 30000}))]
+        (reset! page-ref page)
         (t/is (= 200 (await (request-status (:endpoint owner) "/json/version"))))
         (t/is (= viewport
-                 (js->clj (await (.evaluate ^js (:page owner)
+                 (js->clj (await (.evaluate ^js (:page page)
                                             "({width:innerWidth,height:innerHeight,dpr:devicePixelRatio})"))
                           :keywordize-keys true)))
+        (await (browser/close-page page))
+        (reset! page-ref nil)
         (await (browser/close owner))
         (await (check-stopped owner))
         (t/is (not (.isConnected ^js (:browser owner))))
@@ -99,30 +111,44 @@
         (await (server/close host))
         (t/is (= "ECONNREFUSED" (await (socket-status (:base-url host))))))
       (finally
-        (await (close-fixture @owner-ref host))))))
+        (await (close-fixture @owner-ref @page-ref host))))))
 
-(t/deftest ^:async identity-mismatch-closes-chromium-before-startup-rejects
-  (let [host (await (open-host "wrong-compiled-build" true))]
+(t/deftest ^:async identity-mismatch-rejects-and-the-caller-closes-the-owned-browser
+  (let [host      (await (open-host "wrong-compiled-build" true))
+        owner-ref (atom nil)]
     (try
-      (let [cause (await (rejected (browser/open host {:viewport viewport :startup-timeout-ms 30000})))
+      (let [owner (await (browser/launch {:startup-timeout-ms 30000}))
+            _     (reset! owner-ref owner)
+            cause (await (rejected (browser/new-page
+                                    owner host
+                                    {:viewport viewport :startup-timeout-ms 30000})))
             facts (ex-data cause)]
         (t/is (= ::identity/mismatch (:type facts)))
         (t/is (= "wrong-compiled-build" (:actual-id facts)))
-        (await (check-stopped facts)))
+        (await (browser/close owner))
+        (reset! owner-ref nil)
+        (await (check-stopped owner)))
       (finally
-        (await (server/close host))))))
+        (await (close-fixture @owner-ref nil host))))))
 
 (t/deftest ^:async abort-stops-a-synchronously-blocked-page
   (let [host           (await (open-host build-id true))
         controller     (js/AbortController.)
         owner-ref      (atom nil)
+        page-ref       (atom nil)
         started-ref    (atom nil)
         started        (js/Promise. (fn [resolve _] (reset! started-ref resolve)))]
     (try
-      (let [owner    (await (browser/open host {:viewport viewport :startup-timeout-ms 30000
-                                                :signal (.-signal controller)}))
-            ^js page (:page owner)]
-        (reset! owner-ref owner)
+      (let [owner      (await (browser/launch {:startup-timeout-ms 30000
+                                               :signal (.-signal controller)}))
+            _          (reset! owner-ref owner)
+            page-owner (await (browser/new-page
+                               owner host
+                               {:viewport viewport
+                                :startup-timeout-ms 30000
+                                :signal (.-signal controller)}))
+            ^js page   (:page page-owner)]
+        (reset! page-ref page-owner)
         (await (.exposeFunction page "__blockingStarted" (fn [] (@started-ref))))
         (let [blocking (.evaluate page "window.__blockingStarted(); while (true) {}")]
           (.catch blocking (fn [_] nil))
@@ -133,31 +159,39 @@
                                                         :phase :test-blocked-page})))]
             (t/is (= ::browser/timeout (:type (ex-data cause)))))
           (.abort controller)
+          (reset! page-ref nil)
           (await (browser/close owner))
+          (reset! owner-ref nil)
           (await (check-stopped owner))
           (t/is (not (.isConnected ^js (:browser owner))))
           (t/is (instance? js/Error (await (rejected blocking))))))
       (finally
         (.abort controller)
-        (await (close-fixture @owner-ref host))))))
+        (await (close-fixture @owner-ref @page-ref host))))))
 
 (t/deftest ^:async abort-during-readiness-cleans-the-starting-browser
   (let [host       (await (open-host build-id false))
         controller (js/AbortController.)
+        owner-ref  (atom nil)
         requested  (js/Promise.
                     (fn [resolve _]
-                      (.once ^js (:server host) "request" (fn [_] (resolve nil)))))
-        opening    (browser/open host {:viewport viewport :startup-timeout-ms 30000
-                                       :signal (.-signal controller)})]
-    (.catch opening (fn [_] nil))
+                      (.once ^js (:server host) "request" (fn [_] (resolve nil)))))]
     (try
-      (await (browser/within (fn [] requested)
-                             {:deadline (process/deadline 30000) :phase :test-host-request}))
-      (.abort controller (ex-info "caller cancelled browser readiness" {:type ::caller-abort}))
-      (let [cause (await (rejected opening))]
-        (t/is (= ::caller-abort (:type (ex-data cause))))
-        (await (check-stopped (ex-data cause))))
+      (let [owner (await (browser/launch {:startup-timeout-ms 30000
+                                          :signal (.-signal controller)}))
+            _     (reset! owner-ref owner)
+            page  (browser/new-page owner host {:viewport viewport
+                                                :startup-timeout-ms 30000
+                                                :signal (.-signal controller)})]
+        (.catch page (fn [_] nil))
+        (await (browser/within (fn [] requested)
+                               {:deadline (process/deadline 30000) :phase :test-host-request}))
+        (.abort controller (ex-info "caller cancelled browser readiness" {:type ::caller-abort}))
+        (let [cause (await (rejected page))]
+          (t/is (= ::caller-abort (:type (ex-data cause)))))
+        (await (browser/close owner))
+        (reset! owner-ref nil)
+        (await (check-stopped owner)))
       (finally
         (.abort controller)
-        (let [owner (await (.catch opening (fn [_] nil)))]
-          (await (close-fixture owner host)))))))
+        (await (close-fixture @owner-ref nil host))))))

frontend/test/performance_tests/render_wasm/cli_integration.cjs

@@ -169,6 +169,133 @@ function seedRuns(fixture) {
   }
 }
 
+function runFixtureText({
+  runId,
+  caseId,
+  value,
+  environment = { node: "v24" },
+}) {
+  const k = transit.keyword;
+  const map = (pairs) => transit.map(pairs);
+  const metric = map([
+    k("value"),
+    value,
+    k("unit"),
+    k("ms"),
+    k("semantics"),
+    k("elapsed"),
+    k("basis"),
+    map([k("kind"), k("attempt-total")]),
+    k("zero-policy"),
+    k("positive"),
+    k("invalid-reason"),
+    null,
+  ]);
+  const preparation = map([
+    k("preparation-id"),
+    `${runId}-preparation`,
+    k("case-id"),
+    k(caseId),
+    k("fingerprint"),
+    map([
+      k("encoding-version"),
+      1,
+      k("algorithm"),
+      k("sha-256"),
+      k("digest"),
+      "a".repeat(64),
+    ]),
+    k("effective-graphics"),
+    map([k("webgl2"), true]),
+    k("diagnostics"),
+    map([]),
+  ]);
+  const attempt = map([
+    k("ordinal"),
+    0,
+    k("case-id"),
+    k(caseId),
+    k("preparation-id"),
+    `${runId}-preparation`,
+    k("warmup?"),
+    false,
+    k("outcome"),
+    k("completed"),
+    k("metrics"),
+    map([k("elapsed"), metric]),
+    k("trace"),
+    map([]),
+  ]);
+  const record = map([
+    k("type"),
+    k("run-record"),
+    k("schema-version"),
+    3,
+    k("run-id"),
+    runId,
+    k("started-at"),
+    "2026-10-26T00:00:00.000Z",
+    k("ended-at"),
+    "2026-10-26T00:01:00.000Z",
+    k("plan"),
+    map([
+      k("cases"),
+      [map([k("id"), k(caseId)])],
+      k("warmups"),
+      0,
+      k("repetitions"),
+      1,
+      k("timeout-policy"),
+      map([k("attempt-ms"), 30000]),
+    ]),
+    k("metadata"),
+    map([
+      k("git"),
+      map([k("sha"), runId, k("dirty"), false]),
+      k("build"),
+      map([k("functional"), map([k("mode"), k("release")])]),
+      k("environment"),
+      map(
+        Object.entries(environment).flatMap(([key, entry]) => [k(key), entry]),
+      ),
+      k("scored?"),
+      true,
+      k("diagnostics"),
+      map([]),
+    ]),
+    k("analysis"),
+    map([
+      k("method-version"),
+      1,
+      k("draws"),
+      2000,
+      k("seed"),
+      1885695600,
+      k("confidence"),
+      0.95,
+      k("min-ci"),
+      10,
+      k("min-p95"),
+      100,
+      k("min-p99"),
+      500,
+    ]),
+    k("preparations"),
+    [preparation],
+    k("attempts"),
+    [attempt],
+    k("remaining"),
+    map([
+      k(caseId),
+      map([k("warmups"), 0, k("repetitions"), 0]),
+    ]),
+    k("termination"),
+    map([k("reason"), k("completed")]),
+  ]);
+  return transit.writer("json").write(record);
+}
+
+
 function tree(directory) {
   return fs
     .readdirSync(directory, { withFileTypes: true })
@@ -325,3 +452,145 @@ test(
     }
   },
 );
+
+test(
+  "run validates an explicit output dir before preparing a build",
+  testOptions,
+  async (t) => {
+    const fixture = createFrontend(t);
+    const marker = path.join(fixture.managed, "compiler-ran");
+    fs.writeFileSync(
+      fixture.compiler,
+      `#!/usr/bin/env node
+const fs = require("node:fs");
+fs.writeFileSync(${JSON.stringify(marker)}, "ran");
+`,
+      { mode: 0o755 },
+    );
+    const output = path.join(fixture.managed, "existing-output");
+    fs.mkdirSync(output);
+    const original = tree(fixture.managed);
+
+    const result = await runCli(fixture, [
+      "run",
+      "--output-dir",
+      output,
+      "--warmups",
+      "0",
+      "--repetitions",
+      "1",
+    ]);
+
+    assert.equal(result.exitCode, 1);
+    assert.equal(result.stdout, "");
+    assert.match(result.stderr, /output directory already exists/);
+    assert.equal(fs.existsSync(marker), false);
+    assert.deepEqual(tree(fixture.managed), original);
+  },
+);
+
+test(
+  "run rejects retained-build feature selection before any work",
+  testOptions,
+  async (t) => {
+    const fixture = createFrontend(t);
+    const marker = path.join(fixture.managed, "compiler-ran");
+    fs.writeFileSync(
+      fixture.compiler,
+      `#!/usr/bin/env node
+const fs = require("node:fs");
+fs.writeFileSync(${JSON.stringify(marker)}, "ran");
+`,
+      { mode: 0o755 },
+    );
+    const original = tree(fixture.managed);
+
+    const result = await runCli(fixture, [
+      "run",
+      "--build",
+      "retained-id",
+      "--features",
+      "functional",
+    ]);
+
+    assert.equal(result.exitCode, 1);
+    assert.equal(result.stdout, "");
+    assert.match(result.stderr, /cannot be combined/);
+    assert.equal(fs.existsSync(marker), false);
+    assert.deepEqual(tree(fixture.managed), original);
+  },
+);
+
+test("help lists run and compare without touching managed storage", testOptions, async (t) => {
+  const fixture = createFrontend(t);
+  const original = tree(fixture.managed);
+
+  const result = await runCli(fixture, ["--help"]);
+
+  assert.equal(result.exitCode, 0);
+  assert.match(result.stdout, /runner run/);
+  assert.match(result.stdout, /runner compare/);
+  assert.match(result.stdout, /--compare-cases/);
+  assert.deepEqual(tree(fixture.managed), original);
+});
+
+
+test("compare reads saved run files without starting resources", testOptions, async (t) => {
+  const fixture = createFrontend(t);
+  const baseline = path.join(fixture.managed, "baseline.transit");
+  const candidate = path.join(fixture.managed, "candidate.transit");
+  fs.writeFileSync(
+    baseline,
+    runFixtureText({ runId: "baseline", caseId: "case/load", value: 10 }),
+  );
+  fs.writeFileSync(
+    candidate,
+    runFixtureText({ runId: "candidate", caseId: "case/load", value: 12 }),
+  );
+  const original = tree(fixture.managed);
+
+  const result = await runCli(fixture, ["compare", baseline, candidate]);
+
+  assert.equal(result.exitCode, 0);
+  assert.match(result.stdout, /Baseline/);
+  assert.match(result.stdout, /case\/load/);
+  assert.match(result.stdout, /median change/);
+  assert.deepEqual(tree(fixture.managed), original);
+});
+
+test("compare supports explicit pairs and force from the CLI", testOptions, async (t) => {
+  const fixture = createFrontend(t);
+  const baseline = path.join(fixture.managed, "baseline.transit");
+  const candidate = path.join(fixture.managed, "candidate.transit");
+  fs.writeFileSync(
+    baseline,
+    runFixtureText({ runId: "baseline", caseId: "a/pan", value: 10 }),
+  );
+  fs.writeFileSync(
+    candidate,
+    runFixtureText({
+      runId: "candidate",
+      caseId: "b/pan",
+      value: 12,
+      environment: { node: "v25" },
+    }),
+  );
+  const original = tree(fixture.managed);
+
+  const incompatible = await runCli(fixture, ["compare", baseline, candidate]);
+  assert.equal(incompatible.exitCode, 1);
+  assert.deepEqual(tree(fixture.managed), original);
+
+  const forced = await runCli(fixture, [
+    "compare",
+    baseline,
+    candidate,
+    "--compare-cases",
+    "a/pan,b/pan",
+    "--force",
+  ]);
+  assert.equal(forced.exitCode, 0);
+  assert.match(forced.stdout, /FORCED COMPARISON/);
+  assert.match(forced.stdout, /Pair a\/pan,b\/pan/);
+  assert.deepEqual(tree(fixture.managed), original);
+});

frontend/test/performance_tests/render_wasm/compare_test.cljs

@@ -0,0 +1,150 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns performance-tests.render-wasm.compare-test
+  (:require
+   ["fs" :as fs]
+   ["os" :as os]
+   ["path" :as path]
+   [benches.render-wasm.node.compare :as compare]
+   [benches.render-wasm.report.compare :as report-compare]
+   [benches.render-wasm.result :as result]
+   [cljs.test :as t :include-macros true]
+   [clojure.string :as str]))
+
+(def ^:private temporary-root (atom nil))
+
+(t/use-fixtures :each
+  {:before #(reset! temporary-root
+                    (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-compare-")))
+   :after #(fs/rmSync @temporary-root #js {:recursive true :force true})})
+
+(defn- metric
+  [value]
+  (result/metric {:value value
+                  :unit :ms
+                  :semantics :elapsed
+                  :basis {:kind :attempt-total}
+                  :zero-policy :positive}))
+
+(defn- case-definition
+  [case-id scene]
+  {:id case-id
+   :scene scene
+   :scene-version 1
+   :scene-description "comparison scene"
+   :scene-seed 7
+   :params {:count 10}
+   :view {:scale 1 :x 0 :y 0}
+   :context :fresh
+   :completion :render-full
+   :batch-size 1
+   :preparation {:version 1}})
+
+(defn- make-run
+  [run-id case-id value & {:keys [environment functional fingerprint scene]
+                           :or {environment {:node "v24" :browser "Fake"}
+                                functional {:mode :release}
+                                fingerprint (apply str (repeat 64 "a"))
+                                scene :case}}]
+  (let [preparation-id (str run-id "-preparation")
+        run (result/create-run
+             {:run-id run-id
+              :started-at "2026-10-26T00:00:00.000Z"
+              :plan {:cases [(case-definition case-id scene)]
+                     :warmups 0
+                     :repetitions 1
+                     :timeout-policy {:attempt-ms 30000}}
+              :metadata {:git {:sha run-id :dirty false}
+                         :build {:functional functional}
+                         :environment environment
+                         :scored? true}})
+        run (result/record-preparation
+             run
+             {:preparation-id preparation-id
+              :case-id case-id
+              :fingerprint {:encoding-version 1 :algorithm :sha-256
+                            :digest fingerprint}
+              :effective-graphics {:webgl2 true}
+              :diagnostics {}})
+        run (result/record-attempt
+             run
+             {:case-id case-id
+              :preparation-id preparation-id
+              :warmup? false
+              :outcome :completed
+              :metrics {:elapsed (metric value)}
+              :trace {}})]
+    (result/finish-run run "2026-10-26T00:01:00.000Z" {:reason :completed})))
+
+(defn- write-run
+  [file run]
+  (fs/writeFileSync file (result/encode run) "utf8")
+  file)
+
+(defn- failure
+  [f]
+  (try
+    (f)
+    nil
+    (catch :default cause cause)))
+
+(t/deftest compares-saved-file-paths-from-the-caller-cwd
+  (write-run (path/join @temporary-root "baseline.transit")
+             (make-run "baseline" :case/load 10))
+  (write-run (path/join @temporary-root "candidate.transit")
+             (make-run "candidate" :case/load 12))
+  (let [previous (.cwd js/process)]
+    (try
+      (.chdir js/process @temporary-root)
+      (let [{:keys [comparison text]} (compare/execute "baseline.transit"
+                                                       "candidate.transit" {})]
+        (t/is (= :comparison (:type comparison)))
+        (t/is (= 2 (:schema-version comparison)))
+        (t/is (str/includes? text "Baseline"))
+        (t/is (str/includes? text "Candidate"))
+        (t/is (str/includes? text "case/load"))
+        (t/is (str/includes? text "median change")))
+      (finally
+        (.chdir js/process previous)))))
+
+(t/deftest compare-cases-selects-one-explicit-baseline-candidate-pair
+  (let [baseline  (write-run (path/join @temporary-root "baseline.transit")
+                             (make-run "baseline" :a/pan 10))
+        candidate (write-run (path/join @temporary-root "candidate.transit")
+                             (make-run "candidate" :b/pan 12 :scene :other))
+        {:keys [text]} (compare/execute baseline candidate
+                                        {:compare-cases "a/pan,b/pan"})]
+    (t/is (str/includes? text "Pair a/pan,b/pan"))))
+
+(t/deftest force-bypasses-differences-and-displays-them
+  (let [baseline  (write-run (path/join @temporary-root "baseline.transit")
+                             (make-run "baseline" :case/load 10))
+        candidate (write-run (path/join @temporary-root "candidate.transit")
+                             (make-run "candidate" :case/load 12
+                                       :environment {:node "v25" :browser "Fake"}))]
+    (t/is (= ::report-compare/incompatible
+             (:type (ex-data (failure #(compare/execute baseline candidate {}))))))
+    (let [{:keys [text]} (compare/execute baseline candidate {:force true})]
+      (t/is (str/starts-with? text "FORCED COMPARISON"))
+      (t/is (str/includes? text "MISMATCHES")))))
+
+(t/deftest invalid-selectors-and-unreadable-records-fail-at-the-boundary
+  (let [baseline  (write-run (path/join @temporary-root "baseline.transit")
+                             (make-run "baseline" :case/load 10))
+        candidate (write-run (path/join @temporary-root "candidate.transit")
+                             (make-run "candidate" :case/load 12))]
+    (doseq [selector ["case/load" "case/load,other/load,third/load" ""]]
+      (t/is (= ::report-compare/invalid-selector
+               (:type (ex-data (failure #(compare/execute baseline candidate
+                                                          {:compare-cases selector})))))))
+    (fs/writeFileSync (path/join @temporary-root "broken.transit") "not transit")
+    (t/is (thrown? js/Error (compare/execute baseline
+                                             (path/join @temporary-root "broken.transit")
+                                             {})))
+    (t/is (thrown? js/Error (compare/execute baseline
+                                             (path/join @temporary-root "missing.transit")
+                                             {})))))

frontend/test/performance_tests/render_wasm/lifecycle_test.cljs

@@ -197,7 +197,7 @@
         (t/is (= "aborted" (:phase result)))
         (t/is (< current @@#'browser/owner-epoch*) "current owner disposed")))))
 
-(t/deftest bridge-exposes-pilot-entries
+(t/deftest bridge-exposes-executor-entries
   (doseq [key ["ping" "loadScene" "prepareScene" "runPrepared" "dispose"]]
     (t/is (fn? (unchecked-get browser/bridge key))
           (str key " is callable through the browser ABI")))

frontend/test/performance_tests/render_wasm/pilot_test.cljs

@@ -1,61 +0,0 @@
-;; This Source Code Form is subject to the terms of the Mozilla Public
-;; License, v. 2.0. If a copy of the MPL was not distributed with this
-;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
-;;
-;; Copyright (c) KALEIDOS SUBSIDIARY SL
-
-(ns performance-tests.render-wasm.pilot-test
-  (:require
-   [benches.render-wasm.node.pilot :as pilot]
-   [cljs.test :as t :include-macros true]))
-
-(t/deftest diagnostic-budget-is-fixed-and-explicit
-  (let [defaults (pilot/parse-args [])
-        explicit (pilot/parse-args ["--warmups" "0" "--attempts" "1"])]
-    (t/is (= defaults explicit))
-    (t/is (= {:warmups 0 :attempts 1 :timeout-ms 30000
-              :build-timeout-ms 300000 :startup-timeout-ms 30000}
-             (select-keys defaults [:warmups :attempts :timeout-ms :build-timeout-ms :startup-timeout-ms]))))
-  (doseq [args [["--warmups" "1"] ["--attempts" "0"] ["--attempts" "2"]]]
-    (t/is (thrown-with-msg? js/Error #"requires --warmups 0 and --attempts 1"
-                            (pilot/parse-args args)))))
-
-(t/deftest retained-reuse-rejects-every-explicit-feature-selection
-  (t/is (= "retained-id" (:build-id (pilot/parse-args ["--build" "retained-id"]))))
-  (doseq [args [["--build" "retained-id" "--no-default-features"]
-                ["--no-default-features" "--build" "retained-id"]
-                ["--build" "retained-id" "--features" ""]
-                ["--features" "functional" "--build" "retained-id"]]]
-    (t/is (thrown-with-msg? js/Error #"cannot be combined"
-                            (pilot/parse-args args)))))
-
-(t/deftest numeric-options-reject-prefix-parsing-and-invalid-bounds
-  (doseq [flag ["--seed" "--timeout-ms" "--build-timeout-ms" "--startup-timeout-ms"
-                "--warmups" "--attempts"]
-          value ["1ms" "1.5" "1e3" "NaN" "-1" "9007199254740992"]]
-    (t/is (thrown? js/Error (pilot/parse-args [flag value]))))
-  (doseq [args [["--seed" "4294967296"] ["--timeout-ms" "0"]
-                ["--build-timeout-ms" "0"] ["--startup-timeout-ms" "0"]
-                ["--timeout-ms" "2147483648"]]]
-    (t/is (thrown? js/Error (pilot/parse-args args))))
-  (t/is (= 4294967295 (:seed (pilot/parse-args ["--seed" "4294967295"])))))
-
-(t/deftest diagnostic-options-keep-case-selection-for-the-retained-catalog
-  (let [opts (pilot/parse-args ["--case" "retained-scene/operation" "--seed" "7"
-                                "--timeout-ms" "100" "--build-timeout-ms" "200"
-                                "--startup-timeout-ms" "300" "--no-default-features"
-                                "--features" "first, second" "--screenshot" "/tmp/pilot.png"])]
-    (t/is (= :retained-scene/operation (:case-id opts)))
-    (t/is (= {:default-features? false :features ["first" "second"]} (:features opts)))
-    (t/is (= {:seed 7 :timeout-ms 100 :build-timeout-ms 200 :startup-timeout-ms 300
-              :screenshot "/tmp/pilot.png"}
-             (select-keys opts [:seed :timeout-ms :build-timeout-ms :startup-timeout-ms :screenshot])))))
-
-(t/deftest malformed-options-fail-at-the-cli-boundary
-  (doseq [args [["--build"] ["--case"] ["--seed"] ["--screenshot"]
-                ["--build" ""] ["--screenshot" ""] ["--case" "load"]
-                ["--features" "first,,second"] ["--timeout-ms" "--seed" "42"]
-                ["--unknown"]]]
-    (t/is (thrown? js/Error (pilot/parse-args args))))
-  (t/is (:help? (pilot/parse-args ["--help"])))
-  (t/is (:help? (pilot/parse-args ["-h"]))))

frontend/test/performance_tests/render_wasm/run_test.cljs

@@ -0,0 +1,470 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns performance-tests.render-wasm.run-test
+  (:require
+   ["fs" :as fs]
+   ["os" :as os]
+   ["path" :as path]
+   [app.common.transit :as transit]
+   [benches.render-wasm.node.artifacts :as artifacts]
+   [benches.render-wasm.node.run :as run]
+   [benches.render-wasm.result :as result]
+   [cljs.test :as t :include-macros true]
+   [clojure.string :as str]))
+
+(def ^:private temporary-root (atom nil))
+
+(t/use-fixtures :each
+  {:before #(reset! temporary-root
+                    (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-run-")))
+   :after #(fs/rmSync @temporary-root #js {:recursive true :force true})})
+
+(defn- output-dir
+  []
+  (path/join @temporary-root (str "out-" (random-uuid))))
+
+(defn- build-dir
+  []
+  (let [dir (path/join @temporary-root (str "build-" (random-uuid)))]
+    (fs/mkdirSync dir #js {:recursive true})
+    (fs/writeFileSync (path/join dir "index.html") "<!doctype html>")
+    dir))
+
+(defn- selected-build
+  [dir id]
+  {:id id
+   :dir dir
+   :log-path (path/join dir "build.log")
+   :manifest {:type :renderer-build
+              :schema-version 1
+              :id id
+              :status :ready
+              :functional {:mode :release
+                           :default-features? true
+                           :features []}
+              :provenance {:git {:sha "build-revision" :dirty false}}}})
+
+(defn- collected
+  [id context]
+  {:id id
+   :scene :case
+   :scene-version 1
+   :scene-description "test scene"
+   :scene-seed 1
+   :params {}
+   :view {:scale 1 :x 0 :y 0}
+   :context context
+   :completion :render-full
+   :batch-size 1
+   :preparation {:version 1}})
+
+(defn- next-count
+  [state key]
+  (inc (get @state key 0)))
+
+(defn- check-page
+  [page]
+  (when-not (and (map? page) (some? (:id page)))
+    (throw (ex-info "browser service requires the page object"
+                    {:phase :browser-page-startup}))))
+
+(defn- fake-browser
+  [{:keys [launch-failure new-page-failure close-page-failure
+           prepare-failure run-failure screenshot-failure
+           interruption controller]}]
+  (let [state (atom {:launches 0 :pages 0 :close-pages 0 :closes 0
+                     :host nil :runs {} :preparations {}})]
+    {:state state
+     :service
+     {:launch
+      (fn [_]
+        (let [n (next-count state :launches)]
+          (swap! state assoc :launches n)
+          (when (and launch-failure (launch-failure n))
+            (throw (ex-info "browser launch failed" {:phase :browser-startup})))
+          {:version "FakeBrowser/1" :browser #js {}}))
+      :new-page
+      (fn [_ host _]
+        (when-not (and (string? (:base-url host))
+                       (some? (get-in host [:build :id])))
+          (throw (ex-info "new-page requires the opened host"
+                          {:phase :browser-page-startup})))
+        (swap! state assoc :host host)
+        (let [n (next-count state :pages)]
+          (swap! state assoc :pages n)
+          (when (and new-page-failure (new-page-failure n))
+            (throw (ex-info "browser page failed" {:phase :browser-page-startup})))
+          {:page {:id n} :context {}}))
+      :close-page
+      (fn [_]
+        (let [n (next-count state :close-pages)]
+          (swap! state assoc :close-pages n)
+          (when (and close-page-failure (close-page-failure n))
+            (throw (ex-info "page close failed" {:phase :browser-page-close})))))
+      :close
+      (fn [_]
+        (swap! state update :closes inc))
+      :connected?
+      (fn [_] true)
+      :prepare
+      (fn [page request _]
+        (check-page page)
+        (let [{:keys [preparation-id] case-desc :case} (transit/decode-str request)
+              case-id                       (:id case-desc)
+              failure                       (when prepare-failure
+                                              (prepare-failure case-id))]
+          (case failure
+            :throw
+            (throw (ex-info "scene preparation failed" {:phase :scene-build}))
+
+            :response
+            {:status "failed" :phase "scene-build" :message "scene rejected"}
+
+            (let [preparation {:preparation-id preparation-id
+                               :case-id case-id
+                               :fingerprint {:encoding-version 1
+                                             :algorithm :sha-256
+                                             :digest (apply str (repeat 64 "a"))}
+                               :effective-graphics {}
+                               :diagnostics {}}]
+              (swap! state assoc-in [:preparations preparation-id] case-id)
+              {:status "ok" :preparation preparation}))))
+      :run
+      (fn [page request _]
+        (check-page page)
+        (let [{:keys [preparation-id]} (transit/decode-str request)
+              case-id (get-in @state [:preparations preparation-id])
+              n       (inc (get-in @state [:runs case-id] 0))
+              _       (swap! state assoc-in [:runs case-id] n)
+              signal  (when interruption (interruption case-id n))]
+          (when signal
+            (let [reason (ex-info "run interrupted"
+                                  {:type ::interrupted :signal signal})]
+              (.abort controller reason)
+              (throw reason)))
+          (case (when run-failure (run-failure case-id n))
+            :throw
+            (throw (ex-info "attempt failed" {:phase "interact"}))
+
+            :response
+            {:status "failed" :phase "interact" :message "attempt rejected"}
+
+            {:status "ok" :metrics {} :trace {}})))
+      :screenshot
+      (fn [page file _]
+        (check-page page)
+        (when (and screenshot-failure (screenshot-failure file))
+          (throw (ex-info "screenshot failed" {:phase :screenshot})))
+        (fs/writeFileSync file "png"))}}))
+
+(defn- options
+  [fake out]
+  {:run-id "test-run"
+   :output-dir out
+   :seed 42
+   :filter nil
+   :warmups 0
+   :repetitions 1
+   :timeout-ms 5000
+   :build-timeout-ms 60000
+   :startup-timeout-ms 5000
+   :screenshots false
+   :remove false
+   :browser (:service fake)})
+
+(defn- execute
+  ([fake cases opts]
+   (execute fake cases opts (js/AbortController.)))
+  ([_fake cases opts controller]
+   (run/execute (artifacts/frontend-root)
+                (selected-build (build-dir) "test-build")
+                cases opts (.-signal controller) nil nil)))
+
+(defn- attempts-for
+  [run case-id]
+  (filterv #(= case-id (:case-id %)) (:attempts run)))
+
+(defn- preparations-for
+  [run case-id]
+  (filterv #(= case-id (:case-id %)) (:preparations run)))
+
+(defn- saved-record
+  [result]
+  (result/decode (fs/readFileSync (get-in result [:home :record-path]) "utf8")))
+
+(t/deftest ^:async fresh-and-reuse-cases-follow-their-context-policies
+  (let [fake    (fake-browser {})
+        cases   [(collected :case/fresh :fresh) (collected :case/reuse :reuse)]
+        opts    (assoc (options fake (output-dir)) :warmups 1 :repetitions 2)
+        result  (await (execute fake cases opts))
+        saved   (saved-record result)
+        run     (:run result)
+        fresh   (attempts-for run :case/fresh)
+        reuse   (attempts-for run :case/reuse)]
+    (t/is (= saved run))
+    (t/is (= 6 (count (:attempts run))))
+    (t/is (= 3 (count (preparations-for run :case/fresh))))
+    (t/is (= 1 (count (preparations-for run :case/reuse))))
+    (t/is (= 3 (count (distinct (map :preparation-id fresh)))))
+    (t/is (= 1 (count (distinct (map :preparation-id reuse)))))
+    (t/is (every? #(= :completed (:outcome %)) (:attempts run)))
+    (t/is (every? #(every? zero? (vals %)) (vals (:remaining run))))
+    (t/is (= :completed (get-in run [:termination :reason])))
+    (t/is (zero? (:exit-code result)))
+    (t/is (empty? (get-in run [:metadata :diagnostics :screenshots])))
+    (t/is (empty? (filter #(str/ends-with? % ".png")
+                          (array-seq (fs/readdirSync (get-in result [:home :dir]))))))))
+
+(t/deftest ^:async page-startup-failure-abandons-the-case-and-keeps-its-slots
+  (doseq [context [:fresh :reuse]]
+    (let [fake   (fake-browser {:new-page-failure #(= 1 %)})
+          opts   (assoc (options fake (output-dir)) :repetitions 2)
+          result (await (execute fake [(collected :case/a context)
+                                       (collected :case/b :fresh)]
+                                 opts))
+          run    (:run result)
+          state  @(:state fake)]
+      (t/is (empty? (attempts-for run :case/a)))
+      (t/is (empty? (preparations-for run :case/a)))
+      (t/is (= {:warmups 0 :repetitions 2}
+               (get-in run [:remaining :case/a])))
+      (t/is (= [:completed :completed]
+               (mapv :outcome (attempts-for run :case/b))))
+      (t/is (= :incomplete (get-in run [:termination :reason])))
+      (t/is (= :browser-page (get-in run [:termination :failure :stage])))
+      (t/is (= 1 (:exit-code result)))
+      (t/is (= 2 (:launches state)))
+      (t/is (= 2 (:closes state))))))
+
+(t/deftest ^:async failed-attempts-consume-slots-and-the-runner-continues
+  (let [fake   (fake-browser {:run-failure (fn [case-id n]
+                                             (when (and (= case-id :case/fresh) (= n 1))
+                                               :throw))})
+        opts   (assoc (options fake (output-dir)) :repetitions 2)
+        result (await (execute fake [(collected :case/fresh :fresh)] opts))
+        run    (:run result)
+        tries  (attempts-for run :case/fresh)]
+    (t/is (= 2 (count tries)))
+    (t/is (= [:failed :completed] (mapv :outcome tries)))
+    (t/is (every? zero? (vals (get-in run [:remaining :case/fresh]))))
+    (t/is (= :failed (get-in run [:termination :reason])))
+    (t/is (= 1 (:exit-code result)))
+    (t/is (= 2 (count (preparations-for run :case/fresh))))))
+
+(t/deftest ^:async reuse-preparation-failure-leaves-slots-and-continues-to-the-next-case
+  (let [fake   (fake-browser {:prepare-failure #(when (= % :case/reuse) :response)})
+        opts   (assoc (options fake (output-dir)) :screenshots true)
+        result (await (execute fake [(collected :case/reuse :reuse)
+                                     (collected :case/fresh :fresh)] opts))
+        run    (:run result)
+        shots  (get-in run [:metadata :diagnostics :screenshots])]
+    (t/is (empty? (attempts-for run :case/reuse)))
+    (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/reuse])))
+    (t/is (= 1 (count (attempts-for run :case/fresh))))
+    (t/is (= :incomplete (get-in run [:termination :reason])))
+    (t/is (= 1 (:exit-code result)))
+    (t/is (= :skipped (get-in shots ["case/reuse" :status])))
+    (t/is (= :ok (get-in shots ["case/fresh" :status])))
+    (let [failure (last (get-in run [:metadata :diagnostics :failures]))]
+      (t/is (= :preparation (:stage failure)))
+      (t/is (= :case/reuse (:case-id failure)))
+      (t/is (= "scene-build" (:phase failure))))))
+
+(t/deftest ^:async fresh-preparation-failure-consumes-its-slot
+  (let [fake   (fake-browser {:prepare-failure #(when (= % :case/fresh) :throw)})
+        result (await (execute fake [(collected :case/fresh :fresh)]
+                               (options fake (output-dir))))
+        run    (:run result)
+        tries  (attempts-for run :case/fresh)]
+    (t/is (= 1 (count tries)))
+    (t/is (= :failed (:outcome (first tries))))
+    (t/is (nil? (:preparation-id (first tries))))
+    (t/is (= "scene-build" (get-in (first tries) [:failure :phase])))
+    (t/is (empty? (preparations-for run :case/fresh)))
+    (t/is (every? zero? (vals (get-in run [:remaining :case/fresh]))))
+    (t/is (= :failed (get-in run [:termination :reason])))
+    (t/is (= 1 (:exit-code result)))))
+
+(t/deftest ^:async browser-close-failure-abandons-the-case-and-relaunches-for-the-next
+  (let [fake   (fake-browser {:run-failure (fn [case-id n]
+                                             (when (and (= case-id :case/a) (= n 1))
+                                               :throw))
+                              :close-page-failure #(= % 1)})
+        opts   (assoc (options fake (output-dir)) :repetitions 2)
+        result (await (execute fake [(collected :case/a :fresh)
+                                     (collected :case/b :fresh)] opts))
+        run    (:run result)
+        state  @(:state fake)]
+    (t/is (= 2 (:launches state)))
+    (t/is (= 1 (count (attempts-for run :case/a))))
+    (t/is (= :failed (:outcome (first (attempts-for run :case/a)))))
+    (t/is (= 1 (get-in run [:remaining :case/a :repetitions])))
+    (t/is (= 2 (count (attempts-for run :case/b))))
+    (t/is (every? #(= :completed (:outcome %)) (attempts-for run :case/b)))
+    (t/is (every? zero? (vals (get-in run [:remaining :case/b]))))
+    (t/is (= :incomplete (get-in run [:termination :reason])))
+    (t/is (= 1 (:exit-code result)))))
+
+(t/deftest ^:async page-close-failure-after-the-final-slot-is-a-harness-failure
+  (let [fake   (fake-browser {:close-page-failure #(= % 1)})
+        result (await (execute fake [(collected :case/a :fresh)]
+                               (options fake (output-dir))))
+        run    (:run result)]
+    (t/is (= :completed (get-in run [:termination :reason])))
+    (t/is (every? zero? (vals (get-in run [:remaining :case/a]))))
+    (t/is (= :browser-close
+             (:stage (last (get-in run [:metadata :diagnostics :failures])))))
+    (t/is (= :browser-close (get-in run [:termination :failure :stage])))
+    (t/is (= 1 (:exit-code result)))))
+
+(t/deftest ^:async initial-browser-launch-failure-aborts-before-any-case
+  (let [fake   (fake-browser {:launch-failure #(= % 1)})
+        result (await (execute fake [(collected :case/a :fresh)]
+                               (options fake (output-dir))))
+        run    (:run result)
+        state  @(:state fake)]
+    (t/is (= 1 (:launches state)))
+    (t/is (empty? (:attempts run)))
+    (t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/a])))
+    (t/is (= :incomplete (get-in run [:termination :reason])))
+    (t/is (= :browser-launch (:stage (last (get-in run [:metadata :diagnostics :failures])))))
+    (t/is (= 1 (:exit-code result)))))
+
+(t/deftest ^:async sigint-and-sigterm-persist-partial-interrupted-records
+  (doseq [[signal code] [["SIGINT" 130] ["SIGTERM" 143]]]
+    (let [controller (js/AbortController.)
+          fake       (fake-browser {:controller controller
+                                    :interruption (fn [case-id n]
+                                                    (when (and (= case-id :case/a) (= n 1))
+                                                      signal))})
+          opts       (assoc (options fake (output-dir)) :screenshots true)
+          result     (await (execute fake [(collected :case/a :fresh)
+                                           (collected :case/b :fresh)]
+                                     opts controller))
+          run        (:run result)
+          saved      (saved-record result)
+          shots      (get-in run [:metadata :diagnostics :screenshots])]
+      (t/is (= saved run))
+      (t/is (= :interrupted (get-in run [:termination :reason])))
+      (t/is (= signal (get-in run [:termination :signal])))
+      (t/is (= code (:exit-code result)))
+      (t/is (= 1 (count (attempts-for run :case/a))))
+      (t/is (empty? (attempts-for run :case/b)))
+      (t/is (= 1 (get-in run [:remaining :case/b :repetitions])))
+      (t/is (= :skipped (get-in shots ["case/b" :status])))
+      (t/is (= :not-run (get-in shots ["case/b" :reason])))
+      (t/is (pos? (:closes @(:state fake)))))))
+
+(t/deftest ^:async log-failure-still-closes-pages-browsers-and-hosts
+  (let [out     (output-dir)
+        base    (fake-browser {})
+        prepare (:prepare (:service base))
+        service (assoc (:service base)
+                       :prepare
+                       (fn [page request bounds]
+                         (let [response (prepare page request bounds)]
+                           (fs/renameSync (path/join out "run.log")
+                                          (path/join out "run.log.saved"))
+                           (fs/mkdirSync (path/join out "run.log"))
+                           response)))
+        fake    (assoc base :service service)
+        result  (await (execute fake [(collected :case/a :fresh)]
+                                (options fake out)))
+        state   @(:state fake)
+        host    (:host state)]
+    (t/is (some? (:error result)))
+    (t/is (= 1 (:exit-code result)))
+    (t/is (empty? (attempts-for (:run result) :case/a)))
+    (t/is (= 1 (:close-pages state)))
+    (t/is (= 1 (:closes state)))
+    (t/is (false? (.-listening ^js (:server host))))))
+
+(t/deftest ^:async interruption-during-shutdown-is-persisted
+  (doseq [[signal code] [["SIGINT" 130] ["SIGTERM" 143]]]
+    (let [controller (js/AbortController.)
+          base       (fake-browser {})
+          close      (:close (:service base))
+          service    (assoc (:service base)
+                            :close
+                            (fn [owner]
+                              (when-not (.-aborted (.-signal controller))
+                                (.abort controller
+                                        (ex-info "run interrupted"
+                                                 {:type ::interrupted :signal signal})))
+                              (close owner)))
+          fake       (assoc base :service service)
+          result     (await (execute fake [(collected :case/a :fresh)]
+                                     (options fake (output-dir))
+                                     controller))
+          run        (:run result)]
+      (t/is (= :interrupted (get-in run [:termination :reason])))
+      (t/is (= signal (get-in run [:termination :signal])))
+      (t/is (= code (:exit-code result)))
+      (t/is (= run (saved-record result)))
+      (t/is (= 1 (:closes @(:state fake)))))))
+
+(t/deftest ^:async screenshots-record-files-and-failures-as-diagnostics
+  (let [fake   (fake-browser {})
+        opts   (assoc (options fake (output-dir)) :screenshots true)
+        result (await (execute fake [(collected :case/a :fresh)] opts))
+        run    (:run result)
+        shot   (get-in run [:metadata :diagnostics :screenshots "case/a"])]
+    (t/is (= :ok (:status shot)))
+    (t/is (= "case-a.png" (:file shot)))
+    (t/is (fs/existsSync (path/join (get-in result [:home :dir]) "case-a.png")))
+    (t/is (= :completed (get-in run [:termination :reason])))
+    (t/is (zero? (:exit-code result))))
+  (let [fake   (fake-browser {:screenshot-failure (constantly true)})
+        opts   (assoc (options fake (output-dir)) :screenshots true)
+        result (await (execute fake [(collected :case/a :fresh)] opts))
+        run    (:run result)
+        shot   (get-in run [:metadata :diagnostics :screenshots "case/a"])]
+    (t/is (= :failed (:status shot)))
+    (t/is (= "case-a.png" (:file shot)))
+    (t/is (= :completed (get-in run [:termination :reason])))
+    (t/is (zero? (:exit-code result)))))
+
+(t/deftest ^:async metadata-records-build-and-controller-provenance-and-budgets
+  (let [fake   (fake-browser {})
+        opts   (assoc (options fake (output-dir))
+                      :timeout-ms 1234 :build-timeout-ms 5678 :startup-timeout-ms 9012)
+        result (await (execute fake [(collected :case/a :fresh)] opts))
+        run    (:run result)
+        diag   (get-in run [:metadata :diagnostics])]
+    (t/is (= {:attempt-ms 1234} (get-in run [:plan :timeout-policy])))
+    (t/is (= {:sha "build-revision" :dirty false} (get-in run [:metadata :git])))
+    (t/is (= {:mode :release :default-features? true :features []}
+             (get-in run [:metadata :build :functional])))
+    (t/is (= "test-build" (get-in run [:metadata :build :provenance :build-id])))
+    (t/is (true? (get-in run [:metadata :scored?])))
+    (t/is (= "FakeBrowser/1" (get-in run [:metadata :environment :browser])))
+    (t/is (= 1234 (:timeout-ms diag)))
+    (t/is (= 5678 (:build-timeout-ms diag)))
+    (t/is (= 9012 (:startup-timeout-ms diag)))
+    (t/is (= 42 (:seed diag)))
+    (t/is (= 0 (:warmups diag)))
+    (t/is (= 1 (:repetitions diag)))
+    (t/is (false? (:screenshots? diag)))
+    (t/is (map? (:runner-git diag)))
+    (t/is (= :unverified (:runtime-helper-identity diag)))))
+
+(t/deftest ^:async remove-deletes-the-used-build-after-the-record-is-saved
+  (let [root   @temporary-root
+        id     "removable-build"
+        dir    (path/join (artifacts/managed-root root) "builds" id)
+        _      (fs/mkdirSync dir #js {:recursive true})
+        _      (fs/writeFileSync (path/join dir "index.html") "<!doctype html>")
+        fake   (fake-browser {})
+        out    (path/join @temporary-root "remove-run")
+        result (await (run/execute root (selected-build dir id)
+                                   [(collected :case/a :fresh)]
+                                   (assoc (options fake out) :remove? true)
+                                   (.-signal (js/AbortController.)) nil nil))]
+    (t/is (fs/existsSync (get-in result [:home :record-path])))
+    (t/is (not (fs/existsSync dir)))
+    (t/is (= :completed (get-in result [:run :termination :reason])))
+    (t/is (zero? (:exit-code result)))))

frontend/test/performance_tests/render_wasm/runner.cljs

@@ -15,16 +15,18 @@
    [performance-tests.render-wasm.browser-test]
    [performance-tests.render-wasm.build-test]
    [performance-tests.render-wasm.builder-test]
+   [performance-tests.render-wasm.compare-test]
    [performance-tests.render-wasm.containers-test]
    [performance-tests.render-wasm.contracts-test]
    [performance-tests.render-wasm.fingerprint-test]
    [performance-tests.render-wasm.geometry-scenes-test]
    [performance-tests.render-wasm.lifecycle-test]
-   [performance-tests.render-wasm.pilot-test]
    [performance-tests.render-wasm.process-test]
    [performance-tests.render-wasm.protocol-test]
    [performance-tests.render-wasm.report-test]
+   [performance-tests.render-wasm.run-test]
    [performance-tests.render-wasm.runner-test]
+   [performance-tests.render-wasm.runs-test]
    [performance-tests.render-wasm.scene-snapshot-test]
    [performance-tests.render-wasm.server-test]))
 
@@ -45,16 +47,18 @@
    'performance-tests.render-wasm.browser-test
    'performance-tests.render-wasm.build-test
    'performance-tests.render-wasm.builder-test
+   'performance-tests.render-wasm.compare-test
    'performance-tests.render-wasm.containers-test
    'performance-tests.render-wasm.contracts-test
    'performance-tests.render-wasm.fingerprint-test
    'performance-tests.render-wasm.geometry-scenes-test
    'performance-tests.render-wasm.lifecycle-test
-   'performance-tests.render-wasm.pilot-test
    'performance-tests.render-wasm.process-test
    'performance-tests.render-wasm.protocol-test
    'performance-tests.render-wasm.report-test
+   'performance-tests.render-wasm.run-test
    'performance-tests.render-wasm.runner-test
+   'performance-tests.render-wasm.runs-test
    'performance-tests.render-wasm.scene-snapshot-test
    'performance-tests.render-wasm.server-test])
 

frontend/test/performance_tests/render_wasm/runner_test.cljs

@@ -68,14 +68,112 @@
                             (runner/parse-args args)))))
 
 (t/deftest help-needs-no-command-and-other-command-lines-must-be-valid
-  (doseq [args [["--help"] ["-h"] ["build" "--help"]]]
+  (doseq [args [["--help"] ["-h"] ["build" "--help"] ["run" "--help"]
+                ["compare" "--help"]]]
     (let [{:keys [options summary]} (runner/parse-args args)
           help                    (runner/usage summary)]
       (t/is (true? (:help options)))
       (t/is (str/includes? help "runner build"))
       (t/is (str/includes? help "runner purge"))
-      (t/is (str/includes? help "--build-timeout-ms"))))
-  (t/is (thrown-with-msg? js/Error #"expected build or purge"
+      (t/is (str/includes? help "runner run"))
+      (t/is (str/includes? help "runner compare"))
+      (t/is (str/includes? help "--build-timeout-ms"))
+      (t/is (str/includes? help "--compare-cases"))))
+  (t/is (thrown-with-msg? js/Error #"expected build, purge, run or compare"
                           (runner/parse-args [])))
-  (doseq [args [["unknown"] ["build" "purge"] ["build" "--unknown"]]]
+  (doseq [args [["unknown"] ["build" "purge"] ["build" "--unknown"]
+                ["run" "extra"] ["compare" "only-one"]]]
+    (t/is (thrown? js/Error (runner/parse-args args)))))
+
+(t/deftest run-resolves-default-and-explicit-options-before-resources
+  (let [{:keys [command options arguments]} (runner/parse-args ["run"])]
+    (t/is (= "run" command))
+    (t/is (= ["run"] arguments))
+    (t/is (= {:seed 42 :warmups 3 :repetitions 10 :timeout-ms 30000
+              :build-timeout-ms 300000 :startup-timeout-ms 30000}
+             (select-keys options [:seed :warmups :repetitions :timeout-ms
+                                   :build-timeout-ms :startup-timeout-ms])))
+    (t/is (nil? (:build-id options)))
+    (t/is (nil? (:filter options)))
+    (t/is (not (:remove options)))
+    (t/is (not (:screenshots options))))
+  (let [{:keys [options]} (runner/parse-args
+                           ["run" "--seed" "7" "--filter" "rects"
+                            "--warmups" "0" "--repetitions" "2"
+                            "--timeout-ms" "100" "--build-timeout-ms" "200"
+                            "--startup-timeout-ms" "300" "--output-dir" "out"
+                            "--remove" "--screenshots"])]
+    (t/is (= {:seed 7 :filter "rects" :warmups 0 :repetitions 2
+              :timeout-ms 100 :build-timeout-ms 200 :startup-timeout-ms 300
+              :output-dir "out" :remove true :screenshots true}
+             (select-keys options [:seed :filter :warmups :repetitions
+                                   :timeout-ms :build-timeout-ms
+                                   :startup-timeout-ms :output-dir
+                                   :remove :screenshots])))))
+
+(t/deftest retained-runs-reject-feature-selection-and-build-runs-accept-it
+  (t/is (= "retained-id" (get-in (runner/parse-args ["run" "--build" "retained-id"])
+                                 [:options :build-id])))
+  (doseq [args [["run" "--build" "retained-id" "--no-default-features"]
+                ["run" "--no-default-features" "--build" "retained-id"]
+                ["run" "--build" "retained-id" "--features" "functional"]
+                ["run" "--build" "retained-id" "--default-features"]]]
+    (t/is (thrown-with-msg? js/Error #"cannot be combined"
+                            (runner/parse-args args))))
+  (let [{:keys [options]} (runner/parse-args
+                           ["run" "--no-default-features" "--features" "functional"])]
+    (t/is (false? (:default-features options)))
+    (t/is (= ["functional"] (:features options)))))
+
+(t/deftest compare-takes-two-file-paths-and-its-own-options
+  (let [{:keys [command arguments options]}
+        (runner/parse-args ["compare" "baseline.transit" "candidate.transit"])]
+    (t/is (= "compare" command))
+    (t/is (= ["compare" "baseline.transit" "candidate.transit"] arguments))
+    (t/is (nil? (:compare-cases options)))
+    (t/is (not (:force options))))
+  (let [{:keys [options]}
+        (runner/parse-args ["compare" "a.transit" "b.transit"
+                            "--compare-cases" "rects/pan,paths/pan" "--force"])]
+    (t/is (= "rects/pan,paths/pan" (:compare-cases options)))
+    (t/is (true? (:force options)))))
+
+(t/deftest commands-reject-flags-owned-by-other-commands
+  (doseq [[args message]
+          [[["run" "--only-builds"] #"purge category flags require purge"]
+           [["run" "--compare-cases" "a,b"] #"compare options require compare"]
+           [["compare" "a" "b" "--seed" "1"] #"run options require run"]
+           [["compare" "a" "b" "--features" "functional"] #"build options require build"]
+           [["build" "--build" "retained"] #"run options require run"]
+           [["build" "--only-runs"] #"purge category flags require purge"]
+           [["purge" "--seed" "1"] #"run options require run"]
+           [["purge" "--compare-cases" "a,b"] #"compare options require compare"]]]
+    (t/is (thrown-with-msg? js/Error message (runner/parse-args args)))))
+
+(t/deftest numeric-and-path-options-reject-invalid-bounds
+  (doseq [args [["run" "--seed" "4294967296"]
+                ["run" "--seed" "-1"]
+                ["run" "--warmups" "-1"]
+                ["run" "--warmups" "1.5"]
+                ["run" "--repetitions" "0"]
+                ["run" "--timeout-ms" "0"]
+                ["run" "--timeout-ms" "2147483648"]
+                ["run" "--startup-timeout-ms" "0"]
+                ["run" "--build" ""]
+                ["run" "--output-dir" ""]]]
+    (t/is (thrown? js/Error (runner/parse-args args))))
+  (t/is (= 4294967295 (:seed (:options (runner/parse-args ["run" "--seed" "4294967295"])))))
+  (t/is (= 0 (:warmups (:options (runner/parse-args ["run" "--warmups" "0"]))))))
+
+(t/deftest option-looking-values-and-filenames-are-not-command-flags
+  (let [parsed (runner/parse-args ["compare" "--" "--seed" "candidate.transit"])]
+    (t/is (= ["compare" "--seed" "candidate.transit"] (:arguments parsed))))
+  (let [parsed (runner/parse-args ["run" "--filter" "--force"])]
+    (t/is (= "--force" (:filter (:options parsed))))))
+
+(t/deftest numeric-options-reject-blank-values
+  (doseq [args [["run" "--seed" ""]
+                ["run" "--warmups" " "]
+                ["run" "--repetitions" ""]
+                ["run" "--timeout-ms" " "]]]
     (t/is (thrown? js/Error (runner/parse-args args)))))

frontend/test/performance_tests/render_wasm/runs_test.cljs

@@ -0,0 +1,106 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns performance-tests.render-wasm.runs-test
+  (:require
+   ["fs" :as fs]
+   ["os" :as os]
+   ["path" :as path]
+   [benches.render-wasm.node.artifacts :as artifacts]
+   [benches.render-wasm.node.runs :as runs]
+   [benches.render-wasm.result :as result]
+   [cljs.test :as t :include-macros true]
+   [clojure.string :as str]))
+
+(def ^:private temporary-root (atom nil))
+
+(t/use-fixtures :each
+  {:before #(reset! temporary-root
+                    (fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-runs-")))
+   :after #(fs/rmSync @temporary-root #js {:recursive true :force true})})
+
+(defn- failure
+  [f]
+  (try
+    (f)
+    nil
+    (catch :default error error)))
+
+(defn- sample-run
+  [run-id]
+  (result/create-run
+   {:run-id run-id
+    :started-at "2026-10-26T00:00:00.000Z"
+    :plan {:cases [{:id :rects/load}]
+           :warmups 0
+           :repetitions 1
+           :timeout-policy {:attempt-ms 30000}}
+    :metadata {:git {:sha "abc" :dirty false}
+               :build {:functional {:mode :release}}
+               :environment {:node "v24.0.0"}
+               :scored? true}}))
+
+(t/deftest managed-home-lives-under-the-frontend-root
+  (let [home (runs/create-home {:root @temporary-root :run-id "run-1"})]
+    (t/is (= (path/join (artifacts/managed-root @temporary-root) "runs" "run-1")
+             (:dir home)))
+    (t/is (true? (:managed? home)))
+    (t/is (.isDirectory (fs/statSync (:dir home))))
+    (t/is (= "" (fs/readFileSync (:log-path home) "utf8")))
+    (t/is (not (fs/existsSync (:record-path home))))))
+
+(t/deftest explicit-home-must-not-exist-and-resolves-from-caller-cwd
+  (let [previous (.cwd js/process)
+        relative "explicit-run"]
+    (try
+      (.chdir js/process @temporary-root)
+      (let [home (runs/create-home {:root @temporary-root
+                                    :run-id "run-2"
+                                    :output-dir relative})]
+        (t/is (= (path/join @temporary-root relative) (:dir home)))
+        (t/is (false? (:managed? home)))
+        (t/is (.isDirectory (fs/statSync (:dir home))))
+        (t/is (not (fs/existsSync (path/join (artifacts/managed-root @temporary-root)
+                                             "runs" "run-2")))))
+      (finally
+        (.chdir js/process previous))))
+  (let [directory (path/join @temporary-root "already-there")]
+    (fs/mkdirSync directory)
+    (fs/writeFileSync (path/join directory "sentinel") "keep")
+    (let [error (failure #(runs/create-home {:root @temporary-root
+                                             :run-id "run-3"
+                                             :output-dir directory}))]
+      (t/is (= ::runs/invalid-home (:type (ex-data error))))
+      (t/is (= "output directory already exists" (ex-message error)))
+      (t/is (= "keep" (fs/readFileSync (path/join directory "sentinel") "utf8"))))))
+
+(t/deftest record-writes-are-atomic-and-invalid-records-leave-the-saved-file
+  (let [home (runs/create-home {:root @temporary-root :run-id "run-4"})
+        run  (sample-run "run-4")]
+    (runs/write-record! home run)
+    (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8"))))
+    (t/is (empty? (filter #(str/ends-with? % ".tmp")
+                          (array-seq (fs/readdirSync (:dir home))))))
+    (let [error (failure #(runs/write-record! home (assoc-in run [:metadata :scored?] "yes")))]
+      (t/is (= ::result/invalid-record (:type (ex-data error))))
+      (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8")))))))
+
+(t/deftest log-and-screenshot-paths-stay-in-the-run-home
+  (let [home (runs/create-home {:root @temporary-root :run-id "run-5"})]
+    (runs/append-log! home "first\n")
+    (runs/append-log! home "second\n")
+    (t/is (= "first\nsecond\n" (fs/readFileSync (:log-path home) "utf8")))
+    (let [shot (runs/screenshot-path home :rects/load)]
+      (t/is (= "rects-load.png" (:file shot)))
+      (t/is (= (path/join (:dir home) "rects-load.png") (:path shot)))
+      (t/is (= (path/dirname (:record-path home)) (path/dirname (:path shot)))))))
+
+(t/deftest malformed-ids-and-empty-paths-fail-at-the-home-boundary
+  (doseq [id ["../escape" "" "a/b" "a\\b"]]
+    (t/is (thrown-with-msg? js/Error #"invalid run id"
+                            (runs/create-home {:root @temporary-root :run-id id}))))
+  (t/is (thrown-with-msg? js/Error #"nonempty path"
+                          (runs/check-output-dir "  "))))