From ca6dbbe65813c6de7b62e1416ac5c1727bbeb4b3 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Thu, 29 Oct 2026 22:57:42 +0100 Subject: [PATCH] :construction: WIP 28, enable GPU support --- frontend/package.json | 1 + .../benches/render_wasm/browser/bridge.cljs | 19 ++- .../benches/render_wasm/node/browser.cljs | 56 ++++---- .../benches/render_wasm/node/build.cljs | 9 +- .../benches/render_wasm/node/commands.cljs | 6 + .../render_wasm/node/feature_comparison.cljs | 13 ++ .../benches/render_wasm/node/run.cljs | 124 ++++++++++------ .../benches/render_wasm/node/runner.cljs | 6 +- .../render_wasm/browser_test.cljs | 17 +++ .../render_wasm/cli_integration.cjs | 30 ++++ .../render_wasm/lifecycle_test.cljs | 66 ++++++--- .../render_wasm/run_test.cljs | 132 ++++++++++++++++-- .../render_wasm/runner_test.cljs | 11 ++ 13 files changed, 377 insertions(+), 113 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 38eb0983e8..f4d8118a77 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,6 +21,7 @@ "build:storybook:cljs": "clojure -M:dev:shadow-cljs compile storybook", "build:wasm": "../render-wasm/build frontend", "build:renderer-benchmarks:runner": "clojure -M:dev:renderer-bench release bench-render-wasm-commands bench-render-wasm-runner", + "render-bench": "sh -c 'if [ \"${PENPOT_GPU:-0}\" = 1 ]; then exec xvfb-run -a node target/performance/render-wasm/runner.cjs \"$@\"; else exec node target/performance/render-wasm/runner.cjs \"$@\"; fi'", "build:app:libs": "node ./scripts/build-libs.js", "build:app:main": "clojure -M:dev:shadow-cljs release main worker", "build:app:worker": "clojure -M:dev:shadow-cljs release worker", diff --git a/frontend/performance/benches/render_wasm/browser/bridge.cljs b/frontend/performance/benches/render_wasm/browser/bridge.cljs index af9d0e0e82..79684b2119 100644 --- a/frontend/performance/benches/render_wasm/browser/bridge.cljs +++ b/frontend/performance/benches/render_wasm/browser/bridge.cljs @@ -141,17 +141,20 @@ :factory-ms (- (now) t0)}))))))))) @module-promise*) -(defn- renderer-string - "Reads the unmasked renderer (SwiftShader/llvmpipe detection for the - software-renderer warning). The init helper forces the debug extension." - [context] +(defn- graphics-string + [context parameter] (try (let [ext (.getExtension ^js context "WEBGL_debug_renderer_info")] (if (some? ext) - (str (.getParameter ^js context (unchecked-get ext "UNMASKED_RENDERER_WEBGL"))) + (str (.getParameter ^js context (unchecked-get ext parameter))) "unknown")) (catch :default _ "unknown"))) +(defn- renderer-string + "Reads the unmasked WebGL renderer identity." + [context] + (graphics-string context "UNMASKED_RENDERER_WEBGL")) + (defn- software-renderer? "Recognizes common software graphics implementations from their renderer text." [s] @@ -216,11 +219,7 @@ attr (fn [k] (when (some? attrs) (unchecked-get attrs k))) - vendor (try - (if-let [ext (.getExtension ^js ctx "WEBGL_debug_renderer_info")] - (str (.getParameter ^js ctx (unchecked-get ext "UNMASKED_VENDOR_WEBGL"))) - "unknown") - (catch :default _ "unknown"))] + vendor (graphics-string ctx "UNMASKED_VENDOR_WEBGL")] {:css-width width :css-height height :drawing-buffer-width (.-drawingBufferWidth ^js ctx) diff --git a/frontend/performance/benches/render_wasm/node/browser.cljs b/frontend/performance/benches/render_wasm/node/browser.cljs index 110c2bef15..f3c9622da2 100644 --- a/frontend/performance/benches/render_wasm/node/browser.cljs +++ b/frontend/performance/benches/render_wasm/node/browser.cljs @@ -8,32 +8,40 @@ "Owns Chromium through process/run from spawn, then attaches Playwright by CDP. Shutdown stops the owned process group before disconnecting the client." (:require - ["fs" :as fs] - ["os" :as os] - ["path" :as path] - ["playwright" :as playwright] - [app.common.transit :as transit] - [benches.render-wasm.failures :as failures] - [benches.render-wasm.identity :as identity] - [benches.render-wasm.node.options :as options] - [benches.render-wasm.node.process :as process])) + ["fs" :as fs] + ["os" :as os] + ["path" :as path] + ["playwright" :as playwright] + [app.common.transit :as transit] + [benches.render-wasm.failures :as failures] + [benches.render-wasm.identity :as identity] + [benches.render-wasm.node.options :as options] + [benches.render-wasm.node.process :as process])) -(def launch-settings - "Static settings for the full Chromium executable; profile and port are owned." +(defn gpu-active? + "Returns whether the development container enables GPU access." + [] + (= "1" (.-PENPOT_GPU (.-env js/process)))) + +(defn launch-settings + "Returns the resolved Chromium settings for this environment." + [] {:headless true :transport :cdp - :args ["--headless" - "--no-sandbox" - "--disable-dev-shm-usage" - "--enable-gpu" - "--enable-unsafe-swiftshader" - "--no-first-run" - "--no-default-browser-check" - "--disable-background-networking" - "--disable-background-timer-throttling" - "--disable-renderer-backgrounding" - "--hide-scrollbars" - "--mute-audio"]}) + :args (cond-> ["--headless" + "--no-sandbox" + "--disable-dev-shm-usage" + "--enable-unsafe-swiftshader" + "--no-first-run" + "--no-default-browser-check" + "--disable-background-networking" + "--disable-background-timer-throttling" + "--disable-renderer-backgrounding" + "--hide-scrollbars" + "--mute-audio"] + (gpu-active?) (into ["--enable-gpu" + "--use-gl=angle" + "--use-angle=vulkan"]))}) (defn within "Calls work while its absolute deadline and optional AbortSignal allow it. @@ -225,7 +233,7 @@ controller (js/AbortController.) endpoint (endpoint-waiter) running (process/run {:command command - :args (into (:args launch-settings) + :args (into (:args (launch-settings)) ["--remote-debugging-port=0" "--remote-debugging-address=127.0.0.1" (str "--user-data-dir=" user-data-dir) diff --git a/frontend/performance/benches/render_wasm/node/build.cljs b/frontend/performance/benches/render_wasm/node/build.cljs index 56a7640a02..2f31be71d6 100644 --- a/frontend/performance/benches/render_wasm/node/build.cljs +++ b/frontend/performance/benches/render_wasm/node/build.cljs @@ -9,8 +9,9 @@ The caller owns exclusive preparation for this checkout. The renderer's lock protects Cargo and copying; CLJS compilation follows after it releases. - Preparation uses installed dependencies and offline Cargo. Every attempt - retains its manifest and compiler log, including failed attempts." + Preparation uses installed dependencies. Cargo may fetch missing crates and + the custom Skia binaries on a cold build. Every attempt retains its manifest + and compiler log, including failed attempts." (:require ["fs" :as fs] ["path" :as path] @@ -89,7 +90,7 @@ (log/info "Renderer compiling") (let [build @build-ref facts-path (path/join (:renderer-dir build) "cargo-build.json") - args (cond-> ["frontend" "--offline"] + args (cond-> ["frontend"] (not (:default-features? selection)) (conj "--no-default-features") (seq (:features selection)) (into ["--features" (str/join "," (:features selection))]))] (await (process/run (merge bounds @@ -97,7 +98,7 @@ :args args :cwd (path/resolve root ".." "render-wasm") :capture-output? false :on-line (fn [line _] (log/debug line)) :env {"PENPOT_WASM_PREPARED" "1" "NODE_ENV" "production" - "BUILD_MODE" "release" "CARGO_NET_OFFLINE" "true" + "BUILD_MODE" "release" "VERSION_TAG" (:id build) "FRONTEND_DEST" (:renderer-dir build) "FRONTEND_SHARED_DEST" (path/join root "src" "app" "render_wasm" "api" "shared.js") diff --git a/frontend/performance/benches/render_wasm/node/commands.cljs b/frontend/performance/benches/render_wasm/node/commands.cljs index 295de7be8e..704d8cffd1 100644 --- a/frontend/performance/benches/render_wasm/node/commands.cljs +++ b/frontend/performance/benches/render_wasm/node/commands.cljs @@ -87,6 +87,12 @@ (let [outcome (try (let [{:keys [command options]} (transit/decode-str request) + _ (when (and (:require-gpu options) + (or (= command "run") + (and (= command "ab") (not (:dry-run options)))) + (not= "1" (.-PENPOT_GPU (.-env js/process)))) + (throw (ex-info "--require-gpu needs PENPOT_GPU=1. Start the container with ./manage.sh run-devenv --gpu and use pnpm run render-bench." + {:type ::gpu-required}))) root (artifacts/frontend-root)] (log/configure {:level (options/console-level options) :color (options/color-mode options)}) diff --git a/frontend/performance/benches/render_wasm/node/feature_comparison.cljs b/frontend/performance/benches/render_wasm/node/feature_comparison.cljs index 4bcd35b6b7..d6a0c76b79 100644 --- a/frontend/performance/benches/render_wasm/node/feature_comparison.cljs +++ b/frontend/performance/benches/render_wasm/node/feature_comparison.cljs @@ -439,6 +439,8 @@ exit-code (:exit-code run-result) termination (get-in run-record [:termination :reason]) attempts (:attempts run-record) + gpu-failure (some #(when (= :gpu (:stage %)) %) + (get-in run-record [:metadata :diagnostics :failures])) evidence-error (or (:storage-error run-result) (try @@ -460,6 +462,9 @@ (some? evidence-error) (failure-evidence :run-evidence evidence-error) + (some? gpu-failure) + gpu-failure + (map? (get-in run-record [:termination :failure])) (get-in run-record [:termination :failure]) @@ -514,6 +519,14 @@ :reason (str "not attempted after run evidence failure in configuration " slug) :error run-error} + (some? gpu-failure) + {:stop? true + :reason (str "not attempted after GPU failure in configuration " slug) + :error (ex-info (:message gpu-failure) + {:type ::gpu-required + :log-path (:log-path run-home) + :configuration slug})} + :else (if (empty? attempts) (let [cause (ex-info "configuration run made no attempts" diff --git a/frontend/performance/benches/render_wasm/node/run.cljs b/frontend/performance/benches/render_wasm/node/run.cljs index f6a4dcde9e..8f1c462c96 100644 --- a/frontend/performance/benches/render_wasm/node/run.cljs +++ b/frontend/performance/benches/render_wasm/node/run.cljs @@ -31,7 +31,8 @@ [benches.render-wasm.node.server :as server] [benches.render-wasm.report.format :as format] [benches.render-wasm.report.summarize :as summary] - [benches.render-wasm.result :as result])) + [benches.render-wasm.result :as result] + [clojure.string :as str])) (defn- now [] @@ -252,6 +253,31 @@ (write-log state log/debug (str "Browser " (:version owner))) owner))) +(defn- usable-graphics-identity? + [value] + (and (string? value) + (not (contains? #{"" "unknown" "unavailable" "null" "undefined"} + (str/lower-case (str/trim value)))))) + +(defn- check-gpu + [state preparation] + (let [graphics (:effective-graphics preparation) + accepted (and (true? (:webgl2 graphics)) + (false? (:software graphics)) + (usable-graphics-identity? (:vendor graphics)) + (usable-graphics-identity? (:renderer graphics)))] + (if (and (:require-gpu? (:opts state)) (not accepted)) + (do + (record-failure state (:case-id preparation) :gpu + {:status "failed" :phase "gpu" + :message (str "GPU check failed; renderer: " (pr-str (:renderer graphics)) + ". Hardware WebGL2 with a usable vendor and renderer is required." + " Start the container with ./manage.sh run-devenv --gpu and use pnpm run render-bench.") + :partial {:graphics graphics}}) + (reset! (:abort-run? state) true) + :abort) + :ok))) + (defn- ^:async open-page [state case] (if (interrupted? state) @@ -304,28 +330,30 @@ (let [preparation (:preparation response) prep-id (:preparation-id preparation)] (record-preparation state preparation) - (let [attempted (try - {:evidence (await (run-attempt state page prep-id deadline))} - (catch :default cause {:error cause}))] - (if-let [cause (:error attempted)] - (let [evidence (cause-evidence cause :attempt)] - (record-attempt state - (measurement/attempt case-id prep-id warmup? evidence)) - (when final? - (record-screenshot-skip state case-id :no-completed-measured-attempt)) - (if (interrupted? state) :interrupted :failed)) - (let [attempt (measurement/attempt case-id prep-id warmup? - (:evidence attempted))] - (record-attempt state attempt) - (if (= :completed (:outcome attempt)) - (do - (when final? (await (screenshot-case state case page))) - :ok) - (do - (when final? - (record-screenshot-skip state case-id - :no-completed-measured-attempt)) - :failed)))))) + (if (= :abort (check-gpu state preparation)) + :abort + (let [attempted (try + {:evidence (await (run-attempt state page prep-id deadline))} + (catch :default cause {:error cause}))] + (if-let [cause (:error attempted)] + (let [evidence (cause-evidence cause :attempt)] + (record-attempt state + (measurement/attempt case-id prep-id warmup? evidence)) + (when final? + (record-screenshot-skip state case-id :no-completed-measured-attempt)) + (if (interrupted? state) :interrupted :failed)) + (let [attempt (measurement/attempt case-id prep-id warmup? + (:evidence attempted))] + (record-attempt state attempt) + (if (= :completed (:outcome attempt)) + (do + (when final? (await (screenshot-case state case page))) + :ok) + (do + (when final? + (record-screenshot-skip state case-id + :no-completed-measured-attempt)) + :failed))))))) (let [evidence (failure-evidence response :preparation)] (record-attempt state (measurement/attempt case-id nil warmup? evidence)) (when final? (record-screenshot-skip state case-id :no-completed-measured-attempt)) @@ -388,6 +416,11 @@ (ensure-screenshot-status state case :case-abandoned) :interrupted) + (= :abort outcome) + (do + (ensure-screenshot-status state case :case-abandoned) + :abort) + (not closed?) (do (ensure-screenshot-status state case :case-abandoned) @@ -433,24 +466,26 @@ total (count slots)] (record-preparation state preparation) (let [outcome - (loop [index 0] - (if (>= index total) - :done - (let [warmup? (nth slots index) - result (await (run-reuse-slot state case page prep-id warmup?))] - (cond - (= :ok result) - (if (= index (dec total)) - (do (await (screenshot-case state case page)) :done) - (recur (inc index))) + (if (= :abort (check-gpu state preparation)) + :abort + (loop [index 0] + (if (>= index total) + :done + (let [warmup? (nth slots index) + result (await (run-reuse-slot state case page prep-id warmup?))] + (cond + (= :ok result) + (if (= index (dec total)) + (do (await (screenshot-case state case page)) :done) + (recur (inc index))) - (= index (dec total)) - (do - (record-screenshot-skip state (:id case) - :no-completed-measured-attempt) - result) + (= index (dec total)) + (do + (record-screenshot-skip state (:id case) + :no-completed-measured-attempt) + result) - :else result)))) + :else result))))) closed? (await (close-current-page state))] (ensure-screenshot-status state case :case-abandoned) (cond @@ -458,6 +493,7 @@ (= outcome :interrupted) :interrupted :else outcome)))))))) + (defn- ^:async launch-initial [state] (let [launched (try @@ -625,7 +661,7 @@ :arch (os/arch) :cpu {:models (vec (distinct (map #(.-model %) cpus))) :logical-count (count cpus)} - :launch browser/launch-settings})) + :launch (browser/launch-settings)})) (defn- metadata [root selected-build opts] @@ -645,6 +681,8 @@ :build-timeout-ms (:build-timeout-ms opts) :startup-timeout-ms (:startup-timeout-ms opts) :screenshots? (boolean (:screenshots? opts)) + :require-gpu? (boolean (:require-gpu? opts)) + :penpot-gpu (or (.-PENPOT_GPU (.-env js/process)) nil) :runner-git {:sha nil :dirty nil} :runtime-helper-identity :unverified :screenshots {} @@ -656,8 +694,10 @@ (assoc :screenshots? (boolean (or (:screenshots? options) (:screenshots options))) :remove? (boolean (or (:remove? options) - (:remove options)))) - (dissoc :screenshots :remove))) + (:remove options))) + :require-gpu? (boolean (or (:require-gpu? options) + (:require-gpu options)))) + (dissoc :screenshots :remove :require-gpu))) (defn ^:async execute "Executes a prepared selection in an allocated home and owns host and browser. diff --git a/frontend/performance/benches/render_wasm/node/runner.cljs b/frontend/performance/benches/render_wasm/node/runner.cljs index 489ed539d6..c81f9c3e1b 100644 --- a/frontend/performance/benches/render_wasm/node/runner.cljs +++ b/frontend/performance/benches/render_wasm/node/runner.cljs @@ -79,6 +79,7 @@ :validate nonempty] [nil "--remove" "Remove the used build after shutdown"] [nil "--screenshots" "Capture one screenshot per case"] + [nil "--require-gpu" "Fail when GPU rendering is unavailable"] [nil "--only-builds" "Purge build attempts"] [nil "--only-runs" "Purge managed runs"] [nil "--only-comparisons" "Purge managed feature comparisons"] @@ -104,6 +105,9 @@ "-v/--verbose includes raw build output. --[no-]color overrides terminal\n" "color detection. File logs receive all levels without logger colors.\n" "Builds and managed runs live under target/performance/render-wasm.\n" + "run and ab accept --require-gpu. Start the container with\n" + "./manage.sh run-devenv --gpu and use pnpm run render-bench -- run ...\n" + "to require hardware WebGL2 in preparation metadata. ab --dry-run skips it.\n" "compare reads saved schema:run records and starts no build, host or browser.\n" "Build and scene permissions combine; operation and runtime checks stay strict.\n" "All-differences permits every compatibility difference. Reports retain them.\n\n" @@ -165,7 +169,7 @@ (def ^:private run-flags #{"--build" "--filter" "--seed" "--warmups" "--repetitions" "--timeout-ms" - "--startup-timeout-ms" "--output-dir" "--remove" "--screenshots"}) + "--startup-timeout-ms" "--output-dir" "--remove" "--screenshots" "--require-gpu"}) (def ^:private purge-flags #{"--only-builds" "--only-runs" "--only-comparisons"}) diff --git a/frontend/test/performance_tests/render_wasm/browser_test.cljs b/frontend/test/performance_tests/render_wasm/browser_test.cljs index 611c6bd6a7..47969d3282 100644 --- a/frontend/test/performance_tests/render_wasm/browser_test.cljs +++ b/frontend/test/performance_tests/render_wasm/browser_test.cljs @@ -26,6 +26,23 @@ {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "penpot-browser-test-"))) :after #(fs/rmSync @temporary-root #js {:recursive true :force true})}) +(t/deftest gpu-container-selects-the-vulkan-launch-backend + (let [env (.-env js/process) + previous (.-PENPOT_GPU env)] + (try + (doseq [value [nil "0" "true" "1"]] + (if (nil? value) + (js-delete env "PENPOT_GPU") + (set! (.-PENPOT_GPU env) value)) + (let [args (set (:args (browser/launch-settings)))] + (t/is (= (= value "1") (contains? args "--use-angle=vulkan"))) + (t/is (= (= value "1") (contains? args "--use-gl=angle"))) + (t/is (contains? args "--enable-unsafe-swiftshader")))) + (finally + (if (undefined? previous) + (js-delete env "PENPOT_GPU") + (set! (.-PENPOT_GPU env) previous)))))) + (defn- ^:async open-host [actual-id ready?] (let [identity-text (.stringify js/JSON (transit/encode-str {:build-id actual-id}))] diff --git a/frontend/test/performance_tests/render_wasm/cli_integration.cjs b/frontend/test/performance_tests/render_wasm/cli_integration.cjs index 2554880581..a341f8c15d 100644 --- a/frontend/test/performance_tests/render_wasm/cli_integration.cjs +++ b/frontend/test/performance_tests/render_wasm/cli_integration.cjs @@ -812,6 +812,7 @@ test( testOptions, async (t) => { const fixture = createFrontend(t); + fixture.env.PENPOT_GPU = "0"; installCargoMetadata(fixture, { default: [], a: [], b: ["a"] }); const retained = tree(fixture.managed); @@ -822,6 +823,7 @@ test( "--filter", "rects/load", "--dry-run", + "--require-gpu", ]); assert.equal(result.exitCode, 0); @@ -1110,6 +1112,34 @@ test( }, ); +test( + "required GPU access rejects run and ab before allocating output or checking builds", + testOptions, + async (t) => { + for (const command of [ + ["run"], + ["ab", "--features", "a", "--filter", "rects/load"], + ]) { + const fixture = createFrontend(t); + fixture.env.PENPOT_GPU = ""; + const original = tree(fixture.managed); + const result = await runCli(fixture, [ + ...command, + "--require-gpu", + "--output-dir", + path.join(fixture.managed, "gpu-run"), + ]); + + assert.equal(result.exitCode, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /PENPOT_GPU=1/); + assert.match(result.stderr, /\.\/manage\.sh run-devenv --gpu/); + assert.match(result.stderr, /pnpm run render-bench/); + assert.deepEqual(tree(fixture.managed), original); + } + }, +); + test( "run validates an explicit output dir before preparing a build", testOptions, diff --git a/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs index b74212f1d9..fb5ca7cef0 100644 --- a/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs +++ b/frontend/test/performance_tests/render_wasm/lifecycle_test.cljs @@ -220,26 +220,33 @@ recorded in the shared call log. The context reports a drawing-buffer size distinct from any canvas size, so tests prove `read-graphics` reads it from the context rather than the canvas attributes." - [id] - (let [lose-ext #js {:loseContext (fn [] (record! ["lose" id]) nil)} - context #js {:getExtension (fn [name] - (when (= name "WEBGL_lose_context") - lose-ext)) - :getContextAttributes (fn [] - #js {"antialias" false - "depth" true - "stencil" false - "alpha" false - "preserveDrawingBuffer" false}) - :drawingBufferWidth 1234 - :drawingBufferHeight 5678}] - #js {:id id - :width 0 - :height 0 - :style #js {} - :getContext (fn [_ _] context) - :addEventListener (fn [_ _] nil) - :remove (fn [] (record! ["remove" id]) nil)})) + ([id] + (fake-canvas id nil)) + ([id {:keys [vendor renderer]}] + (let [debug-ext (when (and vendor renderer) + #js {:UNMASKED_VENDOR_WEBGL 1 :UNMASKED_RENDERER_WEBGL 2}) + lose-ext #js {:loseContext (fn [] (record! ["lose" id]) nil)} + context #js {:getExtension (fn [name] + (case name + "WEBGL_lose_context" lose-ext + "WEBGL_debug_renderer_info" debug-ext + nil)) + :getParameter (fn [parameter] (case parameter 1 vendor 2 renderer nil)) + :getContextAttributes (fn [] + #js {"antialias" false + "depth" true + "stencil" false + "alpha" false + "preserveDrawingBuffer" false}) + :drawingBufferWidth 1234 + :drawingBufferHeight 5678}] + #js {:id id + :width 0 + :height 0 + :style #js {} + :getContext (fn [_ _] context) + :addEventListener (fn [_ _] nil) + :remove (fn [] (record! ["remove" id]) nil)}))) (defn- install-fake-dom! "Installs fake DOM, import, clock and scenario globals and returns a @@ -423,6 +430,25 @@ (cleanup) (throw cause))))) +(t/deftest ^:async preparations-record-the-context-identity-and-software-status + (doseq [[identity software] [[{:vendor "NVIDIA" :renderer "ANGLE (NVIDIA, Vulkan)"} false] + [{:vendor "Google" :renderer "ANGLE (SwiftShader)"} true] + [{:vendor "Mesa" :renderer "llvmpipe (LLVM)"} true] + [nil false]]] + (await + (with-entry-env + (^:async fn [cleanup] + (try + (set! (.-createElement js/document) (fn [_] (fake-canvas 1 identity))) + (let [response (transit/decode-str (await (browser/prepare-scene-request (scene-args 7 {})))) + graphics (get-in response [:preparation :effective-graphics])] + (t/is (= "ok" (:status response))) + (t/is (= (or (:vendor identity) "unknown") (:vendor graphics))) + (t/is (= (or (:renderer identity) "unknown") (:renderer graphics))) + (t/is (= software (:software graphics)))) + (finally + (cleanup)))))))) + (t/deftest successful-load-reaches-full (t/async done (with-entry-env diff --git a/frontend/test/performance_tests/render_wasm/run_test.cljs b/frontend/test/performance_tests/render_wasm/run_test.cljs index 27c118f77f..f7909407e4 100644 --- a/frontend/test/performance_tests/render_wasm/run_test.cljs +++ b/frontend/test/performance_tests/render_wasm/run_test.cljs @@ -79,10 +79,14 @@ (throw (ex-info "browser service requires the page object" {:phase :browser-page-startup})))) +(def ^:private hardware-graphics + {:webgl2 true :vendor "NVIDIA" :renderer "ANGLE (NVIDIA, Vulkan, NVIDIA)" :software false}) + (defn- fake-browser [{:keys [launch-failure new-page-failure close-page-failure prepare-failure run-failure screenshot-failure - interruption controller]}] + interruption controller effective-graphics] + :or {effective-graphics (constantly {})}}] (let [state (atom {:launches 0 :pages 0 :close-pages 0 :closes 0 :host nil :runs {} :preparations {}})] {:state state @@ -135,7 +139,7 @@ :case-id case-id :fingerprint {:algorithm :sha-256 :digest (apply str (repeat 64 "a"))} - :effective-graphics {} + :effective-graphics (effective-graphics case-id (:id page)) :diagnostics {}}] (swap! state assoc-in [:preparations preparation-id] case-id) {:status "ok" :preparation preparation})))) @@ -384,6 +388,68 @@ (t/is (= :browser-launch (:stage (last (get-in run [:metadata :diagnostics :failures]))))) (t/is (= 1 (:exit-code result))))) +(t/deftest ^:async hardware-preparations-complete-fresh-and-reuse-cases + (let [fake (fake-browser {:effective-graphics (constantly hardware-graphics)}) + opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1 :repetitions 2) + result (await (execute fake [(collected :case/fresh :fresh) + (collected :case/reuse :reuse)] opts)) + run (saved-record result)] + (t/is (= 0 (:exit-code result))) + (t/is (= :completed (get-in run [:termination :reason]))) + (t/is (= 6 (count (:attempts run)))) + (t/is (every? #(= hardware-graphics (:effective-graphics %)) (:preparations run))) + (t/is (true? (get-in run [:metadata :diagnostics :require-gpu?]))))) + +(t/deftest ^:async required-gpu-failures-preserve-evidence-and-all-attempt-slots + (doseq [context [:fresh :reuse] + graphics [(assoc hardware-graphics :renderer "ANGLE (SwiftShader)" :software true) + (assoc hardware-graphics :webgl2 false) + (assoc hardware-graphics :renderer nil) + (assoc hardware-graphics :renderer " UnKnOwN ") + (assoc hardware-graphics :vendor "") + {}]] + (let [fake (fake-browser {:effective-graphics (constantly graphics)}) + opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1) + result (await (execute fake [(collected :case/a context) + (collected :case/b :reuse)] opts)) + run (saved-record result) + failure (get-in run [:termination :failure]) + state @(:state fake)] + (t/is (= 1 (:exit-code result))) + (t/is (= :incomplete (get-in run [:termination :reason]))) + (t/is (= [graphics] (mapv :effective-graphics (:preparations run)))) + (t/is (= [] (:attempts run))) + (t/is (= {:case/a {:warmups 1 :repetitions 1} + :case/b {:warmups 1 :repetitions 1}} (:remaining run))) + (t/is (= :gpu (:stage failure))) + (t/is (= graphics (get-in failure [:partial :graphics]))) + (t/is (str/includes? (:message failure) (pr-str (:renderer graphics)))) + (t/is (str/includes? (:message failure) "./manage.sh run-devenv --gpu")) + (t/is (str/includes? (:message failure) "pnpm run render-bench")) + (t/is (= 1 (:close-pages state))) + (t/is (= 1 (:closes state)))))) + +(t/deftest ^:async graphics-rejection-preserves-earlier-attempts-and-stops-the-run + (doseq [context [:fresh :reuse]] + (let [software (assoc hardware-graphics :renderer "SwiftShader" :software true) + fake (fake-browser {:effective-graphics (fn [_ page-id] + (if (= page-id 1) hardware-graphics software))}) + opts (assoc (options fake (output-dir)) :require-gpu true :repetitions 2) + result (await (execute fake [(collected :case/a context) + (collected :case/b context)] opts)) + run (saved-record result) + failure (get-in run [:termination :failure])] + (t/is (= 1 (:exit-code result))) + (t/is (= :gpu (:stage failure))) + (t/is (= software (get-in failure [:partial :graphics]))) + (t/is (= (if (= context :fresh) 1 2) (count (:attempts run)))) + (t/is (every? #(= :completed (:outcome %)) (:attempts run))) + (t/is (= [hardware-graphics software] (mapv :effective-graphics (:preparations run)))) + (t/is (= {:case/a {:warmups 0 :repetitions (if (= context :fresh) 1 0)} + :case/b {:warmups 0 :repetitions 2}} (:remaining run))) + (t/is (= 2 (:close-pages @(:state fake)))) + (t/is (= 1 (:closes @(:state fake))))))) + (t/deftest ^:async sigint-and-sigterm-persist-partial-interrupted-records (doseq [[signal code] [["SIGINT" 130] ["SIGTERM" 143]]] (let [controller (js/AbortController.) @@ -573,6 +639,57 @@ (t/is (= "" (fs/readFileSync (path/join dir "index.html") "utf8"))))) +(defn- prepare-comparison-build + [{:keys [selection on-created]}] + (let [dir (build-dir) + build (-> (selected-build dir (str (random-uuid))) + (assoc :manifest-path (path/join dir "build.transit")) + (assoc-in [:manifest :functional :default-features?] false) + (assoc-in [:manifest :functional :features] (:features selection)) + (assoc-in [:manifest :provenance :feature-selection] selection))] + (on-created build) + (js/Promise.resolve build))) + +(t/deftest ^:async gpu-rejection-stops-later-matrix-configurations + (doseq [[hardware-first close-fails] [[false false] [true false] [true true]]] + (let [root @temporary-root + out (output-dir) + cases [(collected :case/a :fresh)] + software (assoc hardware-graphics :renderer "SwiftShader" :software true) + prepared {:plan (feature-plan/plan ["a"] {:dry-run true}) + :cases cases + :output-dir out + :options (assoc (dissoc (options nil out) :run-id :browser) + :require-gpu true :repetitions 2)} + response (await + (feature-comparison/execute + root prepared nil + {:services + {:prepare-build prepare-comparison-build + :collect-catalog (fn [_ _] cases) + :execute-run + (fn [root home selected cases opts signal] + (let [fake (fake-browser + {:close-page-failure (when close-fails #(= % 2)) + :effective-graphics + (fn [_ page-id] + (if (and hardware-first (= page-id 1)) + hardware-graphics + software))})] + (run/execute root home selected cases + (assoc opts :browser (:service fake)) signal)))}})) + saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8")) + run (result/decode (fs/readFileSync (get-in saved [:configurations 0 :run :record-path]) "utf8")) + failure (some #(when (= :gpu (:stage %)) %) + (get-in run [:metadata :diagnostics :failures]))] + (t/is (= 1 (:exit-code response))) + (t/is (= (if hardware-first 1 0) (count (:attempts run)))) + (t/is (= software (get-in failure [:partial :graphics]))) + (t/is (= (if close-fails :browser-close :gpu) (get-in run [:termination :failure :stage]))) + (t/is (= :failed (get-in saved [:configurations 0 :status]))) + (t/is (= :gpu (get-in saved [:configurations 0 :failure :stage]))) + (t/is (= :unattempted (get-in saved [:configurations 1 :status])))))) + (t/deftest ^:async comparison-stops-after-run-record-or-log-storage-fails (doseq [filename ["run.transit" "run.log"]] (let [executed (atom nil) @@ -587,16 +704,7 @@ (feature-comparison/execute root prepared nil {:services - {:prepare-build - (fn [{:keys [selection on-created]}] - (let [dir (build-dir) - build (-> (selected-build dir (str (random-uuid))) - (assoc :manifest-path (path/join dir "build.transit")) - (assoc-in [:manifest :functional :default-features?] false) - (assoc-in [:manifest :functional :features] (:features selection)) - (assoc-in [:manifest :provenance :feature-selection] selection))] - (on-created build) - (js/Promise.resolve build))) + {:prepare-build prepare-comparison-build :collect-catalog (fn [_ _] cases) :execute-run (^:async fn [root home selected cases opts signal] diff --git a/frontend/test/performance_tests/render_wasm/runner_test.cljs b/frontend/test/performance_tests/render_wasm/runner_test.cljs index 071daf08b2..16196bf7a8 100644 --- a/frontend/test/performance_tests/render_wasm/runner_test.cljs +++ b/frontend/test/performance_tests/render_wasm/runner_test.cljs @@ -218,6 +218,17 @@ ["run" "--timeout-ms" " "]]] (t/is (thrown? js/Error (runner/parse-args args))))) +(t/deftest gpu-policy-is-available-for-runs-and-feature-comparisons + (doseq [args [["run" "--require-gpu"] + ["ab" "--features" "a" "--filter" "rects/load" "--require-gpu"] + ["ab" "--features" "a" "--require-gpu" "--dry-run"]]] + (t/is (true? (get-in (runner/parse-args args) [:options :require-gpu])))) + (doseq [args [["build" "--require-gpu"] + ["purge" "--require-gpu"] + ["compare" "a" "b" "--require-gpu"]]] + (t/is (thrown-with-msg? js/Error #"run options require run" (runner/parse-args args)))) + (t/is (re-find #"--require-gpu" (runner/usage "")))) + (t/deftest ab-accepts-run-budgets-and-comparison-permissions (let [{:keys [command options]} (runner/parse-args ["ab" "--features" " b,a,b " "--filter" "rects/load"