From d8a24e3fc520cfef350e0327ed3101116bcaca38 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Mon, 2 Nov 2026 19:54:51 +0100 Subject: [PATCH] :recycle: CLI command execution refactor AI-assisted-by: gpt-6.1-sol, deepseek-flash --- .../benches/render_wasm/node/cli.cljs | 11 +- .../benches/render_wasm/node/cli/build.cljs | 7 + .../render_wasm/node/cli/registry.cljs | 63 +++- .../benches/render_wasm/node/cli/run.cljs | 7 + .../benches/render_wasm/node/commands.cljs | 33 +- .../benches/render_wasm/node/execution.cljs | 139 ++++++++ .../benches/render_wasm/node/process.cljs | 12 - .../render_wasm/compare_test.cljs | 23 ++ .../render_wasm/execution_test.cljs | 331 ++++++++++++++++++ .../render_wasm/feature_comparison_test.cljs | 3 +- .../render_wasm/process_test.cljs | 25 -- .../performance_tests/render_wasm/runner.cljs | 10 +- 12 files changed, 573 insertions(+), 91 deletions(-) create mode 100644 frontend/performance/benches/render_wasm/node/execution.cljs create mode 100644 frontend/test/performance_tests/render_wasm/execution_test.cljs diff --git a/frontend/performance/benches/render_wasm/node/cli.cljs b/frontend/performance/benches/render_wasm/node/cli.cljs index d791c581b9..e15191fc69 100644 --- a/frontend/performance/benches/render_wasm/node/cli.cljs +++ b/frontend/performance/benches/render_wasm/node/cli.cljs @@ -16,14 +16,11 @@ [clojure.string :as str] [clojure.tools.cli :as tools])) -(def ^:private commands - (into {} registry/commands)) - (defn- get-definition [topic] (if (nil? topic) {:usage help/usage :description help/description :options []} - (get commands topic))) + (registry/get-definition topic))) (defn- get-declarations [definition] @@ -46,7 +43,7 @@ first-arg (first argv) topic (when (and first-arg (not (str/starts-with? first-arg "-"))) first-arg) args (if topic (rest argv) argv)] - (when (and topic (not (contains? commands topic))) + (when (and topic (nil? (registry/get-definition topic))) (throw (argument-error topic (str "unknown command: " topic) nil))) (let [definition (get-definition topic)] (try @@ -68,9 +65,7 @@ :options ((:resolve definition) (dissoc options :help :quiet :verbose :color) arguments)})) resolved (request/check (request/apply-defaults partial))] - (if (and (not= "help" (:command resolved)) (:check definition)) - (update resolved :options (:check definition)) - resolved)) + (update resolved :options #(registry/check-options (:command resolved) %))) (catch :default cause (if (contains? #{::shared/invalid-args ::request/invalid-request ::feature-plan/too-many-features ::feature-plan/confirmation-required} diff --git a/frontend/performance/benches/render_wasm/node/cli/build.cljs b/frontend/performance/benches/render_wasm/node/cli/build.cljs index a31d5d25bd..b7bbf72305 100644 --- a/frontend/performance/benches/render_wasm/node/cli/build.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/build.cljs @@ -31,3 +31,10 @@ [options arguments] (shared/check-arguments arguments 0) options) + +(defn check-options + "Checks measurement feature policy and returns the resolved build options." + [resolved] + (options/feature-selection {:default-features? (:default-features resolved) + :features (:features resolved)}) + resolved) diff --git a/frontend/performance/benches/render_wasm/node/cli/registry.cljs b/frontend/performance/benches/render_wasm/node/cli/registry.cljs index bc70470991..6dd2d62a6b 100644 --- a/frontend/performance/benches/render_wasm/node/cli/registry.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/registry.cljs @@ -15,19 +15,54 @@ [benches.render-wasm.node.cli.run :as run])) (def commands - [["build" {:usage build/usage :description build/description - :options build/cli-options :resolve build/resolve-options :schema build/schema:options}] - ["run" {:usage run/usage :description run/description - :options run/cli-options :resolve run/resolve-options :schema run/schema:options - :help-options {:build {:kind :prepare}} :help-defaults run/get-help-defaults}] - ["ab" {:usage ab/usage :description ab/description - :options ab/cli-options :resolve ab/resolve-options :schema ab/schema:options - :check ab/check-options}] - ["compare" {:usage compare/usage :description compare/description - :options compare/cli-options :resolve compare/resolve-options :schema compare/schema:options}] - ["merge" {:usage merge/usage :description merge/description - :options merge/cli-options :resolve merge/resolve-options :schema merge/schema:options}] - ["purge" {:usage purge/usage :description purge/description - :options purge/cli-options :resolve purge/resolve-options :schema purge/schema:options}]]) + [["build" {:usage build/usage + :description build/description + :options build/cli-options + :resolve build/resolve-options + :schema build/schema:options + :check build/check-options}] + ["run" {:usage run/usage + :description run/description + :options run/cli-options + :resolve run/resolve-options + :schema run/schema:options + :help-options {:build {:kind :prepare}} + :help-defaults run/get-help-defaults + :check run/check-options}] + ["ab" {:usage ab/usage + :description ab/description + :options ab/cli-options + :resolve ab/resolve-options + :schema ab/schema:options + :check ab/check-options}] + ["compare" {:usage compare/usage + :description compare/description + :options compare/cli-options + :resolve compare/resolve-options + :schema compare/schema:options}] + ["merge" {:usage merge/usage + :description merge/description + :options merge/cli-options + :resolve merge/resolve-options + :schema merge/schema:options}] + ["purge" {:usage purge/usage + :description purge/description + :options purge/cli-options + :resolve purge/resolve-options + :schema purge/schema:options}]]) (def command-names (mapv first commands)) + +(def ^:private definitions (into {} commands)) + +(defn get-definition + "Returns a command definition, or nil for an unknown command." + [command] + (get definitions command)) + +(defn check-options + "Checks command policy and returns the resolved options." + [command options] + (if-let [check (:check (get-definition command))] + (check options) + options)) diff --git a/frontend/performance/benches/render_wasm/node/cli/run.cljs b/frontend/performance/benches/render_wasm/node/cli/run.cljs index c9232c8081..cc6a65a50c 100644 --- a/frontend/performance/benches/render_wasm/node/cli/run.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/run.cljs @@ -53,6 +53,13 @@ Managed output: frontend/target/performance/render-wasm/runs//run.transi {:kind :existing :id (:build-id options)} (assoc preparation :kind :prepare))))) +(defn check-options + "Checks measurement feature policy for the selected build source." + [options] + (when (= :prepare (get-in options [:build :kind])) + (build/check-options (:build options))) + options) + (defn get-help-defaults "Returns CLI defaults for run options with a preparation source." [options] diff --git a/frontend/performance/benches/render_wasm/node/commands.cljs b/frontend/performance/benches/render_wasm/node/commands.cljs index ec3cd439c3..e7700d3f2e 100644 --- a/frontend/performance/benches/render_wasm/node/commands.cljs +++ b/frontend/performance/benches/render_wasm/node/commands.cljs @@ -12,47 +12,22 @@ [benches.render-wasm.errors :as errors :refer [attempt failure?]] [benches.render-wasm.node.artifacts :as artifacts] [benches.render-wasm.node.build :as build] + [benches.render-wasm.node.execution :as execution] [benches.render-wasm.node.feature-comparison :as feature-comparison] [benches.render-wasm.node.feature-plan :as feature-plan] [benches.render-wasm.node.log :as log] [benches.render-wasm.node.options :as options] [benches.render-wasm.node.outcome :as outcome] - [benches.render-wasm.node.process :as process] [benches.render-wasm.node.run :as run])) -(defn- ^:async with-interruption - "Calls `(f signal)` with an AbortSignal that SIGINT and SIGTERM abort, and - returns its outcome. A failure becomes an outcome with exit code 1. After an - interruption the exit code is 130 or 143 and only fatal failures stay in - the errors: the others come from the aborted work." - [f] - (let [controller (js/AbortController.) - signal (.-signal controller) - interrupted (atom nil) - remove-signals (process/on-signals - (fn [signal-name] - (when-not (.-aborted signal) - (reset! interrupted signal-name) - (.abort controller - (ex-info "interrupted" - {:type ::interrupted :signal signal-name})))))] - (try - (let [result (attempt :catch-fatal (await (f signal))) - finished (if (failure? result) (outcome/failed result {}) result)] - (cond-> finished - @interrupted (-> (update :errors #(filterv errors/fatal? %)) - (assoc :exit-code (outcome/interruption-code @interrupted))))) - (finally - (remove-signals))))) - (defn- run-command [root options] - (with-interruption + (execution/with-interruption #(run/run root (select-keys options (sm/keys run/schema:run-options)) %))) (defn- build-command [root options] - (with-interruption + (execution/with-interruption (^:async fn [signal] (await (build/prepare {:root root :features {:default-features? (:default-features options) @@ -66,7 +41,7 @@ (defn- ab-command [root options] - (with-interruption + (execution/with-interruption (^:async fn [signal] (let [prepared (await (feature-plan/prepare root options signal))] (if (:dry-run options) diff --git a/frontend/performance/benches/render_wasm/node/execution.cljs b/frontend/performance/benches/render_wasm/node/execution.cljs new file mode 100644 index 0000000000..0fed79be09 --- /dev/null +++ b/frontend/performance/benches/render_wasm/node/execution.cljs @@ -0,0 +1,139 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns benches.render-wasm.node.execution + "Checked command execution, final publication and exit status." + (:require + [app.common.schema :as sm] + [benches.render-wasm.errors :as errors :refer [attempt failure?]] + [benches.render-wasm.node.cli.registry :as registry] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.outcome :as outcome] + [benches.render-wasm.node.request :as request])) + +(defn on-signals + "Installs SIGINT/SIGTERM callbacks receiving the signal name. Returns an + idempotent remover. The caller can abort its work and let output drain." + [callback] + (let [interrupt (fn [] (callback "SIGINT")) + terminate (fn [] (callback "SIGTERM"))] + (.on js/process "SIGINT" interrupt) + (.on js/process "SIGTERM" terminate) + (fn [] + (.removeListener js/process "SIGINT" interrupt) + (.removeListener js/process "SIGTERM" terminate)))) + +(defn ^:async with-interruption + "Awaits `(f signal)` and returns its outcome. SIGINT/SIGTERM abort the signal + and select status 130/143. Interrupted outcomes retain fatal errors and partial + evidence. Thrown or rejected failures become outcomes." + [f] + (let [controller (js/AbortController.) + signal (.-signal controller) + interrupted (atom nil) + remove-signals (on-signals + (fn [signal-name] + (when-not (.-aborted signal) + (reset! interrupted signal-name) + (.abort controller + (ex-info "interrupted" + {:type ::interrupted :signal signal-name})))))] + (try + (let [result (attempt :catch-fatal (await (f signal))) + finished (if (failure? result) (outcome/failed result {}) result)] + (cond-> finished + @interrupted (-> (update :errors #(filterv errors/fatal? %)) + (assoc :exit-code (outcome/interruption-code @interrupted))))) + (finally + (remove-signals))))) + +(defn- configure-console + [console local] + (log/configure (merge (select-keys local [:stdout :stderr]) console))) + +(defn- check-gpu + [command options] + (when (and (:require-gpu options) + (or (= command "run") + (and (= command "ab") (not (:dry-run options)))) + (not= "1" (.-PENPOT_GPU (.-env js/process)))) + (throw (ex-info "--require-gpu needs PENPOT_GPU=1. Start the container with ./manage.sh run-devenv --gpu and use pnpm run render-bench." + {:type ::gpu-required})))) + +(defn- log-failures + [finished] + (doseq [[index cause] (map-indexed vector (:errors finished))] + (let [data (ex-data cause) + validation (some-> (::sm/explain data) sm/simplify)] + (log/error {:message (errors/message-of cause) + :fields (into (filterv (comp some? second) + [[:type (:type data)] [:cause-type (:cause-type data)] + [:command (:command data)] + [:phase (:phase data)] [:build (:build-id data)] + [:reason (:reason data)] [:manifest-error (:manifest-error data)] + [:baseline (:baseline data)] [:candidate (:candidate data)] + [:features (:features data)] + [:count (:count data)] [:max-features (:max-features data)] + [:validation validation]]) + (map #(vector :difference (:field %)) (:mismatches data))) + :tail-log (:log-path data) + :tail? (zero? index)})))) + +(defn- report-fallback + [cause local] + (attempt :catch-fatal + (let [message (errors/message-of cause) + stream (or (:stderr local) (.-stderr js/process))] + (.write ^js stream + (str "Could not report benchmark failure: " + (subs message 0 (min 1024 (count message))) "\n"))))) + +(defn- finish + [result local] + (binding [log/*log-path* (:log-path result)] + (let [finished (if-some [report (:report result)] + (outcome/publish result #(log/result report)) + result) + reported (attempt :catch-fatal (log-failures finished))] + (if (failure? reported) + (do + (report-fallback reported local) + (:exit-code (outcome/add-error finished reported))) + (:exit-code finished))))) + +(defn fail + "Prints a failure and returns its numeric status. Optional resolved console + settings and local streams configure its output." + ([cause] + (fail cause {} {})) + ([cause console local] + (configure-console console local) + (finish (outcome/failed (errors/as-failure cause) {}) local))) + +(defn ^:async execute + "Checks a complete received request, awaits its supported handler and returns + a numeric exit status after final publication. Handlers receive local signal, + root access and services plus resolved options. Local settings may supply + streams, a root thunk and services." + ([received handlers] + (execute received handlers {})) + ([received handlers local] + (configure-console {} local) + (let [result (attempt :catch-fatal + (let [{:keys [command console options]} (request/check received)] + (configure-console console local) + (let [handler (or (get handlers command) + (throw (ex-info "command is unavailable at this execution entry" + {:type ::unsupported-command :command command}))) + options (registry/check-options command options)] + (check-gpu command options) + (await (with-interruption + (fn [signal] + (handler (assoc (select-keys local [:get-root :services]) + :signal signal) + options))))))) + result (if (failure? result) (outcome/failed result {}) result)] + (finish result local)))) diff --git a/frontend/performance/benches/render_wasm/node/process.cljs b/frontend/performance/benches/render_wasm/node/process.cljs index 2976cdb635..57f5798a0a 100644 --- a/frontend/performance/benches/render_wasm/node/process.cljs +++ b/frontend/performance/benches/render_wasm/node/process.cljs @@ -286,15 +286,3 @@ (.destroy ^js stream)) (when-let [stream (.-stderr child)] (.destroy ^js stream))))))) - -(defn on-signals - "Installs SIGINT/SIGTERM callbacks receiving the signal name. Returns an - idempotent remover. The caller can abort its work and let output drain." - [callback] - (let [interrupt (fn [] (callback "SIGINT")) - terminate (fn [] (callback "SIGTERM"))] - (.on js/process "SIGINT" interrupt) - (.on js/process "SIGTERM" terminate) - (fn [] - (.removeListener js/process "SIGINT" interrupt) - (.removeListener js/process "SIGTERM" terminate)))) diff --git a/frontend/test/performance_tests/render_wasm/compare_test.cljs b/frontend/test/performance_tests/render_wasm/compare_test.cljs index 9841088559..4fbdd734a1 100644 --- a/frontend/test/performance_tests/render_wasm/compare_test.cljs +++ b/frontend/test/performance_tests/render_wasm/compare_test.cljs @@ -7,10 +7,13 @@ (ns performance-tests.render-wasm.compare-test (:require ["fs" :as fs] + ["node:stream" :as stream] ["os" :as os] ["path" :as path] [app.common.transit :as transit] [benches.render-wasm.node.compare :as compare] + [benches.render-wasm.node.execution :as execution] + [benches.render-wasm.node.log :as log] [benches.render-wasm.report.compare :as report-compare] [benches.render-wasm.result :as result] [cljs.test :as t :include-macros true] @@ -191,6 +194,26 @@ (into {} (map (juxt :field :rule)) (get-in comparison [:pairs 0 :mismatches]))))))) +(t/deftest comparison-failure-identifies-incompatible-fields + (let [baseline (write-run (path/join @temporary-root "baseline.transit") + (make-run "baseline" :case/load 10)) + candidate (write-run (path/join @temporary-root "candidate.transit") + (make-run "candidate" :case/load 12 + :environment {:node "v25" :browser "Fake"})) + cause (failure #(compare/execute baseline candidate {})) + text (atom "") + stderr (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))})] + (try + (t/is (some? cause)) + (t/is (= 1 (execution/fail cause {:color :never} {:stderr stderr}))) + (t/is (str/includes? @text "incompatible saved runs")) + (t/is (str/includes? @text ":environment")) + (finally + (log/configure {}))))) + (t/deftest scoped-permissions-keep-operation-and-runtime-checks-strict (let [baseline (write-run (path/join @temporary-root "baseline.transit") (make-run "baseline" :case/load 10))] diff --git a/frontend/test/performance_tests/render_wasm/execution_test.cljs b/frontend/test/performance_tests/render_wasm/execution_test.cljs new file mode 100644 index 0000000000..ccb005340a --- /dev/null +++ b/frontend/test/performance_tests/render_wasm/execution_test.cljs @@ -0,0 +1,331 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns performance-tests.render-wasm.execution-test + (:require + ["fs" :as fs] + ["node:module" :refer [createRequire]] + ["node:stream" :as stream] + ["node:timers/promises" :as timers] + ["os" :as os] + ["path" :as path] + [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] + [benches.render-wasm.node.cli :as cli] + [benches.render-wasm.node.execution :as execution] + [benches.render-wasm.node.log :as log] + [benches.render-wasm.node.request :as request] + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def temporary-root (atom nil)) + +(defn- within + [promise] + (let [timer (volatile! nil)] + (-> (js/Promise.race + #js [promise + (js/Promise. + (fn [_ reject] + (vreset! timer (js/setTimeout #(reject (js/Error. "Signal test stalled")) 5000))))]) + (.finally #(js/clearTimeout @timer))))) + +(t/deftest ^:async real-signals-reach-the-callback-and-remover-restores-listeners + (let [before-int (.listenerCount js/process "SIGINT") + before-term (.listenerCount js/process "SIGTERM") + received (atom []) + next-signal (volatile! nil) + remove (signals/on-signals + (fn [signal] + (swap! received conj signal) + (@next-signal signal))) + receive (fn [signal] + (within + (js/Promise. + (fn [resolve _] + (vreset! next-signal resolve) + (.kill js/process (.-pid js/process) signal)))))] + (try + (t/is (= "SIGINT" (await (receive "SIGINT")))) + (t/is (= "SIGTERM" (await (receive "SIGTERM")))) + (t/is (= ["SIGINT" "SIGTERM"] @received)) + (finally + (remove))) + (remove) + (t/is (= before-int (.listenerCount js/process "SIGINT"))) + (t/is (= before-term (.listenerCount js/process "SIGTERM"))))) + +(t/use-fixtures :each + {:before #(reset! temporary-root (fs/mkdtempSync (path/join (os/tmpdir) "benchmark-execution-"))) + :after (fn [] + (log/configure {}) + (fs/rmSync @temporary-root #js {:recursive true :force true}))}) + +(defn- capture + [] + (let [output (atom "") + errors (atom "") + writer (fn [text] + (stream/Writable. + #js {:write (fn [chunk _encoding done] + (swap! text str (.toString chunk)) + (done))}))] + {:stdout (writer output) :stderr (writer errors) :output output :errors errors})) + +(defn- resolved + [command options] + (request/apply-defaults {:command command :console {:color :never} :options options})) + +(defn- listeners + [] + [(.listenerCount js/process "SIGINT") (.listenerCount js/process "SIGTERM")]) + +(defn- within + [promise] + (let [timer (volatile! nil)] + (-> (js/Promise.race + #js [promise + (js/Promise. + (fn [_ reject] + (vreset! timer (js/setTimeout #(reject (js/Error. "Execution test stalled")) 5000))))]) + (.finally #(js/clearTimeout @timer))))) + +(t/deftest ^:async sync-and-async-handlers-publish-once-and-release-listeners + (doseq [async [false true]] + (let [local (capture) + before (listeners) + request (cli/parse-resolve ["compare" "--help" "--no-color"]) + handler (fn [_ {:keys [topic]}] + (log/info "Preparing help") + {:exit-code 0 :errors [] :log-path nil :report (cli/get-help topic)}) + handler (if async + (^:async fn [context options] + (await (timers/setTimeout 1)) + (handler context options)) + handler)] + (t/is (= 0 (await (execution/execute request {"help" handler} + (assoc local :get-root #(throw (js/Error. "root unavailable"))))))) + (t/is (= (str (cli/get-help "compare") "\n") @(:output local))) + (t/is (= "Preparing help\n" @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest ^:async handlers-use-local-root-and-services-with-complete-options + (let [local (capture) + file (path/join @temporary-root "input.txt")] + (fs/writeFileSync file "saved input" "utf8") + (t/is (= 0 + (await + (execution/execute + (cli/parse-resolve ["run" "--build" "ready" "--warmups" "0"]) + {"run" (fn [{:keys [get-root services]} {:keys [warmups build]}] + {:exit-code 0 :errors [] :log-path nil + :report (str (:id build) ":" warmups ":" + ((:read-text services) (path/join (get-root) "input.txt")))})} + (assoc local :get-root (constantly @temporary-root) + :services {:read-text #(fs/readFileSync % "utf8")}))))) + (t/is (= "ready:0:saved input\n" @(:output local))))) + +(t/deftest ^:async malformed-and-unsupported-requests-fail-before-output-allocation + (let [valid (resolved "run" {:build {:kind :existing :id "ready"}})] + (doseq [[received field handlers] + [[(dissoc valid :console) "console" nil] + [(update valid :options dissoc :warmups) "warmups" nil] + [(assoc (update valid :options dissoc :warmups) + :console {:level :warn :color :always}) "warmups" nil] + [(assoc-in valid [:options :extra] true) "extra" nil] + [(assoc-in valid [:console :color] :invalid) "color" nil] + [(assoc-in valid [:console :stdout] #js {:write #(throw (js/Error. "untrusted stream"))}) + "stdout" nil] + [valid "unsupported-command" {}]]] + (let [local (capture) + output (path/join @temporary-root (str (random-uuid))) + before (listeners) + handler (fn [_ _] + (fs/mkdirSync output) + {:exit-code 0 :errors [] :log-path nil})] + (t/is (= 1 (await (execution/execute received (or handlers {"run" handler}) + (assoc local :get-root #(fs/mkdirSync output)))))) + (t/is (str/includes? @(:errors local) field)) + (t/is (not (str/includes? @(:errors local) "\u001b["))) + (t/is (not (fs/existsSync output))) + (t/is (= before (listeners))))))) + +(t/deftest ^:async received-command-policies-reject-before-output-allocation + (doseq [[command options message] + [["build" {:features ["stats"]} "diagnostic-features"] + ["run" {:build {:kind :prepare :features ["profile"]}} "diagnostic-features"] + ["ab" {:features ["profile-raf"] :dry-run true} "diagnostic-features"] + ["ab" {:features ["cache"]} "confirmation-required"] + ["ab" {:features ["cache"] :filter " "} "confirmation-required"] + ["ab" {:features ["a" "b"] :max-features 1 :dry-run true} "too-many-features"]]] + (let [local (capture) + output (path/join @temporary-root (str (random-uuid))) + request (request/check (resolved command options))] + (t/is (= 1 (await (execution/execute + request + {command (fn [_ _] + (fs/mkdirSync output) + {:exit-code 0 :errors [] :log-path nil})} + (assoc local :get-root #(fs/mkdirSync output)))))) + (t/is (str/includes? @(:errors local) message)) + (t/is (not (fs/existsSync output)))))) + +(t/deftest ^:async gpu-policy-uses-checked-console-and-dry-run-ab-needs-no-gpu + (let [env (.-env js/process) + old (.-PENPOT_GPU env)] + (try + (js-delete env "PENPOT_GPU") + (doseq [[command options expected] + [["run" {:build {:kind :existing :id "ready"} :require-gpu true} 1] + ["ab" {:features ["cache"] :confirm true :require-gpu true} 1] + ["ab" {:features ["cache"] :dry-run true :require-gpu true} 0]]] + (let [local (capture) + request (assoc (resolved command options) :console {:level :warn :color :always})] + (t/is (= expected + (await (execution/execute + request {command (fn [_ _] {:exit-code 0 :errors [] :log-path nil + :report "dry plan"})} local)))) + (if (zero? expected) + (t/is (= "dry plan\n" @(:output local))) + (do + (t/is (str/includes? @(:errors local) "PENPOT_GPU=1")) + (t/is (str/includes? @(:errors local) "\u001b[")))))) + (finally + (if (nil? old) (js-delete env "PENPOT_GPU") (set! (.-PENPOT_GPU env) old)))))) + +(t/deftest ^:async thrown-and-rejected-failures-use-checked-console-and-release-listeners + (doseq [handler [(fn [_ _] (throw (js/Error. "failed work"))) + (^:async fn [_ _] + (await (timers/setTimeout 1)) + (throw (js/Error. "failed work"))) + (^:async fn [_ _] + (await (timers/setTimeout 1)) + (throw (errors/fatal-error "failed work" {} (js/Error. "storage"))))]] + (let [local (capture) + before (listeners) + request (assoc (resolved "help" {:topic nil}) :console {:level :warn :color :never})] + (t/is (= 1 (await (execution/execute + request {"help" (fn [context options] + (log/info "hidden progress") + (handler context options))} local)))) + (t/is (= "✖ failed work\n" @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest ^:async final-publication-uses-the-outcomes-log-and-numeric-status + (let [local (capture) + file (path/join @temporary-root "command.log")] + (t/is (= 7 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 7 :errors [] :log-path file :report "saved report"})} + local)))) + (t/is (= "saved report\n" (fs/readFileSync file "utf8"))) + (t/is (= "saved report\n" @(:output local))) + (let [silent (capture)] + (t/is (= 0 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path nil})} silent)))) + (t/is (= "" @(:output silent)))))) + +(t/deftest ^:async final-errors-share-the-command-log-and-print-one-tail + (let [local (capture) + file (path/join @temporary-root "failures.log")] + (fs/writeFileSync file "saved evidence\n" "utf8") + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] + {:exit-code 1 :log-path file + :errors [(js/Error. "first failure") (js/Error. "second failure")]})} + local)))) + (t/is (= 2 (count (re-seq #"✖" @(:errors local))))) + (t/is (= 1 (count (re-seq #" tail\n" @(:errors local))))) + (t/is (str/includes? @(:errors local) "saved evidence")) + (t/is (str/includes? (fs/readFileSync file "utf8") "second failure")))) + +(t/deftest ^:async publication-failure-fails-the-command-and-preserves-interruption-status + (doseq [[code expected] [[0 1] [130 130] [143 143]]] + (let [local (capture) + written (atom []) + broken #js {:write (fn [text] + (swap! written conj text) + (throw (js/Error. "stdout closed")))}] + (t/is (= expected (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code code :errors [] :log-path nil :report "result"})} + (assoc local :stdout broken))))) + (t/is (= ["result\n"] @written)) + (t/is (= "✖ stdout closed\n" @(:errors local))))) + (let [local (capture)] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path @temporary-root :report "result"})} + local)))) + (t/is (str/includes? @(:errors local) "cannot append benchmark log")) + (t/is (str/includes? @(:errors local) "Could not append log")))) + +(t/deftest ^:async real-signals-retain-partial-evidence-and-fatal-errors + (doseq [[first-signal second-signal code] [["SIGINT" "SIGTERM" 130] ["SIGTERM" "SIGINT" 143]]] + (let [local (capture) + before (listeners) + file (path/join @temporary-root first-signal) + handler (fn [{:keys [signal]} _] + (js/Promise. + (fn [resolve reject] + (.addEventListener + signal "abort" + (fn [] + (-> (timers/setTimeout 1) + (.then (fn [] + (t/is (.emit js/process second-signal)) + (fs/writeFileSync file "partial evidence" "utf8") + (resolve {:exit-code 1 :log-path nil :report "partial report" + :errors [(js/Error. "aborted work") + (errors/fatal-error "fatal storage" {} (js/Error. "disk"))]}))) + (.catch reject))) + #js {:once true}) + (.kill js/process (.-pid js/process) first-signal))))] + (t/is (= code (await (within (execution/execute (resolved "help" {:topic nil}) + {"help" handler} local))))) + (t/is (= "partial evidence" (fs/readFileSync file "utf8"))) + (t/is (= "partial report\n" @(:output local))) + (t/is (= "✖ fatal storage\n" @(:errors local))) + (t/is (= before (listeners)))))) + +(t/deftest failure-adapters-preserve-usage-validation-and-decoder-diagnostics + (doseq [[get-cause message] + [[#(cli/parse-resolve ["purge"]) "Usage: runner purge"] + [#(request/check (update (resolved "run" {:build {:kind :existing :id "ready"}}) + :options dissoc :warmups)) "warmups"] + [#(transit/decode-str "invalid transit") nil] + [#((createRequire (path/join @temporary-root "entry.cjs")) + (path/join @temporary-root "missing.cjs")) "missing.cjs"]]] + (let [local (capture) + cause (try + (get-cause) + (t/is false "expected a boundary failure") + (catch :default cause cause))] + (t/is (= 1 (execution/fail cause {} local))) + (t/is (str/includes? @(:errors local) (or message (errors/message-of cause)))) + (t/is (= 1 (count (re-seq #"✖" @(:errors local)))))))) + +(t/deftest ^:async error-reporting-failure-uses-a-bounded-fallback + (doseq [[code expected] [[1 1] [130 130]]] + (let [local (capture) + fallback (atom []) + broken #js {:write (fn [text] + (if (str/starts-with? text "Could not report") + (swap! fallback conj text) + (throw (js/Error. (apply str (repeat 2000 "x"))))))}] + (t/is (= expected (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code code :log-path nil :errors [(js/Error. "work failed")]})} + (assoc local :stderr broken))))) + (t/is (= 1 (count @fallback))) + (t/is (< (count (first @fallback)) 1100)))) + (let [broken #js {:write #(throw (js/Error. "closed"))}] + (t/is (= 1 (await (execution/execute + (resolved "help" {:topic nil}) + {"help" (fn [_ _] {:exit-code 0 :errors [] :log-path nil :report "result"})} + {:stdout broken :stderr broken})))))) diff --git a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs index 95806ab898..4335001f2e 100644 --- a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs +++ b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs @@ -15,6 +15,7 @@ [benches.render-wasm.errors :as errors] [benches.render-wasm.feature-plan :as feature-plan] [benches.render-wasm.node.build :as build] + [benches.render-wasm.node.execution :as execution] [benches.render-wasm.node.feature-comparison :as feature-comparison] [benches.render-wasm.node.feature-plan :as node-plan] [benches.render-wasm.node.files :as files] @@ -1102,7 +1103,7 @@ (doseq [[signal exit-code] [["SIGINT" 130] ["SIGTERM" 143]]] (let [prepared (comparison-prepared ["a"] {:output-dir (path/join @temporary-root signal)}) controller (js/AbortController.) - remove (process/on-signals #(.abort controller %)) + remove (execution/on-signals #(.abort controller %)) execute (execute-service (fn [build] (when (seq (get-in build [:manifest :functional :features])) diff --git a/frontend/test/performance_tests/render_wasm/process_test.cljs b/frontend/test/performance_tests/render_wasm/process_test.cljs index e51a8fbcc1..a3a253cb50 100644 --- a/frontend/test/performance_tests/render_wasm/process_test.cljs +++ b/frontend/test/performance_tests/render_wasm/process_test.cljs @@ -452,28 +452,3 @@ (finally (.abort controller "test-cleanup") (await held))))) - -(t/deftest ^:async real-signals-reach-the-callback-and-remover-restores-listeners - (let [before-int (.listenerCount js/process "SIGINT") - before-term (.listenerCount js/process "SIGTERM") - received (atom []) - next-signal (volatile! nil) - remove (process/on-signals - (fn [signal] - (swap! received conj signal) - (@next-signal signal))) - receive (fn [signal] - (within - (js/Promise. - (fn [resolve _] - (vreset! next-signal resolve) - (.kill js/process (.-pid js/process) signal)))))] - (try - (t/is (= "SIGINT" (await (receive "SIGINT")))) - (t/is (= "SIGTERM" (await (receive "SIGTERM")))) - (t/is (= ["SIGINT" "SIGTERM"] @received)) - (finally - (remove))) - (remove) - (t/is (= before-int (.listenerCount js/process "SIGINT"))) - (t/is (= before-term (.listenerCount js/process "SIGTERM"))))) diff --git a/frontend/test/performance_tests/render_wasm/runner.cljs b/frontend/test/performance_tests/render_wasm/runner.cljs index d55629bffd..c7754f2152 100644 --- a/frontend/test/performance_tests/render_wasm/runner.cljs +++ b/frontend/test/performance_tests/render_wasm/runner.cljs @@ -16,10 +16,12 @@ [performance-tests.render-wasm.build-test] [performance-tests.render-wasm.builder-test] [performance-tests.render-wasm.cli-test] + [performance-tests.render-wasm.commands-test] [performance-tests.render-wasm.compare-test] [performance-tests.render-wasm.containers-test] [performance-tests.render-wasm.contracts-test] [performance-tests.render-wasm.errors-test] + [performance-tests.render-wasm.execution-test] [performance-tests.render-wasm.feature-comparison-test] [performance-tests.render-wasm.fingerprint-test] [performance-tests.render-wasm.geometry-scenes-test] @@ -35,7 +37,8 @@ [performance-tests.render-wasm.runner-test] [performance-tests.render-wasm.runs-test] [performance-tests.render-wasm.scene-snapshot-test] - [performance-tests.render-wasm.server-test])) + [performance-tests.render-wasm.server-test] + [performance-tests.render-wasm.signals-test])) (enable-console-print!) @@ -55,10 +58,12 @@ 'performance-tests.render-wasm.build-test 'performance-tests.render-wasm.builder-test 'performance-tests.render-wasm.cli-test + 'performance-tests.render-wasm.commands-test 'performance-tests.render-wasm.compare-test 'performance-tests.render-wasm.containers-test 'performance-tests.render-wasm.contracts-test 'performance-tests.render-wasm.errors-test + 'performance-tests.render-wasm.execution-test 'performance-tests.render-wasm.feature-comparison-test 'performance-tests.render-wasm.fingerprint-test 'performance-tests.render-wasm.geometry-scenes-test @@ -74,7 +79,8 @@ 'performance-tests.render-wasm.runner-test 'performance-tests.render-wasm.runs-test 'performance-tests.render-wasm.scene-snapshot-test - 'performance-tests.render-wasm.server-test]) + 'performance-tests.render-wasm.server-test + 'performance-tests.render-wasm.signals-test]) (assert (every? find-ns-obj test-namespaces) "test-namespaces contains a namespace that isn't required in runner.cljs")