mirror of
https://github.com/penpot/penpot.git
synced 2026-10-31 23:10:31 +00:00
1027 lines
52 KiB
Clojure
1027 lines
52 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns performance-tests.render-wasm.run-test
|
|
(:require
|
|
["fs" :as fs]
|
|
["os" :as os]
|
|
["path" :as path]
|
|
[app.common.transit :as transit]
|
|
[benches.render-wasm.feature-plan :as feature-plan]
|
|
[benches.render-wasm.node.artifacts :as artifacts]
|
|
[benches.render-wasm.node.feature-comparison :as feature-comparison]
|
|
[benches.render-wasm.node.log :as log]
|
|
[benches.render-wasm.node.run :as run]
|
|
[benches.render-wasm.node.runs :as runs]
|
|
[benches.render-wasm.result :as result]
|
|
[cljs.test :as t :include-macros true]
|
|
[clojure.string :as str]))
|
|
|
|
(def ^:private temporary-root (atom nil))
|
|
|
|
(t/use-fixtures :each
|
|
{:before #(reset! temporary-root
|
|
(fs/mkdtempSync (path/join (os/tmpdir) "penpot-render-run-")))
|
|
:after (fn []
|
|
(log/configure {})
|
|
(fs/rmSync @temporary-root #js {:recursive true :force true}))})
|
|
|
|
(defn- output-dir
|
|
[]
|
|
(path/join @temporary-root (str "out-" (random-uuid))))
|
|
|
|
(defn- build-dir
|
|
[]
|
|
(let [dir (path/join @temporary-root (str "build-" (random-uuid)))]
|
|
(fs/mkdirSync dir #js {:recursive true})
|
|
(fs/writeFileSync (path/join dir "index.html") "<!doctype html>")
|
|
dir))
|
|
|
|
(defn- selected-build
|
|
[dir id]
|
|
{:id id
|
|
:dir dir
|
|
:log-path (path/join dir "build.log")
|
|
:manifest {:type :renderer-build
|
|
:id id
|
|
:status :ready
|
|
:created-at "2026-10-27T00:00:00.000Z"
|
|
:updated-at "2026-10-27T00:00:00.000Z"
|
|
:functional {:mode :release
|
|
:default-features? true
|
|
:features []
|
|
:cljs-profile-marks false}
|
|
:provenance {:git {:sha "build-revision" :dirty false}}}})
|
|
|
|
(defn- collected
|
|
[id context]
|
|
{:id id
|
|
:scene :case
|
|
:scene-description "test scene"
|
|
:scene-seed 1
|
|
:params {}
|
|
:view {:scale 1 :x 0 :y 0}
|
|
:context context
|
|
:completion :render-full
|
|
:batch-size 1
|
|
:preparation {}})
|
|
|
|
(defn- next-count
|
|
[state key]
|
|
(inc (get @state key 0)))
|
|
|
|
(defn- check-page
|
|
[page]
|
|
(when-not (and (map? page) (some? (:id page)))
|
|
(throw (ex-info "browser service requires the page object"
|
|
{:phase :browser-page-startup}))))
|
|
|
|
(def ^:private hardware-graphics
|
|
{:webgl2 true :vendor "NVIDIA" :renderer "ANGLE (NVIDIA, Vulkan, NVIDIA)" :software false})
|
|
|
|
(defn- fake-browser
|
|
[{:keys [launch-failure new-page-failure close-page-failure
|
|
prepare-failure run-failure screenshot-failure
|
|
interruption controller effective-graphics metrics]
|
|
:or {effective-graphics (constantly {}) metrics {}}}]
|
|
(let [state (atom {:launches 0 :pages 0 :close-pages 0 :closes 0
|
|
:host nil :runs {} :preparations {}})]
|
|
{:state state
|
|
:service
|
|
{:launch
|
|
(fn [_]
|
|
(let [n (next-count state :launches)]
|
|
(swap! state assoc :launches n)
|
|
(when (and launch-failure (launch-failure n))
|
|
(throw (ex-info "browser launch failed" {:phase :browser-startup})))
|
|
{:version "FakeBrowser/1" :browser #js {}}))
|
|
:new-page
|
|
(fn [_ host _]
|
|
(when-not (and (string? (:base-url host))
|
|
(some? (get-in host [:build :id])))
|
|
(throw (ex-info "new-page requires the opened host"
|
|
{:phase :browser-page-startup})))
|
|
(swap! state assoc :host host)
|
|
(let [n (next-count state :pages)]
|
|
(swap! state assoc :pages n)
|
|
(when (and new-page-failure (new-page-failure n))
|
|
(throw (ex-info "browser page failed" {:phase :browser-page-startup})))
|
|
{:page {:id n} :context {}}))
|
|
:close-page
|
|
(fn [_]
|
|
(let [n (next-count state :close-pages)]
|
|
(swap! state assoc :close-pages n)
|
|
(when (and close-page-failure (close-page-failure n))
|
|
(throw (ex-info "page close failed" {:phase :browser-page-close})))))
|
|
:close
|
|
(fn [_]
|
|
(swap! state update :closes inc))
|
|
:connected?
|
|
(fn [_] true)
|
|
:prepare
|
|
(fn [page request _]
|
|
(check-page page)
|
|
(let [{:keys [preparation-id] case-desc :case} (transit/decode-str request)
|
|
case-id (:id case-desc)
|
|
failure (when prepare-failure
|
|
(prepare-failure case-id))]
|
|
(case failure
|
|
:throw
|
|
(throw (ex-info "scene preparation failed" {:phase :scene-build}))
|
|
|
|
:response
|
|
{:status "failed" :phase "scene-build" :message "scene rejected"}
|
|
|
|
(let [preparation {:preparation-id preparation-id
|
|
:case-id case-id
|
|
:fingerprint {:algorithm :sha-256
|
|
:digest (apply str (repeat 64 "a"))}
|
|
:effective-graphics (effective-graphics case-id (:id page))
|
|
:diagnostics {}}]
|
|
(swap! state assoc-in [:preparations preparation-id] case-id)
|
|
{:status "ok" :preparation preparation}))))
|
|
:run
|
|
(fn [page request _]
|
|
(check-page page)
|
|
(let [{:keys [preparation-id]} (transit/decode-str request)
|
|
case-id (get-in @state [:preparations preparation-id])
|
|
n (inc (get-in @state [:runs case-id] 0))
|
|
_ (swap! state assoc-in [:runs case-id] n)
|
|
signal (when interruption (interruption case-id n))]
|
|
(when signal
|
|
(let [reason (ex-info "run interrupted"
|
|
{:type ::interrupted :signal signal})]
|
|
(.abort controller reason)
|
|
(throw reason)))
|
|
(case (when run-failure (run-failure case-id n))
|
|
:throw
|
|
(throw (ex-info "attempt failed" {:phase "interact"}))
|
|
|
|
:response
|
|
{:status "failed" :phase "interact" :message "attempt rejected"}
|
|
|
|
{:status "ok" :metrics metrics :trace {}})))
|
|
:screenshot
|
|
(fn [page file _]
|
|
(check-page page)
|
|
(when (and screenshot-failure (screenshot-failure file))
|
|
(throw (ex-info "screenshot failed" {:phase :screenshot})))
|
|
(fs/writeFileSync file "png"))}}))
|
|
|
|
(defn- options
|
|
[fake out]
|
|
{:output-dir out
|
|
:seed 42
|
|
:filter nil
|
|
:warmups 0
|
|
:repetitions 1
|
|
:timeout-ms 5000
|
|
:build-timeout-ms 60000
|
|
:startup-timeout-ms 5000
|
|
:screenshots false
|
|
:remove false
|
|
:browser (:service fake)})
|
|
|
|
(defn- execute
|
|
([fake cases opts]
|
|
(execute fake cases opts (js/AbortController.)))
|
|
([_fake cases opts controller]
|
|
(run/execute (artifacts/frontend-root)
|
|
(runs/create-home {:root (artifacts/frontend-root) :run-id "test-run"
|
|
:output-dir (:output-dir opts)})
|
|
(selected-build (build-dir) "test-build")
|
|
cases opts (.-signal controller))))
|
|
|
|
(defn- attempts-for
|
|
[run case-id]
|
|
(filterv #(= case-id (:case-id %)) (:attempts run)))
|
|
|
|
(defn- preparations-for
|
|
[run case-id]
|
|
(filterv #(= case-id (:case-id %)) (:preparations run)))
|
|
|
|
(defn- saved-record
|
|
[result]
|
|
(result/decode (fs/readFileSync (get-in result [:home :record-path]) "utf8")))
|
|
|
|
(t/deftest ^:async run-rejects-invalid-options-before-allocating-output
|
|
(doseq [invalid [{:seed 4294967296}
|
|
{:warmups -1}
|
|
{:repetitions 1.5}
|
|
{:timeout-ms js/Infinity}
|
|
{:build-timeout-ms 2147483648}
|
|
{:startup-timeout-ms 0}
|
|
{:screenshots "false"}
|
|
{:remove nil}
|
|
{:require-gpu 1}
|
|
{:repetitons 1}]]
|
|
(let [out (output-dir)
|
|
opts (merge (dissoc (options nil out) :browser)
|
|
{:build-id "unavailable"} invalid)
|
|
cause (try
|
|
(await (run/run @temporary-root opts nil))
|
|
nil
|
|
(catch :default cause cause))]
|
|
(t/testing (pr-str invalid)
|
|
(t/is (= ::run/invalid-options (:type (ex-data cause))))
|
|
(t/is (not (fs/existsSync out)))))))
|
|
|
|
(t/deftest ^:async execute-rejects-invalid-options-before-writing-evidence
|
|
(let [root @temporary-root
|
|
fake (fake-browser {})
|
|
home (runs/create-home {:root root :run-id "invalid-options"
|
|
:output-dir (output-dir)})
|
|
files (vec (fs/readdirSync (:dir home)))
|
|
log-text (fs/readFileSync (:log-path home) "utf8")
|
|
opts (assoc (options fake (:dir home)) :screenshots "false")
|
|
cause (try
|
|
(await (run/execute root home (selected-build (build-dir) "test-build")
|
|
[(collected :case/a :fresh)] opts nil))
|
|
nil
|
|
(catch :default cause cause))]
|
|
(t/is (= ::run/invalid-options (:type (ex-data cause))))
|
|
(t/is (= files (vec (fs/readdirSync (:dir home)))))
|
|
(t/is (= log-text (fs/readFileSync (:log-path home) "utf8")))))
|
|
|
|
(t/deftest ^:async retained-run-prints-results-and-keeps-all-levels-in-its-log
|
|
(doseq [level [:info :warn :debug]]
|
|
(let [root @temporary-root
|
|
cases [(collected :case/a :fresh) (collected :case/b :reuse)]
|
|
build (artifacts/create-build root {:default-features? true :features []}
|
|
{:git {:sha "fixture" :dirty false}})
|
|
stdout (atom "")
|
|
stderr (atom "")
|
|
fake (fake-browser {})
|
|
out (output-dir)]
|
|
(doseq [file [(:index-path build) (path/join (:browser-dir build) "browser.js")
|
|
(path/join (:renderer-dir build) "render-wasm.js")
|
|
(path/join (:renderer-dir build) "render-wasm.wasm")
|
|
(path/join (:renderer-dir build) "shared.js")]]
|
|
(fs/writeFileSync file "fixture" "utf8"))
|
|
(fs/writeFileSync (:cases-path build)
|
|
(str "module.exports = {buildIdentity() {return "
|
|
(js/JSON.stringify (transit/encode-str {:build-id (:id build)}))
|
|
";}, collectCases() {return "
|
|
(js/JSON.stringify (transit/encode-str {:build-id (:id build) :cases cases}))
|
|
";}};") "utf8")
|
|
(artifacts/write-manifest build (assoc (:manifest build) :status :ready))
|
|
(log/configure {:level level :color :never
|
|
:stdout #js {:write #(swap! stdout str %)}
|
|
:stderr #js {:write #(swap! stderr str %)}})
|
|
(let [response (await (run/run root (assoc (options fake out) :build-id (:id build)
|
|
:warmups 1 :repetitions 2) nil))
|
|
saved (saved-record response)
|
|
text (fs/readFileSync (get-in response [:home :log-path]) "utf8")]
|
|
(t/is (= 0 (:exit-code response)))
|
|
(t/is (= (str "Run " (:run-id saved) "\nOutput " out "\nReport\n"
|
|
(str/replace (:report response) #"\n$" "") "\n") @stdout))
|
|
(t/is (str/includes? text "▸ Case case/a (fresh)\n"))
|
|
(doseq [case-text (rest (str/split text #"▸ Case "))]
|
|
(t/is (str/includes? case-text " attempt 1/3 completed (warmup)\n"))
|
|
(t/is (str/includes? case-text " attempt 2/3 completed\n"))
|
|
(t/is (str/includes? case-text " attempt 3/3 completed\n")))
|
|
(t/is (str/includes? text (:report response)))
|
|
(t/is (str/includes? text "Browser FakeBrowser/1"))
|
|
(t/is (= (not= :warn level) (str/includes? @stderr "▸ Case case/a")))
|
|
(t/is (= (= :debug level) (str/includes? @stderr "Browser FakeBrowser/1")))))))
|
|
|
|
(t/deftest ^:async fresh-and-reuse-cases-follow-their-context-policies
|
|
(let [fake (fake-browser {})
|
|
cases [(collected :case/fresh :fresh) (collected :case/reuse :reuse)]
|
|
opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2)
|
|
result (await (execute fake cases opts))
|
|
saved (saved-record result)
|
|
run (:run result)
|
|
fresh (attempts-for run :case/fresh)
|
|
reuse (attempts-for run :case/reuse)]
|
|
(t/is (= saved run))
|
|
(t/is (= 6 (count (:attempts run))))
|
|
(t/is (= 3 (count (preparations-for run :case/fresh))))
|
|
(t/is (= 1 (count (preparations-for run :case/reuse))))
|
|
(t/is (= 3 (count (distinct (map :preparation-id fresh)))))
|
|
(t/is (= 1 (count (distinct (map :preparation-id reuse)))))
|
|
(t/is (every? #(= :completed (:outcome %)) (:attempts run)))
|
|
(t/is (every? #(every? zero? (vals %)) (vals (:remaining run))))
|
|
(t/is (= :completed (get-in run [:termination :reason])))
|
|
(t/is (zero? (:exit-code result)))
|
|
(t/is (empty? (get-in run [:metadata :diagnostics :screenshots])))
|
|
(t/is (empty? (filter #(str/ends-with? % ".png")
|
|
(array-seq (fs/readdirSync (get-in result [:home :dir]))))))))
|
|
|
|
(t/deftest ^:async page-startup-failure-abandons-the-case-and-keeps-its-slots
|
|
(doseq [context [:fresh :reuse]]
|
|
(let [fake (fake-browser {:new-page-failure #(= 1 %)})
|
|
opts (assoc (options fake (output-dir)) :repetitions 2)
|
|
result (await (execute fake [(collected :case/a context)
|
|
(collected :case/b :fresh)]
|
|
opts))
|
|
run (:run result)
|
|
state @(:state fake)]
|
|
(t/is (empty? (attempts-for run :case/a)))
|
|
(t/is (empty? (preparations-for run :case/a)))
|
|
(t/is (= {:warmups 0 :repetitions 2}
|
|
(get-in run [:remaining :case/a])))
|
|
(t/is (= [:completed :completed]
|
|
(mapv :outcome (attempts-for run :case/b))))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= :browser-page (get-in run [:termination :failure :stage])))
|
|
(t/is (= 1 (:exit-code result)))
|
|
(t/is (= 2 (:launches state)))
|
|
(t/is (= 2 (:closes state))))))
|
|
|
|
(t/deftest ^:async failed-attempts-consume-slots-and-the-runner-continues
|
|
(doseq [failure [:throw :response]]
|
|
(let [fake (fake-browser {:run-failure (fn [case-id n]
|
|
(when (and (= case-id :case/fresh) (= n 1))
|
|
failure))})
|
|
opts (assoc (options fake (output-dir)) :repetitions 2)
|
|
result (await (execute fake [(collected :case/fresh :fresh)] opts))
|
|
run (:run result)
|
|
tries (attempts-for run :case/fresh)]
|
|
(t/is (= 2 (count tries)))
|
|
(t/is (= [:failed :completed] (mapv :outcome tries)))
|
|
(t/is (every? zero? (vals (get-in run [:remaining :case/fresh]))))
|
|
(t/is (= :failed (get-in run [:termination :reason])))
|
|
(t/is (= 1 (:exit-code result)))
|
|
(t/is (= 2 (count (preparations-for run :case/fresh)))))))
|
|
|
|
(t/deftest ^:async reuse-preparation-failure-leaves-slots-and-continues-to-the-next-case
|
|
(doseq [kind [:throw :response]]
|
|
(let [fake (fake-browser {:prepare-failure #(when (= % :case/reuse) kind)})
|
|
opts (assoc (options fake (output-dir)) :screenshots true :repetitions 2)
|
|
result (await (execute fake [(collected :case/reuse :reuse)
|
|
(collected :case/fresh :fresh)] opts))
|
|
run (:run result)
|
|
shots (get-in run [:metadata :diagnostics :screenshots])]
|
|
(t/is (empty? (attempts-for run :case/reuse)))
|
|
(t/is (= {:warmups 0 :repetitions 2} (get-in run [:remaining :case/reuse])))
|
|
(t/is (= [:completed :completed] (mapv :outcome (attempts-for run :case/fresh))))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= 1 (:exit-code result)))
|
|
(t/is (= :skipped (get-in shots ["case/reuse" :status])))
|
|
(t/is (= :ok (get-in shots ["case/fresh" :status])))
|
|
(let [failure (last (get-in run [:metadata :diagnostics :failures]))]
|
|
(t/is (= :preparation (:stage failure)))
|
|
(t/is (= :case/reuse (:case-id failure)))
|
|
(t/is (= "scene-build" (:phase failure)))))))
|
|
|
|
(t/deftest ^:async fresh-preparation-failure-consumes-one-slot-and-retries
|
|
(doseq [kind [:throw :response]]
|
|
(let [preparations (atom 0)
|
|
fake (fake-browser {:prepare-failure #(when (and (= % :case/fresh)
|
|
(= 1 (swap! preparations inc)))
|
|
kind)})
|
|
result (await (execute fake [(collected :case/fresh :fresh)]
|
|
(assoc (options fake (output-dir)) :repetitions 2 :screenshots true)))
|
|
run (:run result)
|
|
tries (attempts-for run :case/fresh)]
|
|
(t/is (= [:failed :completed] (mapv :outcome tries)))
|
|
(t/is (nil? (:preparation-id (first tries))))
|
|
(t/is (= "scene-build" (get-in (first tries) [:failure :phase])))
|
|
(t/is (= 1 (count (preparations-for run :case/fresh))))
|
|
(t/is (= :ok (get-in run [:metadata :diagnostics :screenshots "case/fresh" :status])))
|
|
(t/is (every? zero? (vals (get-in run [:remaining :case/fresh]))))
|
|
(t/is (= :failed (get-in run [:termination :reason])))
|
|
(t/is (= 1 (:exit-code result))))))
|
|
|
|
(t/deftest ^:async reuse-attempt-failure-leaves-pending-slots-and-continues-to-the-next-case
|
|
(doseq [kind [:throw :response]]
|
|
(let [fake (fake-browser {:run-failure (fn [case-id n]
|
|
(when (and (= case-id :case/reuse) (= n 1)) kind))})
|
|
opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2 :screenshots true)
|
|
result (await (execute fake [(collected :case/reuse :reuse)
|
|
(collected :case/fresh :fresh)] opts))
|
|
run (saved-record result)]
|
|
(t/is (= [:failed] (mapv :outcome (attempts-for run :case/reuse))))
|
|
(t/is (= {:warmups 0 :repetitions 2} (get-in run [:remaining :case/reuse])))
|
|
(t/is (= [:completed :completed :completed] (mapv :outcome (attempts-for run :case/fresh))))
|
|
(t/is (= {:status :skipped :reason :case-abandoned}
|
|
(get-in run [:metadata :diagnostics :screenshots "case/reuse"])))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= 1 (:exit-code result))))))
|
|
|
|
(t/deftest ^:async final-attempt-failure-records-why-the-screenshot-was-skipped
|
|
(doseq [context [:fresh :reuse]
|
|
failure [:preparation :attempt]
|
|
:when (or (= context :fresh) (= failure :attempt))]
|
|
(let [fake (fake-browser (if (= failure :preparation)
|
|
{:prepare-failure (constantly :response)}
|
|
{:run-failure (fn [_ _] :response)}))
|
|
opts (assoc (options fake (output-dir)) :screenshots true)
|
|
result (await (execute fake [(collected :case/a context)] opts))
|
|
run (saved-record result)]
|
|
(t/is (= [:failed] (mapv :outcome (:attempts run))))
|
|
(t/is (= {:status :skipped :reason :no-completed-measured-attempt}
|
|
(get-in run [:metadata :diagnostics :screenshots "case/a"])))
|
|
(t/is (= :failed (get-in run [:termination :reason])))
|
|
(t/is (= 1 (:exit-code result))))))
|
|
|
|
(t/deftest ^:async browser-close-failure-abandons-the-case-and-relaunches-for-the-next
|
|
(let [fake (fake-browser {:run-failure (fn [case-id n]
|
|
(when (and (= case-id :case/a) (= n 1))
|
|
:throw))
|
|
:close-page-failure #(= % 1)})
|
|
opts (assoc (options fake (output-dir)) :repetitions 2)
|
|
result (await (execute fake [(collected :case/a :fresh)
|
|
(collected :case/b :fresh)] opts))
|
|
run (:run result)
|
|
state @(:state fake)]
|
|
(t/is (= 2 (:launches state)))
|
|
(t/is (= 1 (count (attempts-for run :case/a))))
|
|
(t/is (= :failed (:outcome (first (attempts-for run :case/a)))))
|
|
(t/is (= 1 (get-in run [:remaining :case/a :repetitions])))
|
|
(t/is (= 2 (count (attempts-for run :case/b))))
|
|
(t/is (every? #(= :completed (:outcome %)) (attempts-for run :case/b)))
|
|
(t/is (every? zero? (vals (get-in run [:remaining :case/b]))))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= 1 (:exit-code result)))))
|
|
|
|
(t/deftest ^:async page-close-failure-after-the-final-slot-is-a-harness-failure
|
|
(let [fake (fake-browser {:close-page-failure #(= % 1)})
|
|
result (await (execute fake [(collected :case/a :fresh)]
|
|
(options fake (output-dir))))
|
|
run (:run result)]
|
|
(t/is (= :completed (get-in run [:termination :reason])))
|
|
(t/is (every? zero? (vals (get-in run [:remaining :case/a]))))
|
|
(t/is (= :browser-close
|
|
(:stage (last (get-in run [:metadata :diagnostics :failures])))))
|
|
(t/is (= :browser-close (get-in run [:termination :failure :stage])))
|
|
(t/is (= 1 (:exit-code result)))))
|
|
|
|
(t/deftest ^:async initial-browser-launch-failure-aborts-before-any-case
|
|
(let [fake (fake-browser {:launch-failure #(= % 1)})
|
|
result (await (execute fake [(collected :case/a :fresh)]
|
|
(options fake (output-dir))))
|
|
run (:run result)
|
|
state @(:state fake)]
|
|
(t/is (= 1 (:launches state)))
|
|
(t/is (empty? (:attempts run)))
|
|
(t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/a])))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= :browser-launch (:stage (last (get-in run [:metadata :diagnostics :failures])))))
|
|
(t/is (= 1 (:exit-code result)))))
|
|
|
|
(t/deftest ^:async hardware-preparations-complete-fresh-and-reuse-cases
|
|
(let [fake (fake-browser {:effective-graphics (constantly hardware-graphics)})
|
|
opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1 :repetitions 2)
|
|
result (await (execute fake [(collected :case/fresh :fresh)
|
|
(collected :case/reuse :reuse)] opts))
|
|
run (saved-record result)]
|
|
(t/is (= 0 (:exit-code result)))
|
|
(t/is (= :completed (get-in run [:termination :reason])))
|
|
(t/is (= 6 (count (:attempts run))))
|
|
(t/is (every? #(= hardware-graphics (:effective-graphics %)) (:preparations run)))
|
|
(t/is (true? (get-in run [:metadata :diagnostics :require-gpu?])))))
|
|
|
|
(t/deftest ^:async required-gpu-failures-preserve-evidence-and-all-attempt-slots
|
|
(doseq [context [:fresh :reuse]
|
|
graphics [(assoc hardware-graphics :renderer "ANGLE (SwiftShader)" :software true)
|
|
(assoc hardware-graphics :webgl2 false)
|
|
(assoc hardware-graphics :renderer nil)
|
|
(assoc hardware-graphics :renderer " UnKnOwN ")
|
|
(assoc hardware-graphics :vendor "")
|
|
{}]]
|
|
(let [fake (fake-browser {:effective-graphics (constantly graphics)})
|
|
opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1)
|
|
result (await (execute fake [(collected :case/a context)
|
|
(collected :case/b :reuse)] opts))
|
|
run (saved-record result)
|
|
failure (get-in run [:termination :failure])
|
|
state @(:state fake)]
|
|
(t/is (= 1 (:exit-code result)))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= [graphics] (mapv :effective-graphics (:preparations run))))
|
|
(t/is (= [] (:attempts run)))
|
|
(t/is (= {:case/a {:warmups 1 :repetitions 1}
|
|
:case/b {:warmups 1 :repetitions 1}} (:remaining run)))
|
|
(t/is (= :gpu (:stage failure)))
|
|
(t/is (= graphics (get-in failure [:partial :graphics])))
|
|
(t/is (str/includes? (:message failure) (pr-str (:renderer graphics))))
|
|
(t/is (str/includes? (:message failure) "./manage.sh run-devenv --gpu"))
|
|
(t/is (str/includes? (:message failure) "pnpm run render-bench"))
|
|
(t/is (= 1 (:close-pages state)))
|
|
(t/is (= 1 (:closes state))))))
|
|
|
|
(t/deftest ^:async graphics-rejection-preserves-earlier-attempts-and-stops-the-run
|
|
(doseq [context [:fresh :reuse]]
|
|
(let [software (assoc hardware-graphics :renderer "SwiftShader" :software true)
|
|
fake (fake-browser {:effective-graphics (fn [_ page-id]
|
|
(if (= page-id 1) hardware-graphics software))})
|
|
opts (assoc (options fake (output-dir)) :require-gpu true :repetitions 2)
|
|
result (await (execute fake [(collected :case/a context)
|
|
(collected :case/b context)] opts))
|
|
run (saved-record result)
|
|
failure (get-in run [:termination :failure])]
|
|
(t/is (= 1 (:exit-code result)))
|
|
(t/is (= :gpu (:stage failure)))
|
|
(t/is (= software (get-in failure [:partial :graphics])))
|
|
(t/is (= (if (= context :fresh) 1 2) (count (:attempts run))))
|
|
(t/is (every? #(= :completed (:outcome %)) (:attempts run)))
|
|
(t/is (= [hardware-graphics software] (mapv :effective-graphics (:preparations run))))
|
|
(t/is (= {:case/a {:warmups 0 :repetitions (if (= context :fresh) 1 0)}
|
|
:case/b {:warmups 0 :repetitions 2}} (:remaining run)))
|
|
(t/is (= 2 (:close-pages @(:state fake))))
|
|
(t/is (= 1 (:closes @(:state fake)))))))
|
|
|
|
(t/deftest ^:async sigint-and-sigterm-persist-partial-interrupted-records
|
|
(doseq [context [:fresh :reuse]
|
|
[signal code] [["SIGINT" 130] ["SIGTERM" 143]]]
|
|
(let [controller (js/AbortController.)
|
|
fake (fake-browser {:controller controller
|
|
:interruption (fn [case-id n]
|
|
(when (and (= case-id :case/a) (= n 1))
|
|
signal))})
|
|
opts (assoc (options fake (output-dir)) :screenshots true)
|
|
result (await (execute fake [(collected :case/a context)
|
|
(collected :case/b context)]
|
|
opts controller))
|
|
run (:run result)
|
|
saved (saved-record result)
|
|
shots (get-in run [:metadata :diagnostics :screenshots])]
|
|
(t/is (= saved run))
|
|
(t/is (= :interrupted (get-in run [:termination :reason])))
|
|
(t/is (= signal (get-in run [:termination :signal])))
|
|
(t/is (= code (:exit-code result)))
|
|
(t/is (= 1 (count (attempts-for run :case/a))))
|
|
(t/is (empty? (attempts-for run :case/b)))
|
|
(t/is (= 1 (get-in run [:remaining :case/b :repetitions])))
|
|
(t/is (= :skipped (get-in shots ["case/b" :status])))
|
|
(t/is (= :not-run (get-in shots ["case/b" :reason])))
|
|
(t/is (pos? (:closes @(:state fake)))))))
|
|
|
|
(t/deftest ^:async interruption-before-an-attempt-preserves-unspent-slots
|
|
(doseq [context [:fresh :reuse]
|
|
method [:launch :new-page :prepare]
|
|
[signal code] [["SIGINT" 130] ["SIGTERM" 143]]]
|
|
(let [controller (js/AbortController.)
|
|
base (fake-browser {})
|
|
stop (^:async fn [& _]
|
|
(let [cause (ex-info "interrupted setup" {:signal signal :phase :preparation})]
|
|
(.abort controller cause)
|
|
(throw cause)))
|
|
fake (assoc base :service (assoc (:service base) method stop))
|
|
opts (assoc (options fake (output-dir)) :warmups 1 :repetitions 2)
|
|
result (await (execute fake [(collected :case/a context)
|
|
(collected :case/b context)] opts controller))
|
|
run (saved-record result)
|
|
consumed (if (and (= method :prepare) (= context :fresh)) 1 0)]
|
|
(t/is (= :interrupted (get-in run [:termination :reason])))
|
|
(t/is (= signal (get-in run [:termination :signal])))
|
|
(t/is (= code (:exit-code result)))
|
|
(t/is (= consumed (count (:attempts run))))
|
|
(t/is (= {:warmups (- 1 consumed) :repetitions 2} (get-in run [:remaining :case/a])))
|
|
(t/is (= {:warmups 1 :repetitions 2} (get-in run [:remaining :case/b])))
|
|
(t/is (= (if (and (= method :prepare) (= context :reuse)) [:preparation] [])
|
|
(mapv :stage (get-in run [:metadata :diagnostics :failures])))))))
|
|
|
|
(t/deftest ^:async fresh-preparation-and-rendering-share-the-attempt-time-budget
|
|
(doseq [context [:fresh :reuse]]
|
|
(let [system-now (.-now js/Date)
|
|
clock (atom (system-now))
|
|
base (fake-browser {})
|
|
work (^:async fn [bounds delegate page request]
|
|
(await (js/Promise.resolve nil))
|
|
(swap! clock + 150)
|
|
(when (>= @clock (:deadline bounds))
|
|
(throw (ex-info "browser work timed out" {:phase (:phase bounds)})))
|
|
(delegate page request bounds))
|
|
fake (assoc base :service
|
|
(assoc (:service base)
|
|
:prepare (fn [page request bounds]
|
|
(work bounds (:prepare (:service base)) page request))
|
|
:run (fn [page request bounds]
|
|
(work bounds (:run (:service base)) page request))))]
|
|
(try
|
|
(set! (.-now js/Date) (fn [] @clock))
|
|
(let [result (await (execute fake [(collected :case/a context)]
|
|
(assoc (options fake (output-dir)) :timeout-ms 250)))
|
|
saved (saved-record result)]
|
|
(t/is (= (if (= context :fresh) :failed :completed) (:outcome (first (:attempts saved)))))
|
|
(t/is (= (if (= context :fresh) 1 0) (:exit-code result))))
|
|
(finally
|
|
(set! (.-now js/Date) system-now))))))
|
|
|
|
(t/deftest ^:async browser-disconnection-leaves-pending-slots-and-relaunches-for-the-next-case
|
|
(let [base (fake-browser {})
|
|
disconnected (atom nil)
|
|
new-page (:new-page (:service base))
|
|
service (assoc (:service base)
|
|
:connected? (fn [owner]
|
|
(compare-and-set! disconnected nil (:browser owner))
|
|
(not (identical? @disconnected (:browser owner))))
|
|
:new-page (fn [owner host options]
|
|
(when (identical? @disconnected (:browser owner))
|
|
(throw (js/Error. "browser disconnected")))
|
|
(new-page owner host options)))
|
|
fake (assoc base :service service)
|
|
opts (assoc (options fake (output-dir)) :repetitions 2)
|
|
result (await (execute fake [(collected :case/a :fresh) (collected :case/b :fresh)] opts))
|
|
run (saved-record result)]
|
|
(t/is (= [:completed] (mapv :outcome (attempts-for run :case/a))))
|
|
(t/is (= {:warmups 0 :repetitions 1} (get-in run [:remaining :case/a])))
|
|
(t/is (= [:completed :completed] (mapv :outcome (attempts-for run :case/b))))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= 1 (:exit-code result)))))
|
|
|
|
(t/deftest ^:async provenance-timeout-saves-the-unattempted-run
|
|
(let [bin (path/join @temporary-root "bin")
|
|
prior (.-PATH (.-env js/process))
|
|
_ (fs/mkdirSync bin)
|
|
_ (fs/writeFileSync (path/join bin "git")
|
|
(str "#!" (.-execPath js/process) "\nsetInterval(() => {}, 1000);\n")
|
|
#js {:mode 8r755})
|
|
fake (fake-browser {})]
|
|
(try
|
|
(set! (.-PATH (.-env js/process)) (str bin path/delimiter prior))
|
|
(let [result (await (execute fake [(collected :case/a :fresh)]
|
|
(assoc (options fake (output-dir)) :startup-timeout-ms 50)))
|
|
run (saved-record result)]
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= :runner-git (get-in run [:termination :failure :stage])))
|
|
(t/is (= {:case/a {:warmups 0 :repetitions 1}} (:remaining run)))
|
|
(t/is (= [] (:attempts run)))
|
|
(t/is (= 1 (:exit-code result))))
|
|
(finally
|
|
(set! (.-PATH (.-env js/process)) prior)))))
|
|
|
|
(t/deftest ^:async unavailable-build-directory-saves-a-server-startup-failure
|
|
(let [root @temporary-root
|
|
selected (selected-build (build-dir) "unavailable-build")
|
|
home (runs/create-home {:root root :run-id "unavailable-build" :output-dir (output-dir)})
|
|
_ (fs/renameSync (:dir selected) (str (:dir selected) ".unavailable"))
|
|
fake (fake-browser {})
|
|
result (await (run/execute root home selected [(collected :case/a :fresh)]
|
|
(options fake (:dir home)) (.-signal (js/AbortController.))))
|
|
run (saved-record result)]
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= :server-startup (get-in run [:termination :failure :stage])))
|
|
(t/is (= {:case/a {:warmups 0 :repetitions 1}} (:remaining run)))
|
|
(t/is (= [] (:attempts run)))
|
|
(t/is (= 1 (:exit-code result)))))
|
|
|
|
(t/deftest ^:async failed-browser-relaunch-aborts-the-pending-cases
|
|
(let [fake (fake-browser {:new-page-failure #(= % 1) :launch-failure #(= % 2)})
|
|
cases [(collected :case/a :fresh) (collected :case/b :reuse) (collected :case/c :fresh)]
|
|
result (await (execute fake cases (options fake (output-dir))))
|
|
run (saved-record result)]
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (= [] (:attempts run)))
|
|
(t/is (= [:browser-page :browser-launch] (mapv :stage (get-in run [:metadata :diagnostics :failures]))))
|
|
(t/is (= {:case/a {:warmups 0 :repetitions 1}
|
|
:case/b {:warmups 0 :repetitions 1}
|
|
:case/c {:warmups 0 :repetitions 1}} (:remaining run)))
|
|
(t/is (= 1 (:exit-code result)))))
|
|
|
|
(t/deftest ^:async interruption-exit-status-takes-precedence-over-finalization-errors
|
|
(doseq [failure [:cleanup :record]
|
|
[signal code] [["SIGINT" 130] ["SIGTERM" 143]]]
|
|
(let [controller (js/AbortController.)
|
|
out (output-dir)
|
|
base (fake-browser {:controller controller :interruption (fn [_ _] signal)})
|
|
close (:close (:service base))
|
|
fake (assoc base :service
|
|
(assoc (:service base) :close
|
|
(fn [owner]
|
|
(close owner)
|
|
(if (= failure :cleanup)
|
|
(throw (js/Error. "cleanup rejected"))
|
|
(let [record (path/join out "run.transit")]
|
|
(fs/renameSync record (str record ".saved"))
|
|
(fs/mkdirSync record))))))
|
|
result (await (execute fake [(collected :case/a :fresh)] (options fake out) controller))]
|
|
(t/is (= code (:exit-code result)))
|
|
(t/is (= :interrupted (get-in result [:run :termination :reason])))
|
|
(t/is (= signal (get-in result [:run :termination :signal])))
|
|
(t/is (some? (:error result))))))
|
|
|
|
(t/deftest ^:async log-failure-still-closes-pages-browsers-and-hosts
|
|
(let [out (output-dir)
|
|
base (fake-browser {})
|
|
prepare (:prepare (:service base))
|
|
service (assoc (:service base)
|
|
:prepare
|
|
(fn [page request bounds]
|
|
(let [response (prepare page request bounds)]
|
|
(fs/renameSync (path/join out "run.log")
|
|
(path/join out "run.log.saved"))
|
|
(fs/mkdirSync (path/join out "run.log"))
|
|
response)))
|
|
fake (assoc base :service service)
|
|
result (await (execute fake [(collected :case/a :fresh)]
|
|
(options fake out)))
|
|
state @(:state fake)
|
|
host (:host state)]
|
|
(t/is (some? (:error result)))
|
|
(t/is (some? (:storage-error result)))
|
|
(t/is (= 1 (:exit-code result)))
|
|
(t/is (empty? (attempts-for (:run result) :case/a)))
|
|
(t/is (= 1 (:close-pages state)))
|
|
(t/is (= 1 (:closes state)))
|
|
(t/is (false? (.-listening ^js (:server host))))))
|
|
|
|
(t/deftest ^:async storage-failure-survives-recovery-and-cleanup-error-takes-precedence
|
|
(doseq [cleanup-fails [false true]]
|
|
(let [out (output-dir)
|
|
file (path/join out "run.log")
|
|
saved (str file ".saved")
|
|
base (fake-browser {})
|
|
prepare (:prepare (:service base))
|
|
close (:close (:service base))
|
|
service (assoc (:service base)
|
|
:prepare (fn [page request bounds]
|
|
(let [response (prepare page request bounds)]
|
|
(fs/renameSync file saved)
|
|
(fs/mkdirSync file)
|
|
response))
|
|
:close (fn [owner]
|
|
(close owner)
|
|
(fs/rmdirSync file)
|
|
(fs/renameSync saved file)
|
|
(when cleanup-fails
|
|
(throw (js/Error. "cleanup rejected")))))
|
|
fake (assoc base :service service)
|
|
result (await (execute fake [(collected :case/a :fresh)] (options fake out)))
|
|
run (saved-record result)]
|
|
(t/is (= 1 (:exit-code result)))
|
|
(t/is (some? (:storage-error result)))
|
|
(t/is (str/includes? (ex-message (:error result))
|
|
(if cleanup-fails "cleanup rejected" "cannot append benchmark log")))
|
|
(t/is (= :incomplete (get-in run [:termination :reason])))
|
|
(t/is (str/includes? (fs/readFileSync file "utf8") "Run incomplete")))))
|
|
|
|
(t/deftest ^:async cleanup-error-keeps-completed-attempt-evidence-and-exits-nonzero
|
|
(let [base (fake-browser {})
|
|
fake (assoc base :service (assoc (:service base) :close
|
|
(fn [_] (throw (js/Error. "cleanup rejected")))))
|
|
result (await (execute fake [(collected :case/a :fresh)] (options fake (output-dir))))
|
|
run (saved-record result)]
|
|
(t/is (= [:completed] (mapv :outcome (:attempts run))))
|
|
(t/is (= :completed (get-in run [:termination :reason])))
|
|
(t/is (= :cleanup (get-in run [:termination :failure :stage])))
|
|
(t/is (= "cleanup rejected" (ex-message (:error result))))
|
|
(t/is (= 1 (:exit-code result)))))
|
|
|
|
(t/deftest ^:async interruption-during-shutdown-is-persisted
|
|
(doseq [[signal code] [["SIGINT" 130] ["SIGTERM" 143]]]
|
|
(let [controller (js/AbortController.)
|
|
base (fake-browser {})
|
|
close (:close (:service base))
|
|
service (assoc (:service base)
|
|
:close
|
|
(fn [owner]
|
|
(when-not (.-aborted (.-signal controller))
|
|
(.abort controller signal))
|
|
(close owner)))
|
|
fake (assoc base :service service)
|
|
result (await (execute fake [(collected :case/a :fresh)]
|
|
(options fake (output-dir))
|
|
controller))
|
|
run (:run result)]
|
|
(t/is (= :interrupted (get-in run [:termination :reason])))
|
|
(t/is (= signal (get-in run [:termination :signal])))
|
|
(t/is (= code (:exit-code result)))
|
|
(t/is (= run (saved-record result)))
|
|
(t/is (= 1 (:closes @(:state fake)))))))
|
|
|
|
(t/deftest ^:async screenshots-record-files-and-failures-as-diagnostics
|
|
(doseq [context [:fresh :reuse]]
|
|
(let [fake (fake-browser {})
|
|
opts (assoc (options fake (output-dir)) :screenshots true)
|
|
result (await (execute fake [(collected :case/a context)] opts))
|
|
run (:run result)
|
|
shot (get-in run [:metadata :diagnostics :screenshots "case/a"])]
|
|
(t/is (= :ok (:status shot)))
|
|
(t/is (= "case-a.png" (:file shot)))
|
|
(t/is (fs/existsSync (path/join (get-in result [:home :dir]) "case-a.png")))
|
|
(t/is (= :completed (get-in run [:termination :reason])))
|
|
(t/is (zero? (:exit-code result)))))
|
|
(let [fake (fake-browser {:screenshot-failure (constantly true)})
|
|
opts (assoc (options fake (output-dir)) :screenshots true)
|
|
result (await (execute fake [(collected :case/a :fresh)] opts))
|
|
run (:run result)
|
|
shot (get-in run [:metadata :diagnostics :screenshots "case/a"])]
|
|
(t/is (= :failed (:status shot)))
|
|
(t/is (= "case-a.png" (:file shot)))
|
|
(t/is (= :completed (get-in run [:termination :reason])))
|
|
(t/is (zero? (:exit-code result)))))
|
|
|
|
(t/deftest ^:async metadata-records-build-and-controller-provenance-and-budgets
|
|
(let [fake (fake-browser {})
|
|
opts (assoc (options fake (output-dir))
|
|
:timeout-ms 1234 :build-timeout-ms 5678 :startup-timeout-ms 9012)
|
|
result (await (execute fake [(collected :case/a :fresh)] opts))
|
|
run (:run result)
|
|
diag (get-in run [:metadata :diagnostics])]
|
|
(t/is (= {:attempt-ms 1234} (get-in run [:plan :timeout-policy])))
|
|
(t/is (= {:sha "build-revision" :dirty false} (get-in run [:metadata :git])))
|
|
(t/is (= {:mode :release :default-features? true :features [] :cljs-profile-marks false}
|
|
(get-in run [:metadata :build :functional])))
|
|
(t/is (= "test-build" (get-in run [:metadata :build :provenance :build-id])))
|
|
(t/is (true? (get-in run [:metadata :scored?])))
|
|
(t/is (= "FakeBrowser/1" (get-in run [:metadata :environment :browser])))
|
|
(t/is (= 1234 (:timeout-ms diag)))
|
|
(t/is (= 5678 (:build-timeout-ms diag)))
|
|
(t/is (= 9012 (:startup-timeout-ms diag)))
|
|
(t/is (= 42 (:seed diag)))
|
|
(t/is (= 0 (:warmups diag)))
|
|
(t/is (= 1 (:repetitions diag)))
|
|
(t/is (false? (:screenshots? diag)))
|
|
(t/is (map? (:runner-git diag)))
|
|
(t/is (= :unverified (:runtime-helper-identity diag)))))
|
|
|
|
(t/deftest ^:async run-metadata-copies-only-saved-build-environment-evidence
|
|
(doseq [environment [::missing {} {"CFLAGS" "" "NODE_ENV" "recorded-build-value"}]]
|
|
(let [fake (fake-browser {})
|
|
root (artifacts/frontend-root)
|
|
opts (options fake (output-dir))
|
|
build (cond-> (selected-build (build-dir) "test-build")
|
|
(not= ::missing environment) (assoc-in [:manifest :environment] environment))
|
|
home (runs/create-home {:root root :run-id "test-run" :output-dir (:output-dir opts)})
|
|
controller (js/AbortController.)
|
|
executed (await (run/execute root home build [(collected :case/a :fresh)]
|
|
opts (.-signal controller)))
|
|
saved (saved-record executed)
|
|
facts (get-in saved [:metadata :build])]
|
|
(t/is (zero? (:exit-code executed)))
|
|
(t/is (= saved (:run executed)))
|
|
(if (= ::missing environment)
|
|
(t/is (not (contains? facts :environment)))
|
|
(t/is (= environment (:environment facts)))))))
|
|
|
|
(t/deftest ^:async remove-deletes-the-used-build-after-the-record-is-saved
|
|
(let [root @temporary-root
|
|
id "removable-build"
|
|
dir (path/join (artifacts/managed-root root) "builds" id)
|
|
_ (fs/mkdirSync dir #js {:recursive true})
|
|
_ (fs/writeFileSync (path/join dir "index.html") "<!doctype html>")
|
|
fake (fake-browser {})
|
|
out (path/join @temporary-root "remove-run")
|
|
home (runs/create-home {:root root :run-id "remove-run"
|
|
:output-dir out :remove true})
|
|
result (await (run/execute root home (selected-build dir id)
|
|
[(collected :case/a :fresh)]
|
|
(assoc (options fake out) :remove true)
|
|
(.-signal (js/AbortController.))))]
|
|
(t/is (fs/existsSync (get-in result [:home :record-path])))
|
|
(t/is (fs/existsSync dir))
|
|
(t/is (not (fs/existsSync (path/join dir "index.html"))))
|
|
(t/is (= :completed (get-in result [:run :termination :reason])))
|
|
(t/is (zero? (:exit-code result)))))
|
|
|
|
(t/deftest ^:async failed-final-record-write-retains-the-selected-build
|
|
(let [root @temporary-root
|
|
id "retained-after-write-failure"
|
|
dir (path/join (artifacts/managed-root root) "builds" id)
|
|
_ (fs/mkdirSync dir #js {:recursive true})
|
|
_ (fs/writeFileSync (path/join dir "index.html") "<!doctype html>")
|
|
out (output-dir)
|
|
record (path/join out "run.transit")
|
|
prior (path/join out "run.before-finish.transit")
|
|
base (fake-browser {})
|
|
close (:close (:service base))
|
|
fake (assoc base :service
|
|
(assoc (:service base) :close
|
|
(fn [owner]
|
|
(close owner)
|
|
;; Make the final atomic rename fail after all
|
|
;; attempts have reached the saved record.
|
|
(fs/renameSync record prior)
|
|
(fs/mkdirSync record))))
|
|
home (runs/create-home {:root root :run-id "failed-write"
|
|
:output-dir out :remove true})
|
|
response (await (run/execute root home (selected-build dir id)
|
|
[(collected :case/a :fresh)]
|
|
(assoc (options fake out) :remove true)
|
|
(.-signal (js/AbortController.))))
|
|
saved (result/decode (fs/readFileSync prior "utf8"))]
|
|
(t/is (= :running (get-in saved [:termination :reason])))
|
|
(t/is (= 1 (count (:attempts saved))))
|
|
(t/is (every? zero? (vals (get-in saved [:remaining :case/a]))))
|
|
(t/is (some? (:error response)))
|
|
(t/is (= 1 (:exit-code response)))
|
|
(t/is (= "<!doctype html>"
|
|
(fs/readFileSync (path/join dir "index.html") "utf8")))))
|
|
|
|
(defn- prepare-comparison-build
|
|
[{:keys [selection on-created]}]
|
|
(let [dir (build-dir)
|
|
build (-> (selected-build dir (str (random-uuid)))
|
|
(assoc :manifest-path (path/join dir "build.transit"))
|
|
(assoc-in [:manifest :functional :default-features?] false)
|
|
(assoc-in [:manifest :functional :features] (:features selection))
|
|
(assoc-in [:manifest :provenance :feature-selection] selection))]
|
|
(on-created build)
|
|
(js/Promise.resolve build)))
|
|
|
|
(t/deftest ^:async feature-comparison-completes-with-managed-output
|
|
(let [root @temporary-root
|
|
cases [(collected :case/a :fresh)]
|
|
fake (fake-browser {:metrics {:first-render-ms
|
|
(result/metric {:value 1 :unit :ms
|
|
:semantics :prepared-scene-to-full
|
|
:basis {:kind :attempt-total}
|
|
:zero-policy :positive})}})
|
|
prepared {:plan (feature-plan/plan ["a"] {:dry-run true})
|
|
:cases cases
|
|
:output-dir nil
|
|
:options (dissoc (options nil nil) :browser)}
|
|
response (await
|
|
(feature-comparison/execute
|
|
root prepared nil
|
|
{:services
|
|
{:prepare-build prepare-comparison-build
|
|
:collect-catalog (fn [_ _] cases)
|
|
:execute-run
|
|
(fn [root home selected cases opts signal]
|
|
(run/execute root home selected cases
|
|
(assoc opts :browser (:service fake)) signal))}}))
|
|
saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))]
|
|
(t/is (= 0 (:exit-code response)))
|
|
(t/is (= [:completed :completed] (mapv :status (:configurations saved))))
|
|
(doseq [configuration (:configurations saved)]
|
|
(let [run (result/decode (fs/readFileSync (get-in configuration [:run :record-path]) "utf8"))]
|
|
(t/is (= :completed (get-in run [:termination :reason])))
|
|
(t/is (= 1 (count (:attempts run))))))))
|
|
|
|
(t/deftest ^:async gpu-rejection-stops-later-matrix-configurations
|
|
(doseq [[hardware-first close-fails] [[false false] [true false] [true true]]]
|
|
(let [root @temporary-root
|
|
out (output-dir)
|
|
cases [(collected :case/a :fresh)]
|
|
software (assoc hardware-graphics :renderer "SwiftShader" :software true)
|
|
prepared {:plan (feature-plan/plan ["a"] {:dry-run true})
|
|
:cases cases
|
|
:output-dir out
|
|
:options (assoc (dissoc (options nil out) :browser)
|
|
:require-gpu true :repetitions 2)}
|
|
response (await
|
|
(feature-comparison/execute
|
|
root prepared nil
|
|
{:services
|
|
{:prepare-build prepare-comparison-build
|
|
:collect-catalog (fn [_ _] cases)
|
|
:execute-run
|
|
(fn [root home selected cases opts signal]
|
|
(let [fake (fake-browser
|
|
{:close-page-failure (when close-fails #(= % 2))
|
|
:effective-graphics
|
|
(fn [_ page-id]
|
|
(if (and hardware-first (= page-id 1))
|
|
hardware-graphics
|
|
software))})]
|
|
(run/execute root home selected cases
|
|
(assoc opts :browser (:service fake)) signal)))}}))
|
|
saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))
|
|
run (result/decode (fs/readFileSync (get-in saved [:configurations 0 :run :record-path]) "utf8"))
|
|
failure (some #(when (= :gpu (:stage %)) %)
|
|
(get-in run [:metadata :diagnostics :failures]))]
|
|
(t/is (= 1 (:exit-code response)))
|
|
(t/is (= (if hardware-first 1 0) (count (:attempts run))))
|
|
(t/is (= software (get-in failure [:partial :graphics])))
|
|
(t/is (= (if close-fails :browser-close :gpu) (get-in run [:termination :failure :stage])))
|
|
(t/is (= :failed (get-in saved [:configurations 0 :status])))
|
|
(t/is (= :gpu (get-in saved [:configurations 0 :failure :stage])))
|
|
(t/is (= :unattempted (get-in saved [:configurations 1 :status]))))))
|
|
|
|
(t/deftest ^:async comparison-stops-after-run-record-or-log-storage-fails
|
|
(doseq [filename ["run.transit" "run.log"]]
|
|
(let [executed (atom nil)
|
|
root @temporary-root
|
|
out (output-dir)
|
|
cases [(collected :case/a :fresh)]
|
|
prepared {:plan (feature-plan/plan ["a"] {:dry-run true})
|
|
:cases cases
|
|
:output-dir out
|
|
:options (dissoc (options nil out) :browser)}
|
|
response (await
|
|
(feature-comparison/execute
|
|
root prepared nil
|
|
{:services
|
|
{:prepare-build prepare-comparison-build
|
|
:collect-catalog (fn [_ _] cases)
|
|
:execute-run
|
|
(^:async fn [root home selected cases opts signal]
|
|
(let [base (fake-browser {})
|
|
close (:close (:service base))
|
|
file (path/join (:dir home) filename)
|
|
service (assoc (:service base) :close
|
|
(fn [owner]
|
|
(close owner)
|
|
(fs/renameSync file (str file ".saved"))
|
|
(fs/mkdirSync file)))
|
|
outcome (await (run/execute root home selected cases
|
|
(assoc opts :browser service) signal))]
|
|
(reset! executed outcome)
|
|
outcome))}}))
|
|
saved (result/decode (fs/readFileSync (get-in response [:home :record-path]) "utf8"))
|
|
evidence (path/join out "configs" "0"
|
|
(if (= filename "run.transit") "run.transit.saved" "run.transit"))]
|
|
(t/is (= 1 (count (:attempts (result/decode (fs/readFileSync evidence "utf8"))))))
|
|
(t/is (= 1 (:exit-code response)))
|
|
(t/is (re-find #"Case case/a" (or (:report @executed) "")))
|
|
(t/is (= :failed (get-in saved [:configurations 0 :status])))
|
|
(t/is (= :unattempted (get-in saved [:configurations 1 :status]))))))
|