diff --git a/frontend/performance/benches/render_wasm/errors.cljc b/frontend/performance/benches/render_wasm/errors.cljc index f933d207e0..6be436d783 100644 --- a/frontend/performance/benches/render_wasm/errors.cljc +++ b/frontend/performance/benches/render_wasm/errors.cljc @@ -69,18 +69,13 @@ [:cause {:optional true} [:vector schema:cause-detail]] [:partial {:optional true} map?] [:reason {:optional true} :any] - [:deadline-exceeded {:optional true} boolean?]]) - - (defn project-diagnostic - "Returns the diagnostic fields of an execution envelope. - - Args: - evidence: Plain execution evidence. - - Returns: - Failure status, location, message and available diagnostic details." - [evidence] - (select-keys evidence [:status :phase :message :cause :partial :reason :deadline-exceeded])) + [:deadline-exceeded {:optional true} boolean?] + [:expected-id {:optional true} [:string {:max 1024}]] + [:actual-id {:optional true} [:string {:max 1024}]] + [:command {:optional true} [:string {:max 1024}]] + [:exit-code {:optional true} :int] + [:signal {:optional true} [:string {:max 1024}]] + [:endpoint {:optional true} [:string {:max 1024}]]]) (def ^:private fatal-cause-depth 8) (def ^:private evidence-cause-depth 3) @@ -160,24 +155,66 @@ [data] (assoc data :status "failed")) - (defn- truncate-cause-str - "Bounds text to 500 UTF-16 units, preserving surrogate pairs. - Returns a fallback string when conversion fails." - [s] + (defn bounded-text + "Returns text within a size limit, preserving UTF-16 boundaries. + + Converts `value` to a string and limits it to max-chars UTF-16 code + units. If conversion fails, it returns 'unrenderable value'. When + truncating, we check whether the cut would leave a high surrogate + without its matching low surrogate. If so, we remove that last code unit + to keep the pair intact. + + Args: + value: Value to convert to text. + max-chars: Positive limit in UTF-16 units. + + Returns: + Bounded text, or a fallback string when conversion fails." + [value max-chars] (or (ex/ignoring - (let [text (str s)] - (if (<= (count text) max-cause-chars) + (let [text (str value)] + (if (<= (count text) max-chars) text - (let [cut (subs text 0 max-cause-chars) - last (.charCodeAt cut (dec max-cause-chars))] + (let [cut (subs text 0 max-chars) + last (.charCodeAt cut (dec max-chars))] (if (and (>= last 0xD800) (<= last 0xDBFF) - (let [nxt (.charCodeAt text max-cause-chars)] + (let [nxt (.charCodeAt text max-chars)] (and (>= nxt 0xDC00) (<= nxt 0xDFFF)))) - (subs cut 0 (dec max-cause-chars)) + (subs cut 0 (dec max-chars)) cut))))) unrenderable-value)) + (defn project-context + "Returns bounded build identity and process details. + + Args: + data: Diagnostic data with optional identity or process fields. + + Returns: + Expected/actual build IDs, command, signal and endpoint limited to + 1024 UTF-16 units, and an integer exit code when supplied." + [data] + (cond-> {} + (string? (:expected-id data)) (assoc :expected-id (bounded-text (:expected-id data) 1024)) + (string? (:actual-id data)) (assoc :actual-id (bounded-text (:actual-id data) 1024)) + (string? (:command data)) (assoc :command (bounded-text (:command data) 1024)) + (integer? (:exit-code data)) (assoc :exit-code (:exit-code data)) + (string? (:signal data)) (assoc :signal (bounded-text (:signal data) 1024)) + (string? (:endpoint data)) (assoc :endpoint (bounded-text (:endpoint data) 1024)))) + + (defn project-diagnostic + "Returns the diagnostic fields of an execution envelope. + + Args: + evidence: Plain execution evidence. + + Returns: + Failure status, location, message and available diagnostic details." + [evidence] + (merge (select-keys evidence [:status :phase :message :cause :partial :reason :deadline-exceeded]) + (project-context evidence))) + (defn- sanitize-cause-value [value] (cond @@ -185,10 +222,10 @@ value (string? value) - (truncate-cause-str value) + (bounded-text value max-cause-chars) :else - (truncate-cause-str (or (ex/ignoring (pr-str value)) unrenderable-value)))) + (bounded-text (or (ex/ignoring (pr-str value)) unrenderable-value) max-cause-chars))) (defn- describe-cause-level [cause] @@ -234,7 +271,8 @@ measurements and a bounded description of the cause." [phase cause] (let [partial (ex/ignoring (:partial (ex-data cause)))] - (cond-> {:phase (if (keyword? phase) (name phase) (str phase)) - :message (message-of cause) - :cause (describe-cause cause)} + (cond-> (merge (project-context (ex-data cause)) + {:phase (if (keyword? phase) (name phase) (str phase)) + :message (message-of cause) + :cause (describe-cause cause)}) partial (assoc :partial partial)))))) diff --git a/frontend/performance/benches/render_wasm/node/cli/execution.cljs b/frontend/performance/benches/render_wasm/node/cli/execution.cljs index 8a272d08f1..1ade131c43 100644 --- a/frontend/performance/benches/render_wasm/node/cli/execution.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/execution.cljs @@ -102,18 +102,30 @@ [finished] (doseq [[index cause] (map-indexed vector (:errors finished))] (let [data (ex-data cause) + context (mapv (fn [[key value]] + [key (if (string? value) + (errors/bounded-text (js/JSON.stringify value) 1024) + value)]) + (errors/project-context data)) + path (when-let [path (:path data)] + (errors/bounded-text (if (string? path) + (js/JSON.stringify path) + (pr-str path)) + 1024)) validation (some-> (::sm/explain data) sm/simplify)] (log/error {:message (errors/message-of cause) :fields (into (filterv (comp some? second) [[:type (:type data)] [:cause-type (:cause-type data)] - [:command (:command data)] + [:path path] + [:role (:role data)] [:operation (:operation data)] [:phase (:phase data)] [:build (:build-id data)] [:reason (:reason data)] [:manifest-error (:manifest-error data)] [:baseline (:baseline data)] [:candidate (:candidate data)] [:features (:features data)] [:count (:count data)] [:max-features (:max-features data)] [:validation validation]]) - (map #(vector :difference (:field %)) (:mismatches data))) + (concat context + (map #(vector :difference (:field %)) (:mismatches data)))) :tail-log (:log-path data) :tail? (zero? index)})))) diff --git a/frontend/performance/benches/render_wasm/node/report/compare.cljs b/frontend/performance/benches/render_wasm/node/report/compare.cljs index ad25932cf5..98261f82e4 100644 --- a/frontend/performance/benches/render_wasm/node/report/compare.cljs +++ b/frontend/performance/benches/render_wasm/node/report/compare.cljs @@ -31,8 +31,8 @@ differences and formatted text." [baseline-path candidate-path {:keys [case-pair allow-build-differences allow-scene-differences allow-all-differences]}] - (let [baseline (runs/get-record baseline-path) - candidate (runs/get-record candidate-path) + (let [baseline (runs/get-record baseline-path {:role :baseline}) + candidate (runs/get-record candidate-path {:role :candidate}) comparison (result/check-record (compare/compare-runs baseline candidate diff --git a/frontend/performance/benches/render_wasm/node/report/merge.cljs b/frontend/performance/benches/render_wasm/node/report/merge.cljs index b501af36cf..d719265544 100644 --- a/frontend/performance/benches/render_wasm/node/report/merge.cljs +++ b/frontend/performance/benches/render_wasm/node/report/merge.cljs @@ -15,8 +15,8 @@ "Reads and merges two saved run files from the caller's cwd, then writes a new run home. Inputs remain unchanged. Returns the merged record and home." [root base-path additional-path {:keys [output-dir]}] - (let [base (runs/get-record base-path) - additional (runs/get-record additional-path) + (let [base (runs/get-record base-path {:role :base}) + additional (runs/get-record additional-path {:role :additional}) run-id (str (random-uuid)) merged (result/merge-runs base additional {:run-id run-id}) home (runs/create-home {:root root diff --git a/frontend/performance/benches/render_wasm/node/runner/browser.cljs b/frontend/performance/benches/render_wasm/node/runner/browser.cljs index 218fc0daba..f1792e9d70 100644 --- a/frontend/performance/benches/render_wasm/node/runner/browser.cljs +++ b/frontend/performance/benches/render_wasm/node/runner/browser.cljs @@ -132,7 +132,8 @@ (when (compare-and-set! settled false true) ((:reject @callbacks) (ex-info "Chromium exited before its CDP endpoint became ready" - {:type ::endpoint-exit :stderr @output} + (merge (errors/project-context (ex-data cause)) + {:type ::endpoint-exit :stderr @output}) cause))))})) (defn- ^:async stop-process @@ -217,7 +218,7 @@ owns the process group, user-data directory and CDP connection. It holds no page or context; use new-page for those. - Startup failures include information about endpoint and profile." + Startup failures include available process details, endpoint and profile." [{:keys [signal startup-timeout-ms deadline] :or {startup-timeout-ms settings/default-startup-timeout-ms}}] (let [deadline (or deadline (process/deadline startup-timeout-ms)) @@ -251,7 +252,8 @@ (.then running (fn [result] ((:failed endpoint) (ex-info "Chromium exited during startup" - {:type ::process-exit :exit-code (:exit-code result)}))) + {:type ::process-exit :command command + :exit-code (:exit-code result)}))) (:failed endpoint)) (reset! (:abort-listener owner) abort) (when signal @@ -271,9 +273,11 @@ (await (close owner)) (catch :default cleanup-cause (throw (ex-info "browser shutdown failed after startup error" - {:type ::startup-cleanup - :endpoint @(:address endpoint) :user-data-dir user-data-dir - :startup-failure (errors/describe-cause cause)} + (merge (errors/project-context (ex-data cause)) + (errors/project-context (ex-data cleanup-cause)) + {:type ::startup-cleanup + :endpoint @(:address endpoint) :user-data-dir user-data-dir + :startup-failure (errors/describe-cause cause)}) cleanup-cause)))) (throw (ex-info (errors/message-of cause) (assoc (ex-data cause) :endpoint @(:address endpoint) :user-data-dir user-data-dir) diff --git a/frontend/performance/benches/render_wasm/node/runner/build.cljs b/frontend/performance/benches/render_wasm/node/runner/build.cljs index d44ae1cc38..78999f1198 100644 --- a/frontend/performance/benches/render_wasm/node/runner/build.cljs +++ b/frontend/performance/benches/render_wasm/node/runner/build.cljs @@ -194,15 +194,17 @@ (artifacts/load-build root (:id ready)))) (catch :default cause (let [build @build-ref - failure {:phase @phase :cause (errors/describe-cause cause)} + context (errors/project-context (ex-data cause)) + failure (merge context {:phase @phase :cause (errors/describe-cause cause)}) saved (when (= :building (get-in build [:manifest :status])) (attempt :catch-fatal (artifacts/write-manifest build (assoc (:manifest build) :status :failed :failure failure)))) manifest-error (when (failure? saved) (errors/message-of saved)) - data (cond-> {:type ::preparation-failed :build-id (:id build) - :phase @phase :log-path (or (:log-path (ex-data cause)) (:log-path build)) - :reason (str/join " -> " (keep :message (:cause failure)))} + data (cond-> (merge context + {:type ::preparation-failed :build-id (:id build) + :phase @phase :log-path (or (:log-path (ex-data cause)) (:log-path build)) + :reason (str/join " -> " (keep :message (:cause failure)))}) manifest-error (assoc :manifest-error manifest-error))] (throw (if manifest-error (errors/fatal-error "renderer build preparation failed" data cause) diff --git a/frontend/performance/benches/render_wasm/node/storage/runs.cljs b/frontend/performance/benches/render_wasm/node/storage/runs.cljs index 5babe7cbac..1dbe608070 100644 --- a/frontend/performance/benches/render_wasm/node/storage/runs.cljs +++ b/frontend/performance/benches/render_wasm/node/storage/runs.cljs @@ -14,6 +14,7 @@ (:require ["fs" :as fs] ["path" :as path] + [benches.render-wasm.errors :as errors] [benches.render-wasm.node.files :as files] [benches.render-wasm.node.storage.artifacts :as artifacts] [benches.render-wasm.result :as result] @@ -32,14 +33,33 @@ (assoc (artifacts/create-home dir "run" "cannot create run home") :run-id run-id))) (defn get-record - "Reads and validates a saved run record from a path relative to cwd." - [file] - (when-not (and (string? file) (fs/existsSync file)) - (throw (ex-info "run record is missing" {:type ::missing-run :path file}))) - (let [record (result/decode (fs/readFileSync file "utf8"))] - (when-not (= :run-record (:type record)) - (throw (ex-info "expected a saved run record" {:type ::invalid-run :path file}))) - record)) + "Reads and validates a saved run record. + + Args: + file: Input path, resolved relative to cwd. + options: Optional :role identifying the input within an operation. + + Returns: + The validated run record. + + Throws: + An error retaining its message, type and cause, with :path, :operation + and the supplied :role in ex-data." + ([file] + (get-record file {})) + ([file {:keys [role]}] + (try + (when-not (and (string? file) (fs/existsSync file)) + (throw (ex-info "run record is missing" {:type ::missing-run}))) + (let [record (result/decode (fs/readFileSync file "utf8"))] + (when-not (= :run-record (:type record)) + (throw (ex-info "expected a saved run record" {:type ::invalid-run}))) + record) + (catch :default cause + (throw (errors/with-data + (errors/as-failure cause) + (cond-> {:path file :operation :read-record} + role (assoc :role role)))))))) (defn write-record! "Validates schema:run and atomically replaces the saved file." diff --git a/frontend/test/performance_tests/render_wasm/browser_test.cljs b/frontend/test/performance_tests/render_wasm/browser_test.cljs index eb04341ed0..8c0e0ccdc1 100644 --- a/frontend/test/performance_tests/render_wasm/browser_test.cljs +++ b/frontend/test/performance_tests/render_wasm/browser_test.cljs @@ -11,12 +11,15 @@ ["net" :as net] ["os" :as os] ["path" :as path] + ["playwright" :as playwright] [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] [benches.render-wasm.identity :as identity] [benches.render-wasm.node.process :as process] [benches.render-wasm.node.runner.browser :as browser] [benches.render-wasm.node.runner.server :as server] - [cljs.test :as t :include-macros true])) + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock])) (def ^:private temporary-root (atom nil)) (def ^:private build-id "owned-browser-fixture") @@ -104,6 +107,24 @@ (finally (await (server/close host))))) +(t/deftest ^:async startup-process-details-survive-browser-failure-wrappers + (let [run-process process/run] + (doseq [exit-code [0 17]] + (await + (mock/with-mocks* + {process/run (fn [options] + (run-process (assoc options :command "/bin/sh" + :args ["-c" (str "exit " exit-code)])))} + (let [cause (await (rejected (browser/launch {:startup-timeout-ms 10000}))) + saved (-> (errors/evidence :browser-startup cause) + transit/encode-str + transit/decode-str) + command (if (zero? exit-code) + (.executablePath (.-chromium playwright)) + "/bin/sh")] + (t/is (= {:command command :exit-code exit-code} + (select-keys saved [:command :exit-code]))))))))) + (t/deftest ^:async successful-close-removes-the-profile-and-cdp-listener (let [host (await (open-host build-id true)) owner-ref (atom nil) diff --git a/frontend/test/performance_tests/render_wasm/cli_integration.cjs b/frontend/test/performance_tests/render_wasm/cli_integration.cjs index 0ed796f27c..8b42d79531 100644 --- a/frontend/test/performance_tests/render_wasm/cli_integration.cjs +++ b/frontend/test/performance_tests/render_wasm/cli_integration.cjs @@ -369,13 +369,15 @@ function installCargoMetadata(fixture, definitions) { return cargo; } -function installPreparedCompiler(fixture, breakLog) { +function installPreparedCompiler(fixture, breakLog, identityId = null) { const k = transit.keyword; const m = (pairs) => transit.map(pairs); const marker = "RENDER_FIXTURE_BUILD_ID"; const identity = transit .writer("json") - .write(m([k("build-id"), marker, k("cljs-profile-marks"), false])); + .write( + m([k("build-id"), identityId ?? marker, k("cljs-profile-marks"), false]), + ); const descriptor = m([ k("id"), k("fixture/load"), @@ -396,7 +398,7 @@ function installPreparedCompiler(fixture, breakLog) { ]); const catalog = transit .writer("json") - .write(m([k("build-id"), marker, k("cases"), [descriptor]])); + .write(m([k("build-id"), identityId ?? marker, k("cases"), [descriptor]])); const frontend = path.resolve(fixture.managed, "../../.."); const host = path.join( frontend, @@ -656,6 +658,8 @@ test( assert.equal(manifest.get(k("status")).toString(), ":failed"); const failure = manifest.get(k("failure")); assert.equal(failure.get(k("phase")).toString(), ":renderer"); + assert.equal(failure.get(k("exit-code")), 23); + assert.equal(failure.get(k("command")), fixture.compiler); assert.equal( failure.get(k("cause"))[0].get(k("type")).toString(), ":process-exit", @@ -678,12 +682,57 @@ test( assert.match(result.stderr, new RegExp(`build\\s+${id}`)); assert.ok(result.stderr.includes(logPath)); assert.match(result.stderr, /phase\s+:renderer/); + assert.match(result.stderr, /exit-code\s+23/); + assert.ok(result.stderr.includes(fixture.compiler), result.stderr); assert.equal(result.stderr.split("✖ ").length - 1, 1); assert.doesNotMatch(result.stderr, /^compiler (stdout|stderr)/m); assert.match(log, /✖ renderer build preparation failed/); }, ); +test( + "build retains selected and loaded identities in saved and terminal failures", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const actualId = `other\n\u001b${"x".repeat(2048)}`; + installPreparedCompiler(fixture, false, actualId); + + const result = await runCli(fixture, [ + "build", + "--no-color", + "--build-timeout-ms", + "10000", + ]); + + assert.equal(result.exitCode, 1, result.stderr); + const id = result.stdout.match(/^Build ([0-9a-f-]+)$/m)?.[1]; + assert.ok(id, result.stdout); + const k = transit.keyword; + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + const failure = manifest.get(k("failure")); + assert.equal(manifest.get(k("status")).toString(), ":failed"); + assert.equal(failure.get(k("phase")).toString(), ":identity"); + assert.equal( + failure.get(k("cause"))[0].get(k("type")).toString(), + ":benches.render-wasm.identity/mismatch", + ); + assert.equal(failure.get(k("expected-id")), id); + assert.equal(failure.get(k("actual-id")), actualId.slice(0, 1024)); + assert.match(result.stderr, new RegExp(`expected-id\\s+"${id}"`)); + const actualField = result.stderr.match(/^ actual-id\s+(.+)$/m)?.[1]; + assert.ok(actualField, result.stderr); + assert.ok(actualField.includes("other"), actualField); + assert.ok(actualField.length <= 1024, actualField.length); + assert.ok( + [...actualField].every((character) => character.charCodeAt(0) >= 32), + actualField, + ); + }, +); + test( "build failure diagnostics retain semantic and storage causes", testOptions, @@ -1455,6 +1504,141 @@ test( }, ); +test( + "compare and merge identify unreadable saved inputs", + testOptions, + async (t) => { + const cases = [ + { + command: "compare", + role: "baseline", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "corrupt", + message: /JSON/, + type: null, + }, + { + command: "compare", + role: "baseline", + kind: "unsupported", + message: /unsupported benchmark format/, + type: /result\/unsupported-format/, + }, + { + command: "compare", + role: "candidate", + kind: "invalid", + message: /invalid schema:run/, + type: /result\/invalid-record/, + }, + { + command: "compare", + role: "candidate", + kind: "directory", + message: /EISDIR/, + type: null, + }, + { + command: "merge", + role: "base", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "merge", + role: "additional", + kind: "corrupt", + message: /JSON/, + type: null, + }, + ]; + for (const { command, role, kind, message, type } of cases) { + await t.test(`${command}: ${role} ${kind}`, async (t) => { + const fixture = createFrontend(t); + const valid = path.join(fixture.managed, "valid.transit"); + const input = path.join(fixture.managed, `${kind}.transit`); + fs.writeFileSync( + valid, + runFixtureText({ runId: "valid", caseId: "case/load", value: 10 }), + ); + if (kind === "directory") { + fs.mkdirSync(input); + } else if (kind !== "missing") { + const text = + kind === "corrupt" + ? "not JSON" + : kind === "unsupported" + ? "{}" + : transit + .writer("json") + .write( + transit.map([ + transit.keyword("type"), + transit.keyword("run-record"), + ]), + ); + fs.writeFileSync(input, text); + } + const inputs = + role === "baseline" || role === "base" + ? [input, valid] + : [valid, input]; + const result = await runCli(fixture, [ + command, + ...inputs, + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, message); + assert.ok(result.stderr.includes(input), result.stderr); + assert.match(result.stderr, new RegExp(`role\\s+:${role}\\b`)); + assert.match(result.stderr, /operation\s+:read-record\b/); + if (type) assert.match(result.stderr, type); + }); + } + }, +); + +test( + "compare bounds and escapes the reported input path", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const prefix = 'missing\n"\\\u001b'; + const prefixLength = JSON.stringify(prefix).length - 1; + const input = `${prefix}${"x".repeat(1023 - prefixLength)}😀.transit`; + const result = await runCli(fixture, [ + "compare", + input, + "unused.transit", + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /role\s+:baseline\b/); + const reportedPath = result.stderr.match(/^ path\s+(.+)$/m)?.[1]; + assert.ok(reportedPath, result.stderr); + assert.match(reportedPath, /^"missing\\n\\"\\\\\\u001b/); + assert.ok(reportedPath.length <= 1024, reportedPath.length); + assert.ok(reportedPath.endsWith("x"), reportedPath); + }, +); + test( "compare reads saved run files without starting resources", testOptions, diff --git a/frontend/test/performance_tests/render_wasm/errors_test.cljs b/frontend/test/performance_tests/render_wasm/errors_test.cljs index ed42f144c6..4f774da050 100644 --- a/frontend/test/performance_tests/render_wasm/errors_test.cljs +++ b/frontend/test/performance_tests/render_wasm/errors_test.cljs @@ -110,6 +110,19 @@ (t/is (= (apply str (repeat 499 "x")) (:hint outer))) (t/is (some #(= "inner boom" (:message %)) (:cause decoded))))) +(t/deftest diagnostics-retain-bounded-identity-and-process-details + (let [long-text (str (apply str (repeat 1023 "x")) "😀") + context {:expected-id long-text :actual-id "other-build" + :command "compiler" :exit-code 23 :signal "SIGTERM" + :endpoint "ws://127.0.0.1/devtools/browser/owner"} + cause (ex-info "failed" context) + decoded (-> (errors/evidence :identity cause) + errors/project-diagnostic + transit/encode-str + transit/decode-str)] + (t/is (= (assoc context :expected-id (apply str (repeat 1023 "x"))) + (select-keys decoded (keys context)))))) + (t/deftest unprintable-values-produce-portable-diagnostics (let [evil (js-obj)] (js/Object.defineProperty evil "evil" diff --git a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs index ca2d851cd7..b42a6559fb 100644 --- a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs +++ b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs @@ -14,6 +14,7 @@ [benches.render-wasm.declarations :as decl] [benches.render-wasm.errors :as errors] [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.identity :as identity] [benches.render-wasm.node.cli.execution :as execution] [benches.render-wasm.node.cli.request :as request] [benches.render-wasm.node.files :as files] @@ -947,6 +948,25 @@ (t/is (= "catalog" (get-in record [:configurations 0 :failure :phase]))) (t/is (= :unattempted (get-in record [:configurations 1 :status]))))) +(t/deftest ^:async catalog-identity-failure-retains-selected-and-loaded-build-ids + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :collect-catalog (fn [selected _] + (identity/check (:id selected) + {:build-id "other-build"})) + :execute-run (execute-service {})})})) + record (saved-comparison outcome) + config (first (:configurations record))] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status config))) + (t/is (= (get-in config [:build :id]) (get-in config [:failure :expected-id]))) + (t/is (= "other-build" (get-in config [:failure :actual-id]))) + (t/is (= ::identity/mismatch (get-in config [:failure :cause 0 :type]))))) + (t/deftest ^:async unacknowledged-differences-save-a-failed-edge-report (create-frontend-root) (let [prepared (comparison-prepared ["a"] {}) diff --git a/frontend/test/performance_tests/render_wasm/run_test.cljs b/frontend/test/performance_tests/render_wasm/run_test.cljs index 40b7ae8490..ecd5999951 100644 --- a/frontend/test/performance_tests/render_wasm/run_test.cljs +++ b/frontend/test/performance_tests/render_wasm/run_test.cljs @@ -443,6 +443,23 @@ (t/is (= :browser-launch (:stage (last (get-in run [:metadata :diagnostics :failures]))))) (t/is (= 1 (:exit-code result))))) +(t/deftest ^:async browser-process-details-survive-saved-run-report + (let [context {:command "chromium" :exit-code 17 :signal "SIGKILL" + :endpoint "ws://127.0.0.1:1234/devtools/browser/owner"} + fake (assoc-in (fake-browser {}) [:service :launch] + (fn [_] + (throw (ex-info "browser launch failed" context)))) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + saved (saved-record result)] + (t/is (= 1 (:exit-code result))) + (t/is (= context (select-keys (get-in saved [:termination :failure]) (keys context)))) + (t/is (= context (select-keys (first (get-in saved [:metadata :diagnostics :failures])) + (keys context)))) + (t/is (str/includes? (:report result) "chromium")) + (t/is (str/includes? (:report result) "SIGKILL")) + (t/is (str/includes? (:report result) (:endpoint context))))) + (t/deftest ^:async hardware-preparations-complete-fresh-and-reuse-cases (let [fake (fake-browser {:effective-graphics (constantly hardware-graphics)}) opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1 :repetitions 2) diff --git a/frontend/test/performance_tests/render_wasm/runs_test.cljs b/frontend/test/performance_tests/render_wasm/runs_test.cljs index ffcb2cf64e..a0af15f0b0 100644 --- a/frontend/test/performance_tests/render_wasm/runs_test.cljs +++ b/frontend/test/performance_tests/render_wasm/runs_test.cljs @@ -9,6 +9,7 @@ ["fs" :as fs] ["os" :as os] ["path" :as path] + [app.common.transit :as transit] [benches.render-wasm.node.storage.artifacts :as artifacts] [benches.render-wasm.node.storage.runs :as runs] [benches.render-wasm.result :as result] @@ -86,6 +87,19 @@ (t/is (= ::result/invalid-record (:type (ex-data error)))) (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8"))))))) +(t/deftest record-read-failures-retain-file-context-and-cause + (let [file (path/join @temporary-root "invalid.transit")] + (fs/writeFileSync file (transit/encode-str (dissoc (sample-run "invalid") :plan))) + (let [error (failure #(runs/get-record file))] + (t/is (= {:path file :operation :read-record :type ::result/invalid-record} + (select-keys (ex-data error) [:path :operation :type]))) + (t/is (= ::result/invalid-record (:type (ex-data (ex-cause error))))) + (t/is (= (ex-message error) (ex-message (ex-cause error)))))) + (let [error (failure #(runs/get-record @temporary-root))] + (t/is (= {:path @temporary-root :operation :read-record} + (select-keys (ex-data error) [:path :operation]))) + (t/is (= "EISDIR" (.-code (ex-cause error)))))) + (t/deftest screenshot-paths-stay-in-the-run-home (let [home (runs/create-home {:root @temporary-root :run-id "run-5"}) shot (runs/screenshot-path home :rects/load)]