From 3446dcfefb4483b7a696d8981b5b4d3fc5b7e665 Mon Sep 17 00:00:00 2001 From: Miguel de Benito Delgado Date: Mon, 9 Nov 2026 11:18:11 +0000 Subject: [PATCH] :bug: Preserve renderer benchmark failure context Retain saved-run input paths and roles in compare and merge errors. Carry bounded build identities and process details through saved diagnostics and terminal reports. Preserve browser startup context through failure wrappers and quote string paths for terminal output. Cover file reads, compiler exits, catalog identities and browser startup with behavioral tests across persistence and CLI reporting. AI-assisted-by: gpt-6.1-sol --- .../benches/render_wasm/errors.cljc | 92 ++++++--- .../render_wasm/node/cli/execution.cljs | 16 +- .../render_wasm/node/report/compare.cljs | 4 +- .../render_wasm/node/report/merge.cljs | 4 +- .../render_wasm/node/runner/browser.cljs | 16 +- .../render_wasm/node/runner/build.cljs | 10 +- .../render_wasm/node/storage/runs.cljs | 36 +++- .../render_wasm/browser_test.cljs | 23 ++- .../render_wasm/cli_integration.cjs | 190 +++++++++++++++++- .../render_wasm/errors_test.cljs | 13 ++ .../render_wasm/feature_comparison_test.cljs | 20 ++ .../render_wasm/run_test.cljs | 17 ++ .../render_wasm/runs_test.cljs | 14 ++ 13 files changed, 400 insertions(+), 55 deletions(-) diff --git a/frontend/performance/benches/render_wasm/errors.cljc b/frontend/performance/benches/render_wasm/errors.cljc index f933d207e0..6be436d783 100644 --- a/frontend/performance/benches/render_wasm/errors.cljc +++ b/frontend/performance/benches/render_wasm/errors.cljc @@ -69,18 +69,13 @@ [:cause {:optional true} [:vector schema:cause-detail]] [:partial {:optional true} map?] [:reason {:optional true} :any] - [:deadline-exceeded {:optional true} boolean?]]) - - (defn project-diagnostic - "Returns the diagnostic fields of an execution envelope. - - Args: - evidence: Plain execution evidence. - - Returns: - Failure status, location, message and available diagnostic details." - [evidence] - (select-keys evidence [:status :phase :message :cause :partial :reason :deadline-exceeded])) + [:deadline-exceeded {:optional true} boolean?] + [:expected-id {:optional true} [:string {:max 1024}]] + [:actual-id {:optional true} [:string {:max 1024}]] + [:command {:optional true} [:string {:max 1024}]] + [:exit-code {:optional true} :int] + [:signal {:optional true} [:string {:max 1024}]] + [:endpoint {:optional true} [:string {:max 1024}]]]) (def ^:private fatal-cause-depth 8) (def ^:private evidence-cause-depth 3) @@ -160,24 +155,66 @@ [data] (assoc data :status "failed")) - (defn- truncate-cause-str - "Bounds text to 500 UTF-16 units, preserving surrogate pairs. - Returns a fallback string when conversion fails." - [s] + (defn bounded-text + "Returns text within a size limit, preserving UTF-16 boundaries. + + Converts `value` to a string and limits it to max-chars UTF-16 code + units. If conversion fails, it returns 'unrenderable value'. When + truncating, we check whether the cut would leave a high surrogate + without its matching low surrogate. If so, we remove that last code unit + to keep the pair intact. + + Args: + value: Value to convert to text. + max-chars: Positive limit in UTF-16 units. + + Returns: + Bounded text, or a fallback string when conversion fails." + [value max-chars] (or (ex/ignoring - (let [text (str s)] - (if (<= (count text) max-cause-chars) + (let [text (str value)] + (if (<= (count text) max-chars) text - (let [cut (subs text 0 max-cause-chars) - last (.charCodeAt cut (dec max-cause-chars))] + (let [cut (subs text 0 max-chars) + last (.charCodeAt cut (dec max-chars))] (if (and (>= last 0xD800) (<= last 0xDBFF) - (let [nxt (.charCodeAt text max-cause-chars)] + (let [nxt (.charCodeAt text max-chars)] (and (>= nxt 0xDC00) (<= nxt 0xDFFF)))) - (subs cut 0 (dec max-cause-chars)) + (subs cut 0 (dec max-chars)) cut))))) unrenderable-value)) + (defn project-context + "Returns bounded build identity and process details. + + Args: + data: Diagnostic data with optional identity or process fields. + + Returns: + Expected/actual build IDs, command, signal and endpoint limited to + 1024 UTF-16 units, and an integer exit code when supplied." + [data] + (cond-> {} + (string? (:expected-id data)) (assoc :expected-id (bounded-text (:expected-id data) 1024)) + (string? (:actual-id data)) (assoc :actual-id (bounded-text (:actual-id data) 1024)) + (string? (:command data)) (assoc :command (bounded-text (:command data) 1024)) + (integer? (:exit-code data)) (assoc :exit-code (:exit-code data)) + (string? (:signal data)) (assoc :signal (bounded-text (:signal data) 1024)) + (string? (:endpoint data)) (assoc :endpoint (bounded-text (:endpoint data) 1024)))) + + (defn project-diagnostic + "Returns the diagnostic fields of an execution envelope. + + Args: + evidence: Plain execution evidence. + + Returns: + Failure status, location, message and available diagnostic details." + [evidence] + (merge (select-keys evidence [:status :phase :message :cause :partial :reason :deadline-exceeded]) + (project-context evidence))) + (defn- sanitize-cause-value [value] (cond @@ -185,10 +222,10 @@ value (string? value) - (truncate-cause-str value) + (bounded-text value max-cause-chars) :else - (truncate-cause-str (or (ex/ignoring (pr-str value)) unrenderable-value)))) + (bounded-text (or (ex/ignoring (pr-str value)) unrenderable-value) max-cause-chars))) (defn- describe-cause-level [cause] @@ -234,7 +271,8 @@ measurements and a bounded description of the cause." [phase cause] (let [partial (ex/ignoring (:partial (ex-data cause)))] - (cond-> {:phase (if (keyword? phase) (name phase) (str phase)) - :message (message-of cause) - :cause (describe-cause cause)} + (cond-> (merge (project-context (ex-data cause)) + {:phase (if (keyword? phase) (name phase) (str phase)) + :message (message-of cause) + :cause (describe-cause cause)}) partial (assoc :partial partial)))))) diff --git a/frontend/performance/benches/render_wasm/node/cli/execution.cljs b/frontend/performance/benches/render_wasm/node/cli/execution.cljs index 8a272d08f1..1ade131c43 100644 --- a/frontend/performance/benches/render_wasm/node/cli/execution.cljs +++ b/frontend/performance/benches/render_wasm/node/cli/execution.cljs @@ -102,18 +102,30 @@ [finished] (doseq [[index cause] (map-indexed vector (:errors finished))] (let [data (ex-data cause) + context (mapv (fn [[key value]] + [key (if (string? value) + (errors/bounded-text (js/JSON.stringify value) 1024) + value)]) + (errors/project-context data)) + path (when-let [path (:path data)] + (errors/bounded-text (if (string? path) + (js/JSON.stringify path) + (pr-str path)) + 1024)) validation (some-> (::sm/explain data) sm/simplify)] (log/error {:message (errors/message-of cause) :fields (into (filterv (comp some? second) [[:type (:type data)] [:cause-type (:cause-type data)] - [:command (:command data)] + [:path path] + [:role (:role data)] [:operation (:operation data)] [:phase (:phase data)] [:build (:build-id data)] [:reason (:reason data)] [:manifest-error (:manifest-error data)] [:baseline (:baseline data)] [:candidate (:candidate data)] [:features (:features data)] [:count (:count data)] [:max-features (:max-features data)] [:validation validation]]) - (map #(vector :difference (:field %)) (:mismatches data))) + (concat context + (map #(vector :difference (:field %)) (:mismatches data)))) :tail-log (:log-path data) :tail? (zero? index)})))) diff --git a/frontend/performance/benches/render_wasm/node/report/compare.cljs b/frontend/performance/benches/render_wasm/node/report/compare.cljs index ad25932cf5..98261f82e4 100644 --- a/frontend/performance/benches/render_wasm/node/report/compare.cljs +++ b/frontend/performance/benches/render_wasm/node/report/compare.cljs @@ -31,8 +31,8 @@ differences and formatted text." [baseline-path candidate-path {:keys [case-pair allow-build-differences allow-scene-differences allow-all-differences]}] - (let [baseline (runs/get-record baseline-path) - candidate (runs/get-record candidate-path) + (let [baseline (runs/get-record baseline-path {:role :baseline}) + candidate (runs/get-record candidate-path {:role :candidate}) comparison (result/check-record (compare/compare-runs baseline candidate diff --git a/frontend/performance/benches/render_wasm/node/report/merge.cljs b/frontend/performance/benches/render_wasm/node/report/merge.cljs index b501af36cf..d719265544 100644 --- a/frontend/performance/benches/render_wasm/node/report/merge.cljs +++ b/frontend/performance/benches/render_wasm/node/report/merge.cljs @@ -15,8 +15,8 @@ "Reads and merges two saved run files from the caller's cwd, then writes a new run home. Inputs remain unchanged. Returns the merged record and home." [root base-path additional-path {:keys [output-dir]}] - (let [base (runs/get-record base-path) - additional (runs/get-record additional-path) + (let [base (runs/get-record base-path {:role :base}) + additional (runs/get-record additional-path {:role :additional}) run-id (str (random-uuid)) merged (result/merge-runs base additional {:run-id run-id}) home (runs/create-home {:root root diff --git a/frontend/performance/benches/render_wasm/node/runner/browser.cljs b/frontend/performance/benches/render_wasm/node/runner/browser.cljs index 218fc0daba..f1792e9d70 100644 --- a/frontend/performance/benches/render_wasm/node/runner/browser.cljs +++ b/frontend/performance/benches/render_wasm/node/runner/browser.cljs @@ -132,7 +132,8 @@ (when (compare-and-set! settled false true) ((:reject @callbacks) (ex-info "Chromium exited before its CDP endpoint became ready" - {:type ::endpoint-exit :stderr @output} + (merge (errors/project-context (ex-data cause)) + {:type ::endpoint-exit :stderr @output}) cause))))})) (defn- ^:async stop-process @@ -217,7 +218,7 @@ owns the process group, user-data directory and CDP connection. It holds no page or context; use new-page for those. - Startup failures include information about endpoint and profile." + Startup failures include available process details, endpoint and profile." [{:keys [signal startup-timeout-ms deadline] :or {startup-timeout-ms settings/default-startup-timeout-ms}}] (let [deadline (or deadline (process/deadline startup-timeout-ms)) @@ -251,7 +252,8 @@ (.then running (fn [result] ((:failed endpoint) (ex-info "Chromium exited during startup" - {:type ::process-exit :exit-code (:exit-code result)}))) + {:type ::process-exit :command command + :exit-code (:exit-code result)}))) (:failed endpoint)) (reset! (:abort-listener owner) abort) (when signal @@ -271,9 +273,11 @@ (await (close owner)) (catch :default cleanup-cause (throw (ex-info "browser shutdown failed after startup error" - {:type ::startup-cleanup - :endpoint @(:address endpoint) :user-data-dir user-data-dir - :startup-failure (errors/describe-cause cause)} + (merge (errors/project-context (ex-data cause)) + (errors/project-context (ex-data cleanup-cause)) + {:type ::startup-cleanup + :endpoint @(:address endpoint) :user-data-dir user-data-dir + :startup-failure (errors/describe-cause cause)}) cleanup-cause)))) (throw (ex-info (errors/message-of cause) (assoc (ex-data cause) :endpoint @(:address endpoint) :user-data-dir user-data-dir) diff --git a/frontend/performance/benches/render_wasm/node/runner/build.cljs b/frontend/performance/benches/render_wasm/node/runner/build.cljs index d44ae1cc38..78999f1198 100644 --- a/frontend/performance/benches/render_wasm/node/runner/build.cljs +++ b/frontend/performance/benches/render_wasm/node/runner/build.cljs @@ -194,15 +194,17 @@ (artifacts/load-build root (:id ready)))) (catch :default cause (let [build @build-ref - failure {:phase @phase :cause (errors/describe-cause cause)} + context (errors/project-context (ex-data cause)) + failure (merge context {:phase @phase :cause (errors/describe-cause cause)}) saved (when (= :building (get-in build [:manifest :status])) (attempt :catch-fatal (artifacts/write-manifest build (assoc (:manifest build) :status :failed :failure failure)))) manifest-error (when (failure? saved) (errors/message-of saved)) - data (cond-> {:type ::preparation-failed :build-id (:id build) - :phase @phase :log-path (or (:log-path (ex-data cause)) (:log-path build)) - :reason (str/join " -> " (keep :message (:cause failure)))} + data (cond-> (merge context + {:type ::preparation-failed :build-id (:id build) + :phase @phase :log-path (or (:log-path (ex-data cause)) (:log-path build)) + :reason (str/join " -> " (keep :message (:cause failure)))}) manifest-error (assoc :manifest-error manifest-error))] (throw (if manifest-error (errors/fatal-error "renderer build preparation failed" data cause) diff --git a/frontend/performance/benches/render_wasm/node/storage/runs.cljs b/frontend/performance/benches/render_wasm/node/storage/runs.cljs index 5babe7cbac..1dbe608070 100644 --- a/frontend/performance/benches/render_wasm/node/storage/runs.cljs +++ b/frontend/performance/benches/render_wasm/node/storage/runs.cljs @@ -14,6 +14,7 @@ (:require ["fs" :as fs] ["path" :as path] + [benches.render-wasm.errors :as errors] [benches.render-wasm.node.files :as files] [benches.render-wasm.node.storage.artifacts :as artifacts] [benches.render-wasm.result :as result] @@ -32,14 +33,33 @@ (assoc (artifacts/create-home dir "run" "cannot create run home") :run-id run-id))) (defn get-record - "Reads and validates a saved run record from a path relative to cwd." - [file] - (when-not (and (string? file) (fs/existsSync file)) - (throw (ex-info "run record is missing" {:type ::missing-run :path file}))) - (let [record (result/decode (fs/readFileSync file "utf8"))] - (when-not (= :run-record (:type record)) - (throw (ex-info "expected a saved run record" {:type ::invalid-run :path file}))) - record)) + "Reads and validates a saved run record. + + Args: + file: Input path, resolved relative to cwd. + options: Optional :role identifying the input within an operation. + + Returns: + The validated run record. + + Throws: + An error retaining its message, type and cause, with :path, :operation + and the supplied :role in ex-data." + ([file] + (get-record file {})) + ([file {:keys [role]}] + (try + (when-not (and (string? file) (fs/existsSync file)) + (throw (ex-info "run record is missing" {:type ::missing-run}))) + (let [record (result/decode (fs/readFileSync file "utf8"))] + (when-not (= :run-record (:type record)) + (throw (ex-info "expected a saved run record" {:type ::invalid-run}))) + record) + (catch :default cause + (throw (errors/with-data + (errors/as-failure cause) + (cond-> {:path file :operation :read-record} + role (assoc :role role)))))))) (defn write-record! "Validates schema:run and atomically replaces the saved file." diff --git a/frontend/test/performance_tests/render_wasm/browser_test.cljs b/frontend/test/performance_tests/render_wasm/browser_test.cljs index eb04341ed0..8c0e0ccdc1 100644 --- a/frontend/test/performance_tests/render_wasm/browser_test.cljs +++ b/frontend/test/performance_tests/render_wasm/browser_test.cljs @@ -11,12 +11,15 @@ ["net" :as net] ["os" :as os] ["path" :as path] + ["playwright" :as playwright] [app.common.transit :as transit] + [benches.render-wasm.errors :as errors] [benches.render-wasm.identity :as identity] [benches.render-wasm.node.process :as process] [benches.render-wasm.node.runner.browser :as browser] [benches.render-wasm.node.runner.server :as server] - [cljs.test :as t :include-macros true])) + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.mock :as mock])) (def ^:private temporary-root (atom nil)) (def ^:private build-id "owned-browser-fixture") @@ -104,6 +107,24 @@ (finally (await (server/close host))))) +(t/deftest ^:async startup-process-details-survive-browser-failure-wrappers + (let [run-process process/run] + (doseq [exit-code [0 17]] + (await + (mock/with-mocks* + {process/run (fn [options] + (run-process (assoc options :command "/bin/sh" + :args ["-c" (str "exit " exit-code)])))} + (let [cause (await (rejected (browser/launch {:startup-timeout-ms 10000}))) + saved (-> (errors/evidence :browser-startup cause) + transit/encode-str + transit/decode-str) + command (if (zero? exit-code) + (.executablePath (.-chromium playwright)) + "/bin/sh")] + (t/is (= {:command command :exit-code exit-code} + (select-keys saved [:command :exit-code]))))))))) + (t/deftest ^:async successful-close-removes-the-profile-and-cdp-listener (let [host (await (open-host build-id true)) owner-ref (atom nil) diff --git a/frontend/test/performance_tests/render_wasm/cli_integration.cjs b/frontend/test/performance_tests/render_wasm/cli_integration.cjs index 0ed796f27c..8b42d79531 100644 --- a/frontend/test/performance_tests/render_wasm/cli_integration.cjs +++ b/frontend/test/performance_tests/render_wasm/cli_integration.cjs @@ -369,13 +369,15 @@ function installCargoMetadata(fixture, definitions) { return cargo; } -function installPreparedCompiler(fixture, breakLog) { +function installPreparedCompiler(fixture, breakLog, identityId = null) { const k = transit.keyword; const m = (pairs) => transit.map(pairs); const marker = "RENDER_FIXTURE_BUILD_ID"; const identity = transit .writer("json") - .write(m([k("build-id"), marker, k("cljs-profile-marks"), false])); + .write( + m([k("build-id"), identityId ?? marker, k("cljs-profile-marks"), false]), + ); const descriptor = m([ k("id"), k("fixture/load"), @@ -396,7 +398,7 @@ function installPreparedCompiler(fixture, breakLog) { ]); const catalog = transit .writer("json") - .write(m([k("build-id"), marker, k("cases"), [descriptor]])); + .write(m([k("build-id"), identityId ?? marker, k("cases"), [descriptor]])); const frontend = path.resolve(fixture.managed, "../../.."); const host = path.join( frontend, @@ -656,6 +658,8 @@ test( assert.equal(manifest.get(k("status")).toString(), ":failed"); const failure = manifest.get(k("failure")); assert.equal(failure.get(k("phase")).toString(), ":renderer"); + assert.equal(failure.get(k("exit-code")), 23); + assert.equal(failure.get(k("command")), fixture.compiler); assert.equal( failure.get(k("cause"))[0].get(k("type")).toString(), ":process-exit", @@ -678,12 +682,57 @@ test( assert.match(result.stderr, new RegExp(`build\\s+${id}`)); assert.ok(result.stderr.includes(logPath)); assert.match(result.stderr, /phase\s+:renderer/); + assert.match(result.stderr, /exit-code\s+23/); + assert.ok(result.stderr.includes(fixture.compiler), result.stderr); assert.equal(result.stderr.split("✖ ").length - 1, 1); assert.doesNotMatch(result.stderr, /^compiler (stdout|stderr)/m); assert.match(log, /✖ renderer build preparation failed/); }, ); +test( + "build retains selected and loaded identities in saved and terminal failures", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const actualId = `other\n\u001b${"x".repeat(2048)}`; + installPreparedCompiler(fixture, false, actualId); + + const result = await runCli(fixture, [ + "build", + "--no-color", + "--build-timeout-ms", + "10000", + ]); + + assert.equal(result.exitCode, 1, result.stderr); + const id = result.stdout.match(/^Build ([0-9a-f-]+)$/m)?.[1]; + assert.ok(id, result.stdout); + const k = transit.keyword; + const manifest = readManifest( + path.join(fixture.builds, id, "build.transit"), + ); + const failure = manifest.get(k("failure")); + assert.equal(manifest.get(k("status")).toString(), ":failed"); + assert.equal(failure.get(k("phase")).toString(), ":identity"); + assert.equal( + failure.get(k("cause"))[0].get(k("type")).toString(), + ":benches.render-wasm.identity/mismatch", + ); + assert.equal(failure.get(k("expected-id")), id); + assert.equal(failure.get(k("actual-id")), actualId.slice(0, 1024)); + assert.match(result.stderr, new RegExp(`expected-id\\s+"${id}"`)); + const actualField = result.stderr.match(/^ actual-id\s+(.+)$/m)?.[1]; + assert.ok(actualField, result.stderr); + assert.ok(actualField.includes("other"), actualField); + assert.ok(actualField.length <= 1024, actualField.length); + assert.ok( + [...actualField].every((character) => character.charCodeAt(0) >= 32), + actualField, + ); + }, +); + test( "build failure diagnostics retain semantic and storage causes", testOptions, @@ -1455,6 +1504,141 @@ test( }, ); +test( + "compare and merge identify unreadable saved inputs", + testOptions, + async (t) => { + const cases = [ + { + command: "compare", + role: "baseline", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "compare", + role: "candidate", + kind: "corrupt", + message: /JSON/, + type: null, + }, + { + command: "compare", + role: "baseline", + kind: "unsupported", + message: /unsupported benchmark format/, + type: /result\/unsupported-format/, + }, + { + command: "compare", + role: "candidate", + kind: "invalid", + message: /invalid schema:run/, + type: /result\/invalid-record/, + }, + { + command: "compare", + role: "candidate", + kind: "directory", + message: /EISDIR/, + type: null, + }, + { + command: "merge", + role: "base", + kind: "missing", + message: /run record is missing/, + type: /storage\.runs\/missing-run/, + }, + { + command: "merge", + role: "additional", + kind: "corrupt", + message: /JSON/, + type: null, + }, + ]; + for (const { command, role, kind, message, type } of cases) { + await t.test(`${command}: ${role} ${kind}`, async (t) => { + const fixture = createFrontend(t); + const valid = path.join(fixture.managed, "valid.transit"); + const input = path.join(fixture.managed, `${kind}.transit`); + fs.writeFileSync( + valid, + runFixtureText({ runId: "valid", caseId: "case/load", value: 10 }), + ); + if (kind === "directory") { + fs.mkdirSync(input); + } else if (kind !== "missing") { + const text = + kind === "corrupt" + ? "not JSON" + : kind === "unsupported" + ? "{}" + : transit + .writer("json") + .write( + transit.map([ + transit.keyword("type"), + transit.keyword("run-record"), + ]), + ); + fs.writeFileSync(input, text); + } + const inputs = + role === "baseline" || role === "base" + ? [input, valid] + : [valid, input]; + const result = await runCli(fixture, [ + command, + ...inputs, + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, message); + assert.ok(result.stderr.includes(input), result.stderr); + assert.match(result.stderr, new RegExp(`role\\s+:${role}\\b`)); + assert.match(result.stderr, /operation\s+:read-record\b/); + if (type) assert.match(result.stderr, type); + }); + } + }, +); + +test( + "compare bounds and escapes the reported input path", + testOptions, + async (t) => { + const fixture = createFrontend(t); + const prefix = 'missing\n"\\\u001b'; + const prefixLength = JSON.stringify(prefix).length - 1; + const input = `${prefix}${"x".repeat(1023 - prefixLength)}😀.transit`; + const result = await runCli(fixture, [ + "compare", + input, + "unused.transit", + "--no-color", + ]); + + assert.equal(result.exitCode, 1); + assert.match(result.stderr, /role\s+:baseline\b/); + const reportedPath = result.stderr.match(/^ path\s+(.+)$/m)?.[1]; + assert.ok(reportedPath, result.stderr); + assert.match(reportedPath, /^"missing\\n\\"\\\\\\u001b/); + assert.ok(reportedPath.length <= 1024, reportedPath.length); + assert.ok(reportedPath.endsWith("x"), reportedPath); + }, +); + test( "compare reads saved run files without starting resources", testOptions, diff --git a/frontend/test/performance_tests/render_wasm/errors_test.cljs b/frontend/test/performance_tests/render_wasm/errors_test.cljs index ed42f144c6..4f774da050 100644 --- a/frontend/test/performance_tests/render_wasm/errors_test.cljs +++ b/frontend/test/performance_tests/render_wasm/errors_test.cljs @@ -110,6 +110,19 @@ (t/is (= (apply str (repeat 499 "x")) (:hint outer))) (t/is (some #(= "inner boom" (:message %)) (:cause decoded))))) +(t/deftest diagnostics-retain-bounded-identity-and-process-details + (let [long-text (str (apply str (repeat 1023 "x")) "😀") + context {:expected-id long-text :actual-id "other-build" + :command "compiler" :exit-code 23 :signal "SIGTERM" + :endpoint "ws://127.0.0.1/devtools/browser/owner"} + cause (ex-info "failed" context) + decoded (-> (errors/evidence :identity cause) + errors/project-diagnostic + transit/encode-str + transit/decode-str)] + (t/is (= (assoc context :expected-id (apply str (repeat 1023 "x"))) + (select-keys decoded (keys context)))))) + (t/deftest unprintable-values-produce-portable-diagnostics (let [evil (js-obj)] (js/Object.defineProperty evil "evil" diff --git a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs index ca2d851cd7..b42a6559fb 100644 --- a/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs +++ b/frontend/test/performance_tests/render_wasm/feature_comparison_test.cljs @@ -14,6 +14,7 @@ [benches.render-wasm.declarations :as decl] [benches.render-wasm.errors :as errors] [benches.render-wasm.features.plan :as feature-plan] + [benches.render-wasm.identity :as identity] [benches.render-wasm.node.cli.execution :as execution] [benches.render-wasm.node.cli.request :as request] [benches.render-wasm.node.files :as files] @@ -947,6 +948,25 @@ (t/is (= "catalog" (get-in record [:configurations 0 :failure :phase]))) (t/is (= :unattempted (get-in record [:configurations 1 :status]))))) +(t/deftest ^:async catalog-identity-failure-retains-selected-and-loaded-build-ids + (create-frontend-root) + (let [prepared (comparison-prepared ["a"] {}) + outcome (await (feature-comparison/execute + (frontend-root) prepared nil + {:services (comparison-services + {:prepare-build (build-service {}) + :collect-catalog (fn [selected _] + (identity/check (:id selected) + {:build-id "other-build"})) + :execute-run (execute-service {})})})) + record (saved-comparison outcome) + config (first (:configurations record))] + (t/is (= 1 (:exit-code outcome))) + (t/is (= :failed (:status config))) + (t/is (= (get-in config [:build :id]) (get-in config [:failure :expected-id]))) + (t/is (= "other-build" (get-in config [:failure :actual-id]))) + (t/is (= ::identity/mismatch (get-in config [:failure :cause 0 :type]))))) + (t/deftest ^:async unacknowledged-differences-save-a-failed-edge-report (create-frontend-root) (let [prepared (comparison-prepared ["a"] {}) diff --git a/frontend/test/performance_tests/render_wasm/run_test.cljs b/frontend/test/performance_tests/render_wasm/run_test.cljs index 40b7ae8490..ecd5999951 100644 --- a/frontend/test/performance_tests/render_wasm/run_test.cljs +++ b/frontend/test/performance_tests/render_wasm/run_test.cljs @@ -443,6 +443,23 @@ (t/is (= :browser-launch (:stage (last (get-in run [:metadata :diagnostics :failures]))))) (t/is (= 1 (:exit-code result))))) +(t/deftest ^:async browser-process-details-survive-saved-run-report + (let [context {:command "chromium" :exit-code 17 :signal "SIGKILL" + :endpoint "ws://127.0.0.1:1234/devtools/browser/owner"} + fake (assoc-in (fake-browser {}) [:service :launch] + (fn [_] + (throw (ex-info "browser launch failed" context)))) + result (await (execute fake [(collected :case/a :fresh)] + (options fake (output-dir)))) + saved (saved-record result)] + (t/is (= 1 (:exit-code result))) + (t/is (= context (select-keys (get-in saved [:termination :failure]) (keys context)))) + (t/is (= context (select-keys (first (get-in saved [:metadata :diagnostics :failures])) + (keys context)))) + (t/is (str/includes? (:report result) "chromium")) + (t/is (str/includes? (:report result) "SIGKILL")) + (t/is (str/includes? (:report result) (:endpoint context))))) + (t/deftest ^:async hardware-preparations-complete-fresh-and-reuse-cases (let [fake (fake-browser {:effective-graphics (constantly hardware-graphics)}) opts (assoc (options fake (output-dir)) :require-gpu true :warmups 1 :repetitions 2) diff --git a/frontend/test/performance_tests/render_wasm/runs_test.cljs b/frontend/test/performance_tests/render_wasm/runs_test.cljs index ffcb2cf64e..a0af15f0b0 100644 --- a/frontend/test/performance_tests/render_wasm/runs_test.cljs +++ b/frontend/test/performance_tests/render_wasm/runs_test.cljs @@ -9,6 +9,7 @@ ["fs" :as fs] ["os" :as os] ["path" :as path] + [app.common.transit :as transit] [benches.render-wasm.node.storage.artifacts :as artifacts] [benches.render-wasm.node.storage.runs :as runs] [benches.render-wasm.result :as result] @@ -86,6 +87,19 @@ (t/is (= ::result/invalid-record (:type (ex-data error)))) (t/is (= run (result/decode (fs/readFileSync (:record-path home) "utf8"))))))) +(t/deftest record-read-failures-retain-file-context-and-cause + (let [file (path/join @temporary-root "invalid.transit")] + (fs/writeFileSync file (transit/encode-str (dissoc (sample-run "invalid") :plan))) + (let [error (failure #(runs/get-record file))] + (t/is (= {:path file :operation :read-record :type ::result/invalid-record} + (select-keys (ex-data error) [:path :operation :type]))) + (t/is (= ::result/invalid-record (:type (ex-data (ex-cause error))))) + (t/is (= (ex-message error) (ex-message (ex-cause error)))))) + (let [error (failure #(runs/get-record @temporary-root))] + (t/is (= {:path @temporary-root :operation :read-record} + (select-keys (ex-data error) [:path :operation]))) + (t/is (= "EISDIR" (.-code (ex-cause error)))))) + (t/deftest screenshot-paths-stay-in-the-run-home (let [home (runs/create-home {:root @temporary-root :run-id "run-5"}) shot (runs/screenshot-path home :rects/load)]