Miguel de Benito Delgado 756541939a
🔧 Add renderer benchmark runner (#12756)
Add a CLI under frontend/performance/benches/render_wasm that builds the
renderer, runs seeded scenes (rects, plus, paths, masks) in headless
Chromium and saves per-case timings.

- build, run: prepare or reuse builds; measure cases with warmups,
  repetitions, deadlines
- ab: build and compare Cargo feature configurations
- compare, merge: median changes with bootstrap CIs between saved runs,
  guarded by scene, build and settings checks
- purge: delete managed builds, runs or comparisons
- GPU runs through `run-devenv --gpu`

Lint, format and pnpm scripts now cover performance/ and its tests.

AI-assisted-by: gpt-6.1-sol, deepseek-flash, muse-spark, glm-5.3, gpt6-astra, claude-5.5-sonnet, claude-haiku-5.5
2026-11-12 11:02:34 +01:00

239 lines
10 KiB
Clojure

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns benches.render-wasm.report.summarize
"Offline analysis of raw command histories, grouped within phase executions."
(:require
[app.common.schema :as sm]
[benches.render-wasm.quantities :as quantities]
[benches.render-wasm.report.analysis :as analysis]))
(def source-coverage
"Recorded inputs and settings checked; runtime-helper identity is unverified. Git SHA and dirty state record provenance.")
(def interpretation
"Intervals are exploratory and conditional on available executions. Incomplete and timed-out cases can select which observations survive. Caches can correlate attempts; repeat whole runs to investigate findings. Full denotes renderer submission completion. Sums combine only one metric type within one phase execution. Finite clock resolution can produce valid zero durations.")
(defn case-attempts
"Returns chronological attempts for an exact saved case identifier."
[run case-id]
(filterv #(= case-id (:case-id %)) (:attempts run)))
(defn get-failed-attempts
"Returns failed warmup and measured attempts with their saved evidence."
[run case-id]
(filterv #(= :failed (:outcome %)) (case-attempts run case-id)))
(defn executions
"Returns actual phase executions with their eligibility and saved identities.
Args:
run: validated saved run.
case-id: exact saved case identifier.
phase: preparation, main or reset.
Returns:
Chronological phase histories with phase and preparation/attempt outcomes."
[run case-id phase]
(let [attempts (case-attempts run case-id)]
(if (= :preparation phase)
(let [definition (some #(when (= case-id (:id %)) %) (get-in run [:plan :cases]))]
(mapv (fn [prep]
(let [block (first (:phases prep))]
{:preparation-id (:preparation-id prep)
:events (:events block)
:warmup? (and (not= :reuse (:context definition)) (:warmup? prep))
:outcome (:outcome block)
:preparation-outcome (if (:failure prep) :failed :completed)}))
(filter #(and (= case-id (:case-id %)) (seq (:phases %))) (:preparations run))))
(into []
(keep (fn [attempt]
(when-let [block (some #(when (= phase (:phase %)) %) (:phases attempt))]
(merge (select-keys attempt [:attempt-id :preparation-id :warmup?])
{:outcome (:outcome block) :attempt-outcome (:outcome attempt)
:events (:events block)}))))
attempts))))
(defn phase-counts
"Counts actual phase executions, including empty histories and excluded work.
Args:
run: validated saved run.
case-id: exact saved case identifier.
Returns:
Execution, completion, failure and warmup counts for each phase."
[run case-id]
(into {}
(map (fn [phase]
(let [blocks (executions run case-id phase)]
[phase {:executions (count blocks)
:completed (count (filter #(= :completed (:outcome %)) blocks))
:failed (count (filter #(= :failed (:outcome %)) blocks))
:warmup (count (filter :warmup? blocks))}])))
[:preparation :main :reset]))
(defn metric-ids
"Returns [phase command-or-global metric-type] keys in first appearance order.
Args:
run: validated saved run.
case-id: exact saved case identifier.
Returns:
Distinct report keys, including quantities recorded during failed work."
[run case-id]
(vec (distinct
(for [phase [:preparation :main :reset]
block (executions run case-id phase)
event (:events block)
metric (keys (:metrics event))
scope [(:command event) :global]]
[phase scope metric]))))
(defn definitions
"Returns valid saved quantity descriptions for a report key.
Args:
run: validated saved run.
_case-id: exact saved case identifier.
id: vector of phase, command or global scope, and metric type.
Returns:
A vector containing the description, or an empty vector for invalid metadata."
[run _case-id [_phase _scope metric]]
(let [description (get-in run [:quantities metric])]
(if (sm/validate quantities/schema:description description) [description] [])))
(defn- invalid-reason
[value]
(cond
(not (js/Number.isFinite value)) :non-finite
(neg? value) :negative
:else nil))
(defn- execution-value
[block scope metric]
(let [events (->> (:events block)
(map-indexed vector)
(filter (fn [[_ event]]
(and (or (= :global scope) (= scope (:command event)))
(contains? (:metrics event) metric)))))
values (vec (mapcat #(get-in (second %) [:metrics metric]) events))
invalid (vec (for [[event-index event] events
[value-index value] (map-indexed vector (get-in event [:metrics metric]))
:let [reason (invalid-reason value)]
:when reason]
(merge (select-keys block [:attempt-id :preparation-id])
{:event-index event-index :value-index value-index
:value value :reason reason})))
total (when (and (seq values) (empty? invalid)) (reduce + 0 values))]
{:invocations (count events) :raw-values (count values)
:diagnostics (cond-> invalid
(and (some? total) (not (js/Number.isFinite total)))
(conj {:reason :non-finite-sum}))
:value total}))
(defn observations
"Sums one metric type per eligible phase execution.
Invocation and raw-value counts cover eligible executions, including invalid
values. Statistical n counts valid execution sums. Failed attempts exclude
both main and reset. Successful preparation survives later attempt failures.
Args:
run: validated saved run.
case-id: exact saved case identifier.
id: vector of phase, command or global scope, and metric type.
Returns:
Valid execution sums, accounting counts and invalidity diagnostics."
[run case-id [phase scope metric :as id]]
(let [description (get-in run [:quantities metric])
diagnostic (when (empty? (definitions run case-id id))
{:reason (if (contains? (:quantities run) metric)
:malformed-quantity :unknown-quantity)
:metric metric :description description})]
(reduce
(fn [out {:keys [warmup? outcome attempt-outcome preparation-outcome] :as block}]
(let [{:keys [value invocations raw-values diagnostics]} (execution-value block scope metric)
kind (cond
warmup? :warmup
(or (= :failed outcome) (= :failed attempt-outcome)
(= :failed preparation-outcome)) :failed
(zero? invocations) :absent
(or diagnostic (seq diagnostics)) :invalid
:else :valid)
eligible? (not (#{:warmup :failed} kind))]
(cond-> (update-in out [:counts kind] inc)
eligible? (update-in [:counts :invocations] + invocations)
eligible? (update-in [:counts :raw-values] + raw-values)
eligible? (update :diagnostics into diagnostics)
(= kind :valid) (update :values conj value))))
{:values [] :diagnostics (if diagnostic [diagnostic] [])
:counts {:valid 0 :invalid 0 :absent 0 :failed 0 :warmup 0
:invocations 0 :raw-values 0
:unattempted (if (= :preparation phase) 0
(get-in run [:remaining case-id :repetitions]))}}
(executions run case-id phase))))
(defn summarize-metric
"Computes statistics across execution sums for one phase, scope and metric type.
Args:
run: validated saved run.
case-id: exact saved case identifier.
id: vector of phase, command or global scope, and metric type.
analysis: resolved statistical settings.
Returns:
Quantity descriptions, counts, diagnostics and statistics across executions."
[run case-id id analysis]
(let [{:keys [values counts diagnostics]} (observations run case-id id)]
{:definitions (definitions run case-id id)
:statistic :sum :counts counts :diagnostics diagnostics
:statistics (analysis/statistics values analysis)}))
(defn case-report
"Reports saved case facts and phase quantities using one analysis method.
Args:
run: validated saved run.
id: exact saved case identifier.
ids: selected report keys.
analysis: resolved statistical settings.
Returns:
Case definition, raw preparation/failure evidence, counts and metric reports."
[run id ids analysis]
{:case-id id
:definition (some #(when (= id (:id %)) %) (get-in run [:plan :cases]))
:outcome (get-in run [:case-outcomes id])
:preparations (filterv #(= id (:case-id %)) (:preparations run))
:phases (phase-counts run id)
:remaining (get-in run [:remaining id])
:failed-attempts (get-failed-attempts run id)
:metrics (into {} (map (fn [metric-id]
[metric-id (summarize-metric run id metric-id analysis)])) ids)})
(defn summarize
"Reports a validated saved run using saved cases and quantity descriptions.
Args:
run: validated saved run.
overrides: optional statistical settings overriding the saved defaults.
Returns:
A report with provenance, termination and separate analyses for each case."
([run] (summarize run nil))
([run overrides]
(let [analysis (analysis/resolve-settings (merge (:analysis run) overrides))]
{:run-id (:run-id run) :metadata (:metadata run) :analysis analysis
:termination (:termination run)
:source-coverage source-coverage :interpretation interpretation
:cases (mapv (fn [{:keys [id]}]
(case-report run id (metric-ids run id) analysis))
(get-in run [:plan :cases]))})))