✨ Implement run merging

AI-assisted-by: deepseek-flash
This commit is contained in:
Miguel de Benito Delgado 2026-10-31 12:12:01 +01:00
parent 0b19800955
commit 1af0e23b96
8 changed files with 610 additions and 32 deletions

View File

@ -871,6 +871,22 @@ pnpm run render-bench -- compare \
--cases rects/load,paths/load --allow-scene-differences
```
Merge an additional saved run into a base run:
```bash
pnpm run render-bench -- merge \
target/performance/render-wasm/runs/<base>/run.transit \
target/performance/render-wasm/runs/<additional>/run.transit \
--output-dir target/performance/render-wasm/runs/<merged>
```
The merged record gets a new run id and a fresh run home. Both inputs must be
completed and compatible: same cases, timeout policy, analysis, build facts,
runtime environment, and Git provenance. The additional run must use the same
warmup count. `merge` appends only the additional measured attempts; its warmup
attempts remain in the additional record. The command starts no build, server,
or browser.
Inspect and clean managed outputs:
```bash
@ -891,6 +907,7 @@ The commands write results to stdout:
- `ab`: `Comparison <id>`, `Output <absolute path>`, and final
`Status <status>`.
- `compare`: the comparison report.
- `merge`: `Run <id>` and `Output <absolute path>`.
- Help: usage text. `purge` produces no output on success.
Progress and diagnostics go to stderr. The default shows build phases, case
@ -912,10 +929,11 @@ Each build writes `build.log`; each A/B comparison writes
`run.transit`, and optional screenshots.
Exit 0 means the command completed successfully: a ready build for `build`,
every allocated attempt for `run`, a valid comparison for `compare`, every
configuration and edge for `ab`, and a clean purge for `purge`. Exit 1 means a
failed preparation, a failed attempt, an incomplete run, an invalid selection,
an unacknowledged comparison difference, or a harness failure. Exits 130 and 143
every allocated attempt for `run`, a valid comparison for `compare`, a
validated merged record for `merge`, every configuration and edge for `ab`, and
a clean purge for `purge`. Exit 1 means a failed preparation, a failed attempt,
an incomplete run, an invalid selection, incompatible merge inputs, an
unacknowledged comparison difference, or a harness failure. Exits 130 and 143
mean interruption by `SIGINT` and `SIGTERM` for `build`, `run` and `ab`. An
interrupted command prints no error for the aborted work; it prints a failed
write of its own record or log.

View File

@ -0,0 +1,31 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns benches.render-wasm.node.merge
"Offline merge of two saved schema:run records."
(:require
["fs" :as fs]
[benches.render-wasm.node.log :as log]
[benches.render-wasm.node.runs :as runs]
[benches.render-wasm.result :as result]))
(defn execute
"Reads and merges two saved run files from the caller's cwd, then writes a
new run home. Inputs remain unchanged. Returns the merged record and home."
[root base-path additional-path {:keys [output-dir]}]
(let [base (runs/get-record base-path)
additional (runs/get-record additional-path)
run-id (str (random-uuid))
merged (result/merge-runs base additional {:run-id run-id})
home (runs/create-home {:root root
:run-id run-id
:output-dir output-dir})]
(runs/write-record! home merged)
(fs/appendFileSync (:log-path home)
(str "merge " (:run-id base) " + " (:run-id additional) "\n"))
(log/result (str "Run " run-id))
(log/result (str "Output " (:dir home)))
{:run merged :home home}))

View File

@ -11,8 +11,10 @@
[app.common.transit :as transit]
[benches.render-wasm.errors :as errors :refer [attempt failure?]]
[benches.render-wasm.feature-plan :as feature-plan]
[benches.render-wasm.node.artifacts :as artifacts]
[benches.render-wasm.node.compare :as compare]
[benches.render-wasm.node.log :as log]
[benches.render-wasm.node.merge :as run-merge]
[benches.render-wasm.node.options :as options]
[clojure.string :as str]
[clojure.tools.cli :refer [parse-opts]]))
@ -98,7 +100,8 @@
" [--confirm] [--dry-run] [run and comparison options]\n"
" runner compare BASELINE CANDIDATE [--cases A,B]\n"
" [--allow-build-differences] [--allow-scene-differences]\n"
" [--allow-all-differences]\n\n"
" [--allow-all-differences]\n"
" runner merge BASE ADDITIONAL [--output-dir DIR]\n\n"
"build prints Build <id> and Output <absolute path> on stdout. Exit 0 means\n"
"ready; errors retain the build.transit and build.log for that ID.\n"
"run prints Run <id>, Output <absolute path> and Report on stdout.\n"
@ -111,7 +114,10 @@
"to require hardware WebGL2 in preparation metadata. ab --dry-run skips it.\n"
"compare reads saved schema:run records and starts no build, host or browser.\n"
"Build and scene permissions combine; operation and runtime checks stay strict.\n"
"All-differences permits every compatibility difference. Reports retain them.\n\n"
"All-differences permits every compatibility difference. Reports retain them.\n"
"merge reads two saved schema:run records and writes a merged record in a fresh\n"
"run home. It appends the additional run's measured attempts, keeps the base\n"
"warmups, and requires compatible completed runs. It starts no build or browser.\n\n"
"ab uses every on/off configuration with Cargo defaults disabled; declared\n"
"defaults must be empty. --max-features defaults to 3 (range 1..6).\n"
"Use --confirm above eight configurations or with an empty filter.\n"
@ -227,7 +233,15 @@
(do
(when purge? (invalid-args "purge category flags require purge"))
(when build? (invalid-args "build options require build"))
(when run? (invalid-args "run options require run"))))))
(when run? (invalid-args "run options require run")))
"merge"
(do
(when purge? (invalid-args "purge category flags require purge"))
(when build? (invalid-args "build options require build"))
(when compare? (invalid-args "compare options require compare"))
(when (some-flags? flags (disj run-flags "--output-dir"))
(invalid-args "run options require run"))))))
(defn- check-arguments!
[command arguments]
@ -237,6 +251,10 @@
(when-not (= 2 (count extra))
(invalid-args "compare requires BASELINE and CANDIDATE paths"))
"merge"
(when-not (= 2 (count extra))
(invalid-args "merge requires BASE and ADDITIONAL paths"))
(when (seq extra)
(invalid-args (str command " takes no positional arguments"))))))
@ -254,8 +272,8 @@
(invalid-args "--quiet and --verbose are mutually exclusive"))
(when-not (:help options)
(when-not (and (= 1 (count (take 1 arguments)))
(contains? #{"build" "purge" "run" "ab" "compare"} command))
(invalid-args "expected build, purge, run, ab or compare; try --help"))
(contains? #{"build" "purge" "run" "ab" "compare" "merge"} command))
(invalid-args "expected build, purge, run, ab, compare or merge; try --help"))
(check-command-flags! command flags)
(check-arguments! command arguments)
(when (and (= command "purge")
@ -290,6 +308,12 @@
:allow-scene-differences
:allow-all-differences]))))
(= command "merge")
(run-merge/execute (artifacts/frontend-root)
(second arguments)
(nth arguments 2)
(select-keys options [:output-dir]))
:else
(let [^js commands (js/require (path/join js/__dirname "commands.cjs"))]
(set! (.-exitCode js/process)

View File

@ -243,6 +243,23 @@
(when-not valid?
(throw (ex-info message (assoc data :type ::invalid-record)))))
(defn- incompatible!
"Rejects runs whose recorded evidence cannot be merged."
[message data]
(throw (ex-info message (assoc data :type ::incompatible-runs))))
(defn- require-compatible!
"Requires equal values at `path`, naming the disagreeing field."
[base additional path label]
(let [base-value (get-in base path)
additional-value (get-in additional path)]
(when-not (= base-value additional-value)
(incompatible! (str "cannot merge runs with different " label)
{:field label
:path path
:base base-value
:additional additional-value}))))
(defn invalid-reason
"Classifies one raw measurement without changing its value.
Nil denotes an explicit missing value; an absent metric has no entry.
@ -382,6 +399,88 @@
(check-completion run termination)
(assoc run :ended-at ended-at :termination termination))
(defn merge-runs
"Returns a new validated run that appends `additional`'s measured attempts to
`base`.
Both runs must be completed and compatible. The merged plan keeps `base`'s
warmups and adds `additional`'s measured repetitions. Warmup attempts
and preparations used only by them are excluded from `additional`.
The caller supplies the merged run id and may supply its end time."
[base additional {:keys [run-id ended-at]}]
(let [base (check-run base)
additional (check-run additional)]
(require-valid! (and (string? run-id) (seq run-id))
"merged run needs a run id" {})
(doseq [[side run] [[:base base] [:additional additional]]]
(when-not (= :completed (get-in run [:termination :reason]))
(incompatible! "only completed runs can be merged"
{:side side
:run-id (:run-id run)
:reason (get-in run [:termination :reason])})))
(when-not (and (string? (:ended-at base))
(string? (:ended-at additional)))
(incompatible! "completed runs need end times"
{:base (:ended-at base)
:additional (:ended-at additional)}))
(doseq [[path label] [[[:plan :cases] "case definitions"]
[[:plan :timeout-policy] "timeout policy"]
[[:plan :warmups] "warmup count"]
[[:analysis] "analysis settings"]
[[:metadata :git] "git provenance"]
[[:metadata :build] "build facts"]
[[:metadata :environment] "runtime environment"]
[[:metadata :scored?] "scored flag"]]]
(require-compatible! base additional path label))
(let [base-attempts (:attempts base)
additional-measured (filterv #(not (:warmup? %)) (:attempts additional))
base-preparations (:preparations base)
base-preparation-ids (into #{} (map :preparation-id) base-preparations)
used-preparation-ids (into #{} (keep :preparation-id) additional-measured)
additional-preparations (filterv #(contains? used-preparation-ids
(:preparation-id %))
(:preparations additional))]
(when-let [duplicate (some #(contains? base-preparation-ids (:preparation-id %))
additional-preparations)]
(incompatible! "cannot merge runs with duplicate preparation ids"
{:preparation-id (:preparation-id duplicate)}))
(let [offset (count base-attempts)
attempts (into (vec base-attempts)
(map-indexed (fn [index attempt]
(assoc attempt :ordinal (+ offset index)))
additional-measured))
plan (assoc (:plan base)
:repetitions (+ (get-in base [:plan :repetitions])
(get-in additional [:plan :repetitions])))
remaining (reduce consume-slot (initial-remaining plan) attempts)
ended-at (or ended-at
(let [base-end (:ended-at base)
additional-end (:ended-at additional)]
(if (neg? (compare base-end additional-end))
additional-end
base-end)))
metadata (update (:metadata base) :diagnostics
(fn [diagnostics]
(assoc (or diagnostics {})
:merge
{:base-run-id (:run-id base)
:additional-run-id (:run-id additional)
:additional-warmups-excluded
(count (filter :warmup? (:attempts additional)))
:merged-at ended-at})))]
(check-run {:type :run-record
:run-id run-id
:started-at (:started-at base)
:ended-at ended-at
:plan plan
:metadata metadata
:analysis (:analysis base)
:preparations (into (vec base-preparations) additional-preparations)
:attempts attempts
:remaining remaining
:termination {:reason :completed}})))))
(defn- check-feature-configuration
"Requires evidence that matches each saved configuration's progress."
[{:keys [status build run effective-features failure reason]}]

View File

@ -319,6 +319,9 @@ function runFixtureText({
fingerprint = "a".repeat(64),
operation = "load",
operationArgs = { distance: 10 },
warmups = 0,
repetitions = 1,
gitSha = runId,
}) {
const k = transit.keyword;
const map = (pairs) => transit.map(pairs);
@ -348,22 +351,29 @@ function runFixtureText({
k("diagnostics"),
map([]),
]);
const attempt = map([
k("ordinal"),
0,
k("case-id"),
k(caseId),
k("preparation-id"),
`${runId}-preparation`,
k("warmup?"),
false,
k("outcome"),
k("completed"),
k("metrics"),
map([k("elapsed"), metric]),
k("trace"),
map([]),
]);
const makeAttempt = (ordinal, warmup) =>
map([
k("ordinal"),
ordinal,
k("case-id"),
k(caseId),
k("preparation-id"),
`${runId}-preparation`,
k("warmup?"),
warmup,
k("outcome"),
k("completed"),
k("metrics"),
map([k("elapsed"), metric]),
k("trace"),
map([]),
]);
const attempts = [
...Array.from({ length: warmups }, (_, index) => makeAttempt(index, true)),
...Array.from({ length: repetitions }, (_, index) =>
makeAttempt(warmups + index, false),
),
];
const record = map([
k("type"),
k("run-record"),
@ -401,16 +411,16 @@ function runFixtureText({
]),
],
k("warmups"),
0,
warmups,
k("repetitions"),
1,
repetitions,
k("timeout-policy"),
map([k("attempt-ms"), 30000]),
]),
k("metadata"),
map([
k("git"),
map([k("sha"), runId, k("dirty"), false]),
map([k("sha"), gitSha, k("dirty"), false]),
k("build"),
map([
k("functional"),
@ -443,7 +453,7 @@ function runFixtureText({
k("preparations"),
[preparation],
k("attempts"),
[attempt],
attempts,
k("remaining"),
map([k(caseId), map([k("warmups"), 0, k("repetitions"), 0])]),
k("termination"),
@ -1252,10 +1262,14 @@ test(
const help = await runCli(fixture, ["--help"]);
assert.equal(help.exitCode, 0, help.stderr);
assert.match(help.stdout, /runner compare/);
assert.match(help.stdout, /runner merge/);
const abHelp = await runCli(fixture, ["ab", "--help"]);
assert.equal(abHelp.exitCode, 0, abHelp.stderr);
assert.match(abHelp.stdout, /runner ab --features/);
assert.match(abHelp.stdout, /--only-comparisons/);
const mergeHelp = await runCli(fixture, ["merge", "--help"]);
assert.equal(mergeHelp.exitCode, 0, mergeHelp.stderr);
assert.match(mergeHelp.stdout, /runner merge BASE ADDITIONAL/);
for (const args of [["ab"], ["ab", "--dry-run"]]) {
const invalid = await runCli(fixture, args);
assert.equal(invalid.exitCode, 1, invalid.stderr);
@ -1298,6 +1312,7 @@ test(
assert.equal(result.exitCode, 0);
assert.match(result.stdout, /runner run/);
assert.match(result.stdout, /runner compare/);
assert.match(result.stdout, /runner merge/);
assert.match(result.stdout, /--cases/);
assert.match(result.stdout, /--allow-build-differences/);
assert.match(result.stdout, /--allow-scene-differences/);
@ -1557,3 +1572,162 @@ test("compare rejects --force without an alias", testOptions, async (t) => {
assert.deepEqual(tree(fixture.managed), original);
}
});
test(
"merge writes a validated run home with appended measured attempts",
testOptions,
async (t) => {
const fixture = createFrontend(t);
const base = path.join(fixture.managed, "base.transit");
const additional = path.join(fixture.managed, "additional.transit");
const baseText = runFixtureText({
runId: "base",
caseId: "case/load",
value: 10,
warmups: 1,
repetitions: 2,
gitSha: "same-sha",
});
const additionalText = runFixtureText({
runId: "additional",
caseId: "case/load",
value: 12,
warmups: 1,
repetitions: 3,
gitSha: "same-sha",
});
fs.writeFileSync(base, baseText);
fs.writeFileSync(additional, additionalText);
const output = path.join(fixture.managed, "merged-run");
const result = await runCli(fixture, [
"merge",
base,
additional,
"--output-dir",
output,
]);
assert.equal(result.exitCode, 0, result.stderr);
const runId = result.stdout.match(/Run ([0-9a-f-]{36})/)[1];
assert.ok(fs.existsSync(path.join(output, "run.transit")));
assert.equal(fs.readFileSync(base, "utf8"), baseText);
assert.equal(fs.readFileSync(additional, "utf8"), additionalText);
const k = transit.keyword;
const merged = readManifest(path.join(output, "run.transit"));
const plan = merged.get(k("plan"));
const attempts = merged.get(k("attempts"));
const remaining = merged.get(k("remaining")).get(k("case/load"));
const mergeInfo = merged
.get(k("metadata"))
.get(k("diagnostics"))
.get(k("merge"));
assert.equal(merged.get(k("run-id")), runId);
assert.equal(plan.get(k("warmups")), 1);
assert.equal(plan.get(k("repetitions")), 5);
assert.deepEqual(
attempts.map((attempt) => attempt.get(k("ordinal"))),
[0, 1, 2, 3, 4, 5],
);
assert.deepEqual(
attempts.map((attempt) => attempt.get(k("warmup?"))),
[true, false, false, false, false, false],
);
assert.equal(remaining.get(k("warmups")), 0);
assert.equal(remaining.get(k("repetitions")), 0);
assert.equal(mergeInfo.get(k("base-run-id")), "base");
assert.equal(mergeInfo.get(k("additional-run-id")), "additional");
assert.equal(mergeInfo.get(k("additional-warmups-excluded")), 1);
assert.match(
fs.readFileSync(path.join(output, "run.log"), "utf8"),
/merge base \+ additional/,
);
},
);
test(
"merge rejects incompatible runs before creating output",
testOptions,
async (t) => {
const fixture = createFrontend(t);
const base = path.join(fixture.managed, "base.transit");
const additional = path.join(fixture.managed, "additional.transit");
fs.writeFileSync(
base,
runFixtureText({
runId: "base",
caseId: "case/load",
value: 10,
gitSha: "sha-a",
}),
);
fs.writeFileSync(
additional,
runFixtureText({
runId: "additional",
caseId: "case/load",
value: 12,
gitSha: "sha-b",
}),
);
const output = path.join(fixture.managed, "merged-run");
const original = tree(fixture.managed);
const result = await runCli(fixture, [
"merge",
base,
additional,
"--output-dir",
output,
]);
assert.equal(result.exitCode, 1);
assert.match(result.stderr, /different git provenance/);
assert.equal(fs.existsSync(output), false);
assert.deepEqual(tree(fixture.managed), original);
},
);
test("merge refuses an existing output directory", testOptions, async (t) => {
const fixture = createFrontend(t);
const base = path.join(fixture.managed, "base.transit");
const additional = path.join(fixture.managed, "additional.transit");
fs.writeFileSync(
base,
runFixtureText({
runId: "base",
caseId: "case/load",
value: 10,
gitSha: "same-sha",
}),
);
fs.writeFileSync(
additional,
runFixtureText({
runId: "additional",
caseId: "case/load",
value: 12,
gitSha: "same-sha",
}),
);
const output = path.join(fixture.managed, "existing-run");
fs.mkdirSync(output);
fs.writeFileSync(path.join(output, "keep.txt"), "keep\n");
const result = await runCli(fixture, [
"merge",
base,
additional,
"--output-dir",
output,
]);
assert.equal(result.exitCode, 1);
assert.match(result.stderr, /already exists/);
assert.equal(
fs.readFileSync(path.join(output, "keep.txt"), "utf8"),
"keep\n",
);
});

View File

@ -0,0 +1,202 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns performance-tests.render-wasm.merge-test
(:require
[benches.render-wasm.result :as result]
[cljs.test :as t :include-macros true]))
(defn- metric
[value]
(result/metric {:value value
:unit :ms
:semantics :elapsed
:basis {:kind :attempt-total}
:zero-policy :positive}))
(defn- case-definition
[case-id]
{:id case-id
:scene :case
:scene-description "merge scene"
:scene-seed 7
:params {:count 10}
:view {:scale 1 :x 0 :y 0}
:context :fresh
:completion :render-full
:batch-size 1
:preparation {}})
(defn- preparation
[preparation-id case-id]
{:preparation-id preparation-id
:case-id case-id
:fingerprint {:algorithm :sha-256
:digest (apply str (repeat 64 "a"))}
:effective-graphics {:webgl2 true}
:diagnostics {}})
(defn- attempt
[case-id preparation-id warmup?]
{:case-id case-id
:preparation-id preparation-id
:warmup? warmup?
:outcome :completed
:metrics {:elapsed (metric 1.0)}
:trace {}})
(defn- make-run
[{:keys [run-id cases warmups repetitions git-sha build environment analysis
timeout-ms started-at ended-at termination preparations]
:or {git-sha "same-sha"
build {:mode :release}
environment {:node "v24"}
timeout-ms 30000
started-at "2026-10-26T00:00:00.000Z"
ended-at "2026-10-26T00:01:00.000Z"
termination {:reason :completed}}}]
(let [run (result/create-run
{:run-id run-id
:started-at started-at
:plan {:cases (mapv case-definition cases)
:warmups warmups
:repetitions repetitions
:timeout-policy {:attempt-ms timeout-ms}}
:metadata {:git {:sha git-sha :dirty false}
:build {:functional build}
:environment environment
:scored? true}
:analysis analysis})
run (reduce
(fn [run case-id]
(let [options (get preparations case-id)
warmup-id (or (:warmup options)
(str run-id "-" (name case-id)))
measured-id (or (:measured options) warmup-id)]
(-> run
(result/record-preparation (preparation warmup-id case-id))
(cond-> (not= warmup-id measured-id)
(result/record-preparation (preparation measured-id case-id)))
(as-> run
(reduce (fn [run _]
(result/record-attempt
run (attempt case-id warmup-id true)))
run (range warmups))
(reduce (fn [run _]
(result/record-attempt
run (attempt case-id measured-id false)))
run (range repetitions))))))
run cases)]
(result/finish-run run ended-at termination)))
(t/deftest merges-measured-attempts-and-updates-the-plan
(let [base (make-run {:run-id "base"
:cases [:case/load :case/pan]
:warmups 1
:repetitions 2
:started-at "2026-10-26T00:00:00.000Z"
:ended-at "2026-10-26T00:01:00.000Z"})
additional (make-run {:run-id "additional"
:cases [:case/load :case/pan]
:warmups 1
:repetitions 3
:started-at "2026-10-26T00:02:00.000Z"
:ended-at "2026-10-26T00:03:00.000Z"})
merged (result/merge-runs base additional {:run-id "merged"})]
(t/is (= "merged" (:run-id merged)))
(t/is (= 1 (get-in merged [:plan :warmups])))
(t/is (= 5 (get-in merged [:plan :repetitions])))
(t/is (= 12 (count (:attempts merged))))
(t/is (= (range 12) (map :ordinal (:attempts merged))))
(t/is (= [true false false true false false false false false false false false]
(map :warmup? (:attempts merged))))
(t/is (every? #(every? zero? (vals %)) (vals (:remaining merged))))
(t/is (= "2026-10-26T00:03:00.000Z" (:ended-at merged)))
(t/is (= 2 (get-in merged [:metadata :diagnostics :merge
:additional-warmups-excluded])))
(t/is (= merged (result/check-run merged)))
(t/is (= (:attempts merged)
(:attempts (result/decode (result/encode merged)))))))
(t/deftest excludes-additional-warmup-only-preparations
(let [base (make-run {:run-id "base"
:cases [:case/load]
:warmups 1
:repetitions 1})
additional (make-run {:run-id "additional"
:cases [:case/load]
:warmups 1
:repetitions 1
:preparations
{:case/load {:warmup "warm-prep"
:measured "measured-prep"}}})
merged (result/merge-runs base additional {:run-id "merged"})
preparation-ids (map :preparation-id (:preparations merged))]
(t/is (some #{"base-load"} preparation-ids))
(t/is (some #{"measured-prep"} preparation-ids))
(t/is (not (some #{"warm-prep"} preparation-ids)))
(t/is (= 3 (count (:attempts merged))))
(t/is (= "measured-prep" (:preparation-id (last (:attempts merged)))))))
(t/deftest rejects-incompatible-runs
(let [base (make-run {:run-id "base"
:cases [:case/load]
:warmups 1
:repetitions 1})]
(doseq [[label options message]
[["case" {:cases [:case/pan]} #"different case definitions"]
["timeout" {:timeout-ms 1000} #"different timeout policy"]
["warmups" {:warmups 0} #"different warmup count"]
["analysis" {:analysis {:draws 1000}} #"different analysis settings"]
["git" {:git-sha "other"} #"different git provenance"]
["build" {:build {:mode :debug}} #"different build facts"]
["environment" {:environment {:node "v25"}}
#"different runtime environment"]]]
(t/testing label
(let [additional (make-run (merge {:run-id "additional"
:cases [:case/load]
:warmups 1
:repetitions 1}
options))]
(t/is (thrown-with-msg? js/Error message
(result/merge-runs base additional
{:run-id "merged"}))))))))
(t/deftest rejects-non-completed-runs
(let [completed (make-run {:run-id "base"
:cases [:case/load]
:warmups 1
:repetitions 1})
interrupted (make-run {:run-id "additional"
:cases [:case/load]
:warmups 1
:repetitions 1
:termination {:reason :interrupted}})]
(t/is (thrown-with-msg? js/Error #"only completed runs can be merged"
(result/merge-runs completed interrupted
{:run-id "merged"})))
(t/is (thrown-with-msg? js/Error #"only completed runs can be merged"
(result/merge-runs interrupted completed
{:run-id "merged"})))))
(t/deftest rejects-duplicate-preparation-ids
(let [base (make-run {:run-id "base"
:cases [:case/load]
:warmups 1
:repetitions 1
:preparations
{:case/load {:warmup "shared"
:measured "shared"}}})
additional (make-run {:run-id "additional"
:cases [:case/load]
:warmups 1
:repetitions 1
:preparations
{:case/load {:warmup "shared"
:measured "shared"}}})]
(t/is (thrown-with-msg? js/Error #"duplicate preparation ids"
(result/merge-runs base additional
{:run-id "merged"})))))

View File

@ -24,6 +24,7 @@
[performance-tests.render-wasm.geometry-scenes-test]
[performance-tests.render-wasm.lifecycle-test]
[performance-tests.render-wasm.log-test]
[performance-tests.render-wasm.merge-test]
[performance-tests.render-wasm.outcome-test]
[performance-tests.render-wasm.process-test]
[performance-tests.render-wasm.protocol-test]
@ -60,6 +61,7 @@
'performance-tests.render-wasm.geometry-scenes-test
'performance-tests.render-wasm.lifecycle-test
'performance-tests.render-wasm.log-test
'performance-tests.render-wasm.merge-test
'performance-tests.render-wasm.outcome-test
'performance-tests.render-wasm.process-test
'performance-tests.render-wasm.protocol-test

View File

@ -13,6 +13,7 @@
(t/deftest output-options-apply-to-every-command
(doseq [args [["build"] ["run"] ["purge"] ["compare" "base" "candidate"]
["merge" "base" "additional"]
["ab" "--features" "a" "--filter" "rects/load"]]]
(t/is (= :info (options/console-level (:options (runner/parse-args args)))))
(t/is (= :auto (options/color-mode (:options (runner/parse-args args)))))
@ -81,7 +82,7 @@
(t/deftest help-needs-no-command-and-other-command-lines-must-be-valid
(doseq [args [["--help"] ["-h"] ["build" "--help"] ["run" "--help"]
["compare" "--help"]]]
["compare" "--help"] ["merge" "--help"]]]
(let [{:keys [options summary]} (runner/parse-args args)
help (runner/usage summary)]
(t/is (true? (:help options)))
@ -89,13 +90,14 @@
(t/is (str/includes? help "runner purge"))
(t/is (str/includes? help "runner run"))
(t/is (str/includes? help "runner compare"))
(t/is (str/includes? help "runner merge"))
(t/is (str/includes? help "--build-timeout-ms"))
(t/is (str/includes? help "--cases"))
(t/is (str/includes? help "--allow-build-differences"))
(t/is (str/includes? help "--allow-scene-differences"))
(t/is (str/includes? help "--allow-all-differences"))
(t/is (not (str/includes? help "--force")))))
(t/is (thrown-with-msg? js/Error #"expected build, purge, run, ab or compare"
(t/is (thrown-with-msg? js/Error #"expected build, purge, run, ab, compare or merge"
(runner/parse-args [])))
(doseq [args [["unknown"] ["build" "purge"] ["build" "--unknown"]
["run" "extra"] ["compare" "only-one"]]]
@ -167,6 +169,27 @@
(t/is (true? (:allow-scene-differences options)))
(t/is (not (:allow-all-differences options)))))
(t/deftest merge-takes-two-paths-and-only-output-dir
(let [{:keys [command arguments options]}
(runner/parse-args ["merge" "base.transit" "additional.transit"
"--output-dir" "merged-home"])]
(t/is (= "merge" command))
(t/is (= ["merge" "base.transit" "additional.transit"] arguments))
(t/is (= "merged-home" (:output-dir options))))
(doseq [args [["merge"] ["merge" "only-base"]
["merge" "base" "additional" "extra"]]]
(t/is (thrown? js/Error (runner/parse-args args))))
(doseq [flags [["--build" "id"]
["--filter" "rects"]
["--seed" "1"]
["--require-gpu"]
["--features" "functional"]
["--cases" "a,b"]
["--allow-build-differences"]
["--only-builds"]
["--dry-run"]]]
(t/is (thrown? js/Error (runner/parse-args (into ["merge" "base" "additional"] flags))))))
(t/deftest force-is-rejected-without-an-alias
(doseq [args [["compare" "a.transit" "b.transit" "--force"]
["compare" "a.transit" "b.transit" "--force" "--allow-all-differences"]
@ -182,7 +205,12 @@
[["build" "--build" "retained"] #"run options require run"]
[["build" "--only-runs"] #"purge category flags require purge"]
[["purge" "--seed" "1"] #"run options require run"]
[["purge" "--cases" "a,b"] #"compare options require compare"]]]
[["purge" "--cases" "a,b"] #"compare options require compare"]
[["merge" "a" "b" "--seed" "1"] #"run options require run"]
[["merge" "a" "b" "--features" "functional"] #"build options require build"]
[["merge" "a" "b" "--cases" "a,b"] #"compare options require compare"]
[["merge" "a" "b" "--only-builds"] #"purge category flags require purge"]
[["merge" "a" "b" "--dry-run"] #"ab planning options require ab"]]]
(t/is (thrown-with-msg? js/Error message (runner/parse-args args))))
(doseq [command ["build" "purge" "run"]
flag ["--allow-build-differences" "--allow-scene-differences"