mirror of
https://github.com/penpot/penpot.git
synced 2026-11-02 16:10:42 +00:00
✨ Implement run merging
AI-assisted-by: deepseek-flash
This commit is contained in:
parent
0b19800955
commit
1af0e23b96
@ -871,6 +871,22 @@ pnpm run render-bench -- compare \
|
||||
--cases rects/load,paths/load --allow-scene-differences
|
||||
```
|
||||
|
||||
Merge an additional saved run into a base run:
|
||||
|
||||
```bash
|
||||
pnpm run render-bench -- merge \
|
||||
target/performance/render-wasm/runs/<base>/run.transit \
|
||||
target/performance/render-wasm/runs/<additional>/run.transit \
|
||||
--output-dir target/performance/render-wasm/runs/<merged>
|
||||
```
|
||||
|
||||
The merged record gets a new run id and a fresh run home. Both inputs must be
|
||||
completed and compatible: same cases, timeout policy, analysis, build facts,
|
||||
runtime environment, and Git provenance. The additional run must use the same
|
||||
warmup count. `merge` appends only the additional measured attempts; its warmup
|
||||
attempts remain in the additional record. The command starts no build, server,
|
||||
or browser.
|
||||
|
||||
Inspect and clean managed outputs:
|
||||
|
||||
```bash
|
||||
@ -891,6 +907,7 @@ The commands write results to stdout:
|
||||
- `ab`: `Comparison <id>`, `Output <absolute path>`, and final
|
||||
`Status <status>`.
|
||||
- `compare`: the comparison report.
|
||||
- `merge`: `Run <id>` and `Output <absolute path>`.
|
||||
- Help: usage text. `purge` produces no output on success.
|
||||
|
||||
Progress and diagnostics go to stderr. The default shows build phases, case
|
||||
@ -912,10 +929,11 @@ Each build writes `build.log`; each A/B comparison writes
|
||||
`run.transit`, and optional screenshots.
|
||||
|
||||
Exit 0 means the command completed successfully: a ready build for `build`,
|
||||
every allocated attempt for `run`, a valid comparison for `compare`, every
|
||||
configuration and edge for `ab`, and a clean purge for `purge`. Exit 1 means a
|
||||
failed preparation, a failed attempt, an incomplete run, an invalid selection,
|
||||
an unacknowledged comparison difference, or a harness failure. Exits 130 and 143
|
||||
every allocated attempt for `run`, a valid comparison for `compare`, a
|
||||
validated merged record for `merge`, every configuration and edge for `ab`, and
|
||||
a clean purge for `purge`. Exit 1 means a failed preparation, a failed attempt,
|
||||
an incomplete run, an invalid selection, incompatible merge inputs, an
|
||||
unacknowledged comparison difference, or a harness failure. Exits 130 and 143
|
||||
mean interruption by `SIGINT` and `SIGTERM` for `build`, `run` and `ab`. An
|
||||
interrupted command prints no error for the aborted work; it prints a failed
|
||||
write of its own record or log.
|
||||
|
||||
31
frontend/performance/benches/render_wasm/node/merge.cljs
Normal file
31
frontend/performance/benches/render_wasm/node/merge.cljs
Normal file
@ -0,0 +1,31 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns benches.render-wasm.node.merge
|
||||
"Offline merge of two saved schema:run records."
|
||||
(:require
|
||||
["fs" :as fs]
|
||||
[benches.render-wasm.node.log :as log]
|
||||
[benches.render-wasm.node.runs :as runs]
|
||||
[benches.render-wasm.result :as result]))
|
||||
|
||||
(defn execute
|
||||
"Reads and merges two saved run files from the caller's cwd, then writes a
|
||||
new run home. Inputs remain unchanged. Returns the merged record and home."
|
||||
[root base-path additional-path {:keys [output-dir]}]
|
||||
(let [base (runs/get-record base-path)
|
||||
additional (runs/get-record additional-path)
|
||||
run-id (str (random-uuid))
|
||||
merged (result/merge-runs base additional {:run-id run-id})
|
||||
home (runs/create-home {:root root
|
||||
:run-id run-id
|
||||
:output-dir output-dir})]
|
||||
(runs/write-record! home merged)
|
||||
(fs/appendFileSync (:log-path home)
|
||||
(str "merge " (:run-id base) " + " (:run-id additional) "\n"))
|
||||
(log/result (str "Run " run-id))
|
||||
(log/result (str "Output " (:dir home)))
|
||||
{:run merged :home home}))
|
||||
@ -11,8 +11,10 @@
|
||||
[app.common.transit :as transit]
|
||||
[benches.render-wasm.errors :as errors :refer [attempt failure?]]
|
||||
[benches.render-wasm.feature-plan :as feature-plan]
|
||||
[benches.render-wasm.node.artifacts :as artifacts]
|
||||
[benches.render-wasm.node.compare :as compare]
|
||||
[benches.render-wasm.node.log :as log]
|
||||
[benches.render-wasm.node.merge :as run-merge]
|
||||
[benches.render-wasm.node.options :as options]
|
||||
[clojure.string :as str]
|
||||
[clojure.tools.cli :refer [parse-opts]]))
|
||||
@ -98,7 +100,8 @@
|
||||
" [--confirm] [--dry-run] [run and comparison options]\n"
|
||||
" runner compare BASELINE CANDIDATE [--cases A,B]\n"
|
||||
" [--allow-build-differences] [--allow-scene-differences]\n"
|
||||
" [--allow-all-differences]\n\n"
|
||||
" [--allow-all-differences]\n"
|
||||
" runner merge BASE ADDITIONAL [--output-dir DIR]\n\n"
|
||||
"build prints Build <id> and Output <absolute path> on stdout. Exit 0 means\n"
|
||||
"ready; errors retain the build.transit and build.log for that ID.\n"
|
||||
"run prints Run <id>, Output <absolute path> and Report on stdout.\n"
|
||||
@ -111,7 +114,10 @@
|
||||
"to require hardware WebGL2 in preparation metadata. ab --dry-run skips it.\n"
|
||||
"compare reads saved schema:run records and starts no build, host or browser.\n"
|
||||
"Build and scene permissions combine; operation and runtime checks stay strict.\n"
|
||||
"All-differences permits every compatibility difference. Reports retain them.\n\n"
|
||||
"All-differences permits every compatibility difference. Reports retain them.\n"
|
||||
"merge reads two saved schema:run records and writes a merged record in a fresh\n"
|
||||
"run home. It appends the additional run's measured attempts, keeps the base\n"
|
||||
"warmups, and requires compatible completed runs. It starts no build or browser.\n\n"
|
||||
"ab uses every on/off configuration with Cargo defaults disabled; declared\n"
|
||||
"defaults must be empty. --max-features defaults to 3 (range 1..6).\n"
|
||||
"Use --confirm above eight configurations or with an empty filter.\n"
|
||||
@ -227,7 +233,15 @@
|
||||
(do
|
||||
(when purge? (invalid-args "purge category flags require purge"))
|
||||
(when build? (invalid-args "build options require build"))
|
||||
(when run? (invalid-args "run options require run"))))))
|
||||
(when run? (invalid-args "run options require run")))
|
||||
|
||||
"merge"
|
||||
(do
|
||||
(when purge? (invalid-args "purge category flags require purge"))
|
||||
(when build? (invalid-args "build options require build"))
|
||||
(when compare? (invalid-args "compare options require compare"))
|
||||
(when (some-flags? flags (disj run-flags "--output-dir"))
|
||||
(invalid-args "run options require run"))))))
|
||||
|
||||
(defn- check-arguments!
|
||||
[command arguments]
|
||||
@ -237,6 +251,10 @@
|
||||
(when-not (= 2 (count extra))
|
||||
(invalid-args "compare requires BASELINE and CANDIDATE paths"))
|
||||
|
||||
"merge"
|
||||
(when-not (= 2 (count extra))
|
||||
(invalid-args "merge requires BASE and ADDITIONAL paths"))
|
||||
|
||||
(when (seq extra)
|
||||
(invalid-args (str command " takes no positional arguments"))))))
|
||||
|
||||
@ -254,8 +272,8 @@
|
||||
(invalid-args "--quiet and --verbose are mutually exclusive"))
|
||||
(when-not (:help options)
|
||||
(when-not (and (= 1 (count (take 1 arguments)))
|
||||
(contains? #{"build" "purge" "run" "ab" "compare"} command))
|
||||
(invalid-args "expected build, purge, run, ab or compare; try --help"))
|
||||
(contains? #{"build" "purge" "run" "ab" "compare" "merge"} command))
|
||||
(invalid-args "expected build, purge, run, ab, compare or merge; try --help"))
|
||||
(check-command-flags! command flags)
|
||||
(check-arguments! command arguments)
|
||||
(when (and (= command "purge")
|
||||
@ -290,6 +308,12 @@
|
||||
:allow-scene-differences
|
||||
:allow-all-differences]))))
|
||||
|
||||
(= command "merge")
|
||||
(run-merge/execute (artifacts/frontend-root)
|
||||
(second arguments)
|
||||
(nth arguments 2)
|
||||
(select-keys options [:output-dir]))
|
||||
|
||||
:else
|
||||
(let [^js commands (js/require (path/join js/__dirname "commands.cjs"))]
|
||||
(set! (.-exitCode js/process)
|
||||
|
||||
@ -243,6 +243,23 @@
|
||||
(when-not valid?
|
||||
(throw (ex-info message (assoc data :type ::invalid-record)))))
|
||||
|
||||
(defn- incompatible!
|
||||
"Rejects runs whose recorded evidence cannot be merged."
|
||||
[message data]
|
||||
(throw (ex-info message (assoc data :type ::incompatible-runs))))
|
||||
|
||||
(defn- require-compatible!
|
||||
"Requires equal values at `path`, naming the disagreeing field."
|
||||
[base additional path label]
|
||||
(let [base-value (get-in base path)
|
||||
additional-value (get-in additional path)]
|
||||
(when-not (= base-value additional-value)
|
||||
(incompatible! (str "cannot merge runs with different " label)
|
||||
{:field label
|
||||
:path path
|
||||
:base base-value
|
||||
:additional additional-value}))))
|
||||
|
||||
(defn invalid-reason
|
||||
"Classifies one raw measurement without changing its value.
|
||||
Nil denotes an explicit missing value; an absent metric has no entry.
|
||||
@ -382,6 +399,88 @@
|
||||
(check-completion run termination)
|
||||
(assoc run :ended-at ended-at :termination termination))
|
||||
|
||||
(defn merge-runs
|
||||
"Returns a new validated run that appends `additional`'s measured attempts to
|
||||
`base`.
|
||||
|
||||
Both runs must be completed and compatible. The merged plan keeps `base`'s
|
||||
warmups and adds `additional`'s measured repetitions. Warmup attempts
|
||||
and preparations used only by them are excluded from `additional`.
|
||||
|
||||
The caller supplies the merged run id and may supply its end time."
|
||||
[base additional {:keys [run-id ended-at]}]
|
||||
(let [base (check-run base)
|
||||
additional (check-run additional)]
|
||||
(require-valid! (and (string? run-id) (seq run-id))
|
||||
"merged run needs a run id" {})
|
||||
(doseq [[side run] [[:base base] [:additional additional]]]
|
||||
(when-not (= :completed (get-in run [:termination :reason]))
|
||||
(incompatible! "only completed runs can be merged"
|
||||
{:side side
|
||||
:run-id (:run-id run)
|
||||
:reason (get-in run [:termination :reason])})))
|
||||
(when-not (and (string? (:ended-at base))
|
||||
(string? (:ended-at additional)))
|
||||
(incompatible! "completed runs need end times"
|
||||
{:base (:ended-at base)
|
||||
:additional (:ended-at additional)}))
|
||||
(doseq [[path label] [[[:plan :cases] "case definitions"]
|
||||
[[:plan :timeout-policy] "timeout policy"]
|
||||
[[:plan :warmups] "warmup count"]
|
||||
[[:analysis] "analysis settings"]
|
||||
[[:metadata :git] "git provenance"]
|
||||
[[:metadata :build] "build facts"]
|
||||
[[:metadata :environment] "runtime environment"]
|
||||
[[:metadata :scored?] "scored flag"]]]
|
||||
(require-compatible! base additional path label))
|
||||
(let [base-attempts (:attempts base)
|
||||
additional-measured (filterv #(not (:warmup? %)) (:attempts additional))
|
||||
base-preparations (:preparations base)
|
||||
base-preparation-ids (into #{} (map :preparation-id) base-preparations)
|
||||
used-preparation-ids (into #{} (keep :preparation-id) additional-measured)
|
||||
additional-preparations (filterv #(contains? used-preparation-ids
|
||||
(:preparation-id %))
|
||||
(:preparations additional))]
|
||||
(when-let [duplicate (some #(contains? base-preparation-ids (:preparation-id %))
|
||||
additional-preparations)]
|
||||
(incompatible! "cannot merge runs with duplicate preparation ids"
|
||||
{:preparation-id (:preparation-id duplicate)}))
|
||||
(let [offset (count base-attempts)
|
||||
attempts (into (vec base-attempts)
|
||||
(map-indexed (fn [index attempt]
|
||||
(assoc attempt :ordinal (+ offset index)))
|
||||
additional-measured))
|
||||
plan (assoc (:plan base)
|
||||
:repetitions (+ (get-in base [:plan :repetitions])
|
||||
(get-in additional [:plan :repetitions])))
|
||||
remaining (reduce consume-slot (initial-remaining plan) attempts)
|
||||
ended-at (or ended-at
|
||||
(let [base-end (:ended-at base)
|
||||
additional-end (:ended-at additional)]
|
||||
(if (neg? (compare base-end additional-end))
|
||||
additional-end
|
||||
base-end)))
|
||||
metadata (update (:metadata base) :diagnostics
|
||||
(fn [diagnostics]
|
||||
(assoc (or diagnostics {})
|
||||
:merge
|
||||
{:base-run-id (:run-id base)
|
||||
:additional-run-id (:run-id additional)
|
||||
:additional-warmups-excluded
|
||||
(count (filter :warmup? (:attempts additional)))
|
||||
:merged-at ended-at})))]
|
||||
(check-run {:type :run-record
|
||||
:run-id run-id
|
||||
:started-at (:started-at base)
|
||||
:ended-at ended-at
|
||||
:plan plan
|
||||
:metadata metadata
|
||||
:analysis (:analysis base)
|
||||
:preparations (into (vec base-preparations) additional-preparations)
|
||||
:attempts attempts
|
||||
:remaining remaining
|
||||
:termination {:reason :completed}})))))
|
||||
|
||||
(defn- check-feature-configuration
|
||||
"Requires evidence that matches each saved configuration's progress."
|
||||
[{:keys [status build run effective-features failure reason]}]
|
||||
|
||||
@ -319,6 +319,9 @@ function runFixtureText({
|
||||
fingerprint = "a".repeat(64),
|
||||
operation = "load",
|
||||
operationArgs = { distance: 10 },
|
||||
warmups = 0,
|
||||
repetitions = 1,
|
||||
gitSha = runId,
|
||||
}) {
|
||||
const k = transit.keyword;
|
||||
const map = (pairs) => transit.map(pairs);
|
||||
@ -348,22 +351,29 @@ function runFixtureText({
|
||||
k("diagnostics"),
|
||||
map([]),
|
||||
]);
|
||||
const attempt = map([
|
||||
k("ordinal"),
|
||||
0,
|
||||
k("case-id"),
|
||||
k(caseId),
|
||||
k("preparation-id"),
|
||||
`${runId}-preparation`,
|
||||
k("warmup?"),
|
||||
false,
|
||||
k("outcome"),
|
||||
k("completed"),
|
||||
k("metrics"),
|
||||
map([k("elapsed"), metric]),
|
||||
k("trace"),
|
||||
map([]),
|
||||
]);
|
||||
const makeAttempt = (ordinal, warmup) =>
|
||||
map([
|
||||
k("ordinal"),
|
||||
ordinal,
|
||||
k("case-id"),
|
||||
k(caseId),
|
||||
k("preparation-id"),
|
||||
`${runId}-preparation`,
|
||||
k("warmup?"),
|
||||
warmup,
|
||||
k("outcome"),
|
||||
k("completed"),
|
||||
k("metrics"),
|
||||
map([k("elapsed"), metric]),
|
||||
k("trace"),
|
||||
map([]),
|
||||
]);
|
||||
const attempts = [
|
||||
...Array.from({ length: warmups }, (_, index) => makeAttempt(index, true)),
|
||||
...Array.from({ length: repetitions }, (_, index) =>
|
||||
makeAttempt(warmups + index, false),
|
||||
),
|
||||
];
|
||||
const record = map([
|
||||
k("type"),
|
||||
k("run-record"),
|
||||
@ -401,16 +411,16 @@ function runFixtureText({
|
||||
]),
|
||||
],
|
||||
k("warmups"),
|
||||
0,
|
||||
warmups,
|
||||
k("repetitions"),
|
||||
1,
|
||||
repetitions,
|
||||
k("timeout-policy"),
|
||||
map([k("attempt-ms"), 30000]),
|
||||
]),
|
||||
k("metadata"),
|
||||
map([
|
||||
k("git"),
|
||||
map([k("sha"), runId, k("dirty"), false]),
|
||||
map([k("sha"), gitSha, k("dirty"), false]),
|
||||
k("build"),
|
||||
map([
|
||||
k("functional"),
|
||||
@ -443,7 +453,7 @@ function runFixtureText({
|
||||
k("preparations"),
|
||||
[preparation],
|
||||
k("attempts"),
|
||||
[attempt],
|
||||
attempts,
|
||||
k("remaining"),
|
||||
map([k(caseId), map([k("warmups"), 0, k("repetitions"), 0])]),
|
||||
k("termination"),
|
||||
@ -1252,10 +1262,14 @@ test(
|
||||
const help = await runCli(fixture, ["--help"]);
|
||||
assert.equal(help.exitCode, 0, help.stderr);
|
||||
assert.match(help.stdout, /runner compare/);
|
||||
assert.match(help.stdout, /runner merge/);
|
||||
const abHelp = await runCli(fixture, ["ab", "--help"]);
|
||||
assert.equal(abHelp.exitCode, 0, abHelp.stderr);
|
||||
assert.match(abHelp.stdout, /runner ab --features/);
|
||||
assert.match(abHelp.stdout, /--only-comparisons/);
|
||||
const mergeHelp = await runCli(fixture, ["merge", "--help"]);
|
||||
assert.equal(mergeHelp.exitCode, 0, mergeHelp.stderr);
|
||||
assert.match(mergeHelp.stdout, /runner merge BASE ADDITIONAL/);
|
||||
for (const args of [["ab"], ["ab", "--dry-run"]]) {
|
||||
const invalid = await runCli(fixture, args);
|
||||
assert.equal(invalid.exitCode, 1, invalid.stderr);
|
||||
@ -1298,6 +1312,7 @@ test(
|
||||
assert.equal(result.exitCode, 0);
|
||||
assert.match(result.stdout, /runner run/);
|
||||
assert.match(result.stdout, /runner compare/);
|
||||
assert.match(result.stdout, /runner merge/);
|
||||
assert.match(result.stdout, /--cases/);
|
||||
assert.match(result.stdout, /--allow-build-differences/);
|
||||
assert.match(result.stdout, /--allow-scene-differences/);
|
||||
@ -1557,3 +1572,162 @@ test("compare rejects --force without an alias", testOptions, async (t) => {
|
||||
assert.deepEqual(tree(fixture.managed), original);
|
||||
}
|
||||
});
|
||||
|
||||
test(
|
||||
"merge writes a validated run home with appended measured attempts",
|
||||
testOptions,
|
||||
async (t) => {
|
||||
const fixture = createFrontend(t);
|
||||
const base = path.join(fixture.managed, "base.transit");
|
||||
const additional = path.join(fixture.managed, "additional.transit");
|
||||
const baseText = runFixtureText({
|
||||
runId: "base",
|
||||
caseId: "case/load",
|
||||
value: 10,
|
||||
warmups: 1,
|
||||
repetitions: 2,
|
||||
gitSha: "same-sha",
|
||||
});
|
||||
const additionalText = runFixtureText({
|
||||
runId: "additional",
|
||||
caseId: "case/load",
|
||||
value: 12,
|
||||
warmups: 1,
|
||||
repetitions: 3,
|
||||
gitSha: "same-sha",
|
||||
});
|
||||
fs.writeFileSync(base, baseText);
|
||||
fs.writeFileSync(additional, additionalText);
|
||||
const output = path.join(fixture.managed, "merged-run");
|
||||
|
||||
const result = await runCli(fixture, [
|
||||
"merge",
|
||||
base,
|
||||
additional,
|
||||
"--output-dir",
|
||||
output,
|
||||
]);
|
||||
|
||||
assert.equal(result.exitCode, 0, result.stderr);
|
||||
const runId = result.stdout.match(/Run ([0-9a-f-]{36})/)[1];
|
||||
assert.ok(fs.existsSync(path.join(output, "run.transit")));
|
||||
assert.equal(fs.readFileSync(base, "utf8"), baseText);
|
||||
assert.equal(fs.readFileSync(additional, "utf8"), additionalText);
|
||||
|
||||
const k = transit.keyword;
|
||||
const merged = readManifest(path.join(output, "run.transit"));
|
||||
const plan = merged.get(k("plan"));
|
||||
const attempts = merged.get(k("attempts"));
|
||||
const remaining = merged.get(k("remaining")).get(k("case/load"));
|
||||
const mergeInfo = merged
|
||||
.get(k("metadata"))
|
||||
.get(k("diagnostics"))
|
||||
.get(k("merge"));
|
||||
|
||||
assert.equal(merged.get(k("run-id")), runId);
|
||||
assert.equal(plan.get(k("warmups")), 1);
|
||||
assert.equal(plan.get(k("repetitions")), 5);
|
||||
assert.deepEqual(
|
||||
attempts.map((attempt) => attempt.get(k("ordinal"))),
|
||||
[0, 1, 2, 3, 4, 5],
|
||||
);
|
||||
assert.deepEqual(
|
||||
attempts.map((attempt) => attempt.get(k("warmup?"))),
|
||||
[true, false, false, false, false, false],
|
||||
);
|
||||
assert.equal(remaining.get(k("warmups")), 0);
|
||||
assert.equal(remaining.get(k("repetitions")), 0);
|
||||
assert.equal(mergeInfo.get(k("base-run-id")), "base");
|
||||
assert.equal(mergeInfo.get(k("additional-run-id")), "additional");
|
||||
assert.equal(mergeInfo.get(k("additional-warmups-excluded")), 1);
|
||||
assert.match(
|
||||
fs.readFileSync(path.join(output, "run.log"), "utf8"),
|
||||
/merge base \+ additional/,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"merge rejects incompatible runs before creating output",
|
||||
testOptions,
|
||||
async (t) => {
|
||||
const fixture = createFrontend(t);
|
||||
const base = path.join(fixture.managed, "base.transit");
|
||||
const additional = path.join(fixture.managed, "additional.transit");
|
||||
fs.writeFileSync(
|
||||
base,
|
||||
runFixtureText({
|
||||
runId: "base",
|
||||
caseId: "case/load",
|
||||
value: 10,
|
||||
gitSha: "sha-a",
|
||||
}),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
additional,
|
||||
runFixtureText({
|
||||
runId: "additional",
|
||||
caseId: "case/load",
|
||||
value: 12,
|
||||
gitSha: "sha-b",
|
||||
}),
|
||||
);
|
||||
const output = path.join(fixture.managed, "merged-run");
|
||||
const original = tree(fixture.managed);
|
||||
|
||||
const result = await runCli(fixture, [
|
||||
"merge",
|
||||
base,
|
||||
additional,
|
||||
"--output-dir",
|
||||
output,
|
||||
]);
|
||||
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.match(result.stderr, /different git provenance/);
|
||||
assert.equal(fs.existsSync(output), false);
|
||||
assert.deepEqual(tree(fixture.managed), original);
|
||||
},
|
||||
);
|
||||
|
||||
test("merge refuses an existing output directory", testOptions, async (t) => {
|
||||
const fixture = createFrontend(t);
|
||||
const base = path.join(fixture.managed, "base.transit");
|
||||
const additional = path.join(fixture.managed, "additional.transit");
|
||||
fs.writeFileSync(
|
||||
base,
|
||||
runFixtureText({
|
||||
runId: "base",
|
||||
caseId: "case/load",
|
||||
value: 10,
|
||||
gitSha: "same-sha",
|
||||
}),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
additional,
|
||||
runFixtureText({
|
||||
runId: "additional",
|
||||
caseId: "case/load",
|
||||
value: 12,
|
||||
gitSha: "same-sha",
|
||||
}),
|
||||
);
|
||||
const output = path.join(fixture.managed, "existing-run");
|
||||
fs.mkdirSync(output);
|
||||
fs.writeFileSync(path.join(output, "keep.txt"), "keep\n");
|
||||
|
||||
const result = await runCli(fixture, [
|
||||
"merge",
|
||||
base,
|
||||
additional,
|
||||
"--output-dir",
|
||||
output,
|
||||
]);
|
||||
|
||||
assert.equal(result.exitCode, 1);
|
||||
assert.match(result.stderr, /already exists/);
|
||||
assert.equal(
|
||||
fs.readFileSync(path.join(output, "keep.txt"), "utf8"),
|
||||
"keep\n",
|
||||
);
|
||||
});
|
||||
|
||||
202
frontend/test/performance_tests/render_wasm/merge_test.cljs
Normal file
202
frontend/test/performance_tests/render_wasm/merge_test.cljs
Normal file
@ -0,0 +1,202 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns performance-tests.render-wasm.merge-test
|
||||
(:require
|
||||
[benches.render-wasm.result :as result]
|
||||
[cljs.test :as t :include-macros true]))
|
||||
|
||||
(defn- metric
|
||||
[value]
|
||||
(result/metric {:value value
|
||||
:unit :ms
|
||||
:semantics :elapsed
|
||||
:basis {:kind :attempt-total}
|
||||
:zero-policy :positive}))
|
||||
|
||||
(defn- case-definition
|
||||
[case-id]
|
||||
{:id case-id
|
||||
:scene :case
|
||||
:scene-description "merge scene"
|
||||
:scene-seed 7
|
||||
:params {:count 10}
|
||||
:view {:scale 1 :x 0 :y 0}
|
||||
:context :fresh
|
||||
:completion :render-full
|
||||
:batch-size 1
|
||||
:preparation {}})
|
||||
|
||||
(defn- preparation
|
||||
[preparation-id case-id]
|
||||
{:preparation-id preparation-id
|
||||
:case-id case-id
|
||||
:fingerprint {:algorithm :sha-256
|
||||
:digest (apply str (repeat 64 "a"))}
|
||||
:effective-graphics {:webgl2 true}
|
||||
:diagnostics {}})
|
||||
|
||||
(defn- attempt
|
||||
[case-id preparation-id warmup?]
|
||||
{:case-id case-id
|
||||
:preparation-id preparation-id
|
||||
:warmup? warmup?
|
||||
:outcome :completed
|
||||
:metrics {:elapsed (metric 1.0)}
|
||||
:trace {}})
|
||||
|
||||
(defn- make-run
|
||||
[{:keys [run-id cases warmups repetitions git-sha build environment analysis
|
||||
timeout-ms started-at ended-at termination preparations]
|
||||
:or {git-sha "same-sha"
|
||||
build {:mode :release}
|
||||
environment {:node "v24"}
|
||||
timeout-ms 30000
|
||||
started-at "2026-10-26T00:00:00.000Z"
|
||||
ended-at "2026-10-26T00:01:00.000Z"
|
||||
termination {:reason :completed}}}]
|
||||
(let [run (result/create-run
|
||||
{:run-id run-id
|
||||
:started-at started-at
|
||||
:plan {:cases (mapv case-definition cases)
|
||||
:warmups warmups
|
||||
:repetitions repetitions
|
||||
:timeout-policy {:attempt-ms timeout-ms}}
|
||||
:metadata {:git {:sha git-sha :dirty false}
|
||||
:build {:functional build}
|
||||
:environment environment
|
||||
:scored? true}
|
||||
:analysis analysis})
|
||||
run (reduce
|
||||
(fn [run case-id]
|
||||
(let [options (get preparations case-id)
|
||||
warmup-id (or (:warmup options)
|
||||
(str run-id "-" (name case-id)))
|
||||
measured-id (or (:measured options) warmup-id)]
|
||||
(-> run
|
||||
(result/record-preparation (preparation warmup-id case-id))
|
||||
(cond-> (not= warmup-id measured-id)
|
||||
(result/record-preparation (preparation measured-id case-id)))
|
||||
(as-> run
|
||||
(reduce (fn [run _]
|
||||
(result/record-attempt
|
||||
run (attempt case-id warmup-id true)))
|
||||
run (range warmups))
|
||||
(reduce (fn [run _]
|
||||
(result/record-attempt
|
||||
run (attempt case-id measured-id false)))
|
||||
run (range repetitions))))))
|
||||
run cases)]
|
||||
(result/finish-run run ended-at termination)))
|
||||
|
||||
(t/deftest merges-measured-attempts-and-updates-the-plan
|
||||
(let [base (make-run {:run-id "base"
|
||||
:cases [:case/load :case/pan]
|
||||
:warmups 1
|
||||
:repetitions 2
|
||||
:started-at "2026-10-26T00:00:00.000Z"
|
||||
:ended-at "2026-10-26T00:01:00.000Z"})
|
||||
additional (make-run {:run-id "additional"
|
||||
:cases [:case/load :case/pan]
|
||||
:warmups 1
|
||||
:repetitions 3
|
||||
:started-at "2026-10-26T00:02:00.000Z"
|
||||
:ended-at "2026-10-26T00:03:00.000Z"})
|
||||
merged (result/merge-runs base additional {:run-id "merged"})]
|
||||
(t/is (= "merged" (:run-id merged)))
|
||||
(t/is (= 1 (get-in merged [:plan :warmups])))
|
||||
(t/is (= 5 (get-in merged [:plan :repetitions])))
|
||||
(t/is (= 12 (count (:attempts merged))))
|
||||
(t/is (= (range 12) (map :ordinal (:attempts merged))))
|
||||
(t/is (= [true false false true false false false false false false false false]
|
||||
(map :warmup? (:attempts merged))))
|
||||
(t/is (every? #(every? zero? (vals %)) (vals (:remaining merged))))
|
||||
(t/is (= "2026-10-26T00:03:00.000Z" (:ended-at merged)))
|
||||
(t/is (= 2 (get-in merged [:metadata :diagnostics :merge
|
||||
:additional-warmups-excluded])))
|
||||
(t/is (= merged (result/check-run merged)))
|
||||
(t/is (= (:attempts merged)
|
||||
(:attempts (result/decode (result/encode merged)))))))
|
||||
|
||||
(t/deftest excludes-additional-warmup-only-preparations
|
||||
(let [base (make-run {:run-id "base"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1})
|
||||
additional (make-run {:run-id "additional"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1
|
||||
:preparations
|
||||
{:case/load {:warmup "warm-prep"
|
||||
:measured "measured-prep"}}})
|
||||
merged (result/merge-runs base additional {:run-id "merged"})
|
||||
preparation-ids (map :preparation-id (:preparations merged))]
|
||||
(t/is (some #{"base-load"} preparation-ids))
|
||||
(t/is (some #{"measured-prep"} preparation-ids))
|
||||
(t/is (not (some #{"warm-prep"} preparation-ids)))
|
||||
(t/is (= 3 (count (:attempts merged))))
|
||||
(t/is (= "measured-prep" (:preparation-id (last (:attempts merged)))))))
|
||||
|
||||
(t/deftest rejects-incompatible-runs
|
||||
(let [base (make-run {:run-id "base"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1})]
|
||||
(doseq [[label options message]
|
||||
[["case" {:cases [:case/pan]} #"different case definitions"]
|
||||
["timeout" {:timeout-ms 1000} #"different timeout policy"]
|
||||
["warmups" {:warmups 0} #"different warmup count"]
|
||||
["analysis" {:analysis {:draws 1000}} #"different analysis settings"]
|
||||
["git" {:git-sha "other"} #"different git provenance"]
|
||||
["build" {:build {:mode :debug}} #"different build facts"]
|
||||
["environment" {:environment {:node "v25"}}
|
||||
#"different runtime environment"]]]
|
||||
(t/testing label
|
||||
(let [additional (make-run (merge {:run-id "additional"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1}
|
||||
options))]
|
||||
(t/is (thrown-with-msg? js/Error message
|
||||
(result/merge-runs base additional
|
||||
{:run-id "merged"}))))))))
|
||||
|
||||
(t/deftest rejects-non-completed-runs
|
||||
(let [completed (make-run {:run-id "base"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1})
|
||||
interrupted (make-run {:run-id "additional"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1
|
||||
:termination {:reason :interrupted}})]
|
||||
(t/is (thrown-with-msg? js/Error #"only completed runs can be merged"
|
||||
(result/merge-runs completed interrupted
|
||||
{:run-id "merged"})))
|
||||
(t/is (thrown-with-msg? js/Error #"only completed runs can be merged"
|
||||
(result/merge-runs interrupted completed
|
||||
{:run-id "merged"})))))
|
||||
|
||||
(t/deftest rejects-duplicate-preparation-ids
|
||||
(let [base (make-run {:run-id "base"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1
|
||||
:preparations
|
||||
{:case/load {:warmup "shared"
|
||||
:measured "shared"}}})
|
||||
additional (make-run {:run-id "additional"
|
||||
:cases [:case/load]
|
||||
:warmups 1
|
||||
:repetitions 1
|
||||
:preparations
|
||||
{:case/load {:warmup "shared"
|
||||
:measured "shared"}}})]
|
||||
(t/is (thrown-with-msg? js/Error #"duplicate preparation ids"
|
||||
(result/merge-runs base additional
|
||||
{:run-id "merged"})))))
|
||||
@ -24,6 +24,7 @@
|
||||
[performance-tests.render-wasm.geometry-scenes-test]
|
||||
[performance-tests.render-wasm.lifecycle-test]
|
||||
[performance-tests.render-wasm.log-test]
|
||||
[performance-tests.render-wasm.merge-test]
|
||||
[performance-tests.render-wasm.outcome-test]
|
||||
[performance-tests.render-wasm.process-test]
|
||||
[performance-tests.render-wasm.protocol-test]
|
||||
@ -60,6 +61,7 @@
|
||||
'performance-tests.render-wasm.geometry-scenes-test
|
||||
'performance-tests.render-wasm.lifecycle-test
|
||||
'performance-tests.render-wasm.log-test
|
||||
'performance-tests.render-wasm.merge-test
|
||||
'performance-tests.render-wasm.outcome-test
|
||||
'performance-tests.render-wasm.process-test
|
||||
'performance-tests.render-wasm.protocol-test
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
|
||||
(t/deftest output-options-apply-to-every-command
|
||||
(doseq [args [["build"] ["run"] ["purge"] ["compare" "base" "candidate"]
|
||||
["merge" "base" "additional"]
|
||||
["ab" "--features" "a" "--filter" "rects/load"]]]
|
||||
(t/is (= :info (options/console-level (:options (runner/parse-args args)))))
|
||||
(t/is (= :auto (options/color-mode (:options (runner/parse-args args)))))
|
||||
@ -81,7 +82,7 @@
|
||||
|
||||
(t/deftest help-needs-no-command-and-other-command-lines-must-be-valid
|
||||
(doseq [args [["--help"] ["-h"] ["build" "--help"] ["run" "--help"]
|
||||
["compare" "--help"]]]
|
||||
["compare" "--help"] ["merge" "--help"]]]
|
||||
(let [{:keys [options summary]} (runner/parse-args args)
|
||||
help (runner/usage summary)]
|
||||
(t/is (true? (:help options)))
|
||||
@ -89,13 +90,14 @@
|
||||
(t/is (str/includes? help "runner purge"))
|
||||
(t/is (str/includes? help "runner run"))
|
||||
(t/is (str/includes? help "runner compare"))
|
||||
(t/is (str/includes? help "runner merge"))
|
||||
(t/is (str/includes? help "--build-timeout-ms"))
|
||||
(t/is (str/includes? help "--cases"))
|
||||
(t/is (str/includes? help "--allow-build-differences"))
|
||||
(t/is (str/includes? help "--allow-scene-differences"))
|
||||
(t/is (str/includes? help "--allow-all-differences"))
|
||||
(t/is (not (str/includes? help "--force")))))
|
||||
(t/is (thrown-with-msg? js/Error #"expected build, purge, run, ab or compare"
|
||||
(t/is (thrown-with-msg? js/Error #"expected build, purge, run, ab, compare or merge"
|
||||
(runner/parse-args [])))
|
||||
(doseq [args [["unknown"] ["build" "purge"] ["build" "--unknown"]
|
||||
["run" "extra"] ["compare" "only-one"]]]
|
||||
@ -167,6 +169,27 @@
|
||||
(t/is (true? (:allow-scene-differences options)))
|
||||
(t/is (not (:allow-all-differences options)))))
|
||||
|
||||
(t/deftest merge-takes-two-paths-and-only-output-dir
|
||||
(let [{:keys [command arguments options]}
|
||||
(runner/parse-args ["merge" "base.transit" "additional.transit"
|
||||
"--output-dir" "merged-home"])]
|
||||
(t/is (= "merge" command))
|
||||
(t/is (= ["merge" "base.transit" "additional.transit"] arguments))
|
||||
(t/is (= "merged-home" (:output-dir options))))
|
||||
(doseq [args [["merge"] ["merge" "only-base"]
|
||||
["merge" "base" "additional" "extra"]]]
|
||||
(t/is (thrown? js/Error (runner/parse-args args))))
|
||||
(doseq [flags [["--build" "id"]
|
||||
["--filter" "rects"]
|
||||
["--seed" "1"]
|
||||
["--require-gpu"]
|
||||
["--features" "functional"]
|
||||
["--cases" "a,b"]
|
||||
["--allow-build-differences"]
|
||||
["--only-builds"]
|
||||
["--dry-run"]]]
|
||||
(t/is (thrown? js/Error (runner/parse-args (into ["merge" "base" "additional"] flags))))))
|
||||
|
||||
(t/deftest force-is-rejected-without-an-alias
|
||||
(doseq [args [["compare" "a.transit" "b.transit" "--force"]
|
||||
["compare" "a.transit" "b.transit" "--force" "--allow-all-differences"]
|
||||
@ -182,7 +205,12 @@
|
||||
[["build" "--build" "retained"] #"run options require run"]
|
||||
[["build" "--only-runs"] #"purge category flags require purge"]
|
||||
[["purge" "--seed" "1"] #"run options require run"]
|
||||
[["purge" "--cases" "a,b"] #"compare options require compare"]]]
|
||||
[["purge" "--cases" "a,b"] #"compare options require compare"]
|
||||
[["merge" "a" "b" "--seed" "1"] #"run options require run"]
|
||||
[["merge" "a" "b" "--features" "functional"] #"build options require build"]
|
||||
[["merge" "a" "b" "--cases" "a,b"] #"compare options require compare"]
|
||||
[["merge" "a" "b" "--only-builds"] #"purge category flags require purge"]
|
||||
[["merge" "a" "b" "--dry-run"] #"ab planning options require ab"]]]
|
||||
(t/is (thrown-with-msg? js/Error message (runner/parse-args args))))
|
||||
(doseq [command ["build" "purge" "run"]
|
||||
flag ["--allow-build-differences" "--allow-scene-differences"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user