diff --git a/performance/run.sh b/performance/run.sh index 96266dcfef..f695701dc7 100755 --- a/performance/run.sh +++ b/performance/run.sh @@ -55,6 +55,7 @@ Commands: media-upload Upload images of varying sizes (direct + chunked) font-upload Upload fonts via chunked upload + create-font-variant concurrent-edit Concurrent editing: same-file or multi-file mode + file-size-matrix Measure latency vs file size (10, 100, 500, 1000 shapes) all Run all scenarios together (orchestrator) clean Remove test results help Show this help @@ -86,6 +87,7 @@ Examples: $(basename "$0") media-upload -u https://penpot.example.com $(basename "$0") concurrent-edit --mode same-file -v 5 -n 10 $(basename "$0") concurrent-edit --mode multi-file --files 3 --vus-per-file 2 -n 10 + $(basename "$0") file-size-matrix -n 10 $(basename "$0") all -v 50 EOF } @@ -274,6 +276,23 @@ cmd_concurrent_edit() { run_script "workspace-edit-concurrent.js" "$label" } +cmd_file_size_matrix() { + check_k6 + + echo "" + echo "=== File Size Matrix ===" + echo " Tiers: small(10), medium(100), large(500), xlarge(1000)" + [[ -n "$ITER" ]] && echo " Iterations: $ITER (per tier)" + echo "" + + # Pass iterations as env var for the script + if [[ -n "$ITER" ]]; then + export PENPOT_MATRIX_ITERATIONS="$ITER" + fi + + run_script "file-size-matrix.js" "file-size-matrix" +} + cmd_clean() { local results_dir="$SCRIPT_DIR/results" if [[ -d "$results_dir" ]]; then @@ -354,6 +373,7 @@ case "$command" in media-upload) cmd_media_upload ;; font-upload) cmd_font_upload ;; concurrent-edit) cmd_concurrent_edit ;; + file-size-matrix) cmd_file_size_matrix ;; all) cmd_all ;; clean) cmd_clean ;; help|-h|--help) usage ;; diff --git a/performance/scripts/file-size-matrix.js b/performance/scripts/file-size-matrix.js new file mode 100644 index 0000000000..b8903d9e4b --- /dev/null +++ b/performance/scripts/file-size-matrix.js @@ -0,0 +1,257 @@ +// File Size Matrix Performance Test +// +// Measures how update-file and get-file latency scales with file size. +// Creates files with different shape counts (10, 100, 500, 1000) and +// benchmarks operations on each. +// +// Usage: +// k6 run scripts/file-size-matrix.js +// k6 run --iterations 10 scripts/file-size-matrix.js +// ./run.sh file-size-matrix +// ./run.sh file-size-matrix -n 10 + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; +const ITERATIONS_PER_TIER = parseInt(__ENV.PENPOT_MATRIX_ITERATIONS || "5"); + +// Shape tiers +const TIERS = [ + { name: "small", shapes: 10, color: "#ff0000" }, + { name: "medium", shapes: 100, color: "#00ff00" }, + { name: "large", shapes: 500, color: "#0000ff" }, + { name: "xlarge", shapes: 1000, color: "#ff00ff" }, +]; + +export const options = { + scenarios: { + file_size_matrix: { + executor: "per-vu-iterations", + vus: 1, + iterations: 1, + maxDuration: "30m", + }, + }, + thresholds: { + http_req_duration: ["p(95)<10000"], + http_req_failed: ["rate<0.01"], + }, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +function makeAddRectChange(pageId, index, color) { + const shapeId = uuidv4(); + const x = 50 + (index % 20) * 30; + const y = 50 + Math.floor(index / 20) * 30; + const w = 100; + const h = 80; + + return { + type: "add-obj", + pageId: pageId, + id: shapeId, + frameId: pageId, + parentId: pageId, + obj: { + id: shapeId, type: "rect", name: `Shape ${index}`, + x, y, width: w, height: h, + fillColor: color, fillOpacity: 0.8, + rotation: 0, hidden: false, locked: false, + selrect: { x, y, width: w, height: h, x1: x, y1: y, x2: x + w, y2: y + h }, + points: [ + { x, y }, { x: x + w, y }, { x: x + w, y: y + h }, { x, y: y + h }, + ], + transform: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + transformInverse: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + parentId: pageId, frameId: pageId, + }, + }; +} + +// Populate a file with N shapes in a single update-file call +function populateFile(client, fileId, pageId, shapeCount, color) { + // Get current file state + const getFileRes = client.getFile(fileId); + if (getFileRes.status !== 200) return null; + let { revn, vern } = getFileRes.body; + + // Add shapes in batches (backend may have limits on changes per call) + const BATCH_SIZE = 100; + let added = 0; + + while (added < shapeCount) { + const batchCount = Math.min(BATCH_SIZE, shapeCount - added); + const changes = []; + for (let i = 0; i < batchCount; i++) { + changes.push(makeAddRectChange(pageId, added + i, color)); + } + + const updateRes = client.updateFile(fileId, revn, vern, client.sessionId, changes); + if (updateRes.status !== 200) { + console.error(`Failed to add batch at ${added}: ${JSON.stringify(updateRes.body)}`); + return null; + } + + // update-file returns {revn, lagged} but not vern + // vern only changes on snapshot restore, so keep the original + revn = updateRes.body.revn; + added += batchCount; + } + + return { revn, vern }; +} + +// --------------------------------------------------------------------------- +// Setup — create files with different shape counts +// --------------------------------------------------------------------------- + +export function setup() { + console.log(`File Size Matrix Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` Iterations/tier: ${ITERATIONS_PER_TIER}`); + console.log(` Tiers: ${TIERS.map(t => `${t.name}(${t.shapes})`).join(", ")}`); + console.log(``); + + const client = createClient(BASE_URL); + if (client.getProfile().status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + // Create demo profile + const userRes = client.rpc("POST", "create-demo-profile", {}); + if (userRes.status !== 200) fail("Failed to create demo profile"); + const user = userRes.json(); + console.log(` Created demo profile: ${user.email}`); + + // Login + if (client.login(user.email, user.password).status !== 200) fail("Login failed"); + const teamId = client.getTeams().body[0].id; + const projectId = client.createProject(teamId, "File Size Matrix Project").body.id; + console.log(` Project: ${projectId}`); + + // Create and populate files for each tier + const tiers = []; + + for (const tier of TIERS) { + console.log(`\n Creating ${tier.name} file (${tier.shapes} shapes)...`); + + // Create file + const fileRes = client.createFile(projectId, `Matrix ${tier.name} (${tier.shapes} shapes)`); + if (fileRes.status !== 200) fail(`Failed to create ${tier.name} file`); + const fileId = fileRes.body.id; + + // Get page ID + const getFileRes = client.getFile(fileId); + if (getFileRes.status !== 200) fail(`Failed to get ${tier.name} file`); + const pageId = getFileRes.body.data.pages[0]; + + // Populate with shapes + const startTime = Date.now(); + const result = populateFile(client, fileId, pageId, tier.shapes, tier.color); + if (!result) fail(`Failed to populate ${tier.name} file`); + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + + console.log(` ${tier.name}: ${tier.shapes} shapes in ${elapsed}s (revn=${result.revn})`); + + tiers.push({ + name: tier.name, + shapes: tier.shapes, + fileId, + pageId, + revn: result.revn, + vern: result.vern, + }); + } + + console.log(`\n Setup complete. ${tiers.length} files ready.`); + + return { baseUrl: BASE_URL, user, tiers, iterationsPerTier: ITERATIONS_PER_TIER }; +} + +// --------------------------------------------------------------------------- +// Main VU Function — benchmark each tier +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Login + if (!assertOk(client.login(data.user.email, data.user.password), "login")) fail("login failed"); + sleep(0.5); + + console.log(`\n=== Starting benchmark (${data.iterationsPerTier} iterations per tier) ===\n`); + + // Benchmark each tier + for (const tier of data.tiers) { + console.log(`--- Tier: ${tier.name} (${tier.shapes} shapes) ---`); + + // Get latest file state + const getFileRes = client.getFile(tier.fileId); + if (!assertOk(getFileRes, `get-file-${tier.name}`)) continue; + let { revn, vern } = getFileRes.body; + + for (let i = 0; i < data.iterationsPerTier; i++) { + // Benchmark get-file + const getRes = client.getFile(tier.fileId); + if (!assertOk(getRes, `get-file-${tier.name}`)) continue; + + sleep(0.2); + + // Benchmark update-file (add 1 shape) + const change = makeAddRectChange(tier.pageId, tier.shapes + i, "#ffaa00"); + const updateRes = client.updateFile(tier.fileId, getRes.body.revn, getRes.body.vern, client.sessionId, [change]); + + if (updateRes.status !== 200) { + console.error(`update-file failed on ${tier.name} iteration ${i}: ${JSON.stringify(updateRes.body)}`); + continue; + } + + // update-file returns {revn, lagged} but not vern + // vern only changes on snapshot restore, so keep the original + revn = updateRes.body.revn; + + sleep(0.3); + } + + console.log(` Completed ${data.iterationsPerTier} iterations on ${tier.name}`); + } + + console.log(`\n=== Benchmark complete ===`); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log(`File size matrix test complete.`); +} diff --git a/plans/2026-06-12-backend-performance-test.md b/plans/2026-06-12-backend-performance-test.md index 74887a1640..4fc6bcc6a8 100644 --- a/plans/2026-06-12-backend-performance-test.md +++ b/plans/2026-06-12-backend-performance-test.md @@ -17,7 +17,7 @@ ### Completed (2026-06-12) -Phase 1 done. Phase 2 done (all core flows + performance optimization). Phase 3 done (orchestrator). Phase 4 concurrent editing done. Phase 4 file size matrix and Phase 5 remain. +Phase 1 done. Phase 2 done (all core flows + performance optimization). Phase 3 done (orchestrator). Phase 4 done (concurrent editing + file size matrix). Phase 5 remains. **What was built:** @@ -32,6 +32,7 @@ performance/ │ ├── workspace-open.js # Read-heavy: file open loop (get-file, libraries, thumbnails) │ ├── workspace-edit.js # Write-heavy: file edit loop (get-file + update-file) │ ├── workspace-edit-concurrent.js # Concurrent editing: same-file or multi-file mode +│ ├── file-size-matrix.js # File size matrix: latency vs shape count (10, 100, 500, 1000) │ ├── media-upload.js # Image uploads: SVG/PNG direct, JPG chunked │ └── font-upload.js # Font uploads: TTF+OTF chunked, create-font-variant ├── results/ # k6 JSON output (gitignored) @@ -98,6 +99,8 @@ Setup is sequential (~0.13ms/user with `derive-password-weak`), excluded from k6 14. **revn conflicts don't happen in normal concurrent editing.** The conflict check in `files_update.clj` is `(> incoming stored)` — only fires when incoming revn is *greater* than stored. If two VUs both read revn=5 and VU A saves first (revn becomes 6), VU B saves with revn=5 → `5 > 6?` → false → no conflict. The real contention point is the **file-level advisory lock** (`db/xact-lock! conn id`) that serializes all `update-file` calls on the same file. More VUs = more lock queuing = higher latency. +15. **`update-file` response doesn't include `vern`.** The response is `{:revn N, :lagged [...]}`. `vern` only changes on snapshot restore, so it can be kept constant across iterations. Get it from the initial `get-file` call. + ### Remaining Work | Phase | Status | Next Actions | @@ -107,13 +110,13 @@ Setup is sequential (~0.13ms/user with `derive-password-weak`), excluded from k6 | Phase 2 – Performance Optimization | **Done** | `derive-password-weak` using pbkdf2+sha256 (100 iter) — ~700x faster than argon2id | | Phase 3 – Scenarios | **Done** | `./run.sh all` runs all flows in parallel | | Phase 4 – Concurrent Editing | **Done** | `workspace-edit-concurrent.js` with same-file and multi-file modes | -| Phase 4 – File Size Matrix | **Not started** | `update-file` latency vs shape count: 10, 100, 500, 1000 shapes | +| Phase 4 – File Size Matrix | **Done** | `file-size-matrix.js` with 4 tiers (10, 100, 500, 1000 shapes) | | Phase 5 – CI & Reporting | **Not started** | Grafana dashboards, regression guard | ### Immediate Next Steps 1. ~~Phase 2 – Fast password for demo users~~ ✅ Done -2. Phase 4: File size matrix (`update-file` latency vs shape count: 10, 100, 500, 1000 shapes). +2. ~~Phase 4: File size matrix (`update-file` latency vs shape count: 10, 100, 500, 1000 shapes).~~ ✅ Done — `file-size-matrix.js` with 4 tiers 3. ~~Phase 4: Concurrent editing test (2–3 VUs per file, measure conflict rate).~~ ✅ Done — `workspace-edit-concurrent.js` with same-file and multi-file modes 4. Phase 5: Grafana dashboard panels (p95 latency by RPC, error rate, JVM, DB pool). 5. ~~Add `--scenario` flag to `run.sh`~~ ✅ Done @@ -578,5 +581,5 @@ Run `workspace-edit.js` against each tier separately and plot: --- **Plan Author:** Senior Software Architect -**Status:** Phase 1–4 complete. Concurrent editing implemented (same-file + multi-file modes). File size matrix and Phase 5 remain. +**Status:** Phase 1–4 complete. All core scripts implemented. Phase 5 (CI & Reporting) remains.