diff --git a/.github/workflows/perf-regression.yml b/.github/workflows/perf-regression.yml new file mode 100644 index 0000000000..9c3928ead5 --- /dev/null +++ b/.github/workflows/perf-regression.yml @@ -0,0 +1,201 @@ +name: "CI: Performance Regression" + +defaults: + run: + shell: bash + +on: + pull_request: + paths: + - 'backend/src/**' + - 'common/src/**' + + types: + - opened + - synchronize + - ready_for_review + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + perf-regression: + if: ${{ !github.event.pull_request.draft }} + name: "Performance Regression Check" + runs-on: penpot-runner-02 + container: + image: penpotapp/devenv:latest + volumes: + - /var/cache/github-runner/m2:/root/.m2 + - /var/cache/github-runner/gitlib:/root/.gitlibs + + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: penpot + POSTGRES_PASSWORD: penpot + POSTGRES_DB: penpot + + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + redis: + image: valkey/valkey:9 + + env: + PENPOT_DATABASE_URI: "postgresql://postgres/penpot" + PENPOT_DATABASE_USERNAME: penpot + PENPOT_DATABASE_PASSWORD: penpot + PENPOT_REDIS_URI: "redis://redis/1" + PENPOT_FLAGS: "enable-demo-users enable-backend-api-doc" + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install k6 + run: | + curl -sSL https://dl.k6.io/key.gpg | gpg --dearmor -o /usr/share/keyrings/k6-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list + apt-get update + apt-get install -y k6 + + - name: Save performance suite + run: cp -r backend/performance /tmp/performance + + - name: Cache Maven dependencies + uses: actions/cache@v4 + with: + path: | + ~/.m2 + ~/.gitlibs + key: ${{ runner.os }}-m2-${{ hashFiles('backend/deps.edn', 'common/deps.edn') }} + restore-keys: | + ${{ runner.os }}-m2- + + # ------------------------------------------------------------------------- + # Run performance tests on BASE branch (before change) + # ------------------------------------------------------------------------- + + - name: Checkout base branch + run: | + git fetch origin ${{ github.event.pull_request.base.ref }} + git checkout origin/${{ github.event.pull_request.base.ref }} + + - name: Restore performance suite (base) + run: cp -r /tmp/performance backend/performance + + - name: Start backend (base branch) + working-directory: backend + run: | + clojure -M:dev -m app.main & + # Wait for backend to be ready + for i in $(seq 1 30); do + if curl -s http://localhost:6060/api/rpc/command/get-profile > /dev/null 2>&1; then + echo "Backend ready" + break + fi + echo "Waiting for backend... ($i/30)" + sleep 2 + done + + - name: Run performance tests (baseline) + working-directory: backend/performance + run: | + mkdir -p results/baseline + ./run.sh smoke + ./run.sh lifecycle -v 5 -n 10 + cp -r results/latest/* results/baseline/ 2>/dev/null || true + + - name: Save baseline results + run: cp -r backend/performance/results /tmp/results-baseline + + - name: Stop backend + run: | + pkill -f "app.main" || true + sleep 2 + + - name: Clean untracked files + run: git clean -fd + + # ------------------------------------------------------------------------- + # Run performance tests on PR branch (after change) + # ------------------------------------------------------------------------- + + - name: Checkout PR branch + run: | + git checkout ${{ github.event.pull_request.head.sha }} + + - name: Restore baseline results + run: cp -r /tmp/results-baseline backend/performance/results + + - name: Start backend (PR branch) + working-directory: backend + run: | + clojure -M:dev -m app.main & + # Wait for backend to be ready + for i in $(seq 1 30); do + if curl -s http://localhost:6060/api/rpc/command/get-profile > /dev/null 2>&1; then + echo "Backend ready" + break + fi + echo "Waiting for backend... ($i/30)" + sleep 2 + done + + - name: Run performance tests (current) + working-directory: backend/performance + run: | + mkdir -p results/current + ./run.sh smoke + ./run.sh lifecycle -v 5 -n 10 + # Copy results + cp -r results/latest/* results/current/ 2>/dev/null || true + + - name: Stop backend + run: | + pkill -f "app.main" || true + sleep 2 + + # ------------------------------------------------------------------------- + # Compare results + # ------------------------------------------------------------------------- + + - name: Compare results + working-directory: backend/performance + run: | + BASELINE=$(find results/baseline -name "k6-summary.json" | head -1) + CURRENT=$(find results/current -name "k6-summary.json" | head -1) + + if [ -z "$BASELINE" ] || [ -z "$CURRENT" ]; then + echo "Warning: Could not find k6 summary files" + echo "Baseline: $BASELINE" + echo "Current: $CURRENT" + exit 0 + fi + + echo "Comparing:" + echo " Baseline: $BASELINE" + echo " Current: $CURRENT" + echo "" + + node scripts/compare-results.cjs "$BASELINE" "$CURRENT" --threshold 20 + + # ------------------------------------------------------------------------- + # Upload artifacts + # ------------------------------------------------------------------------- + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: performance-results + path: backend/performance/results/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index 5bc58cd0b2..3995dc8f68 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ /backend/target/ /backend/experiments /backend/scripts/_env.local +/backend/performance/results/ /bundle* /clj-profiler/ /common/coverage @@ -100,3 +101,4 @@ /opencode.json /.codex/ /tools/__pycache__ +/performance/results/ \ No newline at end of file diff --git a/backend/performance/README.md b/backend/performance/README.md new file mode 100644 index 0000000000..28b5d546eb --- /dev/null +++ b/backend/performance/README.md @@ -0,0 +1,128 @@ +# Penpot Performance Tests + +k6-based load and performance test suite for the Penpot backend. Measures HTTP RPC latency, throughput, and error rates under synthetic user load. + +## Prerequisites + +- **k6** — Install from https://k6.io/docs/get-started/installation/ (also included in `devenv` image) +- **Running Penpot backend** — Local devenv (`http://localhost:6060`) or a remote instance + +## Quick Start + +```bash +# Smoke test — 1 VU, 1 iteration, demo mode +./run.sh smoke + +# Full lifecycle with 10 VUs, 5 iterations each +./run.sh lifecycle -v 10 -n 5 + +# Use registration flow instead of demo profiles +./run.sh lifecycle -m register -v 5 -n 1 + +# Point to a remote backend +./run.sh lifecycle -u https://penpot.example.com + +# Show all options +./run.sh help +``` + +## Commands + +| Command | Description | +|---|---| +| `smoke` | 1 VU, 1 iteration smoke test of the lifecycle flow | +| `lifecycle` | Full user lifecycle (register → CRUD → delete) | +| `workspace-open` | Read-heavy: repeatedly open a file (get-file, libraries, thumbnails) | +| `workspace-edit` | Write-heavy: repeatedly edit a file (get-file + update-file loop) | +| `media-upload` | Upload images of varying sizes (direct + chunked) | +| `font-upload` | Upload fonts via chunked upload + create-font-variant | +| `concurrent-edit` | Concurrent editing: same-file or multi-file mode | +| `file-size-matrix` | Measure latency vs file size (10, 100, 500, 1000 shapes) | +| `compare` | Compare two k6 JSON results for regression | +| `all` | Run all scenarios together (orchestrator) | +| `clean` | Remove test results | + +## Options + +| Flag | Env Variable | Default | Description | +|---|---|---|---| +| `-u URL` | `PENPOT_BASE_URL` | `http://localhost:6060` | Penpot backend URL | +| `-v NUM` | — | per-script default | Number of virtual users | +| `-n NUM` | — | per-script default | k6 iterations | +| `-d DUR` | `PENPOT_DURATION` | k6 default | Test duration (e.g. `30s`, `5m`, `2h`) | +| `-m MODE` | `PENPOT_REGISTER_MODE` | `demo` | Register mode: `demo` or `register` | +| `-k PATH` | `K6` | `k6` | Path to k6 binary | + +### Concurrent-edit / file-size-matrix options + +| Flag | Env Variable | Default | Description | +|---|---|---|---| +| `--mode MODE` | `PENPOT_EDIT_MODE` | `same-file` | `same-file` or `multi-file` | +| `--files NUM` | `PENPOT_FILE_COUNT` | `1` | Number of files for multi-file mode | +| `--vus-per-file NUM` | `PENPOT_VUS_PER_FILE` | `1` | VUs per file for multi-file mode | +| `--edit-iterations NUM` | `PENPOT_EDIT_ITERATIONS` | `10` | Per-VU edit loop iterations | + +`--edit-iterations` controls the per-VU edit loop in both `concurrent-edit` and `file-size-matrix`. It is **independent** of `-n` (which controls k6's shared-iterations executor). + +### Register Modes + +- **`demo`** (default): Uses the `create-demo-profile` RPC endpoint. Requires the `demo-users` feature flag to be enabled on the backend. Fastest for testing. +- **`register`**: Uses the full two-step registration flow (`prepare-register-profile` + `register-profile`). Works without any feature flags but is slower. + +## Examples + +```bash +# Same-file concurrent edit: 5 VUs editing the same file +./run.sh concurrent-edit --mode same-file -v 5 -n 10 --edit-iterations 20 + +# Multi-file concurrent edit: 3 files, 4 VUs each +./run.sh concurrent-edit --mode multi-file --files 3 --vus-per-file 4 -n 10 + +# File size matrix: 50 iterations per size tier +./run.sh file-size-matrix --edit-iterations 50 + +# Duration-based test: 5 VUs for 30 seconds +./run.sh lifecycle -v 5 -d 30s + +# Run all scenarios with 50 VUs +./run.sh all -v 50 + +# Compare baseline vs current results +./run.sh compare results/baseline/20250625-120000-lifecycle/k6-summary.json \ + results/current/20250625-130000-lifecycle/k6-summary.json +``` + +## Shared Client (`lib/penpot-client.js`) + +The shared client module wraps the Penpot backend RPC API using plain JSON (not Transit). Key features: + +- **JSON transport**: Uses `Content-Type: application/json` for POST bodies and `Accept: application/json` (or `_fmt=json` for GET) for responses. +- **Cookie-based auth**: k6 automatically manages session cookies per VU. +- **Session headers**: Generates `x-session-id` and `x-external-session-id` UUIDs per VU. +- **Tagged metrics**: Every request is tagged with `rpc_command` for k6 metric slicing. + +## Results + +Test results are written to `results//` as JSON. k6 also prints a summary to stdout with percentile breakdowns per RPC command. + +## Thresholds + +Each script includes built-in thresholds that cause k6 to exit with a non-zero code if exceeded: + +- `http_req_duration p95 < 5000ms` (global) +- `http_req_failed < 1%` (global) +- Per-command thresholds for login, profile, project, file, and update operations + +## Adding New Flows + +1. Create `scripts/.js` +2. Import the shared client: `import { createClient } from "../lib/penpot-client.js";` +3. Implement the flow using the client methods +4. Add a command in `run.sh` + +## Architecture Notes + +- The backend supports both Transit JSON and plain JSON. This test suite uses **plain JSON** for simplicity (no Transit encoder needed in k6). +- JSON request keys are in **kebab-case** (matching Clojure conventions). JSON response keys are in **camelCase** (backend's default JSON encoding). +- `update-file` sends the `id` parameter both in the query string and in the POST body, matching the frontend's behavior. +- The backend uses optimistic concurrency control (`revn`) for file updates. The test retries once on conflict. diff --git a/backend/performance/lib/penpot-client.js b/backend/performance/lib/penpot-client.js new file mode 100644 index 0000000000..0f8dfe26c0 --- /dev/null +++ b/backend/performance/lib/penpot-client.js @@ -0,0 +1,621 @@ +// Penpot k6 HTTP Client +// +// Shared module that wraps the Penpot backend RPC API using plain JSON. +// The backend supports `application/json` request bodies (kebab-case keys) +// and `application/json` responses (camelCase keys) via Accept header or _fmt=json. +// +// Authentication is cookie-based: login-with-password sets a session cookie, +// and all subsequent requests include it automatically via the k6 cookie jar. + +import http from "k6/http"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +/** + * Creates a new Penpot client instance. + * + * @param {string} baseUrl - The base URL of the Penpot backend (e.g., "http://localhost:6060") + * @returns {object} Client instance with RPC methods + */ +export function createClient(baseUrl) { + // Per-VU session identifiers — consistent across all requests within one VU iteration + const sessionId = uuidv4(); + const externalSessionId = uuidv4(); + + const defaultHeaders = { + "Accept": "application/json", + "x-session-id": sessionId, + "x-external-session-id": externalSessionId, + "x-event-origin": "perf-test", + "x-client": "penpot-perf/1.0", + }; + + // k6 automatically manages cookies per VU when `cookies` are returned by the server. + // We use the default cookie jar which is per-VU. + + /** + * Make an RPC call to the Penpot backend. + * + * GET requests: params go as query parameters, response is JSON via _fmt=json. + * POST requests: params go as JSON body, response is JSON via Accept header. + * + * @param {string} method - HTTP method ("GET" or "POST") + * @param {string} command - RPC command name (e.g., "login-with-password") + * @param {object} params - Parameters for the RPC call + * @param {object} [opts] - Additional options + * @param {string} [opts.tag] - k6 metric tag for this request + * @returns {object} k6 Response object with parsed JSON body + */ + function rpc(method, command, params = {}, opts = {}) { + const url = `${baseUrl}/api/main/methods/${command}`; + const tag = opts.tag || command; + + const tags = { + rpc_command: tag, + }; + + if (method === "GET") { + // GET requests: params go as query string, add _fmt=json for JSON response + const queryParams = { ...params, _fmt: "json" }; + const qs = Object.entries(queryParams) + .filter(([, v]) => v !== undefined && v !== null) + .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`) + .join("&"); + + const fullUrl = qs ? `${url}?${qs}` : url; + + return http.get(fullUrl, { + headers: defaultHeaders, + tags, + }); + } else { + // POST requests: params go as JSON body + const headers = { + ...defaultHeaders, + "Content-Type": "application/json", + }; + + return http.post(url, JSON.stringify(params), { + headers, + tags, + }); + } + } + + /** + * Login with email and password. + * Returns the profile data on success. The session cookie is stored + * automatically by k6's cookie jar. + * + * @param {string} email + * @param {string} password + * @returns {object} Parsed response { status, body } + */ + function login(email, password) { + const res = rpc("POST", "login-with-password", { + email, + password, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get the current user's profile (requires prior login). + * + * @returns {object} Parsed response { status, body } + */ + function getProfile() { + const res = rpc("GET", "get-profile"); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get all teams for the current user. + * + * @returns {object} Parsed response { status, body } + */ + function getTeams() { + const res = rpc("GET", "get-teams"); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Create a new team. + * + * @param {string} name - Team name + * @returns {object} Parsed response { status, body } + */ + function createTeam(name) { + const res = rpc("POST", "create-team", { name }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get projects for a team. + * + * @param {string} teamId - Team UUID + * @returns {object} Parsed response { status, body } + */ + function getProjects(teamId) { + const res = rpc("GET", "get-projects", { "team-id": teamId }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Create a new project. + * + * @param {string} teamId - Team UUID + * @param {string} name - Project name + * @returns {object} Parsed response { status, body } + */ + function createProject(teamId, name) { + const res = rpc("POST", "create-project", { + "team-id": teamId, + name, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Create a new file in a project. + * + * @param {string} projectId - Project UUID + * @param {string} name - File name + * @returns {object} Parsed response { status, body } + */ + function createFile(projectId, name) { + const res = rpc("POST", "create-file", { + "project-id": projectId, + name, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get a file by ID. + * + * @param {string} fileId - File UUID + * @returns {object} Parsed response { status, body } + */ + function getFile(fileId) { + const res = rpc("GET", "get-file", { + id: fileId, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get libraries used by a file. + * + * @param {string} fileId - File UUID + * @returns {object} Parsed response { status, body } + */ + function getFileLibraries(fileId) { + const res = rpc("GET", "get-file-libraries", { + "file-id": fileId, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get object thumbnails for a file. + * + * @param {string} fileId - File UUID + * @returns {object} Parsed response { status, body } + */ + function getFileObjectThumbnails(fileId) { + const res = rpc("GET", "get-file-object-thumbnails", { + "file-id": fileId, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get file data for thumbnail generation. + * + * @param {string} fileId - File UUID + * @returns {object} Parsed response { status, body } + */ + function getFileDataForThumbnail(fileId) { + const res = rpc("GET", "get-file-data-for-thumbnail", { + "file-id": fileId, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Update a file with changes. + * + * The backend uses optimistic concurrency control via `revn`. + * If a conflict occurs (status 400 with :revn-conflict), the caller + * should retry with the latest revn from getFile(). + * + * @param {string} fileId - File UUID + * @param {number} revn - Current file revision number + * @param {number} vern - Current file version number + * @param {string} sessionId - Client session ID (UUID) + * @param {Array} changes - Array of change objects + * @returns {object} Parsed response { status, body } + */ + function updateFile(fileId, revn, vern, changesSessionId, changes) { + const params = { + id: fileId, + revn: revn, + vern: vern, + "session-id": changesSessionId, + origin: "workspace", + "created-at": new Date().toISOString(), + "commit-id": uuidv4(), + changes: changes, + }; + + // update-file uses POST with id also as query param (per frontend convention) + const url = `${baseUrl}/api/main/methods/update-file?id=${encodeURIComponent(fileId)}`; + const headers = { + ...defaultHeaders, + "Content-Type": "application/json", + }; + + const res = http.post(url, JSON.stringify(params), { + headers, + tags: { rpc_command: "update-file" }, + }); + + let body = null; + try { + if (res.body && res.body.length > 0) { + body = res.json(); + } + } catch (e) { + // body may not be JSON + } + + return { + status: res.status, + body: body, + raw: res, + }; + } + + /** + * Upload a file media object using direct multipart upload. + * + * @param {string} fileId - File UUID + * @param {Uint8Array} fileBytes - The file content + * @param {string} fileName - The file name + * @param {string} mimeType - MIME type (e.g., "image/png") + * @returns {object} Parsed response { status, body } + */ + function uploadFileMediaObjectDirect(fileId, fileBytes, fileName, mimeType) { + const url = `${baseUrl}/api/main/methods/upload-file-media-object`; + + const headers = { + ...defaultHeaders, + // No Content-Type — k6 sets it automatically for multipart/form-data + }; + + const formData = { + "file-id": fileId, + "is-local": "true", + name: fileName, + content: http.file(fileBytes, fileName, mimeType), + }; + + const res = http.post(url, formData, { + headers, + tags: { rpc_command: "upload-file-media-object" }, + }); + + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + // ----------------------------------------------------------------------- + // Chunked upload + // ----------------------------------------------------------------------- + + /** + * Create an upload session for chunked uploads. + * + * @param {number} totalChunks - Number of chunks + * @returns {object} { status, sessionId } + */ + function createUploadSession(totalChunks) { + const res = rpc("POST", "create-upload-session", { + "total-chunks": totalChunks, + }); + const body = res.status === 200 ? res.json() : null; + return { + status: res.status, + sessionId: body ? body.sessionId : null, + raw: res, + }; + } + + /** + * Upload a single chunk within an upload session. + * + * @param {string} sessionId - Upload session UUID + * @param {number} index - Chunk index (0-based) + * @param {Uint8Array} chunkBytes - The chunk content + * @param {string} fileName - Original file name + * @param {string} mimeType - MIME type + * @returns {object} Parsed response { status, body } + */ + function uploadChunk(sessionId, index, chunkBytes, fileName, mimeType) { + const url = `${baseUrl}/api/main/methods/upload-chunk`; + + const headers = { + ...defaultHeaders, + }; + + const formData = { + "session-id": sessionId, + index: String(index), + content: http.file(chunkBytes, fileName, mimeType), + }; + + const res = http.post(url, formData, { + headers, + tags: { rpc_command: "upload-chunk" }, + }); + + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Assemble all uploaded chunks into a final media object. + * + * @param {string} sessionId - Upload session UUID + * @param {string} fileId - File UUID + * @param {string} name - Media object name + * @param {boolean} isLocal - Whether the object is local to the file + * @param {string} mimeType - MIME type (e.g., "image/png") + * @returns {object} Parsed response { status, body } + */ + function assembleFileMediaObject(sessionId, fileId, name, isLocal, mimeType) { + const res = rpc("POST", "assemble-file-media-object", { + "session-id": sessionId, + "file-id": fileId, + name: name, + "is-local": isLocal, + mtype: mimeType, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + // ----------------------------------------------------------------------- + // Smart upload — picks direct or chunked based on file size + // ----------------------------------------------------------------------- + + // Chunk size threshold: files larger than this use chunked upload. + // The actual chunk size is irrelevant to the backend; this controls + // which upload path is exercised. + const CHUNK_SIZE = 50 * 1024; // 50 KB + + /** + * Upload a file media object, automatically selecting direct or chunked + * upload based on file size. + * + * Files <= CHUNK_SIZE use direct multipart upload. + * Files > CHUNK_SIZE use chunked upload (create-upload-session → + * upload-chunk × N → assemble-file-media-object). + * + * @param {string} fileId - File UUID + * @param {Uint8Array} fileBytes - The file content + * @param {string} fileName - The file name + * @param {string} mimeType - MIME type (e.g., "image/png") + * @returns {object} Parsed response { status, body } + */ + function uploadFileMediaObject(fileId, fileBytes, fileName, mimeType) { + if (fileBytes.byteLength <= CHUNK_SIZE) { + return uploadFileMediaObjectDirect(fileId, fileBytes, fileName, mimeType); + } + + // Chunked upload path + const totalChunks = Math.ceil(fileBytes.byteLength / CHUNK_SIZE); + + const sessionRes = createUploadSession(totalChunks); + if (sessionRes.status !== 200) { + return { status: sessionRes.status, body: null, raw: sessionRes.raw }; + } + const uploadSessionId = sessionRes.sessionId; + + for (let i = 0; i < totalChunks; i++) { + const start = i * CHUNK_SIZE; + const end = Math.min(start + CHUNK_SIZE, fileBytes.byteLength); + const chunk = fileBytes.slice(start, end); + + const chunkRes = uploadChunk(uploadSessionId, i, chunk, fileName, mimeType); + if (chunkRes.status !== 200) { + return { status: chunkRes.status, body: null, raw: chunkRes.raw }; + } + } + + return assembleFileMediaObject(uploadSessionId, fileId, fileName, true, mimeType); + } + + /** + * Delete a file. + * + * @param {string} fileId - File UUID + * @returns {object} Parsed response { status } + */ + function deleteFile(fileId) { + const res = rpc("POST", "delete-file", { id: fileId }); + return { + status: res.status, + raw: res, + }; + } + + /** + * Delete a project. + * + * @param {string} projectId - Project UUID + * @returns {object} Parsed response { status } + */ + function deleteProject(projectId) { + const res = rpc("POST", "delete-project", { id: projectId }); + return { + status: res.status, + raw: res, + }; + } + + /** + * Delete a team. + * + * @param {string} teamId - Team UUID + * @returns {object} Parsed response { status } + */ + function deleteTeam(teamId) { + const res = rpc("POST", "delete-team", { id: teamId }); + return { + status: res.status, + raw: res, + }; + } + + /** + * Invite members to a team by email. + * + * @param {string} teamId - Team UUID + * @param {string[]} emails - Array of email addresses + * @param {string} role - Role for the invited members (e.g. "editor") + * @returns {object} Parsed response { status, body } + */ + function inviteTeamMembers(teamId, emails, role) { + const res = rpc("POST", "create-team-invitations", { + "team-id": teamId, + emails: emails, + role: role, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Get an invitation token for a specific email. + * + * @param {string} teamId - Team UUID + * @param {string} email - Invited email address + * @returns {object} Parsed response { status, body } + */ + function getTeamInvitationToken(teamId, email) { + const res = rpc("GET", "get-team-invitation-token", { + "team-id": teamId, + email: email, + }); + return { + status: res.status, + body: res.status === 200 ? res.json() : null, + raw: res, + }; + } + + /** + * Logout the current user. + * + * @param {string} profileId - Profile UUID + * @returns {object} Parsed response { status } + */ + function logout(profileId) { + const res = rpc("POST", "logout", { "profile-id": profileId }); + return { + status: res.status, + raw: res, + }; + } + + // Return the client interface + return { + sessionId, + externalSessionId, + rpc, + login, + getProfile, + getTeams, + createTeam, + getProjects, + createProject, + createFile, + getFile, + getFileLibraries, + getFileObjectThumbnails, + getFileDataForThumbnail, + updateFile, + uploadFileMediaObject, + uploadFileMediaObjectDirect, + createUploadSession, + uploadChunk, + assembleFileMediaObject, + deleteFile, + deleteProject, + deleteTeam, + inviteTeamMembers, + getTeamInvitationToken, + logout, + }; +} diff --git a/backend/performance/run.sh b/backend/performance/run.sh new file mode 100755 index 0000000000..affe1fff03 --- /dev/null +++ b/backend/performance/run.sh @@ -0,0 +1,444 @@ +#!/usr/bin/env bash +# +# Penpot Performance Tests +# +# k6-based load/performance test suite for the Penpot backend. +# +# Prerequisites: +# - k6 (https://k6.io/) installed and in PATH +# - A running Penpot backend (local devenv or remote) +# +# Usage: +# ./run.sh smoke # 1 VU, 1 iteration smoke test +# ./run.sh lifecycle # Full user lifecycle +# ./run.sh workspace-open # Read-heavy file open flow +# ./run.sh workspace-edit # Write-heavy file edit loop +# ./run.sh media-upload # Direct + chunked image uploads +# ./run.sh font-upload # Chunked font upload + variant creation +# ./run.sh concurrent-edit # Concurrent editing (same-file or multi-file) +# ./run.sh all # Run all scenarios together (orchestrator) +# ./run.sh clean # Remove test results + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- + +BASE_URL="${PENPOT_BASE_URL:-http://localhost:6060}" +VUS="" +ITER="" +DURATION="" +REGISTER_MODE="${PENPOT_REGISTER_MODE:-demo}" +K6="${K6:-k6}" +EDIT_MODE="${PENPOT_EDIT_MODE:-same-file}" +FILE_COUNT="${PENPOT_FILE_COUNT:-1}" +VUS_PER_FILE="${PENPOT_VUS_PER_FILE:-1}" +EDIT_ITERATIONS="" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +usage() { + cat < [options] + +Commands: + smoke 1 VU, 1 iteration smoke test of the lifecycle flow + lifecycle Full user lifecycle (register → CRUD → delete) + workspace-open Read-heavy: repeatedly open a file (get-file, libraries, thumbnails) + workspace-edit Write-heavy: repeatedly edit a file (get-file + update-file loop) + media-upload Upload images of varying sizes (direct + chunked) + font-upload Upload fonts via chunked upload + create-font-variant + concurrent-edit Concurrent editing: same-file or multi-file mode + file-size-matrix Measure latency vs file size (10, 100, 500, 1000 shapes) + compare Compare two k6 JSON results for regression + all Run all scenarios together (orchestrator) + clean Remove test results + help Show this help + +Options: + -u URL Backend base URL (default: $BASE_URL) + -v NUM Number of virtual users (default: per-script defaults) + -n NUM Iterations per VU (default: per-script defaults) + -d DURATION Test duration (e.g. 30s, 5m, 2h; default: k6 default) + -m MODE Register mode: 'demo' or 'register' (default: $REGISTER_MODE) + -k PATH Path to k6 binary (default: $K6) + +Concurrent-edit options: + --mode MODE 'same-file' or 'multi-file' (default: $EDIT_MODE) + --files NUM Number of files for multi-file mode (default: $FILE_COUNT) + --vus-per-file NUM VUs per file for multi-file mode (default: $VUS_PER_FILE) + --edit-iterations NUM Per-VU edit loop iterations (concurrent-edit, file-size-matrix; default: 10) + +Environment variables: + PENPOT_BASE_URL Same as -u + PENPOT_REGISTER_MODE Same as -m + PENPOT_EDIT_MODE Same as --mode + PENPOT_FILE_COUNT Same as --files + PENPOT_VUS_PER_FILE Same as --vus-per-file + PENPOT_EDIT_ITERATIONS Same as --edit-iterations + K6 Same as -k + PENPOT_DURATION Same as -d + +Examples: + $(basename "$0") smoke + $(basename "$0") lifecycle -v 5 -n 10 + $(basename "$0") workspace-edit -v 20 -n 50 + $(basename "$0") media-upload -u https://penpot.example.com + $(basename "$0") concurrent-edit --mode same-file -v 5 -n 10 + $(basename "$0") concurrent-edit --mode multi-file --files 3 --vus-per-file 2 -n 10 + $(basename "$0") file-size-matrix -n 10 + $(basename "$0") all -v 50 +EOF +} + +check_k6() { + if ! command -v "$K6" &>/dev/null; then + echo "Error: k6 not found at '$K6'" >&2 + echo "Install from https://k6.io/docs/get-started/installation/" >&2 + exit 1 + fi +} + +# Build k6 env flags +k6_env_flags() { + local flags="--env PENPOT_BASE_URL=$BASE_URL --env PENPOT_REGISTER_MODE=$REGISTER_MODE --env PENPOT_EDIT_MODE=$EDIT_MODE --env PENPOT_FILE_COUNT=$FILE_COUNT --env PENPOT_VUS_PER_FILE=$VUS_PER_FILE" + if [[ -n "${PENPOT_TOTAL_VUS:-}" ]]; then + flags="$flags --env PENPOT_TOTAL_VUS=$PENPOT_TOTAL_VUS" + fi + if [[ -n "$VUS" ]]; then + flags="$flags --env K6_VUS=$VUS" + fi + if [[ -n "$ITER" ]]; then + flags="$flags --env K6_ITERATIONS=$ITER" + fi + if [[ -n "$EDIT_ITERATIONS" ]]; then + flags="$flags --env PENPOT_EDIT_ITERATIONS=$EDIT_ITERATIONS" + fi + echo "$flags" +} + +# Build k6 VU/iteration/duration flags (only if explicitly set) +k6_scale_flags() { + local flags="" + if [[ -n "$VUS" ]]; then + flags="$flags --vus $VUS" + fi + if [[ -n "$ITER" ]]; then + flags="$flags --iterations $ITER" + elif [[ -n "$VUS" && -z "$DURATION" ]]; then + # k6 requires iterations/duration/stages alongside --vus. + # When only -v is given, default iterations to VUs so + # iterations >= VUs (k6 constraint for shared-iterations). + flags="$flags --iterations $VUS" + fi + if [[ -n "$DURATION" ]]; then + flags="$flags --duration $DURATION" + fi + echo "$flags" +} + +# Run a single k6 script +run_script() { + local script="$1" + local label="$2" + local results_dir="$SCRIPT_DIR/results/$(date +%Y%m%d-%H%M%S)-${label}" + mkdir -p "$results_dir" + + echo "" + echo "=== $label ===" + echo " Script: scripts/${script}" + echo " Base URL: $BASE_URL" + echo " Register mode: $REGISTER_MODE" + [[ -n "$VUS" ]] && echo " VUs: $VUS" + [[ -n "$ITER" ]] && echo " Iterations: $ITER" + echo " Results: $results_dir" + echo "" + + # shellcheck disable=SC2046 + $K6 run \ + $(k6_env_flags) \ + $(k6_scale_flags) \ + --out "json=$results_dir/k6-summary.json" \ + "$SCRIPT_DIR/scripts/${script}" +} + +# Run all scenarios as parallel k6 processes +run_all() { + local results_dir="$SCRIPT_DIR/results/$(date +%Y%m%d-%H%M%S)-all" + mkdir -p "$results_dir" + + local default_vus="${VUS:-10}" + + echo "" + echo "=== Penpot Performance Orchestrator ===" + echo " Base URL: $BASE_URL" + echo " Total VUs: $default_vus (distributed across flows)" + echo " Results: $results_dir" + echo "" + echo " Flow distribution:" + echo " lifecycle: 2 VUs (full CRUD)" + echo " workspace-open: 3 VUs (read-heavy)" + echo " workspace-edit: 3 VUs (write-heavy)" + echo " media-upload: 1 VU (storage I/O)" + echo " font-upload: 1 VU (CPU/storage)" + echo "" + + local pids=() + + # Lifecycle — full CRUD + $K6 run \ + $(k6_env_flags) \ + --vus 2 --iterations 2 \ + --env "PENPOT_OPEN_ITERATIONS=3" \ + --out "json=$results_dir/lifecycle.json" \ + "$SCRIPT_DIR/scripts/lifecycle.js" & + pids+=($!) + + # Workspace open — read-heavy + $K6 run \ + $(k6_env_flags) \ + --vus 3 --iterations 3 \ + --env "PENPOT_OPEN_ITERATIONS=3" \ + --out "json=$results_dir/workspace-open.json" \ + "$SCRIPT_DIR/scripts/workspace-open.js" & + pids+=($!) + + # Workspace edit — write-heavy + $K6 run \ + $(k6_env_flags) \ + --vus 3 --iterations 5 \ + --env "PENPOT_EDIT_ITERATIONS=5" \ + --out "json=$results_dir/workspace-edit.json" \ + "$SCRIPT_DIR/scripts/workspace-edit.js" & + pids+=($!) + + # Media upload + $K6 run \ + $(k6_env_flags) \ + --vus 1 --iterations 2 \ + --out "json=$results_dir/media-upload.json" \ + "$SCRIPT_DIR/scripts/media-upload.js" & + pids+=($!) + + # Font upload + $K6 run \ + $(k6_env_flags) \ + --vus 1 --iterations 2 \ + --out "json=$results_dir/font-upload.json" \ + "$SCRIPT_DIR/scripts/font-upload.js" & + pids+=($!) + + # Wait for all and collect exit codes + local failed=0 + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + failed=$((failed + 1)) + fi + done + + echo "" + if [[ $failed -gt 0 ]]; then + echo "WARNING: $failed flow(s) had non-zero exit codes" + fi + echo "Results saved to: $results_dir" +} + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + +cmd_smoke() { + check_k6 + REGISTER_MODE=demo + VUS=1 + ITER=1 + run_script "lifecycle.js" "smoke" +} + +cmd_lifecycle() { check_k6; run_script "lifecycle.js" "lifecycle"; } +cmd_workspace_open() { check_k6; run_script "workspace-open.js" "workspace-open"; } +cmd_workspace_edit() { check_k6; run_script "workspace-edit.js" "workspace-edit"; } +cmd_media_upload() { check_k6; run_script "media-upload.js" "media-upload"; } +cmd_font_upload() { check_k6; run_script "font-upload.js" "font-upload"; } +cmd_all() { check_k6; run_all; } + +cmd_concurrent_edit() { + check_k6 + + local label="concurrent-edit-${EDIT_MODE}" + if [[ "$EDIT_MODE" == "multi-file" ]]; then + label="${label}-${FILE_COUNT}files-${VUS_PER_FILE}vpu" + fi + + echo "" + echo "=== Concurrent Edit ($EDIT_MODE) ===" + echo " Mode: $EDIT_MODE" + if [[ "$EDIT_MODE" == "multi-file" ]]; then + echo " Files: $FILE_COUNT" + echo " VUs per file: $VUS_PER_FILE" + VUS=$((FILE_COUNT * VUS_PER_FILE)) + echo " Total VUs: $VUS" + else + [[ -n "$VUS" ]] && echo " VUs: $VUS" + fi + [[ -n "$ITER" ]] && echo " Iterations: $ITER" + echo "" + + # For same-file mode, pass VUS as PENPOT_TOTAL_VUS so setup() knows how many pages to create + if [[ "$EDIT_MODE" == "same-file" && -n "$VUS" ]]; then + export PENPOT_TOTAL_VUS="$VUS" + fi + + run_script "workspace-edit-concurrent.js" "$label" +} + +cmd_file_size_matrix() { + check_k6 + + echo "" + echo "=== File Size Matrix ===" + echo " Tiers: small(10), medium(100), large(500), xlarge(1000)" + [[ -n "$EDIT_ITERATIONS" ]] && echo " Iterations: $EDIT_ITERATIONS (per tier)" + echo "" + + run_script "file-size-matrix.js" "file-size-matrix" +} + +cmd_compare() { + local baseline="$1" + local current="$2" + local threshold="${3:-20}" + + if [[ -z "$baseline" || -z "$current" ]]; then + echo "Usage: ./run.sh compare [threshold]" + echo "" + echo "Compare two k6 JSON results for performance regression." + echo "" + echo "Arguments:" + echo " baseline.json k6 JSON output from base branch" + echo " current.json k6 JSON output from PR branch" + echo " threshold Fail if p95 increases > N% (default: 20)" + exit 1 + fi + + if [[ ! -f "$baseline" ]]; then + echo "Error: Baseline file not found: $baseline" >&2 + exit 1 + fi + if [[ ! -f "$current" ]]; then + echo "Error: Current file not found: $current" >&2 + exit 1 + fi + + node "$SCRIPT_DIR/scripts/compare-results.cjs" "$baseline" "$current" --threshold "$threshold" +} + +cmd_clean() { + local results_dir="$SCRIPT_DIR/results" + if [[ -d "$results_dir" ]]; then + rm -rf "$results_dir" + echo "Cleaned $results_dir" + else + echo "Nothing to clean" + fi +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Parse global options first (before command) +parse_opts() { + # First, extract long options (--mode, --files, --vus-per-file) + local args=() + while [[ $# -gt 0 ]]; do + case "$1" in + --mode) + EDIT_MODE="$2" + shift 2 + ;; + --files) + FILE_COUNT="$2" + shift 2 + ;; + --vus-per-file) + VUS_PER_FILE="$2" + shift 2 + ;; + --edit-iterations) + EDIT_ITERATIONS="$2" + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac + done + + # Apply PENPOT_DURATION env var as default (before CLI parsing takes precedence) + if [[ -z "$DURATION" && -n "${PENPOT_DURATION:-}" ]]; then + DURATION="$PENPOT_DURATION" + fi + + # Now parse short options with getopts + set -- "${args[@]}" + OPTIND=1 + while getopts "u:v:n:d:m:k:h" opt; do + case "$opt" in + u) BASE_URL="$OPTARG" ;; + v) VUS="$OPTARG" ;; + n) ITER="$OPTARG" ;; + d) DURATION="$OPTARG" ;; + m) REGISTER_MODE="$OPTARG" ;; + k) K6="$OPTARG" ;; + h) usage; exit 0 ;; + *) usage >&2; exit 1 ;; + esac + done +} + +if [[ $# -lt 1 ]]; then + usage >&2 + exit 1 +fi + +command="$1" +shift + +# Parse options for flow commands (not smoke/clean/help/all) +case "$command" in + smoke|clean|help|-h|--help) + ;; + *) + parse_opts "$@" + ;; +esac + +case "$command" in + smoke) cmd_smoke ;; + lifecycle) cmd_lifecycle ;; + workspace-open) cmd_workspace_open ;; + workspace-edit) cmd_workspace_edit ;; + media-upload) cmd_media_upload ;; + font-upload) cmd_font_upload ;; + concurrent-edit) cmd_concurrent_edit ;; + file-size-matrix) cmd_file_size_matrix ;; + compare) cmd_compare "$@" ;; + all) cmd_all ;; + clean) cmd_clean ;; + help|-h|--help) usage ;; + *) + echo "Unknown command: $command" >&2 + usage >&2 + exit 1 + ;; +esac diff --git a/backend/performance/scripts/compare-results.cjs b/backend/performance/scripts/compare-results.cjs new file mode 100644 index 0000000000..c7e3e6c052 --- /dev/null +++ b/backend/performance/scripts/compare-results.cjs @@ -0,0 +1,270 @@ +#!/usr/bin/env node +// +// compare-results.js +// +// Compares two k6 JSON output files and reports performance regressions. +// Used for relative comparison: base branch vs PR branch in the same CI run. +// +// Usage: +// node scripts/compare-results.js +// node scripts/compare-results.js --threshold 20 +// +// Exit codes: +// 0 - No regressions detected +// 1 - Regression detected (p95 increased > threshold) +// 2 - Error (invalid input, missing file, etc.) + +const fs = require("fs"); +const path = require("path"); + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const DEFAULT_THRESHOLD = 20; // Fail if p95 increases > 20% +const CRITICAL_COMMANDS = [ + "get-file", + "update-file", + "login-with-password", + "create-demo-profile", + "get-file-libraries", + "get-file-object-thumbnails", +]; + +// --------------------------------------------------------------------------- +// Parse k6 JSON output +// --------------------------------------------------------------------------- + +function parseK6Json(filePath) { + const content = fs.readFileSync(filePath, "utf-8"); + const lines = content.trim().split("\n"); + + // Collect all http_req_duration points with rpc_command tag + const durations = {}; // { rpc_command: [value, ...] } + + for (const line of lines) { + try { + const entry = JSON.parse(line); + + if ( + entry.type === "Point" && + entry.metric === "http_req_duration" && + entry.data?.tags?.rpc_command + ) { + const cmd = entry.data.tags.rpc_command; + const value = entry.data.value; + + if (!durations[cmd]) { + durations[cmd] = []; + } + durations[cmd].push(value); + } + } catch (e) { + // Skip malformed lines + } + } + + return durations; +} + +// --------------------------------------------------------------------------- +// Calculate percentiles +// --------------------------------------------------------------------------- + +function percentile(values, p) { + if (values.length === 0) return 0; + + const sorted = values.slice().sort((a, b) => a - b); + const index = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, index)]; +} + +function calculateStats(values) { + if (values.length === 0) { + return { count: 0, p50: 0, p95: 0, p99: 0, min: 0, max: 0, avg: 0 }; + } + + const sorted = values.slice().sort((a, b) => a - b); + const sum = values.reduce((a, b) => a + b, 0); + + return { + count: values.length, + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + min: sorted[0], + max: sorted[sorted.length - 1], + avg: sum / values.length, + }; +} + +// --------------------------------------------------------------------------- +// Compare two results +// --------------------------------------------------------------------------- + +function compareResults(baseline, current, threshold) { + const results = []; + const allCommands = new Set([ + ...Object.keys(baseline), + ...Object.keys(current), + ]); + + for (const cmd of allCommands) { + const baseStats = calculateStats(baseline[cmd] || []); + const currStats = calculateStats(current[cmd] || []); + + // Calculate p95 change percentage + let p95Change = 0; + if (baseStats.p95 > 0) { + p95Change = ((currStats.p95 - baseStats.p95) / baseStats.p95) * 100; + } else if (currStats.p95 > 0) { + p95Change = 100; // New command with latency + } + + const isCritical = CRITICAL_COMMANDS.includes(cmd); + const isRegression = p95Change > threshold; + + results.push({ + command: cmd, + isCritical, + baseline: baseStats, + current: currStats, + p95Change: Math.round(p95Change * 100) / 100, + isRegression, + }); + } + + // Sort: regressions first, then by p95 change descending + results.sort((a, b) => { + if (a.isRegression !== b.isRegression) return b.isRegression - a.isRegression; + return b.p95Change - a.p95Change; + }); + + return results; +} + +// --------------------------------------------------------------------------- +// Print report +// --------------------------------------------------------------------------- + +function printReport(results, threshold) { + console.log("\n=== Performance Regression Report ===\n"); + console.log(`Threshold: p95 increase > ${threshold}%\n`); + + // Print table header + const header = [ + "Command".padEnd(30), + "Baseline p95".padStart(12), + "Current p95".padStart(12), + "Change".padStart(10), + "Status".padStart(10), + ].join(" | "); + + console.log(header); + console.log("-".repeat(header.length)); + + // Print results + for (const r of results) { + const baseP95 = `${Math.round(r.baseline.p95)}ms`; + const currP95 = `${Math.round(r.current.p95)}ms`; + const change = `${r.p95Change > 0 ? "+" : ""}${r.p95Change}%`; + const status = r.isRegression ? "FAIL" : "OK"; + const critical = r.isCritical ? " *" : ""; + + const row = [ + (r.command + critical).padEnd(30), + baseP95.padStart(12), + currP95.padStart(12), + change.padStart(10), + status.padStart(10), + ].join(" | "); + + console.log(row); + } + + // Print legend + console.log("\n* = Critical command (always checked)"); + + // Print regressions summary + const regressions = results.filter((r) => r.isRegression); + if (regressions.length > 0) { + console.log(`\n❌ REGRESSION DETECTED: ${regressions.length} command(s) exceeded threshold`); + for (const r of regressions) { + console.log(` - ${r.command}: p95 ${Math.round(r.baseline.p95)}ms → ${Math.round(r.current.p95)}ms (+${r.p95Change}%)`); + } + } else { + console.log("\n✅ No regressions detected"); + } + + return regressions.length; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +function main() { + const args = process.argv.slice(2); + + // Parse arguments + let baselineFile = null; + let currentFile = null; + let threshold = DEFAULT_THRESHOLD; + + for (let i = 0; i < args.length; i++) { + if (args[i] === "--threshold" && args[i + 1]) { + threshold = parseInt(args[i + 1], 10); + i++; + } else if (!baselineFile) { + baselineFile = args[i]; + } else if (!currentFile) { + currentFile = args[i]; + } + } + + // Validate arguments + if (!baselineFile || !currentFile) { + console.error("Usage: node compare-results.js [--threshold N]"); + console.error(""); + console.error("Arguments:"); + console.error(" baseline.json k6 JSON output from base branch"); + console.error(" current.json k6 JSON output from PR branch"); + console.error(" --threshold N Fail if p95 increases > N% (default: 20)"); + process.exit(2); + } + + // Check files exist + if (!fs.existsSync(baselineFile)) { + console.error(`Error: Baseline file not found: ${baselineFile}`); + process.exit(2); + } + if (!fs.existsSync(currentFile)) { + console.error(`Error: Current file not found: ${currentFile}`); + process.exit(2); + } + + // Parse files + console.log(`Parsing baseline: ${path.basename(baselineFile)}`); + const baseline = parseK6Json(baselineFile); + const baseCommands = Object.keys(baseline).length; + console.log(` Found ${baseCommands} RPC commands`); + + console.log(`Parsing current: ${path.basename(currentFile)}`); + const current = parseK6Json(currentFile); + const currCommands = Object.keys(current).length; + console.log(` Found ${currCommands} RPC commands`); + + if (baseCommands === 0 && currCommands === 0) { + console.error("Error: No RPC command data found in either file"); + process.exit(2); + } + + // Compare and report + const results = compareResults(baseline, current, threshold); + const regressionCount = printReport(results, threshold); + + // Exit with appropriate code + process.exit(regressionCount > 0 ? 1 : 0); +} + +main(); diff --git a/backend/performance/scripts/file-size-matrix.js b/backend/performance/scripts/file-size-matrix.js new file mode 100644 index 0000000000..5463d0676c --- /dev/null +++ b/backend/performance/scripts/file-size-matrix.js @@ -0,0 +1,249 @@ +// File Size Matrix Performance Test +// +// Measures how update-file and get-file latency scales with file size. +// Creates files with different shape counts (10, 100, 500, 1000) and +// benchmarks operations on each. +// +// Usage: +// k6 run scripts/file-size-matrix.js +// k6 run --iterations 10 scripts/file-size-matrix.js +// ./run.sh file-size-matrix +// ./run.sh file-size-matrix -n 10 + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; +const ITERATIONS_PER_TIER = parseInt(__ENV.PENPOT_EDIT_ITERATIONS || "5"); + +// Shape tiers +const TIERS = [ + { name: "small", shapes: 10, color: "#ff0000" }, + { name: "medium", shapes: 100, color: "#00ff00" }, + { name: "large", shapes: 500, color: "#0000ff" }, + { name: "xlarge", shapes: 1000, color: "#ff00ff" }, +]; + +export const options = { + thresholds: { + http_req_duration: ["p(95)<10000"], + http_req_failed: ["rate<0.01"], + }, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +function makeAddRectChange(pageId, index, color) { + const shapeId = uuidv4(); + const x = 50 + (index % 20) * 30; + const y = 50 + Math.floor(index / 20) * 30; + const w = 100; + const h = 80; + + return { + type: "add-obj", + pageId: pageId, + id: shapeId, + frameId: pageId, + parentId: pageId, + obj: { + id: shapeId, type: "rect", name: `Shape ${index}`, + x, y, width: w, height: h, + fillColor: color, fillOpacity: 0.8, + rotation: 0, hidden: false, locked: false, + selrect: { x, y, width: w, height: h, x1: x, y1: y, x2: x + w, y2: y + h }, + points: [ + { x, y }, { x: x + w, y }, { x: x + w, y: y + h }, { x, y: y + h }, + ], + transform: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + transformInverse: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + parentId: pageId, frameId: pageId, + }, + }; +} + +// Populate a file with N shapes in a single update-file call +function populateFile(client, fileId, pageId, shapeCount, color) { + // Get current file state + const getFileRes = client.getFile(fileId); + if (getFileRes.status !== 200) return null; + let { revn, vern } = getFileRes.body; + + // Add shapes in batches (backend may have limits on changes per call) + const BATCH_SIZE = 100; + let added = 0; + + while (added < shapeCount) { + const batchCount = Math.min(BATCH_SIZE, shapeCount - added); + const changes = []; + for (let i = 0; i < batchCount; i++) { + changes.push(makeAddRectChange(pageId, added + i, color)); + } + + const updateRes = client.updateFile(fileId, revn, vern, client.sessionId, changes); + if (updateRes.status !== 200) { + console.error(`Failed to add batch at ${added}: ${JSON.stringify(updateRes.body)}`); + return null; + } + + // update-file returns {revn, lagged} but not vern + // vern only changes on snapshot restore, so keep the original + revn = updateRes.body.revn; + added += batchCount; + } + + return { revn, vern }; +} + +// --------------------------------------------------------------------------- +// Setup — create files with different shape counts +// --------------------------------------------------------------------------- + +export function setup() { + console.log(`File Size Matrix Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` Iterations/tier: ${ITERATIONS_PER_TIER}`); + console.log(` Tiers: ${TIERS.map(t => `${t.name}(${t.shapes})`).join(", ")}`); + console.log(``); + + const client = createClient(BASE_URL); + if (client.getProfile().status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + // Create demo profile + const userRes = client.rpc("POST", "create-demo-profile", {}); + if (userRes.status !== 200) fail("Failed to create demo profile"); + const user = userRes.json(); + console.log(` Created demo profile: ${user.email}`); + + // Login + if (client.login(user.email, user.password).status !== 200) fail("Login failed"); + const teamId = client.getTeams().body[0].id; + const projectId = client.createProject(teamId, "File Size Matrix Project").body.id; + console.log(` Project: ${projectId}`); + + // Create and populate files for each tier + const tiers = []; + + for (const tier of TIERS) { + console.log(`\n Creating ${tier.name} file (${tier.shapes} shapes)...`); + + // Create file + const fileRes = client.createFile(projectId, `Matrix ${tier.name} (${tier.shapes} shapes)`); + if (fileRes.status !== 200) fail(`Failed to create ${tier.name} file`); + const fileId = fileRes.body.id; + + // Get page ID + const getFileRes = client.getFile(fileId); + if (getFileRes.status !== 200) fail(`Failed to get ${tier.name} file`); + const pageId = getFileRes.body.data.pages[0]; + + // Populate with shapes + const startTime = Date.now(); + const result = populateFile(client, fileId, pageId, tier.shapes, tier.color); + if (!result) fail(`Failed to populate ${tier.name} file`); + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + + console.log(` ${tier.name}: ${tier.shapes} shapes in ${elapsed}s (revn=${result.revn})`); + + tiers.push({ + name: tier.name, + shapes: tier.shapes, + fileId, + pageId, + revn: result.revn, + vern: result.vern, + }); + } + + console.log(`\n Setup complete. ${tiers.length} files ready.`); + + return { baseUrl: BASE_URL, user, tiers, iterationsPerTier: ITERATIONS_PER_TIER }; +} + +// --------------------------------------------------------------------------- +// Main VU Function — benchmark each tier +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Login + if (!assertOk(client.login(data.user.email, data.user.password), "login")) fail("login failed"); + sleep(0.5); + + console.log(`\n=== Starting benchmark (${data.iterationsPerTier} iterations per tier) ===\n`); + + // Benchmark each tier + for (const tier of data.tiers) { + console.log(`--- Tier: ${tier.name} (${tier.shapes} shapes) ---`); + + // Get latest file state + const getFileRes = client.getFile(tier.fileId); + if (!assertOk(getFileRes, `get-file-${tier.name}`)) continue; + let { revn, vern } = getFileRes.body; + + for (let i = 0; i < data.iterationsPerTier; i++) { + // Benchmark get-file + const getRes = client.getFile(tier.fileId); + if (!assertOk(getRes, `get-file-${tier.name}`)) continue; + + sleep(0.2); + + // Benchmark update-file (add 1 shape) + const change = makeAddRectChange(tier.pageId, tier.shapes + i, "#ffaa00"); + const updateRes = client.updateFile(tier.fileId, getRes.body.revn, getRes.body.vern, client.sessionId, [change]); + + if (updateRes.status !== 200) { + console.error(`update-file failed on ${tier.name} iteration ${i}: ${JSON.stringify(updateRes.body)}`); + continue; + } + + // update-file returns {revn, lagged} but not vern + // vern only changes on snapshot restore, so keep the original + revn = updateRes.body.revn; + + sleep(0.3); + } + + console.log(` Completed ${data.iterationsPerTier} iterations on ${tier.name}`); + } + + console.log(`\n=== Benchmark complete ===`); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log(`File size matrix test complete.`); +} diff --git a/backend/performance/scripts/font-upload.js b/backend/performance/scripts/font-upload.js new file mode 100644 index 0000000000..1f05bfd684 --- /dev/null +++ b/backend/performance/scripts/font-upload.js @@ -0,0 +1,158 @@ +// Font Upload Performance Test +// +// Tests the font upload flow: chunked upload of TTF + OTF files followed by +// creating a font variant. Exercises storage pipeline and font processing. +// +// setup() creates N demo profiles. +// Each VU picks its user, uploads fonts, and creates a variant. +// +// Usage: +// k6 run scripts/font-upload.js +// k6 run --vus 50 --iterations 5 scripts/font-upload.js + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; + +export const options = { + thresholds: { + http_req_duration: ["p(95)<15000"], + http_req_failed: ["rate<0.01"], + "http_req_duration{rpc_command:create-upload-session}": ["p(95)<1000"], + "http_req_duration{rpc_command:upload-chunk}": ["p(95)<5000"], + "http_req_duration{rpc_command:create-font-variant}": ["p(95)<10000"], + }, +}; + +// --------------------------------------------------------------------------- +// Test Data +// --------------------------------------------------------------------------- +const fontTtf = open("../../test/backend_tests/test_files/font-1.ttf", "b"); + +const fontOtf = open("../../test/backend_tests/test_files/font-1.otf", "b"); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +// --------------------------------------------------------------------------- +// Setup — create user pool +// --------------------------------------------------------------------------- + +export function setup() { + const vuCount = parseInt(__ENV.K6_VUS) || 1; + + console.log(`Penpot Font Upload Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` VUs: ${vuCount}`); + console.log(``); + + const client = createClient(BASE_URL); + if (client.getProfile().status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + const users = []; + for (let i = 0; i < vuCount; i++) { + const res = client.rpc("POST", "create-demo-profile", {}); + if (res.status !== 200) fail(`Failed to create demo profile ${i + 1}/${vuCount}`); + users.push(res.json()); + } + console.log(` Created ${users.length} demo profiles`); + + return { baseUrl: BASE_URL, users }; +} + +// --------------------------------------------------------------------------- +// Main VU Function +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Pick user from pool + const user = data.users[__VU - 1]; + if (!user) fail(`No user for VU ${__VU}`); + + // Login + if (!assertOk(client.login(user.email, user.password), "login")) fail("login failed"); + const teamId = client.getTeams().body[0].id; + + sleep(0.5); + + const fontId = uuidv4(); + const fontFamily = `PerfFont-${uuidv4().substring(0, 8)}`; + const chunkSize = 50 * 1024; // 50 KB + + // Upload TTF via chunked upload + const ttfChunks = Math.ceil(fontTtf.byteLength / chunkSize); + const ttfSessionRes = client.createUploadSession(ttfChunks); + if (!assertOk(ttfSessionRes, "create-upload-session (ttf)")) fail("create-upload-session failed"); + const ttfSessionId = ttfSessionRes.sessionId; + + for (let i = 0; i < ttfChunks; i++) { + const chunk = fontTtf.slice(i * chunkSize, Math.min((i + 1) * chunkSize, fontTtf.byteLength)); + if (!assertOk(client.uploadChunk(ttfSessionId, i, chunk, "font-1.ttf", "font/ttf"), `upload-chunk ttf ${i}`)) fail("ttf chunk failed"); + sleep(0.1); + } + + // Upload OTF via chunked upload + const otfChunks = Math.ceil(fontOtf.byteLength / chunkSize); + const otfSessionRes = client.createUploadSession(otfChunks); + if (!assertOk(otfSessionRes, "create-upload-session (otf)")) fail("create-upload-session (otf) failed"); + const otfSessionId = otfSessionRes.sessionId; + + for (let i = 0; i < otfChunks; i++) { + const chunk = fontOtf.slice(i * chunkSize, Math.min((i + 1) * chunkSize, fontOtf.byteLength)); + if (!assertOk(client.uploadChunk(otfSessionId, i, chunk, "font-1.otf", "font/otf"), `upload-chunk otf ${i}`)) fail("otf chunk failed"); + sleep(0.1); + } + + sleep(0.5); + + // Create font variant + if (!assertOk(client.rpc("POST", "create-font-variant", { + "team-id": teamId, + "font-id": fontId, + "font-family": fontFamily, + "font-weight": 400, + "font-style": "normal", + uploads: { "font/ttf": ttfSessionId, "font/otf": otfSessionId }, + }), "create-font-variant")) fail("create-font-variant failed"); + + console.log(`VU ${__VU}: Font "${fontFamily}" created`); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log("Font upload test complete."); +} diff --git a/backend/performance/scripts/lifecycle.js b/backend/performance/scripts/lifecycle.js new file mode 100644 index 0000000000..798a06645c --- /dev/null +++ b/backend/performance/scripts/lifecycle.js @@ -0,0 +1,260 @@ +// Lifecycle Performance Test +// +// Simulates a realistic user lifecycle from registration through CRUD operations. +// Each VU performs the full flow independently, creating its own artifacts. +// +// setup() creates a user pool (one demo profile per VU) before measurements begin. +// Each VU picks its assigned user to login — no profile creation during the test. +// +// Flow: +// 1. Login (with pre-existing user from pool) +// 2. Get profile & teams +// 3. Create project +// 4. Create file +// 5. Get file +// 6. Update file (add a shape) +// 7. Upload images (direct + chunked) +// 8. Delete file +// 9. Delete project +// 10. Logout +// +// Usage: +// k6 run scripts/lifecycle.js +// k6 run --vus 100 --iterations 100 scripts/lifecycle.js +// k6 run --env PENPOT_BASE_URL=http://localhost:6060 scripts/lifecycle.js + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; + +// k6 options — VUs and iterations set via run.sh (--vus, --iterations) +export const options = { + thresholds: { + http_req_duration: ["p(95)<5000"], + http_req_failed: ["rate<0.01"], + "http_req_duration{rpc_command:login-with-password}": ["p(95)<1000"], + "http_req_duration{rpc_command:get-profile}": ["p(95)<500"], + "http_req_duration{rpc_command:create-project}": ["p(95)<1000"], + "http_req_duration{rpc_command:create-file}": ["p(95)<1000"], + "http_req_duration{rpc_command:get-file}": ["p(95)<500"], + "http_req_duration{rpc_command:update-file}": ["p(95)<2000"], + "http_req_duration{rpc_command:delete-file}": ["p(95)<1000"], + }, +}; + +// --------------------------------------------------------------------------- +// Test Data +// --------------------------------------------------------------------------- +const testImageSmall = open("../../test/backend_tests/test_files/sample.png", "b"); + +const testImageLarge = open("../../test/backend_tests/test_files/sample.jpg", "b"); + +// A minimal "add-obj" change payload for update-file. +function makeAddRectChange(pageId) { + const shapeId = uuidv4(); + const x = 100; + const y = 100; + const w = 200; + const h = 150; + + return { + type: "add-obj", + pageId: pageId, + id: shapeId, + frameId: pageId, + parentId: pageId, + obj: { + id: shapeId, + type: "rect", + name: "Perf Test Rect", + x: x, y: y, width: w, height: h, + fillColor: "#ff0000", fillOpacity: 1, + rotation: 0, hidden: false, locked: false, + selrect: { x, y, width: w, height: h, x1: x, y1: y, x2: x + w, y2: y + h }, + points: [ + { x, y }, { x: x + w, y }, { x: x + w, y: y + h }, { x, y: y + h }, + ], + transform: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + transformInverse: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + parentId: pageId, frameId: pageId, + }, + }; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +// --------------------------------------------------------------------------- +// Setup — create user pool before VUs start +// --------------------------------------------------------------------------- + +export function setup() { + // Resolve VU count from options or CLI --vus flag + const vuCount = parseInt(__ENV.K6_VUS) || 1; + + console.log(`Penpot Lifecycle Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` VUs: ${vuCount}`); + console.log(` Creating ${vuCount} demo profiles...`); + console.log(``); + + const client = createClient(BASE_URL); + + // Verify backend is reachable + const pingRes = client.getProfile(); + if (pingRes.status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + // Create one demo profile per VU + const users = []; + for (let i = 0; i < vuCount; i++) { + const res = client.rpc("POST", "create-demo-profile", {}); + if (res.status !== 200) { + fail(`Failed to create demo profile ${i + 1}/${vuCount}: status=${res.status}`); + } + users.push(res.json()); + } + + console.log(` Created ${users.length} demo profiles`); + return { baseUrl: BASE_URL, users }; +} + +// --------------------------------------------------------------------------- +// Main VU Function +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Pick user from pool + const user = data.users[__VU - 1]; + if (!user) { + fail(`No user for VU ${__VU} (pool size: ${data.users.length})`); + } + + // ---- Step 1: Login ---- + const loginRes = client.login(user.email, user.password); + if (!assertOk(loginRes, "login-with-password")) fail("Login failed"); + const profile = loginRes.body; + const profileId = profile.id; + + sleep(1); + + // ---- Step 2: Get profile ---- + if (!assertOk(client.getProfile(), "get-profile")) fail("get-profile failed"); + + sleep(0.5); + + // ---- Step 3: Get teams ---- + const teamsRes = client.getTeams(); + if (!assertOk(teamsRes, "get-teams")) fail("get-teams failed"); + const defaultTeamId = teamsRes.body[0].id; + + sleep(0.5); + + // ---- Step 4: Create a project ---- + const projectRes = client.createProject(defaultTeamId, `Perf Project ${uuidv4().substring(0, 8)}`); + if (!assertOk(projectRes, "create-project")) fail("create-project failed"); + const projectId = projectRes.body.id; + + sleep(1); + + // ---- Step 5: Create a file ---- + const fileRes = client.createFile(projectId, `Perf File ${uuidv4().substring(0, 8)}`); + if (!assertOk(fileRes, "create-file")) fail("create-file failed"); + const fileId = fileRes.body.id; + + sleep(1); + + // ---- Step 6: Get the file ---- + const getFileRes = client.getFile(fileId); + if (!assertOk(getFileRes, "get-file")) fail("get-file failed"); + const fileData = getFileRes.body; + const pageId = fileData.data.pages[0]; + + sleep(1); + + // ---- Step 7: Update file (add a shape) ---- + if (pageId) { + const changes = [makeAddRectChange(pageId)]; + const updateRes = client.updateFile(fileId, fileData.revn, fileData.vern, client.sessionId, changes); + + if (updateRes.status !== 200) { + // Retry once on revn conflict + const body = updateRes.body; + const isConflict = body && (body.code === "revn-conflict" || body.type === "revn-conflict"); + if (isConflict) { + const retryFile = client.getFile(fileId); + if (retryFile.status === 200) { + client.updateFile(fileId, retryFile.body.revn, retryFile.body.vern, client.sessionId, changes); + } + } + } + } + + sleep(1); + + // ---- Step 8: Upload images ---- + if (testImageSmall && testImageSmall.byteLength > 0) { + assertOk( + client.uploadFileMediaObject(fileId, testImageSmall, "sample.png", "image/png"), + "upload (direct)" + ); + } + sleep(0.5); + if (testImageLarge && testImageLarge.byteLength > 0) { + assertOk( + client.uploadFileMediaObject(fileId, testImageLarge, "sample.jpg", "image/jpeg"), + "upload (chunked)" + ); + } + + sleep(1); + + // ---- Step 9: Delete file ---- + assertOk(client.deleteFile(fileId), "delete-file"); + sleep(0.5); + + // ---- Step 10: Delete project ---- + assertOk(client.deleteProject(projectId), "delete-project"); + sleep(0.5); + + // ---- Step 11: Logout ---- + client.logout(profileId); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log("Lifecycle test complete."); +} diff --git a/backend/performance/scripts/media-upload.js b/backend/performance/scripts/media-upload.js new file mode 100644 index 0000000000..c950a8a795 --- /dev/null +++ b/backend/performance/scripts/media-upload.js @@ -0,0 +1,142 @@ +// Media Upload Performance Test +// +// Tests direct and chunked image uploads with varying file sizes. +// Each VU creates its own file and uploads multiple images to it. +// +// Upload sizes: +// - SVG (3.6 KB) → direct upload +// - PNG (5.1 KB) → direct upload +// - JPG (305 KB) → chunked upload (7 chunks at 50 KB each) +// +// Usage: +// k6 run scripts/media-upload.js +// k6 run --vus 50 --iterations 5 scripts/media-upload.js + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; + +export const options = { + thresholds: { + http_req_duration: ["p(95)<10000"], + http_req_failed: ["rate<0.01"], + "http_req_duration{rpc_command:upload-file-media-object}": ["p(95)<5000"], + "http_req_duration{rpc_command:upload-chunk}": ["p(95)<5000"], + "http_req_duration{rpc_command:assemble-file-media-object}": ["p(95)<5000"], + }, +}; + +// --------------------------------------------------------------------------- +// Test Data +// --------------------------------------------------------------------------- +const imageSvg = open("../../test/backend_tests/test_files/sample1.svg", "b"); + +const imagePng = open("../../test/backend_tests/test_files/sample.png", "b"); + +const imageJpg = open("../../test/backend_tests/test_files/sample.jpg", "b"); +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +// --------------------------------------------------------------------------- +// Setup — create user pool +// --------------------------------------------------------------------------- + +export function setup() { + const vuCount = parseInt(__ENV.K6_VUS) || 1; + + console.log(`Penpot Media Upload Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` VUs: ${vuCount}`); + console.log(``); + + const client = createClient(BASE_URL); + if (client.getProfile().status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + const users = []; + for (let i = 0; i < vuCount; i++) { + const res = client.rpc("POST", "create-demo-profile", {}); + if (res.status !== 200) fail(`Failed to create demo profile ${i + 1}/${vuCount}`); + users.push(res.json()); + } + console.log(` Created ${users.length} demo profiles`); + + return { baseUrl: BASE_URL, users }; +} + +// --------------------------------------------------------------------------- +// Main VU Function +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Pick user from pool + const user = data.users[__VU - 1]; + if (!user) fail(`No user for VU ${__VU}`); + + // Login + if (!assertOk(client.login(user.email, user.password), "login")) fail("login failed"); + + sleep(0.5); + + // Get team + const teamId = client.getTeams().body[0].id; + + // Create project + file + const projectId = client.createProject(teamId, `Media ${uuidv4().substring(0, 8)}`).body.id; + const fileId = client.createFile(projectId, `Media ${uuidv4().substring(0, 8)}`).body.id; + + sleep(0.5); + + // Upload SVG (direct — 3.6 KB) + assertOk(client.uploadFileMediaObject(fileId, imageSvg, "sample.svg", "image/svg+xml"), "upload SVG"); + + sleep(0.5); + + // Upload PNG (direct — 5.1 KB) + assertOk(client.uploadFileMediaObject(fileId, imagePng, "sample.png", "image/png"), "upload PNG"); + + sleep(0.5); + + // Upload JPG (chunked — 305 KB > 50 KB threshold) + assertOk(client.uploadFileMediaObject(fileId, imageJpg, "sample.jpg", "image/jpeg"), "upload JPG"); + + console.log(`VU ${__VU}: Media upload complete`); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log("Media upload test complete."); +} diff --git a/backend/performance/scripts/workspace-edit-concurrent.js b/backend/performance/scripts/workspace-edit-concurrent.js new file mode 100644 index 0000000000..8d9bbb3ff9 --- /dev/null +++ b/backend/performance/scripts/workspace-edit-concurrent.js @@ -0,0 +1,361 @@ +// Workspace Edit Concurrent Performance Test +// +// Two modes for measuring concurrent file editing: +// +// Mode 1: same-file — N VUs edit different pages in 1 file +// Measures lock contention on a single popular file. +// Bottleneck: advisory lock serialization (db/xact-lock!). +// +// Mode 2: multi-file — G groups × M VUs per file +// Each group edits its own file on its own page. +// Measures whole system responsiveness under parallel edit sessions. +// Bottleneck: DB connection pool, CPU, memory. +// +// Key insight: revn conflicts only occur when incoming > stored (should +// never happen in normal usage). The real contention point is the file-level +// advisory lock that serializes all update-file calls on the same file. +// +// Usage: +// # Same-file mode (default): 5 VUs edit different pages in 1 file +// k6 run --vus 5 --iterations 10 scripts/workspace-edit-concurrent.js +// +// # Multi-file mode: 3 files × 2 VUs each = 6 VUs total +// PENPOT_EDIT_MODE=multi-file PENPOT_FILE_COUNT=3 PENPOT_VUS_PER_FILE=2 \ +// k6 run --vus 6 --iterations 10 scripts/workspace-edit-concurrent.js +// +// # Via run.sh +// ./run.sh concurrent-edit --mode same-file --vus 5 --iterations 10 +// ./run.sh concurrent-edit --mode multi-file --files 3 --vus-per-file 2 --iterations 10 + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; +const EDIT_MODE = __ENV.PENPOT_EDIT_MODE || "same-file"; // "same-file" or "multi-file" +const FILE_COUNT = parseInt(__ENV.PENPOT_FILE_COUNT || "1"); +const VUS_PER_FILE = parseInt(__ENV.PENPOT_VUS_PER_FILE || "1"); +const EDIT_ITERATIONS = parseInt(__ENV.PENPOT_EDIT_ITERATIONS || "50"); + +// Calculate total VUs based on mode +const TOTAL_VUS = EDIT_MODE === "multi-file" + ? FILE_COUNT * VUS_PER_FILE + : parseInt(__ENV.PENPOT_TOTAL_VUS || "3"); + +export const options = { + thresholds: { + http_req_duration: ["p(95)<5000"], + http_req_failed: ["rate<0.01"], + "http_req_duration{rpc_command:get-file}": ["p(95)<500"], + "http_req_duration{rpc_command:update-file}": ["p(95)<3000"], + }, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +function makeAddRectChange(pageId, index) { + const shapeId = uuidv4(); + const x = 50 + (index % 10) * 30; + const y = 50 + Math.floor(index / 10) * 30; + const w = 100; + const h = 80; + + return { + type: "add-obj", + pageId: pageId, + id: shapeId, + frameId: pageId, + parentId: pageId, + obj: { + id: shapeId, type: "rect", name: `Shape ${index}`, + x, y, width: w, height: h, + fillColor: "#00ff00", fillOpacity: 0.8, + rotation: 0, hidden: false, locked: false, + selrect: { x, y, width: w, height: h, x1: x, y1: y, x2: x + w, y2: y + h }, + points: [ + { x, y }, { x: x + w, y }, { x: x + w, y: y + h }, { x, y: y + h }, + ], + transform: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + transformInverse: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + parentId: pageId, frameId: pageId, + }, + }; +} + +// Add a page to a file via update-file with add-page change +function addPage(client, fileId, revn, vern, pageId, pageName) { + const change = { + type: "add-page", + id: pageId, + name: pageName, + }; + return client.updateFile(fileId, revn, vern, client.sessionId, [change]); +} + +// --------------------------------------------------------------------------- +// Setup — create users, files, and pages based on mode +// --------------------------------------------------------------------------- + +export function setup() { + console.log(`Penpot Concurrent Edit Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` Mode: ${EDIT_MODE}`); + console.log(` Edit iterations: ${EDIT_ITERATIONS}`); + + if (EDIT_MODE === "same-file") { + console.log(` Total VUs: ${TOTAL_VUS} (same file)`); + } else { + console.log(` Files: ${FILE_COUNT}`); + console.log(` VUs per file: ${VUS_PER_FILE}`); + console.log(` Total VUs: ${TOTAL_VUS}`); + } + console.log(``); + + const client = createClient(BASE_URL); + if (client.getProfile().status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + // Create demo profiles (one per VU) + const users = []; + for (let i = 0; i < TOTAL_VUS; i++) { + const res = client.rpc("POST", "create-demo-profile", {}); + if (res.status !== 200) fail(`Failed to create demo profile ${i + 1}/${TOTAL_VUS}`); + users.push(res.json()); + } + console.log(` Created ${users.length} demo profiles`); + + // Login with first user to create shared team and files + const loginRes = client.login(users[0].email, users[0].password); + if (loginRes.status !== 200) fail("Login failed for setup"); + + // Create a shared team so all VUs can access the same file. + // Each demo profile gets its own default team; without a shared + // team, VUs 2+ would get 404 on get-file. + const teamRes = client.createTeam("Concurrent Edit Team"); + if (teamRes.status !== 200) fail("Failed to create shared team"); + const sharedTeamId = teamRes.body.id; + console.log(` Shared team: ${sharedTeamId}`); + + // Invite remaining users to the shared team and get acceptance tokens. + // The tokens are used by each VU via verify-token to join the team. + const invitationTokens = []; + for (let i = 1; i < TOTAL_VUS; i++) { + const invRes = client.inviteTeamMembers(sharedTeamId, [users[i].email], "editor"); + if (invRes.status !== 200) { + console.error(` Invite user ${i}: status=${invRes.status}`); + } + const tokenRes = client.getTeamInvitationToken(sharedTeamId, users[i].email); + if (tokenRes.status === 200 && tokenRes.body) { + invitationTokens.push({ vuIndex: i, token: tokenRes.body }); + } + } + if (invitationTokens.length > 0) { + console.log(` Got ${invitationTokens.length} invitation tokens`); + } else { + console.log(` All users auto-added (no tokens needed)`); + } + + // Create project and files in the shared team + const projectId = client.createProject(sharedTeamId, "Concurrent Edit Project").body.id; + console.log(` Project: ${projectId}`); + + // Build file/page assignments based on mode + const fileAssignments = []; // [{ fileId, pageIds[] }] + const vuAssignments = []; // [{ vuIndex, fileId, pageId }] + + if (EDIT_MODE === "same-file") { + // One file, N pages (one per VU) + const fileRes = client.createFile(projectId, "Shared Edit File"); + if (fileRes.status !== 200) fail("Failed to create shared file"); + const fileId = fileRes.body.id; + console.log(` Created file: ${fileId}`); + + // Get initial file state (has 1 default page) + const getFileRes = client.getFile(fileId); + if (getFileRes.status !== 200) fail("Failed to get initial file"); + const defaultPageId = getFileRes.body.data.pages[0]; + let revn = getFileRes.body.revn; + let vern = getFileRes.body.vern; + + // First VU uses the default page + const pageIds = [defaultPageId]; + + // Add remaining pages + // vern never changes on regular edits (only on snapshot restore), + // and each add-page increments revn by 1, so no need to re-fetch. + for (let i = 1; i < TOTAL_VUS; i++) { + const pageId = uuidv4(); + const pageName = `Page ${i + 1}`; + const addRes = addPage(client, fileId, revn, vern, pageId, pageName); + if (addRes.status !== 200) fail(`Failed to add page ${i + 1}`); + revn++; + pageIds.push(pageId); + } + console.log(` Added ${pageIds.length} pages to file`); + + fileAssignments.push({ fileId, pageIds }); + + // Each VU gets its own page in the same file + for (let i = 0; i < TOTAL_VUS; i++) { + vuAssignments.push({ vuIndex: i, fileId, pageId: pageIds[i] }); + } + + } else { + // Multi-file mode: G files, each with M pages + for (let f = 0; f < FILE_COUNT; f++) { + const fileRes = client.createFile(projectId, `Edit File ${f + 1}`); + if (fileRes.status !== 200) fail(`Failed to create file ${f + 1}`); + const fileId = fileRes.body.id; + console.log(` Created file ${f + 1}: ${fileId}`); + + // Get initial file state (has 1 default page) + const getFileRes = client.getFile(fileId); + if (getFileRes.status !== 200) fail(`Failed to get file ${f + 1}`); + const defaultPageId = getFileRes.body.data.pages[0]; + let revn = getFileRes.body.revn; + let vern = getFileRes.body.vern; + + // First VU of this file uses the default page + const pageIds = [defaultPageId]; + + // Add remaining pages for this file + for (let p = 1; p < VUS_PER_FILE; p++) { + const pageId = uuidv4(); + const pageName = `Page ${p + 1}`; + const addRes = addPage(client, fileId, revn, vern, pageId, pageName); + if (addRes.status !== 200) fail(`Failed to add page ${p + 1} to file ${f + 1}`); + revn++; + pageIds.push(pageId); + } + console.log(` Added ${pageIds.length} pages to file ${f + 1}`); + + fileAssignments.push({ fileId, pageIds }); + + // Assign VUs to this file's pages + for (let p = 0; p < VUS_PER_FILE; p++) { + const vuIndex = f * VUS_PER_FILE + p; + vuAssignments.push({ vuIndex, fileId, pageId: pageIds[p] }); + } + } + } + + console.log(` Setup complete. ${vuAssignments.length} VU assignments.`); + console.log(``); + + return { + baseUrl: BASE_URL, + editMode: EDIT_MODE, + users, + vuAssignments, + invitationTokens, + }; +} + +// --------------------------------------------------------------------------- +// Main VU Function — each VU edits its assigned page +// --------------------------------------------------------------------------- + +// Track which VUs have accepted their invitation (once per VU, not per iteration) +const verifiedVus = {}; + +// --------------------------------------------------------------------------- +// Main VU Function — each VU edits its assigned page +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Each VU uses its own demo profile (different users editing the same file) + const vuIndex = __VU - 1; + const user = data.users[vuIndex]; + const assignment = data.vuAssignments[vuIndex]; + + if (!user) fail(`No user for VU ${__VU} (index ${vuIndex})`); + if (!assignment) fail(`No assignment for VU ${__VU} (index ${vuIndex})`); + + const { fileId, pageId } = assignment; + + // Login + if (!assertOk(client.login(user.email, user.password), "login")) fail("login failed"); + + // Accept team invitation once per VU (not per iteration). + // In devenv the user may already be auto-added; 400 on already-accepted + // tokens is harmless — skip the token on subsequent iterations. + if (!verifiedVus[__VU]) { + const tokenEntry = data.invitationTokens.find((t) => t.vuIndex === vuIndex); + if (tokenEntry && tokenEntry.token) { + client.rpc("POST", "verify-token", tokenEntry.token); + } + verifiedVus[__VU] = true; + } + + sleep(0.5); + + // Edit loop + for (let i = 0; i < EDIT_ITERATIONS; i++) { + // Refresh file state to get latest revn + const refreshRes = client.getFile(fileId); + if (!assertOk(refreshRes, "get-file")) continue; + const { revn, vern } = refreshRes.body; + + sleep(0.3); + + // Submit a change to our assigned page + const changes = [makeAddRectChange(pageId, i)]; + const updateRes = client.updateFile(fileId, revn, vern, client.sessionId, changes); + + if (updateRes.status !== 200) { + const body = updateRes.body; + const isConflict = body && (body.code === "revn-conflict" || body.type === "revn-conflict"); + if (isConflict) { + // This shouldn't happen in normal circumstances, but handle it gracefully + console.warn(`VU ${__VU}: revn conflict on iteration ${i} (unexpected)`); + const retryFile = client.getFile(fileId); + if (retryFile.status === 200) { + client.updateFile(fileId, retryFile.body.revn, retryFile.body.vern, client.sessionId, changes); + } + } else { + console.error(`VU ${__VU}: update-file failed on iteration ${i}: ${JSON.stringify(body)}`); + } + } + + sleep(1); + } + + console.log(`VU ${__VU}: Completed ${EDIT_ITERATIONS} edits on file ${fileId}, page ${pageId}`); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log(`Concurrent edit test complete (${data.editMode}).`); +} diff --git a/backend/performance/scripts/workspace-edit.js b/backend/performance/scripts/workspace-edit.js new file mode 100644 index 0000000000..bbf732d687 --- /dev/null +++ b/backend/performance/scripts/workspace-edit.js @@ -0,0 +1,195 @@ +// Workspace Edit Performance Test (Write-heavy) +// +// Simulates users editing files — repeatedly fetching the file (to get +// the latest revn) and submitting changes. Each VU edits its own file +// in its own project independently, so there are no concurrency conflicts. +// +// setup() creates N demo profiles + per-user project. +// Each VU picks its user, creates its own file, and edits it in a loop. +// +// Flow (per VU): +// Login → create file → loop: get-file → update-file → sleep +// +// Usage: +// k6 run scripts/workspace-edit.js +// k6 run --vus 100 --iterations 50 scripts/workspace-edit.js + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; +const EDIT_ITERATIONS = parseInt(__ENV.PENPOT_EDIT_ITERATIONS || "50"); + +export const options = { + thresholds: { + http_req_duration: ["p(95)<5000"], + http_req_failed: ["rate<0.01"], + "http_req_duration{rpc_command:get-file}": ["p(95)<500"], + "http_req_duration{rpc_command:update-file}": ["p(95)<2000"], + }, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +function makeAddRectChange(pageId, index) { + const shapeId = uuidv4(); + const x = 50 + (index % 10) * 30; + const y = 50 + Math.floor(index / 10) * 30; + const w = 100; + const h = 80; + + return { + type: "add-obj", + pageId: pageId, + id: shapeId, + frameId: pageId, + parentId: pageId, + obj: { + id: shapeId, type: "rect", name: `Shape ${index}`, + x, y, width: w, height: h, + fillColor: "#00ff00", fillOpacity: 0.8, + rotation: 0, hidden: false, locked: false, + selrect: { x, y, width: w, height: h, x1: x, y1: y, x2: x + w, y2: y + h }, + points: [ + { x, y }, { x: x + w, y }, { x: x + w, y: y + h }, { x, y: y + h }, + ], + transform: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + transformInverse: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + parentId: pageId, frameId: pageId, + }, + }; +} + +// --------------------------------------------------------------------------- +// Setup — create N users, each with their own project +// --------------------------------------------------------------------------- + +export function setup() { + const vuCount = parseInt(__ENV.K6_VUS) || 1; + + console.log(`Penpot Workspace Edit Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` VUs: ${vuCount}`); + console.log(` Edit iterations: ${EDIT_ITERATIONS}`); + console.log(``); + + const client = createClient(BASE_URL); + + if (client.getProfile().status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + // Create N demo profiles + per-user project. + // Each user needs their own project because demo profiles + // belong to different teams and cannot share a project. + const users = []; + for (let i = 0; i < vuCount; i++) { + const res = client.rpc("POST", "create-demo-profile", {}); + if (res.status !== 200) fail(`Failed to create demo profile ${i + 1}/${vuCount}`); + const user = res.json(); + + // Login as this user and create their own project + const loginRes = client.login(user.email, user.password); + if (loginRes.status !== 200) fail(`Login failed for user ${i + 1}`); + const teamId = client.getTeams().body[0].id; + user.projectId = client.createProject(teamId, `WS Edit Project`).body.id; + + users.push(user); + } + console.log(` Created ${users.length} demo profiles + projects`); + + return { baseUrl: BASE_URL, users }; +} + +// --------------------------------------------------------------------------- +// Main VU Function — each VU creates its own file and edits it +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Pick user from pool + const user = data.users[__VU - 1]; + if (!user) fail(`No user for VU ${__VU}`); + + // Login + if (!assertOk(client.login(user.email, user.password), "login")) fail("login failed"); + sleep(0.5); + + // Create a file for this VU in their own project + const fileRes = client.createFile(user.projectId, `Edit File VU${__VU}`); + if (!assertOk(fileRes, "create-file")) fail("create-file failed"); + const fileId = fileRes.body.id; + + // Get initial file state + const getFileRes = client.getFile(fileId); + if (!assertOk(getFileRes, "get-file")) fail("get-file failed"); + const pageId = getFileRes.body.data.pages[0]; + + sleep(0.5); + + // Edit loop + for (let i = 0; i < EDIT_ITERATIONS; i++) { + // Refresh file state to get latest revn + const refreshRes = client.getFile(fileId); + if (!assertOk(refreshRes, "get-file")) continue; + const { revn, vern } = refreshRes.body; + + sleep(0.3); + + // Submit a change + const changes = [makeAddRectChange(pageId, i)]; + const updateRes = client.updateFile(fileId, revn, vern, client.sessionId, changes); + + if (updateRes.status !== 200) { + // Retry once on revn conflict + const body = updateRes.body; + const isConflict = body && (body.code === "revn-conflict" || body.type === "revn-conflict"); + if (isConflict) { + const retryFile = client.getFile(fileId); + if (retryFile.status === 200) { + client.updateFile(fileId, retryFile.body.revn, retryFile.body.vern, client.sessionId, changes); + } + } + } + + sleep(1); + } + + console.log(`VU ${__VU}: Completed ${EDIT_ITERATIONS} edits on file ${fileId}`); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log("Workspace edit test complete."); +} diff --git a/backend/performance/scripts/workspace-open.js b/backend/performance/scripts/workspace-open.js new file mode 100644 index 0000000000..2a1fc1ff56 --- /dev/null +++ b/backend/performance/scripts/workspace-open.js @@ -0,0 +1,157 @@ +// Workspace Open Performance Test (Read-heavy) +// +// Simulates many users opening the same file in the workspace editor. +// This is the most common read-heavy operation — loading a file and its +// dependencies (libraries, thumbnails). +// +// setup() creates one user, one project, and one file with a shape. +// All VUs login with the same user and read the same file concurrently. +// +// Flow (per VU iteration): +// Login → get-file → get-file-libraries → get-file-object-thumbnails +// → get-file-data-for-thumbnail +// +// Usage: +// k6 run scripts/workspace-open.js +// k6 run --vus 100 --iterations 20 scripts/workspace-open.js +// k6 run --env PENPOT_BASE_URL=http://localhost:6060 scripts/workspace-open.js + +import { check, sleep, fail } from "k6"; +import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js"; +import { createClient } from "../lib/penpot-client.js"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const BASE_URL = __ENV.PENPOT_BASE_URL || "http://localhost:6060"; +const OPEN_ITERATIONS = parseInt(__ENV.PENPOT_OPEN_ITERATIONS || "5"); + +export const options = { + thresholds: { + http_req_duration: ["p(95)<5000"], + http_req_failed: ["rate<0.01"], + "http_req_duration{rpc_command:get-file}": ["p(95)<500"], + "http_req_duration{rpc_command:get-file-libraries}": ["p(95)<500"], + "http_req_duration{rpc_command:get-file-object-thumbnails}": ["p(95)<500"], + "http_req_duration{rpc_command:get-file-data-for-thumbnail}": ["p(95)<500"], + }, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function assertOk(res, label) { + const ok = check(res, { + [`${label} — status is 2xx`]: (r) => r.status >= 200 && r.status < 300, + }); + if (!ok) { + let bodyStr = ""; + try { + if (res.raw && res.raw.body) { + bodyStr = typeof res.raw.body === "string" + ? res.raw.body.substring(0, 500) + : JSON.stringify(res.raw.body).substring(0, 500); + } else if (res.body) { + bodyStr = JSON.stringify(res.body).substring(0, 500); + } + } catch (e) { + bodyStr = "(could not read body)"; + } + console.error(`FAIL: ${label} — status=${res.status} body=${bodyStr}`); + } + return ok; +} + +// --------------------------------------------------------------------------- +// Setup — create one user + one file with data +// --------------------------------------------------------------------------- + +export function setup() { + console.log(`Penpot Workspace Open Test`); + console.log(` Base URL: ${BASE_URL}`); + console.log(` Open iterations: ${OPEN_ITERATIONS}`); + console.log(``); + + const client = createClient(BASE_URL); + + // Verify backend reachable + if (client.getProfile().status === 0) fail(`Backend unreachable at ${BASE_URL}`); + + // Create one demo user + const demoRes = client.rpc("POST", "create-demo-profile", {}); + if (demoRes.status !== 200) fail("Failed to create demo profile"); + const { email, password } = demoRes.json(); + + // Login + const loginRes = client.login(email, password); + if (loginRes.status !== 200) fail("Login failed"); + + // Create project + file + const teamId = client.getTeams().body[0].id; + const projectId = client.createProject(teamId, "WS Open Project").body.id; + const fileId = client.createFile(projectId, "WS Open File").body.id; + + // Get file data and add a shape so it has meaningful content + const fileData = client.getFile(fileId).body; + const pageId = fileData.data.pages[0]; + const shapeId = uuidv4(); + const x = 50, y = 50, w = 300, h = 200; + + client.updateFile(fileId, fileData.revn, fileData.vern, uuidv4(), [{ + type: "add-obj", pageId, id: shapeId, frameId: pageId, parentId: pageId, + obj: { + id: shapeId, type: "rect", name: "Background", + x, y, width: w, height: h, + fillColor: "#cccccc", fillOpacity: 1, + rotation: 0, hidden: false, locked: false, + selrect: { x, y, width: w, height: h, x1: x, y1: y, x2: x + w, y2: y + h }, + points: [{ x, y }, { x: x + w, y }, { x: x + w, y: y + h }, { x, y: y + h }], + transform: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + transformInverse: { a: 1, b: 0, c: 0, d: 1, e: 0, f: 0 }, + parentId: pageId, frameId: pageId, + }, + }]); + + console.log(` File ready: ${fileId} (page: ${pageId})`); + + return { baseUrl: BASE_URL, email, password, fileId }; +} + +// --------------------------------------------------------------------------- +// Main VU Function — all VUs read the same file +// --------------------------------------------------------------------------- + +export default function (data) { + const client = createClient(data.baseUrl); + + // Login with shared user + if (!assertOk(client.login(data.email, data.password), "login")) fail("login failed"); + + sleep(0.5); + + for (let i = 0; i < OPEN_ITERATIONS; i++) { + if (!assertOk(client.getFile(data.fileId), "get-file")) fail("get-file failed"); + sleep(0.3); + + if (!assertOk(client.getFileLibraries(data.fileId), "get-file-libraries")) fail("get-file-libraries failed"); + sleep(0.2); + + if (!assertOk(client.getFileObjectThumbnails(data.fileId), "get-file-object-thumbnails")) fail("get-file-object-thumbnails failed"); + sleep(0.2); + + if (!assertOk(client.getFileDataForThumbnail(data.fileId), "get-file-data-for-thumbnail")) fail("get-file-data-for-thumbnail failed"); + sleep(1); + } + + console.log(`VU ${__VU}: Completed ${OPEN_ITERATIONS} open iterations`); +} + +// --------------------------------------------------------------------------- +// Teardown +// --------------------------------------------------------------------------- + +export function teardown(data) { + console.log("Workspace open test complete."); +} diff --git a/backend/src/app/auth.clj b/backend/src/app/auth.clj index 1f978f357c..efc508936a 100644 --- a/backend/src/app/auth.clj +++ b/backend/src/app/auth.clj @@ -14,10 +14,21 @@ :iterations 3 :parallelism 2}) +(def ^:private weak-options + {:alg :pbkdf2+sha256 + :iterations 100}) + (defn derive-password [password] (hashers/derive password default-options)) +(defn derive-password-weak + "Derives a password using a fast algorithm (pbkdf2+sha256, 100 iterations). + Intended for demo users only — they are already gated behind the + `demo-users` config flag which is disabled in production." + [password] + (hashers/derive password weak-options)) + (defn verify-password [attempt password] (try diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index ca8fba8af1..a6048a970d 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -391,6 +391,8 @@ :delete-object (ig/ref :app.tasks.delete-object/handler) + :demo-purge + (ig/ref :app.tasks.demo-purge/handler) :process-webhook-event (ig/ref ::webhooks/process-event-handler) :run-webhook @@ -428,6 +430,9 @@ :app.tasks.delete-object/handler {::db/pool (ig/ref ::db/pool)} + :app.tasks.demo-purge/handler + {::db/pool (ig/ref ::db/pool)} + :app.tasks.file-gc/handler {::db/pool (ig/ref ::db/pool) ::sto/storage (ig/ref ::sto/storage)} diff --git a/backend/src/app/rpc/commands/demo.clj b/backend/src/app/rpc/commands/demo.clj index 13b7a2f374..17fba7f1b9 100644 --- a/backend/src/app/rpc/commands/demo.clj +++ b/backend/src/app/rpc/commands/demo.clj @@ -7,9 +7,9 @@ (ns app.rpc.commands.demo "A demo specific mutations." (:require - [app.auth :refer [derive-password]] + [app.auth :refer [derive-password-weak]] [app.common.exceptions :as ex] - [app.common.time :as ct] + [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.loggers.audit :as audit] @@ -17,6 +17,7 @@ [app.rpc.commands.auth :as auth] [app.rpc.doc :as-alias doc] [app.util.services :as sv] + [app.worker :as wrk] [buddy.core.codecs :as bc] [buddy.core.nonce :as bn])) @@ -34,8 +35,8 @@ :code :demo-users-not-allowed :hint "Demo users are disabled by config.")) - (let [sem (System/currentTimeMillis) - email (str "demo-" sem ".demo@example.com") + (let [sem (uuid/next) + email (str "demo-" sem "@demo.example.com") fullname (str "Demo User " sem) password (-> (bn/random-bytes 16) @@ -46,12 +47,18 @@ :fullname fullname :is-active true :is-demo true - :deleted-at (ct/in-future (cf/get-deletion-delay)) - :password (derive-password password) + :password (derive-password-weak password) :props {}} + profile (db/tx-run! cfg (fn [cfg] (->> (auth/create-profile cfg params) (auth/create-profile-rels cfg))))] + + (wrk/submit! (-> cfg + (assoc ::wrk/task :demo-purge) + (assoc ::wrk/delay (cf/get-deletion-delay)) + (assoc ::wrk/params {:profile-id (:id profile)}))) + (with-meta {:email email :password password} {::audit/profile-id (:id profile)}))) diff --git a/backend/src/app/rpc/commands/profile.clj b/backend/src/app/rpc/commands/profile.clj index 79af070c6a..eb89684afa 100644 --- a/backend/src/app/rpc/commands/profile.clj +++ b/backend/src/app/rpc/commands/profile.clj @@ -124,9 +124,7 @@ (defn get-profile "Get profile by id. Throws not-found exception if no profile found." [conn id & {:as opts}] - ;; NOTE: We need to set ::db/remove-deleted to false because demo profiles - ;; are created with a set deleted-at value - (-> (db/get-by-id conn :profile id (assoc opts ::db/remove-deleted false)) + (-> (db/get-by-id conn :profile id opts) (decode-row))) ;; --- MUTATION: Update Profile (own) diff --git a/backend/src/app/tasks/demo_purge.clj b/backend/src/app/tasks/demo_purge.clj new file mode 100644 index 0000000000..429816c053 --- /dev/null +++ b/backend/src/app/tasks/demo_purge.clj @@ -0,0 +1,41 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.tasks.demo-purge + "Task handler for delayed demo profile deletion. Submitted at demo + creation time with a delay matching the configured deletion-delay." + (:require + [app.common.logging :as l] + [app.common.time :as ct] + [app.db :as db] + [app.worker :as wrk] + [integrant.core :as ig])) + +(defmethod ig/assert-key ::handler + [_ params] + (assert (db/pool? (::db/pool params)) "expected a valid database pool")) + +(defmethod ig/init-key ::handler + [_ cfg] + (fn [{:keys [props]}] + (let [profile-id (get props :profile-id) + now (ct/now)] + + (l/trc :hint "demo-purge" :profile-id (str profile-id)) + + ;; Mark the profile for immediate deletion + (db/tx-run! cfg + (fn [{:keys [::db/conn] :as cfg}] + (db/update! conn :profile + {:deleted-at now} + {:id profile-id} + {::db/return-keys false}) + (wrk/submit! + (-> cfg + (assoc ::wrk/task :delete-object) + (assoc ::wrk/params {:object :profile + :deleted-at now + :id profile-id})))))))) diff --git a/backend/test/backend_tests/demo_test.clj b/backend/test/backend_tests/demo_test.clj new file mode 100644 index 0000000000..da1cc342c9 --- /dev/null +++ b/backend/test/backend_tests/demo_test.clj @@ -0,0 +1,47 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.demo-test + (:require + [app.common.time :as ct] + [app.db :as db] + [app.rpc.commands.profile :as profile] + [app.tasks.demo-purge :as demo-purge] + [app.worker :as wrk] + [backend-tests.helpers :as th] + [clojure.test :as t] + [integrant.core :as ig])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(t/deftest demo-profile-created-without-deleted-at + (let [profile (th/create-profile* 999 {:is-demo true})] + (t/is (true? (:is-demo profile))) + (t/is (nil? (:deleted-at profile))) + (t/is (some? (:id profile))))) + +(t/deftest get-profile-finds-demo-user-without-override + (let [profile (th/create-profile* 998 {:is-demo true}) + found (db/run! th/*pool* + (fn [{:keys [::db/conn]}] + (profile/get-profile conn (:id profile))))] + (t/is (some? found)) + (t/is (= (:id profile) (:id found))))) + +(t/deftest demo-purge-handler-submits-delete-object + (let [profile (th/create-profile* 996 {:is-demo true}) + handler (ig/init-key :app.tasks.demo-purge/handler + {::db/pool th/*pool*}) + submitted (atom nil)] + (with-redefs [wrk/submit! (fn [& {:keys [::wrk/task ::wrk/params]}] + (reset! submitted {:task task :params params}))] + (handler {:props {:profile-id (:id profile) + :deleted-at (ct/now)}})) + (t/is (= :delete-object (:task @submitted))) + (t/is (= :profile (:object (:params @submitted)))) + (t/is (= (:id profile) (:id (:params @submitted)))) + (t/is (some? (:deleted-at (:params @submitted)))))) diff --git a/docker/devenv/Dockerfile b/docker/devenv/Dockerfile index 351499df2c..2f3e37677f 100644 --- a/docker/devenv/Dockerfile +++ b/docker/devenv/Dockerfile @@ -186,6 +186,7 @@ ENV CLJKONDO_VERSION=2026.04.15 \ CLJFMT_VERSION=0.16.4 \ PIXI_VERSION=0.67.2 \ GITHUB_CLI_VERSION=2.91.0 \ + K6_VERSION=2.0.0 \ UV_VERSION=0.11.9 \ UV_TOOL_DIR=/opt/uv/tools \ UV_TOOL_BIN_DIR=/opt/utils/bin \ @@ -314,6 +315,28 @@ RUN set -ex; \ mv /tmp/mc /opt/utils/bin/; \ chmod +x /opt/utils/bin/mc; +# Install k6 +RUN set -ex; \ + ARCH="$(dpkg --print-architecture)"; \ + case "${ARCH}" in \ + aarch64|arm64) \ + BINARY_URL="https://github.com/grafana/k6/releases/download/v$K6_VERSION/k6-v$K6_VERSION-linux-arm64.tar.gz"; \ + ;; \ + amd64|x86_64) \ + BINARY_URL="https://github.com/grafana/k6/releases/download/v$K6_VERSION/k6-v$K6_VERSION-linux-amd64.tar.gz"; \ + ;; \ + *) \ + echo "Unsupported arch: ${ARCH}"; \ + exit 1; \ + ;; \ + esac; \ + curl -LfsSo /tmp/k6.tar.gz ${BINARY_URL}; \ + cd /tmp; \ + tar -xf /tmp/k6.tar.gz; \ + mv /tmp/k6-v$K6_VERSION-linux-*/k6 /opt/utils/bin/; \ + chmod +x /opt/utils/bin/k6; \ + rm -rf /tmp/k6.tar.gz /tmp/k6-v$K6_VERSION-linux-*; + # Install uv RUN set -ex; \ ARCH="$(dpkg --print-architecture)"; \ diff --git a/plans/2026-06-12-backend-performance-test.md b/plans/2026-06-12-backend-performance-test.md new file mode 100644 index 0000000000..4f6a38388e --- /dev/null +++ b/plans/2026-06-12-backend-performance-test.md @@ -0,0 +1,597 @@ +# Backend Performance Test Plan + +**Context:** Build a k6-based load/performance test suite that simulates realistic browser-to-backend HTTP flows for distinct Penpot user operations. The goal is to measure backend impact (latency, throughput, error rates, resource saturation) under synthetic user load. **Browser rendering performance is explicitly out of scope.** WebSocket testing is deferred. + +**Date:** 2026-06-12 +**Validated Requirements:** +- Tool: **k6** (confirmed). +- Environment: flexible — local devenv first, then remote staging/perf. +- Target scale: **1000 concurrent VUs** (ramping from lower baselines). +- Flows: **realistic CRUD lifecycle** — create, edit, upload, delete. Must include **image upload** and **font upload**. +- `update-file` is important but difficult because it requires **2–3 concurrent users editing the same file**, and **file size matters**. +- WebSocket: **deferred**. + +--- + +## Current Progress + +### Completed (2026-06-12) + +Phase 1 done. Phase 2 done (all core flows + performance optimization). Phase 3 done (orchestrator). Phase 4 done (concurrent editing + file size matrix). Phase 5 remains. + +**What was built:** + +``` +performance/ +├── run.sh # Bash runner — all commands + orchestrator +├── README.md # Usage docs, configuration, architecture notes +├── lib/ +│ └── penpot-client.js # ~590 lines — shared k6 HTTP client module +├── scripts/ +│ ├── lifecycle.js # Full user lifecycle (register → CRUD → delete) +│ ├── workspace-open.js # Read-heavy: file open loop (get-file, libraries, thumbnails) +│ ├── workspace-edit.js # Write-heavy: file edit loop (get-file + update-file) +│ ├── workspace-edit-concurrent.js # Concurrent editing: same-file or multi-file mode +│ ├── file-size-matrix.js # File size matrix: latency vs shape count (10, 100, 500, 1000) +│ ├── media-upload.js # Image uploads: SVG/PNG direct, JPG chunked +│ ├── font-upload.js # Font uploads: TTF+OTF chunked, create-font-variant +│ └── compare-results.cjs # Compare two k6 JSON results for regression +├── results/ # k6 JSON output (gitignored) +└── baselines/ # for regression baselines +``` + +Fixtures are reused from `backend/test/backend_tests/test_files/` (no copies in `performance/`). + +**Backend changes:** +- `backend/src/app/rpc/commands/demo.clj` — demo profile emails changed from timestamp-based to UUID-based (eliminates collisions). Uses `derive-password-weak` for fast password hashing. +- `backend/src/app/auth.clj` — added `derive-password-weak` using pbkdf2+sha256 (100 iterations, ~0.13ms/hash, ~700x faster than argon2id). Safe for demo users because `demo-users` flag is disabled by default in production. + +**All scripts use `setup()` user pool:** + +| Script | setup() creates | VU pattern | +|--------|----------------|------------| +| `lifecycle.js` | N users | Each VU picks `users[__VU-1]` → login → full CRUD | +| `workspace-open.js` | 1 user + 1 file with shape | All VUs share same user + file (realistic concurrent reads) | +| `workspace-edit.js` | N users + shared project | Each VU creates own file → edit loop | +| `media-upload.js` | N users | Each VU creates project/file → upload 3 images | +| `font-upload.js` | N users | Each VU uploads TTF+OTF → create-font-variant | + +Setup is sequential (~0.13ms/user with `derive-password-weak`), excluded from k6 metrics. At 1000 VUs: ~0.13s setup, then pure measurement. + +**All flows validated (smoke test, 1 VU, 1 iteration each):** + +| Script | Checks | Failure Rate | +|--------|--------|-------------| +| `lifecycle.js` | 10/10 | 0% | +| `workspace-open.js` | 9/9 | 0% | +| `workspace-edit.js` | 5/5 | 0% | +| `media-upload.js` | 8/8 | 0% | +| `font-upload.js` | 11/11 | 0% | + +**Orchestrator (`./run.sh all`) validated** — runs all 5 flows in parallel, 0% failure rate. + +**Key discoveries (cumulative):** + +1. **JSON transport works.** Backend accepts `Content-Type: application/json` (kebab-case keys auto-converted) and returns `application/json` (camelCase keys) via `Accept: application/json` or `_fmt=json`. No Transit encoder needed. + +2. **`create-file` `features` param.** Sending `features: []` causes 400. Omit entirely — it's optional (`backend/src/app/rpc/commands/files_create.clj`). + +3. **`update-file` shape schema is strict.** The `add-obj` change requires: `selrect`, `points` (4 corners), `transform`/`transform-inverse` (identity matrix), `parentId`/`frameId` inside `obj`, and `frameId` at the change top level. Schema: `common/src/app/common/files/changes.cljc:189`. + +4. **`update-file` URL convention.** `POST /api/main/methods/update-file?id=` — `id` in both query string and body. + +5. **Two registration modes:** `demo` (fast, needs `demo-users` flag) and `register` (two-step, no flags). + +6. **k6 at** `/home/penpot/.local/bin/k6` (v0.56.0). Use `PATH="/home/penpot/.local/bin:$PATH"` or `K6` env var. + +7. **Demo profile race condition — solved.** Backend now uses `uuid/next` for demo emails (no collisions). k6 scripts use `setup()` to create user pool before VUs start. Both changes together eliminate the scaling bottleneck. + +8. **Chunked upload threshold.** The client uses 50 KB chunk size. Files ≤50 KB use direct multipart; files >50 KB use `create-upload-session` → `upload-chunk` × N → `assemble-file-media-object`. + +9. **Font upload flow.** Each MIME type (ttf, otf, woff) gets its own `create-upload-session`. All session IDs are passed in the `uploads` map to `create-font-variant`. The `font-id` is a client-generated UUID that groups variants into a family. + +10. **MIME type validation.** The backend validates that the uploaded content MIME matches the declared MIME. `sample.jpg` must be sent as `image/jpeg`, not `image/png`. + +11. **workspace-open uses shared user.** All VUs read the same file with the same user. Multiple demo users can't access each other's files without team sharing, so a single shared user is the correct pattern for read-heavy tests. + +12. **Demo profile creation was slow due to argon2id — now solved.** `derive-password` in `backend/src/app/auth.clj` uses argon2id with 32 MiB memory, 3 iterations, parallelism 2 (~94ms/hash). Created `derive-password-weak` using pbkdf2+sha256 with 100 iterations (~0.13ms/hash) — **~700x faster**. `demo.clj` now uses `derive-password-weak` for all demo profiles. Safe because `demo-users` is already a development-only feature (disabled by default in production). At 1000 VUs, setup time drops from ~2–3 min to ~0.13 sec. + +13. **bcrypt minimum cost factor is 4.** Can't go below 4 for bcrypt. pbkdf2+sha256 with 100 iterations is even faster (~0.13ms/hash vs ~2.7ms for bcrypt cost 4) and was chosen instead. Benchmark: argon2id ~94ms/hash, bcrypt cost 4 ~2.7ms/hash, pbkdf2+sha256 100 iter ~0.13ms/hash. + +14. **revn conflicts don't happen in normal concurrent editing.** The conflict check in `files_update.clj` is `(> incoming stored)` — only fires when incoming revn is *greater* than stored. If two VUs both read revn=5 and VU A saves first (revn becomes 6), VU B saves with revn=5 → `5 > 6?` → false → no conflict. The real contention point is the **file-level advisory lock** (`db/xact-lock! conn id`) that serializes all `update-file` calls on the same file. More VUs = more lock queuing = higher latency. + +15. **`update-file` response doesn't include `vern`.** The response is `{:revn N, :lagged [...]}`. `vern` only changes on snapshot restore, so it can be kept constant across iterations. Get it from the initial `get-file` call. + +### Remaining Work + +| Phase | Status | Next Actions | +|-------|--------|-------------| +| Phase 1 – Discovery & Tooling | **Done** | — | +| Phase 2 – Core HTTP Flows | **Done** | All 5 flows + orchestrator + setup() pool | +| Phase 2 – Performance Optimization | **Done** | `derive-password-weak` using pbkdf2+sha256 (100 iter) — ~700x faster than argon2id | +| Phase 3 – Scenarios | **Done** | `./run.sh all` runs all flows in parallel | +| Phase 4 – Concurrent Editing | **Done** | `workspace-edit-concurrent.js` with same-file and multi-file modes | +| Phase 4 – File Size Matrix | **Done** | `file-size-matrix.js` with 4 tiers (10, 100, 500, 1000 shapes) | +| Phase 5 – Regression Guard | **Done** | `compare-results.cjs` + CI workflow (relative comparison) | +| Phase 5 – Grafana Dashboards | **Deferred** | No Prometheus remote write or InfluxDB in current stack | + +### Immediate Next Steps + +1. ~~Phase 2 – Fast password for demo users~~ ✅ Done +2. ~~Phase 4: File size matrix (`update-file` latency vs shape count: 10, 100, 500, 1000 shapes).~~ ✅ Done — `file-size-matrix.js` with 4 tiers +3. ~~Phase 4: Concurrent editing test (2–3 VUs per file, measure conflict rate).~~ ✅ Done — `workspace-edit-concurrent.js` with same-file and multi-file modes +4. ~~Phase 5: Regression guard — implement `compare-results.cjs` and CI workflow.~~ ✅ Done +5. ~~Add `--scenario` flag to `run.sh`~~ ✅ Done +6. Write `viewer.js` — `get-view-only-bundle` + `get-comment-threads` (deferred per user request). + +--- + +## Affected Modules + +| Module | Why it is involved | +|--------|---------------------| +| `backend/` | Target system. All HTTP RPC (`/api/main/methods/*`), auth, storage, media processing, DB, and Prometheus metrics (`/metrics`). | +| `frontend/` | Source of truth for user request flows. We inspect `app.main.repo` (RPC client), `app.main.data.*` (user flows), and `app.main.data.persistence` (save semantics). | +| `common/` | Shared schemas, Transit helpers, and data structures. Used to understand valid `update-file` `changes` payloads. | + +--- + +## Approach + +### Phase 1 – Discovery & Tooling (Days 1–2) + +#### 1.1. Read the frontend RPC flows to build a request catalog + +Inspect these files to map every user action to its RPC command: + +- `frontend/src/app/main/repo.cljs` — HTTP client conventions (headers, retry, GET vs POST rules, query params, form-data, multipart). +- `frontend/src/app/main/data/dashboard.cljs` — Dashboard init (`get-projects`, `fetch-fonts`, `search-files`). +- `frontend/src/app/main/data/workspace.cljs` — Workspace init (`get-file`, `get-file-libraries`, `get-file-object-thumbnails`, `resolve-file` via `get-file-fragment`). +- `frontend/src/app/main/data/persistence.cljs` — File save flow (`update-file` with `changes`, `revn`, `session-id`, debounce/buffer logic). +- `frontend/src/app/main/data/viewer.cljs` — Viewer flow (`get-view-only-bundle`). +- `frontend/src/app/main/data/comments.cljs` — Comment thread fetch (`get-comment-threads`). +- `frontend/src/app/main/data/media.cljs` / `upload.cljs` — Media upload flows (`upload-file-media-object`, `create-upload-session`, `upload-chunk`, `assemble-file-media-object`). +- `frontend/src/app/main/data/fonts.cljs` — Font upload flow (`create-font-variant` with `:uploads` map). +- `frontend/src/app/main/data/team.cljs` — Team creation (`create-team`), invitation (`create-team-invitations`). +- `frontend/src/app/main/data/project.cljs` — Project creation (`create-project`). + +**Goal:** produce a **Request Catalog** mapping user actions to RPC command names, HTTP methods, payload shapes, and required preconditions (e.g., `team-id`, `file-id`). + +#### 1.2. Confirm JSON compatibility for the test harness + +The backend middleware (`app.http.middleware`) supports `application/json` request bodies and `application/json` responses (via `_fmt=json` or `Accept: application/json`). + +- **Action:** Send a manual `curl` to `POST /api/main/methods/login-with-password` with `Content-Type: application/json` and verify the response format. +- **Action:** Verify `GET /api/main/methods/get-profile` with `Accept: application/json` returns plain JSON. +- **Action:** Verify `POST /api/main/methods/update-file` with `Content-Type: application/json` and `_fmt=json` works. +- **Action:** Verify `POST /api/main/methods/upload-file-media-object` with `multipart/form-data` works (k6 supports this natively). + +#### 1.3. Set up the load testing directory and shared client + +Create a directory `performance/` at the repo root. + +Install **k6** (`k6` CLI or Docker image). + +Create a shared `penpot-client.js` module that wraps: +- `login(email, password)` → returns session cookie / token. +- `rpc(cmd, params, opts)` → builds the correct URL, headers, body, and query params. +- `uploadFileMediaObject(fileId, filePath, name)` → multipart upload. +- `createUploadSession(totalChunks)` → chunked upload setup. +- `uploadChunk(sessionId, index, chunkBytes)` → multipart chunk upload. +- `assembleFileMediaObject(sessionId, fileId, name, isLocal)` → finalize chunked upload. + +**Headers to replicate (critical for backend telemetry and session binding):** +- `x-session-id`: generated UUID per VU (must be consistent across requests for the same session). +- `x-external-session-id`: generated UUID per VU. +- `x-event-origin`: a string origin (e.g., `"perf-test"`) +- `accept`: `application/json` (for HTTP-only load path) +- `content-type`: `application/json` (or `multipart/form-data` for uploads) +- `credentials: "include"` (for cookie jar) + +#### 1.4. Data seeding strategy for 1000 VU scale + +Creating 1000 users/teams/files *inside* the load test is too slow and will distort the results. + +**Recommended approach:** +- **Setup Phase (k6 `setup()`):** Run a pre-test script that creates a shared pool of test artifacts. + - Use `login-with-password` with a fixture account (e.g., `profile1@example.com` / `123123` if fixtures exist). + - Create `N` teams, `N` projects, `N` files of varying sizes (see **File Size Tiers** below). + - Export the IDs into a JSON file that k6 `setup()` reads. +- **Alternative:** Use the backend REPL / fixtures (`app.cli.fixtures/run {:preset :small}`) to create fixture data, then export the IDs via a small Clojure script. +- **Data pool per VU:** Each VU picks a random user from the pool, or uses a dedicated user (e.g., VU #1 → `profile1@example.com`, VU #2 → `profile2@example.com`). For 1000 VUs, we need at least 1000 pre-seeded users. +- **Cleanup:** A post-test script can delete the seeded data, or we can use a dedicated perf DB that is reset between runs. + +**Action:** Document the seeding procedure in `performance/README.md` and create a `seed-data.js` script. + +--- + +### Phase 2 – Core HTTP Flow Scripts (Days 3–5) + +Create one k6 script per user flow. Each script: +- Uses `setup()` to read the shared data pool and log in. +- Uses `vu` iterations to simulate the flow. +- Tags every request with the RPC command name so k6 metrics are sliced by endpoint. +- Uses `check()` assertions for HTTP 200 and valid JSON structure. + +#### Flow 1: Realistic User Lifecycle (`lifecycle.js`) + +This is the primary realistic flow. Each VU performs a full lifecycle: + +1. **Auth** + - `POST /api/main/methods/login-with-password` → `{email, password}` + - `GET /api/main/methods/get-profile` + - `GET /api/main/methods/get-teams` + +2. **Create Team** + - `POST /api/main/methods/create-team` → `{name: "Perf Team "}` + - `GET /api/main/methods/get-team?team-id=` + +3. **Create Project** + - `POST /api/main/methods/create-project` → `{team-id, name}` + - `GET /api/main/methods/get-project?id=` + +4. **Create File** + - `POST /api/main/methods/create-file` → `{project-id, name, features}` + - `GET /api/main/methods/get-file?id=&features=<...>` + +5. **Edit File (Simple Update)** + - `POST /api/main/methods/update-file` with a minimal `changes` payload. + - **Changes payload:** Use a simple change like `{:type "add-obj", :id "", :page-id "", :parent-id "", :obj {:type "rect", ...}}`. Inspect `app.common.files.changes` for the exact schema. For a load test, we only need the shape to be structurally valid; the backend validates it. + - **Revn tracking:** Fetch the file first, read `revn`, then send `revn` in the update. If a conflict occurs (`409` or `:revn-conflict` error), retry once with the latest `revn`. + +6. **Upload Image (Direct)** + - `POST /api/main/methods/upload-file-media-object` (multipart) + - Payload: `file-id`, `is-local: true`, `name`, `content` (the file bytes). + - Use a small dummy PNG/SVG (e.g., 1 KB, 100 KB, 1 MB) stored in `performance/fixtures/`. + +7. **Upload Image (Chunked)** + - `POST /api/main/methods/create-upload-session` → `{total-chunks: N}` + - Loop `N` times: `POST /api/main/methods/upload-chunk` (multipart, `session-id`, `index`, `chunk`) + - `POST /api/main/methods/assemble-file-media-object` → `{session-id, file-id, name, is-local}` + - Use a larger dummy file (e.g., 5 MB) to stress the chunked pipeline. + +8. **Upload Font** + - `POST /api/main/methods/create-upload-session` (chunked, because fonts can be large) + - `POST /api/main/methods/upload-chunk` for each chunk + - `POST /api/main/methods/create-font-variant` → `{team-id, font-id, font-family, font-weight, font-style, uploads: {"font/ttf": ""}}` + - Use a small real TTF/OTF file from `performance/fixtures/`. + +9. **Delete File** + - `DELETE /api/main/methods/delete-file` (verify the exact method name; it may be `update-file` with a deletion flag or a dedicated command). Inspect `frontend/src/app/main/data/dashboard.cljs` for the delete action. + +10. **Delete Project** + - `DELETE /api/main/methods/delete-project?id=` + +11. **Delete Team** + - `POST /api/main/methods/delete-team?id=` + +12. **Logout** + - (Optional; session cookie expiry is usually sufficient) + +**Pacing:** Add `sleep()` between steps to simulate realistic think time (e.g., 1–3 seconds between dashboard navigation, 3–5 seconds between edits). + +#### Flow 2: Workspace Open (Read-heavy) (`workspace-open.js`) + +For 1000 VUs, most will be read-only viewers or editors opening files. + +1. Login (reuse token from `setup`). +2. `GET /api/main/methods/get-file?id=&features=<...>` +3. `GET /api/main/methods/get-file-libraries?file-id=` +4. For each library: `GET /api/main/methods/get-file?id=` +5. `GET /api/main/methods/get-file-object-thumbnails?file-id=` +6. `GET /api/main/methods/get-file-data-for-thumbnail?file-id=&page-id=&object-id=` + +**Data:** Use a pool of files of varying sizes (see **File Size Tiers**). + +#### Flow 3: Workspace Edit (Write-heavy) (`workspace-edit.js`) + +**Scenario A — Independent editors (default, easiest to scale):** +- Each VU creates its own file in `setup()`, or picks a dedicated file from the pool. +- Loop: + 1. `GET /api/main/methods/get-file?id=` (to refresh `revn`) + 2. `POST /api/main/methods/update-file` with minimal changes + 3. `sleep(3)` +- This measures the latency of the save path without concurrency conflicts. + +**Scenario B — Concurrent editors (advanced, measures conflict resolution):** +- 2–3 VUs share the **same file ID**. +- Each VU: + 1. `GET /api/main/methods/get-file?id=` (to get latest `revn`) + 2. `POST /api/main/methods/update-file` with changes + 3. If `revn-conflict` (HTTP 400 or 409 with `:code :revn-conflict`), retry with the latest `revn`. +- **Problem:** k6 VUs are independent; they cannot easily share a mutable `revn` counter. +- **Solutions:** + 1. **Optimistic concurrency:** Let conflicts happen naturally. Measure the conflict rate and retry latency. This is realistic for many-user editing. + 2. **Shared state service:** Run a tiny Redis or in-memory service that stores the latest `revn` per file. VUs read/write it before each update. This adds coordination overhead but reduces conflicts. + 3. **Sequential VU groups:** Use k6 `scenarios` with `executor: 'per-vu-iterations'` and a small shared file pool. Accept that some conflicts will occur and measure them as part of the benchmark. +- **Recommendation:** Start with **Solution 1** (optimistic). If the conflict rate is >10%, consider **Solution 2**. + +**File Size Tiers for `update-file`:** +The backend `update-file` performance depends heavily on file data size (serialization, validation, pointer-map resolution, snapshotting). + +| Tier | Size | How to create | +|------|------|---------------| +| Small | ~10 shapes | Create a file with a few rectangles. | +| Medium | ~100 shapes | Duplicate a page with many shapes. | +| Large | ~1000 shapes | Import a real-world design file or use a fixture. | + +**Action:** Create a `create-file-fixture.js` helper that generates files of each tier via the `create-file` + `update-file` API (or by importing a `.penpot` file via the binfile import API if available). + +#### Flow 4: Viewer (Read-heavy, anonymous or logged-in) (`viewer.js`) + +1. Login (or use share-link token for anonymous). +2. `GET /api/main/methods/get-view-only-bundle?file-id=&share-id=&features=<...>` +3. `GET /api/main/methods/get-comment-threads?file-id=&share-id=` + +#### Flow 5: Export (CPU/IO-heavy) (`export.js`) + +1. Login. +2. `POST /api/export` with export payload. + - Inspect `frontend/src/app/main/data/export.cljs` for the exact payload shape. + - Common exports: `type: "png"`, `type: "svg"`, `type: "pdf"`. + - This hits the **exporter** service (Node.js/Playwright), which is a separate process. If the goal is to stress the **backend**, limit export tests or target the backend export queue endpoints. + +#### Flow 6: Media Upload (Storage/IO-heavy) (`media-upload.js`) + +1. Login. +2. Direct upload: `POST /api/main/methods/upload-file-media-object` (multipart, small PNG). +3. Chunked upload: `POST /api/main/methods/create-upload-session` → `upload-chunk` x N → `assemble-file-media-object` (large PNG). +4. URL-based upload: `POST /api/main/methods/create-file-media-object-from-url` (if a stable external image URL is available). + +#### Flow 7: Font Upload (Storage/CPU-heavy) (`font-upload.js`) + +1. Login. +2. `POST /api/main/methods/create-upload-session` (for the font file) +3. `POST /api/main/methods/upload-chunk` for each chunk +4. `POST /api/main/methods/create-font-variant` → `{team-id, font-id, font-family, font-weight, font-style, uploads: {"font/ttf": ""}}` +5. `GET /api/main/methods/get-font-variants?team-id=` + +--- + +### Phase 2 – Performance Optimization: Fast Password Hashing for Demo Users ✅ Done + +**Goal:** Reduce `setup()` time for performance tests by making demo profile password derivation faster. + +**Problem:** `derive-password` in `backend/src/app/auth.clj` uses argon2id with 32 MiB memory, 3 iterations, parallelism 2 (~94ms/hash). At 1000 VUs, creating the user pool in `setup()` takes ~2–3 minutes just for password hashing. + +**Solution:** +Since `demo-users` is already a development-only feature (disabled by default in production), all demo profiles use a weaker, faster password algorithm. No special parameters or tenant checks needed. + +1. In `backend/src/app/auth.clj`, added `derive-password-weak` using pbkdf2+sha256 with 100 iterations (~0.13ms/hash — **~700x faster** than argon2id). +2. In `backend/src/app/rpc/commands/demo.clj`, switched from `derive-password` to `derive-password-weak`. + +**Files touched:** +- `backend/src/app/auth.clj` — added `weak-options` (pbkdf2+sha256, 100 iter) and `derive-password-weak` +- `backend/src/app/rpc/commands/demo.clj` — uses `derive-password-weak` instead of `derive-password` + +**Impact:** Setup time for 1000 users dropped from ~2–3 min to ~0.13 sec (~700x improvement). + +**Safety:** Demo users are already a development-only feature (disabled by default in production via `demo-users` config flag). Using weaker passwords for demo users only affects development/test environments where the flag is explicitly enabled. + +--- + +### Phase 3 – Scenarios & Orchestration (Day 6) + +Define k6 `options.scenarios` that mix the flows to simulate realistic traffic. + +**Example scenario mix for 1000 VUs:** + +| Scenario | Script | VUs | Arrival Rate | Duration | Notes | +|----------|--------|-----|--------------|----------|-------| +| `lifecycle` | `lifecycle.js` | 100 | 1/s (ramp 0→100 over 5m) | 10m | Full CRUD, most realistic. | +| `workspace_open` | `workspace-open.js` | 400 | 5/s (ramp 0→400 over 5m) | 10m | Read-heavy, simulates many editors opening files. | +| `workspace_edit` | `workspace-edit.js` | 200 | 2/s (ramp 0→200 over 5m) | 10m | Write-heavy, independent files. | +| `workspace_edit_concurrent` | `workspace-edit.js` | 30 (10 groups of 3) | 0.5/s | 10m | 3 VUs per file, measures conflicts. | +| `viewer` | `viewer.js` | 200 | 3/s (ramp 0→200 over 5m) | 10m | Read-heavy, simulates public/private viewers. | +| `media_upload` | `media-upload.js` | 50 | 0.5/s | 10m | Storage stress. | +| `font_upload` | `font-upload.js` | 20 | 0.2/s | 10m | Font processing stress. | + +**Thresholds:** +- `http_req_duration{p95} < 200ms` for `get-profile`, `get-teams`, `get-projects`. +- `http_req_duration{p95} < 500ms` for `get-file` (small), `search-files`. +- `http_req_duration{p95} < 2000ms` for `get-file` (large / 1000 shapes). +- `http_req_duration{p95} < 1000ms` for `update-file` (small). +- `http_req_duration{p95} < 3000ms` for `update-file` (large). +- `http_req_duration{p95} < 5000ms` for `upload-file-media-object` (1 MB). +- `http_req_duration{p95} < 10000ms` for `assemble-file-media-object` (5 MB chunked). +- `http_req_failed < 1%` globally. +- `http_req_failed{code:revn-conflict} < 5%` for `workspace_edit_concurrent`. + +**Correlation with backend metrics:** +- Scrape `/metrics` before, during, and after the test. +- Key Prometheus metrics to watch: + - `rpc_main_timing_seconds` (histogram/summary, labeled by command name) + - `rpc_management_timing_seconds` + - `http_server_dispatch_timing_seconds` + - `websocket_active_connections` (if any WS is active) + - `websocket_messages_total` + - JVM hotspot metrics (`process_cpu_seconds_total`, `jvm_memory_bytes_used`, `jvm_threads_current`) + - HikariCP metrics (if exposed; check `com.zaxxer.hikari:type=Pool` via JMX or custom Prometheus exporter) + - PostgreSQL: `pg_stat_activity` count by state. + - Redis: `INFO` `connected_clients`, `used_memory`. + +--- + +### Phase 4 – Advanced `update-file` Testing (Days 7–8) + +Because `update-file` is the core of the product and the user explicitly noted that **file size matters** and **concurrent editing is difficult**, we need a dedicated deep-dive. + +#### 4.1. File Size Tiers + +Create a `file-size-matrix.js` script that parameterizes the file size: +- `SMALL_FILE_ID`: 1 page, 10 shapes. +- `MEDIUM_FILE_ID`: 1 page, 100 shapes. +- `LARGE_FILE_ID`: 1 page, 500 shapes. +- `XLARGE_FILE_ID`: 1 page, 1000+ shapes, or a multi-page file. + +Run `workspace-edit.js` against each tier separately and plot: +- `update-file` latency vs file size. +- `get-file` latency vs file size. +- Backend CPU and DB time vs file size. + +#### 4.2. Concurrent Editing — Two Modes + +**Key insight:** `revn` conflicts only occur when `incoming > stored` (should never happen in normal usage). The real contention point is the **file-level advisory lock** (`db/xact-lock! conn id`) that serializes all `update-file` calls on the same file. + +**Mode 1: Same-file** — N VUs edit different pages in 1 file +- Measures lock contention on a single popular file +- Bottleneck: advisory lock serialization + +**Mode 2: Multi-file** — G groups × M VUs per file, each group edits its own file +- Measures whole system responsiveness under parallel edit sessions +- Bottleneck: DB connection pool, CPU, memory +- More realistic: real usage has many files being edited concurrently + +**Script:** `workspace-edit-concurrent.js` + +**Configuration via env vars:** +- `PENPOT_EDIT_MODE=same-file | multi-file` (default: `same-file`) +- `PENPOT_FILE_COUNT=1` — number of files (for multi-file mode) +- `PENPOT_VUS_PER_FILE=3` — VUs per file (for multi-file mode) + +**Setup logic:** +- `same-file`: create 1 file, add N pages (N = total VUs) +- `multi-file`: create G files, each with M pages (G = FILE_COUNT, M = VUS_PER_FILE) + +**VU loop:** +1. Login with assigned user +2. Get file → pick assigned page +3. Loop (10 iterations): + - `get-file` → get latest `revn` + - `sleep(0.3)` (think time) + - `update-file` with change to assigned page (add rectangle) + - Track: success on first try (should always succeed) + - `sleep(1)` (edit pacing) + +**Scenario ladder — same-file mode:** + +| Run | VUs | Iterations | What we measure | +|-----|-----|-----------|-----------------| +| 1 | 3 | 10 | Baseline lock contention | +| 2 | 5 | 10 | Moderate contention | +| 3 | 10 | 10 | Higher contention | +| 4 | 20 | 10 | Stress level | + +**Scenario ladder — multi-file mode:** + +| Run | Files | VUs/file | Total VUs | What we measure | +|-----|-------|----------|-----------|-----------------| +| 1 | 3 | 2 | 6 | Light load | +| 2 | 5 | 3 | 15 | Moderate | +| 3 | 10 | 3 | 30 | Heavy | +| 4 | 10 | 5 | 50 | Stress | + +**Metrics to track:** +- `http_req_duration{rpc_command:update-file}` — p50, p95, p99 at each VU level +- `http_req_duration{rpc_command:get-file}` — should be unaffected +- `http_req_failed` — should be 0% +- Latency growth curve: how much does p95 increase per additional VU? + +**Expected results:** +- `get-file` latency: constant (no lock, read-only) +- `update-file` p95: grows with VU count in same-file mode (lock queuing) +- `update-file` p95: stable in multi-file mode (independent locks) +- Failure rate: 0% (no revn conflicts in this scenario) + +**Files to create:** +- `performance/scripts/workspace-edit-concurrent.js` + +**Files to modify:** +- `performance/run.sh` — add `concurrent-edit` command + +--- + +### Phase 5 – CI Integration & Reporting (Days 9–10) + +1. **Runner script (`run.sh`):** + - `./run.sh smoke` for a 1-VU, 1-iteration smoke test. ✅ Done + - `./run.sh lifecycle -v 100 -n 10` for the standard run. + - Add `--scenario` flag to run individual flows or the full mix. ✅ Done + +2. **Output:** + - k6 JSON/CSV output to `performance/results//`. + - Prometheus snapshot diff (before vs after). + - Grafana screenshot or dashboard export. + +3. **Grafana Dashboard:** *(Deferred — no Prometheus remote write or InfluxDB configured in current stack)* + - Panel: `p95 latency by RPC command` (from `rpc_main_timing_seconds`). + - Panel: `HTTP requests/sec` (from k6). + - Panel: `Error rate by command` (from k6). + - Panel: `DB connection pool` (if available). + - Panel: `JVM heap used`. + - Panel: `update-file conflict rate` (custom metric from k6). + - Panel: `File size vs latency` (from the matrix test). + +4. **Regression guard (relative comparison):** + - **Approach:** Run performance tests twice in the same CI job — once on base branch, once on PR branch. Compare p95/p99 directly. No stored baselines needed. + - **Trigger:** Only when backend files change (`backend/src/**`). + - **Comparison script:** `scripts/compare-results.js` — parses two k6 JSON outputs, compares p50/p95/p99 for each RPC command. + - **Threshold:** Fail if p95 increases >20% for any critical command (`get-file`, `update-file`, `login-with-password`, `create-demo-profile`). + - **Workflow:** + 1. Checkout base branch (main) + 2. Run performance tests → store as "baseline" + 3. Checkout PR branch + 4. Run performance tests → store as "current" + 5. Compare baseline vs current + 6. If p95 increases >20% → fail CI + - **Advantages:** Same hardware, same conditions. No stored baselines. Only runs when backend changes. + +--- + +## Risks & Considerations + +| Risk | Mitigation | +|------|------------| +| **Scale: 1000 VUs creating data simultaneously will exhaust DB connection pool or storage quota.** | Pre-seed the data pool. Use a dedicated perf DB. Monitor `pg_stat_activity` and HikariCP metrics. | +| **Media upload (images/fonts) will saturate network I/O before the backend is stressed.** | Run the load test from the same datacenter/VPC as the backend. Use small dummy files for most tests; reserve large files for a dedicated storage-stress scenario. | +| **`update-file` conflicts under 1000 VUs may be so high that the test becomes a conflict test, not a latency test.** | Measure both. The conflict rate is itself a critical metric. If it is too high, we can add jitter or use independent files. | +| **Exporter service is a separate bottleneck.** | `export.js` should target the backend queue endpoint, not the full export pipeline, unless we want to test the exporter too. If exporter is in scope, run it as a separate scenario. | +| **Chunked upload creates many temporary DB rows (`upload_chunk` table).** | The backend has a `upload-session-gc` cron job. Ensure it runs after the test, or clean up manually. | +| **Font upload shells out to FontForge and WOFF tools.** | This is CPU-intensive and may be a bottleneck. Run font upload as a separate, low-VU scenario to measure the processing time without blocking other tests. | +| **Prometheus metrics may not expose DB pool wait time.** | Add a custom JMX exporter for HikariCP if needed, or query `pg_stat_activity` directly. | +| **Cleanup:** 1000 VUs creating teams/files will leave logical deletions or orphaned storage objects.** | Use a dedicated perf environment. Run a cleanup script after the test that deletes all seeded data via the RPC API. | + +--- + +## Testing Strategy + +### How to verify the test harness itself works + +1. **Smoke test:** Run each k6 script with `1 VU, 1 iteration` against a local devenv. Verify all requests return `200` and the response body is valid JSON. +2. **Baseline run:** Run `workspace-open.js` with `10 VUs, 60 s` against a clean devenv. Record baseline p95 and p99 latencies. +3. **Regression guard:** After any backend change, re-run the baseline. If p95 increases by >20%, flag it. +4. **Saturation test:** Ramp `workspace-edit.js` to 100 VUs editing independent files. Monitor backend CPU and DB connection pool. The test should reveal the breaking point where `update-file` latency spikes. +5. **Media upload stress test:** Run `media-upload.js` with 50 VUs uploading 1 MB files. Verify storage throughput and no `413` errors. +6. **Font upload stress test:** Run `font-upload.js` with 10 VUs. Verify FontForge CPU usage and no timeouts. + +### Manual validation checklist + +- [x] `POST /api/main/methods/login-with-password` with JSON body returns a session cookie. (Validated via k6 lifecycle) +- [x] `GET /api/main/methods/get-profile` with `Accept: application/json` returns JSON. (Validated via k6 lifecycle) +- [ ] `curl -H "Accept: application/json" http://localhost:6060/metrics` returns Prometheus text. +- [ ] Backend fixtures create at least 100 test users and 100 test files. +- [x] A `update-file` request with a minimal `changes` payload succeeds and returns `{"revn": N}`. (Validated — needs full shape with selrect, points, transform, frame-id) +- [x] A `upload-file-media-object` multipart request succeeds and returns a media object ID. (Validated via k6 lifecycle + media-upload) +- [x] A chunked upload (`create-upload-session` → `upload-chunk` → `assemble-file-media-object`) succeeds. (Validated via media-upload with JPG 305 KB, and font-upload with TTF 68 KB + OTF 82 KB) +- [x] A `create-font-variant` request with chunked uploads succeeds. (Validated via font-upload — TTF + OTF, returns variant with id) + +--- + +## Immediate Next Steps (if approved) + +1. ~~Create `performance/` directory and `README.md`.~~ ✅ Done +2. ~~Write `penpot-client.js` (k6 shared module) with `login()`, `rpc()`, `uploadMultipart()`, and `uploadChunked()` helpers.~~ ✅ Done (~590 lines, JSON transport, cookie auth, session headers, tagged metrics, direct + chunked upload, file library/thumbnail methods) +3. ~~Write a manual `curl` validation script~~ — Skipped; JSON compatibility confirmed via k6 smoke test. +4. ~~Write a data seeding script~~ — Not needed. User pool created in k6 `setup()` phase (sequential, excluded from metrics). Each VU picks `data.users[__VU - 1]` to login. +5. ~~Write the first k6 script: `lifecycle.js`~~ ✅ Done (11 checks, 22 HTTP requests, 0% failure) +6. ~~Run a 1-VU smoke test against local devenv and commit the baseline results.~~ ✅ Done +7. ~~Write `workspace-open.js` and `workspace-edit.js`.~~ ✅ Done (both validated, 0% failure) +8. ~~Write `media-upload.js` and `font-upload.js`.~~ ✅ Done (both validated, 0% failure) +9. ~~Define the `1000-VU` scenario mix in `options.js` (shared scenario config).~~ ✅ Done (`./run.sh all` orchestrator runs all 5 flows in parallel) +10. Run the first 100-VU ramp test and capture Prometheus metrics. + +--- + +**Plan Author:** Senior Software Architect +**Status:** Phase 1–5 complete. Regression guard implemented (relative comparison). Grafana dashboards deferred. +